Skip to main content

lenso_service/
service_backup.rs

1use std::collections::BTreeMap;
2
3use schemars::JsonSchema;
4use serde::{Deserialize, Serialize};
5use serde_json::Value;
6use utoipa::ToSchema;
7
8use crate::extraction_input_digest;
9
10pub const SERVICE_BACKUP_PROTOCOL: &str = "lenso.service-backup.v1";
11pub const SERVICE_RESTORE_EVIDENCE_PROTOCOL: &str = "lenso.service-restore-evidence.v1";
12
13#[derive(
14    Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema, ToSchema,
15)]
16#[serde(rename_all = "snake_case")]
17pub enum RestoreDecision {
18    Passed,
19    Blocked,
20}
21
22#[derive(
23    Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema, ToSchema,
24)]
25#[serde(rename_all = "snake_case")]
26pub enum RestoreIssueCode {
27    BackupIntegrityInvalid,
28    BackupIncomplete,
29    EncryptionBoundaryInvalid,
30    TargetStoreNotClean,
31    RestoreStateMismatch,
32    ReplayBoundaryInvalid,
33    AuthorityConflict,
34    EnvironmentEvidenceInvalid,
35    RecoverySetIncompatible,
36    RecoverySetStale,
37    KeyReferenceUnavailable,
38    ReconciliationIncomplete,
39    CleanupIncomplete,
40}
41
42impl RestoreIssueCode {
43    #[must_use]
44    pub const fn as_str(self) -> &'static str {
45        match self {
46            Self::BackupIntegrityInvalid => "restore_backup_integrity_invalid",
47            Self::BackupIncomplete => "restore_backup_incomplete",
48            Self::EncryptionBoundaryInvalid => "restore_encryption_boundary_invalid",
49            Self::TargetStoreNotClean => "restore_target_store_not_clean",
50            Self::RestoreStateMismatch => "restore_state_mismatch",
51            Self::ReplayBoundaryInvalid => "restore_replay_boundary_invalid",
52            Self::AuthorityConflict => "restore_authority_conflict",
53            Self::EnvironmentEvidenceInvalid => "restore_environment_evidence_invalid",
54            Self::RecoverySetIncompatible => "restore_recovery_set_incompatible",
55            Self::RecoverySetStale => "restore_recovery_set_stale",
56            Self::KeyReferenceUnavailable => "restore_key_reference_unavailable",
57            Self::ReconciliationIncomplete => "restore_reconciliation_incomplete",
58            Self::CleanupIncomplete => "restore_cleanup_incomplete",
59        }
60    }
61}
62
63#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, ToSchema)]
64#[serde(rename_all = "camelCase")]
65pub struct RestoreIssue {
66    pub code: RestoreIssueCode,
67    pub message: String,
68    pub remediation: String,
69    pub next_actions: Vec<String>,
70}
71
72#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, ToSchema)]
73#[serde(rename_all = "camelCase")]
74pub struct ServiceBackupInput {
75    pub service_id: String,
76    pub store_id: String,
77    pub format_version: String,
78    pub schema_version: String,
79    pub release_digest: String,
80    pub config_revision_digest: String,
81    pub contract_version_digests: BTreeMap<String, String>,
82    pub store_checkpoint_digest: String,
83    pub broker_position: Option<u64>,
84    pub restore_preconditions: Vec<String>,
85    pub point_in_time_unix_ms: u64,
86    pub captured_at_unix_ms: u64,
87    pub freshness_horizon_unix_ms: u64,
88    pub snapshot_digest: String,
89    pub post_checkpoint_work_digest: String,
90    pub encryption_key_reference: String,
91    pub encryption_algorithm: String,
92    pub state_digests: BTreeMap<String, String>,
93    pub outbox_sequence: u64,
94    pub inbox_sequence: u64,
95    pub workflow_timer_sequence: u64,
96    pub story_sequence: u64,
97    pub completed: bool,
98}
99
100#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, ToSchema)]
101#[serde(rename_all = "camelCase")]
102pub struct ServiceBackup {
103    pub protocol: String,
104    pub backup_id: String,
105    pub backup_digest: String,
106    #[serde(flatten)]
107    pub input: ServiceBackupInput,
108}
109
110#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, ToSchema)]
111#[serde(rename_all = "camelCase")]
112pub struct PostgresRestoreObservation {
113    pub provider: String,
114    pub version: String,
115    pub instance_identity: String,
116    pub used_real_instance: bool,
117    pub observation_digest: String,
118}
119
120#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, ToSchema)]
121#[serde(rename_all = "camelCase")]
122pub struct ServiceRestoreInput {
123    pub backup: ServiceBackup,
124    pub target_store_id: String,
125    pub target_was_clean: bool,
126    pub expected_service_id: String,
127    pub expected_format_version: String,
128    pub expected_schema_version: String,
129    pub expected_release_digest: String,
130    pub expected_config_revision_digest: String,
131    pub expected_contract_version_digests: BTreeMap<String, String>,
132    pub key_reference_available: bool,
133    pub observed_at_unix_ms: u64,
134    pub restored_snapshot_digest: String,
135    pub restored_state_digests: BTreeMap<String, String>,
136    pub restored_contract_version_digests: BTreeMap<String, String>,
137    pub restored_release_digest: String,
138    pub restored_config_revision_digest: String,
139    pub replay_outbox_from_sequence: u64,
140    pub replay_inbox_from_sequence: u64,
141    pub restored_workflow_timer_sequence: u64,
142    pub restored_story_sequence: u64,
143    pub authoritative_workload_count: u32,
144    pub business_invariants_verified: bool,
145    pub post_checkpoint_work_reconciled: bool,
146    pub recovery_time_ms: u64,
147    pub intentional_loss_bound_ms: u64,
148    pub replay_bound_count: u64,
149    pub remaining_story_gaps: Vec<String>,
150    pub cleanup_complete: bool,
151    pub postgres: PostgresRestoreObservation,
152}
153
154#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, ToSchema)]
155#[serde(rename_all = "camelCase")]
156pub struct ServiceRestoreEvidence {
157    pub protocol: String,
158    pub evidence_id: String,
159    pub evidence_digest: String,
160    pub backup_id: String,
161    pub backup_digest: String,
162    pub service_id: String,
163    pub source_store_id: String,
164    pub target_store_id: String,
165    pub point_in_time_unix_ms: u64,
166    pub format_version: String,
167    pub schema_version: String,
168    pub contract_version_digests: BTreeMap<String, String>,
169    pub restored_state_digests: BTreeMap<String, String>,
170    pub recovery_time_ms: u64,
171    pub intentional_loss_bound_ms: u64,
172    pub replay_bound_count: u64,
173    pub remaining_story_gaps: Vec<String>,
174    pub cleanup_complete: bool,
175    pub decision: RestoreDecision,
176    pub issues: Vec<RestoreIssue>,
177    pub next_actions: Vec<String>,
178    pub production_mutated: bool,
179}
180
181pub fn assemble_service_backup(input: ServiceBackupInput) -> Result<ServiceBackup, RestoreIssue> {
182    if input.service_id.trim().is_empty()
183        || input.store_id.trim().is_empty()
184        || input.format_version.trim().is_empty()
185        || input.schema_version.trim().is_empty()
186        || input.point_in_time_unix_ms == 0
187        || input.captured_at_unix_ms == 0
188        || input.freshness_horizon_unix_ms < input.captured_at_unix_ms
189        || !valid_digest(&input.release_digest)
190        || !valid_digest(&input.config_revision_digest)
191        || input.contract_version_digests.is_empty()
192        || input
193            .contract_version_digests
194            .values()
195            .any(|digest| !valid_digest(digest))
196        || !valid_digest(&input.store_checkpoint_digest)
197        || !valid_digest(&input.snapshot_digest)
198        || !valid_digest(&input.post_checkpoint_work_digest)
199        || input.restore_preconditions.is_empty()
200        || [
201            "business",
202            "inbox",
203            "outbox",
204            "workflows",
205            "workflow_timers",
206            "compensation",
207            "stories",
208            "federation_cursors",
209        ]
210        .into_iter()
211        .any(|partition| !input.state_digests.contains_key(partition))
212        || input
213            .state_digests
214            .values()
215            .any(|digest| !valid_digest(digest))
216    {
217        return Err(issue(
218            RestoreIssueCode::BackupIntegrityInvalid,
219            "The backup is not bound to exact Service, Store, release, configuration, schema, and state identities.",
220            "Capture one content-addressed backup manifest at the authoritative Store boundary.",
221            "Correct the backup inputs and create a new immutable backup.",
222        ));
223    }
224    if !input.completed {
225        return Err(issue(
226            RestoreIssueCode::BackupIncomplete,
227            "The backup did not reach a durable completed state.",
228            "Keep partial snapshots ineligible for restore.",
229            "Finish or discard the partial backup before restore planning.",
230        ));
231    }
232    if input.encryption_key_reference.trim().is_empty()
233        || input.encryption_algorithm.trim().is_empty()
234        || input.encryption_key_reference.contains("BEGIN ")
235        || input.encryption_key_reference.contains("secret=")
236    {
237        return Err(issue(
238            RestoreIssueCode::EncryptionBoundaryInvalid,
239            "Backup encryption must use an opaque key reference without key material.",
240            "Resolve encryption only inside the configured backup provider.",
241            "Replace key material with an opaque provider reference.",
242        ));
243    }
244    let digest = digest_json(&input);
245    Ok(ServiceBackup {
246        protocol: SERVICE_BACKUP_PROTOCOL.to_owned(),
247        backup_id: format!("service-backup:{}", &digest[7..23]),
248        backup_digest: digest,
249        input,
250    })
251}
252
253#[must_use]
254pub fn evaluate_service_restore(input: ServiceRestoreInput) -> ServiceRestoreEvidence {
255    let mut issues = Vec::new();
256    if !service_backup_integrity_valid(&input.backup) {
257        issues.push(issue(
258            RestoreIssueCode::BackupIntegrityInvalid,
259            "Backup content does not match its immutable identity.",
260            "Reject modified or stale backup metadata.",
261            "Load the exact verified backup and repeat restore.",
262        ));
263    }
264    if !input.target_was_clean {
265        issues.push(issue(
266            RestoreIssueCode::TargetStoreNotClean,
267            "Restore target contains pre-existing authoritative state.",
268            "Restore only into an isolated empty Store or use a separately approved destructive plan.",
269            "Provision a clean target Store.",
270        ));
271    }
272    if input.expected_service_id != input.backup.input.service_id
273        || input.expected_format_version != input.backup.input.format_version
274        || input.expected_schema_version != input.backup.input.schema_version
275        || input.expected_release_digest != input.backup.input.release_digest
276        || input.expected_config_revision_digest != input.backup.input.config_revision_digest
277        || input.expected_contract_version_digests != input.backup.input.contract_version_digests
278        || input.restored_contract_version_digests != input.backup.input.contract_version_digests
279    {
280        issues.push(issue(
281            RestoreIssueCode::RecoverySetIncompatible,
282            "Recovery set Service, format, schema, or Contract Versions do not match the isolated target.",
283            "Restore only the exact supported recovery-set identity.",
284            "Select a compatible recovery set or target release.",
285        ));
286    }
287    if input.observed_at_unix_ms == 0
288        || input.observed_at_unix_ms > input.backup.input.freshness_horizon_unix_ms
289    {
290        issues.push(issue(
291            RestoreIssueCode::RecoverySetStale,
292            "Recovery-set evidence is outside its reviewed freshness horizon.",
293            "Capture or explicitly review a fresh recovery set.",
294            "Refresh the backup before restore.",
295        ));
296    }
297    if !input.key_reference_available {
298        issues.push(issue(
299            RestoreIssueCode::KeyReferenceUnavailable,
300            "The opaque backup key reference is unavailable.",
301            "Resolve the key through the configured provider without exposing key material.",
302            "Restore provider access or choose another verified recovery set.",
303        ));
304    }
305    if input.restored_snapshot_digest != input.backup.input.snapshot_digest
306        || input.restored_state_digests != input.backup.input.state_digests
307        || input.restored_release_digest != input.backup.input.release_digest
308        || input.restored_config_revision_digest != input.backup.input.config_revision_digest
309        || input.restored_workflow_timer_sequence != input.backup.input.workflow_timer_sequence
310        || input.restored_story_sequence != input.backup.input.story_sequence
311    {
312        issues.push(issue(
313            RestoreIssueCode::RestoreStateMismatch,
314            "Restored business, Workflow, Story, release, or configuration state differs from the backup.",
315            "Verify every state partition and exact artifact identity before activation.",
316            "Discard the target and repeat restore from verified bytes.",
317        ));
318    }
319    if input.replay_outbox_from_sequence != input.backup.input.outbox_sequence.saturating_add(1)
320        || input.replay_inbox_from_sequence != input.backup.input.inbox_sequence.saturating_add(1)
321    {
322        issues.push(issue(
323            RestoreIssueCode::ReplayBoundaryInvalid,
324            "Inbox or Outbox replay would skip or repeat a committed effect.",
325            "Resume from the first sequence after the backup checkpoint.",
326            "Correct the replay cursor before starting Workloads.",
327        ));
328    }
329    if input.authoritative_workload_count != 0 {
330        issues.push(issue(
331            RestoreIssueCode::AuthorityConflict,
332            "Restore verification ran while an authoritative Workload could still write.",
333            "Keep the restored target passive until an explicit authority cutover.",
334            "Fence active writers and repeat restore verification.",
335        ));
336    }
337    if !input.business_invariants_verified || !input.post_checkpoint_work_reconciled {
338        issues.push(issue(
339            RestoreIssueCode::ReconciliationIncomplete,
340            "Business invariants or identified post-checkpoint work remain unreconciled.",
341            "Reconcile durable delivery, Workflow, compensation, Story, and federation state.",
342            "Keep the target passive and complete reconciliation.",
343        ));
344    }
345    if !input.cleanup_complete {
346        issues.push(issue(
347            RestoreIssueCode::CleanupIncomplete,
348            "Temporary restore resources are neither cleaned nor deterministically isolated.",
349            "Remove temporary credentials and processes while retaining only reviewed passive evidence.",
350            "Complete restore cleanup.",
351        ));
352    }
353    if !input.postgres.used_real_instance
354        || input.postgres.provider.trim().is_empty()
355        || input.postgres.version.trim().is_empty()
356        || input.postgres.instance_identity.trim().is_empty()
357        || !valid_digest(&input.postgres.observation_digest)
358    {
359        issues.push(issue(
360            RestoreIssueCode::EnvironmentEvidenceInvalid,
361            "Restore evidence does not come from a real isolated Postgres instance.",
362            "Use the pinned Environment Verification database lane.",
363            "Repeat backup and restore against the supported Postgres adapter.",
364        ));
365    }
366
367    let decision = if issues.is_empty() {
368        RestoreDecision::Passed
369    } else {
370        RestoreDecision::Blocked
371    };
372    let next_actions = if issues.is_empty() {
373        vec![
374            "Keep the restored Store passive until the disaster-recovery Approval Boundary.".into(),
375        ]
376    } else {
377        issues
378            .iter()
379            .flat_map(|issue| issue.next_actions.iter().cloned())
380            .collect()
381    };
382    let mut evidence = ServiceRestoreEvidence {
383        protocol: SERVICE_RESTORE_EVIDENCE_PROTOCOL.to_owned(),
384        evidence_id: String::new(),
385        evidence_digest: String::new(),
386        backup_id: input.backup.backup_id,
387        backup_digest: input.backup.backup_digest,
388        service_id: input.backup.input.service_id,
389        source_store_id: input.backup.input.store_id,
390        target_store_id: input.target_store_id,
391        point_in_time_unix_ms: input.backup.input.point_in_time_unix_ms,
392        format_version: input.backup.input.format_version,
393        schema_version: input.backup.input.schema_version,
394        contract_version_digests: input.backup.input.contract_version_digests,
395        restored_state_digests: input.restored_state_digests,
396        recovery_time_ms: input.recovery_time_ms,
397        intentional_loss_bound_ms: input.intentional_loss_bound_ms,
398        replay_bound_count: input.replay_bound_count,
399        remaining_story_gaps: input.remaining_story_gaps,
400        cleanup_complete: input.cleanup_complete,
401        decision,
402        issues,
403        next_actions,
404        production_mutated: false,
405    };
406    evidence.evidence_digest = digest_without_identity(&evidence);
407    evidence.evidence_id = format!("service-restore:{}", &evidence.evidence_digest[7..23]);
408    evidence
409}
410
411#[must_use]
412pub fn service_restore_evidence_schema() -> Value {
413    let mut schema = serde_json::to_value(schemars::schema_for!(ServiceRestoreEvidence))
414        .expect("service restore schema serializes");
415    schema["$id"] = Value::String(
416        "https://contracts.lenso.local/ga/lenso.service-restore-evidence.v1.schema.json".to_owned(),
417    );
418    schema
419}
420
421fn service_backup_integrity_valid(backup: &ServiceBackup) -> bool {
422    backup.protocol == SERVICE_BACKUP_PROTOCOL
423        && valid_digest(&backup.backup_digest)
424        && backup.backup_digest == digest_json(&backup.input)
425        && backup.backup_id == format!("service-backup:{}", &backup.backup_digest[7..23])
426}
427
428fn issue(
429    code: RestoreIssueCode,
430    message: impl Into<String>,
431    remediation: impl Into<String>,
432    next_action: impl Into<String>,
433) -> RestoreIssue {
434    RestoreIssue {
435        code,
436        message: message.into(),
437        remediation: remediation.into(),
438        next_actions: vec![next_action.into()],
439    }
440}
441
442fn valid_digest(value: &str) -> bool {
443    value.strip_prefix("sha256:").is_some_and(|digest| {
444        digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit())
445    })
446}
447
448fn digest_json(value: &impl Serialize) -> String {
449    extraction_input_digest(&serde_json::to_vec(value).expect("backup evidence serializes"))
450}
451
452fn digest_without_identity(evidence: &ServiceRestoreEvidence) -> String {
453    let mut canonical = evidence.clone();
454    canonical.evidence_id.clear();
455    canonical.evidence_digest.clear();
456    digest_json(&canonical)
457}
458
459#[must_use]
460pub fn service_restore_integrity_is_valid(evidence: &ServiceRestoreEvidence) -> bool {
461    valid_digest(&evidence.evidence_digest)
462        && evidence.evidence_digest == digest_without_identity(evidence)
463        && evidence.evidence_id == format!("service-restore:{}", &evidence.evidence_digest[7..23])
464}