thruput_node_supervisor/
error.rs

1//! Error types for the node supervisor.
2
3use std::{fmt, io, path::PathBuf, process::ExitStatus, time::Duration};
4
5use thruput_hogd_backend::IntroOutcome;
6use thruput_json_rpc_types::NetworkHashText;
7use thruput_x23_transport::ConnectError;
8
9/// Unix child-process signal used by the supervisor.
10#[cfg(unix)]
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub struct ChildSignal(i32);
13
14#[cfg(unix)]
15impl ChildSignal {
16    /// Create a signal wrapper from a raw libc signal number.
17    pub const fn from_raw(signal: i32) -> Self {
18        Self(signal)
19    }
20
21    /// Raw libc signal number.
22    pub const fn raw(self) -> i32 {
23        self.0
24    }
25
26    const fn name(self) -> Option<&'static str> {
27        match self.0 {
28            libc::SIGTERM => Some("SIGTERM"),
29            libc::SIGKILL => Some("SIGKILL"),
30            _ => None,
31        }
32    }
33}
34
35#[cfg(unix)]
36impl fmt::Display for ChildSignal {
37    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
38        match self.name() {
39            Some(name) => f.write_str(name),
40            None => write!(f, "signal {}", self.raw()),
41        }
42    }
43}
44
45/// Errors returned by the node supervisor.
46#[derive(Debug, thiserror::Error)]
47#[non_exhaustive]
48pub enum NodeSupervisorError {
49    /// The chain's private faucet state cannot be safely loaded or persisted.
50    #[error("invalid local funding state at {}: {reason}", path.display())]
51    LocalFundingState {
52        /// File or directory whose state failed validation.
53        path: PathBuf,
54        /// Diagnostic without private material.
55        reason: String,
56    },
57    /// Local airdrop resource or transfer configuration is invalid.
58    #[error("invalid local airdrop configuration: {reason}")]
59    InvalidLocalAirdropConfig {
60        /// Human-readable validation failure.
61        reason: &'static str,
62    },
63    /// The configured protocol override is not the wire width of a token.
64    #[error("invalid protocol override: {source}")]
65    InvalidProtocolOverride {
66        /// Why the text cannot name a protocol token.
67        #[from]
68        source: thruput_json_rpc_types::protocol_token::ProtocolTokenWidthError,
69    },
70    /// A configured binary path does not exist.
71    #[error("configured hog-devnode binary path does not exist: {}", path.display())]
72    ConfiguredPathMissing {
73        /// Missing configured path.
74        path: PathBuf,
75    },
76
77    /// A configured binary path exists but is not executable.
78    #[error("configured hog-devnode binary path is not executable: {}", path.display())]
79    ConfiguredPathNotExecutable {
80        /// Non-executable configured path.
81        path: PathBuf,
82    },
83
84    /// The supervisor could not locate a devnode binary.
85    #[error("hog-devnode binary not found")]
86    HogdDevNotFound,
87
88    /// The supervisor could not determine its current executable path.
89    #[error("failed to resolve current executable path: {source}")]
90    CurrentExe {
91        /// Underlying I/O error.
92        #[source]
93        source: io::Error,
94    },
95
96    /// The supervisor could not start the configured child process.
97    #[error("failed to spawn child process `{program}`: {source}")]
98    Spawn {
99        /// Program path or name.
100        program: String,
101        /// Underlying I/O error.
102        #[source]
103        source: io::Error,
104    },
105
106    /// The configured hogd-dev bootstrap manifest request could not be represented.
107    #[error("invalid hogd-dev bootstrap manifest request: {reason}")]
108    InvalidBootstrapManifestConfig {
109        /// Human-readable reason.
110        reason: String,
111    },
112
113    /// The supervisor could not create a private bootstrap-manifest directory.
114    #[error("failed to create hogd-dev bootstrap manifest temp dir: {source}")]
115    BootstrapManifestTempDir {
116        /// Underlying I/O error.
117        #[source]
118        source: io::Error,
119    },
120
121    /// The supervisor could not serialize the bootstrap manifest.
122    #[error("failed to serialize hogd-dev bootstrap manifest: {source}")]
123    BootstrapManifestSerialize {
124        /// Underlying JSON error.
125        #[source]
126        source: serde_json::Error,
127    },
128
129    /// The supervisor could not write the bootstrap manifest file.
130    #[error("failed to write hogd-dev bootstrap manifest {}: {source}", path.display())]
131    BootstrapManifestWrite {
132        /// Manifest path.
133        path: PathBuf,
134        /// Underlying I/O error.
135        #[source]
136        source: io::Error,
137    },
138
139    /// The child process was spawned without a stdout pipe.
140    #[error("child process `{program}` did not expose stdout")]
141    MissingStdout {
142        /// Program path or name.
143        program: String,
144    },
145
146    /// The child process was spawned without a stderr pipe.
147    #[error("child process `{program}` did not expose stderr")]
148    MissingStderr {
149        /// Program path or name.
150        program: String,
151    },
152
153    /// The child process did not report readiness before the startup timeout.
154    #[error("child process `{program}` did not report readiness within {timeout:?}")]
155    ReadinessTimeout {
156        /// Program path or name.
157        program: String,
158        /// Startup timeout.
159        timeout: Duration,
160    },
161
162    /// The child process wrote a readiness line exceeding the configured cap.
163    #[error("child process `{program}` wrote a readiness line larger than {max_bytes} bytes")]
164    ReadinessTooLarge {
165        /// Program path or name.
166        program: String,
167        /// Maximum allowed readiness line size.
168        max_bytes: usize,
169    },
170
171    /// The child process exited before reporting readiness.
172    #[error("child process `{program}` exited before readiness with status {status}")]
173    ExitedBeforeReady {
174        /// Program path or name.
175        program: String,
176        /// Child exit status.
177        status: ExitStatus,
178    },
179
180    /// The supervisor could not read the readiness line.
181    #[error("failed to read readiness from child process `{program}`: {source}")]
182    ReadinessRead {
183        /// Program path or name.
184        program: String,
185        /// Underlying I/O error.
186        #[source]
187        source: io::Error,
188    },
189
190    /// The child process wrote malformed readiness JSON.
191    #[error("invalid readiness JSON from child process `{program}`: {source}")]
192    InvalidReadinessJson {
193        /// Program path or name.
194        program: String,
195        /// Raw readiness line.
196        line: String,
197        /// JSON parsing error.
198        #[source]
199        source: serde_json::Error,
200    },
201
202    /// The child process wrote a readiness line with an unsupported schema.
203    #[error(
204        "invalid readiness schema version from child process `{program}`: expected {expected}, got {actual}"
205    )]
206    InvalidReadinessSchemaVersion {
207        /// Program path or name.
208        program: String,
209        /// Expected schema version.
210        expected: u32,
211        /// Actual schema version.
212        actual: u32,
213    },
214
215    /// The child process wrote a readiness line with an unexpected status.
216    #[error("invalid readiness status from child process `{program}`: {status}")]
217    InvalidReadinessStatus {
218        /// Program path or name.
219        program: String,
220        /// Status value from the readiness line.
221        status: String,
222    },
223
224    /// The supervisor could not read the readiness runtime manifest.
225    #[error(
226        "failed to read runtime manifest `{path}` advertised by child process `{program}`: {source}"
227    )]
228    RuntimeManifestRead {
229        /// Program path or name.
230        program: String,
231        /// Manifest path advertised in readiness.
232        path: String,
233        /// Underlying I/O error.
234        #[source]
235        source: io::Error,
236    },
237
238    /// The advertised runtime manifest path or permissions are unsafe.
239    #[error("unsafe runtime manifest `{path}` advertised by child process `{program}`: {reason}")]
240    RuntimeManifestUnsafePermissions {
241        /// Program path or name.
242        program: String,
243        /// Manifest path advertised in readiness.
244        path: String,
245        /// Validation failure.
246        reason: String,
247    },
248
249    /// The child process advertised a malformed runtime manifest hash.
250    #[error("invalid runtime manifest hash from child process `{program}`: {hash}")]
251    InvalidRuntimeManifestHash {
252        /// Program path or name.
253        program: String,
254        /// Advertised hash.
255        hash: String,
256    },
257
258    /// The runtime manifest bytes did not match the advertised hash.
259    #[error(
260        "runtime manifest hash mismatch from child process `{program}` for `{path}`: expected {expected}, got {actual}"
261    )]
262    RuntimeManifestHashMismatch {
263        /// Program path or name.
264        program: String,
265        /// Manifest path advertised in readiness.
266        path: String,
267        /// Hash advertised in readiness.
268        expected: String,
269        /// Hash computed from manifest bytes.
270        actual: String,
271    },
272
273    /// The runtime manifest could not be parsed or projected.
274    #[error("invalid runtime manifest from child process `{program}`: {reason}")]
275    InvalidRuntimeManifest {
276        /// Program path or name.
277        program: String,
278        /// Validation failure.
279        reason: String,
280    },
281
282    /// The runtime manifest advertised a different network than the bootstrap request.
283    #[error(
284        "hogd-dev runtime manifest network hash mismatch: bootstrap requested {expected}, runtime advertised {observed}"
285    )]
286    RuntimeNetworkHashMismatch {
287        /// Network hash requested in the bootstrap manifest.
288        expected: NetworkHashText,
289        /// Network hash advertised by the runtime manifest.
290        observed: NetworkHashText,
291    },
292
293    /// The runtime manifest advertised a different protocol token than requested.
294    #[error(
295        "hogd-dev runtime manifest protocol mismatch: supervisor requested {expected}, runtime advertised {observed:?}"
296    )]
297    RuntimeProtocolMismatch {
298        /// Protocol token requested by the supervisor.
299        expected: String,
300        /// Protocol token advertised by the runtime manifest, if present.
301        observed: Option<String>,
302    },
303
304    /// The supervisor could not signal the child process group.
305    #[cfg(unix)]
306    #[error("failed to send {signal} to child process group `{program}`: {source}")]
307    Signal {
308        /// Program path or name.
309        program: String,
310        /// Signal sent to the child process group.
311        signal: ChildSignal,
312        /// Underlying I/O error.
313        #[source]
314        source: io::Error,
315    },
316
317    /// The supervisor could not query the child process state.
318    #[error("failed to query child process `{program}` status: {source}")]
319    Status {
320        /// Program path or name.
321        program: String,
322        /// Underlying I/O error.
323        #[source]
324        source: io::Error,
325    },
326
327    /// The supervisor could not wait for the child process to exit.
328    #[error("failed to wait for child process `{program}`: {source}")]
329    Wait {
330        /// Program path or name.
331        program: String,
332        /// Underlying I/O error.
333        #[source]
334        source: io::Error,
335    },
336
337    /// The supervisor could not request child process termination.
338    #[error("failed to stop child process `{program}`: {source}")]
339    Stop {
340        /// Program path or name.
341        program: String,
342        /// Underlying I/O error.
343        #[source]
344        source: io::Error,
345    },
346
347    /// The supervisor could not bind the JSON-RPC server to its listen address.
348    #[error("failed to bind JSON-RPC server to {address}: {source}")]
349    RpcBind {
350        /// Configured listen address.
351        address: String,
352        /// Underlying I/O error.
353        #[source]
354        source: io::Error,
355    },
356
357    /// The supervisor could not raise the process file-descriptor limit for JSON-RPC.
358    #[cfg(unix)]
359    #[error("failed to raise JSON-RPC nofile limit toward {target}: {source}")]
360    RpcNofileLimit {
361        /// Desired soft nofile limit.
362        target: u64,
363        /// Underlying OS error.
364        #[source]
365        source: io::Error,
366    },
367
368    /// The configured JSON-RPC connection budget exceeds the hard file-descriptor limit.
369    #[cfg(unix)]
370    #[error("JSON-RPC nofile hard limit {hard} is below requested target {target}")]
371    RpcNofileHardLimit {
372        /// Desired soft nofile limit.
373        target: u64,
374        /// Current hard nofile limit.
375        hard: u64,
376    },
377
378    /// The supervisor could not read the JSON-RPC server's bound address.
379    #[error("failed to read JSON-RPC bound address: {source}")]
380    RpcLocalAddr {
381        /// Underlying I/O error.
382        #[source]
383        source: io::Error,
384    },
385
386    /// The supervisor stop signal failed.
387    #[error("failed to wait for supervisor stop signal: {source}")]
388    StopSignal {
389        /// Underlying I/O error.
390        #[source]
391        source: io::Error,
392    },
393
394    /// The supervisor could not connect to hogd-dev's `SchedulerCoord` x23 endpoint.
395    #[error("failed to connect hogd-dev SchedulerCoord endpoint {endpoint}: {source}")]
396    SchedulerConnect {
397        /// `SchedulerCoord` endpoint advertised by hogd-dev readiness.
398        endpoint: std::net::SocketAddr,
399        /// x23 connection error.
400        #[source]
401        source: ConnectError,
402    },
403
404    /// The supervisor could not connect to hogd-dev's `SchedGateway` x23 endpoint.
405    #[error("failed to connect hogd-dev SchedGateway endpoint {endpoint}: {source}")]
406    SchedulerGatewayConnect {
407        /// `SchedGateway` endpoint advertised by hogd-dev readiness.
408        endpoint: std::net::SocketAddr,
409        /// x23 connection error.
410        #[source]
411        source: ConnectError,
412    },
413
414    /// The scheduler tip belongs to a different network than the runtime manifest.
415    #[error(
416        "hogd-dev scheduler network hash mismatch: runtime manifest expected {expected}, scheduler reported {observed}"
417    )]
418    SchedulerNetworkHashMismatch {
419        /// Network hash advertised by the runtime manifest.
420        expected: NetworkHashText,
421        /// Network hash observed in the scheduler tip proposal header.
422        observed: NetworkHashText,
423    },
424
425    /// The coordinator and public scheduler gateway advertised different startup tips.
426    #[error(
427        "hogd-dev scheduler planes disagree at startup: coordinator {coordinator}, gateway {gateway}"
428    )]
429    SchedulerPlaneTipMismatch {
430        /// Debug representation of the coordinator tip.
431        coordinator: String,
432        /// Debug representation of the scheduler gateway tip.
433        gateway: String,
434    },
435
436    /// A hogd-dev scheduler RPC failed.
437    #[error("hogd-dev scheduler RPC `{op}` failed: {source}")]
438    SchedulerRpc {
439        /// Scheduler RPC operation name.
440        op: &'static str,
441        /// x23 RPC error.
442        #[source]
443        source: rpc_driver::RpcError,
444    },
445
446    /// A hogd-dev scheduler gateway RPC failed during startup.
447    #[error("hogd-dev scheduler gateway RPC `{op}` failed: {source}")]
448    SchedulerGatewayRpc {
449        /// Scheduler gateway RPC operation name.
450        op: &'static str,
451        /// x23 RPC error.
452        #[source]
453        source: rpc_driver::RpcError,
454    },
455
456    /// The scheduler rejected a startup operation.
457    #[error("hogd-dev scheduler rejected `{op}`: {result}")]
458    SchedulerRejected {
459        /// Scheduler RPC operation name.
460        op: &'static str,
461        /// Scheduler result value.
462        result: String,
463    },
464
465    /// The supervisor could not connect to hogd-dev's `IndexerBlockExplorer` x23 endpoint.
466    #[error("failed to connect hogd-dev IndexerBlockExplorer endpoint {endpoint}: {source}")]
467    IndexerConnect {
468        /// `IndexerBlockExplorer` endpoint advertised by hogd-dev readiness.
469        endpoint: std::net::SocketAddr,
470        /// x23 connection error.
471        #[source]
472        source: ConnectError,
473    },
474
475    /// A hogd-dev indexer RPC failed during startup probing.
476    #[error("hogd-dev indexer RPC `{op}` failed: {error}")]
477    IndexerRpc {
478        /// Indexer RPC operation name.
479        op: &'static str,
480        /// x23 RPC error message.
481        error: String,
482    },
483
484    /// The supervisor could not connect to hogd-dev's `DbGateway` x23 endpoint.
485    #[error("failed to connect hogd-dev DbGateway endpoint {endpoint}: {source}")]
486    DbGatewayConnect {
487        /// `DbGateway` endpoint advertised by hogd-dev readiness.
488        endpoint: std::net::SocketAddr,
489        /// x23 connection error.
490        #[source]
491        source: ConnectError,
492    },
493
494    /// A hogd-dev `DbGateway` RPC failed during startup probing.
495    #[error("hogd-dev DbGateway RPC `{op}` failed: {error}")]
496    DbGatewayRpc {
497        /// `DbGateway` RPC operation name.
498        op: &'static str,
499        /// x23 RPC error message.
500        error: String,
501    },
502
503    /// The supervisor could not connect to hogd-dev's `ViewGateway` x23 endpoint.
504    #[error("failed to connect hogd-dev ViewGateway endpoint {endpoint}: {source}")]
505    ViewGatewayConnect {
506        /// `ViewGateway` endpoint advertised by hogd-dev readiness.
507        endpoint: std::net::SocketAddr,
508        /// x23 connection error.
509        #[source]
510        source: ConnectError,
511    },
512
513    /// A hogd-dev `ViewGateway` RPC failed during startup probing.
514    #[error("hogd-dev ViewGateway RPC `{op}` failed: {error}")]
515    ViewGatewayRpc {
516        /// `ViewGateway` RPC operation name.
517        op: &'static str,
518        /// x23 RPC error message.
519        error: String,
520    },
521
522    /// An x23 service rejected the intro handshake with protocol details.
523    #[error(
524        "{}",
525        intro_rejected_message(
526            service,
527            *outcome,
528            outcome_name,
529            *is_protocol_mismatch,
530            protocol,
531            expected_protocol.as_deref()
532        )
533    )]
534    ProtocolIntroRejected {
535        /// x23 service that rejected the intro handshake.
536        service: &'static str,
537        /// Raw outcome discriminant from the x23 service.
538        outcome: i32,
539        /// Decoded intro outcome name when known.
540        outcome_name: &'static str,
541        /// Whether the intro outcome is a protocol mismatch.
542        is_protocol_mismatch: bool,
543        /// Protocol token sent by the Rust supervisor.
544        protocol: String,
545        /// Protocol token advertised by readiness, when available.
546        expected_protocol: Option<String>,
547    },
548
549    /// The supervisor could not operate on the hogd-dev PID file.
550    #[error("hogd-dev PID file operation failed at {}: {source}", path.display())]
551    PidFile {
552        /// PID file path or parent directory involved in the failed operation.
553        path: PathBuf,
554        /// Underlying I/O error.
555        #[source]
556        source: io::Error,
557    },
558
559    /// Another managed local node appears to be running.
560    #[error(
561        "thruput local node already appears to be running (PID file {}, hogd-dev pid {pid:?}); stop it before starting another node",
562        path.display()
563    )]
564    LocalNodeAlreadyRunning {
565        /// PID file path that is locked by the running node.
566        path: PathBuf,
567        /// Existing hogd-dev PID if it could be read from the PID file.
568        pid: Option<u32>,
569    },
570
571    /// Forced shutdown did not conclusively stop the supervised process group.
572    #[error(
573        "child process `{program}` did not fully stop after SIGKILL (process group {process_group:?}); preserving its PID file to prevent unsafe restart"
574    )]
575    ShutdownIncomplete {
576        /// Program path or name used to spawn the child.
577        program: String,
578        /// Process group that may still be alive.
579        process_group: Option<i32>,
580    },
581
582    /// Persistent local-node lifecycle guarantees require Unix process groups.
583    #[error("persistent hog-devnode state is only supported on Unix platforms")]
584    PersistentStateUnsupported,
585
586    /// The supervisor could not prepare a private persistent data directory.
587    #[error("hog-devnode data directory operation failed at {}: {source}", path.display())]
588    DataDir {
589        /// Persistent data directory involved in the failed operation.
590        path: PathBuf,
591        /// Underlying I/O error.
592        #[source]
593        source: io::Error,
594    },
595}
596
597impl NodeSupervisorError {
598    /// Report an x23 service that did not accept the protocol this build
599    /// speaks.
600    ///
601    /// `expected_protocol` is what hogd-dev advertised in its readiness
602    /// payload, when the supervisor has it: naming both sides turns "intro
603    /// rejected" into an actionable version mismatch.
604    pub(crate) fn intro_rejected(
605        outcome: IntroOutcome,
606        protocol: &str,
607        expected_protocol: Option<&str>,
608    ) -> Self {
609        Self::ProtocolIntroRejected {
610            service: outcome.service(),
611            outcome: outcome.code(),
612            outcome_name: outcome.name(),
613            is_protocol_mismatch: outcome.mismatched_protocol(),
614            protocol: protocol.to_string(),
615            expected_protocol: expected_protocol.map(str::to_string),
616        }
617    }
618}
619
620fn intro_rejected_message(
621    service: &str,
622    outcome: i32,
623    outcome_name: &str,
624    is_protocol_mismatch: bool,
625    protocol: &str,
626    expected_protocol: Option<&str>,
627) -> String {
628    if is_protocol_mismatch {
629        return match expected_protocol {
630            Some(expected_protocol) => format!(
631                "{service} intro mismatch: thruput node speaks protocol {protocol}, \
632                 hogd-dev expects {expected_protocol} ({outcome_name}; binary is built against \
633                 a different protocol commit)"
634            ),
635            None => format!(
636                "{service} intro mismatch: thruput node speaks protocol {protocol}, \
637                 but hogd-dev rejected it ({outcome_name})"
638            ),
639        };
640    }
641
642    format!(
643        "hogd-dev {service} intro rejected with outcome {outcome_name} \
644         (discriminant {outcome}); thruput node speaks protocol {protocol}"
645    )
646}
thruput_node_supervisor/error.rs

thruput_node_supervisor/
error.rs
