thruput-node-supervisor 0.43.0

Local node supervisor for Thruput development
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
use std::{fs, time::Duration};

use tempfile::TempDir;

use super::{PidGuard, matches_expected_program};
use crate::{
    ChildProcessSpec, ChildSupervisor, NodeSupervisorError,
    test_support::{FAKE_HOGD_DEV_MODE_ENV, fake_hogd_dev_binary},
};

#[tokio::test]
async fn acquire_no_file_is_clean() {
    let temp = TempDir::new().expect("bug: temp dir should be created");
    let path = temp.path().join("hogd-dev.pid");

    let guard = PidGuard::acquire(
        path.clone(),
        std::ffi::OsStr::new("hogd-dev"),
        Duration::from_millis(10),
    )
    .await
    .expect("bug: acquiring missing PID file should succeed");

    assert!(path.exists());
    drop(guard);
    assert!(!path.exists());
}

#[tokio::test]
async fn write_creates_parent_dirs() {
    let temp = TempDir::new().expect("bug: temp dir should be created");
    let path = temp.path().join("nested").join("hogd-dev.pid");
    let mut guard = PidGuard::acquire(
        path.clone(),
        std::ffi::OsStr::new("hogd-dev"),
        Duration::from_millis(10),
    )
    .await
    .expect("bug: acquiring PID file should create parents");

    guard.write(1234).expect("bug: PID write should succeed");
    assert_eq!(
        fs::read_to_string(&path).expect("bug: PID file should be readable"),
        "1234\n"
    );
    #[cfg(unix)]
    {
        use std::os::unix::fs::PermissionsExt;
        let mode = fs::metadata(path.parent().expect("bug: PID file has a parent"))
            .expect("bug: PID parent metadata should be readable")
            .permissions()
            .mode()
            & 0o777;
        assert_eq!(mode, 0o700);
    }
    #[cfg(unix)]
    {
        use std::os::unix::fs::PermissionsExt;
        let mode = fs::metadata(&path)
            .expect("bug: PID file metadata should be readable")
            .permissions()
            .mode()
            & 0o777;
        assert_eq!(mode, 0o600);
    }
}

#[tokio::test]
async fn written_pid_file_survives_drop_until_explicit_release() {
    let temp = TempDir::new().expect("bug: temp dir should be created");
    let path = temp.path().join("hogd-dev.pid");
    let stale_pid = i32::MAX as u32;
    let mut guard = PidGuard::acquire(
        path.clone(),
        std::ffi::OsStr::new("hogd-dev"),
        Duration::from_millis(10),
    )
    .await
    .expect("bug: PID file should be acquired");
    guard
        .write(stale_pid)
        .expect("bug: PID write should succeed");

    drop(guard);
    assert_eq!(
        fs::read_to_string(&path).expect("bug: orphan recovery PID should remain"),
        format!("{stale_pid}\n")
    );

    let guard = PidGuard::acquire(
        path.clone(),
        std::ffi::OsStr::new("hogd-dev"),
        Duration::from_millis(10),
    )
    .await
    .expect("bug: stale PID should be reacquired");
    guard.release();
    assert!(!path.exists());
}

#[cfg(unix)]
#[tokio::test]
async fn acquire_rejects_symlinked_pid_file() {
    use std::os::unix::fs::symlink;

    let temp = TempDir::new().expect("bug: temp dir should be created");
    let target = temp.path().join("target.pid");
    let path = temp.path().join("hogd-dev.pid");
    fs::write(&target, "").expect("bug: PID target should be written");
    symlink(&target, &path).expect("bug: PID symlink should be created");

    let err = PidGuard::acquire(
        path,
        std::ffi::OsStr::new("hogd-dev"),
        Duration::from_millis(10),
    )
    .await
    .expect_err("bug: symlinked PID file should fail closed");
    assert!(matches!(err, NodeSupervisorError::PidFile { .. }));
}

#[tokio::test]
async fn drop_removes_file() {
    let temp = TempDir::new().expect("bug: temp dir should be created");
    let path = temp.path().join("hogd-dev.pid");
    let guard = PidGuard::acquire(
        path.clone(),
        std::ffi::OsStr::new("hogd-dev"),
        Duration::from_millis(10),
    )
    .await
    .expect("bug: acquiring PID file should succeed");

    assert!(path.exists());
    drop(guard);
    assert!(!path.exists());
}

#[tokio::test]
async fn acquire_stale_pid_removes_file_contents() {
    let temp = TempDir::new().expect("bug: temp dir should be created");
    let path = temp.path().join("hogd-dev.pid");
    fs::write(&path, "999999\n").expect("bug: stale PID should be written");

    let mut guard = PidGuard::acquire(
        path.clone(),
        std::ffi::OsStr::new("hogd-dev"),
        Duration::from_millis(10),
    )
    .await
    .expect("bug: stale PID should not block acquire");

    guard.write(7).expect("bug: PID write should succeed");
    assert_eq!(
        fs::read_to_string(&path).expect("bug: PID file should be readable"),
        "7\n"
    );
}

#[tokio::test]
async fn acquire_malformed_pid_removes_file_contents() {
    let temp = TempDir::new().expect("bug: temp dir should be created");
    let path = temp.path().join("hogd-dev.pid");
    fs::write(&path, "not-a-pid\n").expect("bug: malformed PID should be written");

    let mut guard = PidGuard::acquire(
        path.clone(),
        std::ffi::OsStr::new("hogd-dev"),
        Duration::from_millis(10),
    )
    .await
    .expect("bug: malformed PID should not block acquire");

    guard.write(8).expect("bug: PID write should succeed");
    assert_eq!(
        fs::read_to_string(&path).expect("bug: PID file should be readable"),
        "8\n"
    );
}

#[tokio::test]
async fn contended_acquire_rebinds_to_the_file_currently_at_path() {
    let temp = TempDir::new().expect("bug: temp dir should be created");
    let path = temp.path().join("hogd-dev.pid");
    let holder = PidGuard::acquire(
        path.clone(),
        std::ffi::OsStr::new("hogd-dev"),
        Duration::from_millis(10),
    )
    .await
    .expect("bug: holder should acquire the PID file");

    let contender_path = path.clone();
    let contender = tokio::spawn(async move {
        PidGuard::acquire(
            contender_path,
            std::ffi::OsStr::new("hogd-dev"),
            Duration::from_millis(10),
        )
        .await
    });
    // Dropping the holder unlinks the path before the close releases the
    // lock, so a contender that keeps its pre-unlink descriptor would win
    // the lock on an unreachable inode and write its PID there. Release
    // from a separate OS thread so an overloaded single-threaded Tokio
    // executor cannot delay this past the contender's retry deadline.
    let release = std::thread::spawn(move || {
        std::thread::sleep(Duration::from_millis(100));
        drop(holder);
    });

    let mut guard = contender
        .await
        .expect("bug: contender task should finish")
        .expect("bug: contender should acquire after the holder releases");
    release
        .join()
        .expect("bug: holder release thread should finish");
    guard.write(4321).expect("bug: PID write should succeed");
    assert_eq!(
        fs::read_to_string(&path).expect("bug: written PID should be visible at the path"),
        "4321\n"
    );
}

#[tokio::test]
async fn acquire_locked_pid_file_reports_running_node() {
    let temp = TempDir::new().expect("bug: temp dir should be created");
    let path = temp.path().join("hogd-dev.pid");
    let _guard = PidGuard::acquire(
        path.clone(),
        std::ffi::OsStr::new("hogd-dev"),
        Duration::from_millis(10),
    )
    .await
    .expect("bug: first acquire should succeed");

    let err = PidGuard::acquire(
        path,
        std::ffi::OsStr::new("hogd-dev"),
        Duration::from_millis(10),
    )
    .await
    .expect_err("bug: second acquire should fail while locked");

    assert!(
        matches!(err, NodeSupervisorError::LocalNodeAlreadyRunning { .. }),
        "unexpected locked PID file error: {err:?}"
    );
}

#[test]
fn matches_expected_program_compares_basenames() {
    // `program` is the comm-style basename produced by `ps -o comm=`,
    // so it never contains arguments or path components.
    assert!(matches_expected_program(
        "hogd-dev",
        std::ffi::OsStr::new("hogd-dev")
    ));
    assert!(matches_expected_program(
        "hogd-dev",
        std::ffi::OsStr::new("/usr/local/bin/hogd-dev")
    ));
    assert!(!matches_expected_program(
        "sleep",
        std::ffi::OsStr::new("hogd-dev")
    ));
    // Truncated comm column must not collide with a longer expected
    // basename.
    assert!(!matches_expected_program(
        "hogd-de",
        std::ffi::OsStr::new("hogd-dev")
    ));
}

#[test]
fn matches_expected_program_accepts_legacy_devnode_name() {
    assert!(matches_expected_program(
        "hogd-dev",
        std::ffi::OsStr::new("hog-devnode")
    ));
    assert!(matches_expected_program(
        "hogd-dev",
        std::ffi::OsStr::new("/usr/local/bin/hog-devnode")
    ));
    assert!(!matches_expected_program(
        "hogd-dev",
        std::ffi::OsStr::new("hog-mempool")
    ));
}

#[cfg(target_os = "linux")]
#[test]
fn parse_linux_stat_handles_spaces_and_parens_in_comm() {
    let stat = "123 (hogd dev) S 42 123 123 0 -1 4194560";
    assert_eq!(
        super::parse_linux_stat(stat),
        Some((Some(42), "hogd dev".to_string()))
    );

    let stat = "123 (hogd)dev) S 42 123 123 0 -1 4194560";
    assert_eq!(
        super::parse_linux_stat(stat),
        Some((Some(42), "hogd)dev".to_string()))
    );
}

#[tokio::test]
async fn acquire_stops_recorded_running_process() {
    let temp = TempDir::new().expect("bug: temp dir should be created");
    let path = temp.path().join("hogd-dev.pid");

    // Spawn an orphan-shaped child: own process group, long-running, so
    // `acquire` will find it live, verify the expected basename, and
    // clean it up. We don't try to detach it from the test process — the
    // test just needs `flock`-based cleanup to work even though `ppid`
    // is the test runner (not 1). That's exactly the subreaper case.
    let binary = fake_hogd_dev_binary();
    let spec = ChildProcessSpec::new(&binary)
        .env(FAKE_HOGD_DEV_MODE_ENV, "hang")
        .process_group(0);
    let mut child = ChildSupervisor::new(spec)
        .spawn()
        .expect("bug: fake hogd-dev should spawn");
    let pid = child.id().expect("bug: fake hogd-dev pid should be known");
    fs::write(&path, format!("{pid}\n")).expect("bug: pid file write");
    tokio::time::sleep(Duration::from_millis(50)).await;
    let wait_task = tokio::spawn(async move {
        child
            .wait()
            .await
            .expect("bug: stopped fake hogd-dev should be reaped by the test")
    });

    let guard = PidGuard::acquire(path.clone(), binary.as_os_str(), Duration::from_secs(1))
        .await
        .expect("bug: acquire should stop the recorded fake hogd-dev");

    let status = wait_task
        .await
        .expect("bug: fake hogd-dev wait task should finish");
    assert!(
        !status.success(),
        "expected signal exit from stopped fake hogd-dev, got {status:?}"
    );

    drop(guard);
}

#[tokio::test]
async fn acquire_fails_closed_for_leaderless_process_group() {
    let temp = TempDir::new().expect("bug: temp dir should be created");
    let path = temp.path().join("hogd-dev.pid");
    let binary = fake_hogd_dev_binary();
    let spec = ChildProcessSpec::new(&binary)
        .env(FAKE_HOGD_DEV_MODE_ENV, "spawn-helper-then-exit")
        .process_group(0);
    let mut child = ChildSupervisor::new(spec)
        .spawn()
        .expect("bug: fake hogd-dev should spawn");
    let pid = child.id().expect("bug: fake hogd-dev pid should be known");
    fs::write(&path, format!("{pid}\n")).expect("bug: pid file should be written");
    let status = child
        .wait()
        .await
        .expect("bug: fake leader should be reaped");
    assert!(status.success());
    assert!(super::process_group_exists(pid));

    let err = PidGuard::acquire(path.clone(), binary.as_os_str(), Duration::from_millis(10))
        .await
        .expect_err("bug: leaderless process group should block startup");
    assert!(matches!(
        err,
        NodeSupervisorError::LocalNodeAlreadyRunning {
            pid: Some(found),
            ..
        } if found == pid
    ));

    let target = -i32::try_from(pid).expect("bug: pid should fit pid_t");
    let _ = unsafe { libc::kill(target, libc::SIGKILL) };
    for _ in 0..50 {
        if !super::process_group_exists(pid) {
            break;
        }
        tokio::time::sleep(Duration::from_millis(20)).await;
    }
    assert!(!super::process_group_exists(pid));
    fs::remove_file(path).expect("bug: test should remove preserved PID file");
}

#[tokio::test]
async fn acquire_leaves_unrelated_program_alone() {
    use std::os::unix::process::CommandExt;

    let temp = TempDir::new().expect("bug: temp dir should be created");
    let path = temp.path().join("hogd-dev.pid");

    let mut child = std::process::Command::new(sleep_command())
        .arg("2")
        .process_group(0)
        .spawn()
        .expect("bug: sleep should spawn");
    let pid = child.id();
    fs::write(&path, format!("{pid}\n")).expect("bug: pid file write");

    // Expected program is "hogd-dev" but the recorded PID is "sleep" —
    // the basename mismatch should make acquire treat this as PID
    // reuse and leave the process running.
    let guard = PidGuard::acquire(
        path.clone(),
        std::ffi::OsStr::new("hogd-dev"),
        Duration::from_millis(50),
    )
    .await
    .expect("bug: acquire should succeed past mismatched basename");

    // Confirm the sleep is still alive.
    assert!(
        child
            .try_wait()
            .expect("bug: try_wait should succeed")
            .is_none(),
        "sleep should still be running after acquire ignored it"
    );

    // Clean up so the test doesn't linger.
    let _ = child.kill();
    let _ = child.wait();
    drop(guard);
}

#[tokio::test]
async fn persistent_acquire_rejects_unrelated_live_program() {
    use std::os::unix::process::CommandExt;

    let temp = TempDir::new().expect("bug: temp dir should be created");
    let path = temp.path().join("hogd-dev.pid");
    let mut child = std::process::Command::new(sleep_command())
        .arg("2")
        .process_group(0)
        .spawn()
        .expect("bug: sleep should spawn");
    let pid = child.id();
    fs::write(&path, format!("{pid}\n")).expect("bug: pid file write");

    let err = PidGuard::acquire_persistent(
        path.clone(),
        std::ffi::OsStr::new("hogd-dev"),
        Duration::from_millis(50),
    )
    .await
    .expect_err("bug: persistent startup should reject an unverifiable live process");
    assert!(matches!(
        err,
        NodeSupervisorError::LocalNodeAlreadyRunning {
            pid: Some(found),
            ..
        } if found == pid
    ));
    assert!(
        child
            .try_wait()
            .expect("bug: try_wait should succeed")
            .is_none(),
        "sleep should remain running after fail-closed acquisition"
    );

    let _ = child.kill();
    let _ = child.wait();
    fs::remove_file(path).expect("bug: test should remove preserved PID file");
}

#[test]
fn process_group_zero_is_rejected() {
    let temp = TempDir::new().expect("bug: temp dir should be created");
    let err = super::signal_process_group(temp.path(), 0, libc::SIGTERM)
        .expect_err("bug: process group zero should be rejected");

    assert!(
        matches!(err, NodeSupervisorError::PidFile { .. }),
        "unexpected process group zero error: {err:?}"
    );
    assert!(!super::process_group_exists(0));
}

#[cfg(unix)]
fn sleep_command() -> &'static str {
    if std::path::Path::new("/bin/sleep").exists() {
        "/bin/sleep"
    } else {
        "sleep"
    }
}