wsb-daemon 0.258.6441

wsb's daemon: the workbench, the GitHub mirror's schedule and peer continuity, for every workspace on the machine
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
use std::fs;
use std::io;
use std::path::Path;
use std::process::{Command, Stdio};

// A build script is its own compilation unit and cannot use the crate it
// builds, so the publishing sequence is INCLUDED rather than imported. The
// crate's tests compile the same file, which is what makes the sequence
// testable at all — and it is the only thing standing between an interrupted
// frontend build and a tree that will not compile.
include!("src/build_publish.rs");

/// Everything in `frontend/` the workbench is built from. One list, so what
/// is copied to be built, what is hashed into the build id and what makes this
/// script run again cannot disagree.
const FRONTEND_INPUTS: &[&str] = &[
    "index.html",
    "package.json",
    "package-lock.json",
    "svelte.config.js",
    "tsconfig.json",
    "vite.config.ts",
    "public",
    "src",
];

fn main() {
    println!("cargo:rerun-if-changed=build.rs");
    for input in FRONTEND_INPUTS {
        println!("cargo:rerun-if-changed=frontend/{input}");
    }
    // The library and its migrations are the parent crate's, one level up,
    // when this daemon is built as a member of that workspace.
    let beside_the_library = beside_the_library();
    if beside_the_library {
        println!("cargo:rerun-if-changed=../migrations");
        println!("cargo:rerun-if-changed=../src");
        println!("cargo:rerun-if-changed=../Cargo.toml");
    }

    let frontend_dir = Path::new("frontend");
    // Built assets go in this build script's own OUT_DIR, which
    // `#[folder = "$OUT_DIR/workbench/dist"]` in web/mod.rs reads — one
    // spelling, or the embed silently serves a tree nothing writes to.
    //
    // OUT_DIR and not a directory of the workspace's: cargo decides whether
    // this script runs again, and it only knows about what it owns. The assets
    // were under the build root, which a sweep may clear; a guest's sweep took
    // them and left cargo's record that this script had run, so the next
    // build did not run it and the daemon would not compile ("folder ... does
    // not exist"). Here they go exactly when that record goes.
    let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR not set");
    let workbench_dir = Path::new(&out_dir).join("workbench");
    let workbench_dir = workbench_dir.as_path();
    let dist_index = workbench_dir.join("dist").join("index.html");

    // Every input below is required: a missing one is a broken tree, not a
    // mode to build in. The whole backend: this daemon AND the library it is a
    // server over. A build id that covered only this crate would report a
    // workbench as unchanged after every change to the thing it serves.
    //
    // From the repository the library is a sibling, and its source is hashed.
    // From this crate's package it is a dependency, and every wsb crate is
    // released together at one version — so this crate's own manifest, which
    // carries that version, already says which library it serves.
    let mut backend_inputs = vec!["build.rs", "Cargo.toml", "src"];
    if beside_the_library {
        backend_inputs.extend(["../Cargo.toml", "../Cargo.lock", "../src"]);
    }
    let backend_build_id = hash_paths(&backend_inputs)
        .unwrap_or_else(|error| panic!("failed to hash backend build inputs: {}", error));
    let frontend_inputs: Vec<String> =
        FRONTEND_INPUTS.iter().map(|input| format!("frontend/{input}")).collect();
    let frontend_build_id = hash_paths(&frontend_inputs.iter().map(String::as_str).collect::<Vec<_>>())
        .unwrap_or_else(|error| panic!("failed to hash frontend build inputs: {}", error));
    println!("cargo:rustc-env=WSB_BACKEND_BUILD_ID={}", backend_build_id);
    println!(
        "cargo:rustc-env=WSB_FRONTEND_BUILD_ID={}",
        frontend_build_id
    );

    // Write the frontend build ID to a well-known path so dx bundle can stamp
    // it into the app bundle's Info.plist without parsing cargo's OUT_DIR hash.
    // The OUT_DIR copy is for any tooling that prefers that location.
    let out_dir_file = std::path::Path::new(&out_dir).join("frontend_build_id.txt");
    fs::write(&out_dir_file, &frontend_build_id)
        .unwrap_or_else(|e| panic!("failed to write {}: {}", out_dir_file.display(), e));
    // <target>/<profile>/frontend_build_id.txt — stable location, no hash
    // suffix. DERIVED from OUT_DIR (<target>/<profile>/build/<pkg>-<hash>/out)
    // rather than assumed to be `../target`: the target directory is declared,
    // and it moved with the rest of the disposable state. An assumption about
    // where it is fails by writing nothing, which nothing notices.
    let target_dir = std::path::Path::new(&out_dir)
        .ancestors()
        .nth(3)
        .map(std::path::Path::to_path_buf)
        .unwrap_or_default();
    if target_dir.is_dir() {
        let stable_file = target_dir.join("frontend_build_id.txt");
        fs::write(&stable_file, &frontend_build_id)
            .unwrap_or_else(|e| panic!("failed to write {}: {}", stable_file.display(), e));
    }

    let debug_build = std::env::var("PROFILE").unwrap_or_default() == "debug";

    if !frontend_dir.is_dir() {
        panic!("frontend/ directory not found");
    }
    if !frontend_dir.join("package.json").is_file() {
        panic!("frontend/package.json not found");
    }
    let install_subcommand = if frontend_dir.join("package-lock.json").is_file() {
        "ci"
    } else {
        panic!("frontend/package-lock.json not found; locked frontend dependencies are required");
    };

    // Installed and built in a COPY under OUT_DIR, never in `frontend/`.
    //
    // A build script may write nowhere but OUT_DIR. Built from the published
    // package, `frontend/` is in cargo's registry, shared by every project that
    // uses this version, and `cargo publish` refused the package because `npm
    // ci` had added `node_modules` beside its sources. In the repository the
    // tree's own `frontend/node_modules` is the developer's, which `dx`
    // installs for the dev server and the frontend's tests.
    let build_sources = workbench_dir.join("frontend");
    copy_sources(frontend_dir, &build_sources, FRONTEND_INPUTS)
        .unwrap_or_else(|error| panic!("copying the frontend to build it: {error}"));
    let build_sources = build_sources.as_path();
    let package_lock = build_sources.join("package-lock.json");
    let node_modules_dir = build_sources.join("node_modules");
    let install_stamp = node_modules_dir.join(".wsb-package-lock");

    let package_lock_contents = fs::read(&package_lock)
        .unwrap_or_else(|error| panic!("failed to read {}: {}", package_lock.display(), error));
    let install_stamp_contents = fs::read(&install_stamp).ok();
    let dependencies_ready = node_modules_dir.is_dir()
        && install_stamp_contents
            .as_ref()
            .is_some_and(|bytes| bytes.as_slice() == package_lock_contents.as_slice());

    if !dependencies_ready {
        run_npm(
            build_sources,
            // `--engine-strict`: the frontend DECLARES which Node it needs,
            // and npm only warns when that is violated. A Windows guest built
            // with a Node one minor version below the declared floor got
            // twenty EBADENGINE warnings, then a native binding that could not
            // be found, then a build script exiting 101 — with the actual
            // sentence, "Vite requires Node.js version 20.19+ or 22.12+",
            // buried between them.
            //
            // The declaration is the workspace's own statement about what it
            // builds with. Enforcing it turns three screens of consequence
            // into one line at the top.
            &[install_subcommand, "--engine-strict"],
            "frontend dependency install",
            &frontend_build_id,
            debug_build,
        );
        // The stamp says NPM RAN, and only that: it is written here,
        // before anything has run a single one of the tools npm just
        // installed. What proves those work is the check after this
        // block, which is why that one is not inside it.
        fs::write(&install_stamp, &package_lock_contents).unwrap_or_else(|error| {
            panic!(
                "failed to write frontend dependency stamp {}: {}",
                install_stamp.display(),
                error
            )
        });
    }

    // The install is not finished when npm exits.
    //
    // `npm ci` writes ~250 packages and the shims in `node_modules/.bin`
    // that every script in `package.json` is invoked through. On Windows
    // those shims are `.cmd` files, and the next process — a FRESH `npm.cmd`
    // started milliseconds later — resolved `vite` before the one npm had
    // just written was visible to it:
    //
    //     'vite' is not recognized as an internal or external command
    //
    // The shim was there. `dir node_modules\.bin\vite*` on the guest
    // afterwards listed `vite`, `vite.cmd` and `vite.ps1`, written in the
    // same minute the build failed, and running the identical `npm run
    // build` by hand against that untouched workspace succeeded in 7.65s.
    // It cost a run of about eleven minutes each time and looked like a
    // missing dependency, which is what sent three earlier investigations
    // at npm versions and dependency trees.
    //
    // So the build waits for what it is about to run. Not a sleep: a sleep
    // is a guess that is either too short on a slow machine or wasted on a
    // fast one.
    //
    // And it waits for the tool to RUN, not for its shim to exist. Waiting
    // for the file was the first attempt at this and it failed on a Windows
    // guest with timestamps a second apart — `vite.cmd` written at 12:08:35,
    // `'vite' is not recognized` at 12:08:36. Existing and working are
    // different states, and only the second one is a precondition for the
    // build. `await_tool` carries the evidence.
    //
    // Checked on EVERY build, not only after an install.
    //
    // The first version of this ran inside the branch above, and a Windows
    // guest showed why that is not enough. The stamp is written the moment npm
    // exits, so a build that then failed at `vite` still left `node_modules`
    // present and the stamp matching the lockfile. The NEXT build read that as
    // "dependencies ready", skipped the install and the wait with it, and went
    // straight to the same `'vite' is not recognized` — with no npm output in
    // the log at all, which is what makes it read as a different bug.
    //
    // The stamp records that NPM RAN. It cannot record that the tools work,
    // because it is written before anything runs them. So this asks the
    // question directly, every time: on a warm tree it is one process that
    // answers in milliseconds, and it is the only thing standing between a
    // half-finished `node_modules` and a build that blames the toolchain.
    await_tool(build_sources, &node_modules_dir, "vite");

    // The frontend is built into a SCRATCH directory and published by rename.
    //
    // It used to delete `dist/` and then build into it, which is the same
    // destroy-then-produce shape that made the native bundle scripts fragile,
    // and it fails harder here. `dist/` is not an output — it is an INPUT to
    // `#[derive(RustEmbed)]`, read while the crate's macros expand. So for the
    // whole twenty seconds of a vite build there was no `dist/` at all, and
    // anything that looked in that window got
    //
    //     #[derive(RustEmbed)] folder '.../frontend/dist' does not exist
    //
    // which names neither the cause nor the remedy. Two builds of this crate
    // from one tree — a `cargo install` beside a `cargo build`, which is an
    // ordinary thing to do — destroyed each other. So did an npm build that
    // failed or was interrupted: it left no `dist/` behind, so the NEXT build
    // failed differently and more confusingly than the one that broke.
    //
    // Renaming closes it. `dist/` is either the previous complete build or
    // this one, and the only gap is between two renames rather than across a
    // whole compile. Stale assets cannot survive either, which is what the
    // delete was for: the scratch tree is built from nothing every time.
    let dist_dir = workbench_dir.join("dist");
    let staging_dir = workbench_dir.join("dist.building");
    let previous_dir = workbench_dir.join("dist.previous");
    fs::create_dir_all(workbench_dir).unwrap_or_else(|error| {
        panic!(
            "creating the workbench output directory {}: {error}",
            workbench_dir.display()
        )
    });
    clear_scratch(&staging_dir, &previous_dir)
        .unwrap_or_else(|error| panic!("preparing the frontend build: {error}"));

    // Absolute: vite resolves a relative --outDir against its own project
    // root, the frontend directory.
    let staging_absolute = staging_dir.to_str().expect("OUT_DIR is UTF-8");
    run_npm(
        build_sources,
        &["run", "build", "--", "--outDir", staging_absolute, "--emptyOutDir"],
        "frontend production build",
        &frontend_build_id,
        debug_build,
    );

    // Checked on the STAGING tree, before anything is published: a build that
    // produced no index.html must not replace one that did.
    let staged_index = staging_dir.join("index.html");
    if !staged_index.is_file() {
        panic!(
            "frontend build completed without producing {}",
            staged_index.display()
        );
    }

    publish_by_rename(&staging_dir, &dist_dir, &previous_dir)
        .unwrap_or_else(|error| panic!("publishing the frontend build: {error}"));

    if !dist_index.is_file() {
        panic!("frontend build published nothing at {}", dist_index.display());
    }
}

fn run_npm(frontend_dir: &Path, args: &[&str], phase: &str, build_id: &str, debug: bool) {
    let mut command = Command::new(npm_command());
    command
        .args(args)
        .current_dir(frontend_dir)
        .env("WSB_FRONTEND_BUILD_ID", build_id)
        .stdin(Stdio::null())
        .stdout(Stdio::inherit())
        .stderr(Stdio::inherit());
    if debug {
        command.env("WSB_DEBUG_BUILD", "1");
    }
    let status = command.status().unwrap_or_else(|error| {
        panic!(
            "failed to start npm for {} in {}: {}",
            phase,
            frontend_dir.display(),
            error
        )
    });

    if !status.success() {
        panic!(
            "npm {} failed during {} in {}",
            args.join(" "),
            phase,
            frontend_dir.display()
        );
    }
}

/// Wait until a tool `npm ci` just installed can actually be RUN.
///
/// # Why the shim's existence is not the question
///
/// The first version of this waited for `node_modules/.bin/vite.cmd` and
/// returned the instant it appeared. That is not enough, and a Windows guest
/// proved it with timestamps a second apart:
///
/// ```text
/// 12:08:14  node_modules/vite/bin/vite.js   written
/// 12:08:35  node_modules/.bin/vite.cmd      written
/// 12:08:36  'vite' is not recognized as an internal or external command
/// ```
///
/// The shim was there. The build still failed one second later. A `.cmd` shim
/// is a two-line batch file that does nothing but `node "%dp0%\..\vite\bin\
/// vite.js"`, so it can exist, be complete, and still not run — while npm is
/// finishing 250 packages, while the directory entry is visible but the file's
/// data is not yet durable, or while a scanner holds it open. Waiting for the
/// FILE tests the one thing that was never in doubt.
///
/// So this asks the question the build is about to ask: run the tool — the
/// SHIM the build itself invokes, not `npm exec <name>`, which answers about
/// npm. A tool that answers `--version` is a tool the next process can invoke,
/// which is the actual precondition. It costs one process on a warm machine and it
/// cannot pass while the thing it is testing does not work.
///
/// Bounded, and a timeout is a FAILURE naming the tool and the path. A tool
/// that never becomes runnable is an install that did not produce what it said
/// it did — npm exits 0 having left it unusable — and going on to the build
/// would report that as `'vite' is not recognized`, which names neither the
/// install nor the race.
fn await_tool(frontend_dir: &Path, node_modules_dir: &Path, name: &str) {
    let shim = node_modules_dir
        .join(".bin")
        .join(if cfg!(windows) { format!("{name}.cmd") } else { name.to_string() });
    let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
    let mut last: Option<String>;

    loop {
        // The file first, because running what is not there is a different
        // error and a worse message than waiting for it.
        if shim.is_file() {
            // THE SHIM, which is what `package.json`'s scripts invoke.
            //
            // This went through `npm exec --offline --no -- <name> --version`
            // on the reasoning that npm is the one spelling that runs a tool on
            // both platforms. Measured on the Windows guest that motivated this
            // whole function, that is not what it does:
            //
            //     npm exec --offline --no -- vite --version   ->  10.9.2
            //     node_modules\.bin\vite.cmd --version        ->  vite/8.0.1
            //
            // `10.9.2` is NPM's version. The check was asking npm about itself,
            // exiting 0 whatever state vite was in — so it could not fail when
            // vite was broken, and it spent sixty seconds exiting 1 while the
            // shim beside it ran correctly. A check that cannot answer the
            // question it is named for is worse than no check: it is sixty
            // seconds of latency and a panic blaming the wrong thing.
            //
            // The shim runs directly on both platforms, which was the concern
            // that sent this through npm: on Unix it is a symlink whose shebang
            // runs it, and on Windows `CreateProcess` runs a `.cmd` without a
            // shell. Both answer `vite/8.0.1`.
            //
            // stderr is CAPTURED, not discarded. The reason is the entire value
            // of this message when it times out, and discarding it is why the
            // failure could say no more than "it exited with exit code: 1".
            // Absolute, as OUT_DIR is, and never CANONICALIZED. On Windows
            // `canonicalize` returns a UNC path — `\\?\C:\…` — and a `.cmd` is
            // run by `cmd.exe`, which refuses a UNC working directory.
            // Measured on a Windows guest:
            //
            //     \\?\C:\…\vite.cmd --version  ->  1, cannot find the path
            //     C:\…\vite.cmd --version       ->  0, vite/8.0.1
            //
            // The failure surfaced as `'"node"' is not recognized`, because
            // the shim's own lookup of `node` is what broke — which reads as a
            // missing toolchain and is a path form.
            match Command::new(&shim)
                .arg("--version")
                .current_dir(frontend_dir)
                .stdin(Stdio::null())
                .stdout(Stdio::piped())
                .stderr(Stdio::piped())
                .output()
            {
                Ok(done) if done.status.success() => return,
                Ok(done) => {
                    let said = String::from_utf8_lossy(&done.stderr);
                    let said = said.trim();
                    last = Some(if said.is_empty() {
                        format!("it exited with {} and said nothing", done.status)
                    } else {
                        format!("it exited with {}: {said}", done.status)
                    });
                }
                Err(error) => last = Some(format!("it could not be started: {error}")),
            }
        } else {
            last = Some(format!("{} is not there yet", shim.display()));
        }

        if std::time::Instant::now() >= deadline {
            // WHAT THE SHIM COULD SEE, on the way out.
            //
            // A `.cmd` resolves `node` through this process's PATH, so a
            // failure that says `'"node"' is not recognized` is a statement
            // about the environment the build script was given — not about the
            // shim, and not about the install. Printing it is the difference
            // between one more run and a diagnosis: the same message without it
            // sent three investigations at npm versions and dependency trees.
            let path = std::env::var("PATH").unwrap_or_else(|_| "<unset>".into());
            let node = path
                .split(if cfg!(windows) { ';' } else { ':' })
                .find(|entry| {
                    let entry = entry.to_ascii_lowercase();
                    entry.contains("nodejs") || entry.contains("node")
                })
                .unwrap_or("<no node directory on PATH>");
            panic!(
                "npm installed the frontend's dependencies and `{name}` still cannot \
                 be run after 60s: {}.\nEvery script in package.json is invoked \
                 through {}, so the build below would fail with `'{name}' is not \
                 recognized`.\nThe shim resolves `node` through PATH, and this \
                 build script's PATH carries: {}\nPATH = {}",
                last.as_deref().unwrap_or("no reason was recorded"),
                shim.display(),
                node,
                path
            );
        }
        std::thread::sleep(std::time::Duration::from_millis(100));
    }
}

fn npm_command() -> &'static str {
    if cfg!(windows) {
        "npm.cmd"
    } else {
        "npm"
    }
}

/// Hash of every listed path, in the order given. A path that is not there is
/// an error: the build ID has to cover the whole input, and one that quietly
/// covered less would still look like a valid ID.
/// Whether this crate is being built as a member of the wsb workspace, beside
/// the library it serves, rather than from its published package.
///
/// The parent's manifest says so by declaring this directory a member. A
/// package has no such parent: it is unpacked on its own.
fn beside_the_library() -> bool {
    let Ok(text) = fs::read_to_string("../Cargo.toml") else {
        return false;
    };
    let parent: toml::Table = text
        .parse()
        .unwrap_or_else(|error| panic!("../Cargo.toml is not a manifest: {error}"));
    parent
        .get("workspace")
        .and_then(|workspace| workspace.get("members"))
        .and_then(|members| members.as_array())
        .is_some_and(|members| members.iter().any(|member| member.as_str() == Some("daemon")))
}

fn hash_paths(paths: &[&str]) -> io::Result<String> {
    let mut hasher = Fnv1a64::new();
    for path in paths {
        hash_path(Path::new(path), &mut hasher)?;
    }
    Ok(hasher.finish_hex())
}

fn hash_path(path: &Path, hasher: &mut Fnv1a64) -> io::Result<()> {
    let metadata = fs::metadata(path)?;
    if metadata.is_dir() {
        let mut entries = fs::read_dir(path)?.collect::<Result<Vec<_>, _>>()?;
        entries.sort_by(|left, right| left.path().cmp(&right.path()));
        hasher.write(path.to_string_lossy().as_bytes());
        hasher.write(b"\0dir\0");
        for entry in entries {
            hash_path(&entry.path(), hasher)?;
        }
        return Ok(());
    }

    hasher.write(path.to_string_lossy().as_bytes());
    hasher.write(b"\0file\0");
    hasher.write(&fs::read(path)?);
    hasher.write(b"\0");
    Ok(())
}

struct Fnv1a64(u64);

impl Fnv1a64 {
    const OFFSET_BASIS: u64 = 0xcbf29ce484222325;
    const PRIME: u64 = 0x00000100000001B3;

    fn new() -> Self {
        Self(Self::OFFSET_BASIS)
    }

    fn write(&mut self, bytes: &[u8]) {
        for byte in bytes {
            self.0 ^= u64::from(*byte);
            self.0 = self.0.wrapping_mul(Self::PRIME);
        }
    }

    fn finish_hex(&self) -> String {
        format!("{:016x}", self.0)
    }
}