flodl 0.7.0

floDl — a flow-graph deep learning framework built on libtorch
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

//! Rank-side dashboard intent stash + emit helpers.
//!
//! When the user's harness on a rank process calls
//! [`crate::monitor::Monitor::serve`] / `.watch()` / `.set_metadata()`,
//! the rewired Monitor stashes the intent here instead of binding a
//! local HTTP server (the launcher hosts the dashboard post controller-
//! active refactor). The cluster_worker drains the stash at startup
//! and emits the matching [`crate::distributed::wire::TimingMsgWire`]
//! frames on the coord socket; the launcher's
//! [`crate::distributed::ClusterDashboardSink`] then binds the server
//! lazily on first arrival.
//!
//! Single global stash — every Monitor on the rank process writes here
//! (typical user code constructs one Monitor; multi-Monitor on the
//! same rank is not a documented pattern). The cluster_worker reads
//! once at startup; subsequent stashes after that point are no-ops
//! from the launcher's POV.

use std::sync::Mutex;

/// Snapshot of the user's dashboard intent, populated by Monitor on
/// the rank's harness thread and drained by `cluster_worker` after
/// rendezvous.
#[derive(Debug, Clone, Default)]
pub(crate) struct PendingDashboardConfig {
    /// HTTP port the user requested via `monitor.serve(port)`. `None` = no live
    /// dashboard: `DashboardRegister` is not emitted and resource sampling stays
    /// off. The other `Dashboard*` emits are NOT port-gated — the saved archive
    /// wants the SVG / metadata / hardware and binds no server, so gating them
    /// on a port produced a persisted dashboard with no graph and no config.
    pub port: Option<u16>,
    /// Graph SVG bytes the user pushed via `monitor.watch(&model)` (or
    /// `monitor.set_svg(&svg)`).
    pub svg: Option<String>,
    /// Graph label captured by `monitor.watch`.
    pub label: Option<String>,
    /// Graph structural hash captured by `monitor.watch`.
    pub hash: Option<String>,
    /// Pre-serialized JSON metadata blob from `monitor.set_metadata`.
    pub metadata_json: Option<String>,
    /// Hardware summary string captured at `Monitor::new`. Sent
    /// per-rank so the launcher's dashboard can render per-rank tabs
    /// labelled `host:lr=<local_rank> gr=<global_rank>`.
    pub hardware: Option<String>,
}

static PENDING: Mutex<PendingDashboardConfig> = Mutex::new(PendingDashboardConfig {
    port: None,
    svg: None,
    label: None,
    hash: None,
    metadata_json: None,
    hardware: None,
});

/// Record the dashboard port the user requested. Called from
/// `Monitor::serve` on the cluster-rank path.
pub(crate) fn stash_port(port: u16) {
    PENDING.lock().unwrap().port = Some(port);
}

/// Record the graph SVG + identity. Called from `Monitor::watch` /
/// `Monitor::set_svg` on the cluster-rank path.
pub(crate) fn stash_svg(svg: String, label: Option<String>, hash: Option<String>) {
    let mut p = PENDING.lock().unwrap();
    p.svg = Some(svg);
    if label.is_some() {
        p.label = label;
    }
    if hash.is_some() {
        p.hash = hash;
    }
}

/// Record the dashboard metadata JSON. Called from `Monitor::set_metadata`.
pub(crate) fn stash_metadata(json: String) {
    PENDING.lock().unwrap().metadata_json = Some(json);
}

/// Record the rank's hardware summary string. Called from `Monitor::new`
/// (so every rank's hardware reaches the launcher, even if the user
/// never opts into the dashboard — cheap stash, drained only when port
/// is also set).
pub(crate) fn stash_hardware(hardware: String) {
    PENDING.lock().unwrap().hardware = Some(hardware);
}

/// Drain the stash (atomic snapshot). Called from `cluster_worker` at
/// startup, after rendezvous but before the training loop begins.
/// Returns the current stash and resets it to default, so a hypothetical
/// re-call would not double-emit.
pub(crate) fn drain() -> PendingDashboardConfig {
    let mut p = PENDING.lock().unwrap();
    std::mem::take(&mut *p)
}