zc2 0.0.46

P2P compute broker with yen-denominated billing, WAL, and broker mesh support
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
//! Broker module for routing compute requests to workers.
//!
//! The broker provides:
//! - Worker registry with health monitoring
//! - Yen-denominated billing with dashboard API
//! - Optimal worker selection based on price-per-compute
//! - Low-latency request forwarding
//! - WireGuard-based P2P worker discovery
//! - Live terminal UI for monitoring
//! - Benchmarking tools

pub mod affinity;
pub mod auto_update;
pub mod balance;
pub mod bench;
pub mod bench_mesh;
pub mod capability;
pub mod deploy;
pub mod discovery;
pub mod estimator;
pub mod flush;
pub mod http_adapter;
pub mod hub_registration;
pub mod info;
pub mod key_auth;
pub mod ledger;
pub mod mesh_batch;
pub mod node_health;
pub mod node_identity;
pub mod node_sync;
pub mod peer;
pub mod placement;
pub mod policy;
pub mod pricing;
pub mod quic;
pub mod recovery;
pub mod roster_cache;
pub mod router;
pub mod sampler;
pub mod selection;
pub mod server;
pub mod shadow;
pub mod stats;
pub mod task_board;
pub mod uri;
pub mod voucher;
pub mod wal;
pub mod worker;

use std::sync::Arc;
use std::sync::OnceLock;

use dashmap::DashMap;

pub use balance::BalanceManager;
pub use discovery::DiscoveryConfig;
pub use flush::TransactionBuffer;
pub use ledger::Ledger;
pub use peer::PeerManager;
pub use pricing::BrokerPrice;
pub use router::Router;
pub use server::start_server;
pub use stats::StatsCollector;
pub use wal::Wal;
pub use worker::WorkerRegistry;

/// Infer this node's name from the system hostname.
///
/// This used to query the mesh daemon first -- `wireguard status --json`, then
/// its local API socket over curl -- and fall back to the hostname. WireGuard
/// has no such daemon and no name service: a peer is an address, and the node
/// name is the host's own. So the fallback is now the whole implementation,
/// and two subprocess spawns per call disappear with it.
fn detect_node_name() -> Option<String> {
    std::process::Command::new("hostname")
        .output()
        .ok()
        .and_then(|o| String::from_utf8(o.stdout).ok())
        .map(|s| s.trim().to_string())
        .filter(|s| !s.is_empty())
}

/// This broker's node name (ZAKURO_NODE_NAME or detected), or "unknown".
pub fn node_name_or_default() -> String {
    std::env::var("ZAKURO_NODE_NAME")
        .ok()
        .filter(|s| !s.is_empty())
        .or_else(detect_node_name)
        .unwrap_or_else(|| "unknown".to_string())
}

/// Broker configuration
#[derive(Debug, Clone)]
pub struct BrokerConfig {
    /// Host to bind the broker server
    pub host: String,
    /// Port to bind the broker server
    pub port: u16,
    /// Worker health check interval in seconds
    pub health_check_interval: u64,
    /// Worker timeout before marking unhealthy (seconds)
    pub worker_timeout: u64,
    /// Minimum balance required for any request
    pub min_balance: f64,
    /// Run in daemon mode (background, minimal output)
    pub daemon: bool,
    /// Verbose output (show live transactions)
    pub verbose: bool,
    /// Enable WireGuard-based worker discovery
    pub enable_discovery: bool,
    /// Discovery configuration
    pub discovery: DiscoveryConfig,
    /// Shared secret for worker management endpoints (from ZAKURO_WORKER_KEY env var)
    pub worker_key: Option<String>,
    /// Owner zakuro_user_id — workers discovered by this broker belong to this user
    /// (from ZAKURO_OWNER_ID env var, required for PG worker sync)
    pub owner_user_id: Option<String>,
    /// Human-readable node name for this broker instance (from ZAKURO_NODE_NAME env var)
    /// Used to tag synced workers with their source node.
    pub node_name: Option<String>,
    /// Shared secret for peer-to-peer broker communication (from ZAKURO_PEER_KEY env var)
    pub peer_key: Option<String>,
    /// Enable P2P money operations (from ZAKURO_P2P env var, default false)
    pub enable_p2p: bool,
    /// Dashboard API URL for worker sync (from ZAKURO_API_URL env var)
    pub api_url: Option<String>,
    /// User API key for authentication (from ZAKURO_API_KEY env var)
    /// This is the universal key used for both user services and broker-to-dashboard communication
    pub api_key: Option<String>,
    /// Override WireGuard IP detection (test-only, avoids global env var race).
    /// When set, `BrokerState::own_wireguard_ip` uses this value instead of
    /// probing network interfaces.
    pub wireguard_ip_override: Option<String>,
    /// Explicit QUIC port (UDP).  When `None`, defaults to `port + 1`.
    pub quic_port: Option<u16>,
    /// Override the async runtime's worker-thread count (test-only).
    /// `None` → tokio default (num_cpus). Tests set a small value so the many
    /// broker runtimes a parallel `cargo test` leaks don't oversubscribe the
    /// CPU and flake latency-sensitive tests. See `server::build_broker_runtime`.
    pub runtime_worker_threads: Option<usize>,
}

impl Default for BrokerConfig {
    fn default() -> Self {
        Self {
            host: "0.0.0.0".to_string(),
            port: 9000,
            health_check_interval: 5,
            worker_timeout: 30,
            min_balance: 0.15,
            daemon: false,
            verbose: true,
            enable_discovery: true,
            discovery: DiscoveryConfig::default(),
            worker_key: std::env::var("ZAKURO_WORKER_KEY").ok(),
            owner_user_id: {
                // Always derived from API key format: zk_{user_id}_{hex}.
                // Not user-configurable — ZAKURO_OWNER_ID is intentionally ignored.
                std::env::var("ZAKURO_API_KEY").ok().and_then(|k| {
                    k.strip_prefix("zk_")
                        .and_then(|rest| rest.rfind('_').map(|pos| rest[..pos].to_string()))
                })
            },
            node_name: std::env::var("ZAKURO_NODE_NAME")
                .ok()
                .or_else(detect_node_name),
            // Env first; else the key `zc connect` stored from the hub, so a
            // user's broker peers with the mesh without pasting the secret.
            peer_key: std::env::var("ZAKURO_PEER_KEY")
                .ok()
                .filter(|k| !k.trim().is_empty())
                .or_else(crate::credentials::load_mesh_peer_key),
            enable_p2p: std::env::var("ZAKURO_P2P")
                .map(|v| v == "true" || v == "1")
                .unwrap_or(false),
            api_url: std::env::var("ZAKURO_API_URL").ok(),
            api_key: std::env::var("ZAKURO_API_KEY").ok(),
            wireguard_ip_override: None,
            quic_port: None,
            runtime_worker_threads: None,
        }
    }
}

/// P2P for a broker started from the CLI (`zc broker`, `zc share`): honour an
/// explicit `ZAKURO_P2P`, otherwise turn it on exactly when this machine
/// reaches the mesh, by either route — a mesh broker that does not peer would
/// be reachable by everyone and dial nobody. `BrokerConfig::default()` itself
/// stays env-only so tests constructing configs are not swayed by the host.
pub fn p2p_default() -> bool {
    p2p_default_from(
        std::env::var("ZAKURO_P2P").ok().as_deref(),
        &crate::vpn::route(),
    )
}

/// [`p2p_default`] as a pure function: an explicit `env` value wins ("true"
/// or "1" is on, anything else off); unset, any mesh route turns P2P on. The
/// proxy route counts: its sidecar carries the broker's HTTP both ways, and
/// only QUIC stays off there (mesh-routes §3).
pub(crate) fn p2p_default_from(env: Option<&str>, route: &crate::vpn::Route) -> bool {
    match env {
        Some(v) => v == "true" || v == "1",
        None => !matches!(route, crate::vpn::Route::None),
    }
}

/// Defaults for a broker started from the CLI on a user's machine (`zc broker`,
/// `zc share`, the auto-spawned local broker). Library code reads plain env
/// vars, so these are applied to the process environment BEFORE the config is
/// built or a child is spawned (children inherit them):
///
/// - `ZAKURO_REQUIRE_VOUCHER=1` whenever billing credentials are present and
///   the operator has not set it. The fleet runs with the flag on and rejects
///   unvouched offers with 402, while a requesting broker only mints when ITS
///   flag is on — so a user broker with the default left unset could never
///   dispatch to the fleet, and the failure looked like a dead mesh.
pub fn apply_user_broker_defaults() {
    crate::credentials::load_into_env();
    // `BrokerConfig` and the voucher default below read `ZAKURO_API_URL` only,
    // so a broker under an explicit `ZAKURO_ENV` with no URL ran billing-free
    // ("no API credentials") although the profile names its hub. Fill it in.
    if let Some(url) = profile_api_url_default(|k| std::env::var(k).ok()) {
        std::env::set_var("ZAKURO_API_URL", url);
    }
    let set = |k: &str| {
        std::env::var(k)
            .map(|v| !v.trim().is_empty())
            .unwrap_or(false)
    };
    if set("ZAKURO_API_KEY")
        && set("ZAKURO_API_URL")
        && std::env::var("ZAKURO_REQUIRE_VOUCHER").is_err()
    {
        std::env::set_var("ZAKURO_REQUIRE_VOUCHER", "1");
    }
}

/// The hub URL a broker should use when `ZAKURO_API_URL` is unset but
/// `ZAKURO_ENV` explicitly names an environment: that profile's hub. `None`
/// when the URL is already set or no known `ZAKURO_ENV` is given, so a plain
/// local broker (no env, no URL) keeps running without a hub, as before.
pub(crate) fn profile_api_url_default(var: impl Fn(&str) -> Option<String>) -> Option<String> {
    let set = |k: &str| var(k).map(|v| !v.trim().is_empty()).unwrap_or(false);
    if set(crate::profile::ENV_API_URL) || !set(crate::profile::ENV_VAR) {
        return None;
    }
    if crate::profile::env_with(&var).1.is_some() {
        return None; // ZAKURO_ENV named no known environment
    }
    Some(crate::profile::resolve_with(var).api_url.0)
}

/// Build the client used to forward a job to a worker.
///
/// Pooled, and with a connect timeout so a worker this host cannot reach fails
/// instead of hanging the request (#242). `proxy` is the sidecar's CONNECT
/// proxy address (`host:port`), which is how a proxy-route host reaches a mesh
/// worker at all; `None` dials the worker directly.
pub(crate) fn build_forward_client(proxy: Option<&str>) -> reqwest::Client {
    let mut builder = reqwest::Client::builder()
        .pool_max_idle_per_host(32)
        .connect_timeout(std::time::Duration::from_secs(5));
    if let Some(addr) = proxy {
        if let Ok(proxy) = reqwest::Proxy::all(format!("http://{addr}")) {
            builder = builder.proxy(proxy);
        }
    }
    builder.build().expect("reqwest client builds with rustls")
}

/// A connection-pooled `ureq` agent for worker forwards, optionally dialing
/// through the sidecar's CONNECT proxy.
///
/// The sync counterpart of `build_forward_client`. No global timeout: the
/// forward sets its own per request (`forward_to_worker_with`), and baking one
/// into the agent would cap every request at whatever the first caller wanted.
/// The connect timeout does belong here -- a request-level deadline does not
/// bound the CONNECT.
pub(crate) fn build_forward_agent(proxy: Option<&str>) -> ureq::Agent {
    let mut cfg =
        ureq::Agent::config_builder().timeout_connect(Some(std::time::Duration::from_secs(5)));
    if let Some(addr) = proxy {
        if let Ok(proxy) = ureq::Proxy::new(&format!("http://{addr}")) {
            cfg = cfg.proxy(Some(proxy));
        }
    }
    ureq::Agent::new_with_config(cfg.build())
}

/// Main broker state shared across handlers
pub struct BrokerState {
    /// Worker registry
    pub workers: WorkerRegistry,
    /// Balance manager (legacy, kept for compatibility)
    pub balances: BalanceManager,
    /// Central ledger for balance management
    pub ledger: Ledger,
    /// Router for worker selection
    pub router: Router,
    /// The adaptive policy, running alongside the real one and routing
    /// nothing. Its ledger is what justifies switching, or does not.
    pub shadow: shadow::Shadow,
    /// Configuration
    pub config: BrokerConfig,
    /// Active request tracking (request_id -> worker_id)
    pub active_requests: DashMap<String, String>,
    /// Instance affinity registry (instance_id -> worker_id), bounded and
    /// self-expiring (see `affinity`). Used to pin RemoteProxy instances to the worker that created them.
    pub instance_registry: affinity::InstanceRegistry,
    /// Whether running in local mode (free execution)
    pub local_mode: std::sync::atomic::AtomicBool,
    /// Statistics collector behind `/stats`
    pub stats: Arc<StatsCollector>,
    /// Write-ahead log for crash recovery
    pub wal: Wal,
    /// This node's own WireGuard IP (for detecting local vs remote workers)
    pub own_wireguard_ip: Option<String>,
    /// P2P peer manager for broker-to-broker money operations
    pub peer_manager: PeerManager,
    /// Transaction flush buffer (batched PG writes in P2P mode)
    pub tx_buffer: TransactionBuffer,
    /// QUIC transport for high-throughput peer task offers (None if not started)
    pub quic: OnceLock<Arc<quic::QuicTransport>>,
    /// Actual QUIC port this broker is listening on (set after QUIC starts)
    pub quic_port: std::sync::atomic::AtomicU16,
    /// Shared connection-pooled HTTP client for forwarding to workers. Reused
    /// across requests so each /execute no longer pays a fresh TCP+TLS setup
    /// (audit M3/async-phase-2: connection pooling). Per-request timeouts are
    /// set on the request builder, not the agent.
    pub http_client: ureq::Agent,
    /// The same, but dialing through the sidecar's CONNECT proxy -- the route
    /// a worker on another node takes. `None` when this host reaches the mesh
    /// directly. Held for the same reason as `http_client`: without it every
    /// forward to a peer built a throwaway agent, and a throwaway agent is a
    /// throwaway connection pool, so each one paid a fresh TCP handshake
    /// across the mesh.
    pub http_client_mesh: Option<ureq::Agent>,
    /// Shared, connection-pooled async client for the /execute worker forward.
    pub async_http: reqwest::Client,
    /// The same client, but dialing through the sidecar's CONNECT proxy. On a
    /// proxy-route host that is the only route to a mesh worker; `None` when
    /// this host reaches the mesh directly (#242).
    pub async_http_mesh: Option<reqwest::Client>,
    /// Two-phase peer offers reserved but not yet committed/cancelled, keyed by
    /// task_id. A reserve holds a worker quota slot; commit executes, cancel (or
    /// TTL sweep) releases. See server::{reserve,cancel,take_committed,sweep}.
    pub pending_offers: DashMap<String, PendingOffer>,
    /// Voucher nonces a peer offer has been accepted under on this node, with
    /// the voucher's expiry (unix secs). A second offer under the same nonce is
    /// refused while the first is reserved, running or done, on HTTP and QUIC
    /// alike; a failed run or a cancel releases it (zc#423).
    pub claimed_voucher_nonces: DashMap<String, u64>,
    /// This node's Ed25519 signing identity — signs every outbound /peer/*
    /// request and is the zero-trust replacement for the shared peer key.
    pub node_key: Arc<node_identity::NodeKey>,
    /// Dashboard-synced roster of authorized node pubkeys → `revoked` flag.
    /// Empty = permissive (no signed peers known yet); populated by roster sync.
    pub node_roster: DashMap<String, bool>,
    /// Whether this node is registered with its hub, and why not when it is
    /// not — served on `/health` as `hub_registration` (see that module).
    pub hub_registration: hub_registration::HubRegistration,
    /// Anti-replay guard for inbound signed /peer/* requests.
    pub node_sig_guard: node_identity::ReplayGuard,
    /// Pins the HTTP `/peer/*` node-signature gate on or off instead of
    /// reading `ZAKURO_ENFORCE_NODE_SIG` per request. Unset in production;
    /// tests arm one broker without touching process env (zc#426).
    pub enforce_node_sig_override: std::sync::OnceLock<bool>,
    /// Cached SET of dashboard voucher-signing pubkeys (b64) for offline voucher
    /// verify: the current key plus any still-valid previous keys, so a hub key
    /// rotation does not invalidate vouchers signed just before it (#396).
    /// Empty = nothing cached yet.
    pub dash_voucher_pubkey: std::sync::RwLock<Vec<String>>,
    /// Unix secs of the last on-failure pubkey-set refresh (rate limit, #396).
    pub voucher_pubkey_refreshed_at: std::sync::atomic::AtomicU64,
    /// This broker's advertised price `P` (yen/hour). Live-mutable via
    /// `zc price <value>` / the local-only `POST /price` control endpoint.
    pub price: BrokerPrice,
    /// Requests currently inside `/execute`, `/execute/batch` or `/serve`
    /// (see [`InflightGuard`]). The idle-time benchmark only runs at zero.
    pub inflight: std::sync::atomic::AtomicU64,
    /// Epoch seconds when `inflight` last returned to zero (start-up counts).
    pub last_busy: std::sync::atomic::AtomicU64,
    /// Raised when a request arrives; a running benchmark stops at once.
    pub bench_cancel: Arc<std::sync::atomic::AtomicBool>,
}

/// Marks a request as in flight for its lifetime. The first one in (0 -> 1)
/// cancels any benchmark that is running, so a measurement never slows a job.
pub struct InflightGuard<'a>(&'a BrokerState);

impl BrokerState {
    fn inflight_enter(&self) {
        use std::sync::atomic::Ordering::SeqCst;
        if self.inflight.fetch_add(1, SeqCst) == 0 {
            self.bench_cancel.store(true, SeqCst);
        }
    }

    fn inflight_leave(&self) {
        use std::sync::atomic::Ordering::SeqCst;
        self.last_busy.store(node_identity::now_secs(), SeqCst);
        self.inflight.fetch_sub(1, SeqCst);
    }
}

impl<'a> InflightGuard<'a> {
    pub fn new(state: &'a BrokerState) -> Self {
        state.inflight_enter();
        InflightGuard(state)
    }
}

impl Drop for InflightGuard<'_> {
    fn drop(&mut self) {
        self.0.inflight_leave();
    }
}

/// [`InflightGuard`] that owns its handle on the state, for paid work whose
/// guard has to move between an auth step and the forward step (`/execute`
/// takes it only once the caller is authenticated).
pub struct InflightHold(Arc<BrokerState>);

impl InflightHold {
    pub fn new(state: &Arc<BrokerState>) -> Self {
        state.inflight_enter();
        InflightHold(state.clone())
    }
}

impl Drop for InflightHold {
    fn drop(&mut self) {
        self.0.inflight_leave();
    }
}

/// A peer task offer that was RESERVED (worker slot held) but not yet executed,
/// awaiting a two-phase commit or cancel.
pub struct PendingOffer {
    pub offer: task_board::TaskOffer,
    pub worker_id: String,
    pub created: std::time::Instant,
}

impl BrokerState {
    /// Create a new broker state with default configuration
    pub fn new() -> Self {
        Self::with_config(BrokerConfig::default())
    }

    /// Create a new broker state with custom configuration
    pub fn with_config(config: BrokerConfig) -> Self {
        let ledger = Ledger::new(config.api_url.clone(), config.api_key.clone());
        let wal = {
            // Candidate paths: ZAKURO_WAL_PATH env → $HOME/.zakuro/wal.jsonl → /tmp/zakuro-wal.jsonl
            let mut candidates: Vec<std::path::PathBuf> = vec![];
            if let Ok(p) = std::env::var("ZAKURO_WAL_PATH") {
                candidates.push(std::path::PathBuf::from(p));
            }
            if let Some(home) = std::env::var("HOME")
                .ok()
                .or_else(|| std::env::var("USERPROFILE").ok())
            {
                let dir = std::path::PathBuf::from(home).join(".zakuro");
                let _ = std::fs::create_dir_all(&dir);
                candidates.push(dir.join("wal.jsonl"));
            }
            candidates.push(std::path::PathBuf::from("/tmp/zakuro-wal.jsonl"));

            candidates
                .into_iter()
                .find_map(|p| Wal::open(p.to_str().unwrap_or("/tmp/zakuro-wal.jsonl")).ok())
                .expect("Failed to open WAL on any candidate path")
        };
        let own_wireguard_ip = config
            .wireguard_ip_override
            .clone()
            .or_else(discovery::get_effective_node_ip);

        // Peer broker list: from ZAKURO_PEERS, or discover on localhost when empty.
        // When using discovered brokers, ZAKURO_API_KEY is optional (P2P only, no dashboard).
        let peer_addresses: Vec<String> = if config.discovery.peers.is_empty()
            && config.enable_p2p
            && std::env::var("ZAKURO_DISCOVER_BROKER_PEERS").unwrap_or_else(|_| "true".into())
                != "false"
        {
            let mut discovered =
                discovery::discover_broker_peers_on_localhost(config.port, 9000, 9010);
            if !discovered.is_empty() {
                eprintln!(
                    "  [P2P] Discovered {} broker(s) on localhost (ZAKURO_API_KEY optional)",
                    discovered.len()
                );
            }

            // Best-effort: also probe the WireGuard mesh subnet for peer
            // brokers when this node is actually on the mesh. Strictly
            // gated behind P2P + a present mesh IP so this is a no-op
            // off-mesh; any failure here just means fewer peers, never a
            // hard error (see discover_broker_peers_on_mesh_subnet doc).
            if let Some(mesh_ip) = discovery::get_mesh_ip() {
                let mesh_peers = discovery::discover_broker_peers_on_mesh_subnet(
                    &mesh_ip,
                    config.peer_key.as_deref().unwrap_or(""),
                );
                if !mesh_peers.is_empty() {
                    eprintln!(
                        "  [P2P] Discovered {} broker(s) on the mesh subnet",
                        mesh_peers.len()
                    );
                }
                discovered.extend(mesh_peers);
            }

            discovered
        } else {
            config.discovery.peers.clone()
        };

        // This node's signing identity — shared with the peer manager so every
        // outbound /peer/* request is signed, and kept on state for inbound verify.
        let node_key = Arc::new(node_identity::NodeKey::load_or_create());

        // Initialize P2P peer manager
        let peer_key = config.peer_key.clone().unwrap_or_default();
        let hub_registration = hub_registration::HubRegistration::new(
            config.api_url.as_deref(),
            config.api_key.as_deref(),
        );
        let peer_manager = PeerManager::new_with_node_key(
            own_wireguard_ip.as_deref(),
            &peer_addresses,
            config.port,
            peer_key,
            config.enable_p2p,
            Some(node_key.clone()),
        );

        Self {
            shadow: shadow::Shadow::default(),
            workers: WorkerRegistry::new(),
            balances: BalanceManager::new(),
            ledger,
            // The router needs this node's identity to tell a pin that names
            // THIS broker from one that names another; without it every
            // `target_node` is unmatchable and spills onto the mesh.
            router: Router::new().with_node(node_key.fingerprint()),
            config,
            active_requests: DashMap::new(),
            instance_registry: affinity::InstanceRegistry::from_env(),
            local_mode: std::sync::atomic::AtomicBool::new(false),
            stats: Arc::new(StatsCollector::new()),
            wal,
            own_wireguard_ip,
            peer_manager,
            tx_buffer: TransactionBuffer::new(),
            quic: OnceLock::new(),
            quic_port: std::sync::atomic::AtomicU16::new(0),
            // Pooled client (no global timeout — set per-request in forward_to_worker).
            http_client: ureq::Agent::new_with_config(ureq::Agent::config_builder().build()),
            http_client_mesh: crate::vpn::mesh_proxy_addr()
                .as_deref()
                .map(|proxy| build_forward_agent(Some(proxy))),
            async_http: build_forward_client(None),
            async_http_mesh: crate::vpn::mesh_proxy_addr()
                .as_deref()
                .map(|proxy| build_forward_client(Some(proxy))),
            pending_offers: DashMap::new(),
            claimed_voucher_nonces: DashMap::new(),
            node_key,
            node_roster: DashMap::new(),
            hub_registration,
            node_sig_guard: node_identity::ReplayGuard::new(),
            enforce_node_sig_override: std::sync::OnceLock::new(),
            dash_voucher_pubkey: std::sync::RwLock::new(Vec::new()),
            voucher_pubkey_refreshed_at: std::sync::atomic::AtomicU64::new(0),
            price: BrokerPrice::from_env_or_default(),
            inflight: std::sync::atomic::AtomicU64::new(0),
            last_busy: std::sync::atomic::AtomicU64::new(node_identity::now_secs()),
            bench_cancel: Arc::new(std::sync::atomic::AtomicBool::new(false)),
        }
    }

    /// Check if running in local mode (free execution)
    pub fn is_local_mode(&self) -> bool {
        self.local_mode.load(std::sync::atomic::Ordering::Relaxed)
    }

    /// Set local mode
    pub fn set_local_mode(&self, local: bool) {
        self.local_mode
            .store(local, std::sync::atomic::Ordering::Relaxed);
    }

    /// Billing is only meaningful when a centralized authority (dashboard API)
    /// is configured.  Without it, balances are unbacked local numbers — so we
    /// skip all reserve / commit / balance-check logic and let every execution
    /// run for free.
    pub fn is_billing_enabled(&self) -> bool {
        // API mode: dashboard URL + key configured
        (self.config.api_url.is_some() && self.config.api_key.is_some())
        // Standalone mode: ZAKURO_MASTER_KEY is set (local ledger)
        || !std::env::var("ZAKURO_MASTER_KEY").unwrap_or_default().is_empty()
    }

    /// Check if a worker URI belongs to this node (local = free execution)
    pub fn is_local_worker(&self, worker_uri: &str) -> bool {
        if self.is_local_mode() {
            return true; // Everything is local in local mode
        }
        // Localhost workers are always local
        if worker_uri.contains("127.0.0.1") || worker_uri.contains("localhost") {
            return true;
        }
        if let Some(ref own_ip) = self.own_wireguard_ip {
            worker_uri.contains(own_ip)
        } else {
            false
        }
    }

    /// Verify this broker's identity with the dashboard at startup.
    ///
    /// - Calls `GET /api/auth/me/api-key` to resolve the real `zakuro_user_id`.
    /// - If `owner_user_id` is not set, auto-fills it from the dashboard.
    /// - If `owner_user_id` IS set but doesn't match the API key's user, overrides
    ///   it with the real value and warns.
    /// - Returns the verified `zakuro_user_id`, or `None` if verification failed.
    pub fn verify_owner_with_dashboard(&mut self) -> Option<String> {
        let api_url = self.config.api_url.as_ref()?;
        let api_key = self.config.api_key.as_ref()?;

        let url = format!("{}/api/auth/me/api-key", api_url.trim_end_matches('/'));
        let agent = ureq::Agent::new_with_config(
            ureq::Agent::config_builder()
                .timeout_global(Some(std::time::Duration::from_secs(10)))
                .build(),
        );

        match agent
            .get(&url)
            .header("Authorization", &format!("Bearer {}", api_key))
            .call()
        {
            Ok(resp) => {
                let body = resp.into_body().read_to_string().unwrap_or_default();
                let parsed: serde_json::Value = serde_json::from_str(&body).unwrap_or_default();
                if let Some(uid) = parsed["zakuro_user_id"].as_str() {
                    let derived = self.config.owner_user_id.as_deref();
                    if derived.is_some() && derived != Some(uid) {
                        // Key-derived ID takes precedence; dashboard mismatch is a warning only.
                        eprintln!("  [HANDSHAKE] owner_user_id mismatch: key-derived={}, dashboard={}. Using key-derived value.",
                                  derived.unwrap_or("?"), uid);
                        return derived.map(str::to_string);
                    }
                    if self.config.owner_user_id.is_none() {
                        self.config.owner_user_id = Some(uid.to_string());
                    }
                    Some(
                        self.config
                            .owner_user_id
                            .clone()
                            .unwrap_or_else(|| uid.to_string()),
                    )
                } else {
                    eprintln!("  [HANDSHAKE] Dashboard did not return zakuro_user_id");
                    None
                }
            }
            Err(e) => {
                eprintln!("  [HANDSHAKE] Failed to verify owner with dashboard: {}", e);
                None
            }
        }
    }
}

pub type SharedBrokerState = Arc<BrokerState>;

#[cfg(test)]
mod tests {
    #[test]
    fn a_broker_under_zakuro_env_gets_the_profile_hub_when_no_url_is_set() {
        let vars = |pairs: &'static [(&'static str, &'static str)]| {
            move |k: &str| {
                pairs
                    .iter()
                    .find(|(n, _)| *n == k)
                    .map(|(_, v)| v.to_string())
            }
        };
        assert_eq!(
            super::profile_api_url_default(vars(&[("ZAKURO_ENV", "lab")])).as_deref(),
            Some("http://hub.lab.zakuro.internal")
        );
        // An explicit URL wins; no env (or an unknown one) leaves the broker hub-less.
        assert_eq!(
            super::profile_api_url_default(vars(&[
                ("ZAKURO_ENV", "lab"),
                ("ZAKURO_API_URL", "http://hub:8000")
            ])),
            None
        );
        assert_eq!(super::profile_api_url_default(vars(&[])), None);
        assert_eq!(
            super::profile_api_url_default(vars(&[("ZAKURO_ENV", "dev")])),
            None
        );
    }

    use super::*;

    #[test]
    fn broker_state_has_shared_async_client() {
        let s = BrokerState::new();
        // Cloning a reqwest::Client is a cheap Arc bump over a shared pool; this
        // asserts the field exists and is usable.
        let _c: reqwest::Client = s.async_http.clone();
    }

    /// An explicit `ZAKURO_P2P` wins on every route; unset, the broker peers
    /// on either mesh route, the sidecar's included, and not without one.
    #[test]
    fn p2p_is_the_explicit_setting_else_on_for_any_mesh_route() {
        use crate::vpn::fixtures::{host_route, proxy_route};
        use crate::vpn::Route;
        for route in [Route::None, host_route(), proxy_route()] {
            assert!(p2p_default_from(Some("true"), &route), "{route:?}");
            assert!(p2p_default_from(Some("1"), &route), "{route:?}");
            assert!(!p2p_default_from(Some("false"), &route), "{route:?}");
            assert!(
                !p2p_default_from(Some(""), &route),
                "set but empty is an explicit off: {route:?}"
            );
        }
        assert!(!p2p_default_from(None, &Route::None));
        assert!(p2p_default_from(None, &host_route()));
        assert!(p2p_default_from(None, &proxy_route()));
    }
}