Skip to main content

krot_server/
server.rs

1//! [`Server`] — top-level orchestration.
2//!
3//! Two runtime modes:
4//!
5//! - **Single-core** (tests, small deployments): [`Server::start`] binds a
6//!   single QUIC endpoint on the configured address; [`Server::run`] or
7//!   [`Server::run_until`] drives its accept loop.
8//! - **Thread-per-core** (production): [`Server::run_multicore`] spawns one
9//!   OS thread per core, each running a `current_thread` tokio runtime with
10//!   its own `SO_REUSEPORT`-bound QUIC endpoint and HTTPS listener. Shared
11//!   state (registry, keys, admin, tls) is `Arc`'d across cores.
12//!
13//! Both modes accept a `tokio::sync::watch::Receiver<bool>` for graceful
14//! shutdown: flipping the watch to `true` causes every open session to emit
15//! `ServerBye { SERVER_SHUTDOWN }`, drain its send stream, and exit; the
16//! accept loop then closes the endpoint and returns.
17
18use std::net::SocketAddr;
19use std::sync::Arc;
20use std::time::Duration;
21
22use tokio::net::TcpListener;
23use tokio::sync::watch;
24use tokio::task::JoinSet;
25use tracing::{error, info, warn};
26
27use krot_proto::consts::DEFAULT_GRACE_PERIOD;
28use krot_proto::ErrorCode;
29use krot_transport::{install_crypto_provider, KrotEndpoint};
30
31use crate::admin::AdminTokenStore;
32use crate::config::{DomainTls, Mode, ServerConfig};
33use crate::domain::acme::{acquire_cert, ChallengeStore};
34use crate::domain::{new_challenge_store, run_http_router, run_https_router, TlsSource};
35use crate::error::ServerError;
36use crate::handshake::{self, HandshakeOutcome};
37use crate::identity::ServerIdentity;
38use crate::keys::KeyRegistry;
39use crate::metrics::ServerMetrics;
40use crate::peer_lookup::{NoOpPeerLookup, PeerLabelLookup};
41use crate::peers::PeerRegistry;
42use crate::rate::RateLimitState;
43use crate::registry::TunnelRegistry;
44use crate::session::{spawn_revocation_watcher, Session, SessionOutcome};
45use crate::sockets::{reuseport_tcp, reuseport_udp};
46
47use krot_proto::PubKey;
48use std::collections::HashMap;
49use std::sync::Mutex;
50
51/// Per-identity rate-limit table shared across worker threads.
52pub type RateTable = Mutex<HashMap<PubKey, Arc<RateLimitState>>>;
53
54/// How long a worker waits for in-flight connection tasks to drain after
55/// receiving the shutdown signal.
56const SHUTDOWN_DEADLINE: Duration = Duration::from_secs(5);
57
58/// §10 auth completion deadline. Applies to the entire handshake — from
59/// accept_bi() returning the Control stream to `AuthOk`/`EnrollOk` being
60/// sent. Prevents DoS-by-idle-handshake.
61const AUTH_DEADLINE: Duration = Duration::from_secs(10);
62
63/// How often the §7.3 reaper task walks the registry and drops expired
64/// dangling leases. Independent of `DEFAULT_GRACE_PERIOD` — smaller
65/// values shorten how long a fully-expired lease lingers in memory but
66/// cost more wakeups.
67const REAP_INTERVAL: Duration = Duration::from_secs(5);
68
69/// Wait until `rx` observes `true`. Avoids `wait_for`, which returns a
70/// `Ref` (`RwLockReadGuard`) that is `!Send` and would poison every future
71/// composed with it in `tokio::select!` when spawned onto a `JoinSet`.
72pub(crate) async fn wait_shutdown(mut rx: watch::Receiver<bool>) {
73    if *rx.borrow() {
74        return;
75    }
76    while rx.changed().await.is_ok() {
77        if *rx.borrow() {
78            return;
79        }
80    }
81}
82
83/// Everything a worker thread needs from the parent to run an accept loop.
84#[derive(Debug)]
85pub struct SharedState {
86    pub keys: Arc<KeyRegistry>,
87    pub admin: Arc<AdminTokenStore>,
88    pub registry: Arc<TunnelRegistry>,
89    pub config: ServerConfig,
90    pub tls: Arc<rustls::ServerConfig>,
91    pub apex: Option<String>,
92    pub https_bind: Option<SocketAddr>,
93    /// §9 per-identity rate-limit state. Populated lazily at handshake
94    /// time; entries are keyed by `PubKey`.
95    pub rates: Arc<RateTable>,
96    /// §16.3.2 static federated-peer list. Hot-reloaded on file change.
97    pub peers: Arc<PeerRegistry>,
98    /// §16.3.4 peer-label lookup oracle. Defaults to
99    /// [`NoOpPeerLookup`] — real deployments plug in an HTTP client
100    /// against peer §16.4 admin APIs via
101    /// [`Server::with_peer_lookup`]. Wrapped in a `Mutex` so
102    /// `with_peer_lookup` can swap the inner `Arc` after
103    /// `SharedState` has already been shared with the accept loop.
104    pub peer_lookup: Mutex<Arc<dyn PeerLabelLookup>>,
105    /// Process-wide counters + gauges surfaced on
106    /// `/admin/v1/metrics`. Every code path that observes a
107    /// countable event (auth success/failure, session outcome,
108    /// resume attempt, rate-limit trip, tunnel registration, admin
109    /// API call) does an `Ordering::Relaxed` fetch-add on the
110    /// corresponding atomic here.
111    pub metrics: Arc<ServerMetrics>,
112}
113
114/// A running KROT server: owns the endpoint and all shared state.
115#[derive(Debug)]
116pub struct Server {
117    endpoint: KrotEndpoint,
118    shared: Arc<SharedState>,
119    identity: Option<ServerIdentity>,
120    http_addr: Option<SocketAddr>,
121    https_addr: Option<SocketAddr>,
122    admin_api_addr: Option<SocketAddr>,
123    tcp_fallback_addr: Option<SocketAddr>,
124}
125
126impl Server {
127    /// Assemble and start a server. Spawns HTTP + HTTPS routers immediately
128    /// in DomainMode (so ACME HTTP-01 challenges can be answered during
129    /// cert acquisition).
130    pub async fn start(config: ServerConfig) -> Result<Self, ServerError> {
131        install_crypto_provider();
132
133        let keys = Arc::new(KeyRegistry::open(config.authorized_keys_path.clone())?);
134        // §13: hot-reload watcher. Spawned once per server; each mutation
135        // to authorized_keys re-parses and broadcasts revoked/changed
136        // pubkeys to any live session. Handle is intentionally dropped
137        // (detached) — the task lives for the lifetime of the runtime.
138        drop(Arc::clone(&keys).spawn_watcher());
139        // §16.3.2 static peer list. Same hot-reload pattern as keys.
140        let peers = Arc::new(PeerRegistry::open(config.peer_list_path.clone())?);
141        drop(Arc::clone(&peers).spawn_watcher());
142        let admin = Arc::new(AdminTokenStore::new(config.data_dir.clone()));
143        let port_pool = config.mode.tcp_port_pool();
144        let registry = Arc::new(TunnelRegistry::new(port_pool));
145        // §7.3 reaper: drop dangling tunnels whose grace deadline has
146        // passed. Runs for the lifetime of the runtime; explicit
147        // shutdown is out of scope since exit destroys the runtime.
148        spawn_reaper(Arc::clone(&registry));
149
150        let (tls_cfg, identity, http_addr, https_addr, apex, https_bind, https_listener) =
151            match &config.mode {
152                Mode::Ip { .. } => {
153                    let id = ServerIdentity::load_or_create(&config.data_dir)?;
154                    let tls = id.clone().into_rustls()?;
155                    (tls, Some(id), None, None, None, None, None)
156                }
157                Mode::Domain {
158                    apex,
159                    tls,
160                    http_bind,
161                    https_bind,
162                    ..
163                } => {
164                    let (tls_cfg, http_addr, https_addr, https_listener) = start_domain_plane(
165                        apex.clone(),
166                        tls.clone(),
167                        *http_bind,
168                        *https_bind,
169                        &config.data_dir,
170                        Arc::clone(&registry),
171                    )
172                    .await?;
173                    (
174                        tls_cfg,
175                        None,
176                        Some(http_addr),
177                        Some(https_addr),
178                        Some(apex.clone()),
179                        Some(*https_bind),
180                        Some(https_listener),
181                    )
182                }
183            };
184        let tls_arc = Arc::new(tls_cfg);
185
186        let endpoint = KrotEndpoint::server(config.bind, (*tls_arc).clone())?;
187
188        let shared = Arc::new(SharedState {
189            keys,
190            admin,
191            registry,
192            config,
193            tls: tls_arc,
194            apex: apex.clone(),
195            https_bind,
196            rates: Arc::new(Mutex::new(HashMap::new())),
197            peers,
198            peer_lookup: Mutex::new(Arc::new(NoOpPeerLookup)),
199            metrics: Arc::new(ServerMetrics::new()),
200        });
201
202        // §16.1.8: spawn the HTTPS router with the ALPN dispatcher
203        // now that SharedState exists. The dispatcher needs
204        // `SharedState` to plug `krot-tcp/1` connections into
205        // `handle_connection`.
206        if let (Some(listener), Some(apex)) = (https_listener, apex) {
207            let (never_tx, never_rx) = watch::channel(false);
208            let shared_https = Arc::clone(&shared);
209            tokio::spawn(async move {
210                let _never_tx = never_tx;
211                run_https_router(listener, apex, shared_https, never_rx).await;
212            });
213        }
214
215        // §16.4: bring the structured admin API up alongside the KROT
216        // accept loop if the config asks for it. Bound before we return
217        // Server so callers get the resolved local addr via
218        // `admin_api_addr()`.
219        let admin_api_addr = if let Some(bind) = shared.config.admin_bind {
220            let api_state = Arc::new(crate::admin_api::AdminApiState::new(
221                Arc::clone(&shared.keys),
222                Arc::clone(&shared.registry),
223                Arc::clone(&shared.admin),
224                Arc::clone(&shared.rates),
225                shared.config.admin_session_ttl,
226                Arc::clone(&shared.metrics),
227            ));
228            Some(crate::admin_api::spawn_admin_api(bind, api_state).await?)
229        } else {
230            None
231        };
232
233        // §16.1.4: optional TCP+TLS fallback listener. Uses the same
234        // rustls config as the QUIC endpoint but forces ALPN to
235        // `krot-tcp/1`. Bound before we return so tests can pull the
236        // ephemeral addr via `tcp_fallback_addr()`.
237        let tcp_fallback_addr = if let Some(bind) = shared.config.tcp_fallback_bind {
238            let listener =
239                krot_transport::TcpFallbackListener::bind(bind, (*shared.tls).clone()).await?;
240            let addr = listener.local_addr()?;
241            let shared_c = Arc::clone(&shared);
242            // Sanity: shutdown-aware accept loops for the QUIC path
243            // hook the graceful watch; for TCP fallback we spawn a
244            // simple detached loop whose lifetime is the runtime's,
245            // matching the admin API and reaper patterns.
246            //
247            // The watch sender is created ONCE outside the accept
248            // loop and kept alive for the loop's lifetime. If we
249            // recreated it per iteration, the previous iteration's
250            // receiver — moved into the spawned session task —
251            // would observe its sender drop between accepts and
252            // `wait_shutdown` would return, prematurely tearing
253            // down the session.
254            let (never_tx, never_rx) = watch::channel(false);
255            tokio::spawn(async move {
256                // Keep the sender live for the loop's lifetime.
257                let _never_tx = never_tx;
258                loop {
259                    match listener.accept().await {
260                        Ok(incoming) => {
261                            let shared_conn = Arc::clone(&shared_c);
262                            let session_shutdown = never_rx.clone();
263                            tokio::spawn(async move {
264                                if let Err(e) =
265                                    handle_connection(incoming, shared_conn, session_shutdown).await
266                                {
267                                    warn!("krot-tcp/1 connection ended with error: {e}");
268                                }
269                            });
270                        }
271                        Err(e) => {
272                            // Transient errors (EMFILE, ECONNABORTED,
273                            // ENOBUFS) must not permanently kill the
274                            // listener — back off briefly and retry.
275                            warn!("krot-tcp/1 listener accept: {e}");
276                            tokio::time::sleep(std::time::Duration::from_millis(100)).await;
277                        }
278                    }
279                }
280            });
281            Some(addr)
282        } else {
283            None
284        };
285
286        Ok(Self {
287            endpoint,
288            shared,
289            identity,
290            http_addr,
291            https_addr,
292            admin_api_addr,
293            tcp_fallback_addr,
294        })
295    }
296
297    /// §16.1.4: local address the `krot-tcp/1` fallback listener
298    /// bound to, when enabled.
299    #[must_use]
300    pub fn tcp_fallback_addr(&self) -> Option<SocketAddr> {
301        self.tcp_fallback_addr
302    }
303
304    /// §16.4: local address the admin API bound to, when enabled.
305    #[must_use]
306    pub fn admin_api_addr(&self) -> Option<SocketAddr> {
307        self.admin_api_addr
308    }
309
310    /// §16.3.4: swap in a custom [`PeerLabelLookup`] implementation.
311    /// The default is [`NoOpPeerLookup`] which never reports a
312    /// collision. Tests use this to inject a mock; real deployments
313    /// use it to plug in an HTTP client that queries peer §16.4 admin
314    /// APIs.
315    ///
316    /// Safe to call at any point before / during a running server —
317    /// the field is `Mutex<Arc<...>>` and this method swaps the inner
318    /// `Arc`. Concurrent registrations either see the old value or
319    /// the new one, not something torn.
320    #[must_use]
321    pub fn with_peer_lookup(self, lookup: Arc<dyn PeerLabelLookup>) -> Self {
322        *self.shared.peer_lookup.lock().unwrap() = lookup;
323        self
324    }
325
326    pub fn issue_admin_token(&self) -> Result<String, ServerError> {
327        self.shared.admin.issue()
328    }
329
330    pub fn issue_admin_token_if_needed(&self) -> Result<Option<String>, ServerError> {
331        if self.shared.config.issue_admin_token || self.shared.keys.is_empty() {
332            Ok(Some(self.shared.admin.issue()?))
333        } else {
334            Ok(None)
335        }
336    }
337
338    pub fn fingerprint_hex(&self) -> Option<&str> {
339        self.identity.as_ref().map(|i| i.spki_sha256.as_str())
340    }
341
342    pub fn local_addr(&self) -> std::io::Result<SocketAddr> {
343        self.endpoint.local_addr()
344    }
345
346    pub fn http_addr(&self) -> Option<SocketAddr> {
347        self.http_addr
348    }
349
350    pub fn https_addr(&self) -> Option<SocketAddr> {
351        self.https_addr
352    }
353
354    pub fn registry(&self) -> &Arc<TunnelRegistry> {
355        &self.shared.registry
356    }
357
358    pub fn shared(&self) -> &Arc<SharedState> {
359        &self.shared
360    }
361
362    /// Single-core accept loop, driven on the current tokio runtime.
363    /// Runs forever — for tests where the harness kills the task at the
364    /// end. For graceful shutdown, use [`Self::run_until`].
365    pub async fn run(&self) {
366        let (_never_tx, never_rx) = watch::channel(false);
367        self.run_until(never_rx).await;
368    }
369
370    /// Single-core accept loop with graceful shutdown: exits cleanly when
371    /// `shutdown` flips to `true` (or its sender is dropped).
372    pub async fn run_until(&self, shutdown: watch::Receiver<bool>) {
373        info!(addr = ?self.local_addr().ok(), "accepting connections");
374        let mut tasks: JoinSet<()> = JoinSet::new();
375        loop {
376            tokio::select! {
377                biased;
378                () = wait_shutdown(shutdown.clone()) => {
379                    info!("graceful shutdown");
380                    break;
381                }
382                incoming = self.endpoint.accept() => {
383                    match incoming {
384                        Some(inc) => {
385                            let shared = Arc::clone(&self.shared);
386                            let shutdown_c = shutdown.clone();
387                            tasks.spawn(async move {
388                                if let Err(e) = handle_connection(inc, shared, shutdown_c).await {
389                                    warn!("connection ended with error: {e}");
390                                }
391                            });
392                        }
393                        None => break,
394                    }
395                }
396                Some(_) = tasks.join_next(), if !tasks.is_empty() => {}
397            }
398        }
399        drain(tasks).await;
400        self.endpoint.close(0u32.into(), b"server shutdown");
401    }
402
403    /// Thread-per-core: spawn `cores` OS threads, each running an
404    /// independent `current_thread` tokio runtime with its own
405    /// `SO_REUSEPORT`-bound QUIC endpoint and (in DomainMode) its own
406    /// HTTPS listener.
407    ///
408    /// Consumes the initial single-endpoint bound in [`Self::start`] and
409    /// releases its socket before the workers rebind with `SO_REUSEPORT`.
410    ///
411    /// Blocks until all worker threads join. Graceful shutdown is triggered
412    /// by flipping `shutdown` to `true`.
413    pub fn run_multicore(
414        self,
415        cores: usize,
416        shutdown: &watch::Receiver<bool>,
417    ) -> Result<(), ServerError> {
418        assert!(cores >= 1, "run_multicore requires at least 1 core");
419        let quic_bind = self.shared.config.bind;
420        assert!(
421            quic_bind.port() != 0,
422            "run_multicore requires an explicit QUIC bind port"
423        );
424        if let Some(https_bind) = self.shared.https_bind {
425            assert!(
426                https_bind.port() != 0,
427                "run_multicore requires an explicit HTTPS bind port"
428            );
429        }
430
431        let shared = Arc::clone(&self.shared);
432        drop(self);
433
434        let mut handles = Vec::with_capacity(cores);
435        for core_id in 0..cores {
436            let shared = Arc::clone(&shared);
437            let shutdown = shutdown.clone();
438            let name = format!("krot-worker-{core_id}");
439            let handle = std::thread::Builder::new()
440                .name(name)
441                .spawn(move || {
442                    if let Err(e) = worker(core_id, shared, quic_bind, shutdown) {
443                        error!(core_id, "worker exited with error: {e}");
444                    }
445                })
446                .map_err(ServerError::Io)?;
447            handles.push(handle);
448        }
449        for h in handles {
450            let _ = h.join();
451        }
452        Ok(())
453    }
454}
455
456fn worker(
457    core_id: usize,
458    shared: Arc<SharedState>,
459    quic_bind: SocketAddr,
460    shutdown: watch::Receiver<bool>,
461) -> Result<(), ServerError> {
462    let rt = tokio::runtime::Builder::new_current_thread()
463        .enable_all()
464        .thread_name(format!("krot-worker-{core_id}"))
465        .on_thread_start(move || {
466            // §9.2: pin this worker's rate-limit partition. Set on
467            // every thread the runtime spawns so blocking-pool tasks
468            // also index into their owner-worker's bucket.
469            crate::rate::set_worker_id(core_id);
470        })
471        .build()?;
472    // Also set on the current (block_on) thread — `on_thread_start`
473    // fires only for tokio-owned threads.
474    crate::rate::set_worker_id(core_id);
475    rt.block_on(async move {
476        let udp = reuseport_udp(quic_bind)?;
477        let endpoint = KrotEndpoint::server_on_socket(udp, (*shared.tls).clone())?;
478        info!(core_id, addr = ?endpoint.local_addr().ok(), "worker: QUIC endpoint bound");
479
480        let https_task =
481            if let (Some(https_bind), Some(apex)) = (shared.https_bind, shared.apex.clone()) {
482                let tcp_std = reuseport_tcp(https_bind, 1024)?;
483                let tcp = TcpListener::from_std(tcp_std)?;
484                info!(core_id, addr = ?tcp.local_addr().ok(), "worker: HTTPS listener bound");
485                let (_never_tx, never_rx) = watch::channel(false);
486                // Keep the sender alive for the loop's lifetime; see
487                // §16.1.4 fallback-loop comment for why per-iter senders
488                // cause premature session shutdown.
489                let shared_https = Arc::clone(&shared);
490                let apex_c = apex.clone();
491                let never_rx_c = never_rx.clone();
492                let handle = tokio::spawn(async move {
493                    let _never_tx = _never_tx;
494                    run_https_router(tcp, apex_c, shared_https, never_rx_c).await;
495                });
496                Some(handle)
497            } else {
498                None
499            };
500
501        let mut tasks: JoinSet<()> = JoinSet::new();
502        loop {
503            tokio::select! {
504                biased;
505                () = wait_shutdown(shutdown.clone()) => {
506                    info!(core_id, "worker: graceful shutdown");
507                    break;
508                }
509                incoming = endpoint.accept() => {
510                    match incoming {
511                        Some(inc) => {
512                            let shared = Arc::clone(&shared);
513                            let shutdown_c = shutdown.clone();
514                            tasks.spawn(async move {
515                                if let Err(e) = handle_connection(inc, shared, shutdown_c).await {
516                                    warn!("connection ended with error: {e}");
517                                }
518                            });
519                        }
520                        None => break,
521                    }
522                }
523                Some(_) = tasks.join_next(), if !tasks.is_empty() => {}
524            }
525        }
526
527        info!(
528            core_id,
529            active = tasks.len(),
530            "worker: draining connections"
531        );
532        drain(tasks).await;
533        endpoint.close(0u32.into(), b"server shutdown");
534        if let Some(h) = https_task {
535            h.abort();
536        }
537        Ok::<_, ServerError>(())
538    })
539}
540
541/// Look up (or lazily create) the [`RateLimitState`] for `pubkey`,
542/// initialised from the entry's `bw=` / `quota=`. Existing state is
543/// preserved across reconnects so the period-quota counter keeps
544/// accumulating across a session drop / resume.
545fn ensure_rate_state(
546    shared: &SharedState,
547    pubkey: &PubKey,
548    entry: &crate::keys::AuthorizedEntry,
549) -> Arc<RateLimitState> {
550    let mut table = shared.rates.lock().unwrap();
551    Arc::clone(table.entry(*pubkey).or_insert_with(|| {
552        Arc::new(RateLimitState::from_entry_with_metrics(
553            entry.bw,
554            entry.quota,
555            shared.config.bw_partitions,
556            Some(Arc::clone(&shared.metrics)),
557        ))
558    }))
559}
560
561/// Spawn the §7.3 lease reaper: a single task that periodically walks
562/// the registry and drops any dangling tunnel whose grace deadline has
563/// passed. Detached (no handle) — its lifetime is the runtime's.
564fn spawn_reaper(registry: Arc<TunnelRegistry>) {
565    tokio::spawn(async move {
566        let mut ticker = tokio::time::interval(REAP_INTERVAL);
567        ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
568        loop {
569            ticker.tick().await;
570            let n = registry.reap_expired();
571            if n > 0 {
572                info!(reaped = n, "reaped expired dangling tunnels");
573            }
574        }
575    });
576}
577
578/// Wait for `tasks` to complete with a bounded deadline
579/// ([`SHUTDOWN_DEADLINE`]).
580async fn drain(mut tasks: JoinSet<()>) {
581    let drain_fut = async { while tasks.join_next().await.is_some() {} };
582    if tokio::time::timeout(SHUTDOWN_DEADLINE, drain_fut)
583        .await
584        .is_err()
585    {
586        warn!(remaining = tasks.len(), "shutdown deadline reached");
587    }
588}
589
590async fn start_domain_plane(
591    apex: String,
592    tls: DomainTls,
593    http_bind: SocketAddr,
594    https_bind: SocketAddr,
595    data_dir: &std::path::Path,
596    registry: Arc<TunnelRegistry>,
597) -> Result<
598    (
599        rustls::ServerConfig,
600        SocketAddr,
601        SocketAddr,
602        // The bound HTTPS listener, handed back to the caller to
603        // spawn once `SharedState` exists (the §16.1.8 ALPN
604        // dispatcher needs `SharedState`).
605        TcpListener,
606    ),
607    ServerError,
608> {
609    let http_listener = TcpListener::bind(http_bind).await?;
610    let http_addr = http_listener.local_addr()?;
611    let https_listener = TcpListener::bind(https_bind).await?;
612    let https_addr = https_listener.local_addr()?;
613
614    let challenges: ChallengeStore = new_challenge_store();
615    tokio::spawn(run_http_router(
616        http_listener,
617        apex.clone(),
618        Arc::clone(&registry),
619        Arc::clone(&challenges),
620    ));
621
622    let tls_cfg = match tls {
623        DomainTls::PemFile { cert, key } => {
624            (*TlsSource::from_pem_files(&cert, &key)?.server_config).clone()
625        }
626        DomainTls::Acme { contact, directory } => {
627            let (chain, key) = acquire_cert(
628                &contact,
629                &apex,
630                directory.as_deref(),
631                data_dir,
632                Arc::clone(&challenges),
633            )
634            .await?;
635            rustls::ServerConfig::builder()
636                .with_no_client_auth()
637                .with_single_cert(chain, key)?
638        }
639    };
640
641    Ok((tls_cfg, http_addr, https_addr, https_listener))
642}
643
644/// Public wrapper — same body as `handle_connection`, re-exported so
645/// domain-plane code (§16.1.8 HTTPS ALPN dispatcher) can plug an
646/// `Incoming` into the same session pipeline as the QUIC / TCP
647/// fallback accept loops.
648pub async fn handle_connection_public(
649    incoming: krot_transport::Incoming,
650    shared: Arc<SharedState>,
651    shutdown: watch::Receiver<bool>,
652) -> Result<(), ServerError> {
653    handle_connection(incoming, shared, shutdown).await
654}
655
656async fn handle_connection(
657    incoming: krot_transport::Incoming,
658    shared: Arc<SharedState>,
659    shutdown: watch::Receiver<bool>,
660) -> Result<(), ServerError> {
661    // Bump per-transport accept counter — before the handshake so a
662    // stuck peer that never completes still shows up in transport
663    // split, and dashboards can see the ratio.
664    use std::sync::atomic::Ordering::Relaxed as MetricRelaxed;
665    match incoming.kind() {
666        krot_transport::TransportKind::Quic => {
667            shared
668                .metrics
669                .transport_quic_accepted
670                .fetch_add(1, MetricRelaxed);
671        }
672        krot_transport::TransportKind::TcpFallback => {
673            shared
674                .metrics
675                .transport_tcp_fallback_accepted
676                .fetch_add(1, MetricRelaxed);
677        }
678    }
679    let connection = incoming.accept().await.map_err(ServerError::Transport)?;
680
681    let (mut send, mut recv) = connection
682        .accept_bi()
683        .await
684        .map_err(ServerError::Transport)?;
685
686    let handshake_outcome = tokio::time::timeout(
687        AUTH_DEADLINE,
688        handshake::perform(&mut send, &mut recv, &shared.keys, &shared.admin),
689    )
690    .await;
691    let Ok(res) = handshake_outcome else {
692        warn!("handshake exceeded {AUTH_DEADLINE:?}, closing");
693        shared
694            .metrics
695            .handshake_timed_out
696            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
697        connection.close(
698            u32::from(ErrorCode::AUTHENTICATION_FAILED.raw()),
699            b"handshake timeout",
700        );
701        return Ok(());
702    };
703    let outcome = res?;
704    match outcome {
705        HandshakeOutcome::EnrollmentDone { success } => {
706            use std::sync::atomic::Ordering::Relaxed;
707            if success {
708                shared.metrics.enrollment_ok.fetch_add(1, Relaxed);
709            } else {
710                shared.metrics.enrollment_rejected.fetch_add(1, Relaxed);
711            }
712            let _ = send.finish();
713            connection.close(0, b"enrolled");
714            Ok(())
715        }
716        HandshakeOutcome::Failed(code) => {
717            info!(?code, "handshake failed");
718            // Bucket the failure by the code the handshake picked.
719            let m = &shared.metrics;
720            use std::sync::atomic::Ordering::Relaxed;
721            if code == ErrorCode::UNKNOWN_IDENTITY {
722                m.handshake_unknown_identity.fetch_add(1, Relaxed);
723            } else if code == ErrorCode::AUTHENTICATION_FAILED {
724                m.handshake_signature_invalid.fetch_add(1, Relaxed);
725            } else if code == ErrorCode::TOKEN_EXPIRED || code == ErrorCode::ENROLL_DISABLED {
726                m.enrollment_rejected.fetch_add(1, Relaxed);
727            } else {
728                m.handshake_protocol_violation.fetch_add(1, Relaxed);
729            }
730            // The enclosed `code` is already the correct QUIC application
731            // close code per §5 / §7.1 — handshake::perform picks the right
732            // one at each failure site.
733            connection.close(u32::from(code.raw()), b"handshake failed");
734            Ok(())
735        }
736        HandshakeOutcome::Auth(ok) => {
737            use std::sync::atomic::Ordering::Relaxed;
738            shared.metrics.handshake_auth_ok.fetch_add(1, Relaxed);
739            let revoked = spawn_revocation_watcher(shared.keys.subscribe_revocations(), ok.pubkey);
740            // §9: resolve (or lazily create) the rate-limit state for
741            // this identity and install a fresh control-stream sender.
742            let rate_state = ensure_rate_state(&shared, &ok.pubkey, &ok.entry);
743            let (ctrl_tx, ctrl_rx) = tokio::sync::mpsc::unbounded_channel();
744            rate_state.attach_ctrl(ctrl_tx);
745
746            let session = Session {
747                pubkey: ok.pubkey,
748                session_id: ok.session_id,
749                connection: connection.clone(),
750                registry: Arc::clone(&shared.registry),
751                mode: shared.config.mode.clone(),
752                shutdown,
753                entry: ok.entry,
754                revoked,
755                rate: Arc::clone(&rate_state),
756                ctrl_rx,
757                peers: Arc::clone(&shared.peers),
758                peer_lookup: Arc::clone(&shared.peer_lookup.lock().unwrap()),
759                metrics: Arc::clone(&shared.metrics),
760            };
761            let result = session.run(&mut send, &mut recv).await;
762            rate_state.detach_ctrl();
763            match &result {
764                Ok(SessionOutcome::Dropped) => {
765                    shared.metrics.session_dropped.fetch_add(1, Relaxed);
766                    // §7.3: preserve the client's tunnels in `Dangling`
767                    // state so a fresh QUIC connection can resume them
768                    // within the grace window.
769                    let grace = shared.config.resume_grace.unwrap_or(DEFAULT_GRACE_PERIOD);
770                    shared.registry.mark_dangling(&ok.pubkey, grace);
771                }
772                Ok(SessionOutcome::Bye) => {
773                    shared.metrics.session_bye.fetch_add(1, Relaxed);
774                    shared.registry.remove_all_owned_by(&ok.pubkey);
775                }
776                Ok(SessionOutcome::Terminal) | Err(_) => {
777                    shared.metrics.session_terminal.fetch_add(1, Relaxed);
778                    shared.registry.remove_all_owned_by(&ok.pubkey);
779                }
780            }
781            connection.close(0, b"bye");
782            if let Err(e) = result {
783                error!("session error: {e}");
784            }
785            Ok(())
786        }
787    }
788}