Skip to main content

omni_dev/
snowflake.rs

1//! Account-agnostic Snowflake query engine hosted by the daemon.
2//!
3//! Each `(account, user)` keeps a **bounded pool** of authenticated sessions
4//! (see [`session`]). A query checks one out, applies any per-query context with
5//! `USE` (skipping `USE`s already in effect), runs concurrently with other
6//! checkouts, and returns it. This gives **concurrent queries on a single
7//! authentication identity** while still honoring per-query
8//! `warehouse`/`role`/`database`/`schema`, with the number of browser auths
9//! capped at the pool size and grown lazily.
10//!
11//! A background keep-alive heartbeat ([`SnowflakeEngine::start_heartbeat`])
12//! periodically heartbeats idle sessions so their master tokens stay valid and
13//! an idle pool never re-prompts browser SSO.
14//!
15//! This is the standalone engine, analogous to [`crate::browser`]; the daemon
16//! adapter lives in [`crate::daemon::services::snowflake`].
17
18pub mod client;
19pub mod session;
20
21use std::sync::{Mutex as StdMutex, MutexGuard};
22use std::time::Duration;
23
24use anyhow::{anyhow, bail, Context, Result};
25use chrono::TimeDelta;
26use serde::{Deserialize, Serialize};
27use serde_json::Value;
28use tokio::task::JoinHandle;
29use tokio_util::sync::CancellationToken;
30
31use crate::utils::env::EnvSource;
32use crate::utils::secret::Secret;
33use crate::utils::settings::Settings;
34use client::{
35    AuthMethod, BrowserConfig, BrowserLaunch, Error as ClientError, KeyPairConfig, Row,
36    SnowflakeClient, SnowflakeClientConfig, SnowflakeSession,
37};
38use session::{PoolRegistry, QueryContext, SessionInfo, SessionKey, SessionPool};
39
40/// Env var (with `~/.omni-dev/settings.json` fallback) for the default account.
41const ENV_ACCOUNT: &str = "SNOWFLAKE_ACCOUNT";
42/// Env var for the default user.
43const ENV_USER: &str = "SNOWFLAKE_USER";
44/// Env var overriding the API host (verbatim), instead of deriving
45/// `<account>.snowflakecomputing.com`. Needed for AWS/Azure PrivateLink
46/// endpoints and gov/custom hosts.
47const ENV_HOST: &str = "SNOWFLAKE_HOST";
48/// Env var for the default warehouse.
49const ENV_WAREHOUSE: &str = "SNOWFLAKE_WAREHOUSE";
50/// Env var for the default role.
51const ENV_ROLE: &str = "SNOWFLAKE_ROLE";
52/// Env var for the default database.
53const ENV_DATABASE: &str = "SNOWFLAKE_DATABASE";
54/// Env var for the default schema.
55const ENV_SCHEMA: &str = "SNOWFLAKE_SCHEMA";
56/// Env var for the per-`(account, user)` pool size (max concurrent sessions).
57const ENV_POOL_SIZE: &str = "SNOWFLAKE_POOL_SIZE";
58/// Env var for the per-request HTTP timeout (seconds).
59const ENV_HTTP_TIMEOUT: &str = "SNOWFLAKE_HTTP_TIMEOUT";
60/// Env var for the overall sign-in deadline (seconds).
61const ENV_AUTH_TIMEOUT: &str = "SNOWFLAKE_AUTH_TIMEOUT";
62/// Env var for the overall per-query deadline incl. async polling (seconds).
63const ENV_QUERY_TIMEOUT: &str = "SNOWFLAKE_QUERY_TIMEOUT";
64/// Env var for the idle-session keep-alive heartbeat interval (seconds; `0`
65/// disables the heartbeat).
66const ENV_HEARTBEAT_INTERVAL: &str = "SNOWFLAKE_HEARTBEAT_INTERVAL";
67/// Env var selecting the auth method: `externalbrowser` (default; interactive
68/// SSO), `programmatic_access_token`, or `snowflake_jwt` (both non-interactive).
69const ENV_AUTHENTICATOR: &str = "SNOWFLAKE_AUTHENTICATOR";
70/// Env var for the programmatic access token (PAT auth).
71const ENV_TOKEN: &str = "SNOWFLAKE_TOKEN";
72/// Env var for the path to an unencrypted PKCS#8 PEM private key (JWT auth).
73const ENV_PRIVATE_KEY_PATH: &str = "SNOWFLAKE_PRIVATE_KEY_PATH";
74/// Env var for an inline unencrypted PKCS#8 PEM private key (alternative to the
75/// path).
76const ENV_PRIVATE_KEY: &str = "SNOWFLAKE_PRIVATE_KEY";
77/// Env var for an encrypted key's passphrase. Recognized but not yet supported;
78/// setting it with the JWT method is a clear error.
79const ENV_PRIVATE_KEY_PASSPHRASE: &str = "SNOWFLAKE_PRIVATE_KEY_PASSPHRASE";
80/// Env var for the external-browser SSO launch command: a single command line
81/// with a `{url}` placeholder (or the URL is appended as a trailing arg when the
82/// placeholder is absent). Quote-aware (single/double quotes, backslash escapes)
83/// so program paths and argument values may contain spaces, e.g.
84/// `/Applications/Google Chrome.app/Contents/MacOS/Google Chrome
85/// --profile-directory="Profile 1" --new-window {url}`. Blank/unset opens the OS
86/// default handler ([`BrowserLaunch::Auto`]). Ignored by the non-interactive
87/// auth methods (PAT / key-pair JWT), which open no browser.
88const ENV_BROWSER_COMMAND: &str = "SNOWFLAKE_BROWSER_COMMAND";
89
90/// Default pool size when `SNOWFLAKE_POOL_SIZE` is unset: the max concurrent
91/// queries (and max browser auths) per `(account, user)`.
92const DEFAULT_POOL_SIZE: usize = 4;
93/// Default overall sign-in deadline: comfortably over the SSO callback wait so a
94/// genuine sign-in completes, but bounded so a hung auth can't hold the gate.
95const DEFAULT_AUTH_TIMEOUT: Duration = Duration::from_secs(150);
96/// Max characters of SQL shown in the "running" preview for a busy session.
97const SQL_PREVIEW_MAX: usize = 60;
98/// Default keep-alive heartbeat interval: a quarter of the default 3600s
99/// session-token validity (Snowflake's own drivers clamp their heartbeat
100/// frequency to 900–3600s), so an idle session renews comfortably before
101/// either token lapses.
102const DEFAULT_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(900);
103/// Extra margin past the heartbeat interval when deciding to proactively renew
104/// an idle session's token before it would lapse mid-cycle.
105const HEARTBEAT_RENEW_MARGIN_SECS: i64 = 60;
106
107/// Engine defaults, resolved from environment variables and then
108/// `~/.omni-dev/settings.json` (the Atlassian credential-resolution pattern).
109///
110/// Account/user/context are optional; a request supplies its own and falls back
111/// to these. There is **no** hardcoded account list or alias map.
112#[derive(Clone, Debug)]
113pub struct SnowflakeEngineConfig {
114    /// Default account when a request omits `--account`.
115    pub default_account: Option<String>,
116    /// Default user when a request omits `--user`.
117    pub default_user: Option<String>,
118    /// Override the API host (verbatim) instead of deriving
119    /// `<account>.snowflakecomputing.com`. Set for PrivateLink / gov / custom
120    /// hosts; applies to every session this engine creates.
121    pub default_host: Option<String>,
122    /// Default warehouse applied at session creation.
123    pub default_warehouse: Option<String>,
124    /// Default role applied at session creation.
125    pub default_role: Option<String>,
126    /// Default database applied at session creation.
127    pub default_database: Option<String>,
128    /// Default schema applied at session creation.
129    pub default_schema: Option<String>,
130    /// How sessions authenticate (SSO by default; PAT or key-pair JWT for
131    /// non-interactive/headless use). The credential is the same for every
132    /// session in every pool this engine creates.
133    pub auth: AuthMethod,
134    /// Max concurrent sessions (and browser auths) per `(account, user)`.
135    pub pool_size: usize,
136    /// Per-request HTTP timeout for REST calls.
137    pub http_timeout: Duration,
138    /// Overall deadline for one sign-in (SSO + login) so a hung auth can't hold
139    /// the shared auth gate indefinitely.
140    pub auth_timeout: Duration,
141    /// Overall deadline for one query (submit + async-result polling).
142    pub query_timeout: Duration,
143    /// How often the background task heartbeats idle sessions to keep their
144    /// master token alive. Zero disables the heartbeat.
145    pub heartbeat_interval: Duration,
146}
147
148impl Default for SnowflakeEngineConfig {
149    fn default() -> Self {
150        Self {
151            default_account: None,
152            default_user: None,
153            default_host: None,
154            default_warehouse: None,
155            default_role: None,
156            default_database: None,
157            default_schema: None,
158            auth: AuthMethod::ExternalBrowser(BrowserConfig::default()),
159            pool_size: DEFAULT_POOL_SIZE,
160            http_timeout: client::config::DEFAULT_HTTP_TIMEOUT,
161            auth_timeout: DEFAULT_AUTH_TIMEOUT,
162            query_timeout: client::config::DEFAULT_QUERY_TIMEOUT,
163            heartbeat_interval: DEFAULT_HEARTBEAT_INTERVAL,
164        }
165    }
166}
167
168impl SnowflakeEngineConfig {
169    /// Resolves defaults from env vars (then settings.json). Cheap and
170    /// side-effect-free; never authenticates.
171    ///
172    /// # Errors
173    ///
174    /// If `SNOWFLAKE_AUTHENTICATOR` names an unknown method or a selected
175    /// non-interactive method is missing its credential (see
176    /// [`resolve_auth_method`]).
177    pub fn from_env_and_settings() -> Result<Self> {
178        let settings = Settings::load().unwrap_or_default();
179        let pool_size = settings
180            .get_env_var(ENV_POOL_SIZE)
181            .and_then(|s| s.trim().parse::<usize>().ok())
182            .filter(|&n| n >= 1)
183            .unwrap_or(DEFAULT_POOL_SIZE);
184        let secs = |key: &str| {
185            settings
186                .get_env_var(key)
187                .and_then(|s| s.trim().parse::<u64>().ok())
188                .filter(|&n| n >= 1)
189                .map(Duration::from_secs)
190        };
191        let private_key_pem = match settings.get_env_var(ENV_PRIVATE_KEY_PATH) {
192            Some(path) => Some(
193                std::fs::read_to_string(&path)
194                    .with_context(|| format!("reading {ENV_PRIVATE_KEY_PATH} '{path}'"))?,
195            ),
196            None => settings.get_env_var(ENV_PRIVATE_KEY),
197        };
198        let auth = resolve_auth_method(
199            settings.get_env_var(ENV_AUTHENTICATOR).as_deref(),
200            settings.get_env_var(ENV_BROWSER_COMMAND),
201            settings.get_env_var(ENV_TOKEN),
202            private_key_pem,
203            settings.get_env_var(ENV_PRIVATE_KEY_PASSPHRASE),
204        )?;
205        Ok(Self {
206            default_account: settings.get_env_var(ENV_ACCOUNT),
207            default_user: settings.get_env_var(ENV_USER),
208            default_host: host_override_from(settings.get_env_var(ENV_HOST)),
209            default_warehouse: settings.get_env_var(ENV_WAREHOUSE),
210            default_role: settings.get_env_var(ENV_ROLE),
211            default_database: settings.get_env_var(ENV_DATABASE),
212            default_schema: settings.get_env_var(ENV_SCHEMA),
213            auth,
214            pool_size,
215            http_timeout: secs(ENV_HTTP_TIMEOUT).unwrap_or(client::config::DEFAULT_HTTP_TIMEOUT),
216            auth_timeout: secs(ENV_AUTH_TIMEOUT).unwrap_or(DEFAULT_AUTH_TIMEOUT),
217            query_timeout: secs(ENV_QUERY_TIMEOUT).unwrap_or(client::config::DEFAULT_QUERY_TIMEOUT),
218            heartbeat_interval: heartbeat_interval_from(
219                settings.get_env_var(ENV_HEARTBEAT_INTERVAL),
220            ),
221        })
222    }
223}
224
225/// Normalizes the `SNOWFLAKE_HOST` override: trims surrounding whitespace (e.g. a
226/// trailing newline from a `$(cat …)`-style value) and treats a blank value as
227/// unset, so an empty override never shadows the derived host.
228fn host_override_from(raw: Option<String>) -> Option<String> {
229    raw.map(|s| s.trim().to_string()).filter(|s| !s.is_empty())
230}
231
232/// Parses the heartbeat-interval setting: seconds, with `0` meaning disabled.
233/// Unset or unparseable values fall back to the default. (The `secs` helper in
234/// [`SnowflakeEngineConfig::from_env_and_settings`] rejects `0`, which here is
235/// a meaningful value.)
236fn heartbeat_interval_from(raw: Option<String>) -> Duration {
237    raw.and_then(|s| s.trim().parse::<u64>().ok())
238        .map_or(DEFAULT_HEARTBEAT_INTERVAL, Duration::from_secs)
239}
240
241/// Resolves the [`AuthMethod`] from the `SNOWFLAKE_AUTHENTICATOR` selector and
242/// the method-specific credential vars (`private_key_pem` is the already-read
243/// key material, from a file or inline). An unset or blank selector keeps
244/// external-browser SSO, preserving the pre-#1108 default. The PAT secret is
245/// trimmed (dropping a stray trailing newline from `$(cat …)`-style values).
246///
247/// `browser_command` (the raw `SNOWFLAKE_BROWSER_COMMAND` value) configures how
248/// external-browser SSO opens the sign-in URL: a non-blank value becomes a
249/// [`BrowserLaunch::Command`] (parsed by [`split_browser_command`]); blank/unset
250/// keeps [`BrowserLaunch::Auto`]. It is ignored by the non-interactive methods,
251/// which open no browser.
252///
253/// # Errors
254///
255/// If the selector is unknown, a non-interactive method is selected without its
256/// credential, an encrypted key passphrase is set (unsupported), or (for
257/// external-browser) `browser_command` is present but cannot be tokenized.
258fn resolve_auth_method(
259    authenticator: Option<&str>,
260    browser_command: Option<String>,
261    token: Option<String>,
262    private_key_pem: Option<String>,
263    passphrase: Option<String>,
264) -> Result<AuthMethod> {
265    let selector = authenticator.unwrap_or("").trim().to_ascii_lowercase();
266    match selector.as_str() {
267        "" | "externalbrowser" | "external_browser" => {
268            let launch = match browser_command
269                .map(|c| c.trim().to_string())
270                .filter(|c| !c.is_empty())
271            {
272                Some(cmd) => BrowserLaunch::Command(split_browser_command(&cmd)?),
273                None => BrowserLaunch::Auto,
274            };
275            Ok(AuthMethod::ExternalBrowser(BrowserConfig {
276                launch,
277                ..BrowserConfig::default()
278            }))
279        }
280        "programmatic_access_token" | "pat" => {
281            let token = token
282                .map(|t| t.trim().to_string())
283                .filter(|t| !t.is_empty())
284                .ok_or_else(|| {
285                    anyhow!("{ENV_AUTHENTICATOR}={selector} requires {ENV_TOKEN} to be set")
286                })?;
287            Ok(AuthMethod::ProgrammaticAccessToken {
288                token: Secret::from(token),
289            })
290        }
291        "snowflake_jwt" | "keypair" | "key_pair" | "jwt" => {
292            if passphrase.is_some_and(|p| !p.trim().is_empty()) {
293                bail!(
294                    "{ENV_PRIVATE_KEY_PASSPHRASE} is set, but encrypted private keys are not yet \
295                     supported; decrypt the key with `openssl pkcs8 -in key.p8 -out \
296                     key_unencrypted.p8` and unset {ENV_PRIVATE_KEY_PASSPHRASE}"
297                );
298            }
299            let private_key_pem = private_key_pem
300                .filter(|k| !k.trim().is_empty())
301                .ok_or_else(|| {
302                    anyhow!(
303                        "{ENV_AUTHENTICATOR}={selector} requires {ENV_PRIVATE_KEY_PATH} or \
304                         {ENV_PRIVATE_KEY}"
305                    )
306                })?;
307            Ok(AuthMethod::KeyPairJwt(KeyPairConfig {
308                private_key_pem: Secret::from(private_key_pem),
309            }))
310        }
311        other => Err(anyhow!(
312            "unknown {ENV_AUTHENTICATOR} '{other}' \
313             (expected externalbrowser, programmatic_access_token, or snowflake_jwt)"
314        )),
315    }
316}
317
318/// Tokenizes a `SNOWFLAKE_BROWSER_COMMAND` value into program + args with
319/// POSIX-style quoting so a program path or an argument value may contain
320/// spaces (`Google Chrome.app`, `--profile-directory="Profile 1"`):
321///
322/// - unquoted whitespace separates tokens;
323/// - single quotes take their contents literally;
324/// - double quotes group while a backslash escapes only `"` or `\` (so
325///   Windows-style paths keep their other backslashes);
326/// - an unquoted backslash escapes the next character.
327///
328/// The `{url}` placeholder is left intact here; the client's `open_browser`
329/// substitutes it (or appends the URL) at launch time.
330///
331/// # Errors
332///
333/// If a quote is left unterminated, or the value tokenizes to zero words.
334fn split_browser_command(raw: &str) -> Result<Vec<String>> {
335    let mut words: Vec<String> = Vec::new();
336    let mut current = String::new();
337    // Distinguishes an empty quoted arg (`""`) from "no arg accumulated yet".
338    let mut has_word = false;
339    let mut chars = raw.chars();
340
341    while let Some(c) = chars.next() {
342        match c {
343            c if c.is_whitespace() => {
344                if has_word {
345                    words.push(std::mem::take(&mut current));
346                    has_word = false;
347                }
348            }
349            '\'' => {
350                has_word = true;
351                loop {
352                    match chars.next() {
353                        Some('\'') => break,
354                        Some(ch) => current.push(ch),
355                        None => {
356                            bail!("{ENV_BROWSER_COMMAND} has an unterminated single quote: {raw}")
357                        }
358                    }
359                }
360            }
361            '"' => {
362                has_word = true;
363                loop {
364                    match chars.next() {
365                        Some('"') => break,
366                        Some('\\') => match chars.next() {
367                            Some(ch @ ('"' | '\\')) => current.push(ch),
368                            Some(ch) => {
369                                current.push('\\');
370                                current.push(ch);
371                            }
372                            None => bail!(
373                                "{ENV_BROWSER_COMMAND} has an unterminated double quote: {raw}"
374                            ),
375                        },
376                        Some(ch) => current.push(ch),
377                        None => {
378                            bail!("{ENV_BROWSER_COMMAND} has an unterminated double quote: {raw}")
379                        }
380                    }
381                }
382            }
383            '\\' => {
384                has_word = true;
385                match chars.next() {
386                    Some(ch) => current.push(ch),
387                    None => current.push('\\'),
388                }
389            }
390            ch => {
391                has_word = true;
392                current.push(ch);
393            }
394        }
395    }
396    if has_word {
397        words.push(current);
398    }
399    if words.is_empty() {
400        bail!("{ENV_BROWSER_COMMAND} is set but contains no command");
401    }
402    Ok(words)
403}
404
405/// A single arbitrary-SQL query request routed to the engine.
406///
407/// `account`/`user` and the per-query context default to the engine config when
408/// omitted. Serialized by the CLI client (after [`fill_defaults_from`]
409/// resolution) and deserialized from the daemon `query` op payload.
410///
411/// [`fill_defaults_from`]: QueryRequest::fill_defaults_from
412#[derive(Clone, Debug, Default, Deserialize, Serialize)]
413#[serde(default)]
414pub struct QueryRequest {
415    /// Target account; falls back to `SNOWFLAKE_ACCOUNT`.
416    #[serde(skip_serializing_if = "Option::is_none")]
417    pub account: Option<String>,
418    /// Authenticating user; falls back to `SNOWFLAKE_USER`.
419    #[serde(skip_serializing_if = "Option::is_none")]
420    pub user: Option<String>,
421    /// Per-query `USE WAREHOUSE` override.
422    #[serde(skip_serializing_if = "Option::is_none")]
423    pub warehouse: Option<String>,
424    /// Per-query `USE ROLE` override.
425    #[serde(skip_serializing_if = "Option::is_none")]
426    pub role: Option<String>,
427    /// Per-query `USE DATABASE` override.
428    #[serde(skip_serializing_if = "Option::is_none")]
429    pub database: Option<String>,
430    /// Per-query `USE SCHEMA` override.
431    #[serde(skip_serializing_if = "Option::is_none")]
432    pub schema: Option<String>,
433    /// The SQL to execute.
434    pub sql: String,
435}
436
437impl QueryRequest {
438    /// Fills each unset identity/context field from `env`.
439    ///
440    /// Called by the CLI client with a profile-aware source
441    /// ([`SettingsEnv`](crate::utils::settings::SettingsEnv)) so that
442    /// `--profile` / `OMNI_DEV_PROFILE` resolve in the invoking process —
443    /// the daemon's startup defaults then only back-fill requests that still
444    /// omit a field (e.g. bare socket clients). Explicit values are never
445    /// overwritten.
446    pub fn fill_defaults_from(&mut self, env: &impl EnvSource) {
447        let fill = |slot: &mut Option<String>, key: &str| {
448            if slot.is_none() {
449                *slot = env.var(key);
450            }
451        };
452        fill(&mut self.account, ENV_ACCOUNT);
453        fill(&mut self.user, ENV_USER);
454        fill(&mut self.warehouse, ENV_WAREHOUSE);
455        fill(&mut self.role, ENV_ROLE);
456        fill(&mut self.database, ENV_DATABASE);
457        fill(&mut self.schema, ENV_SCHEMA);
458    }
459    /// The per-query context overrides (the `Some` fields override the session
460    /// base context).
461    fn overrides(&self) -> QueryContext {
462        QueryContext {
463            warehouse: self.warehouse.clone(),
464            role: self.role.clone(),
465            database: self.database.clone(),
466            schema: self.schema.clone(),
467        }
468    }
469}
470
471/// The running keep-alive heartbeat loop: cancelled and awaited on shutdown.
472struct HeartbeatTask {
473    token: CancellationToken,
474    handle: JoinHandle<()>,
475}
476
477/// The account-agnostic Snowflake query engine: lazy multiplexed auth, bounded
478/// per-identity session pools, and concurrent arbitrary-SQL execution.
479///
480/// A background keep-alive heartbeat for idle sessions is started by
481/// [`start_heartbeat`](Self::start_heartbeat) and stopped by
482/// [`shutdown`](Self::shutdown).
483pub struct SnowflakeEngine {
484    config: SnowflakeEngineConfig,
485    registry: PoolRegistry,
486    heartbeat: StdMutex<Option<HeartbeatTask>>,
487}
488
489impl SnowflakeEngine {
490    /// Builds an engine. Cheap — no eager auth or I/O.
491    #[must_use]
492    pub fn new(config: SnowflakeEngineConfig) -> Self {
493        Self {
494            config,
495            registry: PoolRegistry::new(),
496            heartbeat: StdMutex::new(None),
497        }
498    }
499
500    /// Starts the background keep-alive heartbeat loop (idempotent).
501    ///
502    /// Every `heartbeat_interval` the loop heartbeats each pool's idle sessions
503    /// — renewing a session token about to lapse — so the server-side
504    /// `CLIENT_SESSION_KEEP_ALIVE` actually extends the master token and an
505    /// idle pool survives past the token TTL without a new browser SSO (#1107).
506    /// Busy sessions are skipped; the query path keeps them alive inline.
507    ///
508    /// No-op when the interval is zero (disabled) or when called outside a
509    /// tokio runtime. Stopped by [`shutdown`](Self::shutdown).
510    pub fn start_heartbeat(&self) {
511        let interval = self.config.heartbeat_interval;
512        if interval.is_zero() {
513            return;
514        }
515        if tokio::runtime::Handle::try_current().is_err() {
516            tracing::debug!("no tokio runtime; Snowflake keep-alive heartbeat not started");
517            return;
518        }
519        let mut guard = self.lock_heartbeat();
520        if guard.is_some() {
521            return;
522        }
523        let token = CancellationToken::new();
524        let loop_token = token.clone();
525        let registry = self.registry.clone();
526        let handle = tokio::spawn(async move {
527            loop {
528                tokio::select! {
529                    () = loop_token.cancelled() => break,
530                    () = tokio::time::sleep(interval) => {
531                        heartbeat_all_pools(&registry, interval).await;
532                    }
533                }
534            }
535        });
536        *guard = Some(HeartbeatTask { token, handle });
537    }
538
539    fn lock_heartbeat(&self) -> MutexGuard<'_, Option<HeartbeatTask>> {
540        self.heartbeat
541            .lock()
542            .unwrap_or_else(std::sync::PoisonError::into_inner)
543    }
544
545    /// A snapshot of every active pool.
546    #[must_use]
547    pub fn sessions(&self) -> Vec<SessionInfo> {
548        self.registry.snapshot()
549    }
550
551    /// The number of active pools (`(account, user)` identities).
552    #[must_use]
553    pub fn pool_count(&self) -> usize {
554        self.registry.len()
555    }
556
557    /// Evicts the pool for `(account, user)`. Returns whether one existed.
558    pub fn disconnect(&self, account: &str, user: &str) -> bool {
559        let key = SessionKey::new(normalize_account(account), user.trim());
560        self.registry.remove(&key).is_some()
561    }
562
563    /// Evicts the pool with the given id. Returns whether one existed.
564    pub fn disconnect_by_id(&self, id: u64) -> bool {
565        self.registry.remove_by_id(id).is_some()
566    }
567
568    /// Evicts every pool. Returns how many were removed.
569    pub fn disconnect_all(&self) -> usize {
570        self.registry.take_all().len()
571    }
572
573    /// Cancels the running query on the `(account, user)` pool — one specific
574    /// member when `member` is `Some`, else every busy member. Returns how many
575    /// statements an abort was issued for. Does **not** create a pool, so an
576    /// unknown identity cancels nothing.
577    ///
578    /// The pooled session frees itself promptly (its poll loop returns a cancelled
579    /// error within one poll interval) rather than waiting out the query timeout.
580    pub async fn cancel(&self, account: &str, user: &str, member: Option<u64>) -> usize {
581        let key = SessionKey::new(normalize_account(account), user.trim());
582        match self.registry.get(&key) {
583            Some(pool) => cancel_pool(&pool, member).await,
584            None => 0,
585        }
586    }
587
588    /// Like [`cancel`](Self::cancel) but selects the pool by its numeric id (as
589    /// shown by `sessions`).
590    pub async fn cancel_by_id(&self, id: u64, member: Option<u64>) -> usize {
591        match self.registry.get_by_id(id) {
592            Some(pool) => cancel_pool(&pool, member).await,
593            None => 0,
594        }
595    }
596
597    /// Cancels every running query across all pools. Returns how many statements
598    /// an abort was issued for.
599    pub async fn cancel_all(&self) -> usize {
600        let mut cancelled = 0;
601        for pool in self.registry.pools() {
602            cancelled += cancel_pool(&pool, None).await;
603        }
604        cancelled
605    }
606
607    /// Stops the keep-alive heartbeat, then drops every pool (and its sessions).
608    pub async fn shutdown(&self) {
609        let task = self.lock_heartbeat().take();
610        if let Some(task) = task {
611            task.token.cancel();
612            let _ = task.handle.await;
613        }
614        let pools = self.registry.take_all();
615        drop(pools);
616    }
617
618    /// Runs arbitrary SQL against the `(account, user)` pool, authenticating a
619    /// session on first use, and returns a self-describing
620    /// `{ statements: [ { columns, rows }, … ] }` payload — one entry per
621    /// `;`-separated statement, so a single-statement query is a one-element
622    /// `statements` array (multi-statement scripts set `MULTI_STATEMENT_COUNT`).
623    /// Concurrent calls run on separate pooled sessions (up to the pool size).
624    ///
625    /// # Errors
626    ///
627    /// Returns an error if no account/user can be resolved, a context flag is not
628    /// a valid identifier, authentication fails, or the query fails. On a
629    /// session-expiry error that session is discarded and the next query
630    /// re-authenticates.
631    pub async fn query(&self, req: QueryRequest) -> Result<Value> {
632        let account = normalize_account(
633            req.account
634                .as_deref()
635                .or(self.config.default_account.as_deref())
636                .ok_or_else(|| {
637                    anyhow!("no Snowflake account: pass --account or set SNOWFLAKE_ACCOUNT")
638                })?,
639        );
640        let user = req
641            .user
642            .as_deref()
643            .or(self.config.default_user.as_deref())
644            .ok_or_else(|| anyhow!("no Snowflake user: pass --user or set SNOWFLAKE_USER"))?
645            .trim()
646            .to_string();
647        validate_context(&req)?;
648
649        let key = SessionKey::new(account, user);
650        let overrides = req.overrides();
651        let pool = self.registry.get_or_create(&key, self.config.pool_size);
652
653        // Check out a session. The pool reuses an idle one when available
654        // (re-checking after the auth gate so a session freed mid-auth is reused),
655        // and only authenticates a new one — serialized to one browser at a time
656        // by the pool's shared auth gate — when none is idle and it is under
657        // capacity. The permit inside the checkout caps concurrency at pool_size.
658        let cfg = self.config.clone();
659        let create_key = key.clone();
660        let auth_timeout = self.config.auth_timeout;
661        let checkout = pool
662            .checkout(move || async move {
663                // Overall sign-in deadline so a hung auth releases the gate.
664                match tokio::time::timeout(
665                    auth_timeout,
666                    create_session_with_base(&create_key, &cfg),
667                )
668                .await
669                {
670                    Ok(result) => result,
671                    Err(_) => Err(ClientError::Auth(format!(
672                        "Snowflake sign-in timed out after {auth_timeout:?}"
673                    ))),
674                }
675            })
676            .await
677            .map_err(|e| {
678                anyhow::Error::new(e).context(format!(
679                    "failed to authenticate Snowflake session for {} / {}",
680                    key.account, key.user
681                ))
682            })?;
683
684        // Proactively renew a session whose token is about to expire, before use.
685        if checkout
686            .session()
687            .session_expiring_within(TimeDelta::seconds(120))
688            && checkout.session().renew().await.is_err()
689        {
690            pool.discard(checkout);
691            return Err(anyhow!(
692                "Snowflake session expired and was discarded — re-run the query to re-authenticate"
693            ));
694        }
695
696        // Record what this member is now running, so menus/status show it, and
697        // capture an abort handle so a concurrent `cancel` can stop it while the
698        // session is checked out (and thus unreachable through the pool slot).
699        pool.start_query(
700            checkout.id(),
701            sql_preview(&req.sql, SQL_PREVIEW_MAX),
702            Some(checkout.session().abort_handle()),
703        );
704
705        // Apply the requested context and run the query, transparently renewing
706        // the token and retrying once if it expires mid-flight.
707        let target = checkout.base().overlay(&overrides);
708        match run_with_renew(checkout.session(), checkout.current(), &target, &req.sql).await {
709            Ok(statements) => {
710                pool.touch();
711                pool.checkin(checkout, target);
712                Ok(client::rows_to_multi_payload(&statements))
713            }
714            Err(e) if e.is_session_expired() => {
715                pool.discard(checkout);
716                Err(anyhow!(
717                    "Snowflake session expired and was discarded — re-run the query to re-authenticate"
718                ))
719            }
720            Err(e) => {
721                // Log the underlying cause server-side and surface it to the
722                // client (the daemon reply uses the full anyhow chain).
723                tracing::warn!("Snowflake query failed: {e}");
724                // The session's context is uncertain after a failure; check in
725                // with an empty context so the next reuse re-applies every dimension.
726                pool.checkin(checkout, QueryContext::default());
727                Err(anyhow::Error::new(e).context("Snowflake query failed"))
728            }
729        }
730    }
731}
732
733impl Drop for SnowflakeEngine {
734    fn drop(&mut self) {
735        // Best-effort: an engine dropped without `shutdown()` must not leave the
736        // heartbeat loop running forever. Cancellation is sync; the task itself
737        // is detached and exits on its next select.
738        let task = self.lock_heartbeat().take();
739        if let Some(task) = task {
740            task.token.cancel();
741        }
742    }
743}
744
745/// Aborts running statements on one pool (a specific `member`, else all busy
746/// members), returning how many aborts were issued. Handles are snapshotted under
747/// the pool's (sync) slot lock and each abort is awaited **after** the lock is
748/// released. A failed abort is logged and skipped, not surfaced.
749async fn cancel_pool(pool: &SessionPool, member: Option<u64>) -> usize {
750    let handles = pool.abort_handles(member);
751    let mut cancelled = 0;
752    for handle in handles {
753        match handle.abort().await {
754            Ok(true) => cancelled += 1,
755            Ok(false) => {}
756            Err(e) => tracing::warn!(pool = pool.id(), "Snowflake query cancel failed: {e}"),
757        }
758    }
759    cancelled
760}
761
762/// Sends one keep-alive round to every pool's currently-idle sessions,
763/// discarding any session that is dead beyond renewal.
764async fn heartbeat_all_pools(registry: &PoolRegistry, interval: Duration) {
765    for pool in registry.pools() {
766        let checkouts = pool.checkout_all_idle();
767        if checkouts.is_empty() {
768            continue;
769        }
770        let total = checkouts.len();
771        let mut kept = 0usize;
772        for checkout in checkouts {
773            if keep_session_alive(checkout.session(), interval).await {
774                pool.restore(checkout);
775                kept += 1;
776            } else {
777                pool.discard(checkout);
778            }
779        }
780        tracing::debug!(
781            pool = pool.id(),
782            kept,
783            total,
784            "Snowflake keep-alive heartbeat round"
785        );
786    }
787}
788
789/// Keeps one idle session alive: proactively renews a session token that would
790/// lapse before the next tick (the heartbeat itself is authorized by the
791/// session token), then heartbeats so the server extends the master token.
792///
793/// Returns whether the session is still usable: `false` only when the master
794/// token has expired (a full re-auth is unavoidable), so the caller discards
795/// it. Transient errors keep the session for the next tick.
796async fn keep_session_alive(session: &SnowflakeSession, interval: Duration) -> bool {
797    let margin_secs = i64::try_from(interval.as_secs())
798        .unwrap_or(i64::MAX)
799        .saturating_add(HEARTBEAT_RENEW_MARGIN_SECS);
800    let margin = TimeDelta::try_seconds(margin_secs).unwrap_or(TimeDelta::MAX);
801    if session.session_expiring_within(margin) {
802        match session.renew().await {
803            Ok(()) => {}
804            Err(e) if e.is_session_expired() => {
805                tracing::warn!("Snowflake keep-alive: master token expired; discarding session");
806                return false;
807            }
808            Err(e) => {
809                // Transient: keep the session and try again next tick.
810                tracing::warn!("Snowflake keep-alive renew failed: {e}");
811                return true;
812            }
813        }
814    }
815    match session.heartbeat().await {
816        Ok(()) => true,
817        Err(e) if e.is_session_expired() => match session.renew().await {
818            Ok(()) => true,
819            Err(renew_err) if renew_err.is_session_expired() => {
820                tracing::warn!("Snowflake keep-alive: master token expired; discarding session");
821                false
822            }
823            Err(renew_err) => {
824                tracing::warn!("Snowflake keep-alive renew failed: {renew_err}");
825                true
826            }
827        },
828        Err(e) => {
829            tracing::warn!("Snowflake keep-alive heartbeat failed: {e}");
830            true
831        }
832    }
833}
834
835/// Authenticates a session (via the engine's configured auth method), enables
836/// keep-alive, and captures its base (account/user default) context.
837async fn create_session_with_base(
838    key: &SessionKey,
839    config: &SnowflakeEngineConfig,
840) -> client::Result<(SnowflakeSession, QueryContext)> {
841    let mut cfg = SnowflakeClientConfig::external_browser(&key.account, &key.user);
842    cfg.auth = config.auth.clone();
843    cfg.host = config.default_host.clone();
844    cfg.warehouse = config.default_warehouse.clone();
845    cfg.role = config.default_role.clone();
846    cfg.database = config.default_database.clone();
847    cfg.schema = config.default_schema.clone();
848    cfg.http_timeout = config.http_timeout;
849    cfg.query_timeout = config.query_timeout;
850
851    let client = SnowflakeClient::new(cfg)?;
852    let session = client.create_session().await?;
853    session
854        .query("ALTER SESSION SET CLIENT_SESSION_KEEP_ALIVE = true")
855        .await?;
856    let base = capture_base_context(&session).await?;
857    Ok((session, base))
858}
859
860/// Reads the session's effective default context so per-query overrides can
861/// later be reset back to it.
862async fn capture_base_context(session: &SnowflakeSession) -> client::Result<QueryContext> {
863    let rows = session
864        .query("SELECT CURRENT_WAREHOUSE(), CURRENT_ROLE(), CURRENT_DATABASE(), CURRENT_SCHEMA()")
865        .await?;
866    let Some(row) = rows.first() else {
867        return Ok(QueryContext::default());
868    };
869    Ok(QueryContext {
870        warehouse: row.raw_at(0).map(str::to_string),
871        role: row.raw_at(1).map(str::to_string),
872        database: row.raw_at(2).map(str::to_string),
873        schema: row.raw_at(3).map(str::to_string),
874    })
875}
876
877/// Applies the context and runs the SQL, transparently renewing the session
878/// token (via the master token) and retrying once if it expired mid-flight.
879async fn run_with_renew(
880    session: &SnowflakeSession,
881    current: &QueryContext,
882    target: &QueryContext,
883    sql: &str,
884) -> client::Result<Vec<Vec<Row>>> {
885    match apply_and_query(session, current, target, sql).await {
886        Err(e) if e.is_session_expired() => {
887            session.renew().await?;
888            // Re-apply the full context on the renewed session, then retry.
889            apply_and_query(session, &QueryContext::default(), target, sql).await
890        }
891        other => other,
892    }
893}
894
895/// Issues any needed `USE` statements, then runs the SQL — which may itself be a
896/// multi-statement script, so this returns one row set per statement.
897async fn apply_and_query(
898    session: &SnowflakeSession,
899    current: &QueryContext,
900    target: &QueryContext,
901    sql: &str,
902) -> client::Result<Vec<Vec<Row>>> {
903    apply_context(session, current, target).await?;
904    session.query_multi(sql).await
905}
906
907/// Issues `USE` for each context dimension whose target differs from the
908/// session's current value. Target names are either validated user overrides or
909/// Snowflake-reported base names.
910async fn apply_context(
911    session: &SnowflakeSession,
912    current: &QueryContext,
913    target: &QueryContext,
914) -> client::Result<()> {
915    for (keyword, cur, tgt) in [
916        (
917            "WAREHOUSE",
918            current.warehouse.as_deref(),
919            target.warehouse.as_deref(),
920        ),
921        ("ROLE", current.role.as_deref(), target.role.as_deref()),
922        (
923            "DATABASE",
924            current.database.as_deref(),
925            target.database.as_deref(),
926        ),
927        (
928            "SCHEMA",
929            current.schema.as_deref(),
930            target.schema.as_deref(),
931        ),
932    ] {
933        if let Some(name) = tgt {
934            if cur != Some(name) {
935                session
936                    .query(format!("USE {keyword} {name}").as_str())
937                    .await?;
938            }
939        }
940    }
941    Ok(())
942}
943
944/// Normalizes an account identifier for keying (Snowflake is case-insensitive).
945fn normalize_account(account: &str) -> String {
946    account.trim().to_ascii_uppercase()
947}
948
949/// A single-line, length-bounded preview of SQL for the "running" display
950/// (collapses whitespace/newlines; appends `…` when truncated).
951fn sql_preview(sql: &str, max: usize) -> String {
952    let collapsed = sql.split_whitespace().collect::<Vec<_>>().join(" ");
953    if collapsed.chars().count() > max {
954        let head: String = collapsed.chars().take(max).collect();
955        format!("{}…", head.trim_end())
956    } else {
957        collapsed
958    }
959}
960
961/// Validates every present context flag as a safe Snowflake identifier before it
962/// is interpolated into a `USE …` statement.
963fn validate_context(req: &QueryRequest) -> Result<()> {
964    for (name, value) in [
965        ("warehouse", req.warehouse.as_deref()),
966        ("role", req.role.as_deref()),
967        ("database", req.database.as_deref()),
968        ("schema", req.schema.as_deref()),
969    ] {
970        if let Some(value) = value {
971            validate_identifier(name, value)?;
972        }
973    }
974    Ok(())
975}
976
977/// Rejects context values that are not bare Snowflake identifiers (letters,
978/// digits, `_`, `$`, `.`), so a `--warehouse` flag cannot smuggle extra SQL into
979/// the `USE …` statement.
980fn validate_identifier(field: &str, value: &str) -> Result<()> {
981    if value.is_empty() {
982        bail!("--{field} must not be empty");
983    }
984    if !value
985        .chars()
986        .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '$' | '.'))
987    {
988        bail!(
989            "--{field} '{value}' is not a valid Snowflake identifier \
990             (allowed: letters, digits, '_', '$', '.')"
991        );
992    }
993    Ok(())
994}
995
996#[cfg(test)]
997#[allow(clippy::unwrap_used, clippy::expect_used)]
998mod tests {
999    use super::*;
1000    use crate::test_support::env::MapEnv;
1001
1002    #[test]
1003    fn default_config_has_a_nonzero_pool_size() {
1004        assert!(SnowflakeEngineConfig::default().pool_size >= 1);
1005    }
1006
1007    #[test]
1008    fn heartbeat_interval_from_parses_seconds_zero_and_garbage() {
1009        assert_eq!(heartbeat_interval_from(None), DEFAULT_HEARTBEAT_INTERVAL);
1010        assert_eq!(
1011            heartbeat_interval_from(Some("300".to_string())),
1012            Duration::from_secs(300)
1013        );
1014        // `0` is meaningful: it disables the heartbeat.
1015        assert_eq!(
1016            heartbeat_interval_from(Some(" 0 ".to_string())),
1017            Duration::ZERO
1018        );
1019        assert_eq!(
1020            heartbeat_interval_from(Some("garbage".to_string())),
1021            DEFAULT_HEARTBEAT_INTERVAL
1022        );
1023    }
1024
1025    #[test]
1026    fn host_override_from_trims_and_treats_blank_as_unset() {
1027        assert_eq!(host_override_from(None), None);
1028        assert_eq!(host_override_from(Some("   ".to_string())), None);
1029        assert_eq!(
1030            host_override_from(Some(
1031                "  acct.privatelink.snowflakecomputing.com\n".to_string()
1032            )),
1033            Some("acct.privatelink.snowflakecomputing.com".to_string())
1034        );
1035    }
1036
1037    #[test]
1038    fn split_browser_command_splits_on_unquoted_whitespace() {
1039        assert_eq!(
1040            split_browser_command("chrome --new-window {url}").unwrap(),
1041            vec!["chrome", "--new-window", "{url}"]
1042        );
1043    }
1044
1045    #[test]
1046    fn split_browser_command_keeps_quoted_spaces_together() {
1047        // The motivating case: a program path and an argument value both with
1048        // spaces, plus the `{url}` placeholder left intact for later substitution.
1049        assert_eq!(
1050            split_browser_command(
1051                "'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' \
1052                 --profile-directory=\"Profile 1\" --new-window {url}"
1053            )
1054            .unwrap(),
1055            vec![
1056                "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
1057                "--profile-directory=Profile 1",
1058                "--new-window",
1059                "{url}",
1060            ]
1061        );
1062    }
1063
1064    #[test]
1065    fn split_browser_command_handles_backslash_escapes() {
1066        // An unquoted backslash escapes the next char; inside double quotes only
1067        // `\"` and `\\` are escapes, so other backslashes (Windows paths) survive.
1068        assert_eq!(
1069            split_browser_command(r#"chrome a\ b "c\"d" "e\\f" "g\h""#).unwrap(),
1070            vec!["chrome", "a b", "c\"d", "e\\f", "g\\h"]
1071        );
1072    }
1073
1074    #[test]
1075    fn split_browser_command_rejects_unterminated_quotes() {
1076        assert!(split_browser_command("chrome \"--flag").is_err());
1077        assert!(split_browser_command("chrome '--flag").is_err());
1078    }
1079
1080    #[test]
1081    fn split_browser_command_rejects_an_empty_command() {
1082        assert!(split_browser_command("   ").is_err());
1083        assert!(split_browser_command("").is_err());
1084    }
1085
1086    #[test]
1087    fn resolve_auth_method_defaults_to_external_browser() {
1088        // Unset, blank, and the explicit name all keep interactive SSO.
1089        for selector in [
1090            None,
1091            Some(""),
1092            Some("  "),
1093            Some("externalbrowser"),
1094            Some("EXTERNALBROWSER"),
1095        ] {
1096            assert!(matches!(
1097                resolve_auth_method(selector, None, None, None, None).unwrap(),
1098                AuthMethod::ExternalBrowser(BrowserConfig {
1099                    launch: BrowserLaunch::Auto,
1100                    ..
1101                })
1102            ));
1103        }
1104    }
1105
1106    #[test]
1107    fn resolve_auth_method_threads_browser_command() {
1108        let auth = resolve_auth_method(
1109            Some("externalbrowser"),
1110            Some(
1111                "\"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome\" \
1112                 --profile-directory=\"Profile 1\" --new-window {url}"
1113                    .to_string(),
1114            ),
1115            None,
1116            None,
1117            None,
1118        )
1119        .unwrap();
1120        let AuthMethod::ExternalBrowser(BrowserConfig {
1121            launch: BrowserLaunch::Command(args),
1122            ..
1123        }) = auth
1124        else {
1125            panic!("expected an external-browser Command launch");
1126        };
1127        assert_eq!(
1128            args,
1129            vec![
1130                "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome".to_string(),
1131                "--profile-directory=Profile 1".to_string(),
1132                "--new-window".to_string(),
1133                "{url}".to_string(),
1134            ]
1135        );
1136    }
1137
1138    #[test]
1139    fn resolve_auth_method_blank_browser_command_is_auto() {
1140        // A blank/whitespace command is treated as unset, not a parse error.
1141        assert!(matches!(
1142            resolve_auth_method(None, Some("   ".to_string()), None, None, None).unwrap(),
1143            AuthMethod::ExternalBrowser(BrowserConfig {
1144                launch: BrowserLaunch::Auto,
1145                ..
1146            })
1147        ));
1148    }
1149
1150    #[test]
1151    fn resolve_auth_method_rejects_a_malformed_browser_command() {
1152        let err = resolve_auth_method(None, Some("chrome \"--flag".to_string()), None, None, None)
1153            .unwrap_err();
1154        assert!(err.to_string().contains(ENV_BROWSER_COMMAND));
1155    }
1156
1157    #[test]
1158    fn resolve_auth_method_ignores_browser_command_for_non_interactive_methods() {
1159        // PAT and JWT open no browser, so a set command is harmlessly ignored.
1160        assert!(matches!(
1161            resolve_auth_method(
1162                Some("pat"),
1163                Some("chrome {url}".to_string()),
1164                Some("tok".to_string()),
1165                None,
1166                None,
1167            )
1168            .unwrap(),
1169            AuthMethod::ProgrammaticAccessToken { .. }
1170        ));
1171        assert!(matches!(
1172            resolve_auth_method(
1173                Some("snowflake_jwt"),
1174                Some("chrome {url}".to_string()),
1175                None,
1176                Some("pem".to_string()),
1177                None,
1178            )
1179            .unwrap(),
1180            AuthMethod::KeyPairJwt(_)
1181        ));
1182    }
1183
1184    #[test]
1185    fn resolve_auth_method_reads_pat_and_trims_it() {
1186        let auth = resolve_auth_method(
1187            Some("programmatic_access_token"),
1188            None,
1189            Some("  tok-123\n".to_string()),
1190            None,
1191            None,
1192        )
1193        .unwrap();
1194        let AuthMethod::ProgrammaticAccessToken { token } = auth else {
1195            panic!("expected a PAT auth method");
1196        };
1197        assert_eq!(token.expose_secret(), "tok-123");
1198    }
1199
1200    #[test]
1201    fn resolve_auth_method_accepts_the_pat_alias() {
1202        assert!(matches!(
1203            resolve_auth_method(Some("pat"), None, Some("t".to_string()), None, None).unwrap(),
1204            AuthMethod::ProgrammaticAccessToken { .. }
1205        ));
1206    }
1207
1208    #[test]
1209    fn resolve_auth_method_errors_when_pat_is_missing_or_blank() {
1210        assert!(
1211            resolve_auth_method(Some("programmatic_access_token"), None, None, None, None).is_err()
1212        );
1213        assert!(
1214            resolve_auth_method(Some("pat"), None, Some("   ".to_string()), None, None).is_err()
1215        );
1216    }
1217
1218    #[test]
1219    fn resolve_auth_method_reads_key_pair_pem() {
1220        let auth = resolve_auth_method(
1221            Some("snowflake_jwt"),
1222            None,
1223            None,
1224            Some("-----BEGIN PRIVATE KEY-----\nAAAA\n-----END PRIVATE KEY-----".to_string()),
1225            None,
1226        )
1227        .unwrap();
1228        let AuthMethod::KeyPairJwt(cfg) = auth else {
1229            panic!("expected a key-pair auth method");
1230        };
1231        assert!(cfg
1232            .private_key_pem
1233            .expose_secret()
1234            .contains("BEGIN PRIVATE KEY"));
1235    }
1236
1237    #[test]
1238    fn resolve_auth_method_accepts_key_pair_aliases() {
1239        for selector in ["snowflake_jwt", "keypair", "key_pair", "jwt"] {
1240            assert!(matches!(
1241                resolve_auth_method(Some(selector), None, None, Some("pem".to_string()), None)
1242                    .unwrap(),
1243                AuthMethod::KeyPairJwt(_)
1244            ));
1245        }
1246    }
1247
1248    #[test]
1249    fn resolve_auth_method_errors_when_key_is_missing() {
1250        assert!(resolve_auth_method(Some("snowflake_jwt"), None, None, None, None).is_err());
1251        assert!(resolve_auth_method(
1252            Some("snowflake_jwt"),
1253            None,
1254            None,
1255            Some("  ".to_string()),
1256            None
1257        )
1258        .is_err());
1259    }
1260
1261    #[test]
1262    fn resolve_auth_method_rejects_an_encrypted_key_passphrase() {
1263        let err = resolve_auth_method(
1264            Some("snowflake_jwt"),
1265            None,
1266            None,
1267            Some("pem".to_string()),
1268            Some("hunter2".to_string()),
1269        )
1270        .unwrap_err();
1271        assert!(err.to_string().contains(ENV_PRIVATE_KEY_PASSPHRASE));
1272    }
1273
1274    #[test]
1275    fn resolve_auth_method_rejects_an_unknown_selector() {
1276        let err = resolve_auth_method(Some("carrier-pigeon"), None, None, None, None).unwrap_err();
1277        assert!(err.to_string().contains("carrier-pigeon"));
1278    }
1279
1280    #[tokio::test]
1281    async fn start_heartbeat_is_a_noop_when_disabled() {
1282        let engine = SnowflakeEngine::new(SnowflakeEngineConfig {
1283            heartbeat_interval: Duration::ZERO,
1284            ..SnowflakeEngineConfig::default()
1285        });
1286        engine.start_heartbeat();
1287        assert!(engine.lock_heartbeat().is_none());
1288    }
1289
1290    #[test]
1291    fn start_heartbeat_is_a_noop_outside_a_runtime() {
1292        let engine = SnowflakeEngine::new(SnowflakeEngineConfig::default());
1293        engine.start_heartbeat();
1294        assert!(engine.lock_heartbeat().is_none());
1295    }
1296
1297    #[tokio::test]
1298    async fn start_heartbeat_is_idempotent_and_shutdown_stops_it() {
1299        let engine = SnowflakeEngine::new(SnowflakeEngineConfig::default());
1300        engine.start_heartbeat();
1301        // Cancelling the running task's token lets a replacement be detected: a
1302        // second start must keep this task, not spawn (and orphan) a fresh one.
1303        engine.lock_heartbeat().as_ref().unwrap().token.cancel();
1304        engine.start_heartbeat();
1305        assert!(
1306            engine
1307                .lock_heartbeat()
1308                .as_ref()
1309                .unwrap()
1310                .token
1311                .is_cancelled(),
1312            "second start must not replace the running task"
1313        );
1314        engine.shutdown().await;
1315        assert!(engine.lock_heartbeat().is_none());
1316    }
1317
1318    #[tokio::test]
1319    async fn cancel_selectors_are_zero_on_an_empty_engine() {
1320        // No pools exist, so every selector cancels nothing — and crucially does
1321        // not create a pool (offline, no auth).
1322        let engine = SnowflakeEngine::new(SnowflakeEngineConfig::default());
1323        assert_eq!(engine.cancel("ACCT", "me", None).await, 0);
1324        assert_eq!(engine.cancel("ACCT", "me", Some(3)).await, 0);
1325        assert_eq!(engine.cancel_by_id(7, None).await, 0);
1326        assert_eq!(engine.cancel_all().await, 0);
1327        assert_eq!(engine.pool_count(), 0, "cancel must not create a pool");
1328    }
1329
1330    #[tokio::test]
1331    async fn cancel_aborts_a_running_query_on_a_pooled_session() {
1332        use serde_json::json;
1333        use wiremock::matchers::{method, path};
1334        use wiremock::{Mock, MockServer, ResponseTemplate};
1335
1336        let server = MockServer::start().await;
1337        // The query goes async and parks in the poll loop, so its statement stays
1338        // published (and thus cancellable) for the whole test.
1339        Mock::given(method("POST"))
1340            .and(path("/queries/v1/query-request"))
1341            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
1342                "success": true, "code": "333333", "data": { "getResultUrl": "/poll/1" }
1343            })))
1344            .mount(&server)
1345            .await;
1346        Mock::given(method("GET"))
1347            .and(path("/poll/1"))
1348            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
1349                "success": true, "code": "333333", "data": {}
1350            })))
1351            .mount(&server)
1352            .await;
1353        Mock::given(method("POST"))
1354            .and(path("/queries/v1/abort-request"))
1355            .respond_with(
1356                ResponseTemplate::new(200).set_body_json(json!({ "success": true, "data": {} })),
1357            )
1358            .mount(&server)
1359            .await;
1360
1361        // Inject a mock-backed session into a pool, bypassing the (live-only) SSO,
1362        // and record it as running with its abort handle — the state the engine's
1363        // `query` path would set.
1364        let engine = SnowflakeEngine::new(SnowflakeEngineConfig::default());
1365        let pool = engine
1366            .registry
1367            .get_or_create(&SessionKey::new("ACCT", "me"), 4);
1368        let pool_id = pool.id();
1369        let uri = server.uri();
1370        let checkout = pool
1371            .checkout(|| async {
1372                Ok::<_, std::convert::Infallible>((
1373                    client::test_session(&uri, Duration::from_secs(5)),
1374                    QueryContext::default(),
1375                ))
1376            })
1377            .await
1378            .unwrap();
1379        let member_id = checkout.id();
1380        pool.start_query(
1381            member_id,
1382            "SELECT LONG".to_string(),
1383            Some(checkout.session().abort_handle()),
1384        );
1385
1386        // Run the (parked) query concurrently with the cancels. `select!` returns
1387        // as soon as the cancels complete, dropping the still-running query.
1388        tokio::select! {
1389            _ = checkout.session().query("SELECT LONG") => panic!("query should stay parked"),
1390            counts = async {
1391                // Let the query publish its in-flight statement before cancelling.
1392                tokio::time::sleep(Duration::from_millis(100)).await;
1393                let by_pair = engine.cancel("ACCT", "me", None).await;
1394                let by_id = engine.cancel_by_id(pool_id, Some(member_id)).await;
1395                let by_all = engine.cancel_all().await;
1396                (by_pair, by_id, by_all)
1397            } => {
1398                assert_eq!(counts, (1, 1, 1), "every selector aborts the running statement");
1399            }
1400        }
1401
1402        // One abort was posted per cancel call.
1403        let aborts = server
1404            .received_requests()
1405            .await
1406            .unwrap()
1407            .iter()
1408            .filter(|r| r.url.path() == "/queries/v1/abort-request")
1409            .count();
1410        assert_eq!(aborts, 3);
1411    }
1412
1413    #[test]
1414    fn normalize_account_uppercases_and_trims() {
1415        assert_eq!(normalize_account("  my-org.acct  "), "MY-ORG.ACCT");
1416    }
1417
1418    #[test]
1419    fn validate_identifier_accepts_bare_identifiers() {
1420        for value in ["WH", "my_wh", "DB.SCHEMA", "wh$1"] {
1421            assert!(validate_identifier("warehouse", value).is_ok(), "{value}");
1422        }
1423    }
1424
1425    #[test]
1426    fn validate_identifier_rejects_injection_and_empty() {
1427        for value in ["", "wh; DROP TABLE t", "wh OR 1=1", "wh'", "a b"] {
1428            assert!(validate_identifier("warehouse", value).is_err(), "{value}");
1429        }
1430    }
1431
1432    #[test]
1433    fn fill_defaults_from_fills_unset_fields() {
1434        let env = MapEnv::new()
1435            .with(ENV_ACCOUNT, "ACCT")
1436            .with(ENV_USER, "me")
1437            .with(ENV_WAREHOUSE, "WH")
1438            .with(ENV_ROLE, "R")
1439            .with(ENV_DATABASE, "DB")
1440            .with(ENV_SCHEMA, "S");
1441        let mut req = QueryRequest {
1442            sql: "SELECT 1".to_string(),
1443            ..QueryRequest::default()
1444        };
1445        req.fill_defaults_from(&env);
1446        assert_eq!(req.account.as_deref(), Some("ACCT"));
1447        assert_eq!(req.user.as_deref(), Some("me"));
1448        assert_eq!(req.warehouse.as_deref(), Some("WH"));
1449        assert_eq!(req.role.as_deref(), Some("R"));
1450        assert_eq!(req.database.as_deref(), Some("DB"));
1451        assert_eq!(req.schema.as_deref(), Some("S"));
1452    }
1453
1454    #[test]
1455    fn fill_defaults_from_keeps_explicit_values() {
1456        let env = MapEnv::new()
1457            .with(ENV_ACCOUNT, "ENV_ACCT")
1458            .with(ENV_USER, "env_user");
1459        let mut req = QueryRequest {
1460            account: Some("FLAG_ACCT".to_string()),
1461            sql: "SELECT 1".to_string(),
1462            ..QueryRequest::default()
1463        };
1464        req.fill_defaults_from(&env);
1465        assert_eq!(req.account.as_deref(), Some("FLAG_ACCT"));
1466        assert_eq!(req.user.as_deref(), Some("env_user"));
1467    }
1468
1469    #[test]
1470    fn fill_defaults_from_leaves_unresolved_fields_none() {
1471        let mut req = QueryRequest {
1472            sql: "SELECT 1".to_string(),
1473            ..QueryRequest::default()
1474        };
1475        req.fill_defaults_from(&MapEnv::new());
1476        assert!(req.account.is_none());
1477        assert!(req.user.is_none());
1478        assert!(req.warehouse.is_none());
1479    }
1480
1481    #[test]
1482    fn query_request_serializes_without_none_fields() {
1483        let req = QueryRequest {
1484            account: Some("ACCT".to_string()),
1485            sql: "SELECT 1".to_string(),
1486            ..QueryRequest::default()
1487        };
1488        let value = serde_json::to_value(&req).unwrap();
1489        assert_eq!(
1490            value,
1491            serde_json::json!({ "account": "ACCT", "sql": "SELECT 1" })
1492        );
1493    }
1494
1495    #[test]
1496    fn validate_context_checks_each_present_flag() {
1497        let mut req = QueryRequest {
1498            sql: "SELECT 1".to_string(),
1499            ..QueryRequest::default()
1500        };
1501        assert!(validate_context(&req).is_ok());
1502        req.role = Some("good_role".to_string());
1503        assert!(validate_context(&req).is_ok());
1504        req.database = Some("bad; drop".to_string());
1505        assert!(validate_context(&req).is_err());
1506    }
1507
1508    #[tokio::test]
1509    async fn query_without_account_errors_without_network() {
1510        let engine = SnowflakeEngine::new(SnowflakeEngineConfig::default());
1511        let err = engine
1512            .query(QueryRequest {
1513                sql: "SELECT 1".to_string(),
1514                ..QueryRequest::default()
1515            })
1516            .await
1517            .unwrap_err();
1518        assert!(err.to_string().contains("account"));
1519    }
1520
1521    #[tokio::test]
1522    async fn query_without_user_errors_without_network() {
1523        let engine = SnowflakeEngine::new(SnowflakeEngineConfig {
1524            default_account: Some("ACCT".to_string()),
1525            ..SnowflakeEngineConfig::default()
1526        });
1527        let err = engine
1528            .query(QueryRequest {
1529                sql: "SELECT 1".to_string(),
1530                ..QueryRequest::default()
1531            })
1532            .await
1533            .unwrap_err();
1534        assert!(err.to_string().contains("user"));
1535    }
1536
1537    #[test]
1538    fn disconnect_and_sessions_on_empty_engine() {
1539        let engine = SnowflakeEngine::new(SnowflakeEngineConfig::default());
1540        assert_eq!(engine.pool_count(), 0);
1541        assert!(engine.sessions().is_empty());
1542        assert!(!engine.disconnect("ACCT", "user"));
1543        assert!(!engine.disconnect_by_id(1));
1544        assert_eq!(engine.disconnect_all(), 0);
1545    }
1546
1547    #[test]
1548    fn sql_preview_collapses_whitespace_and_truncates() {
1549        // Whitespace/newlines collapse to single spaces; short SQL is unchanged.
1550        assert_eq!(sql_preview("SELECT   1\n  FROM t", 60), "SELECT 1 FROM t");
1551        // Over-length SQL is truncated with an ellipsis.
1552        let long = format!("SELECT {}", "a".repeat(100));
1553        let preview = sql_preview(&long, 20);
1554        assert!(preview.ends_with('…'));
1555        assert!(preview.chars().count() <= 21, "{preview}");
1556    }
1557
1558    #[test]
1559    fn overrides_extracts_only_the_set_dimensions() {
1560        let req = QueryRequest {
1561            warehouse: Some("WH".to_string()),
1562            schema: Some("S".to_string()),
1563            sql: "SELECT 1".to_string(),
1564            ..QueryRequest::default()
1565        };
1566        let overrides = req.overrides();
1567        assert_eq!(overrides.warehouse.as_deref(), Some("WH"));
1568        assert_eq!(overrides.schema.as_deref(), Some("S"));
1569        assert!(overrides.role.is_none());
1570        assert!(overrides.database.is_none());
1571    }
1572
1573    mod orchestration {
1574        use super::*;
1575        use serde_json::json;
1576        use wiremock::matchers::{method, path};
1577        use wiremock::{Mock, MockServer, ResponseTemplate};
1578
1579        /// Mounts a `query-request` handler that returns `data` for every POST.
1580        async fn mount_query(server: &MockServer, data: serde_json::Value) {
1581            Mock::given(method("POST"))
1582                .and(path("/queries/v1/query-request"))
1583                .respond_with(
1584                    ResponseTemplate::new(200)
1585                        .set_body_json(json!({ "success": true, "data": data })),
1586                )
1587                .mount(server)
1588                .await;
1589        }
1590
1591        #[tokio::test]
1592        async fn capture_base_context_reads_current_context() {
1593            let server = MockServer::start().await;
1594            mount_query(
1595                &server,
1596                json!({
1597                    "rowtype": [
1598                        { "name": "CURRENT_WAREHOUSE()", "type": "text" },
1599                        { "name": "CURRENT_ROLE()", "type": "text" },
1600                        { "name": "CURRENT_DATABASE()", "type": "text" },
1601                        { "name": "CURRENT_SCHEMA()", "type": "text" },
1602                    ],
1603                    "rowset": [["WH", "R", "DB", "S"]],
1604                }),
1605            )
1606            .await;
1607            let session = client::test_session(&server.uri(), Duration::from_secs(5));
1608            let base = capture_base_context(&session).await.unwrap();
1609            assert_eq!(base.warehouse.as_deref(), Some("WH"));
1610            assert_eq!(base.role.as_deref(), Some("R"));
1611            assert_eq!(base.database.as_deref(), Some("DB"));
1612            assert_eq!(base.schema.as_deref(), Some("S"));
1613        }
1614
1615        #[tokio::test]
1616        async fn capture_base_context_defaults_when_no_rows() {
1617            let server = MockServer::start().await;
1618            mount_query(
1619                &server,
1620                json!({ "rowtype": [{ "name": "X", "type": "text" }], "rowset": [] }),
1621            )
1622            .await;
1623            let session = client::test_session(&server.uri(), Duration::from_secs(5));
1624            assert_eq!(
1625                capture_base_context(&session).await.unwrap(),
1626                QueryContext::default()
1627            );
1628        }
1629
1630        #[tokio::test]
1631        async fn apply_context_issues_use_only_for_differing_dimensions() {
1632            let server = MockServer::start().await;
1633            mount_query(&server, json!({ "rowtype": [], "rowset": [] })).await;
1634            let session = client::test_session(&server.uri(), Duration::from_secs(5));
1635
1636            let current = QueryContext {
1637                warehouse: Some("WH".to_string()),
1638                ..QueryContext::default()
1639            };
1640            let target = QueryContext {
1641                warehouse: Some("WH".to_string()), // same → no USE
1642                role: Some("R2".to_string()),      // differs → one USE
1643                ..QueryContext::default()
1644            };
1645            apply_context(&session, &current, &target).await.unwrap();
1646
1647            let reqs = server.received_requests().await.unwrap();
1648            assert_eq!(reqs.len(), 1, "only the differing dimension issues a USE");
1649            assert!(String::from_utf8_lossy(&reqs[0].body).contains("USE ROLE R2"));
1650        }
1651
1652        #[tokio::test]
1653        async fn run_with_renew_runs_without_renew_when_not_expired() {
1654            let server = MockServer::start().await;
1655            mount_query(
1656                &server,
1657                json!({ "rowtype": [{ "name": "N", "type": "text" }], "rowset": [["x"]] }),
1658            )
1659            .await;
1660            let session = client::test_session(&server.uri(), Duration::from_secs(5));
1661            let ctx = QueryContext::default();
1662            let rows = run_with_renew(&session, &ctx, &ctx, "SELECT 1")
1663                .await
1664                .unwrap();
1665            assert_eq!(rows.len(), 1);
1666        }
1667
1668        #[tokio::test]
1669        async fn run_with_renew_renews_and_retries_once_on_expiry() {
1670            let server = MockServer::start().await;
1671            // First query attempt: session expired (then this rule is exhausted).
1672            Mock::given(method("POST"))
1673                .and(path("/queries/v1/query-request"))
1674                .respond_with(ResponseTemplate::new(200).set_body_json(json!({
1675                    "success": false, "code": "390112", "message": "expired", "data": {}
1676                })))
1677                .up_to_n_times(1)
1678                .with_priority(1)
1679                .mount(&server)
1680                .await;
1681            // Renew succeeds.
1682            Mock::given(method("POST"))
1683                .and(path("/session/token-request"))
1684                .respond_with(ResponseTemplate::new(200).set_body_json(json!({
1685                    "success": true,
1686                    "data": { "sessionToken": "fresh", "validityInSecondsST": 3600 }
1687                })))
1688                .mount(&server)
1689                .await;
1690            // The retried query succeeds.
1691            Mock::given(method("POST"))
1692                .and(path("/queries/v1/query-request"))
1693                .respond_with(ResponseTemplate::new(200).set_body_json(json!({
1694                    "success": true,
1695                    "data": { "rowtype": [{ "name": "N", "type": "text" }], "rowset": [["x"]] }
1696                })))
1697                .with_priority(2)
1698                .mount(&server)
1699                .await;
1700
1701            let session = client::test_session(&server.uri(), Duration::from_secs(5));
1702            let ctx = QueryContext::default();
1703            let rows = run_with_renew(&session, &ctx, &ctx, "SELECT 1")
1704                .await
1705                .unwrap();
1706            assert_eq!(rows.len(), 1, "renewed and retried transparently");
1707        }
1708    }
1709
1710    mod keep_alive {
1711        use super::*;
1712        use serde_json::json;
1713        use wiremock::matchers::{method, path};
1714        use wiremock::{Mock, MockServer, ResponseTemplate};
1715
1716        /// A short interval so `session_expiring_within(interval + margin)` is
1717        /// false for the test session's fresh 3600s token.
1718        const INTERVAL: Duration = Duration::from_secs(60);
1719
1720        /// Mounts a `session/heartbeat` handler answering with `body`.
1721        async fn mount_heartbeat(server: &MockServer, body: serde_json::Value) {
1722            Mock::given(method("POST"))
1723                .and(path("/session/heartbeat"))
1724                .respond_with(ResponseTemplate::new(200).set_body_json(body))
1725                .mount(server)
1726                .await;
1727        }
1728
1729        /// Mounts a `session/token-request` (renew) handler answering with `body`.
1730        async fn mount_renew(server: &MockServer, body: serde_json::Value) {
1731            Mock::given(method("POST"))
1732                .and(path("/session/token-request"))
1733                .respond_with(ResponseTemplate::new(200).set_body_json(body))
1734                .mount(server)
1735                .await;
1736        }
1737
1738        fn ok_body() -> serde_json::Value {
1739            json!({ "success": true, "data": {} })
1740        }
1741
1742        fn renew_ok_body() -> serde_json::Value {
1743            json!({
1744                "success": true,
1745                "data": { "sessionToken": "fresh", "validityInSecondsST": 3600 }
1746            })
1747        }
1748
1749        fn expired_body() -> serde_json::Value {
1750            json!({ "success": false, "code": "390112", "message": "expired", "data": {} })
1751        }
1752
1753        #[tokio::test]
1754        async fn keep_session_alive_heartbeats_a_healthy_session() {
1755            let server = MockServer::start().await;
1756            mount_heartbeat(&server, ok_body()).await;
1757            let session = client::test_session(&server.uri(), Duration::from_secs(5));
1758            assert!(keep_session_alive(&session, INTERVAL).await);
1759            let reqs = server.received_requests().await.unwrap();
1760            assert_eq!(reqs.len(), 1, "one heartbeat, no renew");
1761        }
1762
1763        #[tokio::test]
1764        async fn keep_session_alive_renews_when_the_heartbeat_reports_expiry() {
1765            let server = MockServer::start().await;
1766            mount_heartbeat(&server, expired_body()).await;
1767            mount_renew(&server, renew_ok_body()).await;
1768            let session = client::test_session(&server.uri(), Duration::from_secs(5));
1769            assert!(keep_session_alive(&session, INTERVAL).await);
1770            let reqs = server.received_requests().await.unwrap();
1771            assert!(
1772                reqs.iter()
1773                    .any(|r| r.url.path() == "/session/token-request"),
1774                "renewed after the expired heartbeat"
1775            );
1776        }
1777
1778        #[tokio::test]
1779        async fn keep_session_alive_discards_when_the_master_token_is_dead() {
1780            let server = MockServer::start().await;
1781            mount_heartbeat(&server, expired_body()).await;
1782            mount_renew(&server, expired_body()).await;
1783            let session = client::test_session(&server.uri(), Duration::from_secs(5));
1784            assert!(!keep_session_alive(&session, INTERVAL).await);
1785        }
1786
1787        #[tokio::test]
1788        async fn keep_session_alive_keeps_the_session_on_transient_errors() {
1789            let server = MockServer::start().await;
1790            mount_heartbeat(
1791                &server,
1792                json!({ "success": false, "code": "390001", "message": "hiccup", "data": {} }),
1793            )
1794            .await;
1795            let session = client::test_session(&server.uri(), Duration::from_secs(5));
1796            assert!(keep_session_alive(&session, INTERVAL).await);
1797        }
1798
1799        #[tokio::test]
1800        async fn keep_session_alive_proactively_renews_a_token_expiring_before_the_next_tick() {
1801            let server = MockServer::start().await;
1802            mount_heartbeat(&server, ok_body()).await;
1803            mount_renew(&server, renew_ok_body()).await;
1804            let session = client::test_session(&server.uri(), Duration::from_secs(5));
1805            // interval + margin exceeds the fresh 3600s validity → renew first.
1806            assert!(keep_session_alive(&session, Duration::from_secs(7200)).await);
1807            let reqs = server.received_requests().await.unwrap();
1808            assert_eq!(
1809                reqs[0].url.path(),
1810                "/session/token-request",
1811                "renew ran first"
1812            );
1813            assert_eq!(reqs[1].url.path(), "/session/heartbeat");
1814        }
1815
1816        #[tokio::test]
1817        async fn engine_heartbeat_loop_beats_idle_sessions_and_stops_on_shutdown() {
1818            let server = MockServer::start().await;
1819            mount_heartbeat(&server, ok_body()).await;
1820
1821            let engine = SnowflakeEngine::new(SnowflakeEngineConfig {
1822                heartbeat_interval: Duration::from_millis(50),
1823                ..SnowflakeEngineConfig::default()
1824            });
1825            // Park one idle session in a pool, bypassing the (live-only) SSO.
1826            let pool = engine
1827                .registry
1828                .get_or_create(&SessionKey::new("ACCT", "user"), 2);
1829            let uri = server.uri();
1830            let checkout = pool
1831                .checkout(|| async {
1832                    Ok::<_, std::convert::Infallible>((
1833                        client::test_session(&uri, Duration::from_secs(5)),
1834                        QueryContext::default(),
1835                    ))
1836                })
1837                .await
1838                .unwrap();
1839            pool.checkin(checkout, QueryContext::default());
1840
1841            engine.start_heartbeat();
1842            let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
1843            while server.received_requests().await.unwrap().is_empty() {
1844                assert!(
1845                    tokio::time::Instant::now() < deadline,
1846                    "no heartbeat within the deadline"
1847                );
1848                tokio::time::sleep(Duration::from_millis(10)).await;
1849            }
1850            // The borrowed session was restored, not consumed.
1851            assert_eq!(pool.live(), 1);
1852
1853            engine.shutdown().await;
1854            let after = server.received_requests().await.unwrap().len();
1855            tokio::time::sleep(Duration::from_millis(150)).await;
1856            assert_eq!(
1857                server.received_requests().await.unwrap().len(),
1858                after,
1859                "no heartbeats after shutdown"
1860            );
1861            assert_eq!(engine.pool_count(), 0, "pools drained on shutdown");
1862        }
1863    }
1864}