Skip to main content

tauri_plugin_licensemanager/core/
manager.rs

1//! Port of `license-manager.ts`.
2
3use std::panic::{catch_unwind, AssertUnwindSafe};
4use std::sync::atomic::{AtomicU64, Ordering};
5use std::sync::{Arc, Mutex, MutexGuard, OnceLock, PoisonError, Weak};
6use std::time::Duration;
7
8use base64::engine::general_purpose::STANDARD;
9use base64::Engine;
10use futures::future::{BoxFuture, Shared};
11use futures::FutureExt;
12use tokio::sync::watch;
13use tokio::task::AbortHandle;
14use tokio_util::sync::CancellationToken;
15
16use super::api::{self, encode_uri_component, ValidateKeyRequest, DEFAULT_API_BASE_URL};
17use super::clock::{from_millis, Clock, ClockGuard, SystemClock};
18use super::environment;
19use super::error::{ConfigError, LatestAllowedVersion, LicenseError, LicenseErrorCode};
20use super::fingerprint::{resolve_machine_fingerprint, MachineFingerprint};
21use super::options::LicenseManagerOptions;
22use super::retry::{with_retry, RetryPolicy, BACKGROUND_RETRY, INTERACTIVE_RETRY};
23use super::state::{locked_state, LicenseState, OfflineState};
24use super::store::{default_backend, SecureStore};
25use super::token::{compute_refresh_at, TokenPayload, TokenVerifier, ValidatedToken};
26use super::transport::{ReqwestTransport, Transport};
27use super::version::is_version_allowed;
28
29const INSTALL_ID_KEY: &str = "installId";
30const LICENSE_KEY_KEY: &str = "licenseKey";
31const LICENSE_TOKEN_KEY: &str = "licenseToken";
32
33const PORTAL_BASE_URL: &str = "https://app.licensemanager.dev/portal";
34
35const MINUTE_MS: i64 = 60 * 1000;
36/// Retry backoff between scheduled re-validations while the server is unreachable.
37const RETRY_INITIAL_MS: i64 = 5 * MINUTE_MS;
38const RETRY_MAX_MS: i64 = 60 * MINUTE_MS;
39/// Longest single timer. Longer waits are re-armed in steps, re-checking the
40/// wall clock each time, so system sleep or clock changes can't push a
41/// validation far past its target date.
42const MAX_TIMEOUT_MS: i64 = 60 * MINUTE_MS;
43
44type Listener = Arc<dyn Fn(LicenseState) + Send + Sync>;
45type SharedRequest = Shared<BoxFuture<'static, Result<ValidatedToken, LicenseError>>>;
46
47fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
48    mutex.lock().unwrap_or_else(PoisonError::into_inner)
49}
50
51/// The license state machine: one per app, created by the plugin and reached
52/// with [`LicensemanagerExt::license`](crate::LicensemanagerExt::license).
53///
54/// On creation it validates the stored license key (in the background when the
55/// stored token is valid) and begins a periodic clock-tamper check. The plugin
56/// disposes it when the app exits.
57#[derive(Clone)]
58pub struct LicenseManager {
59    inner: Arc<Inner>,
60}
61
62struct Inner {
63    product_id: String,
64    fingerprint_option: Option<MachineFingerprint>,
65    version: String,
66    machine_name: String,
67    platform: String,
68    min_offline_ms: i64,
69    api_base_url: String,
70    store: SecureStore,
71    verifier: TokenVerifier,
72    clock_guard: ClockGuard,
73    clock: Arc<dyn Clock>,
74    transport: Arc<dyn Transport>,
75    runtime: tokio::runtime::Handle,
76    /// Cancelled on dispose, to stop pending retries and timers.
77    cancel: CancellationToken,
78    fingerprint: tokio::sync::OnceCell<String>,
79    state_tx: watch::Sender<LicenseState>,
80    listeners: Mutex<Vec<(u64, Listener)>>,
81    next_id: AtomicU64,
82    ready: OnceLock<Shared<BoxFuture<'static, LicenseState>>>,
83    m: Mutex<Mutable>,
84}
85
86struct Mutable {
87    key: Option<String>,
88    state: LicenseState,
89    /// Payload of the current valid token.
90    token: Option<TokenPayload>,
91    /// False once a request failed to reach a definitive server answer.
92    online: bool,
93    latest_allowed_version: LatestAllowedVersion,
94    retry_delay_ms: i64,
95    timer: Option<(u64, AbortHandle)>,
96    inflight: Option<Inflight>,
97    disposed: bool,
98}
99
100struct Inflight {
101    key: String,
102    policy: RetryPolicy,
103    id: u64,
104    request: SharedRequest,
105}
106
107/// What the stored token allows without asking the server.
108enum Stored {
109    Active(TokenPayload),
110    Refused(LicenseState),
111}
112
113impl Stored {
114    fn state(&self) -> LicenseState {
115        match self {
116            Self::Active(_) => LicenseState::Active,
117            Self::Refused(state) => *state,
118        }
119    }
120}
121
122/// Keeps a listener registered by
123/// [`LicenseManager::on_state_change`] until dropped.
124#[must_use = "the listener is removed when the Subscription is dropped"]
125pub struct Subscription {
126    id: u64,
127    inner: Weak<Inner>,
128}
129
130impl Drop for Subscription {
131    fn drop(&mut self) {
132        if let Some(inner) = self.inner.upgrade() {
133            lock(&inner.listeners).retain(|(id, _)| *id != self.id);
134        }
135    }
136}
137
138impl std::fmt::Debug for Subscription {
139    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
140        f.debug_struct("Subscription")
141            .field("id", &self.id)
142            .finish()
143    }
144}
145
146impl LicenseManager {
147    /// Created by the plugin's setup. Runs on the given Tokio handle. Reads
148    /// the stored key synchronously, then spawns startup validation and the
149    /// clock guard.
150    pub(crate) fn new(options: LicenseManagerOptions) -> Result<Self, ConfigError> {
151        if options.product_id.is_empty() {
152            return Err(ConfigError::MissingProductId);
153        }
154        if options.public_key.is_empty() {
155            return Err(ConfigError::MissingPublicKey);
156        }
157        let hooks = options.testing.unwrap_or_default();
158        let clock = hooks.clock.unwrap_or_else(|| Arc::new(SystemClock));
159        let transport = hooks
160            .transport
161            .unwrap_or_else(|| Arc::new(ReqwestTransport::new()));
162        let backend = hooks.secret_backend.unwrap_or_else(default_backend);
163
164        let verifier = TokenVerifier::new(&options.public_key, &options.product_id, clock.clone())?;
165        let store = SecureStore::new(&options.product_id, options.storage_dirs, backend);
166        let runtime = options.runtime;
167        let clock_guard =
168            ClockGuard::new(store.clone(), clock.clone(), transport.clone(), &runtime);
169        let key = store.get(LICENSE_KEY_KEY);
170        let (state_tx, _) = watch::channel(LicenseState::Loading);
171
172        let inner = Arc::new(Inner {
173            fingerprint_option: options.fingerprint,
174            version: options.version,
175            machine_name: options.machine_name.unwrap_or_else(environment::hostname),
176            platform: options
177                .platform
178                .unwrap_or_else(environment::default_platform),
179            min_offline_ms: i64::try_from(options.min_offline_duration.as_millis())
180                .unwrap_or(i64::MAX),
181            api_base_url: options
182                .api_base_url
183                .as_deref()
184                .unwrap_or(DEFAULT_API_BASE_URL)
185                .trim_end_matches('/')
186                .to_owned(),
187            product_id: options.product_id,
188            store,
189            verifier,
190            clock_guard,
191            clock,
192            transport,
193            runtime,
194            cancel: CancellationToken::new(),
195            fingerprint: tokio::sync::OnceCell::new(),
196            state_tx,
197            listeners: Mutex::new(Vec::new()),
198            next_id: AtomicU64::new(0),
199            ready: OnceLock::new(),
200            m: Mutex::new(Mutable {
201                key,
202                state: LicenseState::Loading,
203                token: None,
204                online: true,
205                latest_allowed_version: LatestAllowedVersion::NotApplicable,
206                retry_delay_ms: RETRY_INITIAL_MS,
207                timer: None,
208                inflight: None,
209                disposed: false,
210            }),
211        });
212        let ready = {
213            let inner = inner.clone();
214            async move { inner.init().await }.boxed().shared()
215        };
216        let _ = inner.ready.set(ready.clone());
217        inner.runtime.spawn(ready);
218        Ok(Self { inner })
219    }
220
221    /// Stable per-machine, per-install, per-product fingerprint:
222    /// `{base64 sha256 of the machine id}:{random install id}:{product id}`.
223    /// Cached after the first call.
224    pub async fn fingerprint(&self) -> String {
225        self.inner.fingerprint().await
226    }
227
228    /// The current state.
229    pub fn state(&self) -> LicenseState {
230        lock(&self.inner.m).state
231    }
232
233    /// Resolves with the state once startup validation has completed, or as
234    /// soon as the stored token is found valid; the online validation then
235    /// continues in the background. Every caller gets the same state.
236    pub async fn ready(&self) -> LicenseState {
237        self.inner.ready().await
238    }
239
240    /// Call `listener` on every state change, until the returned
241    /// [`Subscription`] is dropped. A panicking listener is ignored.
242    pub fn on_state_change<F>(&self, listener: F) -> Subscription
243    where
244        F: Fn(LicenseState) + Send + Sync + 'static,
245    {
246        let id = self.inner.next_id.fetch_add(1, Ordering::SeqCst);
247        lock(&self.inner.listeners).push((id, Arc::new(listener)));
248        Subscription {
249            id,
250            inner: Arc::downgrade(&self.inner),
251        }
252    }
253
254    /// A receiver of the current state and every change.
255    pub fn subscribe(&self) -> watch::Receiver<LicenseState> {
256        self.inner.state_tx.subscribe()
257    }
258
259    /// Validate a key the user typed and, on success, store it and activate.
260    /// The key is trimmed but its case is kept.
261    ///
262    /// Fails when validation fails for any reason, including the server being
263    /// unreachable. Nothing is stored and the current state is left untouched.
264    pub async fn set_license_key(&self, input: &str) -> Result<LicenseState, LicenseError> {
265        let key = input.trim();
266        if key.is_empty() {
267            return Err(LicenseError::new(
268                LicenseErrorCode::ValidationError,
269                "The license key is empty",
270            ));
271        }
272        let validated = self.inner.request(key, INTERACTIVE_RETRY).await?;
273        self.inner
274            .store
275            .set_async(LICENSE_KEY_KEY, key.to_owned())
276            .await;
277        lock(&self.inner.m).key = Some(key.to_owned());
278        self.inner.activate(validated).await;
279        Ok(self.state())
280    }
281
282    /// Re-validate the stored key now, e.g. after the customer renewed an
283    /// expired license. Never fails.
284    pub async fn refresh(&self) -> LicenseState {
285        self.inner.ready().await;
286        let key = lock(&self.inner.m).key.clone();
287        match key {
288            Some(key) => self.inner.revalidate(key, INTERACTIVE_RETRY).await,
289            None => self.state(),
290        }
291    }
292
293    /// The stored license key, or `None` before one was set.
294    pub fn license_key(&self) -> Option<String> {
295        lock(&self.inner.m).key.clone()
296    }
297
298    /// Link to the customer portal for this product.
299    pub fn portal_url(&self) -> String {
300        format!(
301            "{PORTAL_BASE_URL}/{}",
302            encode_uri_component(&self.inner.product_id)
303        )
304    }
305
306    /// In state `VERSION_NOT_ALLOWED`: the latest version the license allows.
307    /// [`NotApplicable`](LatestAllowedVersion::NotApplicable) otherwise, and
308    /// when the refusal came from the stored token while offline (it only
309    /// knows the max version, not the latest release).
310    pub fn latest_allowed_version(&self) -> LatestAllowedVersion {
311        lock(&self.inner.m).latest_allowed_version.clone()
312    }
313
314    /// Whether the last validation reached the server. When offline, the app
315    /// keeps running on the stored token until `valid_until`.
316    pub fn offline_state(&self) -> OfflineState {
317        let m = lock(&self.inner.m);
318        if m.online {
319            return OfflineState::Online;
320        }
321        match &m.token {
322            Some(token) if m.state == LicenseState::Active => {
323                let expires_at = token.exp.saturating_mul(1000);
324                let remaining = (expires_at - self.inner.clock.now_ms()).max(0);
325                OfflineState::Offline {
326                    valid_until: Some(from_millis(expires_at)),
327                    remaining: Duration::from_millis(remaining.unsigned_abs()),
328                }
329            }
330            _ => OfflineState::Offline {
331                valid_until: None,
332                remaining: Duration::ZERO,
333            },
334        }
335    }
336
337    /// True once a backwards clock jump has been observed.
338    pub fn is_clock_tampered(&self) -> bool {
339        self.inner.clock_guard.is_tampered()
340    }
341
342    /// Stop the periodic clock check, retries and scheduled re-validations.
343    /// Idempotent. The plugin calls it when the app exits.
344    pub fn dispose(&self) {
345        self.inner.dispose();
346    }
347
348    #[cfg(test)]
349    pub(crate) fn is_disposed(&self) -> bool {
350        lock(&self.inner.m).disposed
351    }
352}
353
354impl std::fmt::Debug for LicenseManager {
355    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
356        f.debug_struct("LicenseManager")
357            .field("product_id", &self.inner.product_id)
358            .field("state", &self.state())
359            .finish_non_exhaustive()
360    }
361}
362
363impl Drop for Inner {
364    fn drop(&mut self) {
365        self.cancel.cancel();
366        self.clock_guard.dispose();
367    }
368}
369
370impl Inner {
371    async fn ready(&self) -> LicenseState {
372        match self.ready.get() {
373            Some(ready) => ready.clone().await,
374            None => lock(&self.m).state,
375        }
376    }
377
378    fn state(&self) -> LicenseState {
379        lock(&self.m).state
380    }
381
382    fn key_is(&self, key: &str) -> bool {
383        lock(&self.m).key.as_deref() == Some(key)
384    }
385
386    fn dispose(&self) {
387        let mut m = lock(&self.m);
388        m.disposed = true;
389        self.cancel.cancel();
390        clear_timer(&mut m);
391        drop(m);
392        self.clock_guard.dispose();
393    }
394
395    async fn fingerprint(&self) -> String {
396        self.fingerprint
397            .get_or_init(|| async {
398                let machine = resolve_machine_fingerprint(self.fingerprint_option.as_ref()).await;
399                let install_id = self.get_or_create_install_id().await;
400                format!("{machine}:{install_id}:{}", self.product_id)
401            })
402            .await
403            .clone()
404    }
405
406    async fn get_or_create_install_id(&self) -> String {
407        if let Some(existing) = self
408            .store
409            .get_async(INSTALL_ID_KEY)
410            .await
411            .filter(|v| !v.is_empty())
412        {
413            return existing;
414        }
415        let created = STANDARD.encode(rand::random::<[u8; 32]>());
416        self.store.set_async(INSTALL_ID_KEY, created.clone()).await;
417        created
418    }
419
420    async fn init(self: Arc<Self>) -> LicenseState {
421        let Some(key) = lock(&self.m).key.clone() else {
422            self.set_state(LicenseState::MissingKey);
423            return self.state();
424        };
425        let stored = self.check_stored_token(&key).await;
426        if !self.key_is(&key) {
427            return self.state();
428        }
429        let Stored::Active(payload) = stored else {
430            return self.revalidate(key, INTERACTIVE_RETRY).await;
431        };
432        lock(&self.m).token = Some(payload);
433        self.set_state(LicenseState::Active);
434        // Confirm with the server without holding up startup.
435        let this = self.clone();
436        self.runtime.spawn(async move {
437            this.revalidate(key, BACKGROUND_RETRY).await;
438        });
439        self.state()
440    }
441
442    /// Validate online; on an error that says nothing about the license, fall
443    /// back to the stored token. Never fails.
444    async fn revalidate(self: &Arc<Self>, key: String, policy: RetryPolicy) -> LicenseState {
445        clear_timer(&mut lock(&self.m));
446        match self.request(&key, policy).await {
447            Ok(validated) => {
448                if !self.key_is(&key) {
449                    return self.state();
450                }
451                self.activate(validated).await;
452            }
453            Err(error) => {
454                if !self.key_is(&key) {
455                    return self.state();
456                }
457                if error.is_locking() {
458                    self.lock_license(error.code, error.latest_allowed_version)
459                        .await;
460                } else {
461                    self.validate_offline(&key).await;
462                }
463            }
464        }
465        self.state()
466    }
467
468    /// Store a freshly validated token and schedule its refresh.
469    async fn activate(self: &Arc<Self>, ValidatedToken { token, payload }: ValidatedToken) {
470        self.store.set_async(LICENSE_TOKEN_KEY, token).await;
471        {
472            let mut m = lock(&self.m);
473            m.token = Some(payload);
474            m.online = true;
475            m.retry_delay_ms = RETRY_INITIAL_MS;
476            m.latest_allowed_version = LatestAllowedVersion::NotApplicable;
477        }
478        self.set_state(LicenseState::Active);
479        self.schedule_next();
480    }
481
482    /// The server refused the license: drop the token and stop refreshing.
483    async fn lock_license(
484        &self,
485        code: LicenseErrorCode,
486        latest_allowed_version: LatestAllowedVersion,
487    ) {
488        self.store.delete_async(LICENSE_TOKEN_KEY).await;
489        let Some(state) = locked_state(code) else {
490            return;
491        };
492        {
493            let mut m = lock(&self.m);
494            m.token = None;
495            m.online = true;
496            m.latest_allowed_version = match (code, latest_allowed_version) {
497                (LicenseErrorCode::VersionNotAllowed, LatestAllowedVersion::NotApplicable) => {
498                    LatestAllowedVersion::NoneAllowed
499                }
500                (LicenseErrorCode::VersionNotAllowed, latest) => latest,
501                _ => LatestAllowedVersion::NotApplicable,
502            };
503        }
504        self.set_state(state);
505    }
506
507    /// Server unreachable: trust the stored token, unless the clock was rolled
508    /// back or the token's max version excludes this app version.
509    async fn validate_offline(self: &Arc<Self>, key: &str) {
510        lock(&self.m).online = false;
511        let stored = self.check_stored_token(key).await;
512        if !self.key_is(key) {
513            return;
514        }
515        let state = stored.state();
516        {
517            let mut m = lock(&self.m);
518            m.token = match stored {
519                Stored::Active(payload) => Some(payload),
520                Stored::Refused(_) => None,
521            };
522            if state == LicenseState::VersionNotAllowed {
523                m.latest_allowed_version = LatestAllowedVersion::NotApplicable;
524            }
525        }
526        self.set_state(state);
527        self.schedule_next();
528        let mut m = lock(&self.m);
529        m.retry_delay_ms = (m.retry_delay_ms * 2).min(RETRY_MAX_MS);
530    }
531
532    async fn check_stored_token(&self, key: &str) -> Stored {
533        self.clock_guard.ready().await;
534        if self.clock_guard.is_tampered() {
535            return Stored::Refused(LicenseState::ClockTampered);
536        }
537        let token = self
538            .store
539            .get_async(LICENSE_TOKEN_KEY)
540            .await
541            .filter(|t| !t.is_empty());
542        let payload = match token {
543            Some(token) => {
544                let fingerprint = self.fingerprint().await;
545                self.verifier.verify(&token, key, &fingerprint).ok()
546            }
547            None => None,
548        };
549        let Some(payload) = payload else {
550            return Stored::Refused(LicenseState::OfflineExpired);
551        };
552        // Keep the token: it works again if the app is downgraded.
553        if !is_version_allowed(&self.version, payload.mv.as_deref()) {
554            return Stored::Refused(LicenseState::VersionNotAllowed);
555        }
556        Stored::Active(payload)
557    }
558
559    /// Arm the next re-validation: at the refresh point when online; when
560    /// offline, retry with backoff but never later than the token expiry.
561    fn schedule_next(self: &Arc<Self>) {
562        let mut m = lock(&self.m);
563        clear_timer(&mut m);
564        let Some(key) = m.key.clone().filter(|_| !m.disposed) else {
565            return;
566        };
567        let now = self.clock.now_ms();
568        let at = match (&m.token, m.state) {
569            (Some(token), LicenseState::Active) => {
570                let mut at = compute_refresh_at(token, self.min_offline_ms);
571                if !m.online {
572                    at = at.max(now + m.retry_delay_ms);
573                }
574                at.min(token.exp.saturating_mul(1000))
575            }
576            (_, LicenseState::OfflineExpired | LicenseState::ClockTampered) => {
577                now + m.retry_delay_ms
578            }
579            // Refused by the stored token: the server has the final word.
580            (_, LicenseState::VersionNotAllowed) if !m.online => now + m.retry_delay_ms,
581            _ => return,
582        };
583        let id = self.next_id.fetch_add(1, Ordering::SeqCst);
584        // The first step is measured now, not when the task first runs.
585        let first_step = tokio::time::Instant::now() + step_until(at, now);
586        let handle = self
587            .runtime
588            .spawn(arm(Arc::downgrade(self), id, key, at, first_step));
589        m.timer = Some((id, handle.abort_handle()));
590    }
591
592    /// Call validate-key, sharing one in-flight request per key and policy,
593    /// so a caller is never held up by a longer background retry budget.
594    fn request(self: &Arc<Self>, key: &str, policy: RetryPolicy) -> SharedRequest {
595        let mut m = lock(&self.m);
596        if let Some(inflight) = &m.inflight {
597            if inflight.key == key && inflight.policy == policy {
598                return inflight.request.clone();
599            }
600        }
601        let id = self.next_id.fetch_add(1, Ordering::SeqCst);
602        let request = {
603            let this = self.clone();
604            let key = key.to_owned();
605            async move {
606                let result = this.fetch_token(&key, policy).await;
607                let mut m = lock(&this.m);
608                if m.inflight.as_ref().is_some_and(|i| i.id == id) {
609                    m.inflight = None;
610                }
611                result
612            }
613            .boxed()
614            .shared()
615        };
616        m.inflight = Some(Inflight {
617            key: key.to_owned(),
618            policy,
619            id,
620            request: request.clone(),
621        });
622        // Runs to completion even if every caller goes away, like a promise.
623        self.runtime.spawn(request.clone());
624        request
625    }
626
627    async fn fetch_token(
628        &self,
629        key: &str,
630        policy: RetryPolicy,
631    ) -> Result<ValidatedToken, LicenseError> {
632        let fingerprint = self.fingerprint().await;
633        let request = ValidateKeyRequest {
634            key: key.to_owned(),
635            fingerprint: fingerprint.clone(),
636            version: self.version.clone(),
637            name: self.machine_name.clone(),
638            platform: self.platform.clone(),
639        };
640        let (token, status) = with_retry(
641            |remaining| {
642                api::post_validate_key(
643                    &*self.transport,
644                    &self.api_base_url,
645                    &self.product_id,
646                    &request,
647                    remaining,
648                )
649            },
650            api::is_transient,
651            policy,
652            &self.cancel,
653        )
654        .await?;
655        match self.verifier.verify(&token, key, &fingerprint) {
656            Ok(payload) => Ok(ValidatedToken { token, payload }),
657            Err(_) => Err(LicenseError::new(
658                LicenseErrorCode::InvalidResponse,
659                "The license token failed verification",
660            )
661            .with_status(status)),
662        }
663    }
664
665    fn set_state(&self, state: LicenseState) {
666        {
667            let mut m = lock(&self.m);
668            if m.state == state {
669                return;
670            }
671            m.state = state;
672        }
673        self.state_tx.send_replace(state);
674        let listeners: Vec<Listener> = lock(&self.listeners)
675            .iter()
676            .map(|(_, listener)| listener.clone())
677            .collect();
678        for listener in listeners {
679            // A listener must not break the state machine.
680            let _ = catch_unwind(AssertUnwindSafe(|| listener(state)));
681        }
682    }
683}
684
685/// Time to sleep towards `at`, capped at [`MAX_TIMEOUT_MS`].
686fn step_until(at: i64, now: i64) -> Duration {
687    Duration::from_millis((at - now).clamp(0, MAX_TIMEOUT_MS).unsigned_abs())
688}
689
690fn clear_timer(m: &mut Mutable) {
691    if let Some((_, timer)) = m.timer.take() {
692        timer.abort();
693    }
694}
695
696/// Wait until `at` (ms since the epoch) in steps of at most
697/// [`MAX_TIMEOUT_MS`], re-reading the wall clock at each step, then
698/// re-validate with the background policy.
699async fn arm(inner: Weak<Inner>, id: u64, key: String, at: i64, first_step: tokio::time::Instant) {
700    let (clock, cancel) = match inner.upgrade() {
701        Some(inner) => (inner.clock.clone(), inner.cancel.clone()),
702        None => return,
703    };
704    let mut step = first_step;
705    loop {
706        tokio::select! {
707            () = tokio::time::sleep_until(step) => {}
708            () = cancel.cancelled() => return,
709        }
710        let now = clock.now_ms();
711        if now >= at {
712            break;
713        }
714        step = tokio::time::Instant::now() + step_until(at, now);
715    }
716    let Some(inner) = inner.upgrade() else {
717        return;
718    };
719    {
720        let mut m = lock(&inner.m);
721        if m.timer.as_ref().map(|(timer_id, _)| *timer_id) != Some(id) {
722            return;
723        }
724        // Fired: a later clear must not abort the validation below.
725        m.timer = None;
726    }
727    inner.revalidate(key, BACKGROUND_RETRY).await;
728}
729
730#[cfg(test)]
731mod tests;