Skip to main content

khive_runtime/
ann_registry.rs

1//! Durable ANN-consumer registration lifecycle and compaction.
2//!
3//! A registration starts at the closed `PENDING_WATERMARK` value and blocks
4//! compaction while its first checkpoint is being built.  Publishing that
5//! checkpoint atomically moves it to an active watermark `S >= 0` and removes
6//! the pending timestamp.  A pending registration which never activates is
7//! retired after [`PENDING_GRACE_US`] by the same write transaction which
8//! compacts the log; returning consumers therefore observe registry loss and
9//! must rebuild before serving.
10
11use std::any::Any;
12
13use khive_storage::types::{SqlStatement, SqlValue};
14use khive_storage::{AtomicUnitOp, SqlAccess, StorageError, StorageResult};
15
16/// Closed watermark for a registered consumer which has not yet published its
17/// first durable checkpoint.  It sorts below every active watermark and below
18/// the `-1` authoritative-rebuild fence, so it always blocks compaction.
19pub const PENDING_WATERMARK: i64 = -2;
20
21/// Closed watermark used while an authoritative registry-loss rebuild is in
22/// flight.  Unlike a pending registration, this state never expires.
23pub const RECOVERING_WATERMARK: i64 = -1;
24
25/// A first checkpoint gets one day to finish before its never-activated
26/// registration is retired.  Active (`S >= 0`) and recovering (`-1`) rows are
27/// never age-retired.
28pub const PENDING_GRACE_US: i64 = 24 * 60 * 60 * 1_000_000;
29
30/// Registry state authorized to publish a checkpoint watermark.
31#[derive(Clone, Copy, Debug, Eq, PartialEq)]
32pub enum WatermarkAuthority {
33    /// A first full checkpoint may activate a pending row; a rebuild of an
34    /// already-active consumer may also advance it.
35    PendingOrActive,
36    /// An incremental or ordinary registered checkpoint requires `S >= 0`.
37    Active,
38    /// An authoritative recovery checkpoint requires exactly `-1`.
39    Recovering,
40}
41
42/// Scope of one compaction operation.
43#[derive(Clone, Debug, Eq, PartialEq)]
44pub enum CompactionScope {
45    /// Compact one namespace/model pair, including wildcard consumers in its
46    /// protection minimum.
47    Namespace(String),
48    /// Compact every namespace for a model, with a correlated per-namespace
49    /// minimum.
50    Model,
51}
52
53/// Identity of one pending consumer retired by compaction.
54#[derive(Clone, Debug, Eq, PartialEq)]
55pub struct RetiredConsumer {
56    /// Stable consumer identity.
57    pub consumer: String,
58    /// Registry namespace (`*` for a global consumer).
59    pub namespace: String,
60    /// Embedding model whose write log was protected.
61    pub embedding_model: String,
62    /// Original pending-registration timestamp in microseconds since epoch.
63    pub registered_at_us: i64,
64}
65
66/// Observable result of one lifecycle-aware compaction transaction.
67#[derive(Clone, Debug, Default, Eq, PartialEq)]
68pub struct CompactionOutcome {
69    /// Raw write-log rows deleted after evaluating the post-retirement minimum.
70    pub deleted_log_rows: u64,
71    /// Pending consumers retired and warned during this transaction.
72    pub retired_consumers: Vec<RetiredConsumer>,
73}
74
75fn stmt(sql: impl Into<String>, params: Vec<SqlValue>, label: &str) -> SqlStatement {
76    SqlStatement {
77        sql: sql.into(),
78        params,
79        label: Some(label.to_owned()),
80    }
81}
82
83fn integer_range_error(value: u64) -> StorageError {
84    StorageError::Internal(format!(
85        "ANN watermark {value} exceeds SQLite INTEGER range"
86    ))
87}
88
89/// Register a consumer in the pending state without changing an existing
90/// active or recovering registration.
91pub async fn register_pending(
92    sql: &dyn SqlAccess,
93    consumer: &str,
94    namespace: &str,
95    model: &str,
96) -> StorageResult<()> {
97    register_pending_at(
98        sql,
99        consumer,
100        namespace,
101        model,
102        chrono::Utc::now().timestamp_micros(),
103    )
104    .await
105}
106
107async fn register_pending_at(
108    sql: &dyn SqlAccess,
109    consumer: &str,
110    namespace: &str,
111    model: &str,
112    registered_at_us: i64,
113) -> StorageResult<()> {
114    let consumer = consumer.to_owned();
115    let namespace = namespace.to_owned();
116    let model = model.to_owned();
117    let op: AtomicUnitOp = Box::new(move |writer| {
118        Box::pin(async move {
119            let inserted = writer
120                .execute(stmt(
121                    "INSERT OR IGNORE INTO ann_consumer_watermark \
122                     (consumer, namespace, embedding_model, watermark) \
123                     VALUES (?1, ?2, ?3, ?4)",
124                    vec![
125                        SqlValue::Text(consumer.clone()),
126                        SqlValue::Text(namespace.clone()),
127                        SqlValue::Text(model.clone()),
128                        SqlValue::Integer(PENDING_WATERMARK),
129                    ],
130                    "ann_registry_register_pending",
131                ))
132                .await?;
133            let pending_insert = if inserted == 1 {
134                // An orphan metadata row can exist only after manual surgery,
135                // but it must not make a genuinely new registration inherit
136                // an already-expired timestamp.
137                "INSERT INTO ann_consumer_pending \
138                 (consumer, namespace, embedding_model, registered_at_us) \
139                 VALUES (?1, ?2, ?3, ?4) \
140                 ON CONFLICT(consumer, namespace, embedding_model) \
141                 DO UPDATE SET registered_at_us = excluded.registered_at_us"
142            } else {
143                // Repeated calls while the same first build is in flight must
144                // not refresh its grace window indefinitely.
145                "INSERT OR IGNORE INTO ann_consumer_pending \
146                 (consumer, namespace, embedding_model, registered_at_us) \
147                 SELECT ?1, ?2, ?3, ?4 \
148                 WHERE EXISTS (SELECT 1 FROM ann_consumer_watermark \
149                               WHERE consumer = ?1 AND namespace = ?2 \
150                                 AND embedding_model = ?3 AND watermark = ?5)"
151            };
152            let mut pending_params = vec![
153                SqlValue::Text(consumer),
154                SqlValue::Text(namespace),
155                SqlValue::Text(model),
156                SqlValue::Integer(registered_at_us),
157            ];
158            if inserted != 1 {
159                pending_params.push(SqlValue::Integer(PENDING_WATERMARK));
160            }
161            writer
162                .execute(stmt(
163                    pending_insert,
164                    pending_params,
165                    "ann_registry_stamp_pending",
166                ))
167                .await?;
168            Ok(Box::new(()) as Box<dyn Any + Send>)
169        })
170    });
171    sql.atomic_unit(op).await?;
172    Ok(())
173}
174
175/// Publish the non-expiring authoritative-rebuild fence and clear any pending
176/// lifecycle timestamp in the same transaction.
177pub async fn mark_recovering(
178    sql: &dyn SqlAccess,
179    consumer: &str,
180    namespace: &str,
181    model: &str,
182) -> StorageResult<()> {
183    let warning_consumer = consumer.to_owned();
184    let warning_namespace = namespace.to_owned();
185    let warning_model = model.to_owned();
186    let consumer = consumer.to_owned();
187    let namespace = namespace.to_owned();
188    let model = model.to_owned();
189    let op: AtomicUnitOp = Box::new(move |writer| {
190        Box::pin(async move {
191            writer
192                .execute(stmt(
193                    "INSERT INTO ann_consumer_watermark \
194                     (consumer, namespace, embedding_model, watermark) \
195                     VALUES (?1, ?2, ?3, ?4) \
196                     ON CONFLICT(consumer, namespace, embedding_model) \
197                     DO UPDATE SET watermark = excluded.watermark",
198                    vec![
199                        SqlValue::Text(consumer.clone()),
200                        SqlValue::Text(namespace.clone()),
201                        SqlValue::Text(model.clone()),
202                        SqlValue::Integer(RECOVERING_WATERMARK),
203                    ],
204                    "ann_registry_mark_recovering",
205                ))
206                .await?;
207            writer
208                .execute(stmt(
209                    "DELETE FROM ann_consumer_pending \
210                     WHERE consumer = ?1 AND namespace = ?2 AND embedding_model = ?3",
211                    vec![
212                        SqlValue::Text(consumer),
213                        SqlValue::Text(namespace),
214                        SqlValue::Text(model),
215                    ],
216                    "ann_registry_clear_pending_for_recovery",
217                ))
218                .await?;
219            Ok(Box::new(()) as Box<dyn Any + Send>)
220        })
221    });
222    sql.atomic_unit(op).await?;
223    tracing::warn!(
224        consumer = %warning_consumer,
225        namespace = %warning_namespace,
226        embedding_model = %warning_model,
227        "ANN consumer entered the non-expiring recovery fence; compaction remains blocked \
228         until an authoritative full checkpoint succeeds"
229    );
230    Ok(())
231}
232
233/// Conditionally publish a durable checkpoint and activate the consumer.
234///
235/// Returns `false` when retirement or a competing checkpoint changed the
236/// registry state first.  The conditional update and pending-metadata cleanup
237/// share one writer transaction, so compaction cannot observe an active
238/// watermark with stale pending metadata or retire between those steps.
239pub async fn raise_watermark(
240    sql: &dyn SqlAccess,
241    consumer: &str,
242    namespace: &str,
243    model: &str,
244    watermark: u64,
245    authority: WatermarkAuthority,
246) -> StorageResult<bool> {
247    let watermark = i64::try_from(watermark).map_err(|_| integer_range_error(watermark))?;
248    let predicate = match authority {
249        WatermarkAuthority::PendingOrActive => {
250            "(watermark = -2 OR (watermark >= 0 AND watermark <= ?4))"
251        }
252        WatermarkAuthority::Active => "watermark >= 0 AND watermark <= ?4",
253        WatermarkAuthority::Recovering => "watermark = -1",
254    };
255    let consumer = consumer.to_owned();
256    let namespace = namespace.to_owned();
257    let model = model.to_owned();
258    let op: AtomicUnitOp = Box::new(move |writer| {
259        Box::pin(async move {
260            let affected = writer
261                .execute(stmt(
262                    format!(
263                        "UPDATE ann_consumer_watermark SET watermark = ?4 \
264                         WHERE consumer = ?1 AND namespace = ?2 AND embedding_model = ?3 \
265                           AND {predicate}"
266                    ),
267                    vec![
268                        SqlValue::Text(consumer.clone()),
269                        SqlValue::Text(namespace.clone()),
270                        SqlValue::Text(model.clone()),
271                        SqlValue::Integer(watermark),
272                    ],
273                    "ann_registry_raise_watermark",
274                ))
275                .await?;
276            if affected == 1 {
277                writer
278                    .execute(stmt(
279                        "DELETE FROM ann_consumer_pending \
280                         WHERE consumer = ?1 AND namespace = ?2 AND embedding_model = ?3",
281                        vec![
282                            SqlValue::Text(consumer),
283                            SqlValue::Text(namespace),
284                            SqlValue::Text(model),
285                        ],
286                        "ann_registry_activate_consumer",
287                    ))
288                    .await?;
289            }
290            Ok(Box::new(affected == 1) as Box<dyn Any + Send>)
291        })
292    });
293    let result = sql.atomic_unit(op).await?;
294    Ok(*result
295        .downcast::<bool>()
296        .expect("ANN watermark atomic unit returns bool"))
297}
298
299/// Retire expired never-activated consumers and compact the write log in one
300/// writer-serialized transaction.
301pub async fn compact_write_log(
302    sql: &dyn SqlAccess,
303    scope: CompactionScope,
304    model: &str,
305) -> StorageResult<CompactionOutcome> {
306    compact_write_log_at(sql, scope, model, chrono::Utc::now().timestamp_micros()).await
307}
308
309async fn compact_write_log_at(
310    sql: &dyn SqlAccess,
311    scope: CompactionScope,
312    model: &str,
313    now_us: i64,
314) -> StorageResult<CompactionOutcome> {
315    let cutoff_us = now_us.saturating_sub(PENDING_GRACE_US);
316    let model = model.to_owned();
317    let op: AtomicUnitOp = Box::new(move |writer| {
318        Box::pin(async move {
319            let (scope_filter, scope_params) = match &scope {
320                CompactionScope::Namespace(namespace) => (
321                    " AND (watermark.namespace = ?3 OR watermark.namespace = '*')",
322                    vec![
323                        SqlValue::Integer(cutoff_us),
324                        SqlValue::Text(model.clone()),
325                        SqlValue::Text(namespace.clone()),
326                    ],
327                ),
328                CompactionScope::Model => (
329                    "",
330                    vec![SqlValue::Integer(cutoff_us), SqlValue::Text(model.clone())],
331                ),
332            };
333
334            // Repair a pending row created by an interrupted/manual writer
335            // without lifecycle metadata.  It starts a fresh grace interval
336            // instead of pinning silently forever or being retired without an
337            // age proof.
338            let backfill_sql = format!(
339                "INSERT OR IGNORE INTO ann_consumer_pending \
340                 (consumer, namespace, embedding_model, registered_at_us) \
341                 SELECT watermark.consumer, watermark.namespace, \
342                        watermark.embedding_model, ?1 \
343                 FROM ann_consumer_watermark watermark \
344                 WHERE watermark.watermark = -2 AND watermark.embedding_model = ?2{scope_filter}"
345            );
346            let mut backfill_params = scope_params.clone();
347            backfill_params[0] = SqlValue::Integer(now_us);
348            writer
349                .execute(stmt(
350                    backfill_sql,
351                    backfill_params,
352                    "ann_registry_backfill_pending_timestamp",
353                ))
354                .await?;
355
356            let retired_rows = writer
357                .query_all(stmt(
358                    format!(
359                        "SELECT watermark.consumer, watermark.namespace, \
360                                watermark.embedding_model, pending.registered_at_us \
361                         FROM ann_consumer_watermark watermark \
362                         JOIN ann_consumer_pending pending \
363                           ON pending.consumer = watermark.consumer \
364                          AND pending.namespace = watermark.namespace \
365                          AND pending.embedding_model = watermark.embedding_model \
366                         WHERE pending.registered_at_us <= ?1 \
367                           AND watermark.watermark = -2 \
368                           AND watermark.embedding_model = ?2{scope_filter} \
369                         ORDER BY watermark.consumer, watermark.namespace"
370                    ),
371                    scope_params.clone(),
372                    "ann_registry_find_expired_pending",
373                ))
374                .await?;
375
376            writer
377                .execute(stmt(
378                    format!(
379                        "DELETE FROM ann_consumer_watermark AS watermark \
380                         WHERE watermark.watermark = -2 \
381                           AND watermark.embedding_model = ?2{scope_filter} \
382                           AND EXISTS (SELECT 1 FROM ann_consumer_pending pending \
383                                       WHERE pending.consumer = watermark.consumer \
384                                         AND pending.namespace = watermark.namespace \
385                                         AND pending.embedding_model = watermark.embedding_model \
386                                         AND pending.registered_at_us <= ?1)"
387                    ),
388                    scope_params.clone(),
389                    "ann_registry_retire_expired_pending",
390                ))
391                .await?;
392
393            // Drop metadata for retired rows and for consumers activated by a
394            // legacy/manual checkpoint writer.  The watermark row is the
395            // authoritative state.
396            let pending_scope_filter = scope_filter.replace("watermark.", "pending.");
397            writer
398                .execute(stmt(
399                    format!(
400                        "DELETE FROM ann_consumer_pending AS pending \
401                         WHERE pending.embedding_model = ?2{pending_scope_filter} \
402                           AND NOT EXISTS (SELECT 1 FROM ann_consumer_watermark watermark \
403                                           WHERE watermark.consumer = pending.consumer \
404                                             AND watermark.namespace = pending.namespace \
405                                             AND watermark.embedding_model = pending.embedding_model \
406                                             AND watermark.watermark = -2)"
407                    ),
408                    scope_params,
409                    "ann_registry_prune_pending_metadata",
410                ))
411                .await?;
412
413            let deleted_log_rows = match &scope {
414                CompactionScope::Namespace(namespace) => {
415                    writer
416                        .execute(stmt(
417                            "DELETE FROM ann_write_log \
418                             WHERE namespace = ?1 AND embedding_model = ?2 \
419                               AND seq <= (SELECT MIN(watermark) \
420                                           FROM ann_consumer_watermark \
421                                           WHERE (namespace = ?1 OR namespace = '*') \
422                                             AND embedding_model = ?2)",
423                            vec![
424                                SqlValue::Text(namespace.clone()),
425                                SqlValue::Text(model.clone()),
426                            ],
427                            "ann_registry_compact_namespace",
428                        ))
429                        .await?
430                }
431                CompactionScope::Model => {
432                    writer
433                        .execute(stmt(
434                            "DELETE FROM ann_write_log \
435                             WHERE embedding_model = ?1 \
436                               AND seq <= (SELECT MIN(watermark.watermark) \
437                                           FROM ann_consumer_watermark watermark \
438                                           WHERE (watermark.namespace = ann_write_log.namespace \
439                                                  OR watermark.namespace = '*') \
440                                             AND watermark.embedding_model = ?1)",
441                            vec![SqlValue::Text(model.clone())],
442                            "ann_registry_compact_model",
443                        ))
444                        .await?
445                }
446            };
447
448            let mut retired_consumers = Vec::with_capacity(retired_rows.len());
449            for row in retired_rows {
450                let text = |column: &str| match row.get(column) {
451                    Some(SqlValue::Text(value)) => Ok(value.clone()),
452                    other => Err(StorageError::Internal(format!(
453                        "ANN retired-consumer {column}: unexpected {other:?}"
454                    ))),
455                };
456                let registered_at_us = match row.get("registered_at_us") {
457                    Some(SqlValue::Integer(value)) => *value,
458                    other => {
459                        return Err(StorageError::Internal(format!(
460                            "ANN retired-consumer registered_at_us: unexpected {other:?}"
461                        )))
462                    }
463                };
464                retired_consumers.push(RetiredConsumer {
465                    consumer: text("consumer")?,
466                    namespace: text("namespace")?,
467                    embedding_model: text("embedding_model")?,
468                    registered_at_us,
469                });
470            }
471            Ok(Box::new(CompactionOutcome {
472                deleted_log_rows,
473                retired_consumers,
474            }) as Box<dyn Any + Send>)
475        })
476    });
477
478    let result = sql.atomic_unit(op).await?;
479    let outcome = *result
480        .downcast::<CompactionOutcome>()
481        .expect("ANN compaction atomic unit returns CompactionOutcome");
482    for retired in &outcome.retired_consumers {
483        tracing::warn!(
484            consumer = %retired.consumer,
485            namespace = %retired.namespace,
486            embedding_model = %retired.embedding_model,
487            registered_at_us = retired.registered_at_us,
488            grace_hours = PENDING_GRACE_US / (60 * 60 * 1_000_000),
489            "retired dormant ANN consumer which never published a checkpoint; \
490             a returning consumer must rebuild before serving"
491        );
492    }
493    Ok(outcome)
494}
495
496#[cfg(test)]
497mod tests {
498    use super::*;
499    use crate::KhiveRuntime;
500
501    async fn execute(sql: &dyn SqlAccess, sql_text: &str, params: Vec<SqlValue>) {
502        sql.writer()
503            .await
504            .expect("writer")
505            .execute(stmt(sql_text, params, "ann_registry_test"))
506            .await
507            .expect("execute");
508    }
509
510    async fn scalar_i64(sql: &dyn SqlAccess, sql_text: &str, params: Vec<SqlValue>) -> i64 {
511        match sql
512            .reader()
513            .await
514            .expect("reader")
515            .query_scalar(stmt(sql_text, params, "ann_registry_test_scalar"))
516            .await
517            .expect("query scalar")
518        {
519            Some(SqlValue::Integer(value)) => value,
520            other => panic!("unexpected scalar: {other:?}"),
521        }
522    }
523
524    #[tokio::test]
525    async fn expired_pending_consumer_retires_without_weakening_active_minimum() {
526        let rt = KhiveRuntime::memory().expect("runtime");
527        let sql = rt.sql();
528        let model = "ann-registry-expiry";
529        register_pending_at(sql.as_ref(), "dormant", "local", model, 1)
530            .await
531            .expect("register dormant");
532        register_pending_at(sql.as_ref(), "active", "local", model, 1)
533            .await
534            .expect("register active");
535        assert!(raise_watermark(
536            sql.as_ref(),
537            "active",
538            "local",
539            model,
540            2,
541            WatermarkAuthority::PendingOrActive,
542        )
543        .await
544        .expect("activate"));
545        for seq in 1..=3 {
546            execute(
547                sql.as_ref(),
548                "INSERT INTO ann_write_log \
549                 (seq, namespace, embedding_model, kind, field, subject_id, op) \
550                 VALUES (?1, 'local', ?2, 'note', 'note.content', ?3, 'upsert')",
551                vec![
552                    SqlValue::Integer(seq),
553                    SqlValue::Text(model.into()),
554                    SqlValue::Text(format!("subject-{seq}")),
555                ],
556            )
557            .await;
558        }
559
560        let outcome = compact_write_log_at(
561            sql.as_ref(),
562            CompactionScope::Namespace("local".into()),
563            model,
564            PENDING_GRACE_US + 2,
565        )
566        .await
567        .expect("compact");
568        assert_eq!(outcome.retired_consumers.len(), 1);
569        assert_eq!(outcome.retired_consumers[0].consumer, "dormant");
570        assert_eq!(outcome.deleted_log_rows, 2);
571        assert_eq!(
572            scalar_i64(
573                sql.as_ref(),
574                "SELECT COUNT(*) FROM ann_write_log WHERE embedding_model = ?1",
575                vec![SqlValue::Text(model.into())],
576            )
577            .await,
578            1
579        );
580        assert_eq!(
581            scalar_i64(
582                sql.as_ref(),
583                "SELECT COUNT(*) FROM ann_consumer_watermark \
584                 WHERE consumer = 'active' AND embedding_model = ?1 AND watermark = 2",
585                vec![SqlValue::Text(model.into())],
586            )
587            .await,
588            1
589        );
590    }
591
592    #[tokio::test]
593    async fn retirement_wins_before_delayed_checkpoint_publication() {
594        let rt = KhiveRuntime::memory().expect("runtime");
595        let sql = rt.sql();
596        let model = "ann-registry-retire-race";
597        register_pending_at(sql.as_ref(), "dormant", "local", model, 1)
598            .await
599            .expect("register");
600        let outcome = compact_write_log_at(
601            sql.as_ref(),
602            CompactionScope::Namespace("local".into()),
603            model,
604            PENDING_GRACE_US + 2,
605        )
606        .await
607        .expect("compact");
608        assert_eq!(outcome.retired_consumers.len(), 1);
609        assert!(!raise_watermark(
610            sql.as_ref(),
611            "dormant",
612            "local",
613            model,
614            7,
615            WatermarkAuthority::PendingOrActive,
616        )
617        .await
618        .expect("delayed raise"));
619    }
620
621    #[tokio::test]
622    async fn pending_row_just_inside_grace_survives_and_still_pins_the_log() {
623        let rt = KhiveRuntime::memory().expect("runtime");
624        let sql = rt.sql();
625        let model = "ann-registry-grace-inside";
626        // cutoff = now - PENDING_GRACE_US = 2; registered_at 3 is one
627        // microsecond inside the grace window.
628        register_pending_at(sql.as_ref(), "fresh", "local", model, 3)
629            .await
630            .expect("register");
631        register_pending_at(sql.as_ref(), "peer", "local", model, 3)
632            .await
633            .expect("register peer");
634        assert!(raise_watermark(
635            sql.as_ref(),
636            "peer",
637            "local",
638            model,
639            5,
640            WatermarkAuthority::PendingOrActive,
641        )
642        .await
643        .expect("seed active peer"));
644        execute(
645            sql.as_ref(),
646            "INSERT INTO ann_write_log \
647             (seq, namespace, embedding_model, kind, field, subject_id, op) \
648             VALUES (1, 'local', ?1, 'note', 'note.content', 'subject-1', 'upsert')",
649            vec![SqlValue::Text(model.into())],
650        )
651        .await;
652
653        let outcome = compact_write_log_at(
654            sql.as_ref(),
655            CompactionScope::Namespace("local".into()),
656            model,
657            PENDING_GRACE_US + 2,
658        )
659        .await
660        .expect("compact");
661        assert!(
662            outcome.retired_consumers.is_empty(),
663            "a row inside the grace window must not retire: {:?}",
664            outcome.retired_consumers
665        );
666        // The surviving pending row (-2) keeps the registry minimum below
667        // every log seq, so nothing compacts away underneath the live build.
668        assert_eq!(outcome.deleted_log_rows, 0);
669        assert!(
670            raise_watermark(
671                sql.as_ref(),
672                "fresh",
673                "local",
674                model,
675                1,
676                WatermarkAuthority::PendingOrActive,
677            )
678            .await
679            .expect("in-grace consumer must still be able to publish"),
680            "the surviving registration must accept its checkpoint"
681        );
682    }
683
684    #[tokio::test]
685    async fn pending_row_at_exact_grace_boundary_retires() {
686        let rt = KhiveRuntime::memory().expect("runtime");
687        let sql = rt.sql();
688        let model = "ann-registry-grace-boundary";
689        // cutoff = now - PENDING_GRACE_US = 2; registered_at 2 sits exactly
690        // on the boundary, which the <= comparator retires.
691        register_pending_at(sql.as_ref(), "boundary", "local", model, 2)
692            .await
693            .expect("register");
694        let outcome = compact_write_log_at(
695            sql.as_ref(),
696            CompactionScope::Namespace("local".into()),
697            model,
698            PENDING_GRACE_US + 2,
699        )
700        .await
701        .expect("compact");
702        assert_eq!(outcome.retired_consumers.len(), 1);
703        assert_eq!(outcome.retired_consumers[0].consumer, "boundary");
704        assert_eq!(outcome.retired_consumers[0].registered_at_us, 2);
705        assert!(!raise_watermark(
706            sql.as_ref(),
707            "boundary",
708            "local",
709            model,
710            4,
711            WatermarkAuthority::PendingOrActive,
712        )
713        .await
714        .expect("raise after boundary retirement"));
715    }
716
717    #[tokio::test]
718    async fn checkpoint_publication_before_compaction_defeats_age_retirement() {
719        // Writer serialization makes the retirement/raise decision atomic in
720        // whichever order the two transactions commit. The retire-first
721        // ordering is pinned by `retirement_wins_before_delayed_checkpoint_
722        // publication`; this is the raise-first ordering: an activation that
723        // commits before compaction must protect the row from age
724        // retirement, however far past its grace the registration is.
725        let rt = KhiveRuntime::memory().expect("runtime");
726        let sql = rt.sql();
727        let model = "ann-registry-activate-race";
728        register_pending_at(sql.as_ref(), "late-build", "local", model, 1)
729            .await
730            .expect("register");
731        assert!(raise_watermark(
732            sql.as_ref(),
733            "late-build",
734            "local",
735            model,
736            9,
737            WatermarkAuthority::PendingOrActive,
738        )
739        .await
740        .expect("activation commits first"));
741        let outcome = compact_write_log_at(
742            sql.as_ref(),
743            CompactionScope::Namespace("local".into()),
744            model,
745            10 * PENDING_GRACE_US,
746        )
747        .await
748        .expect("compact");
749        assert!(
750            outcome.retired_consumers.is_empty(),
751            "an activated consumer must never age-retire: {:?}",
752            outcome.retired_consumers
753        );
754        assert_eq!(
755            scalar_i64(
756                sql.as_ref(),
757                "SELECT watermark FROM ann_consumer_watermark \
758                 WHERE consumer = 'late-build' AND embedding_model = ?1",
759                vec![SqlValue::Text(model.into())],
760            )
761            .await,
762            9
763        );
764    }
765
766    #[tokio::test]
767    async fn repeated_pending_registration_does_not_refresh_grace() {
768        let rt = KhiveRuntime::memory().expect("runtime");
769        let sql = rt.sql();
770        let model = "ann-registry-stable-grace";
771        register_pending_at(sql.as_ref(), "pending", "local", model, 11)
772            .await
773            .expect("first registration");
774        register_pending_at(sql.as_ref(), "pending", "local", model, 99)
775            .await
776            .expect("repeat registration");
777        assert_eq!(
778            scalar_i64(
779                sql.as_ref(),
780                "SELECT registered_at_us FROM ann_consumer_pending \
781                 WHERE consumer = 'pending' AND embedding_model = ?1",
782                vec![SqlValue::Text(model.into())],
783            )
784            .await,
785            11
786        );
787    }
788
789    #[tokio::test]
790    async fn recovering_and_active_zero_consumers_never_age_retire() {
791        let rt = KhiveRuntime::memory().expect("runtime");
792        let sql = rt.sql();
793        let model = "ann-registry-protected-states";
794        register_pending_at(sql.as_ref(), "active-zero", "local", model, 1)
795            .await
796            .expect("register active zero");
797        assert!(raise_watermark(
798            sql.as_ref(),
799            "active-zero",
800            "local",
801            model,
802            0,
803            WatermarkAuthority::PendingOrActive,
804        )
805        .await
806        .expect("activate at zero"));
807        mark_recovering(sql.as_ref(), "recovering", "local", model)
808            .await
809            .expect("recovering");
810
811        let outcome = compact_write_log_at(
812            sql.as_ref(),
813            CompactionScope::Namespace("local".into()),
814            model,
815            PENDING_GRACE_US * 10,
816        )
817        .await
818        .expect("compact");
819        assert!(outcome.retired_consumers.is_empty());
820        assert_eq!(
821            scalar_i64(
822                sql.as_ref(),
823                "SELECT COUNT(*) FROM ann_consumer_watermark \
824                 WHERE embedding_model = ?1",
825                vec![SqlValue::Text(model.into())],
826            )
827            .await,
828            2
829        );
830    }
831}