1use std::any::Any;
12
13use khive_storage::types::{SqlStatement, SqlValue};
14use khive_storage::{AtomicUnitOp, SqlAccess, StorageError, StorageResult};
15
16pub const PENDING_WATERMARK: i64 = -2;
20
21pub const RECOVERING_WATERMARK: i64 = -1;
24
25pub const PENDING_GRACE_US: i64 = 24 * 60 * 60 * 1_000_000;
29
30#[derive(Clone, Copy, Debug, Eq, PartialEq)]
32pub enum WatermarkAuthority {
33 PendingOrActive,
36 Active,
38 Recovering,
40}
41
42#[derive(Clone, Debug, Eq, PartialEq)]
44pub enum CompactionScope {
45 Namespace(String),
48 Model,
51}
52
53#[derive(Clone, Debug, Eq, PartialEq)]
55pub struct RetiredConsumer {
56 pub consumer: String,
58 pub namespace: String,
60 pub embedding_model: String,
62 pub registered_at_us: i64,
64}
65
66#[derive(Clone, Debug, Default, Eq, PartialEq)]
68pub struct CompactionOutcome {
69 pub deleted_log_rows: u64,
71 pub retired_consumers: Vec<RetiredConsumer>,
73}
74
75fn stmt(sql: impl Into<String>, params: Vec<SqlValue>, label: &str) -> SqlStatement {
76 SqlStatement {
77 sql: sql.into(),
78 params,
79 label: Some(label.to_owned()),
80 }
81}
82
83fn integer_range_error(value: u64) -> StorageError {
84 StorageError::Internal(format!(
85 "ANN watermark {value} exceeds SQLite INTEGER range"
86 ))
87}
88
89pub async fn register_pending(
92 sql: &dyn SqlAccess,
93 consumer: &str,
94 namespace: &str,
95 model: &str,
96) -> StorageResult<()> {
97 register_pending_at(
98 sql,
99 consumer,
100 namespace,
101 model,
102 chrono::Utc::now().timestamp_micros(),
103 )
104 .await
105}
106
107async fn register_pending_at(
108 sql: &dyn SqlAccess,
109 consumer: &str,
110 namespace: &str,
111 model: &str,
112 registered_at_us: i64,
113) -> StorageResult<()> {
114 let consumer = consumer.to_owned();
115 let namespace = namespace.to_owned();
116 let model = model.to_owned();
117 let op: AtomicUnitOp = Box::new(move |writer| {
118 Box::pin(async move {
119 let inserted = writer
120 .execute(stmt(
121 "INSERT OR IGNORE INTO ann_consumer_watermark \
122 (consumer, namespace, embedding_model, watermark) \
123 VALUES (?1, ?2, ?3, ?4)",
124 vec![
125 SqlValue::Text(consumer.clone()),
126 SqlValue::Text(namespace.clone()),
127 SqlValue::Text(model.clone()),
128 SqlValue::Integer(PENDING_WATERMARK),
129 ],
130 "ann_registry_register_pending",
131 ))
132 .await?;
133 let pending_insert = if inserted == 1 {
134 "INSERT INTO ann_consumer_pending \
138 (consumer, namespace, embedding_model, registered_at_us) \
139 VALUES (?1, ?2, ?3, ?4) \
140 ON CONFLICT(consumer, namespace, embedding_model) \
141 DO UPDATE SET registered_at_us = excluded.registered_at_us"
142 } else {
143 "INSERT OR IGNORE INTO ann_consumer_pending \
146 (consumer, namespace, embedding_model, registered_at_us) \
147 SELECT ?1, ?2, ?3, ?4 \
148 WHERE EXISTS (SELECT 1 FROM ann_consumer_watermark \
149 WHERE consumer = ?1 AND namespace = ?2 \
150 AND embedding_model = ?3 AND watermark = ?5)"
151 };
152 let mut pending_params = vec![
153 SqlValue::Text(consumer),
154 SqlValue::Text(namespace),
155 SqlValue::Text(model),
156 SqlValue::Integer(registered_at_us),
157 ];
158 if inserted != 1 {
159 pending_params.push(SqlValue::Integer(PENDING_WATERMARK));
160 }
161 writer
162 .execute(stmt(
163 pending_insert,
164 pending_params,
165 "ann_registry_stamp_pending",
166 ))
167 .await?;
168 Ok(Box::new(()) as Box<dyn Any + Send>)
169 })
170 });
171 sql.atomic_unit(op).await?;
172 Ok(())
173}
174
175pub async fn mark_recovering(
178 sql: &dyn SqlAccess,
179 consumer: &str,
180 namespace: &str,
181 model: &str,
182) -> StorageResult<()> {
183 let warning_consumer = consumer.to_owned();
184 let warning_namespace = namespace.to_owned();
185 let warning_model = model.to_owned();
186 let consumer = consumer.to_owned();
187 let namespace = namespace.to_owned();
188 let model = model.to_owned();
189 let op: AtomicUnitOp = Box::new(move |writer| {
190 Box::pin(async move {
191 writer
192 .execute(stmt(
193 "INSERT INTO ann_consumer_watermark \
194 (consumer, namespace, embedding_model, watermark) \
195 VALUES (?1, ?2, ?3, ?4) \
196 ON CONFLICT(consumer, namespace, embedding_model) \
197 DO UPDATE SET watermark = excluded.watermark",
198 vec![
199 SqlValue::Text(consumer.clone()),
200 SqlValue::Text(namespace.clone()),
201 SqlValue::Text(model.clone()),
202 SqlValue::Integer(RECOVERING_WATERMARK),
203 ],
204 "ann_registry_mark_recovering",
205 ))
206 .await?;
207 writer
208 .execute(stmt(
209 "DELETE FROM ann_consumer_pending \
210 WHERE consumer = ?1 AND namespace = ?2 AND embedding_model = ?3",
211 vec![
212 SqlValue::Text(consumer),
213 SqlValue::Text(namespace),
214 SqlValue::Text(model),
215 ],
216 "ann_registry_clear_pending_for_recovery",
217 ))
218 .await?;
219 Ok(Box::new(()) as Box<dyn Any + Send>)
220 })
221 });
222 sql.atomic_unit(op).await?;
223 tracing::warn!(
224 consumer = %warning_consumer,
225 namespace = %warning_namespace,
226 embedding_model = %warning_model,
227 "ANN consumer entered the non-expiring recovery fence; compaction remains blocked \
228 until an authoritative full checkpoint succeeds"
229 );
230 Ok(())
231}
232
233pub async fn raise_watermark(
240 sql: &dyn SqlAccess,
241 consumer: &str,
242 namespace: &str,
243 model: &str,
244 watermark: u64,
245 authority: WatermarkAuthority,
246) -> StorageResult<bool> {
247 let watermark = i64::try_from(watermark).map_err(|_| integer_range_error(watermark))?;
248 let predicate = match authority {
249 WatermarkAuthority::PendingOrActive => {
250 "(watermark = -2 OR (watermark >= 0 AND watermark <= ?4))"
251 }
252 WatermarkAuthority::Active => "watermark >= 0 AND watermark <= ?4",
253 WatermarkAuthority::Recovering => "watermark = -1",
254 };
255 let consumer = consumer.to_owned();
256 let namespace = namespace.to_owned();
257 let model = model.to_owned();
258 let op: AtomicUnitOp = Box::new(move |writer| {
259 Box::pin(async move {
260 let affected = writer
261 .execute(stmt(
262 format!(
263 "UPDATE ann_consumer_watermark SET watermark = ?4 \
264 WHERE consumer = ?1 AND namespace = ?2 AND embedding_model = ?3 \
265 AND {predicate}"
266 ),
267 vec![
268 SqlValue::Text(consumer.clone()),
269 SqlValue::Text(namespace.clone()),
270 SqlValue::Text(model.clone()),
271 SqlValue::Integer(watermark),
272 ],
273 "ann_registry_raise_watermark",
274 ))
275 .await?;
276 if affected == 1 {
277 writer
278 .execute(stmt(
279 "DELETE FROM ann_consumer_pending \
280 WHERE consumer = ?1 AND namespace = ?2 AND embedding_model = ?3",
281 vec![
282 SqlValue::Text(consumer),
283 SqlValue::Text(namespace),
284 SqlValue::Text(model),
285 ],
286 "ann_registry_activate_consumer",
287 ))
288 .await?;
289 }
290 Ok(Box::new(affected == 1) as Box<dyn Any + Send>)
291 })
292 });
293 let result = sql.atomic_unit(op).await?;
294 Ok(*result
295 .downcast::<bool>()
296 .expect("ANN watermark atomic unit returns bool"))
297}
298
299pub async fn compact_write_log(
302 sql: &dyn SqlAccess,
303 scope: CompactionScope,
304 model: &str,
305) -> StorageResult<CompactionOutcome> {
306 compact_write_log_at(sql, scope, model, chrono::Utc::now().timestamp_micros()).await
307}
308
309async fn compact_write_log_at(
310 sql: &dyn SqlAccess,
311 scope: CompactionScope,
312 model: &str,
313 now_us: i64,
314) -> StorageResult<CompactionOutcome> {
315 let cutoff_us = now_us.saturating_sub(PENDING_GRACE_US);
316 let model = model.to_owned();
317 let op: AtomicUnitOp = Box::new(move |writer| {
318 Box::pin(async move {
319 let (scope_filter, scope_params) = match &scope {
320 CompactionScope::Namespace(namespace) => (
321 " AND (watermark.namespace = ?3 OR watermark.namespace = '*')",
322 vec![
323 SqlValue::Integer(cutoff_us),
324 SqlValue::Text(model.clone()),
325 SqlValue::Text(namespace.clone()),
326 ],
327 ),
328 CompactionScope::Model => (
329 "",
330 vec![SqlValue::Integer(cutoff_us), SqlValue::Text(model.clone())],
331 ),
332 };
333
334 let backfill_sql = format!(
339 "INSERT OR IGNORE INTO ann_consumer_pending \
340 (consumer, namespace, embedding_model, registered_at_us) \
341 SELECT watermark.consumer, watermark.namespace, \
342 watermark.embedding_model, ?1 \
343 FROM ann_consumer_watermark watermark \
344 WHERE watermark.watermark = -2 AND watermark.embedding_model = ?2{scope_filter}"
345 );
346 let mut backfill_params = scope_params.clone();
347 backfill_params[0] = SqlValue::Integer(now_us);
348 writer
349 .execute(stmt(
350 backfill_sql,
351 backfill_params,
352 "ann_registry_backfill_pending_timestamp",
353 ))
354 .await?;
355
356 let retired_rows = writer
357 .query_all(stmt(
358 format!(
359 "SELECT watermark.consumer, watermark.namespace, \
360 watermark.embedding_model, pending.registered_at_us \
361 FROM ann_consumer_watermark watermark \
362 JOIN ann_consumer_pending pending \
363 ON pending.consumer = watermark.consumer \
364 AND pending.namespace = watermark.namespace \
365 AND pending.embedding_model = watermark.embedding_model \
366 WHERE pending.registered_at_us <= ?1 \
367 AND watermark.watermark = -2 \
368 AND watermark.embedding_model = ?2{scope_filter} \
369 ORDER BY watermark.consumer, watermark.namespace"
370 ),
371 scope_params.clone(),
372 "ann_registry_find_expired_pending",
373 ))
374 .await?;
375
376 writer
377 .execute(stmt(
378 format!(
379 "DELETE FROM ann_consumer_watermark AS watermark \
380 WHERE watermark.watermark = -2 \
381 AND watermark.embedding_model = ?2{scope_filter} \
382 AND EXISTS (SELECT 1 FROM ann_consumer_pending pending \
383 WHERE pending.consumer = watermark.consumer \
384 AND pending.namespace = watermark.namespace \
385 AND pending.embedding_model = watermark.embedding_model \
386 AND pending.registered_at_us <= ?1)"
387 ),
388 scope_params.clone(),
389 "ann_registry_retire_expired_pending",
390 ))
391 .await?;
392
393 let pending_scope_filter = scope_filter.replace("watermark.", "pending.");
397 writer
398 .execute(stmt(
399 format!(
400 "DELETE FROM ann_consumer_pending AS pending \
401 WHERE pending.embedding_model = ?2{pending_scope_filter} \
402 AND NOT EXISTS (SELECT 1 FROM ann_consumer_watermark watermark \
403 WHERE watermark.consumer = pending.consumer \
404 AND watermark.namespace = pending.namespace \
405 AND watermark.embedding_model = pending.embedding_model \
406 AND watermark.watermark = -2)"
407 ),
408 scope_params,
409 "ann_registry_prune_pending_metadata",
410 ))
411 .await?;
412
413 let deleted_log_rows = match &scope {
414 CompactionScope::Namespace(namespace) => {
415 writer
416 .execute(stmt(
417 "DELETE FROM ann_write_log \
418 WHERE namespace = ?1 AND embedding_model = ?2 \
419 AND seq <= (SELECT MIN(watermark) \
420 FROM ann_consumer_watermark \
421 WHERE (namespace = ?1 OR namespace = '*') \
422 AND embedding_model = ?2)",
423 vec![
424 SqlValue::Text(namespace.clone()),
425 SqlValue::Text(model.clone()),
426 ],
427 "ann_registry_compact_namespace",
428 ))
429 .await?
430 }
431 CompactionScope::Model => {
432 writer
433 .execute(stmt(
434 "DELETE FROM ann_write_log \
435 WHERE embedding_model = ?1 \
436 AND seq <= (SELECT MIN(watermark.watermark) \
437 FROM ann_consumer_watermark watermark \
438 WHERE (watermark.namespace = ann_write_log.namespace \
439 OR watermark.namespace = '*') \
440 AND watermark.embedding_model = ?1)",
441 vec![SqlValue::Text(model.clone())],
442 "ann_registry_compact_model",
443 ))
444 .await?
445 }
446 };
447
448 let mut retired_consumers = Vec::with_capacity(retired_rows.len());
449 for row in retired_rows {
450 let text = |column: &str| match row.get(column) {
451 Some(SqlValue::Text(value)) => Ok(value.clone()),
452 other => Err(StorageError::Internal(format!(
453 "ANN retired-consumer {column}: unexpected {other:?}"
454 ))),
455 };
456 let registered_at_us = match row.get("registered_at_us") {
457 Some(SqlValue::Integer(value)) => *value,
458 other => {
459 return Err(StorageError::Internal(format!(
460 "ANN retired-consumer registered_at_us: unexpected {other:?}"
461 )))
462 }
463 };
464 retired_consumers.push(RetiredConsumer {
465 consumer: text("consumer")?,
466 namespace: text("namespace")?,
467 embedding_model: text("embedding_model")?,
468 registered_at_us,
469 });
470 }
471 Ok(Box::new(CompactionOutcome {
472 deleted_log_rows,
473 retired_consumers,
474 }) as Box<dyn Any + Send>)
475 })
476 });
477
478 let result = sql.atomic_unit(op).await?;
479 let outcome = *result
480 .downcast::<CompactionOutcome>()
481 .expect("ANN compaction atomic unit returns CompactionOutcome");
482 for retired in &outcome.retired_consumers {
483 tracing::warn!(
484 consumer = %retired.consumer,
485 namespace = %retired.namespace,
486 embedding_model = %retired.embedding_model,
487 registered_at_us = retired.registered_at_us,
488 grace_hours = PENDING_GRACE_US / (60 * 60 * 1_000_000),
489 "retired dormant ANN consumer which never published a checkpoint; \
490 a returning consumer must rebuild before serving"
491 );
492 }
493 Ok(outcome)
494}
495
496#[cfg(test)]
497mod tests {
498 use super::*;
499 use crate::KhiveRuntime;
500
501 async fn execute(sql: &dyn SqlAccess, sql_text: &str, params: Vec<SqlValue>) {
502 sql.writer()
503 .await
504 .expect("writer")
505 .execute(stmt(sql_text, params, "ann_registry_test"))
506 .await
507 .expect("execute");
508 }
509
510 async fn scalar_i64(sql: &dyn SqlAccess, sql_text: &str, params: Vec<SqlValue>) -> i64 {
511 match sql
512 .reader()
513 .await
514 .expect("reader")
515 .query_scalar(stmt(sql_text, params, "ann_registry_test_scalar"))
516 .await
517 .expect("query scalar")
518 {
519 Some(SqlValue::Integer(value)) => value,
520 other => panic!("unexpected scalar: {other:?}"),
521 }
522 }
523
524 #[tokio::test]
525 async fn expired_pending_consumer_retires_without_weakening_active_minimum() {
526 let rt = KhiveRuntime::memory().expect("runtime");
527 let sql = rt.sql();
528 let model = "ann-registry-expiry";
529 register_pending_at(sql.as_ref(), "dormant", "local", model, 1)
530 .await
531 .expect("register dormant");
532 register_pending_at(sql.as_ref(), "active", "local", model, 1)
533 .await
534 .expect("register active");
535 assert!(raise_watermark(
536 sql.as_ref(),
537 "active",
538 "local",
539 model,
540 2,
541 WatermarkAuthority::PendingOrActive,
542 )
543 .await
544 .expect("activate"));
545 for seq in 1..=3 {
546 execute(
547 sql.as_ref(),
548 "INSERT INTO ann_write_log \
549 (seq, namespace, embedding_model, kind, field, subject_id, op) \
550 VALUES (?1, 'local', ?2, 'note', 'note.content', ?3, 'upsert')",
551 vec![
552 SqlValue::Integer(seq),
553 SqlValue::Text(model.into()),
554 SqlValue::Text(format!("subject-{seq}")),
555 ],
556 )
557 .await;
558 }
559
560 let outcome = compact_write_log_at(
561 sql.as_ref(),
562 CompactionScope::Namespace("local".into()),
563 model,
564 PENDING_GRACE_US + 2,
565 )
566 .await
567 .expect("compact");
568 assert_eq!(outcome.retired_consumers.len(), 1);
569 assert_eq!(outcome.retired_consumers[0].consumer, "dormant");
570 assert_eq!(outcome.deleted_log_rows, 2);
571 assert_eq!(
572 scalar_i64(
573 sql.as_ref(),
574 "SELECT COUNT(*) FROM ann_write_log WHERE embedding_model = ?1",
575 vec![SqlValue::Text(model.into())],
576 )
577 .await,
578 1
579 );
580 assert_eq!(
581 scalar_i64(
582 sql.as_ref(),
583 "SELECT COUNT(*) FROM ann_consumer_watermark \
584 WHERE consumer = 'active' AND embedding_model = ?1 AND watermark = 2",
585 vec![SqlValue::Text(model.into())],
586 )
587 .await,
588 1
589 );
590 }
591
592 #[tokio::test]
593 async fn retirement_wins_before_delayed_checkpoint_publication() {
594 let rt = KhiveRuntime::memory().expect("runtime");
595 let sql = rt.sql();
596 let model = "ann-registry-retire-race";
597 register_pending_at(sql.as_ref(), "dormant", "local", model, 1)
598 .await
599 .expect("register");
600 let outcome = compact_write_log_at(
601 sql.as_ref(),
602 CompactionScope::Namespace("local".into()),
603 model,
604 PENDING_GRACE_US + 2,
605 )
606 .await
607 .expect("compact");
608 assert_eq!(outcome.retired_consumers.len(), 1);
609 assert!(!raise_watermark(
610 sql.as_ref(),
611 "dormant",
612 "local",
613 model,
614 7,
615 WatermarkAuthority::PendingOrActive,
616 )
617 .await
618 .expect("delayed raise"));
619 }
620
621 #[tokio::test]
622 async fn pending_row_just_inside_grace_survives_and_still_pins_the_log() {
623 let rt = KhiveRuntime::memory().expect("runtime");
624 let sql = rt.sql();
625 let model = "ann-registry-grace-inside";
626 register_pending_at(sql.as_ref(), "fresh", "local", model, 3)
629 .await
630 .expect("register");
631 register_pending_at(sql.as_ref(), "peer", "local", model, 3)
632 .await
633 .expect("register peer");
634 assert!(raise_watermark(
635 sql.as_ref(),
636 "peer",
637 "local",
638 model,
639 5,
640 WatermarkAuthority::PendingOrActive,
641 )
642 .await
643 .expect("seed active peer"));
644 execute(
645 sql.as_ref(),
646 "INSERT INTO ann_write_log \
647 (seq, namespace, embedding_model, kind, field, subject_id, op) \
648 VALUES (1, 'local', ?1, 'note', 'note.content', 'subject-1', 'upsert')",
649 vec![SqlValue::Text(model.into())],
650 )
651 .await;
652
653 let outcome = compact_write_log_at(
654 sql.as_ref(),
655 CompactionScope::Namespace("local".into()),
656 model,
657 PENDING_GRACE_US + 2,
658 )
659 .await
660 .expect("compact");
661 assert!(
662 outcome.retired_consumers.is_empty(),
663 "a row inside the grace window must not retire: {:?}",
664 outcome.retired_consumers
665 );
666 assert_eq!(outcome.deleted_log_rows, 0);
669 assert!(
670 raise_watermark(
671 sql.as_ref(),
672 "fresh",
673 "local",
674 model,
675 1,
676 WatermarkAuthority::PendingOrActive,
677 )
678 .await
679 .expect("in-grace consumer must still be able to publish"),
680 "the surviving registration must accept its checkpoint"
681 );
682 }
683
684 #[tokio::test]
685 async fn pending_row_at_exact_grace_boundary_retires() {
686 let rt = KhiveRuntime::memory().expect("runtime");
687 let sql = rt.sql();
688 let model = "ann-registry-grace-boundary";
689 register_pending_at(sql.as_ref(), "boundary", "local", model, 2)
692 .await
693 .expect("register");
694 let outcome = compact_write_log_at(
695 sql.as_ref(),
696 CompactionScope::Namespace("local".into()),
697 model,
698 PENDING_GRACE_US + 2,
699 )
700 .await
701 .expect("compact");
702 assert_eq!(outcome.retired_consumers.len(), 1);
703 assert_eq!(outcome.retired_consumers[0].consumer, "boundary");
704 assert_eq!(outcome.retired_consumers[0].registered_at_us, 2);
705 assert!(!raise_watermark(
706 sql.as_ref(),
707 "boundary",
708 "local",
709 model,
710 4,
711 WatermarkAuthority::PendingOrActive,
712 )
713 .await
714 .expect("raise after boundary retirement"));
715 }
716
717 #[tokio::test]
718 async fn checkpoint_publication_before_compaction_defeats_age_retirement() {
719 let rt = KhiveRuntime::memory().expect("runtime");
726 let sql = rt.sql();
727 let model = "ann-registry-activate-race";
728 register_pending_at(sql.as_ref(), "late-build", "local", model, 1)
729 .await
730 .expect("register");
731 assert!(raise_watermark(
732 sql.as_ref(),
733 "late-build",
734 "local",
735 model,
736 9,
737 WatermarkAuthority::PendingOrActive,
738 )
739 .await
740 .expect("activation commits first"));
741 let outcome = compact_write_log_at(
742 sql.as_ref(),
743 CompactionScope::Namespace("local".into()),
744 model,
745 10 * PENDING_GRACE_US,
746 )
747 .await
748 .expect("compact");
749 assert!(
750 outcome.retired_consumers.is_empty(),
751 "an activated consumer must never age-retire: {:?}",
752 outcome.retired_consumers
753 );
754 assert_eq!(
755 scalar_i64(
756 sql.as_ref(),
757 "SELECT watermark FROM ann_consumer_watermark \
758 WHERE consumer = 'late-build' AND embedding_model = ?1",
759 vec![SqlValue::Text(model.into())],
760 )
761 .await,
762 9
763 );
764 }
765
766 #[tokio::test]
767 async fn repeated_pending_registration_does_not_refresh_grace() {
768 let rt = KhiveRuntime::memory().expect("runtime");
769 let sql = rt.sql();
770 let model = "ann-registry-stable-grace";
771 register_pending_at(sql.as_ref(), "pending", "local", model, 11)
772 .await
773 .expect("first registration");
774 register_pending_at(sql.as_ref(), "pending", "local", model, 99)
775 .await
776 .expect("repeat registration");
777 assert_eq!(
778 scalar_i64(
779 sql.as_ref(),
780 "SELECT registered_at_us FROM ann_consumer_pending \
781 WHERE consumer = 'pending' AND embedding_model = ?1",
782 vec![SqlValue::Text(model.into())],
783 )
784 .await,
785 11
786 );
787 }
788
789 #[tokio::test]
790 async fn recovering_and_active_zero_consumers_never_age_retire() {
791 let rt = KhiveRuntime::memory().expect("runtime");
792 let sql = rt.sql();
793 let model = "ann-registry-protected-states";
794 register_pending_at(sql.as_ref(), "active-zero", "local", model, 1)
795 .await
796 .expect("register active zero");
797 assert!(raise_watermark(
798 sql.as_ref(),
799 "active-zero",
800 "local",
801 model,
802 0,
803 WatermarkAuthority::PendingOrActive,
804 )
805 .await
806 .expect("activate at zero"));
807 mark_recovering(sql.as_ref(), "recovering", "local", model)
808 .await
809 .expect("recovering");
810
811 let outcome = compact_write_log_at(
812 sql.as_ref(),
813 CompactionScope::Namespace("local".into()),
814 model,
815 PENDING_GRACE_US * 10,
816 )
817 .await
818 .expect("compact");
819 assert!(outcome.retired_consumers.is_empty());
820 assert_eq!(
821 scalar_i64(
822 sql.as_ref(),
823 "SELECT COUNT(*) FROM ann_consumer_watermark \
824 WHERE embedding_model = ?1",
825 vec![SqlValue::Text(model.into())],
826 )
827 .await,
828 2
829 );
830 }
831}