use super::map;
use crate::crypto;
use crate::http::{HttpError, RestClient};
use crate::store::{queries, Store, StoreError};
use std::time::{Duration, Instant};
use tracing::warn;
pub struct BackfillBudget {
pub max_wall: Duration,
pub max_clips: u32,
}
impl Default for BackfillBudget {
fn default() -> Self {
Self {
max_wall: Duration::from_secs(2),
max_clips: 500,
}
}
}
fn ulid_to_datetime(ulid_str: &str) -> Option<chrono::DateTime<chrono::Utc>> {
use std::time::UNIX_EPOCH;
let ulid = ulid::Ulid::from_string(ulid_str).ok()?;
let ms = ulid.datetime().duration_since(UNIX_EPOCH).ok()?.as_millis() as i64;
chrono::DateTime::from_timestamp_millis(ms)
}
pub async fn backfill_once(
store: &Store,
client: &RestClient,
budget: BackfillBudget,
enc_key: Option<&[u8; 32]>,
) -> Result<usize, BackfillError> {
let start = Instant::now();
let since = queries::watermark(store)
.map_err(BackfillError::Store)?
.and_then(|w| ulid_to_datetime(&w));
let clips = client
.list_clips_since(since, budget.max_clips)
.await
.map_err(BackfillError::Http)?;
let mut inserted = 0usize;
let mut max_id: Option<String> = None;
for mut clip in clips {
if start.elapsed() >= budget.max_wall {
break;
}
if clip.encrypted {
match enc_key {
None => {
warn!(
clip_id = %clip.clip_id,
"backfill: skipping encrypted clip — no encryption key available"
);
continue;
}
Some(key) => {
match crypto::decrypt(key, &clip.content) {
Ok(plaintext) => {
clip.content = String::from_utf8_lossy(&plaintext).into_owned();
clip.encrypted = false;
}
Err(e) => {
warn!(
clip_id = %clip.clip_id,
error = %e,
"backfill: skipping encrypted clip — decryption failed"
);
continue;
}
}
}
}
}
let stored = match map::clip_wire_to_stored(&clip).map_err(BackfillError::Map)? {
Some(c) => c,
None => continue,
};
if max_id
.as_deref()
.map(|m| stored.id.as_str() > m)
.unwrap_or(true)
{
max_id = Some(stored.id.clone());
}
queries::insert_clip(store, &stored).map_err(BackfillError::Store)?;
inserted += 1;
}
if let Some(id) = max_id {
queries::set_watermark(store, &id).map_err(BackfillError::Store)?;
}
Ok(inserted)
}
#[derive(Debug, thiserror::Error)]
pub enum BackfillError {
#[error("store: {0}")]
Store(#[from] StoreError),
#[error("http: {0}")]
Http(HttpError),
#[error("map: {0}")]
Map(String),
}