Skip to main content

rostra_client_db/
lib.rs

1mod current_state;
2mod event_order;
3mod events_content_missing_ops;
4mod extension;
5mod id_nodes_ops;
6mod ids_full;
7mod migration_ops;
8mod models;
9pub mod news;
10mod paginate;
11mod process_event_content_ops;
12mod process_event_ops;
13mod reception_order_ops;
14mod self_followee;
15pub mod social;
16mod social_post_materialization;
17mod social_post_scan;
18mod social_post_scan_ops;
19#[cfg(test)]
20mod social_post_scan_tests;
21mod table_ops;
22mod tables;
23mod tx_ops;
24
25use std::borrow::Cow;
26use std::collections::{BTreeMap, HashMap, HashSet};
27use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind};
28use std::path::{Path, PathBuf};
29use std::sync::Arc;
30use std::{io, ops, result};
31
32use event::ContentStoreRecord;
33pub use ids::{IdsFolloweesRecord, IdsFollowersRecord};
34use itertools::Itertools as _;
35use process_event_content_ops::ProcessEventError;
36use redb_bincode::{ReadTransaction, ReadableTable, WriteTransaction};
37use rostra_core::event::{
38    EventAuxKey, EventContentRaw, EventExt as _, EventKind, IrohNodeId, PersonasTagsSelector,
39    VerifiedEvent, VerifiedEventContent, content_kind,
40};
41use rostra_core::id::{RostraId, ShortRostraId, ToShort as _};
42use rostra_core::{ExternalEventId, ShortEventId, Timestamp};
43use rostra_util_error::{BoxedError, FmtCompact as _};
44use snafu::{Location, ResultExt as _, Snafu};
45use tokio::sync::{Notify, broadcast, watch};
46use tokio::task::JoinError;
47use tracing::{debug, error, info, instrument};
48
49pub use self::current_state::{CurrentState, CurrentStateClosed};
50pub use self::extension::{
51    EXTENSION_RESERVED_TABLE_PREFIXES, ExtensionReadTransaction, ExtensionTableDefinition,
52    ExtensionWriteTransaction,
53};
54pub use self::self_followee::SelfFollowee;
55pub use self::social_post_materialization::{
56    SOCIAL_POST_MATERIALIZATION_SCAN_MAX, SocialPostMaterialization,
57    SocialPostMaterializationCursor, SocialPostMaterializationPage,
58};
59pub use self::social_post_scan::{
60    SocialPostScanPage, SocialPostScanRequest, SocialPostScanRow, SocialPostScanStop,
61};
62pub(crate) use self::tables::*;
63pub use self::tables::{
64    ContentStoreRecordOwned, EventContentResult, EventContentState, EventReceivedRecord,
65    EventReceivedSource, EventRecord, EventsHeadsTableRecord, IdSocialProfileRecord,
66    IdsDataUsageRecord, IrohNodeRecord, IrohNodeStats, Latest, SocialNewsRankRecord,
67    SocialPostRecord, SocialPostsReactionsRecord, SocialPostsRepliesRecord, SocialVoteScore,
68    SocialVoteSumRecord,
69};
70
71/// Web of Trust data - contains direct followees and extended followees.
72///
73/// Extended followees are the followees of your direct followees, excluding
74/// those you already follow directly.
75#[derive(Debug, Clone, Default)]
76pub struct WotData {
77    /// Direct followees with their persona selectors
78    pub followees: HashMap<RostraId, ids::IdsFolloweesRecord>,
79    /// Extended followees (followees of followees), excluding direct followees
80    pub extended: HashSet<RostraId>,
81}
82
83impl WotData {
84    /// Returns true if the given id is in our web of trust (self, direct
85    /// followee, or extended)
86    pub fn contains(&self, id: RostraId, self_id: RostraId) -> bool {
87        id == self_id || self.followees.contains_key(&id) || self.extended.contains(&id)
88    }
89
90    /// Returns the total number of IDs in the web of trust (excluding self)
91    pub fn len(&self) -> usize {
92        self.followees.len() + self.extended.len()
93    }
94
95    /// Returns true if there are no followees
96    pub fn is_empty(&self) -> bool {
97        self.followees.is_empty()
98    }
99
100    /// Returns an iterator over all IDs in the web of trust (direct +
101    /// extended), excluding self
102    pub fn iter_all(&self) -> impl Iterator<Item = RostraId> + '_ {
103        self.followees
104            .keys()
105            .copied()
106            .chain(self.extended.iter().copied())
107    }
108}
109
110const LOG_TARGET: &str = "rostra::db";
111
112type CommitHook = Box<dyn FnOnce() + 'static>;
113
114pub(crate) struct WriteTransactionCtx {
115    dbtx: WriteTransaction,
116    on_commit: std::sync::Mutex<Option<Vec<CommitHook>>>,
117    materialization_emission_enabled: std::sync::atomic::AtomicBool,
118}
119
120impl From<WriteTransaction> for WriteTransactionCtx {
121    fn from(dbtx: WriteTransaction) -> Self {
122        Self {
123            dbtx,
124            on_commit: std::sync::Mutex::new(Some(vec![])),
125            materialization_emission_enabled: std::sync::atomic::AtomicBool::new(true),
126        }
127    }
128}
129impl std::ops::Deref for WriteTransactionCtx {
130    type Target = WriteTransaction;
131
132    fn deref(&self) -> &Self::Target {
133        &self.dbtx
134    }
135}
136
137impl std::ops::DerefMut for WriteTransactionCtx {
138    fn deref_mut(&mut self) -> &mut Self::Target {
139        &mut self.dbtx
140    }
141}
142
143impl WriteTransactionCtx {
144    /// Registers an action to run in registration order after a successful
145    /// commit.
146    ///
147    /// A hook panic does not suppress later hooks. After every hook has been
148    /// attempted, the first panic resumes.
149    pub(crate) fn on_commit(&self, f: impl FnOnce() + 'static) {
150        if let Some(hooks) = self.on_commit.lock().expect("Locking failed").as_mut() {
151            hooks.push(Box::new(f));
152        }
153    }
154
155    /// Discard existing and future commit hooks for a bulk internal rebuild.
156    ///
157    /// Total replay reconstructs durable state before the database is exposed
158    /// to subscribers. Retaining one publication closure per source event
159    /// would make an otherwise streaming rebuild consume memory
160    /// proportional to database size.
161    pub(crate) fn discard_commit_hooks(&self) {
162        *self.on_commit.lock().expect("Locking failed") = None;
163    }
164
165    /// Return whether this transaction publishes incremental commit hooks.
166    pub(crate) fn commit_hooks_enabled(&self) -> bool {
167        self.on_commit.lock().expect("Locking failed").is_some()
168    }
169
170    /// Suppress durable occurrence emission during a projection rebuild.
171    pub(crate) fn suppress_materialization_emission(&self) {
172        self.materialization_emission_enabled
173            .store(false, std::sync::atomic::Ordering::Release);
174    }
175
176    /// Return whether durable materialization occurrences should be emitted.
177    pub(crate) fn materialization_emission_enabled(&self) -> bool {
178        self.materialization_emission_enabled
179            .load(std::sync::atomic::Ordering::Acquire)
180    }
181
182    fn commit(self) -> result::Result<(), redb::CommitError> {
183        let Self {
184            dbtx,
185            on_commit,
186            materialization_emission_enabled: _,
187        } = self;
188
189        dbtx.commit()?;
190
191        let mut first_panic = None;
192        if let Some(hooks) = on_commit.lock().expect("Locking failed").as_mut() {
193            for hook in hooks.drain(..) {
194                if let Err(payload) = catch_unwind(AssertUnwindSafe(hook)) {
195                    first_panic.get_or_insert(payload);
196                }
197            }
198        }
199        if let Some(payload) = first_panic {
200            resume_unwind(payload);
201        }
202        Ok(())
203    }
204}
205
206#[derive(Debug, Snafu)]
207pub enum TableDumpError {
208    #[snafu(display("Unknown table `{name}`"))]
209    UnknownTable { name: String },
210}
211pub type TableDumpResult<T> = std::result::Result<T, TableDumpError>;
212
213#[derive(Debug, Snafu)]
214pub enum DbError {
215    #[snafu(display("Database error"))]
216    Database {
217        source: redb::DatabaseError,
218        #[snafu(implicit)]
219        location: Location,
220    },
221    #[snafu(transparent)]
222    Table {
223        source: redb::TableError,
224        #[snafu(implicit)]
225        location: Location,
226    },
227    #[snafu(transparent)]
228    Storage {
229        source: redb::StorageError,
230        #[snafu(implicit)]
231        location: Location,
232    },
233    #[snafu(display("Database transaction error"))]
234    Transaction {
235        #[snafu(source(from(redb::TransactionError, Box::new)))]
236        source: Box<redb::TransactionError>,
237        #[snafu(implicit)]
238        location: Location,
239    },
240    #[snafu(display("Database commit error"))]
241    Commit {
242        source: redb::CommitError,
243        #[snafu(implicit)]
244        location: Location,
245    },
246    #[snafu(transparent)]
247    StoredDecode {
248        source: bincode::error::DecodeError,
249        #[snafu(implicit)]
250        location: Location,
251    },
252    #[snafu(display("Database version {db_ver} is newer than supported version {code_ver}"))]
253    DbVersionTooHigh {
254        db_ver: u64,
255        code_ver: u64,
256        #[snafu(implicit)]
257        location: Location,
258    },
259    #[snafu(display("Pending migration stash is missing required table `{table}`"))]
260    MissingMigrationStashTable {
261        table: String,
262        #[snafu(implicit)]
263        location: Location,
264    },
265    #[snafu(display("Database task failed"))]
266    Join {
267        source: JoinError,
268        #[snafu(implicit)]
269        location: Location,
270    },
271    #[snafu(transparent)]
272    DbTxLogic {
273        source: BoxedError,
274        #[snafu(implicit)]
275        location: Location,
276    },
277    #[snafu(visibility(pub))]
278    #[snafu(display("Provided Id does not match one used previously"))]
279    DbIdMismatch {
280        #[snafu(implicit)]
281        location: Location,
282    },
283    #[snafu(display("Database file-format upgrade failed"))]
284    Upgrade {
285        source: redb::UpgradeError,
286        #[snafu(implicit)]
287        location: Location,
288    },
289    #[snafu(display("Integer overflow"))]
290    Overflow,
291    #[snafu(display(
292        "Reception-order key ({received_at:?}, {reception_order}) already exists in table {table}"
293    ))]
294    ReceptionOrderCollision {
295        table: String,
296        received_at: Timestamp,
297        reception_order: u64,
298        #[snafu(implicit)]
299        location: Location,
300    },
301    #[snafu(display("Social post {event_id} already has a reception-order key"))]
302    SocialPostReceiptAlreadyIndexed {
303        event_id: ShortEventId,
304        #[snafu(implicit)]
305        location: Location,
306    },
307    #[snafu(display(
308        "Reception-order key for social post {event_id} references {actual_event_id:?}"
309    ))]
310    SocialPostReceiptMismatch {
311        event_id: ShortEventId,
312        actual_event_id: Option<ShortEventId>,
313        #[snafu(implicit)]
314        location: Location,
315    },
316    #[snafu(display("Deletion attribution references missing event {event_id}"))]
317    MissingDeletionAttribution {
318        event_id: ShortEventId,
319        #[snafu(implicit)]
320        location: Location,
321    },
322    #[snafu(display("Extension access to built-in table `{name}` is forbidden"))]
323    ReservedExtensionTable { name: String },
324    #[snafu(display(
325        "SocialPost materialization cursor {position} exceeds durable position {durable_next}"
326    ))]
327    SocialPostMaterializationCursorOutOfRange { position: u64, durable_next: u64 },
328    #[snafu(display(
329        "SocialPost materialization scan limit {requested} exceeds maximum {maximum}"
330    ))]
331    SocialPostMaterializationScanLimitTooHigh { requested: usize, maximum: usize },
332    #[snafu(display(
333        "SocialPost materialization feed has a gap: expected {expected}, found {actual}"
334    ))]
335    SocialPostMaterializationLogGap { expected: u64, actual: u64 },
336    #[snafu(display("SocialPost materialization {event_id} references a missing event"))]
337    MissingSocialPostMaterializationEvent { event_id: ShortEventId },
338    #[snafu(display("SocialPost materialization {event_id} references event kind {kind} instead"))]
339    InvalidSocialPostMaterializationKind {
340        event_id: ShortEventId,
341        kind: EventKind,
342    },
343    #[snafu(display("SocialPost materialization {event_id} unexpectedly became Missing"))]
344    MissingSocialPostMaterializationState { event_id: ShortEventId },
345    #[snafu(display("SocialPost materialization {event_id} unexpectedly became Invalid"))]
346    InvalidSocialPostMaterializationState { event_id: ShortEventId },
347    #[snafu(display("SocialPost materialization {event_id} has no current content"))]
348    MissingSocialPostMaterializationContent { event_id: ShortEventId },
349    #[snafu(display("SocialPost materialization {event_id} has invalid processed content"))]
350    InvalidSocialPostMaterializationContent {
351        event_id: ShortEventId,
352        source: BoxedError,
353    },
354    #[snafu(display("Stored vote singleton {event_id} has no inline vote projection"))]
355    MissingVoteSingletonProjection {
356        event_id: ShortEventId,
357        #[snafu(implicit)]
358        location: Location,
359    },
360    #[snafu(display(
361        "Stored vote singleton {event_id} has a cached target outside its shortened key"
362    ))]
363    InvalidVoteSingletonProjection {
364        event_id: ShortEventId,
365        #[snafu(implicit)]
366        location: Location,
367    },
368    #[snafu(display(
369        "Shortened identity prefix {prefix} maps to both {existing_id} and {incoming_id}"
370    ))]
371    IdentityPrefixCollision {
372        prefix: ShortRostraId,
373        existing_id: RostraId,
374        incoming_id: RostraId,
375        #[snafu(implicit)]
376        location: Location,
377    },
378}
379pub type DbResult<T> = std::result::Result<T, DbError>;
380
381/// The authoritative event store and projection boundary for one local
382/// identity.
383///
384/// Built-in reducers and projection tables are not part of the external API:
385///
386/// ```compile_fail
387/// let _ = rostra_client_db::Database::process_event_content_inserted_tx;
388/// ```
389///
390/// ```compile_fail
391/// let _ = rostra_client_db::events_singletons_new::TABLE;
392/// ```
393///
394/// ```compile_fail
395/// let _ = rostra_client_db::Database::write_with::<()>;
396/// ```
397///
398/// ```compile_fail
399/// let _ = rostra_client_db::Database::write_with_inner::<()>;
400/// ```
401///
402/// ```compile_fail
403/// let _ = rostra_client_db::Database::read_with::<()>;
404/// ```
405///
406/// ```compile_fail
407/// let _: Option<rostra_client_db::WriteTransactionCtx> = None;
408/// ```
409///
410/// ```compile_fail
411/// rostra_client_db::def_table!(forbidden: u64 => u64);
412/// ```
413#[derive(Debug)]
414pub struct Database {
415    inner: redb_bincode::Database,
416    self_id: RostraId,
417    iroh_secret: iroh::SecretKey,
418
419    /// Timestamp when this database was first created.
420    ///
421    /// Used with the current follow epoch as a notification cutoff: posts older
422    /// than both are historical syncs and should not appear as newly received.
423    db_init_time: Timestamp,
424
425    /// Serializes writes through their post-commit current-state publication.
426    ///
427    /// Acquiring it before transaction creation keeps redb writer order and
428    /// publication order identical. Commit hooks run while it is held and must
429    /// not synchronously re-enter `write_with`. A hook panic propagates after
430    /// commit and poisons this mutex; the next write recovers the guard because
431    /// the durable transaction is still valid.
432    write_and_publish_lock: std::sync::Mutex<()>,
433
434    self_followees_updated: watch::Sender<Arc<HashMap<RostraId, IdsFolloweesRecord>>>,
435    self_followers_updated: watch::Sender<Arc<HashMap<RostraId, IdsFollowersRecord>>>,
436    self_wot_updated: watch::Sender<Arc<WotData>>,
437    self_head_updated: watch::Sender<Option<ShortEventId>>,
438    new_content_tx: broadcast::Sender<VerifiedEventContent>,
439    new_posts_tx: broadcast::Sender<(VerifiedEventContent, content_kind::SocialPost)>,
440    new_shoutbox_tx: broadcast::Sender<(VerifiedEventContent, content_kind::Shoutbox)>,
441    new_heads_tx: broadcast::Sender<(RostraId, ShortEventId)>,
442    ids_with_missing_events_tx: dedup_chan::Sender<RostraId>,
443    news_score_updates_tx: dedup_chan::Sender<ExternalEventId>,
444
445    /// Notification for when new content is added to `events_content_missing`.
446    ///
447    /// The `MissingEventContentFetcher` task waits on this to wake up
448    /// immediately when new missing content arrives, instead of polling.
449    content_missing_notify: Arc<Notify>,
450}
451
452impl Database {
453    const MAX_CONTENT_LEN: u32 = 10_000_000u32;
454
455    pub async fn mk_db_path(
456        data_dir: &Path,
457        self_id: RostraId,
458    ) -> std::result::Result<PathBuf, io::Error> {
459        tokio::fs::create_dir_all(&data_dir).await?;
460
461        let legacy_path_unprefixed_z32 =
462            data_dir.join(format!("{}.redb", self_id.to_unprefixed_z32_string()));
463        if legacy_path_unprefixed_z32.exists() {
464            return Ok(legacy_path_unprefixed_z32);
465        }
466        let legacy_path_bech32 = data_dir.join(format!("{}.redb", self_id.to_bech32_string()));
467        if legacy_path_bech32.exists() {
468            return Ok(legacy_path_bech32);
469        }
470        Ok(data_dir.join(format!("{self_id}.redb")))
471    }
472
473    pub async fn new_in_memory(self_id: RostraId) -> DbResult<Database> {
474        debug!(target: LOG_TARGET, id = %self_id, "Opening in-memory database");
475        let inner = redb::Database::builder()
476            .create_with_file_format_v3(true)
477            .create_with_backend(redb::backends::InMemoryBackend::new())
478            .context(DatabaseSnafu)?;
479        Self::open_inner(inner, self_id).await
480    }
481
482    /// Open an identity database, creating it when absent.
483    ///
484    /// Opening storage older than schema version 25 performs the total rebuild
485    /// before returning. This can hold a long atomic write transaction and
486    /// requires deployment-specific RAM and temporary disk headroom. See
487    /// `specs/ARCH-client-database.md#total-rebuild`.
488    pub async fn open(path: impl Into<PathBuf>, self_id: RostraId) -> DbResult<Database> {
489        let path = path.into();
490        debug!(target: LOG_TARGET, id = %self_id, path = %path.display(), "Opening database");
491
492        let mut inner = tokio::task::spawn_blocking(move || {
493            redb::Database::builder()
494                .create_with_file_format_v3(true)
495                .create(path)
496        })
497        .await
498        .context(JoinSnafu)?
499        .context(DatabaseSnafu)?;
500
501        inner.upgrade().context(UpgradeSnafu)?;
502
503        Self::open_inner(inner, self_id).await
504    }
505
506    #[instrument(skip_all)]
507    async fn open_inner(inner: redb::Database, self_id: RostraId) -> DbResult<Database> {
508        let inner = redb_bincode::Database::from(inner);
509
510        // Run migrations (may stash tables for total migration reprocessing)
511        Self::write_with_inner(&inner, |tx| {
512            // Opening table definitions is raw and does not decode rows. Check
513            // the schema version before reading typed identity bytes so a newer
514            // database always fails with DbVersionTooHigh.
515            Self::init_tables_tx(tx)?;
516            Self::handle_db_ver_migrations(tx)?;
517            Self::verify_self_tx(self_id, &mut tx.open_table(&ids_self::TABLE)?)?;
518            Ok(())
519        })
520        .await?;
521
522        // Check if there's a pending migration stash (either from this run or a
523        // previous interrupted run). Using stash existence as the marker ensures
524        // we retry if reprocessing fails/panics.
525        let needs_reprocessing =
526            Self::write_with_inner(&inner, Self::has_pending_migration_stash).await?;
527
528        let (self_head, iroh_secret, self_followees, self_followers, self_wot, db_init_time) =
529            Self::read_with_inner(&inner, |tx| {
530                let ids_followees_table = tx.open_table(&ids_followees::TABLE)?;
531                let self_followees = Self::read_followees_tx(self_id, &ids_followees_table)?;
532                let self_wot =
533                    Self::compute_wot_tx(self_id, &self_followees, &ids_followees_table)?;
534                let db_init_time = tx
535                    .open_table(&db_init_time::TABLE)?
536                    .get(&())?
537                    .map(|g| g.value())
538                    .unwrap_or(Timestamp::ZERO);
539                Ok((
540                    Self::read_head_tx(self_id, &tx.open_table(&events_heads::TABLE)?)?,
541                    Self::read_iroh_secret_tx(&tx.open_table(&ids_self::TABLE)?)?,
542                    self_followees,
543                    Self::read_followers_tx(self_id, &tx.open_table(&ids_followers::TABLE)?)?,
544                    self_wot,
545                    db_init_time,
546                ))
547            })
548            .await?;
549
550        let (self_followees_updated, _) = watch::channel(Arc::new(self_followees));
551        let (self_followers_updated, _) = watch::channel(Arc::new(self_followers));
552        let (self_wot_updated, _) = watch::channel(Arc::new(self_wot));
553        let (self_head_updated, _) = watch::channel(self_head);
554        let (new_content_tx, _) = broadcast::channel(100);
555        let (new_posts_tx, _) = broadcast::channel(100);
556        let (new_shoutbox_tx, _) = broadcast::channel(100);
557        let (new_heads_tx, _) = broadcast::channel(100);
558
559        let db = Self {
560            inner,
561            self_id,
562            iroh_secret,
563            db_init_time,
564            write_and_publish_lock: std::sync::Mutex::new(()),
565            self_followees_updated,
566            self_followers_updated,
567            self_wot_updated,
568            self_head_updated,
569            new_content_tx,
570            new_posts_tx,
571            new_shoutbox_tx,
572            new_heads_tx,
573            ids_with_missing_events_tx: dedup_chan::Sender::new(),
574            news_score_updates_tx: dedup_chan::Sender::new(),
575            content_missing_notify: Arc::new(Notify::new()),
576        };
577
578        // If total migration stashed events, reprocess them now using the real
579        // Database. The stash existence check ensures we retry if this
580        // fails/panics.
581        if needs_reprocessing {
582            db.write_with(|tx| db.reprocess_migration_stash(tx)).await?;
583            db.refresh_current_state_after_replay().await?;
584        }
585
586        Ok(db)
587    }
588
589    /// Publish one coherent current-state snapshot after silent total replay.
590    async fn refresh_current_state_after_replay(&self) -> DbResult<()> {
591        let (self_head, self_followees, self_followers, self_wot) = self
592            .read_with(|tx| {
593                let followees = tx.open_table(&ids_followees::TABLE)?;
594                let self_followees = Self::read_followees_tx(self.self_id, &followees)?;
595                let self_wot = Self::compute_wot_tx(self.self_id, &self_followees, &followees)?;
596                Ok((
597                    Self::read_head_tx(self.self_id, &tx.open_table(&events_heads::TABLE)?)?,
598                    self_followees,
599                    Self::read_followers_tx(self.self_id, &tx.open_table(&ids_followers::TABLE)?)?,
600                    self_wot,
601                ))
602            })
603            .await?;
604
605        self.self_head_updated.send_replace(self_head);
606        self.self_followees_updated
607            .send_replace(Arc::new(self_followees));
608        self.self_followers_updated
609            .send_replace(Arc::new(self_followers));
610        self.self_wot_updated.send_replace(Arc::new(self_wot));
611        Ok(())
612    }
613
614    pub async fn compact(&mut self) -> Result<bool, redb::CompactionError> {
615        tokio::task::block_in_place(|| self.inner.as_raw_mut().compact())
616    }
617
618    pub async fn dump_table(&self, name: &str) -> TableDumpResult<()> {
619        self.read_with(|tx| {
620            match name {
621                "events" => Self::dump_table_dbtx(tx, &tables::events::TABLE)?,
622                "content_store" => Self::dump_table_dbtx(tx, &tables::content_store::TABLE)?,
623                "events_content_state" => {
624                    Self::dump_table_dbtx(tx, &tables::events_content_state::TABLE)?
625                }
626                "events_content_missing" => {
627                    Self::dump_table_dbtx(tx, &tables::events_content_missing::TABLE)?
628                }
629                "social_posts" => Self::dump_table_dbtx(tx, &tables::social_posts::TABLE)?,
630                "social_posts_replies" => {
631                    Self::dump_table_dbtx(tx, &tables::social_posts_replies::TABLE)?
632                }
633                "social_posts_reactions" => {
634                    Self::dump_table_dbtx(tx, &tables::social_posts_reactions::TABLE)?
635                }
636                "social_vote_sums" => Self::dump_table_dbtx(tx, &tables::social_vote_sums::TABLE)?,
637                "social_news_rank_by_post_id" => {
638                    Self::dump_table_dbtx(tx, &tables::social_news_rank_by_post_id::TABLE)?
639                }
640                "social_news_rank_by_score" => {
641                    Self::dump_table_dbtx(tx, &tables::social_news_rank_by_score::TABLE)?
642                }
643                "social_news_rank_by_time" => {
644                    Self::dump_table_dbtx(tx, &tables::social_news_rank_by_time::TABLE)?
645                }
646                _ => {
647                    return Ok(Err(UnknownTableSnafu {
648                        name: name.to_string(),
649                    }
650                    .build()));
651                }
652            }
653            Ok(Ok(()))
654        })
655        .await
656        .expect("Database panic")
657    }
658
659    /// Subscribe to owned snapshots of the retained current self-followee
660    /// projection.
661    pub fn self_followees_subscribe(
662        &self,
663    ) -> CurrentState<Arc<HashMap<RostraId, IdsFolloweesRecord>>> {
664        CurrentState::new(self.self_followees_updated.subscribe())
665    }
666
667    /// Subscribe to owned snapshots of the retained current self-follower
668    /// projection.
669    pub fn self_followers_subscribe(
670        &self,
671    ) -> CurrentState<Arc<HashMap<RostraId, IdsFollowersRecord>>> {
672        CurrentState::new(self.self_followers_updated.subscribe())
673    }
674
675    /// Subscribe to owned snapshots of the retained current Web-of-Trust
676    /// projection.
677    pub fn self_wot_subscribe(&self) -> CurrentState<Arc<WotData>> {
678        CurrentState::new(self.self_wot_updated.subscribe())
679    }
680
681    /// Subscribe to owned snapshots of the retained deterministic self-head
682    /// representative.
683    ///
684    /// The value is the minimum current `ShortEventId`. It is a stable default,
685    /// not a claim that the current head set is a singleton. Every committed
686    /// self-event insertion publishes, even when the representative is
687    /// unchanged.
688    pub fn self_head_subscribe(&self) -> CurrentState<Option<ShortEventId>> {
689        CurrentState::new(self.self_head_updated.subscribe())
690    }
691
692    pub fn new_content_subscribe(&self) -> broadcast::Receiver<VerifiedEventContent> {
693        self.new_content_tx.subscribe()
694    }
695    pub fn new_posts_subscribe(
696        &self,
697    ) -> broadcast::Receiver<(VerifiedEventContent, content_kind::SocialPost)> {
698        self.new_posts_tx.subscribe()
699    }
700
701    pub fn new_shoutbox_subscribe(
702        &self,
703    ) -> broadcast::Receiver<(VerifiedEventContent, content_kind::Shoutbox)> {
704        self.new_shoutbox_tx.subscribe()
705    }
706
707    /// Subscribe to incremental exact new-head signals.
708    ///
709    /// This lossy channel carries the event that became a head. Consumers that
710    /// lag must recover from the durable complete head set.
711    pub fn new_heads_subscribe(&self) -> broadcast::Receiver<(RostraId, ShortEventId)> {
712        self.new_heads_tx.subscribe()
713    }
714
715    pub fn ids_with_missing_events_subscribe(
716        &self,
717        capacity: usize,
718    ) -> dedup_chan::Receiver<RostraId> {
719        self.ids_with_missing_events_tx.subscribe(capacity)
720    }
721
722    pub fn news_score_updates_subscribe(
723        &self,
724        capacity: usize,
725    ) -> dedup_chan::Receiver<ExternalEventId> {
726        self.news_score_updates_tx.subscribe(capacity)
727    }
728
729    /// Get a handle to the content-missing notification.
730    ///
731    /// The `MissingEventContentFetcher` calls `notified()` on this to wake up
732    /// when new missing content is inserted into the database.
733    pub fn content_missing_notify(&self) -> Arc<Notify> {
734        self.content_missing_notify.clone()
735    }
736
737    pub async fn has_event(&self, event_id: impl Into<ShortEventId>) -> bool {
738        let event_id = event_id.into();
739        self.read_with(|tx| {
740            let events_table = tx.open_table(&events::TABLE).expect("Storage error");
741            Database::has_event_tx(event_id, &events_table)
742        })
743        .await
744        .expect("Database panic")
745    }
746
747    pub async fn get_missing_events_for_id(&self, id: RostraId) -> Vec<ShortEventId> {
748        self.read_with(|tx| {
749            let events_missing_tbl = tx.open_table(&events_missing::TABLE)?;
750            Ok(
751                Database::get_missing_events_for_id_tx(id, &events_missing_tbl)?
752                    .into_iter()
753                    .collect(),
754            )
755        })
756        .await
757        .expect("Database panic")
758    }
759
760    /// Lists unique authors that still have missing event envelopes.
761    ///
762    /// The page starts strictly after `after`, is ordered by author ID, and
763    /// contains at most `limit` authors. A zero limit returns an empty page.
764    ///
765    /// Each call uses an independent read transaction. Callers that reconcile
766    /// multiple pages should capture [`Self::get_last_id_with_missing_events`]
767    /// first and stop at that high-water author.
768    ///
769    /// This compatibility query panics if the database read fails.
770    pub async fn get_ids_with_missing_events(
771        &self,
772        after: Option<RostraId>,
773        limit: usize,
774    ) -> Vec<RostraId> {
775        self.read_with(|tx| {
776            let events_missing_tbl = tx.open_table(&events_missing::TABLE)?;
777            Database::get_ids_with_missing_events_tx(after, limit, &events_missing_tbl)
778        })
779        .await
780        .expect("Database panic")
781    }
782
783    /// Returns the greatest author ID that currently has missing event
784    /// envelopes.
785    ///
786    /// This compatibility query panics if the database read fails.
787    pub async fn get_last_id_with_missing_events(&self) -> Option<RostraId> {
788        self.read_with(|tx| {
789            let events_missing_tbl = tx.open_table(&events_missing::TABLE)?;
790            Database::get_last_id_with_missing_events_tx(&events_missing_tbl)
791        })
792        .await
793        .expect("Database panic")
794    }
795
796    pub async fn get_heads_events_for_id(&self, id: RostraId) -> Vec<ShortEventId> {
797        self.read_with(|tx| {
798            let events_heads_tbl = tx.open_table(&events_heads::TABLE)?;
799            Ok(Database::get_heads_events_tx(id, &events_heads_tbl)?
800                .into_iter()
801                .collect())
802        })
803        .await
804        .expect("Database panic")
805    }
806
807    /// Lists a bounded ordered page of unique authors with current durable
808    /// heads.
809    ///
810    /// The page starts strictly after `after`. A zero limit returns an empty
811    /// page. This compatibility query panics if the database read fails.
812    pub async fn get_ids_with_heads(&self, after: Option<RostraId>, limit: usize) -> Vec<RostraId> {
813        self.read_with(|tx| {
814            let events_heads_tbl = tx.open_table(&events_heads::TABLE)?;
815            Database::get_ids_with_heads_tx(after, limit, &events_heads_tbl)
816        })
817        .await
818        .expect("Database panic")
819    }
820
821    pub async fn count_missing_events_for_id(&self, id: RostraId) -> usize {
822        self.read_with(|tx| {
823            let events_missing_tbl = tx.open_table(&events_missing::TABLE)?;
824            Database::count_missing_events_for_id_tx(id, &events_missing_tbl)
825        })
826        .await
827        .expect("Database panic")
828    }
829
830    pub async fn count_heads_events_for_id(&self, id: RostraId) -> usize {
831        self.read_with(|tx| {
832            let events_heads_tbl = tx.open_table(&events_heads::TABLE)?;
833            Database::count_heads_events_tx(id, &events_heads_tbl)
834        })
835        .await
836        .expect("Database panic")
837    }
838
839    pub async fn get_data_usage(&self, id: RostraId) -> IdsDataUsageRecord {
840        self.read_with(|tx| {
841            let ids_data_usage_tbl = tx.open_table(&ids_data_usage::TABLE)?;
842            Database::get_data_usage_tx(id, &ids_data_usage_tbl)
843        })
844        .await
845        .expect("Database panic")
846    }
847
848    pub async fn get_self_followees(&self) -> Vec<(RostraId, PersonasTagsSelector)> {
849        self.get_followees(self.self_id).await
850    }
851
852    pub async fn get_followees(&self, id: RostraId) -> Vec<(RostraId, PersonasTagsSelector)> {
853        self.read_with(|tx| {
854            let ids_followees_table = tx.open_table(&ids_followees::TABLE)?;
855            Ok(Database::read_followees_tx(id, &ids_followees_table)?
856                .into_iter()
857                .map(|(id, record)| (id, record.effective_tags_selector()))
858                .collect())
859        })
860        .await
861        .expect("Database panic")
862    }
863
864    pub async fn get_followees_extended(
865        &self,
866        id: RostraId,
867    ) -> (HashMap<RostraId, PersonasTagsSelector>, HashSet<RostraId>) {
868        self.read_with(|tx| {
869            let ids_followees_table = tx.open_table(&ids_followees::TABLE)?;
870            let followees: HashMap<RostraId, PersonasTagsSelector> =
871                Database::read_followees_tx_iter(id, &ids_followees_table)?
872                    .map_ok(|(id, record)| (id, record.effective_tags_selector()))
873                    .collect::<Result<_, _>>()?;
874
875            let mut extended = HashSet::new();
876
877            for followee in followees.keys() {
878                for extended_followee in
879                    Database::read_followees_tx_iter(*followee, &ids_followees_table)?
880                        .map_ok(|(id, _record)| id)
881                {
882                    let extended_followee = extended_followee?;
883                    if !followees.contains_key(&extended_followee) {
884                        extended.insert(extended_followee);
885                    }
886                }
887            }
888            Ok((followees, extended))
889        })
890        .await
891        .expect("Database panic")
892    }
893
894    pub async fn get_self_followers(&self) -> Vec<RostraId> {
895        self.get_followers(self.self_id).await
896    }
897
898    pub async fn get_followers(&self, id: RostraId) -> Vec<RostraId> {
899        self.read_with(|tx| {
900            let ids_followers_table = tx.open_table(&ids_followers::TABLE)?;
901            Ok(Database::read_followers_tx(id, &ids_followers_table)?
902                .into_keys()
903                .collect())
904        })
905        .await
906        .expect("Database panic")
907    }
908
909    pub async fn get_event(
910        &self,
911        event_id: impl Into<ShortEventId>,
912    ) -> Option<crate::event::EventRecord> {
913        let event_id = event_id.into();
914        self.read_with(|tx| {
915            let events_table = tx.open_table(&events::TABLE)?;
916            Database::get_event_tx(event_id, &events_table)
917        })
918        .await
919        .expect("Database panic")
920    }
921
922    pub async fn get_event_content(
923        &self,
924        event_id: impl Into<ShortEventId>,
925    ) -> Option<EventContentRaw> {
926        let event_id = event_id.into();
927        self.read_with(|tx| {
928            let events_table = tx.open_table(&events::TABLE)?;
929            let events_content_state_table = tx.open_table(&events_content_state::TABLE)?;
930            let content_store_table = tx.open_table(&content_store::TABLE)?;
931
932            // Get the event to find its content hash
933            let Some(event_record) = Database::get_event_tx(event_id, &events_table)? else {
934                return Ok(None);
935            };
936            let content_hash = event_record.content_hash();
937
938            Ok(Database::get_event_content_full_tx(
939                event_id,
940                content_hash,
941                &events_content_state_table,
942                &content_store_table,
943            )?
944            .and_then(|result| result.content().cloned()))
945        })
946        .await
947        .expect("Database panic")
948    }
949
950    /// Return the lifecycle state for event content that is not currently
951    /// stored.
952    ///
953    /// For an existing event, `None` normally means the content was processed.
954    /// A terminal state such as `Deleted`, `Pruned`, or `Invalid` means callers
955    /// must not wait for payload availability.
956    pub async fn get_event_content_state(
957        &self,
958        event_id: impl Into<ShortEventId>,
959    ) -> Option<EventContentState> {
960        let event_id = event_id.into();
961        self.read_with(|tx| {
962            Self::get_event_content_state_tx(
963                event_id,
964                &tx.open_table(&events_content_state::TABLE)?,
965            )
966        })
967        .await
968        .expect("Database panic")
969    }
970
971    /// Return the minimum current self-head as a deterministic representative.
972    ///
973    /// Use [`Self::get_heads_self`] when the complete set is required.
974    pub async fn get_self_current_head(&self) -> Option<ShortEventId> {
975        self.read_with(|tx| {
976            let events_heads_table = tx.open_table(&events_heads::TABLE)?;
977
978            Database::read_head_tx(self.self_id, &events_heads_table)
979        })
980        .await
981        .expect("Storage error")
982    }
983
984    pub async fn get_self_random_eventid(&self) -> Option<ShortEventId> {
985        self.read_with(|tx| {
986            let events_self_table = tx.open_table(&events_self::TABLE)?;
987
988            Database::get_random_self_event(&events_self_table)
989        })
990        .await
991        .expect("Storage error")
992    }
993
994    /// Fallibly process a verified event envelope.
995    ///
996    /// # Errors
997    ///
998    /// Returns storage and transaction errors, including
999    /// [`DbError::IdentityPrefixCollision`]. An error leaves the ingestion
1000    /// transaction uncommitted.
1001    pub async fn try_process_event(
1002        &self,
1003        event: &VerifiedEvent,
1004    ) -> DbResult<(InsertEventOutcome, ProcessEventState)> {
1005        let now = Timestamp::now();
1006        self.write_with(|tx| self.process_event_tx(event, now, tx))
1007            .await
1008    }
1009
1010    /// Process a verified event envelope, panicking on storage failure.
1011    ///
1012    /// # Panics
1013    ///
1014    /// Panics after rolling back if storage fails or the event author's
1015    /// shortened identity prefix is already mapped to a different full
1016    /// identity.
1017    pub async fn process_event(
1018        &self,
1019        event: &VerifiedEvent,
1020    ) -> (InsertEventOutcome, ProcessEventState) {
1021        self.try_process_event(event).await.expect("Storage error")
1022    }
1023
1024    /// Fallibly process a verified envelope and its verified content
1025    /// atomically.
1026    ///
1027    /// This operation is idempotent whether or not the envelope or content was
1028    /// processed previously.
1029    ///
1030    /// # Errors
1031    ///
1032    /// Returns storage and transaction errors, including
1033    /// [`DbError::IdentityPrefixCollision`]. An error rolls back the envelope,
1034    /// content, lifecycle, and projection changes together.
1035    pub async fn try_process_event_with_content(
1036        &self,
1037        content: &VerifiedEventContent,
1038    ) -> DbResult<(InsertEventOutcome, ProcessEventState)> {
1039        let now = Timestamp::now();
1040        self.write_with(|tx| {
1041            let res = self.process_event_tx(&content.event, now, tx)?;
1042            self.process_event_content_tx(content, now, tx)?;
1043            Ok(res)
1044        })
1045        .await
1046    }
1047
1048    /// Process a verified envelope and its verified content, panicking on
1049    /// storage failure.
1050    ///
1051    /// This operation is idempotent whether or not the envelope or content was
1052    /// processed previously.
1053    ///
1054    /// # Panics
1055    ///
1056    /// Panics after rolling back if storage fails or a stored invariant is
1057    /// violated, including a vote winner with a missing or invalid inline
1058    /// projection or a shortened identity prefix mapped to a different full
1059    /// identity.
1060    pub async fn process_event_with_content(
1061        &self,
1062        content: &VerifiedEventContent,
1063    ) -> (InsertEventOutcome, ProcessEventState) {
1064        self.try_process_event_with_content(content)
1065            .await
1066            .expect("Storage error")
1067    }
1068
1069    /// Fallibly process verified content and its carried envelope atomically.
1070    ///
1071    /// This is the safe public content-ingestion boundary. If the envelope is
1072    /// absent, it is inserted before content processing in the same
1073    /// transaction. If the envelope is already present, normal
1074    /// lifecycle-state checks make the operation idempotent and prevent
1075    /// repeated accounting or projections.
1076    ///
1077    /// # Errors
1078    ///
1079    /// Returns errors under the same conditions as
1080    /// [`Database::try_process_event_with_content`].
1081    pub async fn try_process_event_content(
1082        &self,
1083        event_content: &VerifiedEventContent,
1084    ) -> DbResult<()> {
1085        self.try_process_event_with_content(event_content)
1086            .await
1087            .map(|_| ())
1088    }
1089
1090    /// Process verified content and its carried envelope, panicking on storage
1091    /// failure.
1092    ///
1093    /// # Panics
1094    ///
1095    /// Panics under the same conditions as
1096    /// [`Database::process_event_with_content`].
1097    pub async fn process_event_content(&self, event_content: &VerifiedEventContent) {
1098        self.try_process_event_content(event_content)
1099            .await
1100            .expect("Storage error");
1101    }
1102
1103    /// Process event content.
1104    ///
1105    /// Ordinary processing requires Missing state, applies kind-specific side
1106    /// effects, stores the content, removes fetch scheduling, and transitions
1107    /// to Processed. RC was already incremented at event insertion.
1108    ///
1109    /// A verified, below-limit Deleted social-post edit is the sole
1110    /// terminal-state exception. It may add only immutable forward and reverse
1111    /// replacement rows; it does not store the supplied bytes or change
1112    /// lifecycle bookkeeping or other projections.
1113    ///
1114    /// The `now` parameter should be `Timestamp::now()` for normal operation,
1115    /// but can be set to a specific value for testing or migration.
1116    pub(crate) fn process_event_content_tx(
1117        &self,
1118        event_content: &VerifiedEventContent,
1119        now: Timestamp,
1120        tx: &WriteTransactionCtx,
1121    ) -> DbResult<()> {
1122        {
1123            let events_table = tx.open_table(&events::TABLE)?;
1124            let has_event = Database::has_event_tx(event_content.event.event_id, &events_table)?;
1125            if !has_event {
1126                // Event doesn't exist - this shouldn't happen in normal operation.
1127                // It means process_event_content_tx was called without first inserting the
1128                // event.
1129                debug_assert!(false, "Processing content for non-existent event");
1130                error!(
1131                    target: LOG_TARGET,
1132                    event_id = %event_content.event.event_id,
1133                    "Processing content for non-existent event - possible bug"
1134                );
1135                return Ok(());
1136            }
1137        }
1138
1139        // Check if content should be processed (not deleted/pruned, is Missing)
1140        let (can_insert, is_deleted) = if u32::from(event_content.event.event.content_len)
1141            < Self::MAX_CONTENT_LEN
1142        {
1143            let events_content_state_table = tx.open_table(&events_content_state::TABLE)?;
1144            let state = events_content_state_table
1145                .get(&event_content.event_id().to_short())?
1146                .map(|state| state.value());
1147            (
1148                Database::can_insert_event_content_tx(event_content, &events_content_state_table)?,
1149                matches!(state, Some(EventContentState::Deleted { .. })),
1150            )
1151        } else {
1152            (false, false)
1153        };
1154
1155        if can_insert {
1156            // Remove eligible content from the missing list.
1157            {
1158                let event_short_id = event_content.event_id().to_short();
1159                let events_content_state_table = tx.open_table(&events_content_state::TABLE)?;
1160                let mut events_content_missing_table =
1161                    tx.open_table(&tables::events_content_missing::TABLE)?;
1162                let state = events_content_state_table
1163                    .get(&event_short_id)?
1164                    .map(|g| g.value());
1165                if let Some(EventContentState::Missing {
1166                    next_fetch_attempt, ..
1167                }) = state
1168                {
1169                    events_content_missing_table.remove(&(next_fetch_attempt, event_short_id))?;
1170                }
1171            }
1172
1173            if let Some(content) = event_content.content.as_ref() {
1174                let content_hash = event_content.content_hash();
1175                let event_short_id = event_content.event_id().to_short();
1176
1177                // Process side effects
1178                let is_valid = match self.process_event_content_inserted_tx(event_content, now, tx)
1179                {
1180                    Ok(()) => {
1181                        if tx.commit_hooks_enabled() {
1182                            info!(target: LOG_TARGET,
1183                                kind = %event_content.kind(),
1184                                event_id = %event_short_id,
1185                                author = %event_content.author().to_short(),
1186                                len = %event_content.content_len(),
1187                                "New event content inserted"
1188                            );
1189                        }
1190                        true
1191                    }
1192                    Err(ProcessEventError::Invalid { source, location }) => {
1193                        if tx.commit_hooks_enabled() {
1194                            info!(
1195                                target: LOG_TARGET,
1196                                err = %source.as_ref().fmt_compact(),
1197                                %location,
1198                                "Invalid event content"
1199                            );
1200                        }
1201                        false
1202                    }
1203                    Err(ProcessEventError::Db { source }) => {
1204                        return Err(source);
1205                    }
1206                };
1207
1208                if is_valid {
1209                    // Store content in content_store if not already there
1210                    {
1211                        let mut content_store_table = tx.open_table(&content_store::TABLE)?;
1212                        if content_store_table.get(&content_hash)?.is_none() {
1213                            content_store_table.insert(
1214                                &content_hash,
1215                                &ContentStoreRecord(Cow::Owned(content.clone())),
1216                            )?;
1217                        }
1218                    }
1219
1220                    // Remove the Missing marker now that content is processed
1221                    {
1222                        let mut events_content_state_table =
1223                            tx.open_table(&events_content_state::TABLE)?;
1224                        events_content_state_table.remove(&event_short_id)?;
1225                    }
1226
1227                    // Track payload as processed (missing → current)
1228                    {
1229                        let mut ids_data_usage_table =
1230                            tx.open_table(&tables::ids_data_usage::TABLE)?;
1231                        Database::track_payload_processed_tx(
1232                            event_content.author(),
1233                            event_content.content_len(),
1234                            &mut ids_data_usage_table,
1235                        )?;
1236                    }
1237
1238                    // Notify about new content
1239                    if tx.commit_hooks_enabled() {
1240                        tx.on_commit({
1241                            let new_content_tx = self.new_content_tx.clone();
1242                            let event_content = event_content.clone();
1243                            move || {
1244                                let _ = new_content_tx.send(event_content);
1245                            }
1246                        });
1247                    }
1248                } else {
1249                    // Content failed validation — mark as Invalid, decrement RC,
1250                    // discard content bytes
1251                    {
1252                        let mut events_content_state_table =
1253                            tx.open_table(&events_content_state::TABLE)?;
1254                        events_content_state_table
1255                            .insert(&event_short_id, &EventContentState::Invalid)?;
1256                    }
1257                    {
1258                        let mut content_rc_table = tx.open_table(&tables::content_rc::TABLE)?;
1259                        Database::decrement_content_rc_tx(content_hash, &mut content_rc_table)?;
1260                    }
1261
1262                    // Track payload as invalid (missing → invalid)
1263                    {
1264                        let mut ids_data_usage_table =
1265                            tx.open_table(&tables::ids_data_usage::TABLE)?;
1266                        Database::track_payload_invalid_tx(
1267                            event_content.author(),
1268                            event_content.content_len(),
1269                            &mut ids_data_usage_table,
1270                        )?;
1271                    }
1272                }
1273            }
1274        } else if is_deleted && event_content.content.is_some() {
1275            match Self::process_deleted_social_post_replacement_tx(event_content, tx) {
1276                Ok(_) => {}
1277                Err(ProcessEventError::Invalid { source, location }) => {
1278                    debug!(
1279                        target: LOG_TARGET,
1280                        err = %source.as_ref().fmt_compact(),
1281                        %location,
1282                        "Ignoring malformed Deleted social-post content"
1283                    );
1284                }
1285                Err(ProcessEventError::Db { source }) => return Err(source),
1286            }
1287        }
1288        Ok(())
1289    }
1290
1291    pub async fn wants_content(
1292        &self,
1293        event_id: impl Into<ShortEventId>,
1294        process_state: ProcessEventState,
1295    ) -> bool {
1296        match process_state.wants_content() {
1297            ContentWantState::DoesNotWant => {
1298                return false;
1299            }
1300            ContentWantState::Wants => {
1301                return true;
1302            }
1303            ContentWantState::MaybeWants => {}
1304        }
1305
1306        let event_id = event_id.into();
1307        self.read_with(|tx| {
1308            let events_table = tx.open_table(&events::TABLE)?;
1309            let events_content_state_table = tx.open_table(&events_content_state::TABLE)?;
1310            let content_store_table = tx.open_table(&content_store::TABLE)?;
1311
1312            // Get event to find content_hash
1313            let Some(event) = Database::get_event_tx(event_id, &events_table)? else {
1314                return Ok(false);
1315            };
1316            let content_hash = event.content_hash();
1317
1318            // We want content if we DON'T have it yet
1319            Ok(!Database::has_event_content_tx(
1320                event_id,
1321                content_hash,
1322                &events_content_state_table,
1323                &content_store_table,
1324            )?)
1325        })
1326        .await
1327        .expect("Storage error")
1328    }
1329
1330    pub fn db_init_time(&self) -> Timestamp {
1331        self.db_init_time
1332    }
1333
1334    pub fn iroh_secret(&self) -> iroh::SecretKey {
1335        self.iroh_secret.clone()
1336    }
1337}
1338
1339impl Database {
1340    fn write_with_inner_blocking<T>(
1341        inner: &redb_bincode::Database,
1342        f: impl FnOnce(&'_ WriteTransactionCtx) -> DbResult<T>,
1343    ) -> DbResult<T> {
1344        let mut dbtx = WriteTransactionCtx::from(inner.begin_write().context(TransactionSnafu)?);
1345        let res = f(&mut dbtx)?;
1346
1347        dbtx.commit().context(CommitSnafu)?;
1348
1349        Ok(res)
1350    }
1351
1352    /// Runs a low-level write transaction without a `Database` publication
1353    /// boundary.
1354    ///
1355    /// The synchronous transaction closure must not start another write.
1356    pub(crate) async fn write_with_inner<T>(
1357        inner: &redb_bincode::Database,
1358        f: impl FnOnce(&'_ WriteTransactionCtx) -> DbResult<T>,
1359    ) -> DbResult<T> {
1360        tokio::task::block_in_place(|| Self::write_with_inner_blocking(inner, f))
1361    }
1362
1363    /// Runs a serialized write transaction and its internal post-commit
1364    /// actions.
1365    ///
1366    /// The mutex spans transaction creation, durable commit, and all
1367    /// post-commit actions so their publication order matches writer order.
1368    /// The synchronous transaction closure and post-commit actions must not
1369    /// re-enter database writes.
1370    ///
1371    /// A post-commit action panic propagates to the caller after the
1372    /// transaction has committed. Every registered action is attempted, then
1373    /// the first panic resumes. A later write recovers the poisoned
1374    /// serialization mutex and proceeds.
1375    pub(crate) async fn write_with<T>(
1376        &self,
1377        f: impl FnOnce(&'_ WriteTransactionCtx) -> DbResult<T>,
1378    ) -> DbResult<T> {
1379        tokio::task::block_in_place(|| {
1380            let _write_and_publish_guard = self
1381                .write_and_publish_lock
1382                .lock()
1383                .unwrap_or_else(std::sync::PoisonError::into_inner);
1384            Self::write_with_inner_blocking(&self.inner, f)
1385        })
1386    }
1387
1388    /// Runs a serialized transaction over caller-owned extension tables.
1389    ///
1390    /// The closure cannot access built-in graph, lifecycle, or projection
1391    /// tables. It must not re-enter database writes. This trusted persistence
1392    /// boundary does not include extension data in core replay or convergence;
1393    /// callers own their schema, invariants, and compatibility. Total database
1394    /// migrations preserve extension tables byte-for-byte.
1395    ///
1396    /// Returning `Ok` commits all extension-table mutations atomically.
1397    /// Returning `Err`, panicking, or failing a reserved-name/table
1398    /// operation leaves the transaction uncommitted. Post-commit built-in
1399    /// projection hooks are not available at this boundary.
1400    ///
1401    /// # Errors
1402    ///
1403    /// Returns the closure's error and propagates transaction, table, and
1404    /// commit errors. Opening a reserved table returns
1405    /// [`DbError::ReservedExtensionTable`].
1406    pub async fn extension_write<T>(
1407        &self,
1408        f: impl FnOnce(&ExtensionWriteTransaction<'_>) -> DbResult<T>,
1409    ) -> DbResult<T> {
1410        self.write_with(|tx| f(&ExtensionWriteTransaction::new(tx)))
1411            .await
1412    }
1413
1414    pub(crate) async fn read_with_inner<T>(
1415        inner: &redb_bincode::Database,
1416        f: impl FnOnce(&'_ ReadTransaction) -> DbResult<T>,
1417    ) -> DbResult<T> {
1418        tokio::task::block_in_place(|| {
1419            let mut dbtx = inner.begin_read().context(TransactionSnafu)?;
1420
1421            f(&mut dbtx)
1422        })
1423    }
1424
1425    pub(crate) async fn read_with<T>(
1426        &self,
1427        f: impl FnOnce(&'_ ReadTransaction) -> DbResult<T>,
1428    ) -> DbResult<T> {
1429        Self::read_with_inner(&self.inner, f).await
1430    }
1431
1432    /// Runs a read transaction over trusted, caller-owned extension tables.
1433    ///
1434    /// Callers own extension schemas, invariants, and compatibility; core event
1435    /// replay does not validate or rebuild their contents.
1436    ///
1437    /// # Errors
1438    ///
1439    /// Returns the closure's error and propagates transaction and table errors.
1440    /// Opening a reserved table returns [`DbError::ReservedExtensionTable`].
1441    pub async fn extension_read<T>(
1442        &self,
1443        f: impl FnOnce(&ExtensionReadTransaction<'_>) -> DbResult<T>,
1444    ) -> DbResult<T> {
1445        self.read_with(|tx| f(&ExtensionReadTransaction::new(tx)))
1446            .await
1447    }
1448
1449    pub(crate) fn verify_self_tx(
1450        self_id: RostraId,
1451        ids_self_t: &mut ids_self::Table,
1452    ) -> DbResult<()> {
1453        match Self::read_self_id_tx(ids_self_t)? {
1454            Some(existing_self_id_record) => {
1455                if existing_self_id_record.rostra_id != self_id {
1456                    return DbIdMismatchSnafu.fail();
1457                }
1458            }
1459            _ => {
1460                Self::write_self_id_tx(self_id, ids_self_t)?;
1461            }
1462        };
1463        Ok(())
1464    }
1465
1466    /// Return the minimum current head as a deterministic representative.
1467    ///
1468    /// Use [`Self::get_heads`] when the complete set is required.
1469    pub async fn get_head(&self, id: RostraId) -> Option<ShortEventId> {
1470        self.read_with(|tx| {
1471            let events_heads = tx.open_table(&events_heads::TABLE)?;
1472
1473            Self::read_head_tx(id, &events_heads)
1474        })
1475        .await
1476        .expect("Database panic")
1477    }
1478
1479    /// Return the complete current head set for an identity.
1480    pub async fn get_heads(&self, id: RostraId) -> HashSet<ShortEventId> {
1481        self.read_with(|tx| {
1482            let events_heads = tx.open_table(&events_heads::TABLE)?;
1483
1484            Self::get_heads_tx(id, &events_heads)
1485        })
1486        .await
1487        .expect("Database panic")
1488    }
1489
1490    /// Return the complete current head set for the local identity.
1491    pub async fn get_heads_self(&self) -> HashSet<ShortEventId> {
1492        self.read_with(|tx| {
1493            let events_heads = tx.open_table(&events_heads::TABLE)?;
1494
1495            Self::get_heads_tx(self.self_id, &events_heads)
1496        })
1497        .await
1498        .expect("Database panic")
1499    }
1500
1501    pub async fn get_social_profile(&self, id: RostraId) -> Option<IdSocialProfileRecord> {
1502        self.read_with(|tx| {
1503            let events_heads = tx.open_table(&social_profiles::TABLE)?;
1504
1505            Self::get_social_profile_tx(id, &events_heads)
1506        })
1507        .await
1508        .expect("Database panic")
1509    }
1510
1511    pub async fn get_latest_singleton_event(
1512        &self,
1513        rostra_id: RostraId,
1514        kind: EventKind,
1515        aux_key: EventAuxKey,
1516    ) -> Option<ShortEventId> {
1517        self.read_with(|tx| {
1518            let singletons_table = tx.open_table(&events_singletons_new::TABLE)?;
1519
1520            Ok(singletons_table
1521                .get(&(rostra_id, kind, aux_key))?
1522                .map(|record| record.value().inner.event_id))
1523        })
1524        .await
1525        .expect("Database panic")
1526    }
1527
1528    /// Returns an identity's singleton winners for one event kind in descending
1529    /// `(timestamp, ShortEventId)` order.
1530    pub async fn get_latest_singleton_events(
1531        &self,
1532        rostra_id: RostraId,
1533        kind: EventKind,
1534    ) -> Vec<ShortEventId> {
1535        self.read_with(|tx| {
1536            let singletons = tx.open_table(&events_singletons_new::TABLE)?;
1537            let start = (rostra_id, kind, EventAuxKey::ZERO);
1538            let end = (rostra_id, kind, EventAuxKey::MAX);
1539            let mut events = singletons
1540                .range(start..=end)?
1541                .map(|entry| {
1542                    let (_, value) = entry?;
1543                    let value = value.value();
1544                    Ok((value.ts, value.inner.event_id))
1545                })
1546                .collect::<DbResult<Vec<_>>>()?;
1547            events.sort_unstable_by_key(|event| std::cmp::Reverse(*event));
1548            Ok(events.into_iter().map(|(_, event_id)| event_id).collect())
1549        })
1550        .await
1551        .expect("Database panic")
1552    }
1553
1554    pub async fn get_id_endpoints(
1555        &self,
1556        id: RostraId,
1557    ) -> BTreeMap<(Timestamp, IrohNodeId), IrohNodeRecord> {
1558        self.write_with(|tx| {
1559            let mut table = tx.open_table(&ids_nodes::TABLE)?;
1560
1561            Self::get_id_endpoints_tx(id, &mut table)
1562        })
1563        .await
1564        .expect("Database panic")
1565    }
1566
1567    /// Register an iroh node endpoint for an identity.
1568    ///
1569    /// Useful for test setups where peer node addresses need to be manually
1570    /// registered.
1571    pub async fn insert_id_node(&self, id: RostraId, node_id: IrohNodeId, ts: Timestamp) {
1572        self.write_with(|tx| {
1573            let mut table = tx.open_table(&ids_nodes::TABLE)?;
1574            table.insert(
1575                &(id, node_id),
1576                &IrohNodeRecord {
1577                    announcement_ts: ts,
1578                    stats: Default::default(),
1579                },
1580            )?;
1581            Ok(())
1582        })
1583        .await
1584        .expect("Database panic")
1585    }
1586
1587    /// Get events for an identity, sorted by timestamp (most recent first).
1588    ///
1589    /// Returns a vector of (EventRecord, Timestamp, EventContentState
1590    /// option) limited to the specified count.
1591    pub async fn get_events_for_id(
1592        &self,
1593        id: RostraId,
1594        limit: usize,
1595    ) -> Vec<(event::EventRecord, Timestamp, Option<EventContentState>)> {
1596        self.read_with(|tx| {
1597            let events_table = tx.open_table(&events::TABLE)?;
1598            let events_by_time_table = tx.open_table(&events_by_time::TABLE)?;
1599            let events_content_state_table = tx.open_table(&events_content_state::TABLE)?;
1600
1601            let mut results = Vec::new();
1602
1603            // Iterate events_by_time in reverse (newest first)
1604            for entry in events_by_time_table.range(..)?.rev() {
1605                if limit <= results.len() {
1606                    break;
1607                }
1608
1609                let entry = entry?;
1610                let (ts, event_id) = entry.0.value();
1611
1612                // Get the event record
1613                let Some(event_record) = events_table.get(&event_id)?.map(|g| g.value()) else {
1614                    continue;
1615                };
1616
1617                // Check if this event belongs to the requested identity
1618                if event_record.signed.event.author != id {
1619                    continue;
1620                }
1621
1622                // Get content state if available
1623                let content_state = events_content_state_table
1624                    .get(&event_id)?
1625                    .map(|g| g.value());
1626
1627                results.push((event_record, ts, content_state));
1628            }
1629
1630            Ok(results)
1631        })
1632        .await
1633        .expect("Database panic")
1634    }
1635
1636    /// Get all identities that have authored retained events.
1637    pub async fn get_known_identities(&self) -> Vec<RostraId> {
1638        self.read_with(ids_full::read_all)
1639            .await
1640            .expect("Database panic")
1641    }
1642
1643    /// Resolve a shortened identity that has authored a retained event.
1644    ///
1645    /// Event ingestion rejects identity-prefix collisions, so a returned
1646    /// identity is unambiguous.
1647    pub async fn get_known_identity(&self, short_id: ShortRostraId) -> Option<RostraId> {
1648        self.read_with(|tx| {
1649            Ok(ids_full::get(tx, short_id)?.map(|rest| RostraId::assemble(short_id, rest)))
1650        })
1651        .await
1652        .expect("Database panic")
1653    }
1654}
1655
1656fn get_first_in_range<K, V>(
1657    events_table: &impl ReadableTable<K, V>,
1658    range: impl ops::RangeBounds<K>,
1659) -> Result<Option<K>, DbError>
1660where
1661    K: bincode::Decode<()> + bincode::Encode,
1662    V: bincode::Decode<()> + bincode::Encode,
1663{
1664    Ok(events_table
1665        .range(range)?
1666        .next()
1667        .transpose()?
1668        .map(|(k, _)| k.value()))
1669}
1670
1671fn get_last_in_range<K, V>(
1672    events_table: &impl ReadableTable<K, V>,
1673    range: impl ops::RangeBounds<K>,
1674) -> Result<Option<K>, DbError>
1675where
1676    K: bincode::Decode<()> + bincode::Encode,
1677    V: bincode::Decode<()> + bincode::Encode,
1678{
1679    Ok(events_table
1680        .range(range)?
1681        .next_back()
1682        .transpose()?
1683        .map(|(k, _)| k.value()))
1684}
1685
1686#[derive(Debug, Clone)]
1687pub enum InsertEventOutcome {
1688    /// The event already existed, so graph, lifecycle, and projection state did
1689    /// not change.
1690    ///
1691    /// Identity registration is still validated and may restore an absent
1692    /// shortened/full mapping.
1693    AlreadyPresent,
1694    Inserted {
1695        /// An event already had a child reporting its existence.
1696        ///
1697        /// This also implies that the event can't be a "head event"
1698        /// as we already have a child of it.
1699        was_missing: bool,
1700        /// This event was already marked as deleted by some processed children
1701        /// event.
1702        ///
1703        /// This also implies that the event can't be a "head event"
1704        /// as we already have a child of it.
1705        is_deleted: bool,
1706        /// An existing parent event had its content marked as deleted by this
1707        /// event.
1708        ///
1709        /// Note, if the parent event was marked for deletion, but it was not
1710        /// processed yet, this will not be set, and instead `is_deleted` will
1711        /// be set to true, when the deleted parent is processed.
1712        deleted_parent: Option<ShortEventId>,
1713        /// Parent content to be reverted.
1714        ///
1715        /// If Some - deletion of the `deleted_parent` is cusing revertion of
1716        /// this content, which should be processed.
1717        reverted_parent_content: Option<EventContentRaw>,
1718
1719        /// Ids of parents we don't have yet, so they are now marked
1720        /// as "missing".
1721        missing_parents: Vec<ShortEventId>,
1722    },
1723}
1724
1725impl InsertEventOutcome {
1726    fn validate(self) -> Self {
1727        if let InsertEventOutcome::Inserted {
1728            deleted_parent,
1729            reverted_parent_content,
1730            ..
1731        } = &self
1732        {
1733            if reverted_parent_content.is_some() {
1734                assert!(deleted_parent.is_some());
1735            }
1736        }
1737        self
1738    }
1739}
1740
1741#[derive(Clone, Copy, PartialEq, Eq, Debug)]
1742pub enum ProcessEventState {
1743    New,
1744    Existing,
1745    Pruned,
1746    Deleted,
1747}
1748
1749pub enum ContentWantState {
1750    Wants,
1751    MaybeWants,
1752    DoesNotWant,
1753}
1754
1755impl ProcessEventState {
1756    pub fn wants_content(self) -> ContentWantState {
1757        match self {
1758            ProcessEventState::New => ContentWantState::Wants,
1759            ProcessEventState::Existing => ContentWantState::MaybeWants,
1760            ProcessEventState::Pruned => ContentWantState::DoesNotWant,
1761            ProcessEventState::Deleted => ContentWantState::DoesNotWant,
1762        }
1763    }
1764}
1765#[cfg(test)]
1766mod content_ingestion_tests;
1767#[cfg(test)]
1768mod deleted_replacement_tests;
1769#[cfg(test)]
1770mod follow_epoch_tests;
1771#[cfg(test)]
1772mod identity_collision_tests;
1773#[cfg(test)]
1774mod reception_order_tests;
1775#[cfg(test)]
1776mod social_post_materialization_tests;
1777#[cfg(test)]
1778mod social_post_projection_tests;
1779#[cfg(test)]
1780mod social_post_receipt_tests;
1781#[cfg(test)]
1782mod tests;