Skip to main content

rostra_client_db/
lib.rs

1mod current_state;
2mod event_order;
3mod events_content_missing_ops;
4mod extension;
5mod id_nodes_ops;
6mod ids_full;
7mod migration_ops;
8mod models;
9pub mod news;
10mod paginate;
11mod process_event_content_ops;
12mod process_event_ops;
13mod reception_order_ops;
14mod self_followee;
15pub mod social;
16mod social_post_materialization;
17mod table_ops;
18mod tables;
19mod tx_ops;
20
21use std::borrow::Cow;
22use std::collections::{BTreeMap, HashMap, HashSet};
23use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind};
24use std::path::{Path, PathBuf};
25use std::sync::Arc;
26use std::{io, ops, result};
27
28use event::ContentStoreRecord;
29pub use ids::{IdsFolloweesRecord, IdsFollowersRecord};
30use itertools::Itertools as _;
31use process_event_content_ops::ProcessEventError;
32use redb_bincode::{ReadTransaction, ReadableTable, WriteTransaction};
33use rostra_core::event::{
34    EventAuxKey, EventContentRaw, EventExt as _, EventKind, IrohNodeId, PersonasTagsSelector,
35    VerifiedEvent, VerifiedEventContent, content_kind,
36};
37use rostra_core::id::{RostraId, ShortRostraId, ToShort as _};
38use rostra_core::{ExternalEventId, ShortEventId, Timestamp};
39use rostra_util_error::{BoxedError, FmtCompact as _};
40use snafu::{Location, ResultExt as _, Snafu};
41use tokio::sync::{Notify, broadcast, watch};
42use tokio::task::JoinError;
43use tracing::{debug, error, info, instrument};
44
45pub use self::current_state::{CurrentState, CurrentStateClosed};
46pub use self::extension::{
47    EXTENSION_RESERVED_TABLE_PREFIXES, ExtensionReadTransaction, ExtensionTableDefinition,
48    ExtensionWriteTransaction,
49};
50pub use self::self_followee::SelfFollowee;
51pub use self::social_post_materialization::{
52    SOCIAL_POST_MATERIALIZATION_SCAN_MAX, SocialPostMaterialization,
53    SocialPostMaterializationCursor, SocialPostMaterializationPage,
54};
55pub(crate) use self::tables::*;
56pub use self::tables::{
57    ContentStoreRecordOwned, EventContentResult, EventContentState, EventReceivedRecord,
58    EventReceivedSource, EventRecord, EventsHeadsTableRecord, IdSocialProfileRecord,
59    IdsDataUsageRecord, IrohNodeRecord, IrohNodeStats, Latest, SocialNewsRankRecord,
60    SocialPostRecord, SocialPostsReactionsRecord, SocialPostsRepliesRecord, SocialVoteScore,
61    SocialVoteSumRecord,
62};
63
64/// Web of Trust data - contains direct followees and extended followees.
65///
66/// Extended followees are the followees of your direct followees, excluding
67/// those you already follow directly.
68#[derive(Debug, Clone, Default)]
69pub struct WotData {
70    /// Direct followees with their persona selectors
71    pub followees: HashMap<RostraId, ids::IdsFolloweesRecord>,
72    /// Extended followees (followees of followees), excluding direct followees
73    pub extended: HashSet<RostraId>,
74}
75
76impl WotData {
77    /// Returns true if the given id is in our web of trust (self, direct
78    /// followee, or extended)
79    pub fn contains(&self, id: RostraId, self_id: RostraId) -> bool {
80        id == self_id || self.followees.contains_key(&id) || self.extended.contains(&id)
81    }
82
83    /// Returns the total number of IDs in the web of trust (excluding self)
84    pub fn len(&self) -> usize {
85        self.followees.len() + self.extended.len()
86    }
87
88    /// Returns true if there are no followees
89    pub fn is_empty(&self) -> bool {
90        self.followees.is_empty()
91    }
92
93    /// Returns an iterator over all IDs in the web of trust (direct +
94    /// extended), excluding self
95    pub fn iter_all(&self) -> impl Iterator<Item = RostraId> + '_ {
96        self.followees
97            .keys()
98            .copied()
99            .chain(self.extended.iter().copied())
100    }
101}
102
103const LOG_TARGET: &str = "rostra::db";
104
105type CommitHook = Box<dyn FnOnce() + 'static>;
106
107pub(crate) struct WriteTransactionCtx {
108    dbtx: WriteTransaction,
109    on_commit: std::sync::Mutex<Option<Vec<CommitHook>>>,
110    materialization_emission_enabled: std::sync::atomic::AtomicBool,
111}
112
113impl From<WriteTransaction> for WriteTransactionCtx {
114    fn from(dbtx: WriteTransaction) -> Self {
115        Self {
116            dbtx,
117            on_commit: std::sync::Mutex::new(Some(vec![])),
118            materialization_emission_enabled: std::sync::atomic::AtomicBool::new(true),
119        }
120    }
121}
122impl std::ops::Deref for WriteTransactionCtx {
123    type Target = WriteTransaction;
124
125    fn deref(&self) -> &Self::Target {
126        &self.dbtx
127    }
128}
129
130impl std::ops::DerefMut for WriteTransactionCtx {
131    fn deref_mut(&mut self) -> &mut Self::Target {
132        &mut self.dbtx
133    }
134}
135
136impl WriteTransactionCtx {
137    /// Registers an action to run in registration order after a successful
138    /// commit.
139    ///
140    /// A hook panic does not suppress later hooks. After every hook has been
141    /// attempted, the first panic resumes.
142    pub(crate) fn on_commit(&self, f: impl FnOnce() + 'static) {
143        if let Some(hooks) = self.on_commit.lock().expect("Locking failed").as_mut() {
144            hooks.push(Box::new(f));
145        }
146    }
147
148    /// Discard existing and future commit hooks for a bulk internal rebuild.
149    ///
150    /// Total replay reconstructs durable state before the database is exposed
151    /// to subscribers. Retaining one publication closure per source event
152    /// would make an otherwise streaming rebuild consume memory
153    /// proportional to database size.
154    pub(crate) fn discard_commit_hooks(&self) {
155        *self.on_commit.lock().expect("Locking failed") = None;
156    }
157
158    /// Return whether this transaction publishes incremental commit hooks.
159    pub(crate) fn commit_hooks_enabled(&self) -> bool {
160        self.on_commit.lock().expect("Locking failed").is_some()
161    }
162
163    /// Suppress durable occurrence emission during a projection rebuild.
164    pub(crate) fn suppress_materialization_emission(&self) {
165        self.materialization_emission_enabled
166            .store(false, std::sync::atomic::Ordering::Release);
167    }
168
169    /// Return whether durable materialization occurrences should be emitted.
170    pub(crate) fn materialization_emission_enabled(&self) -> bool {
171        self.materialization_emission_enabled
172            .load(std::sync::atomic::Ordering::Acquire)
173    }
174
175    fn commit(self) -> result::Result<(), redb::CommitError> {
176        let Self {
177            dbtx,
178            on_commit,
179            materialization_emission_enabled: _,
180        } = self;
181
182        dbtx.commit()?;
183
184        let mut first_panic = None;
185        if let Some(hooks) = on_commit.lock().expect("Locking failed").as_mut() {
186            for hook in hooks.drain(..) {
187                if let Err(payload) = catch_unwind(AssertUnwindSafe(hook)) {
188                    first_panic.get_or_insert(payload);
189                }
190            }
191        }
192        if let Some(payload) = first_panic {
193            resume_unwind(payload);
194        }
195        Ok(())
196    }
197}
198
199#[derive(Debug, Snafu)]
200pub enum TableDumpError {
201    #[snafu(display("Unknown table `{name}`"))]
202    UnknownTable { name: String },
203}
204pub type TableDumpResult<T> = std::result::Result<T, TableDumpError>;
205
206#[derive(Debug, Snafu)]
207pub enum DbError {
208    #[snafu(display("Database error"))]
209    Database {
210        source: redb::DatabaseError,
211        #[snafu(implicit)]
212        location: Location,
213    },
214    #[snafu(transparent)]
215    Table {
216        source: redb::TableError,
217        #[snafu(implicit)]
218        location: Location,
219    },
220    #[snafu(transparent)]
221    Storage {
222        source: redb::StorageError,
223        #[snafu(implicit)]
224        location: Location,
225    },
226    #[snafu(display("Database transaction error"))]
227    Transaction {
228        #[snafu(source(from(redb::TransactionError, Box::new)))]
229        source: Box<redb::TransactionError>,
230        #[snafu(implicit)]
231        location: Location,
232    },
233    #[snafu(display("Database commit error"))]
234    Commit {
235        source: redb::CommitError,
236        #[snafu(implicit)]
237        location: Location,
238    },
239    #[snafu(transparent)]
240    StoredDecode {
241        source: bincode::error::DecodeError,
242        #[snafu(implicit)]
243        location: Location,
244    },
245    #[snafu(display("Database version {db_ver} is newer than supported version {code_ver}"))]
246    DbVersionTooHigh {
247        db_ver: u64,
248        code_ver: u64,
249        #[snafu(implicit)]
250        location: Location,
251    },
252    #[snafu(display("Pending migration stash is missing required table `{table}`"))]
253    MissingMigrationStashTable {
254        table: String,
255        #[snafu(implicit)]
256        location: Location,
257    },
258    #[snafu(display("Database task failed"))]
259    Join {
260        source: JoinError,
261        #[snafu(implicit)]
262        location: Location,
263    },
264    #[snafu(transparent)]
265    DbTxLogic {
266        source: BoxedError,
267        #[snafu(implicit)]
268        location: Location,
269    },
270    #[snafu(visibility(pub))]
271    #[snafu(display("Provided Id does not match one used previously"))]
272    DbIdMismatch {
273        #[snafu(implicit)]
274        location: Location,
275    },
276    #[snafu(display("Database file-format upgrade failed"))]
277    Upgrade {
278        source: redb::UpgradeError,
279        #[snafu(implicit)]
280        location: Location,
281    },
282    #[snafu(display("Integer overflow"))]
283    Overflow,
284    #[snafu(display(
285        "Reception-order key ({received_at:?}, {reception_order}) already exists in table {table}"
286    ))]
287    ReceptionOrderCollision {
288        table: String,
289        received_at: Timestamp,
290        reception_order: u64,
291        #[snafu(implicit)]
292        location: Location,
293    },
294    #[snafu(display("Social post {event_id} already has a reception-order key"))]
295    SocialPostReceiptAlreadyIndexed {
296        event_id: ShortEventId,
297        #[snafu(implicit)]
298        location: Location,
299    },
300    #[snafu(display(
301        "Reception-order key for social post {event_id} references {actual_event_id:?}"
302    ))]
303    SocialPostReceiptMismatch {
304        event_id: ShortEventId,
305        actual_event_id: Option<ShortEventId>,
306        #[snafu(implicit)]
307        location: Location,
308    },
309    #[snafu(display("Deletion attribution references missing event {event_id}"))]
310    MissingDeletionAttribution {
311        event_id: ShortEventId,
312        #[snafu(implicit)]
313        location: Location,
314    },
315    #[snafu(display("Extension access to built-in table `{name}` is forbidden"))]
316    ReservedExtensionTable { name: String },
317    #[snafu(display(
318        "SocialPost materialization cursor {position} exceeds durable position {durable_next}"
319    ))]
320    SocialPostMaterializationCursorOutOfRange { position: u64, durable_next: u64 },
321    #[snafu(display(
322        "SocialPost materialization scan limit {requested} exceeds maximum {maximum}"
323    ))]
324    SocialPostMaterializationScanLimitTooHigh { requested: usize, maximum: usize },
325    #[snafu(display(
326        "SocialPost materialization feed has a gap: expected {expected}, found {actual}"
327    ))]
328    SocialPostMaterializationLogGap { expected: u64, actual: u64 },
329    #[snafu(display("SocialPost materialization {event_id} references a missing event"))]
330    MissingSocialPostMaterializationEvent { event_id: ShortEventId },
331    #[snafu(display("SocialPost materialization {event_id} references event kind {kind} instead"))]
332    InvalidSocialPostMaterializationKind {
333        event_id: ShortEventId,
334        kind: EventKind,
335    },
336    #[snafu(display("SocialPost materialization {event_id} unexpectedly became Missing"))]
337    MissingSocialPostMaterializationState { event_id: ShortEventId },
338    #[snafu(display("SocialPost materialization {event_id} unexpectedly became Invalid"))]
339    InvalidSocialPostMaterializationState { event_id: ShortEventId },
340    #[snafu(display("SocialPost materialization {event_id} has no current content"))]
341    MissingSocialPostMaterializationContent { event_id: ShortEventId },
342    #[snafu(display("SocialPost materialization {event_id} has invalid processed content"))]
343    InvalidSocialPostMaterializationContent {
344        event_id: ShortEventId,
345        source: BoxedError,
346    },
347    #[snafu(display("Stored vote singleton {event_id} has no inline vote projection"))]
348    MissingVoteSingletonProjection {
349        event_id: ShortEventId,
350        #[snafu(implicit)]
351        location: Location,
352    },
353    #[snafu(display(
354        "Stored vote singleton {event_id} has a cached target outside its shortened key"
355    ))]
356    InvalidVoteSingletonProjection {
357        event_id: ShortEventId,
358        #[snafu(implicit)]
359        location: Location,
360    },
361    #[snafu(display(
362        "Shortened identity prefix {prefix} maps to both {existing_id} and {incoming_id}"
363    ))]
364    IdentityPrefixCollision {
365        prefix: ShortRostraId,
366        existing_id: RostraId,
367        incoming_id: RostraId,
368        #[snafu(implicit)]
369        location: Location,
370    },
371}
372pub type DbResult<T> = std::result::Result<T, DbError>;
373
374/// The authoritative event store and projection boundary for one local
375/// identity.
376///
377/// Built-in reducers and projection tables are not part of the external API:
378///
379/// ```compile_fail
380/// let _ = rostra_client_db::Database::process_event_content_inserted_tx;
381/// ```
382///
383/// ```compile_fail
384/// let _ = rostra_client_db::events_singletons_new::TABLE;
385/// ```
386///
387/// ```compile_fail
388/// let _ = rostra_client_db::Database::write_with::<()>;
389/// ```
390///
391/// ```compile_fail
392/// let _ = rostra_client_db::Database::write_with_inner::<()>;
393/// ```
394///
395/// ```compile_fail
396/// let _ = rostra_client_db::Database::read_with::<()>;
397/// ```
398///
399/// ```compile_fail
400/// let _: Option<rostra_client_db::WriteTransactionCtx> = None;
401/// ```
402///
403/// ```compile_fail
404/// rostra_client_db::def_table!(forbidden: u64 => u64);
405/// ```
406#[derive(Debug)]
407pub struct Database {
408    inner: redb_bincode::Database,
409    self_id: RostraId,
410    iroh_secret: iroh::SecretKey,
411
412    /// Timestamp when this database was first created.
413    ///
414    /// Used with the current follow epoch as a notification cutoff: posts older
415    /// than both are historical syncs and should not appear as newly received.
416    db_init_time: Timestamp,
417
418    /// Serializes writes through their post-commit current-state publication.
419    ///
420    /// Acquiring it before transaction creation keeps redb writer order and
421    /// publication order identical. Commit hooks run while it is held and must
422    /// not synchronously re-enter `write_with`. A hook panic propagates after
423    /// commit and poisons this mutex; the next write recovers the guard because
424    /// the durable transaction is still valid.
425    write_and_publish_lock: std::sync::Mutex<()>,
426
427    self_followees_updated: watch::Sender<Arc<HashMap<RostraId, IdsFolloweesRecord>>>,
428    self_followers_updated: watch::Sender<Arc<HashMap<RostraId, IdsFollowersRecord>>>,
429    self_wot_updated: watch::Sender<Arc<WotData>>,
430    self_head_updated: watch::Sender<Option<ShortEventId>>,
431    new_content_tx: broadcast::Sender<VerifiedEventContent>,
432    new_posts_tx: broadcast::Sender<(VerifiedEventContent, content_kind::SocialPost)>,
433    new_shoutbox_tx: broadcast::Sender<(VerifiedEventContent, content_kind::Shoutbox)>,
434    new_heads_tx: broadcast::Sender<(RostraId, ShortEventId)>,
435    ids_with_missing_events_tx: dedup_chan::Sender<RostraId>,
436    news_score_updates_tx: dedup_chan::Sender<ExternalEventId>,
437
438    /// Notification for when new content is added to `events_content_missing`.
439    ///
440    /// The `MissingEventContentFetcher` task waits on this to wake up
441    /// immediately when new missing content arrives, instead of polling.
442    content_missing_notify: Arc<Notify>,
443}
444
445impl Database {
446    const MAX_CONTENT_LEN: u32 = 10_000_000u32;
447
448    pub async fn mk_db_path(
449        data_dir: &Path,
450        self_id: RostraId,
451    ) -> std::result::Result<PathBuf, io::Error> {
452        tokio::fs::create_dir_all(&data_dir).await?;
453
454        let legacy_path_unprefixed_z32 =
455            data_dir.join(format!("{}.redb", self_id.to_unprefixed_z32_string()));
456        if legacy_path_unprefixed_z32.exists() {
457            return Ok(legacy_path_unprefixed_z32);
458        }
459        let legacy_path_bech32 = data_dir.join(format!("{}.redb", self_id.to_bech32_string()));
460        if legacy_path_bech32.exists() {
461            return Ok(legacy_path_bech32);
462        }
463        Ok(data_dir.join(format!("{self_id}.redb")))
464    }
465
466    pub async fn new_in_memory(self_id: RostraId) -> DbResult<Database> {
467        debug!(target: LOG_TARGET, id = %self_id, "Opening in-memory database");
468        let inner = redb::Database::builder()
469            .create_with_file_format_v3(true)
470            .create_with_backend(redb::backends::InMemoryBackend::new())
471            .context(DatabaseSnafu)?;
472        Self::open_inner(inner, self_id).await
473    }
474
475    /// Open an identity database, creating it when absent.
476    ///
477    /// Opening storage older than schema version 25 performs the total rebuild
478    /// before returning. This can hold a long atomic write transaction and
479    /// requires deployment-specific RAM and temporary disk headroom. See
480    /// `specs/ARCH-client-database.md#total-rebuild`.
481    pub async fn open(path: impl Into<PathBuf>, self_id: RostraId) -> DbResult<Database> {
482        let path = path.into();
483        debug!(target: LOG_TARGET, id = %self_id, path = %path.display(), "Opening database");
484
485        let mut inner = tokio::task::spawn_blocking(move || {
486            redb::Database::builder()
487                .create_with_file_format_v3(true)
488                .create(path)
489        })
490        .await
491        .context(JoinSnafu)?
492        .context(DatabaseSnafu)?;
493
494        inner.upgrade().context(UpgradeSnafu)?;
495
496        Self::open_inner(inner, self_id).await
497    }
498
499    #[instrument(skip_all)]
500    async fn open_inner(inner: redb::Database, self_id: RostraId) -> DbResult<Database> {
501        let inner = redb_bincode::Database::from(inner);
502
503        // Run migrations (may stash tables for total migration reprocessing)
504        Self::write_with_inner(&inner, |tx| {
505            // Opening table definitions is raw and does not decode rows. Check
506            // the schema version before reading typed identity bytes so a newer
507            // database always fails with DbVersionTooHigh.
508            Self::init_tables_tx(tx)?;
509            Self::handle_db_ver_migrations(tx)?;
510            Self::verify_self_tx(self_id, &mut tx.open_table(&ids_self::TABLE)?)?;
511            Ok(())
512        })
513        .await?;
514
515        // Check if there's a pending migration stash (either from this run or a
516        // previous interrupted run). Using stash existence as the marker ensures
517        // we retry if reprocessing fails/panics.
518        let needs_reprocessing =
519            Self::write_with_inner(&inner, Self::has_pending_migration_stash).await?;
520
521        let (self_head, iroh_secret, self_followees, self_followers, self_wot, db_init_time) =
522            Self::read_with_inner(&inner, |tx| {
523                let ids_followees_table = tx.open_table(&ids_followees::TABLE)?;
524                let self_followees = Self::read_followees_tx(self_id, &ids_followees_table)?;
525                let self_wot =
526                    Self::compute_wot_tx(self_id, &self_followees, &ids_followees_table)?;
527                let db_init_time = tx
528                    .open_table(&db_init_time::TABLE)?
529                    .get(&())?
530                    .map(|g| g.value())
531                    .unwrap_or(Timestamp::ZERO);
532                Ok((
533                    Self::read_head_tx(self_id, &tx.open_table(&events_heads::TABLE)?)?,
534                    Self::read_iroh_secret_tx(&tx.open_table(&ids_self::TABLE)?)?,
535                    self_followees,
536                    Self::read_followers_tx(self_id, &tx.open_table(&ids_followers::TABLE)?)?,
537                    self_wot,
538                    db_init_time,
539                ))
540            })
541            .await?;
542
543        let (self_followees_updated, _) = watch::channel(Arc::new(self_followees));
544        let (self_followers_updated, _) = watch::channel(Arc::new(self_followers));
545        let (self_wot_updated, _) = watch::channel(Arc::new(self_wot));
546        let (self_head_updated, _) = watch::channel(self_head);
547        let (new_content_tx, _) = broadcast::channel(100);
548        let (new_posts_tx, _) = broadcast::channel(100);
549        let (new_shoutbox_tx, _) = broadcast::channel(100);
550        let (new_heads_tx, _) = broadcast::channel(100);
551
552        let db = Self {
553            inner,
554            self_id,
555            iroh_secret,
556            db_init_time,
557            write_and_publish_lock: std::sync::Mutex::new(()),
558            self_followees_updated,
559            self_followers_updated,
560            self_wot_updated,
561            self_head_updated,
562            new_content_tx,
563            new_posts_tx,
564            new_shoutbox_tx,
565            new_heads_tx,
566            ids_with_missing_events_tx: dedup_chan::Sender::new(),
567            news_score_updates_tx: dedup_chan::Sender::new(),
568            content_missing_notify: Arc::new(Notify::new()),
569        };
570
571        // If total migration stashed events, reprocess them now using the real
572        // Database. The stash existence check ensures we retry if this
573        // fails/panics.
574        if needs_reprocessing {
575            db.write_with(|tx| db.reprocess_migration_stash(tx)).await?;
576            db.refresh_current_state_after_replay().await?;
577        }
578
579        Ok(db)
580    }
581
582    /// Publish one coherent current-state snapshot after silent total replay.
583    async fn refresh_current_state_after_replay(&self) -> DbResult<()> {
584        let (self_head, self_followees, self_followers, self_wot) = self
585            .read_with(|tx| {
586                let followees = tx.open_table(&ids_followees::TABLE)?;
587                let self_followees = Self::read_followees_tx(self.self_id, &followees)?;
588                let self_wot = Self::compute_wot_tx(self.self_id, &self_followees, &followees)?;
589                Ok((
590                    Self::read_head_tx(self.self_id, &tx.open_table(&events_heads::TABLE)?)?,
591                    self_followees,
592                    Self::read_followers_tx(self.self_id, &tx.open_table(&ids_followers::TABLE)?)?,
593                    self_wot,
594                ))
595            })
596            .await?;
597
598        self.self_head_updated.send_replace(self_head);
599        self.self_followees_updated
600            .send_replace(Arc::new(self_followees));
601        self.self_followers_updated
602            .send_replace(Arc::new(self_followers));
603        self.self_wot_updated.send_replace(Arc::new(self_wot));
604        Ok(())
605    }
606
607    pub async fn compact(&mut self) -> Result<bool, redb::CompactionError> {
608        tokio::task::block_in_place(|| self.inner.as_raw_mut().compact())
609    }
610
611    pub async fn dump_table(&self, name: &str) -> TableDumpResult<()> {
612        self.read_with(|tx| {
613            match name {
614                "events" => Self::dump_table_dbtx(tx, &tables::events::TABLE)?,
615                "content_store" => Self::dump_table_dbtx(tx, &tables::content_store::TABLE)?,
616                "events_content_state" => {
617                    Self::dump_table_dbtx(tx, &tables::events_content_state::TABLE)?
618                }
619                "events_content_missing" => {
620                    Self::dump_table_dbtx(tx, &tables::events_content_missing::TABLE)?
621                }
622                "social_posts" => Self::dump_table_dbtx(tx, &tables::social_posts::TABLE)?,
623                "social_posts_replies" => {
624                    Self::dump_table_dbtx(tx, &tables::social_posts_replies::TABLE)?
625                }
626                "social_posts_reactions" => {
627                    Self::dump_table_dbtx(tx, &tables::social_posts_reactions::TABLE)?
628                }
629                "social_vote_sums" => Self::dump_table_dbtx(tx, &tables::social_vote_sums::TABLE)?,
630                "social_news_rank_by_post_id" => {
631                    Self::dump_table_dbtx(tx, &tables::social_news_rank_by_post_id::TABLE)?
632                }
633                "social_news_rank_by_score" => {
634                    Self::dump_table_dbtx(tx, &tables::social_news_rank_by_score::TABLE)?
635                }
636                "social_news_rank_by_time" => {
637                    Self::dump_table_dbtx(tx, &tables::social_news_rank_by_time::TABLE)?
638                }
639                _ => {
640                    return Ok(Err(UnknownTableSnafu {
641                        name: name.to_string(),
642                    }
643                    .build()));
644                }
645            }
646            Ok(Ok(()))
647        })
648        .await
649        .expect("Database panic")
650    }
651
652    /// Subscribe to owned snapshots of the retained current self-followee
653    /// projection.
654    pub fn self_followees_subscribe(
655        &self,
656    ) -> CurrentState<Arc<HashMap<RostraId, IdsFolloweesRecord>>> {
657        CurrentState::new(self.self_followees_updated.subscribe())
658    }
659
660    /// Subscribe to owned snapshots of the retained current self-follower
661    /// projection.
662    pub fn self_followers_subscribe(
663        &self,
664    ) -> CurrentState<Arc<HashMap<RostraId, IdsFollowersRecord>>> {
665        CurrentState::new(self.self_followers_updated.subscribe())
666    }
667
668    /// Subscribe to owned snapshots of the retained current Web-of-Trust
669    /// projection.
670    pub fn self_wot_subscribe(&self) -> CurrentState<Arc<WotData>> {
671        CurrentState::new(self.self_wot_updated.subscribe())
672    }
673
674    /// Subscribe to owned snapshots of the retained deterministic self-head
675    /// representative.
676    ///
677    /// The value is the minimum current `ShortEventId`. It is a stable default,
678    /// not a claim that the current head set is a singleton. Every committed
679    /// self-event insertion publishes, even when the representative is
680    /// unchanged.
681    pub fn self_head_subscribe(&self) -> CurrentState<Option<ShortEventId>> {
682        CurrentState::new(self.self_head_updated.subscribe())
683    }
684
685    pub fn new_content_subscribe(&self) -> broadcast::Receiver<VerifiedEventContent> {
686        self.new_content_tx.subscribe()
687    }
688    pub fn new_posts_subscribe(
689        &self,
690    ) -> broadcast::Receiver<(VerifiedEventContent, content_kind::SocialPost)> {
691        self.new_posts_tx.subscribe()
692    }
693
694    pub fn new_shoutbox_subscribe(
695        &self,
696    ) -> broadcast::Receiver<(VerifiedEventContent, content_kind::Shoutbox)> {
697        self.new_shoutbox_tx.subscribe()
698    }
699
700    /// Subscribe to incremental exact new-head signals.
701    ///
702    /// This lossy channel carries the event that became a head. Consumers that
703    /// lag must recover from the durable complete head set.
704    pub fn new_heads_subscribe(&self) -> broadcast::Receiver<(RostraId, ShortEventId)> {
705        self.new_heads_tx.subscribe()
706    }
707
708    pub fn ids_with_missing_events_subscribe(
709        &self,
710        capacity: usize,
711    ) -> dedup_chan::Receiver<RostraId> {
712        self.ids_with_missing_events_tx.subscribe(capacity)
713    }
714
715    pub fn news_score_updates_subscribe(
716        &self,
717        capacity: usize,
718    ) -> dedup_chan::Receiver<ExternalEventId> {
719        self.news_score_updates_tx.subscribe(capacity)
720    }
721
722    /// Get a handle to the content-missing notification.
723    ///
724    /// The `MissingEventContentFetcher` calls `notified()` on this to wake up
725    /// when new missing content is inserted into the database.
726    pub fn content_missing_notify(&self) -> Arc<Notify> {
727        self.content_missing_notify.clone()
728    }
729
730    pub async fn has_event(&self, event_id: impl Into<ShortEventId>) -> bool {
731        let event_id = event_id.into();
732        self.read_with(|tx| {
733            let events_table = tx.open_table(&events::TABLE).expect("Storage error");
734            Database::has_event_tx(event_id, &events_table)
735        })
736        .await
737        .expect("Database panic")
738    }
739
740    pub async fn get_missing_events_for_id(&self, id: RostraId) -> Vec<ShortEventId> {
741        self.read_with(|tx| {
742            let events_missing_tbl = tx.open_table(&events_missing::TABLE)?;
743            Ok(
744                Database::get_missing_events_for_id_tx(id, &events_missing_tbl)?
745                    .into_iter()
746                    .collect(),
747            )
748        })
749        .await
750        .expect("Database panic")
751    }
752
753    /// Lists unique authors that still have missing event envelopes.
754    ///
755    /// The page starts strictly after `after`, is ordered by author ID, and
756    /// contains at most `limit` authors. A zero limit returns an empty page.
757    ///
758    /// Each call uses an independent read transaction. Callers that reconcile
759    /// multiple pages should capture [`Self::get_last_id_with_missing_events`]
760    /// first and stop at that high-water author.
761    ///
762    /// This compatibility query panics if the database read fails.
763    pub async fn get_ids_with_missing_events(
764        &self,
765        after: Option<RostraId>,
766        limit: usize,
767    ) -> Vec<RostraId> {
768        self.read_with(|tx| {
769            let events_missing_tbl = tx.open_table(&events_missing::TABLE)?;
770            Database::get_ids_with_missing_events_tx(after, limit, &events_missing_tbl)
771        })
772        .await
773        .expect("Database panic")
774    }
775
776    /// Returns the greatest author ID that currently has missing event
777    /// envelopes.
778    ///
779    /// This compatibility query panics if the database read fails.
780    pub async fn get_last_id_with_missing_events(&self) -> Option<RostraId> {
781        self.read_with(|tx| {
782            let events_missing_tbl = tx.open_table(&events_missing::TABLE)?;
783            Database::get_last_id_with_missing_events_tx(&events_missing_tbl)
784        })
785        .await
786        .expect("Database panic")
787    }
788
789    pub async fn get_heads_events_for_id(&self, id: RostraId) -> Vec<ShortEventId> {
790        self.read_with(|tx| {
791            let events_heads_tbl = tx.open_table(&events_heads::TABLE)?;
792            Ok(Database::get_heads_events_tx(id, &events_heads_tbl)?
793                .into_iter()
794                .collect())
795        })
796        .await
797        .expect("Database panic")
798    }
799
800    /// Lists a bounded ordered page of unique authors with current durable
801    /// heads.
802    ///
803    /// The page starts strictly after `after`. A zero limit returns an empty
804    /// page. This compatibility query panics if the database read fails.
805    pub async fn get_ids_with_heads(&self, after: Option<RostraId>, limit: usize) -> Vec<RostraId> {
806        self.read_with(|tx| {
807            let events_heads_tbl = tx.open_table(&events_heads::TABLE)?;
808            Database::get_ids_with_heads_tx(after, limit, &events_heads_tbl)
809        })
810        .await
811        .expect("Database panic")
812    }
813
814    pub async fn count_missing_events_for_id(&self, id: RostraId) -> usize {
815        self.read_with(|tx| {
816            let events_missing_tbl = tx.open_table(&events_missing::TABLE)?;
817            Database::count_missing_events_for_id_tx(id, &events_missing_tbl)
818        })
819        .await
820        .expect("Database panic")
821    }
822
823    pub async fn count_heads_events_for_id(&self, id: RostraId) -> usize {
824        self.read_with(|tx| {
825            let events_heads_tbl = tx.open_table(&events_heads::TABLE)?;
826            Database::count_heads_events_tx(id, &events_heads_tbl)
827        })
828        .await
829        .expect("Database panic")
830    }
831
832    pub async fn get_data_usage(&self, id: RostraId) -> IdsDataUsageRecord {
833        self.read_with(|tx| {
834            let ids_data_usage_tbl = tx.open_table(&ids_data_usage::TABLE)?;
835            Database::get_data_usage_tx(id, &ids_data_usage_tbl)
836        })
837        .await
838        .expect("Database panic")
839    }
840
841    pub async fn get_self_followees(&self) -> Vec<(RostraId, PersonasTagsSelector)> {
842        self.get_followees(self.self_id).await
843    }
844
845    pub async fn get_followees(&self, id: RostraId) -> Vec<(RostraId, PersonasTagsSelector)> {
846        self.read_with(|tx| {
847            let ids_followees_table = tx.open_table(&ids_followees::TABLE)?;
848            Ok(Database::read_followees_tx(id, &ids_followees_table)?
849                .into_iter()
850                .map(|(id, record)| (id, record.effective_tags_selector()))
851                .collect())
852        })
853        .await
854        .expect("Database panic")
855    }
856
857    pub async fn get_followees_extended(
858        &self,
859        id: RostraId,
860    ) -> (HashMap<RostraId, PersonasTagsSelector>, HashSet<RostraId>) {
861        self.read_with(|tx| {
862            let ids_followees_table = tx.open_table(&ids_followees::TABLE)?;
863            let followees: HashMap<RostraId, PersonasTagsSelector> =
864                Database::read_followees_tx_iter(id, &ids_followees_table)?
865                    .map_ok(|(id, record)| (id, record.effective_tags_selector()))
866                    .collect::<Result<_, _>>()?;
867
868            let mut extended = HashSet::new();
869
870            for followee in followees.keys() {
871                for extended_followee in
872                    Database::read_followees_tx_iter(*followee, &ids_followees_table)?
873                        .map_ok(|(id, _record)| id)
874                {
875                    let extended_followee = extended_followee?;
876                    if !followees.contains_key(&extended_followee) {
877                        extended.insert(extended_followee);
878                    }
879                }
880            }
881            Ok((followees, extended))
882        })
883        .await
884        .expect("Database panic")
885    }
886
887    pub async fn get_self_followers(&self) -> Vec<RostraId> {
888        self.get_followers(self.self_id).await
889    }
890
891    pub async fn get_followers(&self, id: RostraId) -> Vec<RostraId> {
892        self.read_with(|tx| {
893            let ids_followers_table = tx.open_table(&ids_followers::TABLE)?;
894            Ok(Database::read_followers_tx(id, &ids_followers_table)?
895                .into_keys()
896                .collect())
897        })
898        .await
899        .expect("Database panic")
900    }
901
902    pub async fn get_event(
903        &self,
904        event_id: impl Into<ShortEventId>,
905    ) -> Option<crate::event::EventRecord> {
906        let event_id = event_id.into();
907        self.read_with(|tx| {
908            let events_table = tx.open_table(&events::TABLE)?;
909            Database::get_event_tx(event_id, &events_table)
910        })
911        .await
912        .expect("Database panic")
913    }
914
915    pub async fn get_event_content(
916        &self,
917        event_id: impl Into<ShortEventId>,
918    ) -> Option<EventContentRaw> {
919        let event_id = event_id.into();
920        self.read_with(|tx| {
921            let events_table = tx.open_table(&events::TABLE)?;
922            let events_content_state_table = tx.open_table(&events_content_state::TABLE)?;
923            let content_store_table = tx.open_table(&content_store::TABLE)?;
924
925            // Get the event to find its content hash
926            let Some(event_record) = Database::get_event_tx(event_id, &events_table)? else {
927                return Ok(None);
928            };
929            let content_hash = event_record.content_hash();
930
931            Ok(Database::get_event_content_full_tx(
932                event_id,
933                content_hash,
934                &events_content_state_table,
935                &content_store_table,
936            )?
937            .and_then(|result| result.content().cloned()))
938        })
939        .await
940        .expect("Database panic")
941    }
942
943    /// Return the lifecycle state for event content that is not currently
944    /// stored.
945    ///
946    /// For an existing event, `None` normally means the content was processed.
947    /// A terminal state such as `Deleted`, `Pruned`, or `Invalid` means callers
948    /// must not wait for payload availability.
949    pub async fn get_event_content_state(
950        &self,
951        event_id: impl Into<ShortEventId>,
952    ) -> Option<EventContentState> {
953        let event_id = event_id.into();
954        self.read_with(|tx| {
955            Self::get_event_content_state_tx(
956                event_id,
957                &tx.open_table(&events_content_state::TABLE)?,
958            )
959        })
960        .await
961        .expect("Database panic")
962    }
963
964    /// Return the minimum current self-head as a deterministic representative.
965    ///
966    /// Use [`Self::get_heads_self`] when the complete set is required.
967    pub async fn get_self_current_head(&self) -> Option<ShortEventId> {
968        self.read_with(|tx| {
969            let events_heads_table = tx.open_table(&events_heads::TABLE)?;
970
971            Database::read_head_tx(self.self_id, &events_heads_table)
972        })
973        .await
974        .expect("Storage error")
975    }
976
977    pub async fn get_self_random_eventid(&self) -> Option<ShortEventId> {
978        self.read_with(|tx| {
979            let events_self_table = tx.open_table(&events_self::TABLE)?;
980
981            Database::get_random_self_event(&events_self_table)
982        })
983        .await
984        .expect("Storage error")
985    }
986
987    /// Fallibly process a verified event envelope.
988    ///
989    /// # Errors
990    ///
991    /// Returns storage and transaction errors, including
992    /// [`DbError::IdentityPrefixCollision`]. An error leaves the ingestion
993    /// transaction uncommitted.
994    pub async fn try_process_event(
995        &self,
996        event: &VerifiedEvent,
997    ) -> DbResult<(InsertEventOutcome, ProcessEventState)> {
998        let now = Timestamp::now();
999        self.write_with(|tx| self.process_event_tx(event, now, tx))
1000            .await
1001    }
1002
1003    /// Process a verified event envelope, panicking on storage failure.
1004    ///
1005    /// # Panics
1006    ///
1007    /// Panics after rolling back if storage fails or the event author's
1008    /// shortened identity prefix is already mapped to a different full
1009    /// identity.
1010    pub async fn process_event(
1011        &self,
1012        event: &VerifiedEvent,
1013    ) -> (InsertEventOutcome, ProcessEventState) {
1014        self.try_process_event(event).await.expect("Storage error")
1015    }
1016
1017    /// Fallibly process a verified envelope and its verified content
1018    /// atomically.
1019    ///
1020    /// This operation is idempotent whether or not the envelope or content was
1021    /// processed previously.
1022    ///
1023    /// # Errors
1024    ///
1025    /// Returns storage and transaction errors, including
1026    /// [`DbError::IdentityPrefixCollision`]. An error rolls back the envelope,
1027    /// content, lifecycle, and projection changes together.
1028    pub async fn try_process_event_with_content(
1029        &self,
1030        content: &VerifiedEventContent,
1031    ) -> DbResult<(InsertEventOutcome, ProcessEventState)> {
1032        let now = Timestamp::now();
1033        self.write_with(|tx| {
1034            let res = self.process_event_tx(&content.event, now, tx)?;
1035            self.process_event_content_tx(content, now, tx)?;
1036            Ok(res)
1037        })
1038        .await
1039    }
1040
1041    /// Process a verified envelope and its verified content, panicking on
1042    /// storage failure.
1043    ///
1044    /// This operation is idempotent whether or not the envelope or content was
1045    /// processed previously.
1046    ///
1047    /// # Panics
1048    ///
1049    /// Panics after rolling back if storage fails or a stored invariant is
1050    /// violated, including a vote winner with a missing or invalid inline
1051    /// projection or a shortened identity prefix mapped to a different full
1052    /// identity.
1053    pub async fn process_event_with_content(
1054        &self,
1055        content: &VerifiedEventContent,
1056    ) -> (InsertEventOutcome, ProcessEventState) {
1057        self.try_process_event_with_content(content)
1058            .await
1059            .expect("Storage error")
1060    }
1061
1062    /// Fallibly process verified content and its carried envelope atomically.
1063    ///
1064    /// This is the safe public content-ingestion boundary. If the envelope is
1065    /// absent, it is inserted before content processing in the same
1066    /// transaction. If the envelope is already present, normal
1067    /// lifecycle-state checks make the operation idempotent and prevent
1068    /// repeated accounting or projections.
1069    ///
1070    /// # Errors
1071    ///
1072    /// Returns errors under the same conditions as
1073    /// [`Database::try_process_event_with_content`].
1074    pub async fn try_process_event_content(
1075        &self,
1076        event_content: &VerifiedEventContent,
1077    ) -> DbResult<()> {
1078        self.try_process_event_with_content(event_content)
1079            .await
1080            .map(|_| ())
1081    }
1082
1083    /// Process verified content and its carried envelope, panicking on storage
1084    /// failure.
1085    ///
1086    /// # Panics
1087    ///
1088    /// Panics under the same conditions as
1089    /// [`Database::process_event_with_content`].
1090    pub async fn process_event_content(&self, event_content: &VerifiedEventContent) {
1091        self.try_process_event_content(event_content)
1092            .await
1093            .expect("Storage error");
1094    }
1095
1096    /// Process event content.
1097    ///
1098    /// Ordinary processing requires Missing state, applies kind-specific side
1099    /// effects, stores the content, removes fetch scheduling, and transitions
1100    /// to Processed. RC was already incremented at event insertion.
1101    ///
1102    /// A verified, below-limit Deleted social-post edit is the sole
1103    /// terminal-state exception. It may add only immutable forward and reverse
1104    /// replacement rows; it does not store the supplied bytes or change
1105    /// lifecycle bookkeeping or other projections.
1106    ///
1107    /// The `now` parameter should be `Timestamp::now()` for normal operation,
1108    /// but can be set to a specific value for testing or migration.
1109    pub(crate) fn process_event_content_tx(
1110        &self,
1111        event_content: &VerifiedEventContent,
1112        now: Timestamp,
1113        tx: &WriteTransactionCtx,
1114    ) -> DbResult<()> {
1115        {
1116            let events_table = tx.open_table(&events::TABLE)?;
1117            let has_event = Database::has_event_tx(event_content.event.event_id, &events_table)?;
1118            if !has_event {
1119                // Event doesn't exist - this shouldn't happen in normal operation.
1120                // It means process_event_content_tx was called without first inserting the
1121                // event.
1122                debug_assert!(false, "Processing content for non-existent event");
1123                error!(
1124                    target: LOG_TARGET,
1125                    event_id = %event_content.event.event_id,
1126                    "Processing content for non-existent event - possible bug"
1127                );
1128                return Ok(());
1129            }
1130        }
1131
1132        // Check if content should be processed (not deleted/pruned, is Missing)
1133        let (can_insert, is_deleted) = if u32::from(event_content.event.event.content_len)
1134            < Self::MAX_CONTENT_LEN
1135        {
1136            let events_content_state_table = tx.open_table(&events_content_state::TABLE)?;
1137            let state = events_content_state_table
1138                .get(&event_content.event_id().to_short())?
1139                .map(|state| state.value());
1140            (
1141                Database::can_insert_event_content_tx(event_content, &events_content_state_table)?,
1142                matches!(state, Some(EventContentState::Deleted { .. })),
1143            )
1144        } else {
1145            (false, false)
1146        };
1147
1148        if can_insert {
1149            // Remove eligible content from the missing list.
1150            {
1151                let event_short_id = event_content.event_id().to_short();
1152                let events_content_state_table = tx.open_table(&events_content_state::TABLE)?;
1153                let mut events_content_missing_table =
1154                    tx.open_table(&tables::events_content_missing::TABLE)?;
1155                let state = events_content_state_table
1156                    .get(&event_short_id)?
1157                    .map(|g| g.value());
1158                if let Some(EventContentState::Missing {
1159                    next_fetch_attempt, ..
1160                }) = state
1161                {
1162                    events_content_missing_table.remove(&(next_fetch_attempt, event_short_id))?;
1163                }
1164            }
1165
1166            if let Some(content) = event_content.content.as_ref() {
1167                let content_hash = event_content.content_hash();
1168                let event_short_id = event_content.event_id().to_short();
1169
1170                // Process side effects
1171                let is_valid = match self.process_event_content_inserted_tx(event_content, now, tx)
1172                {
1173                    Ok(()) => {
1174                        if tx.commit_hooks_enabled() {
1175                            info!(target: LOG_TARGET,
1176                                kind = %event_content.kind(),
1177                                event_id = %event_short_id,
1178                                author = %event_content.author().to_short(),
1179                                len = %event_content.content_len(),
1180                                "New event content inserted"
1181                            );
1182                        }
1183                        true
1184                    }
1185                    Err(ProcessEventError::Invalid { source, location }) => {
1186                        if tx.commit_hooks_enabled() {
1187                            info!(
1188                                target: LOG_TARGET,
1189                                err = %source.as_ref().fmt_compact(),
1190                                %location,
1191                                "Invalid event content"
1192                            );
1193                        }
1194                        false
1195                    }
1196                    Err(ProcessEventError::Db { source }) => {
1197                        return Err(source);
1198                    }
1199                };
1200
1201                if is_valid {
1202                    // Store content in content_store if not already there
1203                    {
1204                        let mut content_store_table = tx.open_table(&content_store::TABLE)?;
1205                        if content_store_table.get(&content_hash)?.is_none() {
1206                            content_store_table.insert(
1207                                &content_hash,
1208                                &ContentStoreRecord(Cow::Owned(content.clone())),
1209                            )?;
1210                        }
1211                    }
1212
1213                    // Remove the Missing marker now that content is processed
1214                    {
1215                        let mut events_content_state_table =
1216                            tx.open_table(&events_content_state::TABLE)?;
1217                        events_content_state_table.remove(&event_short_id)?;
1218                    }
1219
1220                    // Track payload as processed (missing → current)
1221                    {
1222                        let mut ids_data_usage_table =
1223                            tx.open_table(&tables::ids_data_usage::TABLE)?;
1224                        Database::track_payload_processed_tx(
1225                            event_content.author(),
1226                            event_content.content_len(),
1227                            &mut ids_data_usage_table,
1228                        )?;
1229                    }
1230
1231                    // Notify about new content
1232                    if tx.commit_hooks_enabled() {
1233                        tx.on_commit({
1234                            let new_content_tx = self.new_content_tx.clone();
1235                            let event_content = event_content.clone();
1236                            move || {
1237                                let _ = new_content_tx.send(event_content);
1238                            }
1239                        });
1240                    }
1241                } else {
1242                    // Content failed validation — mark as Invalid, decrement RC,
1243                    // discard content bytes
1244                    {
1245                        let mut events_content_state_table =
1246                            tx.open_table(&events_content_state::TABLE)?;
1247                        events_content_state_table
1248                            .insert(&event_short_id, &EventContentState::Invalid)?;
1249                    }
1250                    {
1251                        let mut content_rc_table = tx.open_table(&tables::content_rc::TABLE)?;
1252                        Database::decrement_content_rc_tx(content_hash, &mut content_rc_table)?;
1253                    }
1254
1255                    // Track payload as invalid (missing → invalid)
1256                    {
1257                        let mut ids_data_usage_table =
1258                            tx.open_table(&tables::ids_data_usage::TABLE)?;
1259                        Database::track_payload_invalid_tx(
1260                            event_content.author(),
1261                            event_content.content_len(),
1262                            &mut ids_data_usage_table,
1263                        )?;
1264                    }
1265                }
1266            }
1267        } else if is_deleted && event_content.content.is_some() {
1268            match Self::process_deleted_social_post_replacement_tx(event_content, tx) {
1269                Ok(_) => {}
1270                Err(ProcessEventError::Invalid { source, location }) => {
1271                    debug!(
1272                        target: LOG_TARGET,
1273                        err = %source.as_ref().fmt_compact(),
1274                        %location,
1275                        "Ignoring malformed Deleted social-post content"
1276                    );
1277                }
1278                Err(ProcessEventError::Db { source }) => return Err(source),
1279            }
1280        }
1281        Ok(())
1282    }
1283
1284    pub async fn wants_content(
1285        &self,
1286        event_id: impl Into<ShortEventId>,
1287        process_state: ProcessEventState,
1288    ) -> bool {
1289        match process_state.wants_content() {
1290            ContentWantState::DoesNotWant => {
1291                return false;
1292            }
1293            ContentWantState::Wants => {
1294                return true;
1295            }
1296            ContentWantState::MaybeWants => {}
1297        }
1298
1299        let event_id = event_id.into();
1300        self.read_with(|tx| {
1301            let events_table = tx.open_table(&events::TABLE)?;
1302            let events_content_state_table = tx.open_table(&events_content_state::TABLE)?;
1303            let content_store_table = tx.open_table(&content_store::TABLE)?;
1304
1305            // Get event to find content_hash
1306            let Some(event) = Database::get_event_tx(event_id, &events_table)? else {
1307                return Ok(false);
1308            };
1309            let content_hash = event.content_hash();
1310
1311            // We want content if we DON'T have it yet
1312            Ok(!Database::has_event_content_tx(
1313                event_id,
1314                content_hash,
1315                &events_content_state_table,
1316                &content_store_table,
1317            )?)
1318        })
1319        .await
1320        .expect("Storage error")
1321    }
1322
1323    pub fn db_init_time(&self) -> Timestamp {
1324        self.db_init_time
1325    }
1326
1327    pub fn iroh_secret(&self) -> iroh::SecretKey {
1328        self.iroh_secret.clone()
1329    }
1330}
1331
1332impl Database {
1333    fn write_with_inner_blocking<T>(
1334        inner: &redb_bincode::Database,
1335        f: impl FnOnce(&'_ WriteTransactionCtx) -> DbResult<T>,
1336    ) -> DbResult<T> {
1337        let mut dbtx = WriteTransactionCtx::from(inner.begin_write().context(TransactionSnafu)?);
1338        let res = f(&mut dbtx)?;
1339
1340        dbtx.commit().context(CommitSnafu)?;
1341
1342        Ok(res)
1343    }
1344
1345    /// Runs a low-level write transaction without a `Database` publication
1346    /// boundary.
1347    ///
1348    /// The synchronous transaction closure must not start another write.
1349    pub(crate) async fn write_with_inner<T>(
1350        inner: &redb_bincode::Database,
1351        f: impl FnOnce(&'_ WriteTransactionCtx) -> DbResult<T>,
1352    ) -> DbResult<T> {
1353        tokio::task::block_in_place(|| Self::write_with_inner_blocking(inner, f))
1354    }
1355
1356    /// Runs a serialized write transaction and its internal post-commit
1357    /// actions.
1358    ///
1359    /// The mutex spans transaction creation, durable commit, and all
1360    /// post-commit actions so their publication order matches writer order.
1361    /// The synchronous transaction closure and post-commit actions must not
1362    /// re-enter database writes.
1363    ///
1364    /// A post-commit action panic propagates to the caller after the
1365    /// transaction has committed. Every registered action is attempted, then
1366    /// the first panic resumes. A later write recovers the poisoned
1367    /// serialization mutex and proceeds.
1368    pub(crate) async fn write_with<T>(
1369        &self,
1370        f: impl FnOnce(&'_ WriteTransactionCtx) -> DbResult<T>,
1371    ) -> DbResult<T> {
1372        tokio::task::block_in_place(|| {
1373            let _write_and_publish_guard = self
1374                .write_and_publish_lock
1375                .lock()
1376                .unwrap_or_else(std::sync::PoisonError::into_inner);
1377            Self::write_with_inner_blocking(&self.inner, f)
1378        })
1379    }
1380
1381    /// Runs a serialized transaction over caller-owned extension tables.
1382    ///
1383    /// The closure cannot access built-in graph, lifecycle, or projection
1384    /// tables. It must not re-enter database writes. This trusted persistence
1385    /// boundary does not include extension data in core replay or convergence;
1386    /// callers own their schema, invariants, and compatibility. Total database
1387    /// migrations preserve extension tables byte-for-byte.
1388    ///
1389    /// Returning `Ok` commits all extension-table mutations atomically.
1390    /// Returning `Err`, panicking, or failing a reserved-name/table
1391    /// operation leaves the transaction uncommitted. Post-commit built-in
1392    /// projection hooks are not available at this boundary.
1393    ///
1394    /// # Errors
1395    ///
1396    /// Returns the closure's error and propagates transaction, table, and
1397    /// commit errors. Opening a reserved table returns
1398    /// [`DbError::ReservedExtensionTable`].
1399    pub async fn extension_write<T>(
1400        &self,
1401        f: impl FnOnce(&ExtensionWriteTransaction<'_>) -> DbResult<T>,
1402    ) -> DbResult<T> {
1403        self.write_with(|tx| f(&ExtensionWriteTransaction::new(tx)))
1404            .await
1405    }
1406
1407    pub(crate) async fn read_with_inner<T>(
1408        inner: &redb_bincode::Database,
1409        f: impl FnOnce(&'_ ReadTransaction) -> DbResult<T>,
1410    ) -> DbResult<T> {
1411        tokio::task::block_in_place(|| {
1412            let mut dbtx = inner.begin_read().context(TransactionSnafu)?;
1413
1414            f(&mut dbtx)
1415        })
1416    }
1417
1418    pub(crate) async fn read_with<T>(
1419        &self,
1420        f: impl FnOnce(&'_ ReadTransaction) -> DbResult<T>,
1421    ) -> DbResult<T> {
1422        Self::read_with_inner(&self.inner, f).await
1423    }
1424
1425    /// Runs a read transaction over trusted, caller-owned extension tables.
1426    ///
1427    /// Callers own extension schemas, invariants, and compatibility; core event
1428    /// replay does not validate or rebuild their contents.
1429    ///
1430    /// # Errors
1431    ///
1432    /// Returns the closure's error and propagates transaction and table errors.
1433    /// Opening a reserved table returns [`DbError::ReservedExtensionTable`].
1434    pub async fn extension_read<T>(
1435        &self,
1436        f: impl FnOnce(&ExtensionReadTransaction<'_>) -> DbResult<T>,
1437    ) -> DbResult<T> {
1438        self.read_with(|tx| f(&ExtensionReadTransaction::new(tx)))
1439            .await
1440    }
1441
1442    pub(crate) fn verify_self_tx(
1443        self_id: RostraId,
1444        ids_self_t: &mut ids_self::Table,
1445    ) -> DbResult<()> {
1446        match Self::read_self_id_tx(ids_self_t)? {
1447            Some(existing_self_id_record) => {
1448                if existing_self_id_record.rostra_id != self_id {
1449                    return DbIdMismatchSnafu.fail();
1450                }
1451            }
1452            _ => {
1453                Self::write_self_id_tx(self_id, ids_self_t)?;
1454            }
1455        };
1456        Ok(())
1457    }
1458
1459    /// Return the minimum current head as a deterministic representative.
1460    ///
1461    /// Use [`Self::get_heads`] when the complete set is required.
1462    pub async fn get_head(&self, id: RostraId) -> Option<ShortEventId> {
1463        self.read_with(|tx| {
1464            let events_heads = tx.open_table(&events_heads::TABLE)?;
1465
1466            Self::read_head_tx(id, &events_heads)
1467        })
1468        .await
1469        .expect("Database panic")
1470    }
1471
1472    /// Return the complete current head set for an identity.
1473    pub async fn get_heads(&self, id: RostraId) -> HashSet<ShortEventId> {
1474        self.read_with(|tx| {
1475            let events_heads = tx.open_table(&events_heads::TABLE)?;
1476
1477            Self::get_heads_tx(id, &events_heads)
1478        })
1479        .await
1480        .expect("Database panic")
1481    }
1482
1483    /// Return the complete current head set for the local identity.
1484    pub async fn get_heads_self(&self) -> HashSet<ShortEventId> {
1485        self.read_with(|tx| {
1486            let events_heads = tx.open_table(&events_heads::TABLE)?;
1487
1488            Self::get_heads_tx(self.self_id, &events_heads)
1489        })
1490        .await
1491        .expect("Database panic")
1492    }
1493
1494    pub async fn get_social_profile(&self, id: RostraId) -> Option<IdSocialProfileRecord> {
1495        self.read_with(|tx| {
1496            let events_heads = tx.open_table(&social_profiles::TABLE)?;
1497
1498            Self::get_social_profile_tx(id, &events_heads)
1499        })
1500        .await
1501        .expect("Database panic")
1502    }
1503
1504    pub async fn get_latest_singleton_event(
1505        &self,
1506        rostra_id: RostraId,
1507        kind: EventKind,
1508        aux_key: EventAuxKey,
1509    ) -> Option<ShortEventId> {
1510        self.read_with(|tx| {
1511            let singletons_table = tx.open_table(&events_singletons_new::TABLE)?;
1512
1513            Ok(singletons_table
1514                .get(&(rostra_id, kind, aux_key))?
1515                .map(|record| record.value().inner.event_id))
1516        })
1517        .await
1518        .expect("Database panic")
1519    }
1520
1521    /// Returns an identity's singleton winners for one event kind in descending
1522    /// `(timestamp, ShortEventId)` order.
1523    pub async fn get_latest_singleton_events(
1524        &self,
1525        rostra_id: RostraId,
1526        kind: EventKind,
1527    ) -> Vec<ShortEventId> {
1528        self.read_with(|tx| {
1529            let singletons = tx.open_table(&events_singletons_new::TABLE)?;
1530            let start = (rostra_id, kind, EventAuxKey::ZERO);
1531            let end = (rostra_id, kind, EventAuxKey::MAX);
1532            let mut events = singletons
1533                .range(start..=end)?
1534                .map(|entry| {
1535                    let (_, value) = entry?;
1536                    let value = value.value();
1537                    Ok((value.ts, value.inner.event_id))
1538                })
1539                .collect::<DbResult<Vec<_>>>()?;
1540            events.sort_unstable_by_key(|event| std::cmp::Reverse(*event));
1541            Ok(events.into_iter().map(|(_, event_id)| event_id).collect())
1542        })
1543        .await
1544        .expect("Database panic")
1545    }
1546
1547    pub async fn get_id_endpoints(
1548        &self,
1549        id: RostraId,
1550    ) -> BTreeMap<(Timestamp, IrohNodeId), IrohNodeRecord> {
1551        self.write_with(|tx| {
1552            let mut table = tx.open_table(&ids_nodes::TABLE)?;
1553
1554            Self::get_id_endpoints_tx(id, &mut table)
1555        })
1556        .await
1557        .expect("Database panic")
1558    }
1559
1560    /// Register an iroh node endpoint for an identity.
1561    ///
1562    /// Useful for test setups where peer node addresses need to be manually
1563    /// registered.
1564    pub async fn insert_id_node(&self, id: RostraId, node_id: IrohNodeId, ts: Timestamp) {
1565        self.write_with(|tx| {
1566            let mut table = tx.open_table(&ids_nodes::TABLE)?;
1567            table.insert(
1568                &(id, node_id),
1569                &IrohNodeRecord {
1570                    announcement_ts: ts,
1571                    stats: Default::default(),
1572                },
1573            )?;
1574            Ok(())
1575        })
1576        .await
1577        .expect("Database panic")
1578    }
1579
1580    /// Get events for an identity, sorted by timestamp (most recent first).
1581    ///
1582    /// Returns a vector of (EventRecord, Timestamp, EventContentState
1583    /// option) limited to the specified count.
1584    pub async fn get_events_for_id(
1585        &self,
1586        id: RostraId,
1587        limit: usize,
1588    ) -> Vec<(event::EventRecord, Timestamp, Option<EventContentState>)> {
1589        self.read_with(|tx| {
1590            let events_table = tx.open_table(&events::TABLE)?;
1591            let events_by_time_table = tx.open_table(&events_by_time::TABLE)?;
1592            let events_content_state_table = tx.open_table(&events_content_state::TABLE)?;
1593
1594            let mut results = Vec::new();
1595
1596            // Iterate events_by_time in reverse (newest first)
1597            for entry in events_by_time_table.range(..)?.rev() {
1598                if limit <= results.len() {
1599                    break;
1600                }
1601
1602                let entry = entry?;
1603                let (ts, event_id) = entry.0.value();
1604
1605                // Get the event record
1606                let Some(event_record) = events_table.get(&event_id)?.map(|g| g.value()) else {
1607                    continue;
1608                };
1609
1610                // Check if this event belongs to the requested identity
1611                if event_record.signed.event.author != id {
1612                    continue;
1613                }
1614
1615                // Get content state if available
1616                let content_state = events_content_state_table
1617                    .get(&event_id)?
1618                    .map(|g| g.value());
1619
1620                results.push((event_record, ts, content_state));
1621            }
1622
1623            Ok(results)
1624        })
1625        .await
1626        .expect("Database panic")
1627    }
1628
1629    /// Get all identities that have authored retained events.
1630    pub async fn get_known_identities(&self) -> Vec<RostraId> {
1631        self.read_with(ids_full::read_all)
1632            .await
1633            .expect("Database panic")
1634    }
1635
1636    /// Resolve a shortened identity that has authored a retained event.
1637    ///
1638    /// Event ingestion rejects identity-prefix collisions, so a returned
1639    /// identity is unambiguous.
1640    pub async fn get_known_identity(&self, short_id: ShortRostraId) -> Option<RostraId> {
1641        self.read_with(|tx| {
1642            Ok(ids_full::get(tx, short_id)?.map(|rest| RostraId::assemble(short_id, rest)))
1643        })
1644        .await
1645        .expect("Database panic")
1646    }
1647}
1648
1649fn get_first_in_range<K, V>(
1650    events_table: &impl ReadableTable<K, V>,
1651    range: impl ops::RangeBounds<K>,
1652) -> Result<Option<K>, DbError>
1653where
1654    K: bincode::Decode<()> + bincode::Encode,
1655    V: bincode::Decode<()> + bincode::Encode,
1656{
1657    Ok(events_table
1658        .range(range)?
1659        .next()
1660        .transpose()?
1661        .map(|(k, _)| k.value()))
1662}
1663
1664fn get_last_in_range<K, V>(
1665    events_table: &impl ReadableTable<K, V>,
1666    range: impl ops::RangeBounds<K>,
1667) -> Result<Option<K>, DbError>
1668where
1669    K: bincode::Decode<()> + bincode::Encode,
1670    V: bincode::Decode<()> + bincode::Encode,
1671{
1672    Ok(events_table
1673        .range(range)?
1674        .next_back()
1675        .transpose()?
1676        .map(|(k, _)| k.value()))
1677}
1678
1679#[derive(Debug, Clone)]
1680pub enum InsertEventOutcome {
1681    /// The event already existed, so graph, lifecycle, and projection state did
1682    /// not change.
1683    ///
1684    /// Identity registration is still validated and may restore an absent
1685    /// shortened/full mapping.
1686    AlreadyPresent,
1687    Inserted {
1688        /// An event already had a child reporting its existence.
1689        ///
1690        /// This also implies that the event can't be a "head event"
1691        /// as we already have a child of it.
1692        was_missing: bool,
1693        /// This event was already marked as deleted by some processed children
1694        /// event.
1695        ///
1696        /// This also implies that the event can't be a "head event"
1697        /// as we already have a child of it.
1698        is_deleted: bool,
1699        /// An existing parent event had its content marked as deleted by this
1700        /// event.
1701        ///
1702        /// Note, if the parent event was marked for deletion, but it was not
1703        /// processed yet, this will not be set, and instead `is_deleted` will
1704        /// be set to true, when the deleted parent is processed.
1705        deleted_parent: Option<ShortEventId>,
1706        /// Parent content to be reverted.
1707        ///
1708        /// If Some - deletion of the `deleted_parent` is cusing revertion of
1709        /// this content, which should be processed.
1710        reverted_parent_content: Option<EventContentRaw>,
1711
1712        /// Ids of parents we don't have yet, so they are now marked
1713        /// as "missing".
1714        missing_parents: Vec<ShortEventId>,
1715    },
1716}
1717
1718impl InsertEventOutcome {
1719    fn validate(self) -> Self {
1720        if let InsertEventOutcome::Inserted {
1721            deleted_parent,
1722            reverted_parent_content,
1723            ..
1724        } = &self
1725        {
1726            if reverted_parent_content.is_some() {
1727                assert!(deleted_parent.is_some());
1728            }
1729        }
1730        self
1731    }
1732}
1733
1734#[derive(Clone, Copy, PartialEq, Eq, Debug)]
1735pub enum ProcessEventState {
1736    New,
1737    Existing,
1738    Pruned,
1739    Deleted,
1740}
1741
1742pub enum ContentWantState {
1743    Wants,
1744    MaybeWants,
1745    DoesNotWant,
1746}
1747
1748impl ProcessEventState {
1749    pub fn wants_content(self) -> ContentWantState {
1750        match self {
1751            ProcessEventState::New => ContentWantState::Wants,
1752            ProcessEventState::Existing => ContentWantState::MaybeWants,
1753            ProcessEventState::Pruned => ContentWantState::DoesNotWant,
1754            ProcessEventState::Deleted => ContentWantState::DoesNotWant,
1755        }
1756    }
1757}
1758#[cfg(test)]
1759mod content_ingestion_tests;
1760#[cfg(test)]
1761mod deleted_replacement_tests;
1762#[cfg(test)]
1763mod follow_epoch_tests;
1764#[cfg(test)]
1765mod identity_collision_tests;
1766#[cfg(test)]
1767mod reception_order_tests;
1768#[cfg(test)]
1769mod social_post_materialization_tests;
1770#[cfg(test)]
1771mod social_post_projection_tests;
1772#[cfg(test)]
1773mod social_post_receipt_tests;
1774#[cfg(test)]
1775mod tests;