1use super::impl_config_secret::seed_storage_deploy_config;
13use super::*;
14
15pub(crate) fn view_records_to_entities(
24 table: &str,
25 records: &[crate::storage::query::unified::UnifiedRecord],
26) -> Vec<crate::storage::UnifiedEntity> {
27 use std::collections::HashMap;
28 let table_arc: std::sync::Arc<str> = std::sync::Arc::from(table);
29 let mut out = Vec::with_capacity(records.len());
30 for record in records {
31 let mut named: HashMap<String, reddb_types::Value> = HashMap::new();
32 for (name, value) in record.iter_fields() {
33 named.insert(name.to_string(), value.clone());
34 }
35 let entity = crate::storage::UnifiedEntity::new(
36 crate::storage::EntityId::new(0),
37 crate::storage::EntityKind::TableRow {
38 table: std::sync::Arc::clone(&table_arc),
39 row_id: 0,
40 },
41 crate::storage::EntityData::Row(crate::storage::RowData {
42 columns: Vec::new(),
43 named: Some(named),
44 schema: None,
45 }),
46 );
47 out.push(entity);
48 }
49 out
50}
51
52fn system_keyed_collection_contract(
53 name: &str,
54 model: crate::catalog::CollectionModel,
55) -> crate::physical::CollectionContract {
56 let now = crate::utils::now_unix_millis() as u128;
57 crate::physical::CollectionContract {
58 name: name.to_string(),
59 declared_model: model,
60 schema_mode: crate::catalog::SchemaMode::Dynamic,
61 origin: crate::physical::ContractOrigin::Implicit,
62 version: 1,
63 created_at_unix_ms: now,
64 updated_at_unix_ms: now,
65 default_ttl_ms: None,
66 vector_dimension: None,
67 vector_metric: None,
68 context_index_fields: Vec::new(),
69 declared_columns: Vec::new(),
70 table_def: None,
71 timestamps_enabled: false,
72 context_index_enabled: false,
73 metrics_raw_retention_ms: None,
74 metrics_rollup_policies: Vec::new(),
75 metrics_tenant_identity: None,
76 metrics_namespace: None,
77 append_only: false,
78 subscriptions: Vec::new(),
79 analytics_config: Vec::new(),
80 session_key: None,
81 session_gap_ms: None,
82 retention_duration_ms: None,
83 analytical_storage: None,
84
85 ai_policy: None,
86 }
87}
88
89pub(crate) fn table_row_index_fields(
90 entity: &crate::storage::unified::entity::UnifiedEntity,
91) -> Vec<(String, reddb_types::Value)> {
92 let crate::storage::EntityData::Row(row) = &entity.data else {
93 return Vec::new();
94 };
95 if let Some(named) = &row.named {
96 return named
97 .iter()
98 .map(|(name, value)| (name.clone(), value.clone()))
99 .collect();
100 }
101 if let Some(schema) = &row.schema {
102 return schema
103 .iter()
104 .zip(row.columns.iter())
105 .map(|(name, value)| (name.clone(), value.clone()))
106 .collect();
107 }
108 Vec::new()
109}
110
111impl RedDBRuntime {
112 pub fn in_memory() -> RedDBResult<Self> {
113 Self::with_options(RedDBOptions::in_memory())
114 }
115
116 pub fn flush(&self) -> RedDBResult<()> {
117 self.inner
118 .db
119 .flush()
120 .map_err(|err| RedDBError::Internal(err.to_string()))
121 }
122
123 pub fn lock_manager(&self) -> std::sync::Arc<crate::runtime::lock_manager::LockManager> {
127 self.inner.lock_manager.clone()
128 }
129
130 pub fn config_registry(&self) -> std::sync::Arc<crate::auth::registry::ConfigRegistry> {
132 self.inner.config_registry.clone()
133 }
134
135 pub fn query_audit(&self) -> std::sync::Arc<crate::runtime::query_audit::QueryAuditStream> {
136 self.inner.query_audit.clone()
137 }
138
139 pub fn control_events_require_persistence(&self) -> bool {
140 self.inner.control_event_config.require_persistence()
141 }
142
143 pub fn control_event_config(&self) -> crate::runtime::control_events::ControlEventConfig {
144 self.inner.control_event_config
145 }
146
147 pub fn control_event_ledger(
148 &self,
149 ) -> Arc<dyn crate::runtime::control_events::ControlEventLedger> {
150 self.inner.control_event_ledger.read().clone()
151 }
152
153 #[doc(hidden)]
154 pub fn replace_control_event_ledger_for_tests(
155 &self,
156 ledger: Arc<dyn crate::runtime::control_events::ControlEventLedger>,
157 ) {
158 *self.inner.control_event_ledger.write() = ledger;
159 }
160
161 #[inline(never)]
162 pub fn with_options(options: RedDBOptions) -> RedDBResult<Self> {
163 Self::with_pool(options, ConnectionPoolConfig::default())
164 }
165
166 pub fn memory_budget(&self) -> crate::storage::memory_budget::MemoryBudget {
170 self.inner.memory_budget
171 }
172
173 pub fn memory_accounting(&self) -> &crate::storage::memory_pools::MemoryAccounting {
177 &self.inner.memory_accounting
178 }
179
180 pub fn with_pool(
181 options: RedDBOptions,
182 pool_config: ConnectionPoolConfig,
183 ) -> RedDBResult<Self> {
184 Self::with_pool_and_limits(
185 options,
186 pool_config,
187 super::resource_limits::ResourceLimits::from_env(),
188 )
189 }
190
191 pub(super) fn with_pool_and_limits(
192 options: RedDBOptions,
193 pool_config: ConnectionPoolConfig,
194 resource_limits: super::resource_limits::ResourceLimits,
195 ) -> RedDBResult<Self> {
196 options
197 .replication
198 .validate_failover_profile()
199 .map_err(RedDBError::InvalidConfig)?;
200 let boot_open_start_ms = std::time::SystemTime::now()
208 .duration_since(std::time::UNIX_EPOCH)
209 .map(|d| d.as_millis() as u64)
210 .unwrap_or(0);
211 let memory_budget = crate::storage::memory_budget::resolve_for_boot(
215 options.storage_profile.deploy_profile,
216 options.memory_budget_bytes,
217 )
218 .map_err(|err| RedDBError::InvalidConfig(err.to_string()))?;
219 crate::storage::memory_budget::log_resolved_once(&memory_budget);
220 let memory_shares = crate::storage::memory_pools::BudgetShares::resolve(
224 memory_budget,
225 options.storage_profile.deploy_profile,
226 );
227 memory_shares.log_once();
228 let embedded_single_file = options.storage_profile.deploy_profile
229 == crate::storage::DeployProfile::Embedded
230 && options.storage_profile.packaging == crate::storage::StoragePackaging::SingleFile;
231 let db = Arc::new(RedDB::open_with_options(&options).map_err(|err| {
232 match err.downcast::<crate::storage::StoreError>() {
233 Ok(store_err) => RedDBError::from(*store_err),
234 Err(err) => RedDBError::Internal(err.to_string()),
235 }
236 })?);
237 let result_blob_cache_config = if embedded_single_file {
241 crate::storage::cache::BlobCacheConfig::default()
242 } else {
243 crate::storage::cache::BlobCacheConfig::default().with_l2_path(
244 reddb_file::layout::result_cache_l2_path(
245 &options.resolved_path(reddb_file::default_database_path()),
246 ),
247 )
248 }
249 .with_l1_bytes_max(memory_shares.blob_cache_l1_bytes());
250 let result_blob_cache =
251 crate::storage::cache::BlobCache::open_with_l2(result_blob_cache_config).map_err(
252 |err| RedDBError::Internal(format!("open result Blob Cache L2 failed: {err:?}")),
253 )?;
254 let storage_ready_ms = std::time::SystemTime::now()
255 .duration_since(std::time::UNIX_EPOCH)
256 .map(|d| d.as_millis() as u64)
257 .unwrap_or(0);
258
259 let no_auth_boot = crate::service_cli::no_auth_active(&options);
260 let runtime = Self {
261 background_tasks: Some(Arc::new(background_tasks::RuntimeBackgroundTasks::default())),
262 inner: Arc::new(RuntimeInner {
263 functions: parking_lot::RwLock::new(
264 super::function_catalog::FunctionCatalog::load(&db)?,
265 ),
266 db: db.clone(),
267 layout: PhysicalLayout::from_options(&options),
268 embedded_single_file,
269 memory_budget,
270 memory_accounting: Arc::new(
271 crate::storage::memory_pools::MemoryAccounting::from_shares(
272 memory_budget,
273 memory_shares,
274 ),
275 ),
276 memory_reservations: parking_lot::Mutex::new(
277 super::memory_admission::MemoryReservations::default(),
278 ),
279 indices: IndexCatalog::register_default_vector_graph(
280 options.has_capability(crate::api::Capability::Table),
281 options.has_capability(crate::api::Capability::Graph),
282 ),
283 pool_config,
284 pool: Mutex::new(PoolState::default()),
285 started_at_unix_ms: SystemTime::now()
286 .duration_since(UNIX_EPOCH)
287 .unwrap_or_default()
288 .as_millis(),
289 probabilistic: super::probabilistic_store::ProbabilisticStore::new(),
290 index_store: super::index_store::IndexStore::new(),
291 cdc: crate::replication::cdc::CdcBuffer::new(100_000),
292 checkpoint_projection_stats: super::CheckpointProjectionStats::default(),
293 scrub_state: parking_lot::Mutex::new(super::ScrubRuntimeState::default()),
294 checkpoint_columnar_emission_budget_chunks: options
295 .checkpoint_columnar_emission_budget_chunks,
296 columnar_projection_size_floor_rows: options.columnar_projection_size_floor_rows,
297 backup_scheduler: crate::replication::scheduler::BackupScheduler::new(3600),
298 query_cache: parking_lot::RwLock::new(
299 crate::storage::query::planner::cache::PlanCache::new(1000),
300 ),
301 result_cache: parking_lot::RwLock::new((
302 HashMap::new(),
303 std::collections::VecDeque::new(),
304 )),
305 result_blob_cache,
306 result_blob_entries: parking_lot::RwLock::new((
307 HashMap::new(),
308 std::collections::VecDeque::new(),
309 )),
310 ask_answer_cache_entries: parking_lot::RwLock::new((
311 HashSet::new(),
312 std::collections::VecDeque::new(),
313 )),
314 result_cache_shadow_divergences: std::sync::atomic::AtomicU64::new(0),
315 result_cache_hits: std::sync::atomic::AtomicU64::new(0),
316 result_cache_misses: std::sync::atomic::AtomicU64::new(0),
317 result_cache_evictions: std::sync::atomic::AtomicU64::new(0),
318 ask_daily_spend: parking_lot::RwLock::new(HashMap::new()),
319 queue_message_locks: parking_lot::RwLock::new(HashMap::new()),
320 rmw_locks: RmwLockTable::new(),
321 planner_dirty_tables: parking_lot::RwLock::new(HashSet::new()),
322 ec_registry: Arc::new(crate::ec::config::EcRegistry::new()),
323 config_registry: Arc::new(crate::auth::registry::ConfigRegistry::new()),
324 ec_worker: crate::ec::worker::EcWorker::new(),
325 auth_store: parking_lot::RwLock::new(None),
326 oauth_validator: parking_lot::RwLock::new(None),
327 browser_token_authority: parking_lot::RwLock::new(None),
328 views: parking_lot::RwLock::new(HashMap::new()),
329 materialized_views: parking_lot::RwLock::new(
330 crate::storage::cache::result::MaterializedViewCache::new(),
331 ),
332 retention_sweeper: parking_lot::RwLock::new(
333 crate::runtime::retention_sweeper::RetentionSweeperState::new(),
334 ),
335 transaction_state: crate::runtime::transaction_state::TransactionState::new(),
336 env_config_overrides: crate::runtime::config_overlay::collect_env_overrides(),
337 lock_manager: Arc::new({
338 let env = crate::runtime::config_overlay::collect_env_overrides();
343 let timeout_ms = env
344 .get("concurrency.locking.deadlock_timeout_ms")
345 .and_then(|raw| raw.parse::<u64>().ok())
346 .unwrap_or_else(|| {
347 match crate::runtime::config_matrix::default_for(
348 "concurrency.locking.deadlock_timeout_ms",
349 ) {
350 Some(crate::serde_json::Value::Number(n)) => n as u64,
351 _ => 5000,
352 }
353 });
354 let cfg = crate::runtime::lock_manager::LockConfig {
355 default_timeout: std::time::Duration::from_millis(timeout_ms),
356 ..Default::default()
357 };
358 crate::runtime::lock_manager::LockManager::new(cfg)
359 }),
360 rls_policies: parking_lot::RwLock::new(HashMap::new()),
361 rls_enabled_tables: parking_lot::RwLock::new(HashSet::new()),
362 foreign_tables: Arc::new(crate::storage::fdw::ForeignTableRegistry::with_builtins()),
363 pending_tombstones: parking_lot::RwLock::new(HashMap::new()),
364 pending_versioned_updates: parking_lot::RwLock::new(HashMap::new()),
365 pending_queue_dedup: parking_lot::RwLock::new(HashMap::new()),
366 pending_kv_watch_events: parking_lot::RwLock::new(HashMap::new()),
367 pending_vault_writes: parking_lot::RwLock::new(HashMap::new()),
368 pending_store_wal_actions: parking_lot::RwLock::new(HashMap::new()),
369 pending_claim_locks: parking_lot::RwLock::new(HashMap::new()),
370 queue_wait_registry: std::sync::Arc::new(
371 crate::runtime::queue_wait_registry::QueueWaitRegistry::new(),
372 ),
373 pending_queue_wakes: parking_lot::RwLock::new(HashMap::new()),
374 tenant_tables: parking_lot::RwLock::new(HashMap::new()),
375 ddl_epoch: std::sync::atomic::AtomicU64::new(0),
376 write_gate: Arc::new(crate::runtime::write_gate::WriteGate::from_options(
377 &options,
378 )),
379 lifecycle: crate::runtime::lifecycle::Lifecycle::new(),
380 resource_limits,
381 query_budget_slots: AtomicU64::new(0),
382 audit_log: {
383 let data_path = if embedded_single_file {
399 std::env::temp_dir()
400 .join("reddb-embedded-runtime")
401 .join(format!("audit-{}", std::process::id()))
402 } else {
403 options
404 .data_path
405 .clone()
406 .unwrap_or_else(|| std::env::temp_dir().join("reddb"))
407 };
408 let (audit_dest, _) = crate::api::tier_wiring::current_log_destinations();
409 if !matches!(audit_dest, crate::storage::layout::LogDestination::File(_))
410 && (embedded_single_file
411 || options
412 .metadata
413 .contains_key(crate::api::EPHEMERAL_RUNTIME_METADATA_KEY))
414 {
415 let audit_path = reddb_file::layout::sibling_path(
425 &data_path,
426 &reddb_file::layout::sidecar_file_name(&data_path, "audit.log"),
427 );
428 Arc::new(crate::runtime::audit_log::AuditLogger::with_path(
429 audit_path,
430 ))
431 } else {
432 Arc::new(crate::runtime::audit_log::AuditLogger::for_destination(
433 &audit_dest,
434 &data_path,
435 ))
436 }
437 },
438 control_event_ledger: parking_lot::RwLock::new(Arc::new(
439 crate::runtime::control_events::RuntimeLedger::new(db.store()),
440 )),
441 control_event_config: options.control_events,
442 query_audit: Arc::new(crate::runtime::query_audit::QueryAuditStream::new(
443 db.store(),
444 options.query_audit.clone(),
445 )),
446 lease_lifecycle: std::sync::OnceLock::new(),
447 replica_apply_metrics: std::sync::Arc::new(
448 crate::replication::logical::ReplicaApplyMetrics::default(),
449 ),
450 replica_link_metrics: std::sync::Arc::new(
451 crate::replication::reconnect::ReplicaLinkMetrics::default(),
452 ),
453 quota_bucket: crate::runtime::quota_bucket::QuotaBucket::from_env(),
454 schema_vocabulary: parking_lot::RwLock::new(
455 crate::runtime::schema_vocabulary::SchemaVocabulary::new(),
456 ),
457 slow_query_logger: {
458 let fallback_dir = if embedded_single_file {
475 std::env::temp_dir()
476 .join("reddb-embedded-runtime")
477 .join(format!("slow-{}", std::process::id()))
478 } else {
479 options
480 .data_path
481 .as_ref()
482 .and_then(|p| p.parent().map(std::path::PathBuf::from))
483 .unwrap_or_else(|| std::env::temp_dir().join("reddb"))
484 };
485 let threshold_ms = std::env::var("RED_SLOW_QUERY_THRESHOLD_MS")
486 .ok()
487 .and_then(|s| s.parse::<u64>().ok())
488 .unwrap_or(1000);
489 let sample_pct = std::env::var("RED_SLOW_QUERY_SAMPLE_PCT")
490 .ok()
491 .and_then(|s| s.parse::<u8>().ok())
492 .unwrap_or(100);
493 let (_, slow_dest) = crate::api::tier_wiring::current_log_destinations();
494 crate::telemetry::slow_query_logger::SlowQueryLogger::for_destination(
495 &slow_dest,
496 &fallback_dir,
497 threshold_ms,
498 sample_pct,
499 )
500 },
501 slow_query_store: crate::telemetry::slow_query_store::SlowQueryStore::new(
502 crate::telemetry::slow_query_store::DEFAULT_CAP,
503 ),
504 kv_stats: crate::runtime::KvStatsCounters::default(),
505 metrics_ingest_stats: crate::runtime::MetricsIngestCounters::default(),
506 metrics_tenant_activity_stats:
507 crate::runtime::MetricsTenantActivityCounters::default(),
508 claim_telemetry: Arc::new(
509 crate::runtime::claim_telemetry::ClaimTelemetryCounters::default(),
510 ),
511 queue_telemetry: Arc::new(
512 crate::runtime::queue_telemetry::QueueTelemetryCounters::default(),
513 ),
514 query_latency_telemetry: Arc::new(
515 crate::runtime::query_latency_telemetry::QueryLatencyTelemetry::default(),
516 ),
517 occupancy_sampler: Arc::new(
518 crate::runtime::occupancy_sampler::OccupancySampler::new(),
519 ),
520 node_load_telemetry: Arc::new(
521 crate::runtime::node_load_telemetry::NodeLoadTelemetry::default(),
522 ),
523 queue_presence: Arc::new(
524 crate::storage::queue::presence::ConsumerPresenceRegistry::new(),
525 ),
526 vector_introspection: Arc::new(
527 crate::storage::vector::introspection::VectorIntrospectionRegistry::new(),
528 ),
529 kv_tag_index: crate::runtime::KvTagIndex::default(),
530 chain_tip_cache: parking_lot::Mutex::new(HashMap::new()),
531 chain_integrity_broken: parking_lot::Mutex::new(HashMap::new()),
532 integrity_tombstones: parking_lot::Mutex::new(Vec::new()),
533 integrity_tombstones_state: std::sync::atomic::AtomicU8::new(0),
534 }),
535 };
536
537 crate::telemetry::operator_event::install_global_audit_sink(Arc::clone(
543 &runtime.inner.audit_log,
544 ));
545
546 runtime
549 .inner
550 .slow_query_logger
551 .attach_store(Arc::clone(&runtime.inner.slow_query_store));
552
553 runtime
561 .inner
562 .lifecycle
563 .set_restore_started_at_ms(boot_open_start_ms);
564 runtime
565 .inner
566 .lifecycle
567 .set_restore_ready_at_ms(storage_ready_ms);
568 runtime
569 .inner
570 .lifecycle
571 .set_wal_replay_started_at_ms(boot_open_start_ms);
572 runtime
573 .inner
574 .lifecycle
575 .set_wal_replay_ready_at_ms(storage_ready_ms);
576
577 let restored_cdc_lsn = runtime
578 .inner
579 .db
580 .replication
581 .as_ref()
582 .map(|repl| {
583 repl.logical_wal_spool
584 .as_ref()
585 .map(|spool| spool.current_lsn())
586 .unwrap_or(0)
587 })
588 .unwrap_or(0)
589 .max(runtime.config_u64("red.config.timeline.last_archived_lsn", 0));
590 runtime.inner.cdc.set_current_lsn(restored_cdc_lsn);
591 runtime.rehydrate_snapshot_xid_floor();
592 assert!(
593 runtime
594 .inner
595 .db
596 .store()
597 .snapshot_manager
598 .set(runtime.snapshot_manager())
599 .is_ok(),
600 "a new runtime owns the store transaction outcomes"
601 );
602 runtime
603 .bootstrap_system_keyed_collections()
604 .map_err(|err| RedDBError::Internal(format!("bootstrap system collections: {err}")))?;
605 runtime.rehydrate_declared_column_schemas();
606 runtime.rehydrate_runtime_index_registry()?;
607 runtime.rehydrate_kv_tags()?;
608 runtime
609 .load_probabilistic_state()
610 .map_err(|err| RedDBError::Internal(format!("load probabilistic state: {err}")))?;
611
612 runtime.rehydrate_tenant_tables();
616 runtime.rehydrate_materialized_view_descriptors();
621 if let Some(repl) = &runtime.inner.db.replication {
622 repl.wal_buffer.set_current_lsn(restored_cdc_lsn);
623 }
624
625 {
627 let sys = SystemInfo::collect();
628 runtime.inner.db.store().set_config_tree(
629 "red.system",
630 &crate::serde_json::json!({
631 "pid": sys.pid,
632 "cpu_cores": sys.cpu_cores,
633 "total_memory_bytes": sys.total_memory_bytes,
634 "available_memory_bytes": sys.available_memory_bytes,
635 "os": sys.os,
636 "arch": sys.arch,
637 "hostname": sys.hostname,
638 "started_at": SystemTime::now()
639 .duration_since(UNIX_EPOCH)
640 .unwrap_or_default()
641 .as_millis() as u64
642 }),
643 );
644
645 let store = runtime.inner.db.store();
647 if store
648 .get_collection("red_config")
649 .map(|m| m.query_all(|_| true).len())
650 .unwrap_or(0)
651 <= 10
652 {
653 store.set_config_tree("red.ai", &crate::json!({
654 "default": crate::json!({
655 "provider": "openai",
656 "model": crate::ai::DEFAULT_OPENAI_PROMPT_MODEL
657 }),
658 "max_embedding_inputs": 256,
659 "max_prompt_batch": 256,
660 "timeout": crate::json!({ "connect_secs": 10, "read_secs": 90, "write_secs": 30 })
661 }));
662 store.set_config_tree(
663 "red.server",
664 &crate::json!({
665 "max_scan_limit": 1000,
666 "max_body_size": 1048576,
667 "read_timeout_ms": 5000,
668 "write_timeout_ms": 5000
669 }),
670 );
671 store.set_config_tree(
672 "red.storage",
673 &crate::json!({
674 "page_size": 4096,
675 "page_cache_capacity": 100000,
676 "auto_checkpoint_pages": 1000,
677 "snapshot_retention": 16,
678 "verify_checksums": true,
679 "segment": crate::json!({
680 "max_entities": 100000,
681 "max_bytes": 268435456_u64,
682 "compression_level": 6
683 }),
684 "hnsw": crate::json!({ "m": 16, "ef_construction": 100, "ef_search": 50 }),
685 "ivf": crate::json!({ "n_lists": 100, "n_probes": 10 }),
686 "bm25": crate::json!({ "k1": 1.2, "b": 0.75 })
687 }),
688 );
689 store.set_config_tree(
690 "red.search",
691 &crate::json!({
692 "rag": crate::json!({
693 "max_chunks_per_source": 10,
694 "max_total_chunks": 25,
695 "similarity_threshold": 0.8,
696 "graph_depth": 2,
697 "min_relevance": 0.3
698 }),
699 "fusion": crate::json!({
700 "vector_weight": 0.5,
701 "graph_weight": 0.3,
702 "table_weight": 0.2,
703 "dedup_threshold": 0.85
704 })
705 }),
706 );
707 store.set_config_tree(
708 "red.auth",
709 &crate::json!({
710 "enabled": false,
711 "session_ttl_secs": 3600,
712 "require_auth": false
713 }),
714 );
715 store.set_config_tree(
716 "red.query",
717 &crate::json!({
718 "connection_pool": crate::json!({ "max_connections": 64, "max_idle": 16 }),
719 "max_recursion_depth": 1000
720 }),
721 );
722 store.set_config_tree(
723 "red.indexes",
724 &crate::json!({
725 "auto_select": true,
726 "bloom_filter": crate::json!({
727 "enabled": true,
728 "false_positive_rate": 0.01,
729 "prune_on_scan": true
730 }),
731 "hash": crate::json!({ "enabled": true }),
732 "bitmap": crate::json!({ "enabled": true, "max_cardinality": 1000 }),
733 "spatial": crate::json!({ "enabled": true })
734 }),
735 );
736 store.set_config_tree(
737 "red.probabilistic",
738 &crate::json!({
739 "hll_registers": 16384,
740 "sketch_default_width": 1000,
741 "sketch_default_depth": 5,
742 "filter_default_capacity": 100000
743 }),
744 );
745 store.set_config_tree(
746 "red.timeseries",
747 &crate::json!({
748 "default_chunk_size": 1024,
749 "compression": crate::json!({
750 "timestamps": "delta_of_delta",
751 "values": "gorilla_xor"
752 }),
753 "default_retention_days": 0
754 }),
755 );
756 store.set_config_tree(
757 "red.queue",
758 &crate::json!({
759 "default_max_size": 0,
760 "default_max_attempts": 3,
761 "visibility_timeout_ms": 30000,
762 "consumer_idle_timeout_ms": 60000
763 }),
764 );
765 store.set_config_tree(
766 "red.backup",
767 &crate::json!({
768 "enabled": false,
769 "interval_secs": 3600,
770 "retention_count": 24,
771 "upload": false,
772 "backend": "local"
773 }),
774 );
775 store.set_config_tree(
776 "red.wal",
777 &crate::json!({
778 "archive": crate::json!({
779 "enabled": false,
780 "retention_hours": 168,
781 "prefix": reddb_file::backup_wal_prefix("")
782 })
783 }),
784 );
785 store.set_config_tree(
786 "red.cdc",
787 &crate::json!({
788 "enabled": true,
789 "buffer_size": 100000
790 }),
791 );
792 store.set_config_tree(
793 "red.config.secret",
794 &crate::json!({
795 "auto_encrypt": true,
796 "auto_decrypt": true
797 }),
798 );
799 }
800
801 crate::runtime::config_matrix::heal_critical_keys(store.as_ref());
808 seed_storage_deploy_config(store.as_ref(), options.storage_profile);
809
810 let lehman_yao = runtime.config_bool("storage.btree.lehman_yao", true);
817 crate::storage::engine::btree::lehman_yao::set_enabled(lehman_yao);
818 if lehman_yao {
819 tracing::info!(
820 "storage.btree.lehman_yao=true — lock-free concurrent descent enabled"
821 );
822 }
823
824 let overlay_path = crate::runtime::config_overlay::config_file_path();
829 let _ =
830 crate::runtime::config_overlay::apply_config_file(store.as_ref(), &overlay_path);
831 }
832
833 {
837 let store = runtime.inner.db.store();
838 for name in crate::application::vcs_collections::ALL {
839 let _ = store.get_or_create_collection(*name);
840 }
841 store.set_config_tree(
844 crate::application::vcs_collections::CONFIG_NAMESPACE,
845 &crate::json!({
846 "default_branch": "main",
847 "author": crate::json!({
848 "name": "reddb",
849 "email": "reddb@localhost"
850 }),
851 "protected_branches": crate::json!(["main"]),
852 "closure": crate::json!({
853 "enabled": true,
854 "lazy": true
855 }),
856 "merge": crate::json!({
857 "default_strategy": "auto",
858 "fast_forward": true
859 })
860 }),
861 );
862 }
863
864 {
867 let store = runtime.inner.db.store();
868 for name in crate::application::migration_collections::ALL {
869 let _ = store.get_or_create_collection(*name);
870 }
871 }
872
873 let _ = crate::application::topology_collections::ensure(&runtime);
877
878 runtime
886 .inner
887 .db
888 .store()
889 .register_entity_id(crate::storage::EntityId::new(
890 crate::storage::FIRST_USER_ENTITY_ID - 1,
891 ));
892
893 {
907 let weak = Arc::downgrade(&runtime.inner);
908 runtime
909 .background_tasks
910 .as_ref()
911 .expect("foreground runtime")
912 .spawn(
913 std::thread::Builder::new().name("reddb-maintenance".into()),
914 move |stopped| {
915 let tick = std::time::Duration::from_millis(200);
916 let work_interval = std::time::Duration::from_secs(60);
917 let mut last_work = std::time::Instant::now();
918 let mut mvcc_maintenance =
919 super::mvcc_maintenance::MvccMaintenance::default();
920 loop {
921 std::thread::park_timeout(tick);
922 if stopped.load(AtomicOrdering::Acquire) {
923 break;
924 }
925 let Some(inner) = weak.upgrade() else {
926 break;
929 };
930 if last_work.elapsed() >= work_interval {
931 let _stats = inner.db.store().context_index().stats();
932 last_work = std::time::Instant::now();
933 }
934 let rt = RedDBRuntime {
935 background_tasks: None,
936 inner,
937 };
938 rt.reclaim_history_tick(&mut mvcc_maintenance);
939 }
940 },
941 );
942 }
943
944 {
946 let store = runtime.inner.db.store();
947 let mut backup_enabled = false;
948 let mut backup_interval = 3600u64;
949
950 if let Some(manager) = store.get_collection("red_config") {
951 manager.for_each_entity(|entity| {
952 if let Some(row) = entity.data.as_row() {
953 let key = row.get_field("key").and_then(|v| match v {
954 reddb_types::Value::Text(s) => Some(s.as_ref()),
955 _ => None,
956 });
957 let val = row.get_field("value");
958 if key == Some("red.config.backup.enabled") {
959 backup_enabled = match val {
960 Some(reddb_types::Value::Boolean(true)) => true,
961 Some(reddb_types::Value::Text(s)) => &**s == "true",
962 _ => false,
963 };
964 } else if key == Some("red.config.backup.interval_secs") {
965 if let Some(reddb_types::Value::Integer(n)) = val {
966 backup_interval = *n as u64;
967 }
968 }
969 }
970 true
971 });
972 }
973
974 if backup_enabled {
975 runtime.inner.backup_scheduler.set_interval(backup_interval);
976 let rt = runtime.clone();
977 runtime
978 .inner
979 .backup_scheduler
980 .start(move || rt.trigger_backup().map_err(|e| format!("{}", e)));
981 }
982 }
983
984 {
986 runtime
987 .inner
988 .ec_registry
989 .load_from_config_store(runtime.inner.db.store().as_ref());
990 if !runtime.inner.ec_registry.async_configs().is_empty() {
991 runtime.inner.ec_worker.start(
992 Arc::clone(&runtime.inner.ec_registry),
993 Arc::clone(&runtime.inner.db.store()),
994 );
995 }
996 }
997
998 if let (true, crate::replication::ReplicationRole::Replica { primary_addr }) = (
999 runtime.inner.db.options().replica_loop_enabled,
1000 runtime.inner.db.options().replication.role.clone(),
1001 ) {
1002 let rt = runtime.clone();
1003 std::thread::Builder::new()
1004 .name("reddb-replica".into())
1005 .spawn(move || rt.run_replica_loop(primary_addr))
1006 .ok();
1007 }
1008
1009 runtime.inner.lifecycle.mark_ready();
1014
1015 {
1024 let weak_inner = Arc::downgrade(&runtime.inner);
1025 runtime
1026 .background_tasks
1027 .as_ref()
1028 .expect("foreground runtime")
1029 .spawn(
1030 std::thread::Builder::new()
1031 .name("reddb-mv-scheduler".into())
1032 .stack_size(8 * 1024 * 1024),
1037 move |stopped| loop {
1038 std::thread::park_timeout(std::time::Duration::from_millis(50));
1039 if stopped.load(AtomicOrdering::Acquire) {
1040 break;
1041 }
1042 let Some(inner) = weak_inner.upgrade() else {
1043 break;
1044 };
1045 let rt = RedDBRuntime {
1046 background_tasks: None,
1047 inner,
1048 };
1049 rt.refresh_due_materialized_views();
1050 },
1051 );
1052 }
1053
1054 if !runtime.write_gate().is_read_only() {
1064 let weak_inner = Arc::downgrade(&runtime.inner);
1065 runtime
1066 .background_tasks
1067 .as_ref()
1068 .expect("foreground runtime")
1069 .spawn(
1070 std::thread::Builder::new().name("reddb-retention-sweeper".into()),
1071 move |stopped| loop {
1072 std::thread::park_timeout(std::time::Duration::from_millis(500));
1073 if stopped.load(AtomicOrdering::Acquire) {
1074 break;
1075 }
1076 let Some(inner) = weak_inner.upgrade() else {
1077 break;
1078 };
1079 let rt = RedDBRuntime {
1080 background_tasks: None,
1081 inner,
1082 };
1083 rt.sweep_retention_tick(
1084 crate::runtime::retention_sweeper::DEFAULT_SWEEPER_BATCH,
1085 );
1086 },
1087 );
1088 }
1089
1090 if no_auth_boot {
1094 use crate::runtime::control_events::{
1095 ActorRef, ControlEvent, ControlEventCtx, EventKind, Outcome,
1096 };
1097 let ledger = std::sync::Arc::clone(&runtime.inner.control_event_ledger.read());
1098 let ctx = ControlEventCtx {
1099 actor: ActorRef::System("boot"),
1100 scope: None,
1101 request_id: None,
1102 trace_id: None,
1103 };
1104 let event = ControlEvent {
1105 kind: EventKind::AuthDisabled,
1106 outcome: Outcome::Allowed,
1107 action: std::borrow::Cow::Borrowed("auth.disabled"),
1108 resource: Some("server".to_string()),
1109 reason: Some("--no-auth / --dev".to_string()),
1110 matched_policy_id: None,
1111 fields: std::collections::HashMap::new(),
1112 };
1113 if let Err(err) = ledger.emit(&ctx, event) {
1114 tracing::warn!(
1115 target: "reddb::security",
1116 error = %err,
1117 "could not record the auth-disabled boot in the control-event ledger"
1118 );
1119 }
1120 }
1121
1122 Ok(runtime)
1123 }
1124
1125 fn rehydrate_snapshot_xid_floor(&self) {
1126 let store = self.inner.db.store();
1127 for collection in store.list_collections() {
1128 let Some(manager) = store.get_collection(&collection) else {
1129 continue;
1130 };
1131 for entity in manager.query_all(|_| true) {
1132 let snapshot_manager = self.inner.transaction_state.snapshot_manager();
1133 snapshot_manager.observe_committed_xid(entity.xmin);
1134 snapshot_manager.observe_committed_xid(entity.xmax);
1135 }
1136 }
1137 }
1138
1139 pub(crate) fn ensure_materialized_view_backing(&self, name: &str) -> RedDBResult<()> {
1152 let store = self.inner.db.store();
1153 let mut changed = false;
1154 if store.get_collection(name).is_none() {
1155 store.get_or_create_collection(name);
1156 changed = true;
1157 }
1158 if self.inner.db.collection_contract(name).is_none() {
1159 self.inner
1160 .db
1161 .save_collection_contract(system_keyed_collection_contract(
1162 name,
1163 crate::catalog::CollectionModel::Table,
1164 ))
1165 .map_err(|err| RedDBError::Internal(err.to_string()))?;
1166 changed = true;
1167 }
1168 if changed {
1169 self.inner
1170 .db
1171 .persist_metadata()
1172 .map_err(|err| RedDBError::Internal(err.to_string()))?;
1173 }
1174 Ok(())
1175 }
1176
1177 pub(crate) fn drop_materialized_view_backing(&self, name: &str) -> RedDBResult<()> {
1182 let store = self.inner.db.store();
1183 if store.get_collection(name).is_none() {
1184 return Ok(());
1185 }
1186 store
1187 .drop_collection(name)
1188 .map_err(|err| RedDBError::Internal(err.to_string()))?;
1189 if self.inner.db.collection_contract(name).is_some() {
1192 self.inner
1193 .db
1194 .remove_collection_contract(name)
1195 .map_err(|err| RedDBError::Internal(err.to_string()))?;
1196 }
1197 self.invalidate_result_cache();
1198 self.inner
1199 .db
1200 .persist_metadata()
1201 .map_err(|err| RedDBError::Internal(err.to_string()))?;
1202 Ok(())
1203 }
1204
1205 fn bootstrap_system_keyed_collections(&self) -> RedDBResult<()> {
1206 let mut changed = false;
1207 for (name, model) in [
1208 ("red.config", crate::catalog::CollectionModel::Config),
1209 ("red.vault", crate::catalog::CollectionModel::Vault),
1210 (
1214 crate::runtime::continuous_materialized_view::CATALOG_COLLECTION,
1215 crate::catalog::CollectionModel::Config,
1216 ),
1217 ] {
1218 if self.inner.db.store().get_collection(name).is_none() {
1219 self.inner.db.store().get_or_create_collection(name);
1220 changed = true;
1221 }
1222 if self.inner.db.collection_contract(name).is_none() {
1223 self.inner
1224 .db
1225 .save_collection_contract(system_keyed_collection_contract(name, model))
1226 .map_err(|err| RedDBError::Internal(err.to_string()))?;
1227 changed = true;
1228 }
1229 }
1230 if changed {
1231 self.inner
1232 .db
1233 .persist_metadata()
1234 .map_err(|err| RedDBError::Internal(err.to_string()))?;
1235 }
1236 Ok(())
1237 }
1238
1239 pub fn db(&self) -> Arc<RedDB> {
1240 Arc::clone(&self.inner.db)
1241 }
1242
1243 pub fn index_store_ref(&self) -> &super::index_store::IndexStore {
1248 &self.inner.index_store
1249 }
1250
1251 pub(crate) fn schema_vocabulary_apply(
1256 &self,
1257 event: crate::runtime::schema_vocabulary::DdlEvent,
1258 ) {
1259 self.inner.schema_vocabulary.write().on_ddl(event);
1260 }
1261
1262 pub fn schema_vocabulary_lookup(
1267 &self,
1268 token: &str,
1269 ) -> Vec<crate::runtime::schema_vocabulary::VocabHit> {
1270 self.inner.schema_vocabulary.read().lookup(token).to_vec()
1271 }
1272
1273 pub fn set_auth_store(&self, store: Arc<crate::auth::store::AuthStore>) {
1277 *self.inner.auth_store.write() = Some(store);
1278 }
1279
1280 pub fn auth_store(&self) -> Option<Arc<crate::auth::store::AuthStore>> {
1283 self.inner.auth_store.read().clone()
1284 }
1285
1286 pub fn set_oauth_validator(&self, validator: Option<Arc<crate::auth::oauth::OAuthValidator>>) {
1290 *self.inner.oauth_validator.write() = validator;
1291 }
1292
1293 pub fn oauth_validator(&self) -> Option<Arc<crate::auth::oauth::OAuthValidator>> {
1297 self.inner.oauth_validator.read().clone()
1298 }
1299
1300 pub fn set_browser_token_authority(
1306 &self,
1307 authority: Option<Arc<crate::auth::browser_token::BrowserTokenAuthority>>,
1308 ) {
1309 *self.inner.browser_token_authority.write() = authority;
1310 }
1311
1312 pub fn browser_token_authority(
1316 &self,
1317 ) -> Option<Arc<crate::auth::browser_token::BrowserTokenAuthority>> {
1318 self.inner.browser_token_authority.read().clone()
1319 }
1320}