use opentelemetry::metrics::{Counter, Histogram};
use opentelemetry::KeyValue;
use skilj_core::access_control::{AccessLevel, JwksCache, RevocationBroadcaster, Role};
use skilj_core::bootstrap::BootstrapSecret;
use skilj_core::db::Pool;
use skilj_core::event_cache::EventCache;
use skilj_core::event_store::{Error as EventStoreError, Event, EventBroadcaster};
use skilj_core::plugin::{BoundedContextEvent, CommandDispatcher};
use skilj_core::projections::{ProjectionRebuildStatus, ProjectionRegistration};
use skilj_core::shared::CommandDecision;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::LazyLock;
use tracing::Instrument;
static BACKGROUND_TASK_TICK_DURATION: LazyLock<Histogram<f64>> = LazyLock::new(|| {
opentelemetry::global::meter("skilj")
.f64_histogram("skilj.background_task.tick.duration")
.with_unit("s")
.with_description("Duration of one tick of a skilj background task.")
.build()
});
static BACKGROUND_TASK_ERRORS: LazyLock<Counter<u64>> = LazyLock::new(|| {
opentelemetry::global::meter("skilj")
.u64_counter("skilj.background_task.errors")
.with_description("Errors encountered by a skilj background task.")
.build()
});
pub use inventory;
pub use skilj_core::access_control::{IdpConfig, SigningAlgorithm};
pub use skilj_core::encryption::EncryptionMasterKey;
pub use skilj_core::plugin::{
requires_role, CommandType, EventType, Projection, Snapshot, DEFAULT_BOUNDED_CONTEXT,
};
pub use skilj_macros::auto_register;
struct IdentityProvider {
config: IdpConfig,
cache: Arc<JwksCache>,
}
pub struct Skilj {
pool: Pool,
command_types: Arc<HashMap<(String, String), RegisteredCommandType>>,
projections: Arc<HashMap<(String, String), RegisteredProjection>>,
event_types: Arc<HashMap<(String, String), RegisteredEventType>>,
snapshots: Arc<HashMap<(String, String), RegisteredSnapshot>>,
bootstrap_secret: Option<BootstrapSecret>,
identity_provider: Option<Arc<IdentityProvider>>,
projection_query_wait_timeout: std::time::Duration,
encryption_master_key: Option<EncryptionMasterKey>,
event_broadcaster: EventBroadcaster,
revocation_broadcaster: RevocationBroadcaster,
event_cache: EventCache,
schema_registry: Arc<skilj_graphql::schema::SchemaRegistry>,
}
struct Dispatcher {
command_types: Arc<HashMap<(String, String), RegisteredCommandType>>,
}
impl CommandDispatcher for Dispatcher {
fn dispatch(
&self,
bounded_context: &str,
command_type: &str,
payload: &str,
matching_events: &[Event],
) -> Option<skilj_core::error::Result<CommandDecision>> {
let registered = self
.command_types
.get(&(bounded_context.to_string(), command_type.to_string()))?;
Some((registered.decide)(payload, matching_events))
}
fn required_role(
&self,
bounded_context: &str,
command_type: &str,
) -> Option<Option<&'static str>> {
let registered = self
.command_types
.get(&(bounded_context.to_string(), command_type.to_string()))?;
Some(registered.required_role)
}
fn snapshot_name(
&self,
bounded_context: &str,
command_type: &str,
) -> Option<Option<&'static str>> {
let registered = self
.command_types
.get(&(bounded_context.to_string(), command_type.to_string()))?;
Some(registered.snapshot_name)
}
fn dispatch_from_snapshot(
&self,
bounded_context: &str,
command_type: &str,
payload: &str,
snapshot_state_json: &str,
events_since_snapshot: &[Event],
) -> Option<skilj_core::error::Result<CommandDecision>> {
let registered = self
.command_types
.get(&(bounded_context.to_string(), command_type.to_string()))?;
Some((registered.decide_from_snapshot)(
payload,
snapshot_state_json,
events_since_snapshot,
))
}
}
struct ProjectionDispatcherImpl {
projections: Arc<HashMap<(String, String), RegisteredProjection>>,
}
impl skilj_core::plugin::ProjectionDispatcher for ProjectionDispatcherImpl {
fn keys(
&self,
bounded_context: &str,
projection_name: &str,
event: &Event,
) -> Option<Vec<String>> {
let registered = self
.projections
.get(&(bounded_context.to_string(), projection_name.to_string()))?;
Some((registered.keys)(event))
}
fn project(
&self,
bounded_context: &str,
projection_name: &str,
state_json: &str,
event: &Event,
key: &str,
) -> Option<skilj_core::error::Result<String>> {
let registered = self
.projections
.get(&(bounded_context.to_string(), projection_name.to_string()))?;
Some((registered.project)(state_json, event, key))
}
fn default_state(&self, bounded_context: &str, projection_name: &str) -> Option<String> {
let registered = self
.projections
.get(&(bounded_context.to_string(), projection_name.to_string()))?;
Some(registered.default_state_json.clone())
}
}
struct EventDispatcherImpl {
event_types: Arc<HashMap<(String, String), RegisteredEventType>>,
}
impl skilj_core::plugin::EventDispatcher for EventDispatcherImpl {
fn scheduled_payload(&self, bounded_context: &str, event_type: &str) -> Option<String> {
let registered = self
.event_types
.get(&(bounded_context.to_string(), event_type.to_string()))?;
Some((registered.scheduled_payload)())
}
}
struct SnapshotDispatcherImpl {
snapshots: Arc<HashMap<(String, String), RegisteredSnapshot>>,
}
impl skilj_core::plugin::SnapshotDispatcher for SnapshotDispatcherImpl {
fn snapshot_names(&self, bounded_context: &str) -> Vec<&'static str> {
self.snapshots
.iter()
.filter(|((bc, _), _)| bc == bounded_context)
.map(|(_, registered)| registered.name)
.collect()
}
fn tag_key(&self, bounded_context: &str, snapshot_name: &str) -> Option<&'static str> {
let registered = self
.snapshots
.get(&(bounded_context.to_string(), snapshot_name.to_string()))?;
Some(registered.tag_key)
}
fn version(&self, bounded_context: &str, snapshot_name: &str) -> Option<u64> {
let registered = self
.snapshots
.get(&(bounded_context.to_string(), snapshot_name.to_string()))?;
Some(registered.version)
}
fn fold(
&self,
bounded_context: &str,
snapshot_name: &str,
state_json: &str,
event: &Event,
) -> Option<skilj_core::error::Result<String>> {
let registered = self
.snapshots
.get(&(bounded_context.to_string(), snapshot_name.to_string()))?;
Some((registered.fold)(state_json, event))
}
fn default_state(&self, bounded_context: &str, snapshot_name: &str) -> Option<String> {
let registered = self
.snapshots
.get(&(bounded_context.to_string(), snapshot_name.to_string()))?;
Some(registered.default_state_json.clone())
}
}
impl Skilj {
pub fn builder(database_url: impl Into<String>) -> SkiljBuilder {
SkiljBuilder {
database_url: database_url.into(),
current_bounded_context: skilj_core::plugin::DEFAULT_BOUNDED_CONTEXT.to_string(),
reconciliation_role: None,
identity_provider: None,
event_types: HashMap::new(),
command_types: HashMap::new(),
projections: HashMap::new(),
snapshots: HashMap::new(),
async_projection_poll_interval: std::time::Duration::from_millis(500),
snapshot_poll_interval: std::time::Duration::from_millis(500),
scheduler_poll_interval: std::time::Duration::from_secs(1),
projection_query_wait_timeout: std::time::Duration::from_secs(5),
encryption_master_key: None,
event_broadcast_capacity: 1024,
event_cache_warm_up_count: 1000,
}
}
pub fn command_dispatcher(&self) -> Arc<dyn CommandDispatcher> {
Arc::new(Dispatcher {
command_types: self.command_types.clone(),
})
}
pub fn projection_dispatcher(&self) -> Arc<dyn skilj_core::plugin::ProjectionDispatcher> {
Arc::new(ProjectionDispatcherImpl {
projections: self.projections.clone(),
})
}
pub fn event_dispatcher(&self) -> Arc<dyn skilj_core::plugin::EventDispatcher> {
Arc::new(EventDispatcherImpl {
event_types: self.event_types.clone(),
})
}
pub fn snapshot_dispatcher(&self) -> Arc<dyn skilj_core::plugin::SnapshotDispatcher> {
Arc::new(SnapshotDispatcherImpl {
snapshots: self.snapshots.clone(),
})
}
pub fn bootstrap_secret(&self) -> Option<&str> {
self.bootstrap_secret.as_ref().map(|s| s.secret.as_str())
}
pub fn rest_router(&self) -> axum::Router {
skilj_rest::router(
self.pool.clone(),
self.command_dispatcher(),
self.projection_dispatcher(),
self.snapshot_dispatcher(),
self.encryption_master_key.clone(),
self.event_broadcaster.clone(),
self.event_cache.clone(),
)
}
pub async fn graphql_router(&self) -> skilj_core::error::Result<axum::Router> {
skilj_graphql::router(Arc::clone(&self.schema_registry), self.graphql_state()).await
}
fn graphql_state(&self) -> skilj_graphql::GraphqlState {
skilj_graphql::GraphqlState {
pool: self.pool.clone(),
bootstrap_secret: self.bootstrap_secret.clone(),
identity: self
.identity_provider
.as_ref()
.map(|ip| skilj_graphql::auth::Identity {
config: ip.config.clone(),
cache: Arc::clone(&ip.cache),
}),
dispatcher: self.command_dispatcher(),
projection_dispatcher: self.projection_dispatcher(),
snapshot_dispatcher: self.snapshot_dispatcher(),
projection_query_wait_timeout: self.projection_query_wait_timeout,
encryption_master_key: self.encryption_master_key.clone(),
event_broadcaster: self.event_broadcaster.clone(),
revocation_broadcaster: self.revocation_broadcaster.clone(),
event_cache: self.event_cache.clone(),
}
}
}
#[derive(Debug, Default)]
pub struct ReconciliationReport {
pub registered: Vec<String>,
pub skipped_no_access: Vec<String>,
}
type ScheduledPayloadFn = Box<dyn Fn() -> String + Send + Sync>;
struct RegisteredEventType {
schema: String,
tag_mappings: Vec<skilj_core::shared::TagMapping>,
sensitive_fields: Vec<skilj_core::shared::SensitiveField>,
external_creation_allowed: bool,
direct_creation_allowed: bool,
event_read_allowed: bool,
system_triggered_allowed: bool,
system_triggered_schedule: Option<String>,
missed_occurrence_policy: Option<skilj_core::event_store::MissedOccurrencePolicy>,
scheduled_payload: ScheduledPayloadFn,
}
fn registered_event_type<T: EventType + 'static>() -> RegisteredEventType {
let schema = schemars::schema_for!(T::Payload);
RegisteredEventType {
schema: serde_json::to_string(&schema).expect("JSON Schema serialization is infallible"),
tag_mappings: T::tag_mappings(),
sensitive_fields: T::sensitive_fields(),
external_creation_allowed: T::external_creation_allowed(),
direct_creation_allowed: T::direct_creation_allowed(),
event_read_allowed: T::event_read_allowed(),
system_triggered_allowed: T::system_triggered_allowed(),
system_triggered_schedule: T::system_triggered_schedule(),
missed_occurrence_policy: T::missed_occurrence_policy(),
scheduled_payload: Box::new(|| {
serde_json::to_string(&T::scheduled_payload())
.expect("JSON serialization of Payload is infallible")
}),
}
}
type DeciderFn =
Box<dyn Fn(&str, &[Event]) -> skilj_core::error::Result<CommandDecision> + Send + Sync>;
type DecideFromSnapshotFn =
Box<dyn Fn(&str, &str, &[Event]) -> skilj_core::error::Result<CommandDecision> + Send + Sync>;
struct RegisteredCommandType {
schema: String,
tag_mappings: Vec<skilj_core::shared::TagMapping>,
sensitive_fields: Vec<skilj_core::shared::SensitiveField>,
rest_trigger_allowed: bool,
required_role: Option<&'static str>,
snapshot_name: Option<&'static str>,
decide: DeciderFn,
decide_from_snapshot: DecideFromSnapshotFn,
}
fn registered_command_type<T: CommandType + 'static>() -> RegisteredCommandType {
let schema = schemars::schema_for!(T::Payload);
RegisteredCommandType {
schema: serde_json::to_string(&schema).expect("JSON Schema serialization is infallible"),
tag_mappings: T::tag_mappings(),
sensitive_fields: T::sensitive_fields(),
rest_trigger_allowed: T::rest_trigger_allowed(),
required_role: T::required_role(),
snapshot_name: T::snapshot(),
decide: Box::new(|payload_json, raw_events| {
let payload: T::Payload = serde_json::from_str(payload_json)
.map_err(|e| EventStoreError::PayloadDecodeFailed(e.to_string()))?;
let mut matching = Vec::with_capacity(raw_events.len());
for event in raw_events {
if let Some(converted) = T::Event::try_from_event(event) {
matching.push(
converted
.map_err(|e| EventStoreError::PayloadDecodeFailed(e.to_string()))?,
);
}
}
Ok(T::decide(&payload, &matching))
}),
decide_from_snapshot: Box::new(|payload_json, snapshot_state_json, raw_events| {
let payload: T::Payload = serde_json::from_str(payload_json)
.map_err(|e| EventStoreError::PayloadDecodeFailed(e.to_string()))?;
let mut matching = Vec::with_capacity(raw_events.len());
for event in raw_events {
if let Some(converted) = T::Event::try_from_event(event) {
matching.push(
converted
.map_err(|e| EventStoreError::PayloadDecodeFailed(e.to_string()))?,
);
}
}
Ok(T::decide_from_snapshot(
&payload,
snapshot_state_json,
&matching,
))
}),
}
}
type KeysFn = Box<dyn Fn(&Event) -> Vec<String> + Send + Sync>;
type ProjectFn = Box<dyn Fn(&str, &Event, &str) -> skilj_core::error::Result<String> + Send + Sync>;
struct RegisteredProjection {
schema: String,
consumed_event_types: Vec<&'static str>,
sync: bool,
default_state_json: String,
keys: KeysFn,
project: ProjectFn,
}
fn registered_projection<T: Projection + 'static>() -> RegisteredProjection {
let schema = schemars::schema_for!(T::State);
let consumed_event_types = T::consumed_event_types();
let consumed_for_keys = consumed_event_types.clone();
let consumed_for_project = consumed_event_types.clone();
let default_state_json = serde_json::to_string(&T::State::default())
.expect("JSON serialization of a Default::default() State is infallible");
RegisteredProjection {
schema: serde_json::to_string(&schema).expect("JSON Schema serialization is infallible"),
consumed_event_types,
sync: T::sync(),
default_state_json,
keys: Box::new(move |event| {
if !consumed_for_keys
.iter()
.any(|&name| name == event.event_type.name)
{
return Vec::new();
}
let Some(Ok(converted)) = T::Event::try_from_event(event) else {
return Vec::new();
};
T::keys(&converted)
}),
project: Box::new(move |state_json, event, key| {
if !consumed_for_project
.iter()
.any(|&name| name == event.event_type.name)
{
return Ok(state_json.to_string());
}
let mut state: T::State = serde_json::from_str(state_json)
.map_err(|e| EventStoreError::PayloadDecodeFailed(e.to_string()))?;
let Some(converted) = T::Event::try_from_event(event) else {
return Ok(state_json.to_string());
};
let converted =
converted.map_err(|e| EventStoreError::PayloadDecodeFailed(e.to_string()))?;
T::project(&mut state, &converted, key);
Ok(serde_json::to_string(&state).expect("JSON serialization of State is infallible"))
}),
}
}
type SnapshotFoldFn = Box<dyn Fn(&str, &Event) -> skilj_core::error::Result<String> + Send + Sync>;
struct RegisteredSnapshot {
name: &'static str,
tag_key: &'static str,
version: u64,
default_state_json: String,
fold: SnapshotFoldFn,
}
fn registered_snapshot<T: Snapshot + 'static>() -> RegisteredSnapshot {
let default_state_json = serde_json::to_string(&T::State::default())
.expect("JSON serialization of a Default::default() State is infallible");
RegisteredSnapshot {
name: T::NAME,
tag_key: T::TAG_KEY,
version: T::VERSION,
default_state_json,
fold: Box::new(|state_json, event| {
let mut state: T::State = serde_json::from_str(state_json)
.map_err(|e| EventStoreError::PayloadDecodeFailed(e.to_string()))?;
let converted = T::Event::try_from_event(event).ok_or_else(|| {
EventStoreError::PayloadDecodeFailed(format!(
"event type {} does not convert via this bounded context's own \
BoundedContextEvent - snapshot {} can't fold it",
event.event_type.name,
T::NAME
))
})?;
let converted =
converted.map_err(|e| EventStoreError::PayloadDecodeFailed(e.to_string()))?;
T::fold(&mut state, &converted);
Ok(serde_json::to_string(&state).expect("JSON serialization of State is infallible"))
}),
}
}
pub struct EventTypeRegistrar(pub fn(SkiljBuilder) -> SkiljBuilder);
inventory::collect!(EventTypeRegistrar);
pub struct CommandTypeRegistrar(pub fn(SkiljBuilder) -> SkiljBuilder);
inventory::collect!(CommandTypeRegistrar);
pub struct ProjectionRegistrar(pub fn(SkiljBuilder) -> SkiljBuilder);
inventory::collect!(ProjectionRegistrar);
pub struct SnapshotRegistrar(pub fn(SkiljBuilder) -> SkiljBuilder);
inventory::collect!(SnapshotRegistrar);
pub struct SkiljBuilder {
database_url: String,
current_bounded_context: String,
reconciliation_role: Option<String>,
identity_provider: Option<IdpConfig>,
event_types: HashMap<(String, String), RegisteredEventType>,
command_types: HashMap<(String, String), RegisteredCommandType>,
projections: HashMap<(String, String), RegisteredProjection>,
snapshots: HashMap<(String, String), RegisteredSnapshot>,
async_projection_poll_interval: std::time::Duration,
snapshot_poll_interval: std::time::Duration,
scheduler_poll_interval: std::time::Duration,
projection_query_wait_timeout: std::time::Duration,
encryption_master_key: Option<EncryptionMasterKey>,
event_broadcast_capacity: usize,
event_cache_warm_up_count: usize,
}
impl SkiljBuilder {
pub fn bounded_context(mut self, name: impl Into<String>) -> Self {
self.current_bounded_context = name.into();
self
}
fn current_bounded_context(&self) -> String {
self.current_bounded_context.clone()
}
pub fn auto_register(mut self) -> Self {
for registrar in inventory::iter::<EventTypeRegistrar> {
self = (registrar.0)(self);
}
for registrar in inventory::iter::<CommandTypeRegistrar> {
self = (registrar.0)(self);
}
for registrar in inventory::iter::<ProjectionRegistrar> {
self = (registrar.0)(self);
}
for registrar in inventory::iter::<SnapshotRegistrar> {
self = (registrar.0)(self);
}
self
}
pub fn event_type<T: EventType + 'static>(mut self) -> Self {
let bc = self.current_bounded_context();
self.event_types
.insert((bc, T::NAME.to_string()), registered_event_type::<T>());
self
}
pub fn command_type<T: CommandType + 'static>(mut self) -> Self {
let bc = self.current_bounded_context();
self.command_types
.insert((bc, T::NAME.to_string()), registered_command_type::<T>());
self
}
pub fn projection<T: Projection + 'static>(mut self) -> Self {
let bc = self.current_bounded_context();
self.projections
.insert((bc, T::NAME.to_string()), registered_projection::<T>());
self
}
pub fn snapshot<T: Snapshot + 'static>(mut self) -> Self {
let bc = self.current_bounded_context();
self.snapshots
.insert((bc, T::NAME.to_string()), registered_snapshot::<T>());
self
}
pub fn reconciliation_role(mut self, external_subject: impl Into<String>) -> Self {
self.reconciliation_role = Some(external_subject.into());
self
}
pub fn identity_provider(mut self, config: IdpConfig) -> Self {
self.identity_provider = Some(config);
self
}
pub fn async_projection_poll_interval(mut self, interval: std::time::Duration) -> Self {
self.async_projection_poll_interval = interval;
self
}
pub fn snapshot_poll_interval(mut self, interval: std::time::Duration) -> Self {
self.snapshot_poll_interval = interval;
self
}
pub fn scheduler_poll_interval(mut self, interval: std::time::Duration) -> Self {
self.scheduler_poll_interval = interval;
self
}
pub fn projection_query_wait_timeout(mut self, timeout: std::time::Duration) -> Self {
self.projection_query_wait_timeout = timeout;
self
}
pub fn encryption_master_key(mut self, master_key: EncryptionMasterKey) -> Self {
self.encryption_master_key = Some(master_key);
self
}
pub fn event_broadcast_capacity(mut self, capacity: usize) -> Self {
self.event_broadcast_capacity = capacity;
self
}
pub fn event_cache_warm_up_count(mut self, count: usize) -> Self {
self.event_cache_warm_up_count = count;
self
}
pub async fn build(self) -> Result<(Skilj, ReconciliationReport), skilj_core::Error> {
let pool = skilj_core::db::connect(&self.database_url).await?;
skilj_core::db::migrate(&pool).await?;
let existing_admin_context = skilj_core::db::get_bounded_context(
&pool,
skilj_core::bootstrap::ADMIN_BOUNDED_CONTEXT_NAME,
)
.await?;
if let Some(admin_context) = skilj_core::bootstrap::stamp_admin_bounded_context(
existing_admin_context.as_ref(),
chrono::Utc::now(),
) {
if let Err(e) = skilj_core::db::insert_bounded_context(&pool, &admin_context).await {
if skilj_core::db::get_bounded_context(
&pool,
skilj_core::bootstrap::ADMIN_BOUNDED_CONTEXT_NAME,
)
.await?
.is_none()
{
return Err(e);
}
}
}
let roles = skilj_core::db::list_roles(&pool).await?;
let bootstrap_secret = skilj_core::bootstrap::generate_bootstrap_secret(&roles);
if let Some(secret) = &bootstrap_secret {
eprintln!(
"skilj: no active superadmin Role exists yet - bootstrap secret (use once, via \
the createSuperadmin GraphQL mutation): {}",
secret.secret
);
}
let projections = Arc::new(self.projections);
let command_types = Arc::new(self.command_types);
let snapshots = Arc::new(self.snapshots);
let mut report = ReconciliationReport::default();
if let Some(external_subject) = &self.reconciliation_role {
let role = skilj_core::access_control::resolve_role_by_external_subject(
external_subject,
&roles,
)?
.clone();
reconcile_event_types(
&pool,
&role,
&self.event_types,
&mut report,
chrono::Utc::now(),
)
.await?;
reconcile_command_types(&pool, &role, &command_types, &mut report).await?;
let reconciliation_dispatcher = ProjectionDispatcherImpl {
projections: projections.clone(),
};
reconcile_projections(
&pool,
&role,
&projections,
&mut report,
&reconciliation_dispatcher,
)
.await?;
}
let identity_provider = self.identity_provider.map(|config| {
Arc::new(IdentityProvider {
cache: Arc::new(JwksCache::new(config.jwks_endpoint.clone())),
config,
})
});
let poll_interval = self.async_projection_poll_interval;
let projection_query_wait_timeout = self.projection_query_wait_timeout;
let encryption_master_key = self.encryption_master_key;
let event_broadcaster = EventBroadcaster::new(self.event_broadcast_capacity);
let revocation_broadcaster = RevocationBroadcaster::new();
let event_cache = EventCache::new(self.event_cache_warm_up_count);
for bc in skilj_core::db::list_bounded_contexts(&pool).await? {
event_cache.warm(&pool, &bc.name).await?;
}
let schema_registry = Arc::new(
skilj_graphql::schema::SchemaRegistry::build(skilj_graphql::GraphqlState {
pool: pool.clone(),
bootstrap_secret: bootstrap_secret.clone(),
identity: identity_provider
.as_ref()
.map(|ip| skilj_graphql::auth::Identity {
config: ip.config.clone(),
cache: Arc::clone(&ip.cache),
}),
dispatcher: Arc::new(Dispatcher {
command_types: command_types.clone(),
}),
projection_dispatcher: Arc::new(ProjectionDispatcherImpl {
projections: projections.clone(),
}),
snapshot_dispatcher: Arc::new(SnapshotDispatcherImpl {
snapshots: snapshots.clone(),
}),
projection_query_wait_timeout,
encryption_master_key: encryption_master_key.clone(),
event_broadcaster: event_broadcaster.clone(),
revocation_broadcaster: revocation_broadcaster.clone(),
event_cache: event_cache.clone(),
})
.await?,
);
let skilj = Skilj {
pool,
command_types,
projections,
snapshots,
event_types: Arc::new(self.event_types),
bootstrap_secret,
identity_provider,
projection_query_wait_timeout,
encryption_master_key,
event_broadcaster,
revocation_broadcaster,
event_cache,
schema_registry,
};
let poll_pool = skilj.pool.clone();
let poll_dispatcher = skilj.projection_dispatcher();
tokio::spawn(async move {
loop {
let start = std::time::Instant::now();
async {
match skilj_core::db::list_bounded_contexts(&poll_pool).await {
Ok(bounded_contexts) => {
for bc in bounded_contexts {
if let Err(e) = skilj_core::db::catch_up_bounded_context(
&poll_pool,
&bc.name,
poll_dispatcher.as_ref(),
)
.await
{
tracing::warn!(
bounded_context = %bc.name,
error = %e,
"async projection catch-up failed"
);
BACKGROUND_TASK_ERRORS.add(
1,
&[
KeyValue::new("task", "async_projection"),
KeyValue::new("reason", "catch_up_failed"),
],
);
}
}
}
Err(e) => {
tracing::warn!(
error = %e,
"async projection catch-up failed to list bounded contexts"
);
BACKGROUND_TASK_ERRORS.add(
1,
&[
KeyValue::new("task", "async_projection"),
KeyValue::new("reason", "list_bounded_contexts_failed"),
],
);
}
}
}
.instrument(tracing::info_span!("async_projection_tick"))
.await;
BACKGROUND_TASK_TICK_DURATION.record(
start.elapsed().as_secs_f64(),
&[KeyValue::new("task", "async_projection")],
);
tokio::time::sleep(poll_interval).await;
}
});
let snapshot_pool = skilj.pool.clone();
let snapshot_dispatcher = skilj.snapshot_dispatcher();
let snapshot_interval = self.snapshot_poll_interval;
tokio::spawn(async move {
loop {
let start = std::time::Instant::now();
async {
match skilj_core::db::list_bounded_contexts(&snapshot_pool).await {
Ok(bounded_contexts) => {
for bc in bounded_contexts {
if let Err(e) = skilj_core::db::catch_up_snapshots(
&snapshot_pool,
&bc.name,
snapshot_dispatcher.as_ref(),
)
.await
{
tracing::warn!(
bounded_context = %bc.name,
error = %e,
"snapshot catch-up failed"
);
BACKGROUND_TASK_ERRORS.add(
1,
&[
KeyValue::new("task", "snapshot"),
KeyValue::new("reason", "catch_up_failed"),
],
);
}
}
}
Err(e) => {
tracing::warn!(
error = %e,
"snapshot catch-up failed to list bounded contexts"
);
BACKGROUND_TASK_ERRORS.add(
1,
&[
KeyValue::new("task", "snapshot"),
KeyValue::new("reason", "list_bounded_contexts_failed"),
],
);
}
}
}
.instrument(tracing::info_span!("snapshot_tick"))
.await;
BACKGROUND_TASK_TICK_DURATION.record(
start.elapsed().as_secs_f64(),
&[KeyValue::new("task", "snapshot")],
);
tokio::time::sleep(snapshot_interval).await;
}
});
let scheduler_pool = skilj.pool.clone();
let scheduler_projection_dispatcher = skilj.projection_dispatcher();
let scheduler_event_dispatcher = skilj.event_dispatcher();
let scheduler_broadcaster = skilj.event_broadcaster.clone();
let scheduler_event_cache = skilj.event_cache.clone();
let scheduler_encryption_master_key = skilj.encryption_master_key.clone();
let scheduler_interval = self.scheduler_poll_interval;
tokio::spawn(async move {
loop {
let start = std::time::Instant::now();
scheduler_tick(
&scheduler_pool,
scheduler_projection_dispatcher.as_ref(),
scheduler_event_dispatcher.as_ref(),
&scheduler_broadcaster,
&scheduler_event_cache,
scheduler_encryption_master_key.as_ref(),
chrono::Utc::now(),
)
.instrument(tracing::info_span!("scheduler_tick"))
.await;
BACKGROUND_TASK_TICK_DURATION.record(
start.elapsed().as_secs_f64(),
&[KeyValue::new("task", "scheduler")],
);
tokio::time::sleep(scheduler_interval).await;
}
});
let cross_instance_pool = skilj.pool.clone();
let cross_instance_event_broadcaster = skilj.event_broadcaster.clone();
let cross_instance_revocation_broadcaster = skilj.revocation_broadcaster.clone();
let cross_instance_schema_registry = Arc::clone(&skilj.schema_registry);
let cross_instance_state = skilj.graphql_state();
tokio::spawn(async move {
let mut listener = loop {
match skilj_core::cross_instance::Listener::connect(&cross_instance_pool).await {
Ok(listener) => break listener,
Err(err) => {
tracing::warn!(
error = %err,
"cross-instance listener failed to connect, retrying"
);
BACKGROUND_TASK_ERRORS.add(
1,
&[
KeyValue::new("task", "cross_instance"),
KeyValue::new("reason", "connect_failed"),
],
);
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
}
}
};
loop {
let message = match listener
.recv()
.instrument(tracing::info_span!("cross_instance_recv"))
.await
{
Ok(message) => message,
Err(err) => {
tracing::warn!(error = %err, "cross-instance listener error, retrying");
BACKGROUND_TASK_ERRORS.add(
1,
&[
KeyValue::new("task", "cross_instance"),
KeyValue::new("reason", "recv_failed"),
],
);
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
continue;
}
};
match message {
skilj_core::cross_instance::Message::EventAppended {
bounded_context,
sequence,
origin_instance_id,
} => {
if origin_instance_id == cross_instance_event_broadcaster.instance_id() {
continue;
}
match skilj_core::db::get_event_by_sequence(
&cross_instance_pool,
&bounded_context,
sequence,
)
.await
{
Ok(Some(event)) => cross_instance_event_broadcaster.publish(&event),
Ok(None) => {}
Err(err) => tracing::warn!(
error = %err,
bounded_context = %bounded_context,
sequence,
"cross-instance event refetch failed"
),
}
}
skilj_core::cross_instance::Message::Revoked {
revoked,
origin_instance_id,
} => {
if origin_instance_id != cross_instance_revocation_broadcaster.instance_id()
{
cross_instance_revocation_broadcaster.publish(revoked);
}
}
skilj_core::cross_instance::Message::RegistrationChanged => {
if let Err(err) = cross_instance_schema_registry
.rebuild(cross_instance_state.clone())
.await
{
tracing::warn!(error = %err, "cross-instance schema rebuild failed");
}
}
}
}
});
Ok((skilj, report))
}
}
const MAX_OCCURRENCES_PER_TICK: usize = 1000;
async fn scheduler_tick(
pool: &Pool,
projection_dispatcher: &dyn skilj_core::plugin::ProjectionDispatcher,
event_dispatcher: &dyn skilj_core::plugin::EventDispatcher,
broadcaster: &EventBroadcaster,
event_cache: &EventCache,
encryption_master_key: Option<&EncryptionMasterKey>,
now: chrono::DateTime<chrono::Utc>,
) {
let bounded_contexts = match skilj_core::db::list_bounded_contexts(pool).await {
Ok(bcs) => bcs,
Err(e) => {
tracing::warn!(error = %e, "scheduler failed to list bounded contexts");
BACKGROUND_TASK_ERRORS.add(
1,
&[
KeyValue::new("task", "scheduler"),
KeyValue::new("reason", "list_bounded_contexts_failed"),
],
);
return;
}
};
for bc in &bounded_contexts {
if bc.status != skilj_core::event_store::BoundedContextStatus::Active {
continue;
}
let scheduled = match skilj_core::db::list_scheduled_event_types(pool, &bc.name).await {
Ok(scheduled) => scheduled,
Err(e) => {
tracing::warn!(
bounded_context = %bc.name,
error = %e,
"scheduler failed to list scheduled event types"
);
BACKGROUND_TASK_ERRORS.add(
1,
&[
KeyValue::new("task", "scheduler"),
KeyValue::new("reason", "list_scheduled_event_types_failed"),
],
);
continue;
}
};
for et in &scheduled {
let (Some(schedule), Some(policy), Some(initial_position)) = (
&et.system_triggered_schedule,
et.missed_occurrence_policy,
et.schedule_position,
) else {
continue;
};
let mut cursor = initial_position;
for _ in 0..MAX_OCCURRENCES_PER_TICK {
let Some(occurrence_at) =
skilj_core::event_store::next_occurrence_after(schedule, cursor)
else {
break;
};
if occurrence_at > now {
break;
}
let nothing_later_is_due =
skilj_core::event_store::next_occurrence_after(schedule, occurrence_at)
.is_none_or(|next| next > now);
if policy == skilj_core::event_store::MissedOccurrencePolicy::Skip
&& !nothing_later_is_due
{
if let Err(e) = skilj_core::db::skip_missed_occurrences_for_event_type(
pool, &bc.name, &et.name, now,
)
.await
{
tracing::warn!(
bounded_context = %bc.name,
event_type = %et.name,
error = %e,
"SkipMissedOccurrences failed"
);
BACKGROUND_TASK_ERRORS.add(
1,
&[
KeyValue::new("task", "scheduler"),
KeyValue::new("reason", "skip_missed_occurrences_failed"),
],
);
}
break;
}
if let Err(e) = skilj_core::db::fire_system_event(
pool,
projection_dispatcher,
event_dispatcher,
broadcaster,
event_cache,
&bc.name,
&et.name,
occurrence_at,
now,
encryption_master_key,
)
.await
{
tracing::warn!(
bounded_context = %bc.name,
event_type = %et.name,
error = %e,
"CreateSystemEvent failed"
);
BACKGROUND_TASK_ERRORS.add(
1,
&[
KeyValue::new("task", "scheduler"),
KeyValue::new("reason", "create_system_event_failed"),
],
);
break;
}
cursor = occurrence_at;
}
}
}
}
async fn active_admin_mapping(
pool: &Pool,
role_id: &str,
bounded_context: &str,
) -> skilj_core::error::Result<Option<skilj_core::access_control::RoleAccessMapping>> {
Ok(
skilj_core::db::get_active_role_access_mapping(pool, role_id, bounded_context)
.await?
.filter(|m| m.level == AccessLevel::Admin),
)
}
async fn resolve_bounded_context_and_admin_mapping(
pool: &Pool,
role_id: &str,
bounded_context_name: &str,
) -> skilj_core::error::Result<
Option<(
skilj_core::event_store::BoundedContext,
skilj_core::access_control::RoleAccessMapping,
)>,
> {
let Some(bc) = skilj_core::db::get_bounded_context(pool, bounded_context_name).await? else {
return Ok(None);
};
let Some(mapping) = active_admin_mapping(pool, role_id, bounded_context_name).await? else {
return Ok(None);
};
Ok(Some((bc, mapping)))
}
async fn reconcile_event_types(
pool: &Pool,
role: &Role,
event_types: &HashMap<(String, String), RegisteredEventType>,
report: &mut ReconciliationReport,
now: chrono::DateTime<chrono::Utc>,
) -> Result<(), skilj_core::Error> {
for ((bounded_context_name, name), registered) in event_types {
let key = format!("{bounded_context_name}/{name}");
let Some((bc, mapping)) =
resolve_bounded_context_and_admin_mapping(pool, &role.id, bounded_context_name).await?
else {
report.skipped_no_access.push(key);
continue;
};
let existing = skilj_core::db::get_event_type(pool, bounded_context_name, name).await?;
let registration = skilj_core::event_store::register_event_type(
&mapping,
&bc,
name.clone(),
registered.schema.clone(),
registered.tag_mappings.clone(),
registered.sensitive_fields.clone(),
registered.external_creation_allowed,
registered.direct_creation_allowed,
registered.system_triggered_allowed,
registered.system_triggered_schedule.clone(),
registered.missed_occurrence_policy,
registered.event_read_allowed,
existing.as_ref(),
now,
)?;
skilj_core::db::upsert_event_type(pool, registration.event_type()).await?;
report.registered.push(key);
}
Ok(())
}
async fn reconcile_command_types(
pool: &Pool,
role: &Role,
command_types: &HashMap<(String, String), RegisteredCommandType>,
report: &mut ReconciliationReport,
) -> Result<(), skilj_core::Error> {
for ((bounded_context_name, name), registered) in command_types {
let key = format!("{bounded_context_name}/{name}");
let Some((bc, mapping)) =
resolve_bounded_context_and_admin_mapping(pool, &role.id, bounded_context_name).await?
else {
report.skipped_no_access.push(key);
continue;
};
let existing = skilj_core::db::get_command_type(pool, bounded_context_name, name).await?;
let registration = skilj_core::event_store::register_command_type(
&mapping,
&bc,
name.clone(),
registered.schema.clone(),
registered.tag_mappings.clone(),
registered.sensitive_fields.clone(),
registered.rest_trigger_allowed,
existing.as_ref(),
)?;
skilj_core::db::upsert_command_type(pool, registration.command_type()).await?;
report.registered.push(key);
}
Ok(())
}
async fn resolve_consumed_event_types(
pool: &Pool,
bounded_context: &str,
names: &[&'static str],
) -> skilj_core::error::Result<Option<Vec<skilj_core::event_store::EventType>>> {
let mut event_types = Vec::with_capacity(names.len());
for name in names {
let Some(et) = skilj_core::db::get_event_type(pool, bounded_context, name).await? else {
return Ok(None);
};
event_types.push(et);
}
Ok(Some(event_types))
}
async fn reconcile_projections(
pool: &Pool,
role: &Role,
projections: &HashMap<(String, String), RegisteredProjection>,
report: &mut ReconciliationReport,
dispatcher: &dyn skilj_core::plugin::ProjectionDispatcher,
) -> Result<(), skilj_core::Error> {
for ((bounded_context_name, name), registered) in projections {
let key = format!("{bounded_context_name}/{name}");
let Some((bc, mapping)) =
resolve_bounded_context_and_admin_mapping(pool, &role.id, bounded_context_name).await?
else {
report.skipped_no_access.push(key);
continue;
};
let Some(consumed_event_types) = resolve_consumed_event_types(
pool,
bounded_context_name,
®istered.consumed_event_types,
)
.await?
else {
report.skipped_no_access.push(key);
continue;
};
let existing = skilj_core::db::get_projection(pool, bounded_context_name, name).await?;
let staged = skilj_core::db::get_projection_rebuild(
pool,
bounded_context_name,
name,
ProjectionRebuildStatus::Pending,
)
.await?;
let bounded_context_events =
skilj_core::db::list_events_for_bounded_context(pool, bounded_context_name).await?;
let registration = skilj_core::projections::register_projection(
&mapping,
&bc,
name.clone(),
registered.schema.clone(),
consumed_event_types,
registered.sync,
existing.as_ref(),
staged.as_ref(),
&bounded_context_events,
)?;
match registration {
ProjectionRegistration::Created {
projection,
needs_history_fold,
} => {
skilj_core::db::upsert_projection(pool, &projection).await?;
if needs_history_fold {
skilj_core::db::fold_history_into_new_sync_projection(
pool,
&projection,
dispatcher,
)
.await?;
}
}
ProjectionRegistration::ReconciledTrivially(projection) => {
skilj_core::db::upsert_projection(pool, &projection).await?;
}
ProjectionRegistration::RebuildStaged(rebuild) => {
skilj_core::db::upsert_projection_rebuild(pool, &rebuild).await?;
}
}
report.registered.push(key);
}
Ok(())
}