use std::sync::Arc;
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
use datafusion::datasource::MemTable;
use datafusion::error::DataFusionError;
use datafusion::execution::context::SessionContext;
use crate::decode;
use crate::decode::persona::{
ParticipationRow, PersonaCredentialRow, PersonaIdentityRow, PersonaRow, PersonaSpendPolicyRow,
PersonaUsageRow, PersonaWalletRow,
};
use crate::views::{
APPROVALS_REDACTED_VIEW_SQL, APPROVALS_VIEW_SQL, ATTRIBUTION_REDACTED_VIEW_SQL,
ATTRIBUTION_VIEW_SQL, FIRES_OWNED_VIEW_SQL, FIRES_VIEW_SQL, GRANT_REPLAYS_REDACTED_VIEW_SQL,
GRANT_REPLAYS_VIEW_SQL, HANDOFFS_REDACTED_VIEW_SQL, HANDOFFS_VIEW_SQL,
MESSAGES_REDACTED_VIEW_SQL, MESSAGES_VIEW_SQL, MODEL_CALL_VIEW_SQL, PAYMENTS_REDACTED_VIEW_SQL,
PAYMENTS_VIEW_SQL, REFUSALS_REDACTED_VIEW_SQL, REFUSALS_VIEW_SQL,
ROUTINE_ACTIVE_GRANTS_VIEW_SQL, ROUTINE_GRANTS_VIEW_SQL, ROUTINE_LIFECYCLE_VIEW_SQL,
ROUTINE_OVERVIEW_VIEW_SQL, ROUTINE_SETUP_OWNED_VIEW_SQL, ROUTINE_SETUP_VIEW_SQL,
SUMMARY_VIEW_SQL, TOOL_CALLS_REDACTED_VIEW_SQL, TOOL_CALLS_VIEW_SQL, TURN_DISPATCH_VIEW_SQL,
TURN_FAILED_VIEW_SQL, USAGE_VIEW_SQL, WALLET_LINK_LIFECYCLE_REDACTED_VIEW_SQL,
WALLET_LINK_LIFECYCLE_VIEW_SQL,
};
#[allow(unused_imports)] use super::QueryEngine;
use super::tables::{
APPROVALS_RAW_TABLE, APPROVALS_TABLE, ATTRIBUTION_RAW_TABLE, ATTRIBUTION_TABLE,
DASHBOARD_TABLE, FIRES_RAW_TABLE, FIRES_TABLE, GRANT_REPLAYS_RAW_TABLE, GRANT_REPLAYS_TABLE,
HANDOFFS_RAW_TABLE, HANDOFFS_TABLE, MESSAGES_RAW_TABLE, MESSAGES_TABLE, MODEL_CALL_RAW_TABLE,
MODEL_CALL_TABLE, PARTICIPATIONS_TABLE, PAYMENTS_RAW_TABLE, PAYMENTS_TABLE,
PERSONA_CREDENTIALS_TABLE, PERSONA_IDENTITIES_TABLE, PERSONA_SPEND_POLICIES_TABLE,
PERSONA_USAGE_TABLE, PERSONA_WALLETS_TABLE, PERSONAS_TABLE, REFUSALS_RAW_TABLE, REFUSALS_TABLE,
ROUTINE_ACTIVE_GRANTS_TABLE, ROUTINE_GRANTS_TABLE, ROUTINE_LIFECYCLE_RAW_TABLE,
ROUTINE_LIFECYCLE_TABLE, ROUTINE_OVERVIEW_TABLE, ROUTINE_SETUP_RAW_TABLE, ROUTINE_SETUP_TABLE,
ROUTINES_TABLE, SUMMARY_RAW_TABLE, SUMMARY_TABLE, TOOL_CALLS_RAW_TABLE, TOOL_CALLS_TABLE,
TURN_DISPATCH_RAW_TABLE, TURN_DISPATCH_TABLE, TURN_FAILED_RAW_TABLE, TURN_FAILED_TABLE,
USAGE_RAW_TABLE, USAGE_TABLE, WALLET_LINK_LIFECYCLE_RAW_TABLE, WALLET_LINK_LIFECYCLE_TABLE,
};
use super::{PartitionTables, QueryEngineError, ReferenceData};
use crate::session::QueryScope;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum RegistrationScope {
Fleet,
Owner,
Grant,
}
impl RegistrationScope {
pub(super) const fn new(scope: &QueryScope, register_owner_scoped_routines: bool) -> Self {
match scope {
QueryScope::Fleet => Self::Fleet,
QueryScope::Conversations(_) if register_owner_scoped_routines => Self::Owner,
QueryScope::Conversations(_) => Self::Grant,
}
}
pub(super) const fn is_fleet(self) -> bool {
matches!(self, Self::Fleet)
}
pub(super) const fn is_owner(self) -> bool {
matches!(self, Self::Owner)
}
}
fn mem_table(schema: SchemaRef, batches: Vec<RecordBatch>) -> Result<MemTable, DataFusionError> {
let partitions = if batches.is_empty() {
vec![vec![]]
} else {
batches.into_iter().map(|batch| vec![batch]).collect()
};
MemTable::try_new(schema, partitions)
}
pub(super) fn register_typed_table(
ctx: &SessionContext,
name: &str,
schema: SchemaRef,
batches: Vec<RecordBatch>,
) -> Result<(), QueryEngineError> {
let table = mem_table(schema, batches)?;
ctx.register_table(name, Arc::new(table))?;
Ok(())
}
type TypedTableRegistration = (
&'static str,
fn() -> SchemaRef,
fn(&PartitionTables) -> &RecordBatch,
);
const JOURNAL_TABLE_REGISTRATIONS: [TypedTableRegistration; 15] = [
(USAGE_RAW_TABLE, decode::usage::schema, |t| &t.usage_raw),
(MODEL_CALL_RAW_TABLE, decode::model_call::schema, |t| {
&t.model_call_raw
}),
(ATTRIBUTION_RAW_TABLE, decode::attribution::schema, |t| {
&t.attribution_raw
}),
(TURN_FAILED_RAW_TABLE, decode::turn_failed::schema, |t| {
&t.turn_failed_raw
}),
(
TURN_DISPATCH_RAW_TABLE,
decode::turn_dispatch::schema,
|t| &t.turn_dispatch_raw,
),
(PAYMENTS_RAW_TABLE, decode::payments::schema, |t| {
&t.payments_raw
}),
(REFUSALS_RAW_TABLE, decode::refusals::schema, |t| {
&t.refusals_raw
}),
(
WALLET_LINK_LIFECYCLE_RAW_TABLE,
decode::wallet_link_lifecycle::schema,
|t| &t.wallet_link_lifecycle_raw,
),
(APPROVALS_RAW_TABLE, decode::approvals::schema, |t| {
&t.approvals_raw
}),
(HANDOFFS_RAW_TABLE, decode::handoffs::schema, |t| {
&t.handoffs_raw
}),
(
GRANT_REPLAYS_RAW_TABLE,
decode::grant_replays::schema,
|t| &t.grant_replays_raw,
),
(SUMMARY_RAW_TABLE, decode::summary::schema, |t| {
&t.summary_raw
}),
(FIRES_RAW_TABLE, decode::fires::schema, |t| &t.fires_raw),
(
ROUTINE_LIFECYCLE_RAW_TABLE,
decode::routine_lifecycle::schema,
|t| &t.routine_lifecycle_raw,
),
(
ROUTINE_SETUP_RAW_TABLE,
decode::routine_setup::schema,
|t| &t.routine_setup_raw,
),
];
const MESSAGE_CONTENT_TABLE_REGISTRATIONS: [TypedTableRegistration; 2] = [
(
MESSAGES_RAW_TABLE,
decode::message_content::messages_schema,
|t| &t.messages_raw,
),
(
TOOL_CALLS_RAW_TABLE,
decode::message_content::tool_calls_schema,
|t| &t.tool_calls_raw,
),
];
fn register_typed_table_recipe(
ctx: &SessionContext,
recipe: TypedTableRegistration,
partition_tables: &[PartitionTables],
) -> Result<(), QueryEngineError> {
let (name, schema_fn, accessor) = recipe;
register_typed_table(
ctx,
name,
schema_fn(),
partition_tables
.iter()
.map(|tables| accessor(tables).clone())
.collect(),
)
}
pub(super) fn register_typed_journal_tables(
ctx: &SessionContext,
partition_tables: &[PartitionTables],
) -> Result<(), QueryEngineError> {
for recipe in JOURNAL_TABLE_REGISTRATIONS {
register_typed_table_recipe(ctx, recipe, partition_tables)?;
}
Ok(())
}
pub(super) fn register_message_content_tables(
ctx: &SessionContext,
partition_tables: &[PartitionTables],
) -> Result<(), QueryEngineError> {
for recipe in MESSAGE_CONTENT_TABLE_REGISTRATIONS {
register_typed_table_recipe(ctx, recipe, partition_tables)?;
}
Ok(())
}
#[allow(clippy::too_many_lines)] pub(super) async fn create_scope_dependent_views(
ctx: &SessionContext,
scope: RegistrationScope,
) -> Result<(), QueryEngineError> {
let is_fleet = scope.is_fleet();
let attribution_view_body = if is_fleet {
ATTRIBUTION_VIEW_SQL
} else {
ATTRIBUTION_REDACTED_VIEW_SQL
};
let create_attribution_view_sql =
format!("CREATE VIEW {ATTRIBUTION_TABLE} AS {attribution_view_body}");
ctx.sql(&create_attribution_view_sql)
.await?
.collect()
.await?;
let payments_view_body = if is_fleet {
PAYMENTS_VIEW_SQL
} else {
PAYMENTS_REDACTED_VIEW_SQL
};
let create_payments_view_sql = format!("CREATE VIEW {PAYMENTS_TABLE} AS {payments_view_body}");
ctx.sql(&create_payments_view_sql).await?.collect().await?;
let refusals_view_body = if is_fleet {
REFUSALS_VIEW_SQL
} else {
REFUSALS_REDACTED_VIEW_SQL
};
let create_refusals_view_sql = format!("CREATE VIEW {REFUSALS_TABLE} AS {refusals_view_body}");
ctx.sql(&create_refusals_view_sql).await?.collect().await?;
let wallet_link_lifecycle_view_body = if is_fleet {
WALLET_LINK_LIFECYCLE_VIEW_SQL
} else {
WALLET_LINK_LIFECYCLE_REDACTED_VIEW_SQL
};
let create_wallet_link_lifecycle_view_sql =
format!("CREATE VIEW {WALLET_LINK_LIFECYCLE_TABLE} AS {wallet_link_lifecycle_view_body}");
ctx.sql(&create_wallet_link_lifecycle_view_sql)
.await?
.collect()
.await?;
let messages_view_body = if is_fleet {
MESSAGES_VIEW_SQL
} else {
MESSAGES_REDACTED_VIEW_SQL
};
let create_messages_view_sql = format!("CREATE VIEW {MESSAGES_TABLE} AS {messages_view_body}");
ctx.sql(&create_messages_view_sql).await?.collect().await?;
let tool_calls_view_body = if is_fleet {
TOOL_CALLS_VIEW_SQL
} else {
TOOL_CALLS_REDACTED_VIEW_SQL
};
let create_tool_calls_view_sql =
format!("CREATE VIEW {TOOL_CALLS_TABLE} AS {tool_calls_view_body}");
ctx.sql(&create_tool_calls_view_sql)
.await?
.collect()
.await?;
let approvals_view_body = if is_fleet {
APPROVALS_VIEW_SQL
} else {
APPROVALS_REDACTED_VIEW_SQL
};
let create_approvals_view_sql =
format!("CREATE VIEW {APPROVALS_TABLE} AS {approvals_view_body}");
ctx.sql(&create_approvals_view_sql).await?.collect().await?;
let handoffs_view_body = if is_fleet {
HANDOFFS_VIEW_SQL
} else {
HANDOFFS_REDACTED_VIEW_SQL
};
let create_handoffs_view_sql = format!("CREATE VIEW {HANDOFFS_TABLE} AS {handoffs_view_body}");
ctx.sql(&create_handoffs_view_sql).await?.collect().await?;
let grant_replays_view_body = if is_fleet {
GRANT_REPLAYS_VIEW_SQL
} else {
GRANT_REPLAYS_REDACTED_VIEW_SQL
};
let create_grant_replays_view_sql =
format!("CREATE VIEW {GRANT_REPLAYS_TABLE} AS {grant_replays_view_body}");
ctx.sql(&create_grant_replays_view_sql)
.await?
.collect()
.await?;
let create_usage_view_sql = format!("CREATE VIEW {USAGE_TABLE} AS {USAGE_VIEW_SQL}");
ctx.sql(&create_usage_view_sql).await?.collect().await?;
let create_model_call_view_sql =
format!("CREATE VIEW {MODEL_CALL_TABLE} AS {MODEL_CALL_VIEW_SQL}");
ctx.sql(&create_model_call_view_sql)
.await?
.collect()
.await?;
let create_turn_failed_view_sql =
format!("CREATE VIEW {TURN_FAILED_TABLE} AS {TURN_FAILED_VIEW_SQL}");
ctx.sql(&create_turn_failed_view_sql)
.await?
.collect()
.await?;
let create_turn_dispatch_view_sql =
format!("CREATE VIEW {TURN_DISPATCH_TABLE} AS {TURN_DISPATCH_VIEW_SQL}");
ctx.sql(&create_turn_dispatch_view_sql)
.await?
.collect()
.await?;
if is_fleet {
let create_summary_view_sql = format!("CREATE VIEW {SUMMARY_TABLE} AS {SUMMARY_VIEW_SQL}");
ctx.sql(&create_summary_view_sql).await?.collect().await?;
}
match scope {
RegistrationScope::Fleet => {
let create_fires_view_sql = format!("CREATE VIEW {FIRES_TABLE} AS {FIRES_VIEW_SQL}");
ctx.sql(&create_fires_view_sql).await?.collect().await?;
}
RegistrationScope::Owner => {
let create_fires_view_sql =
format!("CREATE VIEW {FIRES_TABLE} AS {FIRES_OWNED_VIEW_SQL}");
ctx.sql(&create_fires_view_sql).await?.collect().await?;
}
RegistrationScope::Grant => {}
}
if is_fleet {
let create_routine_lifecycle_view_sql =
format!("CREATE VIEW {ROUTINE_LIFECYCLE_TABLE} AS {ROUTINE_LIFECYCLE_VIEW_SQL}");
ctx.sql(&create_routine_lifecycle_view_sql)
.await?
.collect()
.await?;
}
match scope {
RegistrationScope::Fleet => {
let sql = format!("CREATE VIEW {ROUTINE_SETUP_TABLE} AS {ROUTINE_SETUP_VIEW_SQL}");
ctx.sql(&sql).await?.collect().await?;
}
RegistrationScope::Owner => {
let sql =
format!("CREATE VIEW {ROUTINE_SETUP_TABLE} AS {ROUTINE_SETUP_OWNED_VIEW_SQL}");
ctx.sql(&sql).await?.collect().await?;
}
RegistrationScope::Grant => {}
}
if matches!(scope, RegistrationScope::Fleet | RegistrationScope::Owner) {
let sql = format!("CREATE VIEW {ROUTINE_GRANTS_TABLE} AS {ROUTINE_GRANTS_VIEW_SQL}");
ctx.sql(&sql).await?.collect().await?;
let sql = format!(
"CREATE VIEW {ROUTINE_ACTIVE_GRANTS_TABLE} AS {ROUTINE_ACTIVE_GRANTS_VIEW_SQL}"
);
ctx.sql(&sql).await?.collect().await?;
let sql = format!("CREATE VIEW {ROUTINE_OVERVIEW_TABLE} AS {ROUTINE_OVERVIEW_VIEW_SQL}");
ctx.sql(&sql).await?.collect().await?;
}
Ok(())
}
#[allow(clippy::too_many_lines)] pub(super) fn register_reference_tables(
ctx: &SessionContext,
reference: &ReferenceData,
) -> Result<(), QueryEngineError> {
let persona_rows: Vec<PersonaRow> = reference.personas.iter().map(PersonaRow::from).collect();
let persona_batch =
decode::persona::decode_personas_batch(&persona_rows).map_err(|source| {
QueryEngineError::Decode {
table: PERSONAS_TABLE,
source,
}
})?;
register_typed_table(
ctx,
PERSONAS_TABLE,
decode::persona::personas_schema(),
vec![persona_batch],
)?;
let participation_rows: Vec<ParticipationRow> = reference
.participations
.iter()
.map(|(persona_id, participation)| ParticipationRow::new(persona_id, participation))
.collect();
let participation_batch = decode::persona::decode_participations_batch(&participation_rows)
.map_err(|source| QueryEngineError::Decode {
table: PARTICIPATIONS_TABLE,
source,
})?;
register_typed_table(
ctx,
PARTICIPATIONS_TABLE,
decode::persona::participations_schema(),
vec![participation_batch],
)?;
let persona_identity_rows: Vec<PersonaIdentityRow> = reference
.personas
.iter()
.flat_map(|profile| {
profile
.identities
.iter()
.map(move |identity| PersonaIdentityRow::new(&profile.persona_id, identity))
})
.collect();
let persona_identity_batch = decode::persona::decode_persona_identities_batch(
&persona_identity_rows,
)
.map_err(|source| QueryEngineError::Decode {
table: PERSONA_IDENTITIES_TABLE,
source,
})?;
register_typed_table(
ctx,
PERSONA_IDENTITIES_TABLE,
decode::persona::persona_identities_schema(),
vec![persona_identity_batch],
)?;
let wallet_rows: Vec<PersonaWalletRow> = reference
.wallets
.iter()
.map(|(persona_id, wallet)| PersonaWalletRow::new(persona_id, wallet))
.collect();
let wallet_batch =
decode::persona::decode_persona_wallets_batch(&wallet_rows).map_err(|source| {
QueryEngineError::Decode {
table: PERSONA_WALLETS_TABLE,
source,
}
})?;
register_typed_table(
ctx,
PERSONA_WALLETS_TABLE,
decode::persona::persona_wallets_schema(),
vec![wallet_batch],
)?;
let spend_policy_rows: Vec<PersonaSpendPolicyRow> = reference
.spend_policies
.iter()
.map(|(persona_id, policy)| PersonaSpendPolicyRow::new(persona_id, policy))
.collect();
let spend_policy_batch = decode::persona::decode_persona_spend_policies_batch(
&spend_policy_rows,
)
.map_err(|source| QueryEngineError::Decode {
table: PERSONA_SPEND_POLICIES_TABLE,
source,
})?;
register_typed_table(
ctx,
PERSONA_SPEND_POLICIES_TABLE,
decode::persona::persona_spend_policies_schema(),
vec![spend_policy_batch],
)?;
let credential_rows: Vec<PersonaCredentialRow> = reference
.credentials
.iter()
.map(|(persona_id, credential)| PersonaCredentialRow::new(persona_id, credential))
.collect();
let credential_batch = decode::persona::decode_persona_credentials_batch(&credential_rows)
.map_err(|source| QueryEngineError::Decode {
table: PERSONA_CREDENTIALS_TABLE,
source,
})?;
register_typed_table(
ctx,
PERSONA_CREDENTIALS_TABLE,
decode::persona::persona_credentials_schema(),
vec![credential_batch],
)?;
let usage_rows: Vec<PersonaUsageRow> = reference
.usage_rollups
.iter()
.map(|(persona_id, rollup)| PersonaUsageRow::new(persona_id, rollup))
.collect();
let usage_batch =
decode::persona::decode_persona_usage_batch(&usage_rows).map_err(|source| {
QueryEngineError::Decode {
table: PERSONA_USAGE_TABLE,
source,
}
})?;
register_typed_table(
ctx,
PERSONA_USAGE_TABLE,
decode::persona::persona_usage_schema(),
vec![usage_batch],
)?;
let dashboard_batch =
decode::dashboard::decode_dashboard_batch(&reference.dashboard).map_err(|source| {
QueryEngineError::Decode {
table: DASHBOARD_TABLE,
source,
}
})?;
register_typed_table(
ctx,
DASHBOARD_TABLE,
decode::dashboard::schema(),
vec![dashboard_batch],
)?;
Ok(())
}
pub(super) fn register_routines_table(
ctx: &SessionContext,
reference: &ReferenceData,
) -> Result<(), QueryEngineError> {
let routines_batch =
decode::routines::decode_routines_batch(&reference.routines).map_err(|source| {
QueryEngineError::Decode {
table: ROUTINES_TABLE,
source,
}
})?;
register_typed_table(
ctx,
ROUTINES_TABLE,
decode::routines::schema(),
vec![routines_batch],
)
}