pub(crate) mod ipc;
use std::collections::{BTreeSet, VecDeque};
use std::fmt;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use connectrpc::client::ClientTransport;
use datafusion::common::config::Dialect;
use datafusion::error::DataFusionError;
use datafusion::execution::SessionStateDefaults;
use datafusion::execution::config::SessionConfig;
use datafusion::execution::context::SessionState;
use datafusion::execution::memory_pool::FairSpillPool;
use datafusion::execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};
use datafusion::execution::session_state::SessionStateBuilder;
use futures::{Stream, StreamExt as _};
use polyc_crypto::signing_role::{RoleTrustSet, TurnReadRole};
use polyc_query_model::{
CatalogColumn, CatalogColumnType, CatalogReply, CatalogTable, DataFrame, ErrorClass,
QueryOutcome, QueryRequest, ResultFrame, SchemaFrame, TerminalFrame, Truncation,
};
use polyc_state::id::{Audience, NamespaceId, OwnerId};
use polyc_state::immutable::AtRestProtection;
use polyc_state::projection::artifact::{ManifestTrust, ObjectNamespace};
use polyc_state::query_audit::{QueryId, RequesterId};
use polyc_state_connect::journal::client::JournalClient;
use polyc_state_connect::projection::client::ProjectionCatalogClient;
use polyc_state_connect::query_audit::client::QueryAuditClient;
use polyc_state_connect::wire::DeclaredCall;
use polyc_storage_gcs::GcsReadClient;
use sha2::Digest as _;
use tokio_util::sync::CancellationToken;
use crate::core_execution::{
CoreArtifactAuthority, CoreExecutionAdmission, CoreExecutionAdmissionInput, CoreExecutionError,
CoreResultStream, classify_error, classify_resolution,
};
use crate::core_production::{
ConnectCoreMetadata, FleetGcsSource, GcsReadNamespace, VisibleGcsSource, fleet_gcs_artifacts,
visible_gcs_artifacts,
};
use crate::core_resolution::{
CatalogCompiler, CoreAuditContext, CoreConsistency, CoreMetadataAuthority,
CoreOperationContext, CoreParameter, CorePlanOutcome, CorePlanningAuthority, CoreQueryRequest,
CoreRealm, CoreRequestedBounds, CoreResolutionError, ProjectedCorePolicy,
ProjectedCorePolicyInput, catalog_tables,
};
use crate::limits::{QueryLimits, SpillDirGuard};
use polyc_query_credential::credential::{
CredentialAuthority, CredentialWitness, SessionVerification, SystemUnixClock, UnixClock,
};
use polyc_query_credential::principal::{PersonaSource, PrincipalError, PrincipalKind, Scoping};
use polyc_query_credential::session::QueryScope;
pub use ipc::FrameEncodeError;
use ipc::FrameEncoder;
pub struct QueryCredential(String);
impl QueryCredential {
#[must_use]
pub const fn from_bearer(token: String) -> Self {
Self(token)
}
}
impl fmt::Debug for QueryCredential {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("QueryCredential([REDACTED])")
}
}
#[derive(Clone)]
pub struct ArtifactNamespace {
pub namespace: String,
pub protection: AtRestProtection,
}
pub struct ArtifactSource {
pub artifacts: GcsReadClient,
pub namespaces: Vec<ArtifactNamespace>,
}
impl fmt::Debug for ArtifactSource {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ArtifactSource")
.field("namespaces", &self.namespaces.len())
.finish_non_exhaustive()
}
}
impl fmt::Debug for ArtifactNamespace {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ArtifactNamespace")
.field("protection", &self.protection)
.finish_non_exhaustive()
}
}
pub enum QueryServiceRealm {
Visible {
artifacts: GcsReadClient,
namespaces: Vec<ArtifactNamespace>,
additional_artifacts: Vec<ArtifactSource>,
fleet_namespaces: Vec<String>,
},
Fleet {
visible_artifacts: GcsReadClient,
visible_namespaces: Vec<ArtifactNamespace>,
additional_visible_artifacts: Vec<ArtifactSource>,
fleet_artifacts: GcsReadClient,
fleet_namespaces: Vec<ArtifactNamespace>,
additional_fleet_artifacts: Vec<ArtifactSource>,
},
}
impl fmt::Debug for QueryServiceRealm {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(match self {
Self::Visible { .. } => "visible",
Self::Fleet { .. } => "fleet",
})
}
}
#[derive(Debug, Clone, Copy)]
pub struct QueryServicePolicy {
pub result_release_bytes: u64,
pub response_frame_bytes: u64,
pub manifest_bytes: u64,
pub artifact_file_bytes: u64,
pub artifact_range_bytes: u64,
pub source_decode_bytes: u64,
pub max_concurrent_executions: usize,
pub revalidation_interval: Duration,
pub execution_memory_bytes: usize,
}
#[derive(Debug, thiserror::Error)]
pub enum QueryServiceError {
#[error("the query service composition is invalid")]
InvalidComposition,
#[error("the presented credential does not authorize this query")]
Unauthorized,
#[error("the query could not be planned")]
Resolution(ErrorClass),
#[error("this query identity was already recorded")]
AlreadyRecorded,
#[error("the query could not be executed")]
Execution(ErrorClass),
#[error("the result could not be encoded")]
Encode(#[source] FrameEncodeError),
#[error("the result source evidence could not be reported")]
Evidence,
}
pub struct CredentialSource {
pub persona: Arc<dyn PersonaSource>,
pub turn_read_trust: RoleTrustSet<TurnReadRole>,
pub sessions: Arc<dyn SessionVerification>,
}
impl fmt::Debug for CredentialSource {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("CredentialSource")
.finish_non_exhaustive()
}
}
pub struct ProjectedCoreComposition<T> {
pub namespace: String,
pub persona_memory_namespace: String,
pub projection_owner: String,
pub policy: QueryServicePolicy,
pub limits: QueryLimits,
pub realm: QueryServiceRealm,
pub trust: Arc<dyn ManifestTrust>,
pub journal: JournalClient<T>,
pub projections: ProjectionCatalogClient<T>,
pub audit: QueryAuditClient<T>,
pub versioned: polyc_state_connect::versioned_source::VersionedSourceClient<T>,
pub persona_memory: polyc_state_connect::persona_memory_journal::PersonaMemoryJournalClient<T>,
pub observation: polyc_state_connect::observation::ObservationClient<T>,
}
impl<T> fmt::Debug for ProjectedCoreComposition<T> {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ProjectedCoreComposition")
.field("policy", &self.policy)
.field("realm", &self.realm)
.finish_non_exhaustive()
}
}
pub struct ProjectedCoreService {
credential: Arc<CredentialAuthority>,
clock: Arc<dyn UnixClock>,
planning: CorePlanningAuthority,
compiler: CatalogCompiler,
realm: ComposedRealm,
trust: Arc<dyn ManifestTrust>,
admission: Arc<CoreExecutionAdmission>,
limits: QueryLimits,
revalidation_interval: Duration,
composite_traces: CompositeTraceCache,
#[allow(dead_code, reason = "held only for its Drop side effect")]
spill_guard: SpillDirGuard,
}
const MAX_COMPOSITE_TRACE_GRANTS: usize = 256;
#[derive(Clone, Default)]
struct CompositeTraceCache {
entries: Arc<Mutex<VecDeque<CompositeTraceGrant>>>,
}
#[derive(Clone)]
struct CompositeTraceGrant {
key: [u8; 32],
conversation_id: String,
personas: Vec<String>,
}
impl CompositeTraceCache {
fn remember(&self, grant: CompositeTraceGrant) {
let Ok(mut entries) = self.entries.lock() else {
return;
};
entries.retain(|entry| entry.key != grant.key);
entries.push_back(grant);
while entries.len() > MAX_COMPOSITE_TRACE_GRANTS {
entries.pop_front();
}
}
fn personas(&self, key: &[u8; 32], conversation_id: &str) -> Option<Vec<String>> {
let entries = self.entries.lock().ok()?;
entries
.iter()
.find(|entry| &entry.key == key && entry.conversation_id == conversation_id)
.map(|entry| entry.personas.clone())
}
}
struct CompositeTraceCollector {
cache: CompositeTraceCache,
key: [u8; 32],
conversation_id: String,
caller_column: usize,
record_column: usize,
personas: BTreeSet<String>,
}
impl CompositeTraceCollector {
fn try_new(
cache: CompositeTraceCache,
key: [u8; 32],
conversation_id: String,
schema: &arrow::datatypes::Schema,
) -> Result<Self, QueryServiceError> {
let caller_column = schema
.index_of("caller_persona_id")
.map_err(|_missing| QueryServiceError::Unauthorized)?;
let record_column = schema
.index_of("record_persona_id")
.map_err(|_missing| QueryServiceError::Unauthorized)?;
Ok(Self {
cache,
key,
conversation_id,
caller_column,
record_column,
personas: BTreeSet::new(),
})
}
fn observe(&mut self, batch: &arrow::record_batch::RecordBatch) -> Result<(), ErrorClass> {
for index in [self.caller_column, self.record_column] {
let values = batch
.column(index)
.as_any()
.downcast_ref::<arrow::array::StringArray>()
.ok_or(ErrorClass::Internal)?;
for value in values.iter().flatten().filter(|value| !value.is_empty()) {
self.personas.insert(value.to_owned());
if self.personas.len() >= polyc_query_model::MAX_SOURCE_PINS {
return Err(ErrorClass::Bounds);
}
}
}
Ok(())
}
fn finish(self) {
self.cache.remember(CompositeTraceGrant {
key: self.key,
conversation_id: self.conversation_id,
personas: self.personas.into_iter().collect(),
});
}
}
impl fmt::Debug for ProjectedCoreService {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ProjectedCoreService")
.field("realm", &self.realm.kind())
.field("revalidation_interval", &self.revalidation_interval)
.finish_non_exhaustive()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum RealmKind {
Visible,
Fleet,
}
impl fmt::Display for RealmKind {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(match self {
Self::Visible => "visible",
Self::Fleet => "fleet",
})
}
}
enum ComposedRealm {
Visible {
sources: Vec<VisibleGcsSource>,
fleet_namespaces: Vec<ObjectNamespace>,
},
Fleet {
visible_sources: Vec<VisibleGcsSource>,
fleet_sources: Vec<FleetGcsSource>,
},
}
impl ComposedRealm {
const fn kind(&self) -> RealmKind {
match self {
Self::Visible { .. } => RealmKind::Visible,
Self::Fleet { .. } => RealmKind::Fleet,
}
}
fn authority(
&self,
trust: Arc<dyn ManifestTrust>,
witness: Arc<CredentialWitness>,
revalidation_interval: Duration,
admission: Arc<CoreExecutionAdmission>,
) -> Result<CoreArtifactAuthority, CoreExecutionError> {
match self {
Self::Visible {
sources,
fleet_namespaces,
} => visible_gcs_artifacts(
sources.clone(),
fleet_namespaces.clone(),
trust,
witness,
revalidation_interval,
admission,
),
Self::Fleet {
visible_sources,
fleet_sources,
} => fleet_gcs_artifacts(
visible_sources.clone(),
fleet_sources.clone(),
trust,
witness,
revalidation_interval,
admission,
),
}
}
}
fn read_namespaces(
namespaces: Vec<ArtifactNamespace>,
) -> Result<Vec<GcsReadNamespace>, QueryServiceError> {
if namespaces.is_empty() {
return Err(QueryServiceError::InvalidComposition);
}
namespaces
.into_iter()
.map(|entry| {
ObjectNamespace::try_new(entry.namespace)
.map(|namespace| GcsReadNamespace::new(namespace, entry.protection))
.map_err(|_invalid| QueryServiceError::InvalidComposition)
})
.collect()
}
fn topology_namespaces(namespaces: Vec<String>) -> Result<Vec<ObjectNamespace>, QueryServiceError> {
namespaces
.into_iter()
.map(|namespace| {
ObjectNamespace::try_new(namespace)
.map_err(|_invalid| QueryServiceError::InvalidComposition)
})
.collect()
}
pub(crate) fn projected_session_state(
runtime: Arc<RuntimeEnv>,
) -> Result<SessionState, DataFusionError> {
let mut config = SessionConfig::new()
.with_information_schema(false)
.with_target_partitions(1)
.with_batch_size(8192);
{
let options = config.options_mut();
options.execution.enable_recursive_ctes = false;
options.execution.planning_concurrency = 1;
options.execution.collect_statistics = true;
options.execution.parquet.pushdown_filters = false;
options.execution.parquet.reorder_filters = false;
options.execution.parquet.enable_page_index = true;
options.execution.parquet.bloom_filter_on_read = true;
options.catalog.information_schema = false;
options.catalog.default_catalog = "datafusion".to_string();
options.catalog.default_schema = "public".to_string();
options.catalog.create_default_catalog_and_schema = true;
options.sql_parser.dialect = Dialect::Generic;
options.sql_parser.enable_ident_normalization = true;
options.sql_parser.parse_float_as_decimal = false;
}
let mut state = SessionStateBuilder::new()
.with_runtime_env(runtime)
.with_config(config)
.with_expr_planners(SessionStateDefaults::default_expr_planners())
.with_scalar_functions(crate::function_surface::closed_scalar_functions())
.with_aggregate_functions(crate::function_surface::closed_aggregate_functions())
.with_window_functions(crate::function_surface::closed_window_functions())
.build();
polyc_query_json::register_all(&mut state)?;
Ok(state)
}
impl ProjectedCoreService {
#[allow(
clippy::too_many_lines,
reason = "the constructor validates and transfers every realm-typed composition field in one audit boundary"
)]
pub fn try_new<T>(
credential: CredentialSource,
composition: ProjectedCoreComposition<T>,
) -> Result<Self, QueryServiceError>
where
T: ClientTransport + Send + Sync + 'static,
<T::ResponseBody as connectrpc::http_body::Body>::Error: fmt::Display,
{
let ProjectedCoreComposition {
namespace,
persona_memory_namespace,
projection_owner,
policy,
limits,
realm,
trust,
journal,
projections,
audit,
versioned,
persona_memory,
observation,
} = composition;
let namespace = namespace.as_str();
let projection_owner = projection_owner.as_str();
let core_policy = ProjectedCorePolicy::try_from(ProjectedCorePolicyInput {
result_release_bytes: policy.result_release_bytes,
response_frame_bytes: policy.response_frame_bytes,
manifest_bytes: policy.manifest_bytes,
artifact_file_bytes: policy.artifact_file_bytes,
artifact_range_bytes: policy.artifact_range_bytes,
source_decode_bytes: policy.source_decode_bytes,
})
.map_err(|_invalid| QueryServiceError::InvalidComposition)?;
let metadata: Arc<dyn CoreMetadataAuthority> = Arc::new(ConnectCoreMetadata::new(
journal,
projections,
audit,
versioned,
persona_memory,
observation,
persona_memory_namespace,
));
let planning = CorePlanningAuthority::new(
NamespaceId::new(namespace),
OwnerId::new(projection_owner),
core_policy,
metadata,
)
.map_err(|_invalid| QueryServiceError::InvalidComposition)?;
let realm = match realm {
QueryServiceRealm::Visible {
artifacts,
namespaces,
additional_artifacts,
fleet_namespaces,
} => {
let mut sources = vec![VisibleGcsSource::new(
artifacts,
read_namespaces(namespaces)?,
)];
for source in additional_artifacts {
sources.push(VisibleGcsSource::new(
source.artifacts,
read_namespaces(source.namespaces)?,
));
}
ComposedRealm::Visible {
sources,
fleet_namespaces: topology_namespaces(fleet_namespaces)?,
}
}
QueryServiceRealm::Fleet {
visible_artifacts,
visible_namespaces,
additional_visible_artifacts,
fleet_artifacts,
fleet_namespaces,
additional_fleet_artifacts,
} => {
let mut visible_sources = vec![VisibleGcsSource::new(
visible_artifacts,
read_namespaces(visible_namespaces)?,
)];
for source in additional_visible_artifacts {
visible_sources.push(VisibleGcsSource::new(
source.artifacts,
read_namespaces(source.namespaces)?,
));
}
let mut fleet_sources = vec![FleetGcsSource::new(
fleet_artifacts,
read_namespaces(fleet_namespaces)?,
)];
for source in additional_fleet_artifacts {
fleet_sources.push(FleetGcsSource::new(
source.artifacts,
read_namespaces(source.namespaces)?,
));
}
ComposedRealm::Fleet {
visible_sources,
fleet_sources,
}
}
};
let admission = Arc::new(
CoreExecutionAdmission::try_from(CoreExecutionAdmissionInput {
max_concurrent_executions: policy.max_concurrent_executions,
})
.map_err(|_invalid| QueryServiceError::InvalidComposition)?,
);
if policy.revalidation_interval.is_zero()
|| policy.execution_memory_bytes == 0
|| limits.spill_quota_bytes == 0
{
return Err(QueryServiceError::InvalidComposition);
}
let spill_guard = SpillDirGuard::new(limits.spill_dir.clone());
let runtime = RuntimeEnvBuilder::new()
.with_memory_pool(Arc::new(FairSpillPool::new(policy.execution_memory_bytes)))
.with_temp_file_path(&limits.spill_dir)
.with_max_temp_directory_size(limits.spill_quota_bytes)
.build_arc()
.map_err(|_datafusion| QueryServiceError::InvalidComposition)?;
let session = projected_session_state(runtime)
.map_err(|_datafusion| QueryServiceError::InvalidComposition)?;
Ok(Self {
credential: Arc::new(CredentialAuthority::verifying(
credential.persona,
credential.turn_read_trust,
credential.sessions,
)),
clock: Arc::new(SystemUnixClock),
planning,
compiler: CatalogCompiler::new(session),
realm,
trust,
admission,
limits,
revalidation_interval: policy.revalidation_interval,
composite_traces: CompositeTraceCache::default(),
spill_guard,
})
}
pub async fn start_projected(
&self,
credential: QueryCredential,
query_id: &str,
request: &QueryRequest,
) -> Result<ProjectedResultStream, QueryServiceError> {
self.start_projected_admitted(credential, query_id, request, Admission::Internal)
.await
}
pub async fn start_projected_edge(
&self,
credential: QueryCredential,
query_id: &str,
request: &QueryRequest,
) -> Result<ProjectedResultStream, QueryServiceError> {
self.start_projected_admitted(credential, query_id, request, Admission::Edge)
.await
}
pub async fn describe_catalog(
&self,
credential: QueryCredential,
) -> Result<CatalogReply, QueryServiceError> {
self.describe_catalog_admitted(credential, Admission::Internal)
.await
}
pub async fn describe_catalog_edge(
&self,
credential: QueryCredential,
) -> Result<CatalogReply, QueryServiceError> {
self.describe_catalog_admitted(credential, Admission::Edge)
.await
}
async fn describe_catalog_admitted(
&self,
credential: QueryCredential,
admission: Admission,
) -> Result<CatalogReply, QueryServiceError> {
let (_witness, scoping) = self.admit_credential(credential, admission).await?;
admit_edge_principal_kind(&scoping, self.realm.kind())?;
let operation = CoreOperationContext::from_declared(
&DeclaredCall::live(polyc_state_connect::state_audience(), self.limits.timeout),
self.limits.timeout,
);
let _permit = self
.admission
.acquire(&operation, &CancellationToken::new())
.await
.map_err(|error| execution_refusal(&error))?;
let realm = CoreRealm::from_scope(scoping.scope());
let reply = catalog_reply(realm, scoping.scope())?;
crate::metrics::record_catalog_described(
catalog_realm_label(realm),
catalog_principal_label(scoping.principal_kind()),
);
tracing::debug!(
realm = catalog_realm_label(realm),
principal_kind = catalog_principal_label(scoping.principal_kind()),
tables = reply.tables().len(),
"described the query catalog"
);
Ok(reply)
}
#[allow(
clippy::too_many_lines,
reason = "one method keeps credential admission, fixed-statement scoping, planning, binding, and observed release in order"
)]
async fn start_projected_admitted(
&self,
credential: QueryCredential,
query_id: &str,
request: &QueryRequest,
admission: Admission,
) -> Result<ProjectedResultStream, QueryServiceError> {
let credential_key: [u8; 32] = sha2::Sha256::digest(credential.0.as_bytes()).into();
let (mut witness, scoping) = self.admit_credential(credential, admission).await?;
admit_control_fleet(&scoping, self.realm.kind(), request.sql())?;
let composite_statement = admit_composite_trace(&scoping, self.realm.kind(), request)?;
let composite_scope = match composite_statement {
Some(CompositeTraceStatement::Trace) | None => None,
Some(CompositeTraceStatement::Memory) => {
let conversation = scoping
.conversation_id()
.ok_or(QueryServiceError::Unauthorized)?;
let personas = self
.composite_traces
.personas(&credential_key, conversation)
.ok_or(QueryServiceError::Unauthorized)?;
witness.retain_composite_trace_memory_sources(personas.clone());
Some(
self.credential
.composite_trace_memory_scope(&scoping, &personas)
.await
.map_err(refusal)?,
)
}
Some(CompositeTraceStatement::Routine) => {
witness.retain_composite_trace_routine();
Some(QueryScope::Fleet)
}
Some(CompositeTraceStatement::Addresses) => {
let conversation = scoping
.conversation_id()
.ok_or(QueryServiceError::Unauthorized)?
.to_owned();
witness.retain_composite_trace_addresses();
Some(QueryScope::FleetConversation { conversation })
}
};
let requester = requester_of(&scoping)?;
let caller_bounds = requested_bounds(request);
let bounds = self
.planning
.effective_bounds(&self.limits, caller_bounds)
.map_err(|error| resolution_refusal(&error, &scoping))?;
let declared = DeclaredCall::live(Audience::new("state"), bounds.timeout());
let audit =
CoreAuditContext::from_scoped(QueryId::new(query_id), requester, &declared, bounds);
let core_request = CoreQueryRequest::new(
request.sql().to_owned(),
request.parameters().iter().map(parameter_of).collect(),
consistency_of(request),
caller_bounds,
);
let planned = if composite_statement == Some(CompositeTraceStatement::Memory) {
self.planning
.plan_composite_trace_memory(
&self.compiler,
&self.limits,
composite_scope.as_ref().unwrap_or_else(|| scoping.scope()),
scoping.allow_explain(),
audit,
core_request,
)
.await
} else {
self.planning
.plan(
&self.compiler,
&self.limits,
composite_scope.as_ref().unwrap_or_else(|| scoping.scope()),
scoping.allow_explain(),
audit,
core_request,
)
.await
};
let prepared = match planned.map_err(|error| resolution_refusal(&error, &scoping))? {
CorePlanOutcome::Granted(prepared) => *prepared,
CorePlanOutcome::AlreadyRecorded(_) => {
return Err(QueryServiceError::AlreadyRecorded);
}
};
let artifacts = self
.realm
.authority(
Arc::clone(&self.trust),
Arc::new(witness),
self.revalidation_interval,
Arc::clone(&self.admission),
)
.map_err(|error| execution_refusal(&error))?;
let bound = artifacts.bind(prepared).await;
drop(artifacts);
let rows = bound.map_err(|error| execution_refusal(&error))?.execute();
let collects_memory_sources = scoping
.composite_trace()
.is_some_and(|capability| capability.memory_statement_digest().is_some());
let collector = if composite_statement == Some(CompositeTraceStatement::Trace)
&& collects_memory_sources
{
Some(CompositeTraceCollector::try_new(
self.composite_traces.clone(),
credential_key,
scoping
.conversation_id()
.map(str::to_owned)
.ok_or(QueryServiceError::Unauthorized)?,
rows.schema(),
)?)
} else {
None
};
ProjectedResultStream::start_observed(
rows,
bounds.response_frame_bytes(),
bounds.result_release_bytes(),
collector,
)
}
async fn admit_credential(
&self,
credential: QueryCredential,
admission: Admission,
) -> Result<(CredentialWitness, Scoping), QueryServiceError> {
let (witness, scoping) = match admission {
Admission::Internal => {
CredentialWitness::admit_bearer(
credential.0,
Arc::clone(&self.credential),
Arc::clone(&self.clock),
)
.await
}
Admission::Edge => {
CredentialWitness::admit_grant_only(
credential.0,
Arc::clone(&self.credential),
Arc::clone(&self.clock),
)
.await
}
}
.map_err(refusal)?;
if admission == Admission::Edge {
admit_edge_principal_kind(&scoping, self.realm.kind())?;
}
admit_admin_fleet(&scoping, self.realm.kind())?;
Ok((witness, scoping))
}
}
fn resolution_refusal(error: &CoreResolutionError, scoping: &Scoping) -> QueryServiceError {
let realm = catalog_realm_label(CoreRealm::from_scope(scoping.scope()));
let principal_kind = catalog_principal_label(scoping.principal_kind());
if let CoreResolutionError::Statement(rejected) = error {
crate::metrics::record_statement_refusal(realm, principal_kind, rejected.reason_key());
tracing::warn!(
realm,
principal_kind,
reason = rejected.reason_key(),
"the projected read path refused a query statement"
);
} else {
tracing::warn!(
?error,
realm,
principal_kind,
"the projected read path could not resolve a query"
);
}
QueryServiceError::Resolution(crate::core_evidence::error_class_of(classify_resolution(
error,
)))
}
fn execution_refusal(error: &CoreExecutionError) -> QueryServiceError {
tracing::warn!(%error, source = ?std::error::Error::source(error), "the projected read path could not execute a query");
QueryServiceError::Execution(crate::core_evidence::error_class_of(classify_error(error)))
}
const fn refusal(_error: PrincipalError) -> QueryServiceError {
QueryServiceError::Unauthorized
}
const fn catalog_realm_label(realm: CoreRealm) -> &'static str {
match realm {
CoreRealm::Visible => "visible",
CoreRealm::Fleet => "fleet",
}
}
const fn catalog_principal_label(kind: PrincipalKind) -> &'static str {
match kind {
PrincipalKind::Admin => "admin",
PrincipalKind::ConversationGrantTurn => "conversation_grant_turn",
PrincipalKind::ConversationGrantWebSession => "conversation_grant_web_session",
PrincipalKind::ConversationGrantPersona => "conversation_grant_persona",
PrincipalKind::ConversationGrantCompositeTrace => "conversation_grant_composite_trace",
PrincipalKind::ConversationGrantAdminCompositeTrace => {
"conversation_grant_admin_composite_trace"
}
PrincipalKind::Persona => "persona",
PrincipalKind::ControlFleet => "control_fleet",
PrincipalKind::AdminFleet => "admin_fleet",
}
}
fn catalog_reply(realm: CoreRealm, scope: &QueryScope) -> Result<CatalogReply, QueryServiceError> {
let internal = || QueryServiceError::Execution(ErrorClass::Internal);
let mut tables = Vec::new();
for table in catalog_tables(realm, scope) {
let columns = table
.public_schema()
.fields()
.iter()
.map(|field| {
CatalogColumn::try_new(
field.name().clone(),
catalog_column_type(field.data_type())?,
field.is_nullable(),
)
.map_err(|_| internal())
})
.collect::<Result<Vec<_>, QueryServiceError>>()?;
tables
.push(CatalogTable::try_new(table.name().to_owned(), columns).map_err(|_| internal())?);
}
tables.sort_by(|a, b| a.name().cmp(b.name()));
CatalogReply::try_new(tables).map_err(|_| internal())
}
fn catalog_column_type(
data_type: &arrow::datatypes::DataType,
) -> Result<CatalogColumnType, QueryServiceError> {
match data_type {
arrow::datatypes::DataType::Utf8 => Ok(CatalogColumnType::Utf8),
arrow::datatypes::DataType::FixedSizeBinary(len) => Ok(CatalogColumnType::FixedBytes {
len: u32::try_from(*len)
.map_err(|_| QueryServiceError::Execution(ErrorClass::Internal))?,
}),
arrow::datatypes::DataType::UInt64 => Ok(CatalogColumnType::UInt64),
arrow::datatypes::DataType::Boolean => Ok(CatalogColumnType::Boolean),
_ => Err(QueryServiceError::Execution(ErrorClass::Internal)),
}
}
const ADMIN_COMPOSITE_TRACE_PURPOSE: &str = "conversation.trace";
fn requester_of(scoping: &Scoping) -> Result<RequesterId, QueryServiceError> {
if let Some(control_fleet) = scoping.control_fleet() {
return Ok(RequesterId::new(format!(
"control-fleet:{}",
control_fleet.purpose()
)));
}
if scoping.principal_kind() == PrincipalKind::ConversationGrantAdminCompositeTrace {
return Ok(RequesterId::new(format!(
"control-fleet:{ADMIN_COMPOSITE_TRACE_PURPOSE}"
)));
}
if let Some(admin_fleet) = scoping.admin_fleet() {
return Ok(RequesterId::new(format!(
"admin-fleet:{}:{}",
admin_fleet.admin_persona(),
admin_fleet.session()
)));
}
scoping.caller_identity().map_or_else(
|| {
scoping
.conversation_id()
.map_or(Err(QueryServiceError::Unauthorized), |conversation| {
Ok(RequesterId::new(format!("conversation:{conversation}")))
})
},
|persona| Ok(RequesterId::new(format!("persona:{persona}"))),
)
}
fn admit_control_fleet(
scoping: &Scoping,
realm: RealmKind,
sql: &str,
) -> Result<(), QueryServiceError> {
let Some(control_fleet) = scoping.control_fleet() else {
return Ok(());
};
match realm {
RealmKind::Fleet => {}
RealmKind::Visible => {
tracing::warn!(
%realm,
purpose = control_fleet.purpose(),
"a control-fleet grant was presented outside the Fleet realm"
);
return Err(QueryServiceError::Unauthorized);
}
}
if scoping.scope() != &QueryScope::Fleet {
tracing::warn!(
purpose = control_fleet.purpose(),
"a control-fleet grant resolved to a non-Fleet scope"
);
return Err(QueryServiceError::Unauthorized);
}
let actual: [u8; 32] = sha2::Sha256::digest(sql.as_bytes()).into();
if &actual != control_fleet.statement_digest() {
tracing::warn!(
purpose = control_fleet.purpose(),
"a control-fleet grant's statement digest did not match the statement presented"
);
return Err(QueryServiceError::Unauthorized);
}
Ok(())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Admission {
Internal,
Edge,
}
fn admit_edge_principal_kind(scoping: &Scoping, realm: RealmKind) -> Result<(), QueryServiceError> {
let admitted = matches!(
(scoping.principal_kind(), realm),
(
PrincipalKind::Persona
| PrincipalKind::ConversationGrantWebSession
| PrincipalKind::ConversationGrantPersona,
RealmKind::Visible
) | (PrincipalKind::AdminFleet, RealmKind::Fleet)
);
if admitted {
Ok(())
} else {
tracing::warn!(
%realm,
principal_kind = ?scoping.principal_kind(),
"a grant resolved to a principal kind this realm's edge listener does not admit"
);
Err(QueryServiceError::Unauthorized)
}
}
fn admit_admin_fleet(scoping: &Scoping, realm: RealmKind) -> Result<(), QueryServiceError> {
let Some(admin_fleet) = scoping.admin_fleet() else {
return Ok(());
};
match realm {
RealmKind::Fleet => {}
RealmKind::Visible => {
tracing::warn!(
%realm,
admin_persona = admin_fleet.admin_persona(),
"an admin-fleet grant was presented outside the Fleet realm"
);
return Err(QueryServiceError::Unauthorized);
}
}
if scoping.scope() != &QueryScope::Fleet {
tracing::warn!(
admin_persona = admin_fleet.admin_persona(),
"an admin-fleet grant resolved to a non-Fleet scope"
);
return Err(QueryServiceError::Unauthorized);
}
Ok(())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum CompositeTraceStatement {
Trace,
Memory,
Routine,
Addresses,
}
fn admit_composite_trace(
scoping: &Scoping,
realm: RealmKind,
request: &QueryRequest,
) -> Result<Option<CompositeTraceStatement>, QueryServiceError> {
let Some(capability) = scoping.composite_trace() else {
return Ok(None);
};
let conversation = scoping
.conversation_id()
.ok_or(QueryServiceError::Unauthorized)?;
let digest = polyc_crypto::hex::lower(&sha2::Sha256::digest(request.sql().as_bytes()));
let parameters = request.parameters();
let conversation_parameter = |index: usize| matches!(parameters.get(index), Some(polyc_query_model::Parameter::Utf8(value)) if value == conversation);
let conversation_partition = format!("conv-{conversation}");
let partition_parameter = |index: usize| matches!(parameters.get(index), Some(polyc_query_model::Parameter::Utf8(value)) if value == &conversation_partition);
if realm == RealmKind::Visible
&& digest == capability.trace_statement_digest()
&& parameters.len() == 1
&& partition_parameter(0)
{
return Ok(Some(CompositeTraceStatement::Trace));
}
if let (Some(memory_digest), Some(owner)) = (
capability.memory_statement_digest(),
scoping.caller_identity(),
) && realm == RealmKind::Visible
&& digest == memory_digest
&& parameters.len() == 3
&& matches!(parameters.first(), Some(polyc_query_model::Parameter::Utf8(value)) if value == &format!("persona-{owner}-mem"))
&& conversation_parameter(1)
&& partition_parameter(2)
{
return Ok(Some(CompositeTraceStatement::Memory));
}
if realm == RealmKind::Fleet
&& digest == capability.routine_statement_digest()
&& parameters.len() == 1
&& conversation_parameter(0)
{
return Ok(Some(CompositeTraceStatement::Routine));
}
let own_address_digest = polyc_crypto::hex::lower(&sha2::Sha256::digest(
polyc_query_model::statements::COMPOSITE_TRACE_ADDRESSES_SQL.as_bytes(),
));
if let Some(address_digest) = capability.address_statement_digest()
&& scoping.principal_kind() == PrincipalKind::ConversationGrantAdminCompositeTrace
&& realm == RealmKind::Fleet
&& digest == address_digest
&& digest == own_address_digest
&& parameters.len() == 1
&& partition_parameter(0)
{
return Ok(Some(CompositeTraceStatement::Addresses));
}
tracing::warn!(%realm, "a composite-trace grant did not match its fixed statement");
Err(QueryServiceError::Unauthorized)
}
const fn consistency_of(request: &QueryRequest) -> CoreConsistency {
match request.consistency() {
polyc_query_model::Consistency::Projected => CoreConsistency::Projected,
polyc_query_model::Consistency::RequireProjectedThrough(position) => {
CoreConsistency::RequireProjectedThrough(polyc_state::revision::JournalPosition::new(
position,
))
}
}
}
fn parameter_of(parameter: &polyc_query_model::Parameter) -> CoreParameter {
match parameter {
polyc_query_model::Parameter::Utf8(value) => CoreParameter::Utf8(value.clone()),
polyc_query_model::Parameter::UInt64(value) => CoreParameter::UInt64(*value),
polyc_query_model::Parameter::Boolean(value) => CoreParameter::Boolean(*value),
polyc_query_model::Parameter::Null => CoreParameter::Null,
}
}
const fn requested_bounds(request: &QueryRequest) -> CoreRequestedBounds {
let bounds = request.bounds();
CoreRequestedBounds::from_requested(
bounds.timeout(),
bounds.rows(),
bounds.result_bytes(),
bounds.frame_bytes(),
)
}
pub struct ProjectedResultStream {
rows: CoreResultStream,
encoder: FrameEncoder,
queued: Option<ResultFrame>,
stage: Stage,
released_rows: u64,
released_bytes: u64,
release_ceiling: u64,
byte_truncated: bool,
composite_trace: Option<CompositeTraceCollector>,
}
impl fmt::Debug for ProjectedResultStream {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ProjectedResultStream")
.field("stage", &self.stage)
.field("queued", &self.queued.is_some())
.finish_non_exhaustive()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Stage {
Streaming,
Terminated,
}
impl ProjectedResultStream {
#[cfg(test)]
pub(crate) fn start(
rows: CoreResultStream,
frame_ceiling: u64,
release_ceiling: u64,
) -> Result<Self, QueryServiceError> {
Self::start_observed(rows, frame_ceiling, release_ceiling, None)
}
fn start_observed(
rows: CoreResultStream,
frame_ceiling: u64,
release_ceiling: u64,
composite_trace: Option<CompositeTraceCollector>,
) -> Result<Self, QueryServiceError> {
let Ok(ceiling) = usize::try_from(frame_ceiling) else {
let _ = rows.report_failure(polyc_state::query_audit::ErrorClass::Internal);
return Err(QueryServiceError::InvalidComposition);
};
let (encoder, schema) = match FrameEncoder::start(rows.schema().clone(), ceiling) {
Ok(started) => started,
Err(error) => {
let class = error.class();
tracing::warn!(%error, ?class, "the result schema does not fit the frame ceiling");
let _ = rows.report_failure(crate::core_evidence::durable_class_of(class));
return Err(QueryServiceError::Encode(error));
}
};
rows.account_at_consumer();
Ok(Self {
rows,
encoder,
queued: Some(ResultFrame::Schema(schema)),
stage: Stage::Streaming,
released_rows: 0,
released_bytes: 0,
release_ceiling,
byte_truncated: false,
composite_trace,
})
}
fn release_terminal(
&mut self,
outcome: QueryOutcome,
) -> Result<ResultFrame, QueryServiceError> {
self.stage = Stage::Terminated;
if matches!(&outcome, QueryOutcome::Succeeded)
&& !self.byte_truncated
&& let Some(observer) = self.composite_trace.take()
{
observer.finish();
}
self.terminal(outcome)
}
fn terminal(&self, outcome: QueryOutcome) -> Result<ResultFrame, QueryServiceError> {
let evidence = crate::core_evidence::evidence_of(self.rows.source())
.map_err(|_unrepresentable| QueryServiceError::Evidence)?;
let truncation = if self.byte_truncated || self.rows.delivered().truncated() {
Truncation::TruncatedAt(self.released_rows)
} else {
Truncation::Complete
};
Ok(ResultFrame::Terminal(TerminalFrame::new(
outcome,
self.rows.elapsed(),
self.released_rows,
self.released_bytes,
truncation,
evidence,
)))
}
fn settle_at_release_ceiling(&mut self) -> Option<Result<ResultFrame, QueryServiceError>> {
let settled = self.rows.settle_consumer_bound()?;
self.byte_truncated = true;
debug_assert_eq!(settled.rows(), self.released_rows);
Some(self.release_terminal(QueryOutcome::Succeeded))
}
fn release_next_data_frame(&mut self) -> Option<Result<ResultFrame, QueryServiceError>> {
let data = match self.encoder.next() {
Ok(Some(data)) => data,
Ok(None) => return None,
Err(error) => {
let class = error.class();
let selected = self
.rows
.report_failure(crate::core_evidence::durable_class_of(class));
tracing::warn!(%error, ?class, "a released batch could not be framed");
if selected {
return Some(self.release_terminal(QueryOutcome::Failed(class)));
}
self.encoder.discard();
return None;
}
};
let bytes = data.arrow_ipc().len() as u64;
let rows = data.rows();
let total = self.released_bytes.saturating_add(bytes);
if total > self.release_ceiling {
let settled = self.settle_at_release_ceiling();
if settled.is_none() {
self.encoder.discard();
}
return settled;
}
if !self.rows.admit_release(rows, bytes) {
self.encoder.discard();
return None;
}
self.released_rows = self.released_rows.saturating_add(rows);
self.released_bytes = total;
Some(Ok(ResultFrame::Data(data)))
}
pub async fn next_frame(&mut self) -> Option<Result<ResultFrame, QueryServiceError>> {
loop {
if let Some(frame) = self.queued.take() {
return Some(Ok(frame));
}
if self.stage == Stage::Terminated {
return None;
}
if self.encoder.has_rows() {
if let Some(frame) = self.release_next_data_frame() {
return Some(frame);
}
continue;
}
match self.rows.next().await {
Some(Ok(batch)) => {
if let Some(observer) = &mut self.composite_trace
&& let Err(class) = observer.observe(&batch)
{
let _ = self
.rows
.report_failure(crate::core_evidence::durable_class_of(class));
return Some(self.release_terminal(QueryOutcome::Failed(class)));
}
self.encoder.begin(batch);
}
Some(Err(error)) => {
let outcome = QueryOutcome::Failed(crate::core_evidence::error_class_of(
classify_error(&error),
));
return Some(self.release_terminal(outcome));
}
None => {
return Some(self.release_terminal(QueryOutcome::Succeeded));
}
}
}
}
#[cfg(test)]
pub(crate) const fn frames_built(&self) -> u64 {
self.encoder.built()
}
#[cfg(test)]
pub(crate) fn request_buffered_batch(&mut self) {
self.rows.request_buffered_batch();
}
#[cfg(test)]
pub(crate) fn buffered_frames(&self) -> usize {
self.rows.buffered_frames()
}
#[cfg(test)]
pub(crate) fn terminal_selected(&self) -> bool {
self.rows.terminal_selected()
}
}
#[derive(Debug)]
pub struct MaterializedResult {
frames: Vec<DataFrame>,
schema: SchemaFrame,
terminal: TerminalFrame,
}
impl MaterializedResult {
#[must_use]
pub const fn schema(&self) -> &SchemaFrame {
&self.schema
}
#[must_use]
pub fn frames(&self) -> &[DataFrame] {
&self.frames
}
#[must_use]
pub const fn terminal(&self) -> &TerminalFrame {
&self.terminal
}
}
impl ProjectedResultStream {
pub async fn materialize(mut self) -> Result<MaterializedResult, QueryServiceError> {
let mut schema = None;
let mut frames = Vec::new();
let mut terminal = None;
while let Some(frame) = self.next_frame().await {
match frame? {
ResultFrame::Schema(value) => schema = Some(value),
ResultFrame::Data(value) => frames.push(value),
ResultFrame::Terminal(value) => terminal = Some(value),
}
}
match (schema, terminal) {
(Some(schema), Some(terminal)) => Ok(MaterializedResult {
frames,
schema,
terminal,
}),
_ => Err(QueryServiceError::Evidence),
}
}
}
pub fn frames(
stream: ProjectedResultStream,
) -> impl Stream<Item = Result<ResultFrame, QueryServiceError>> {
futures::stream::unfold(Some(stream), |state| async move {
let mut stream = state?;
let frame = stream.next_frame().await?;
Some((frame, Some(stream)))
})
}
#[cfg(test)]
mod control_fleet_tests {
#![allow(clippy::pedantic, clippy::nursery, missing_docs, clippy::unwrap_used)]
use polyc_query_credential::principal::ControlFleetPrincipal;
use super::*;
const STATEMENT_A: &str = "SELECT partition FROM turns";
const STATEMENT_B: &str = "SELECT partition FROM usage";
fn digest_of(sql: &str) -> [u8; 32] {
sha2::Sha256::digest(sql.as_bytes()).into()
}
fn control_fleet_scoping(purpose: &str, digest: [u8; 32]) -> Scoping {
Scoping::for_test(
QueryScope::Fleet,
false,
None,
None,
None,
None,
Some(ControlFleetPrincipal::minted(purpose.to_owned(), digest)),
None,
None,
PrincipalKind::ControlFleet,
)
}
fn admin_fleet_scoping(admin_persona: &str, session: &str) -> Scoping {
Scoping::for_test(
QueryScope::Fleet,
true,
Some(admin_persona.to_owned()),
None,
None,
None,
None,
Some(
polyc_query_credential::principal::AdminFleetPrincipal::minted(
admin_persona.to_owned(),
session.to_owned(),
),
),
None,
PrincipalKind::AdminFleet,
)
}
#[test]
fn a_matching_grant_is_admitted() {
let scoping = control_fleet_scoping("dashboard.conversations", digest_of(STATEMENT_A));
assert!(admit_control_fleet(&scoping, RealmKind::Fleet, STATEMENT_A).is_ok());
}
#[test]
fn a_grant_for_one_statement_is_refused_for_another() {
let scoping = control_fleet_scoping("dashboard.conversations", digest_of(STATEMENT_A));
assert!(matches!(
admit_control_fleet(&scoping, RealmKind::Fleet, STATEMENT_B),
Err(QueryServiceError::Unauthorized)
));
}
#[test]
fn a_matching_grant_is_refused_outside_the_fleet_realm() {
let scoping = control_fleet_scoping("dashboard.conversations", digest_of(STATEMENT_A));
assert!(matches!(
admit_control_fleet(&scoping, RealmKind::Visible, STATEMENT_A),
Err(QueryServiceError::Unauthorized)
));
}
#[test]
fn a_matching_grant_is_refused_under_a_conversation_scope() {
let mut scoping = control_fleet_scoping("dashboard.conversations", digest_of(STATEMENT_A));
scoping.set_scope_for_test(QueryScope::Conversations {
conversations: vec!["conv-1".to_owned()],
memory: polyc_query_credential::session::MemorySources::default(),
});
assert!(matches!(
admit_control_fleet(&scoping, RealmKind::Fleet, STATEMENT_A),
Err(QueryServiceError::Unauthorized)
));
}
#[test]
fn a_non_control_fleet_scope_is_never_refused_here() {
let scoping = Scoping::for_test(
QueryScope::Fleet,
true,
Some("persona-1".to_owned()),
None,
None,
None,
None,
None,
None,
PrincipalKind::Admin,
);
assert!(admit_control_fleet(&scoping, RealmKind::Visible, "SELECT 1").is_ok());
}
#[test]
fn the_requester_names_the_control_fleet_purpose() {
let scoping = control_fleet_scoping("dashboard.spend", digest_of(STATEMENT_A));
let requester = requester_of(&scoping).expect("a control-fleet scope is attributable");
assert_eq!(requester.as_str(), "control-fleet:dashboard.spend");
}
#[test]
fn an_unattributed_scope_still_refuses() {
let scoping = Scoping::for_test(
QueryScope::Fleet,
false,
None,
None,
None,
None,
None,
None,
None,
PrincipalKind::Admin,
);
assert!(matches!(
requester_of(&scoping),
Err(QueryServiceError::Unauthorized)
));
}
#[test]
fn an_admin_fleet_grant_is_refused_outside_the_fleet_realm() {
let scoping = admin_fleet_scoping("persona-admin", "session-a");
assert!(admit_admin_fleet(&scoping, RealmKind::Fleet).is_ok());
assert!(matches!(
admit_admin_fleet(&scoping, RealmKind::Visible),
Err(QueryServiceError::Unauthorized)
));
}
#[test]
fn an_admin_fleet_grant_is_refused_under_a_conversation_scope() {
let mut scoping = admin_fleet_scoping("persona-admin", "session-a");
scoping.set_scope_for_test(QueryScope::Conversations {
conversations: vec!["conv-1".to_owned()],
memory: polyc_query_credential::session::MemorySources::default(),
});
assert!(matches!(
admit_admin_fleet(&scoping, RealmKind::Fleet),
Err(QueryServiceError::Unauthorized)
));
}
#[test]
fn the_requester_names_the_admin_fleet_persona_and_session() {
let scoping = admin_fleet_scoping("persona-admin", "session-a");
let requester = requester_of(&scoping).expect("an admin-fleet scope is attributable");
assert_eq!(requester.as_str(), "admin-fleet:persona-admin:session-a");
}
fn persona_principal_scoping() -> Scoping {
Scoping::for_test(
QueryScope::Conversations {
conversations: vec!["conv-1".to_owned()],
memory: polyc_query_credential::session::MemorySources::default(),
},
false,
Some("persona-1".to_owned()),
None,
None,
None,
None,
None,
None,
PrincipalKind::Persona,
)
}
fn websession_conversation_grant_scoping() -> Scoping {
Scoping::for_test(
QueryScope::Conversations {
conversations: vec!["conv-1".to_owned()],
memory: polyc_query_credential::session::MemorySources::default(),
},
false,
None,
Some("conv-1".to_owned()),
None,
Some("session-1".to_owned()),
None,
None,
None,
PrincipalKind::ConversationGrantWebSession,
)
}
fn persona_conversation_grant_scoping() -> Scoping {
Scoping::for_test(
QueryScope::Conversations {
conversations: vec!["conv-1".to_owned()],
memory: polyc_query_credential::session::MemorySources::default(),
},
false,
Some("persona-1".to_owned()),
Some("conv-1".to_owned()),
None,
None,
None,
None,
None,
PrincipalKind::ConversationGrantPersona,
)
}
fn turn_conversation_grant_scoping() -> Scoping {
Scoping::for_test(
QueryScope::Conversations {
conversations: vec!["conv-1".to_owned()],
memory: polyc_query_credential::session::MemorySources::default(),
},
false,
None,
Some("conv-1".to_owned()),
Some("turn-1".to_owned()),
None,
None,
None,
None,
PrincipalKind::ConversationGrantTurn,
)
}
#[test]
fn edge_refuses_a_control_fleet_grant_on_either_realm() {
let scoping = control_fleet_scoping("dashboard.conversations", digest_of(STATEMENT_A));
assert!(matches!(
admit_edge_principal_kind(&scoping, RealmKind::Visible),
Err(QueryServiceError::Unauthorized)
));
assert!(matches!(
admit_edge_principal_kind(&scoping, RealmKind::Fleet),
Err(QueryServiceError::Unauthorized)
));
}
#[test]
fn visible_edge_admits_persona_and_websession_shapes_fleet_edge_refuses_them() {
for scoping in [
persona_principal_scoping(),
websession_conversation_grant_scoping(),
persona_conversation_grant_scoping(),
] {
assert!(admit_edge_principal_kind(&scoping, RealmKind::Visible).is_ok());
assert!(matches!(
admit_edge_principal_kind(&scoping, RealmKind::Fleet),
Err(QueryServiceError::Unauthorized)
));
}
}
#[test]
fn edge_refuses_a_turn_grant_on_either_realm() {
let scoping = turn_conversation_grant_scoping();
assert!(matches!(
admit_edge_principal_kind(&scoping, RealmKind::Visible),
Err(QueryServiceError::Unauthorized)
));
assert!(matches!(
admit_edge_principal_kind(&scoping, RealmKind::Fleet),
Err(QueryServiceError::Unauthorized)
));
}
#[test]
fn visible_edge_refuses_admin_fleet() {
let scoping = admin_fleet_scoping("persona-admin", "session-a");
assert!(matches!(
admit_edge_principal_kind(&scoping, RealmKind::Visible),
Err(QueryServiceError::Unauthorized)
));
assert!(admit_edge_principal_kind(&scoping, RealmKind::Fleet).is_ok());
}
#[test]
fn each_real_control_fleet_statement_is_refused_for_the_others() {
let statements = [
(
"dashboard.conversations",
polyc_query_model::statements::DASHBOARD_CONVERSATIONS_SQL,
),
(
"dashboard.spend",
polyc_query_model::statements::DASHBOARD_SPEND_SQL,
),
(
"dashboard.attribution",
polyc_query_model::statements::DASHBOARD_ATTRIBUTION_SQL,
),
(
"dashboard.context",
polyc_query_model::statements::DASHBOARD_CONTEXT_SQL,
),
(
"routines.fires",
polyc_query_model::statements::ROUTINE_FIRES_SQL,
),
(
"routines.fire",
polyc_query_model::statements::ROUTINE_FIRE_SQL,
),
];
for (purpose, sql) in statements {
let scoping = control_fleet_scoping(purpose, digest_of(sql));
assert!(
admit_control_fleet(&scoping, RealmKind::Fleet, sql).is_ok(),
"{purpose}'s own grant must admit its own statement"
);
for (_, other_sql) in statements {
if other_sql == sql {
continue;
}
assert!(
matches!(
admit_control_fleet(&scoping, RealmKind::Fleet, other_sql),
Err(QueryServiceError::Unauthorized)
),
"{purpose}'s grant must be refused for a different fixed statement"
);
}
}
}
}
#[cfg(test)]
mod composite_trace_tests {
#![allow(clippy::pedantic, clippy::nursery, missing_docs, clippy::unwrap_used)]
use arrow::array::{ArrayRef, StringArray};
use arrow::datatypes::{DataType, Field, Schema};
use arrow::record_batch::RecordBatch;
use polyc_query_model::{Consistency, Parameter, RequestedBounds};
use polyc_query_credential::principal::CompositeTracePrincipal;
use super::*;
const TRACE: &str = polyc_query_model::statements::COMPOSITE_TRACE_SQL;
const MEMORY: &str = polyc_query_model::statements::COMPOSITE_TRACE_MEMORY_SQL;
const ROUTINE: &str = polyc_query_model::statements::COMPOSITE_TRACE_ROUTINES_SQL;
const ADDRESSES: &str = polyc_query_model::statements::COMPOSITE_TRACE_ADDRESSES_SQL;
fn digest(sql: &str) -> [u8; 32] {
sha2::Sha256::digest(sql.as_bytes()).into()
}
fn scoping() -> Scoping {
Scoping::for_test(
QueryScope::Conversations {
conversations: vec!["conversation-a".to_owned()],
memory: polyc_query_credential::session::MemorySources {
owner: Some("persona-a".to_owned()),
participants: Vec::new(),
},
},
false,
Some("persona-a".to_owned()),
Some("conversation-a".to_owned()),
None,
None,
None,
None,
Some(CompositeTracePrincipal::minted(
&polyc_crypto::hex::lower(&digest(TRACE)),
&polyc_crypto::hex::lower(&digest(MEMORY)),
&polyc_crypto::hex::lower(&digest(ROUTINE)),
)),
PrincipalKind::ConversationGrantCompositeTrace,
)
}
fn request(sql: &str, parameters: Vec<Parameter>) -> QueryRequest {
QueryRequest::try_new(
sql.to_owned(),
parameters,
Consistency::Projected,
RequestedBounds::try_new(Duration::from_secs(5), 100, 1024 * 1024, 64 * 1024).unwrap(),
)
.unwrap()
}
#[test]
fn only_the_three_exact_statements_and_parameters_are_admitted() {
let scoped = scoping();
let trace = request(
TRACE,
vec![Parameter::Utf8("conv-conversation-a".to_owned())],
);
assert_eq!(
admit_composite_trace(&scoped, RealmKind::Visible, &trace).unwrap(),
Some(CompositeTraceStatement::Trace)
);
let memory = request(
MEMORY,
vec![
Parameter::Utf8("persona-persona-a-mem".to_owned()),
Parameter::Utf8("conversation-a".to_owned()),
Parameter::Utf8("conv-conversation-a".to_owned()),
],
);
assert_eq!(
admit_composite_trace(&scoped, RealmKind::Visible, &memory).unwrap(),
Some(CompositeTraceStatement::Memory)
);
let routine = request(ROUTINE, vec![Parameter::Utf8("conversation-a".to_owned())]);
assert_eq!(
admit_composite_trace(&scoped, RealmKind::Fleet, &routine).unwrap(),
Some(CompositeTraceStatement::Routine)
);
let caller_statement = request(
&format!("{TRACE}\n-- caller text"),
vec![Parameter::Utf8("conversation-a".to_owned())],
);
assert!(matches!(
admit_composite_trace(&scoped, RealmKind::Visible, &caller_statement),
Err(QueryServiceError::Unauthorized)
));
}
fn admin_scoping() -> Scoping {
Scoping::for_test(
QueryScope::Conversations {
conversations: vec!["conversation-a".to_owned()],
memory: polyc_query_credential::session::MemorySources::default(),
},
false,
None,
Some("conversation-a".to_owned()),
None,
None,
None,
None,
Some(CompositeTracePrincipal::minted_without_memory(
&polyc_crypto::hex::lower(&digest(TRACE)),
&polyc_crypto::hex::lower(&digest(ROUTINE)),
&polyc_crypto::hex::lower(&digest(ADDRESSES)),
)),
PrincipalKind::ConversationGrantAdminCompositeTrace,
)
}
fn memory_request(owner_partition: &str) -> QueryRequest {
request(
MEMORY,
vec![
Parameter::Utf8(owner_partition.to_owned()),
Parameter::Utf8("conversation-a".to_owned()),
Parameter::Utf8("conv-conversation-a".to_owned()),
],
)
}
#[test]
fn the_admin_subject_never_admits_the_memory_statement() {
let scoped = admin_scoping();
for owner_partition in ["persona-x-mem", "persona--mem"] {
assert!(matches!(
admit_composite_trace(
&scoped,
RealmKind::Visible,
&memory_request(owner_partition)
),
Err(QueryServiceError::Unauthorized)
));
}
}
#[test]
fn the_admin_subject_admits_trace_and_routine_under_the_same_rules() {
let scoped = admin_scoping();
let trace = request(
TRACE,
vec![Parameter::Utf8("conv-conversation-a".to_owned())],
);
assert_eq!(
admit_composite_trace(&scoped, RealmKind::Visible, &trace).unwrap(),
Some(CompositeTraceStatement::Trace)
);
let routine = request(ROUTINE, vec![Parameter::Utf8("conversation-a".to_owned())]);
assert_eq!(
admit_composite_trace(&scoped, RealmKind::Fleet, &routine).unwrap(),
Some(CompositeTraceStatement::Routine)
);
for (realm, refused) in [
(
RealmKind::Visible,
request(
TRACE,
vec![Parameter::Utf8("conv-conversation-b".to_owned())],
),
),
(RealmKind::Fleet, trace.clone()),
(
RealmKind::Fleet,
request(ROUTINE, vec![Parameter::Utf8("conversation-b".to_owned())]),
),
(RealmKind::Visible, routine.clone()),
] {
assert!(matches!(
admit_composite_trace(&scoped, realm, &refused),
Err(QueryServiceError::Unauthorized)
));
}
}
fn address_request(parameters: Vec<Parameter>) -> QueryRequest {
request(ADDRESSES, parameters)
}
#[test]
fn the_admin_subject_admits_the_address_statement_only_on_fleet() {
let scoped = admin_scoping();
let own = || vec![Parameter::Utf8("conv-conversation-a".to_owned())];
let admitted = admit_composite_trace(&scoped, RealmKind::Fleet, &address_request(own()));
assert!(
matches!(admitted, Ok(Some(CompositeTraceStatement::Addresses))),
"the admin subject reads its own addresses on Fleet: {admitted:?}"
);
for (realm, refused) in [
(RealmKind::Visible, address_request(own())),
(
RealmKind::Fleet,
address_request(vec![Parameter::Utf8("conv-conversation-b".to_owned())]),
),
(
RealmKind::Fleet,
address_request(vec![Parameter::Utf8("conversation-a".to_owned())]),
),
(
RealmKind::Fleet,
address_request(vec![
Parameter::Utf8("conv-conversation-a".to_owned()),
Parameter::Utf8("conv-conversation-a".to_owned()),
]),
),
(RealmKind::Fleet, address_request(Vec::new())),
] {
assert!(
matches!(
admit_composite_trace(&scoped, realm, &refused),
Err(QueryServiceError::Unauthorized)
),
"refused on {realm} with {:?}",
refused.parameters()
);
}
}
#[test]
fn a_persona_kind_with_an_address_digest_is_refused() {
let scoped = Scoping::for_test(
QueryScope::Conversations {
conversations: vec!["conversation-a".to_owned()],
memory: polyc_query_credential::session::MemorySources::default(),
},
false,
Some("persona-a".to_owned()),
Some("conversation-a".to_owned()),
None,
None,
None,
None,
Some(CompositeTracePrincipal::minted_without_memory(
&polyc_crypto::hex::lower(&digest(TRACE)),
&polyc_crypto::hex::lower(&digest(ROUTINE)),
&polyc_crypto::hex::lower(&digest(ADDRESSES)),
)),
PrincipalKind::ConversationGrantCompositeTrace,
);
assert!(matches!(
admit_composite_trace(
&scoped,
RealmKind::Fleet,
&address_request(vec![Parameter::Utf8("conv-conversation-a".to_owned())]),
),
Err(QueryServiceError::Unauthorized)
));
}
#[test]
fn an_address_digest_for_other_sql_is_refused() {
const OTHER: &str = "SELECT address FROM trace_participant_addresses";
let scoped = Scoping::for_test(
QueryScope::Conversations {
conversations: vec!["conversation-a".to_owned()],
memory: polyc_query_credential::session::MemorySources::default(),
},
false,
None,
Some("conversation-a".to_owned()),
None,
None,
None,
None,
Some(CompositeTracePrincipal::minted_without_memory(
&polyc_crypto::hex::lower(&digest(TRACE)),
&polyc_crypto::hex::lower(&digest(ROUTINE)),
&polyc_crypto::hex::lower(&digest(OTHER)),
)),
PrincipalKind::ConversationGrantAdminCompositeTrace,
);
assert!(matches!(
admit_composite_trace(
&scoped,
RealmKind::Fleet,
&request(
OTHER,
vec![Parameter::Utf8("conv-conversation-a".to_owned())]
),
),
Err(QueryServiceError::Unauthorized)
));
}
#[test]
fn no_persona_subject_admits_the_address_statement() {
let scoped = scoping();
for realm in [RealmKind::Visible, RealmKind::Fleet] {
assert!(matches!(
admit_composite_trace(
&scoped,
realm,
&address_request(vec![Parameter::Utf8("conv-conversation-a".to_owned())]),
),
Err(QueryServiceError::Unauthorized)
));
}
}
#[test]
fn the_admin_composite_trace_requester_is_never_an_empty_persona() {
let requester = requester_of(&admin_scoping()).expect("the admin read is attributable");
assert_eq!(requester.as_str(), "control-fleet:conversation.trace");
let persona = requester_of(&scoping()).expect("the persona read is attributable");
assert_eq!(persona.as_str(), "persona:persona-a");
}
#[test]
fn trace_rows_supply_the_complete_memory_source_proposal() {
let cache = CompositeTraceCache::default();
let schema = Arc::new(Schema::new(vec![
Field::new("caller_persona_id", DataType::Utf8, false),
Field::new("record_persona_id", DataType::Utf8, false),
]));
let mut collector = CompositeTraceCollector::try_new(
cache.clone(),
[7; 32],
"conversation-a".to_owned(),
&schema,
)
.unwrap();
let columns: Vec<ArrayRef> = vec![
Arc::new(StringArray::from(vec!["persona-a", "persona-a"])),
Arc::new(StringArray::from(vec!["", "persona-b"])),
];
collector
.observe(&RecordBatch::try_new(schema, columns).unwrap())
.unwrap();
collector.finish();
assert_eq!(
cache.personas(&[7; 32], "conversation-a").unwrap(),
vec!["persona-a".to_owned(), "persona-b".to_owned()]
);
assert!(cache.personas(&[7; 32], "conversation-b").is_none());
}
}
#[cfg(test)]
mod catalog_tests {
#![allow(clippy::pedantic, clippy::nursery, missing_docs, clippy::unwrap_used)]
use std::collections::BTreeMap;
use std::task::{Context, Poll};
use bytes::Bytes;
use connectrpc::client::{ClientBody, ClientConfig, ClientTransport};
use futures::future::BoxFuture;
use polyc_crypto::session::{AuthorizedSessionClaims, SessionScope, SessionSubject};
use polyc_crypto::signing_role::RoleSigner;
use polyc_persona::{ActivePersona, PersonaError, PersonaReferenceSnapshot, ScopeResolution};
use polyc_proto::proto::polychrome::persona::v1::Participation;
use polyc_query_model::GrantSubject;
use polyc_session_family::authority::SessionAuthorityError;
use polyc_state::projection::artifact::ManifestVerdict;
use polyc_state_connect::observation::ObservationClient;
use polyc_state_connect::persona_memory_journal::PersonaMemoryJournalClient;
use polyc_state_connect::versioned_source::VersionedSourceClient;
use polyc_storage_gcs::{GcsConfig, GcsCredentials};
use super::*;
use crate::core_resolution::all_tables;
const CONVERSATION: &str = "conv-a";
const PERSONA: &str = "persona-a";
const ADMIN: &str = "persona-admin";
const PERSONA_BEARER: &str = "bearer-persona-a";
const ADMIN_BEARER: &str = "bearer-persona-admin";
const EXPIRES: u64 = u64::MAX / 2;
fn signer() -> RoleSigner<TurnReadRole> {
RoleSigner::from_key_bytes(&[23_u8; 32]).expect("a well-formed signing key")
}
struct FakePersonas {
personas: BTreeMap<String, ActivePersona>,
participations: BTreeMap<String, Vec<Participation>>,
}
#[async_trait::async_trait]
impl PersonaSource for FakePersonas {
async fn active_persona(
&self,
persona_id: String,
) -> Result<Option<ActivePersona>, PersonaError> {
Ok(self.personas.get(&persona_id).cloned())
}
async fn participations(
&self,
persona_id: String,
) -> Result<Vec<Participation>, PersonaError> {
Ok(self
.participations
.get(&persona_id)
.cloned()
.unwrap_or_default())
}
async fn participation_scope(
&self,
_persona_id: String,
_cap: usize,
) -> Result<ScopeResolution, PersonaError> {
Err(PersonaError::Authority(
"the catalog path never resolves a search scope".to_owned(),
))
}
async fn usage_rollup_index(&self) -> Result<Vec<String>, PersonaError> {
Ok(Vec::new())
}
async fn reference_snapshot(
&self,
_persona_id: String,
) -> Result<Option<PersonaReferenceSnapshot>, PersonaError> {
Ok(None)
}
}
struct FakeSessions;
#[async_trait::async_trait]
impl SessionVerification for FakeSessions {
async fn verify_bearer(
&self,
token: &str,
_now_ms: u64,
) -> Result<AuthorizedSessionClaims, SessionAuthorityError> {
let (session_id, persona_id) = match token {
PERSONA_BEARER => ("session-persona-a", PERSONA),
ADMIN_BEARER => ("session-persona-admin", ADMIN),
_ => return Err(SessionAuthorityError::Invalid),
};
Ok(AuthorizedSessionClaims {
issuer: "session-test".to_owned(),
key_id: "key-test".to_owned(),
session_id: session_id.to_owned(),
authorization_epoch: 1,
subject: SessionSubject::Persona {
persona_id: persona_id.to_owned(),
},
scopes: vec![SessionScope::ExplorerRead],
issued_ms: 1,
expires_ms: EXPIRES,
})
}
}
#[derive(Clone)]
struct NeverTransport;
struct NeverBody;
impl connectrpc::http_body::Body for NeverBody {
type Data = Bytes;
type Error = std::io::Error;
fn poll_frame(
self: std::pin::Pin<&mut Self>,
_context: &mut Context<'_>,
) -> Poll<Option<Result<connectrpc::http_body::Frame<Bytes>, Self::Error>>> {
Poll::Ready(None)
}
}
impl ClientTransport for NeverTransport {
type ResponseBody = NeverBody;
type Error = std::io::Error;
fn send(
&self,
_request: http::Request<ClientBody>,
) -> BoxFuture<'static, Result<http::Response<Self::ResponseBody>, Self::Error>> {
Box::pin(async {
Err(std::io::Error::other(
"the catalog path makes no state call",
))
})
}
}
#[derive(Debug)]
struct NoTrust;
impl ManifestTrust for NoTrust {
fn classify(
&self,
_signer: &[u8],
_canonical: &[u8],
_signature: &[u8],
) -> ManifestVerdict {
ManifestVerdict::Untrusted
}
}
fn personas() -> FakePersonas {
FakePersonas {
personas: BTreeMap::from([
(
PERSONA.to_owned(),
ActivePersona {
persona_id: PERSONA.to_owned(),
admin: false,
provisional: false,
},
),
(
ADMIN.to_owned(),
ActivePersona {
persona_id: ADMIN.to_owned(),
admin: true,
provisional: false,
},
),
]),
participations: BTreeMap::from([(
PERSONA.to_owned(),
vec![Participation {
conversation_id: CONVERSATION.to_owned(),
role: "participant".to_owned(),
first_at_ms: 1,
via_persona_id: String::new(),
__buffa_unknown_fields: buffa::UnknownFields::new(),
}],
)]),
}
}
fn dead_reader() -> GcsReadClient {
GcsReadClient::new(GcsConfig {
bucket: "catalog-tests".to_owned(),
prefix: "unused".to_owned(),
credentials: GcsCredentials::Anonymous,
api_base: "http://127.0.0.1:1".to_owned(),
http_timeout: Duration::from_secs(1),
})
.expect("an anonymous read client composes")
}
fn service(realm_fleet: bool, concurrency: usize, timeout: Duration) -> ProjectedCoreService {
service_with_personas(personas(), realm_fleet, concurrency, timeout)
}
fn service_with_personas(
source: FakePersonas,
realm_fleet: bool,
concurrency: usize,
timeout: Duration,
) -> ProjectedCoreService {
let config = || ClientConfig::new("http://127.0.0.1:1".parse().unwrap());
let composition = ProjectedCoreComposition {
namespace: "conversation-core".to_owned(),
persona_memory_namespace: "polychrome".to_owned(),
projection_owner: "projector".to_owned(),
policy: QueryServicePolicy {
result_release_bytes: 1024 * 1024,
response_frame_bytes: 256 * 1024,
manifest_bytes: 256 * 1024,
artifact_file_bytes: 64 * 1024 * 1024,
artifact_range_bytes: 16 * 1024 * 1024,
source_decode_bytes: 512 * 1024 * 1024,
max_concurrent_executions: concurrency,
revalidation_interval: Duration::from_secs(30),
execution_memory_bytes: 8 * 1024 * 1024,
},
limits: {
let defaults = QueryLimits::default();
QueryLimits {
timeout,
row_cap: defaults.row_cap,
spill_dir: std::env::temp_dir().join(format!(
"polyc-query-catalog-test-{}-{realm_fleet}-{concurrency}",
std::process::id()
)),
spill_quota_bytes: 1024 * 1024,
}
},
realm: if realm_fleet {
QueryServiceRealm::Fleet {
visible_artifacts: dead_reader(),
visible_namespaces: vec![ArtifactNamespace {
namespace: "conversation-core-visible".to_owned(),
protection: AtRestProtection::TenantKey,
}],
additional_visible_artifacts: Vec::new(),
fleet_artifacts: dead_reader(),
fleet_namespaces: vec![ArtifactNamespace {
namespace: "conversation-core-fleet".to_owned(),
protection: AtRestProtection::TenantKey,
}],
additional_fleet_artifacts: Vec::new(),
}
} else {
QueryServiceRealm::Visible {
artifacts: dead_reader(),
namespaces: vec![ArtifactNamespace {
namespace: "conversation-core-visible".to_owned(),
protection: AtRestProtection::TenantKey,
}],
additional_artifacts: Vec::new(),
fleet_namespaces: vec!["conversation-core-fleet".to_owned()],
}
},
trust: Arc::new(NoTrust),
journal: JournalClient::new(NeverTransport, config()),
projections: ProjectionCatalogClient::new(NeverTransport, config()),
audit: QueryAuditClient::new(NeverTransport, config()),
versioned: VersionedSourceClient::new(NeverTransport, config()),
persona_memory: PersonaMemoryJournalClient::new(NeverTransport, config()),
observation: ObservationClient::new(NeverTransport, config()),
};
ProjectedCoreService::try_new(
CredentialSource {
persona: Arc::new(source),
turn_read_trust: RoleTrustSet::current(&signer()),
sessions: Arc::new(FakeSessions),
},
composition,
)
.expect("the test service composes")
}
fn bearer(token: &str) -> QueryCredential {
QueryCredential::from_bearer(token.to_owned())
}
fn conversation_grant(subject: GrantSubject) -> String {
polyc_query_model::mint_conversation_grant(&signer(), CONVERSATION, subject, EXPIRES)
}
fn names(reply: &CatalogReply) -> Vec<&str> {
reply.tables().iter().map(|table| table.name()).collect()
}
#[tokio::test]
async fn the_internal_listener_describes_the_visible_catalog() {
let service = service(false, 2, Duration::from_secs(30));
let reply = service
.describe_catalog(bearer(PERSONA_BEARER))
.await
.expect("a persona session describes its catalog");
let names = names(&reply);
assert!(names.contains(&"turns"), "a persona sees turns: {names:?}");
assert!(names.windows(2).all(|pair| pair[0] < pair[1]));
for credential in [
conversation_grant(GrantSubject::WebSession("session-1".to_owned())),
conversation_grant(GrantSubject::Persona(PERSONA.to_owned())),
] {
service
.describe_catalog(bearer(&credential))
.await
.expect("a conversation grant describes its catalog");
}
}
#[tokio::test]
async fn a_visible_caller_never_sees_a_fleet_only_table() {
let service = service(false, 2, Duration::from_secs(30));
let reply = service
.describe_catalog(bearer(PERSONA_BEARER))
.await
.unwrap();
let scope = QueryScope::Conversations {
conversations: vec![CONVERSATION.to_owned()],
memory: polyc_query_credential::session::MemorySources {
owner: Some(PERSONA.to_owned()),
participants: Vec::new(),
},
};
let listed = names(&reply);
for table in all_tables() {
if !table.visible_in(CoreRealm::Visible) {
assert!(
!listed.contains(&table.name()),
"a Fleet-only table name reached a Visible caller: `{}`",
table.name()
);
}
assert_eq!(
listed.contains(&table.name()),
catalog_tables(CoreRealm::Visible, &scope).contains(&table),
"the reply must be exactly the listing rule's answer for `{}`",
table.name()
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn refused_principal_kinds_and_the_edge_session_bearer() {
let service = service(false, 2, Duration::from_secs(30));
service
.describe_catalog(bearer(PERSONA_BEARER))
.await
.expect("the internal listener admits a session bearer");
assert!(matches!(
service.describe_catalog_edge(bearer(PERSONA_BEARER)).await,
Err(QueryServiceError::Unauthorized)
));
let refused = [
conversation_grant(GrantSubject::Turn("turn-1".to_owned())),
conversation_grant(GrantSubject::CompositeTrace {
persona_id: PERSONA.to_owned(),
trace_statement_digest: polyc_crypto::hex::lower(&[1; 32]),
memory_statement_digest: polyc_crypto::hex::lower(&[2; 32]),
routine_statement_digest: polyc_crypto::hex::lower(&[3; 32]),
}),
polyc_query_model::mint_control_fleet_grant(
&signer(),
"dashboard.conversations",
&polyc_crypto::hex::lower(&[4; 32]),
EXPIRES,
),
polyc_query_model::mint_admin_fleet_grant(&signer(), ADMIN, "session-a", EXPIRES),
ADMIN_BEARER.to_owned(),
];
for credential in &refused {
assert!(
matches!(
service.describe_catalog(bearer(credential)).await,
Err(QueryServiceError::Unauthorized)
),
"the internal listener refuses it"
);
assert!(
matches!(
service.describe_catalog_edge(bearer(credential)).await,
Err(QueryServiceError::Unauthorized)
),
"the edge listener refuses it"
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn the_fleet_realm_describes_its_catalog_to_admin_fleet_only() {
let service = service(true, 2, Duration::from_secs(30));
let admin_fleet =
polyc_query_model::mint_admin_fleet_grant(&signer(), ADMIN, "session-a", EXPIRES);
let reply = service
.describe_catalog(bearer(&admin_fleet))
.await
.expect("an admin-fleet grant describes the fleet catalog");
assert_eq!(
names(&reply).len(),
catalog_tables(CoreRealm::Fleet, &QueryScope::Fleet).len()
);
assert!(
all_tables().any(|table| !table.visible_in(CoreRealm::Visible)
&& names(&reply).contains(&table.name())),
"a fleet-only table name is present"
);
for credential in [
PERSONA_BEARER.to_owned(),
ADMIN_BEARER.to_owned(),
conversation_grant(GrantSubject::WebSession("session-1".to_owned())),
conversation_grant(GrantSubject::Turn("turn-1".to_owned())),
polyc_query_model::mint_control_fleet_grant(
&signer(),
"dashboard.conversations",
&polyc_crypto::hex::lower(&[4; 32]),
EXPIRES,
),
] {
assert!(
matches!(
service.describe_catalog(bearer(&credential)).await,
Err(QueryServiceError::Unauthorized)
),
"the Fleet realm refuses a non-admin-fleet caller"
);
}
assert!(matches!(
service.describe_catalog_edge(bearer(PERSONA_BEARER)).await,
Err(QueryServiceError::Unauthorized)
));
}
#[tokio::test]
async fn a_saturated_permit_refuses_the_catalog_call() {
let service = service(false, 1, Duration::from_millis(80));
let operation = CoreOperationContext::for_test(Duration::from_secs(30));
let held = service
.admission
.acquire(&operation, &CancellationToken::new())
.await
.expect("the one slot is free");
let outcome = service.describe_catalog(bearer(PERSONA_BEARER)).await;
assert!(
matches!(outcome, Err(QueryServiceError::Execution(_))),
"a saturated admission refuses the catalog: {outcome:?}"
);
drop(held);
service
.describe_catalog(bearer(PERSONA_BEARER))
.await
.expect("the released slot admits the catalog again");
}
#[tokio::test]
async fn the_reply_reports_public_schema_names() {
let service = service(true, 2, Duration::from_secs(30));
let admin_fleet =
polyc_query_model::mint_admin_fleet_grant(&signer(), ADMIN, "session-a", EXPIRES);
let reply = service
.describe_catalog(bearer(&admin_fleet))
.await
.unwrap();
let renamed = all_tables()
.find(|table| {
let Some(declared) = table
.family()
.tables()
.iter()
.find(|schema| schema.table().as_str() == table.name())
else {
return false;
};
let declared_names: Vec<&str> =
declared.fields().iter().map(|field| field.name()).collect();
let public_schema = table.public_schema();
let public_names: Vec<&str> = public_schema
.fields()
.iter()
.map(|field| field.name().as_str())
.collect();
declared_names.contains(&"summary_id") && public_names.contains(&"turn_id")
})
.expect("the registry renames at least one table's public schema");
let listed = reply
.tables()
.iter()
.find(|table| table.name() == renamed.name())
.expect("the reply lists the renaming table");
let columns: Vec<&str> = listed
.columns()
.iter()
.map(|column| column.name())
.collect();
let public_schema = renamed.public_schema();
let public_names: Vec<&str> = public_schema
.fields()
.iter()
.map(|field| field.name().as_str())
.collect();
assert_eq!(columns, public_names, "the reply speaks the view's names");
assert!(
columns.contains(&"turn_id"),
"the view's renamed column is present: {columns:?}"
);
assert!(
!columns.contains(&"summary_id"),
"the declared column name never leaks: {columns:?}"
);
}
const GET_MY_PAYMENTS_SQL: &str = "SELECT partition, position, turn_id, direction, reference, \
amount_base_units, asset, recipient, method, tool_call_id, payer_kind, timestamp_unix \
FROM payments WHERE subject = $1 \
UNION ALL \
SELECT partition, position, turn_id, direction, reference, amount_base_units, asset, \
recipient, method, tool_call_id, payer_kind, timestamp_unix FROM outbound_payments \
WHERE subject = $1 \
ORDER BY timestamp_unix DESC, partition DESC, position DESC LIMIT 500";
const GET_MY_REFUSALS_SQL: &str = "SELECT partition, position, turn_id, reason, reason_detail, \
merchant_host, requested_base_units, permitted_base_units, tool_call_id, timestamp_unix \
FROM refusals WHERE subject = $1 \
ORDER BY timestamp_unix DESC, partition DESC, position DESC LIMIT 500";
fn personas_in(conversations: usize) -> FakePersonas {
let mut source = personas();
source.participations.insert(
PERSONA.to_owned(),
(0..conversations)
.map(|index| Participation {
conversation_id: format!("c-{index}"),
role: "participant".to_owned(),
first_at_ms: 1,
via_persona_id: String::new(),
__buffa_unknown_fields: buffa::UnknownFields::new(),
})
.collect(),
);
source
}
fn my_statement_request(sql: &str) -> QueryRequest {
QueryRequest::try_new(
sql.to_owned(),
vec![polyc_query_model::Parameter::Utf8(PERSONA.to_owned())],
polyc_query_model::Consistency::Projected,
polyc_query_model::RequestedBounds::try_new(
Duration::from_secs(30),
500,
1024 * 1024,
256 * 1024,
)
.unwrap(),
)
.unwrap()
}
#[tokio::test]
async fn a_persona_wide_grant_past_the_source_pin_bound_is_refused_bounds() {
let over = usize::try_from(polyc_state::query_audit::MAX_SOURCE_PINS).unwrap() + 1;
for (label, sql) in [
("payments", GET_MY_PAYMENTS_SQL),
("refusals", GET_MY_REFUSALS_SQL),
] {
let service =
service_with_personas(personas_in(over), false, 2, Duration::from_secs(30));
let grant = polyc_query_model::mint_persona_wide_grant(&signer(), PERSONA, EXPIRES);
let error = service
.start_projected(
bearer(&grant),
&format!("q-479-{label}"),
&my_statement_request(sql),
)
.await
.expect_err("a persona-wide statement past the source-pin bound must refuse");
assert!(
matches!(error, QueryServiceError::Resolution(ErrorClass::Bounds)),
"{label} refused for the wrong reason: {error:?}"
);
}
}
#[tokio::test]
async fn a_persona_wide_grant_at_the_source_pin_bound_passes_the_bound() {
let at = usize::try_from(polyc_state::query_audit::MAX_SOURCE_PINS).unwrap();
for (label, sql) in [
("payments", GET_MY_PAYMENTS_SQL),
("refusals", GET_MY_REFUSALS_SQL),
] {
let service = service_with_personas(personas_in(at), false, 2, Duration::from_secs(30));
let grant = polyc_query_model::mint_persona_wide_grant(&signer(), PERSONA, EXPIRES);
let error = service
.start_projected(
bearer(&grant),
&format!("q-479-at-{label}"),
&my_statement_request(sql),
)
.await
.expect_err("the dead transport must refuse once the bound passes");
assert!(
matches!(error, QueryServiceError::Resolution(class) if class != ErrorClass::Bounds),
"{label} must pass the bound and fail in source resolution: {error:?}"
);
}
}
}
#[cfg(test)]
mod session_tests {
use super::*;
use datafusion::execution::FunctionRegistry;
use datafusion::execution::context::SessionContext;
fn composed() -> SessionState {
let runtime = RuntimeEnvBuilder::new()
.build_arc()
.expect("a bare runtime env builds");
projected_session_state(runtime).expect("the projected session composes")
}
fn per_request(state: &SessionState) -> SessionState {
SessionStateBuilder::new_from_existing(state.clone()).build()
}
#[test]
fn the_composed_session_carries_every_pinned_option() {
let state = composed();
let request_state = per_request(&state);
for options in [state.config_options(), request_state.config_options()] {
assert!(
!options.execution.enable_recursive_ctes,
"recursive CTE planning is off"
);
assert!(!options.catalog.information_schema);
assert_eq!(options.execution.target_partitions, 1);
assert_eq!(options.execution.planning_concurrency, 1);
assert_eq!(options.execution.batch_size.get(), 8192);
assert!(options.execution.collect_statistics);
assert!(!options.execution.parquet.pushdown_filters);
assert!(!options.execution.parquet.reorder_filters);
assert!(options.execution.parquet.enable_page_index);
assert!(options.execution.parquet.bloom_filter_on_read);
assert_eq!(options.catalog.default_catalog, "datafusion");
assert_eq!(options.catalog.default_schema, "public");
assert_eq!(options.sql_parser.dialect, Dialect::Generic);
assert!(options.sql_parser.enable_ident_normalization);
assert!(!options.sql_parser.parse_float_as_decimal);
}
assert!(
state
.config_options()
.catalog
.create_default_catalog_and_schema
);
assert!(
!request_state
.config_options()
.catalog
.create_default_catalog_and_schema
);
}
#[tokio::test]
async fn the_planner_alone_refuses_a_recursive_statement() {
let context = SessionContext::new_with_state(composed());
let refusal = context
.sql(
"WITH RECURSIVE n AS (SELECT 1 AS v UNION ALL SELECT v + 1 FROM n) \
SELECT v FROM n",
)
.await;
assert!(refusal.is_err(), "planning refuses recursion: {refusal:?}");
}
#[tokio::test]
async fn a_url_table_reference_fails_to_plan() {
let context = SessionContext::new_with_state(composed());
let refusal = context.sql("SELECT * FROM 'file.csv'").await;
assert!(refusal.is_err(), "URL tables stay off: {refusal:?}");
}
#[tokio::test]
async fn the_per_request_session_carries_the_json_surface() {
let state = per_request(&composed());
let context = SessionContext::new_with_state(state);
context
.sql("SELECT json_get_str('{\"q\":\"a\"}', 'q')")
.await
.expect("json_get_str is registered on the per-request session");
let operator_plan = context
.sql("SELECT '{\"q\":\"a\"}' -> 'q'")
.await
.expect("`->` reaches the JSON expression planner")
.logical_plan()
.display_indent()
.to_string();
assert!(
operator_plan.contains("json_get"),
"`->` plans as json_get: {operator_plan}"
);
let batches = context
.sql("SELECT CAST(json_get('{\"q\":1}', 'q') AS int)")
.await
.expect("the cast-collapse query plans")
.collect()
.await
.expect("the JSON rewrite made the union cast executable");
let value = batches[0]
.column(0)
.as_any()
.downcast_ref::<arrow::array::Int64Array>()
.expect("the collapsed getter returns Int64")
.value(0);
assert_eq!(value, 1);
let question = context.sql("SELECT '{\"q\":\"a\"}' ? 'q'").await;
assert!(
question.is_err(),
"`?` does not plan under the pinned dialect: {question:?}"
);
}
#[test]
fn the_composed_session_registers_only_the_allowed_functions() {
let allowed = crate::function_surface::allowed_names();
for state in [&composed(), &per_request(&composed())] {
for name in state.scalar_functions().keys() {
assert!(
allowed.contains(name),
"scalar registry holds unlisted name {name:?}"
);
}
for name in state
.aggregate_functions()
.keys()
.chain(state.window_functions().keys())
{
assert!(
allowed.contains(name),
"aggregate/window registry holds unlisted name {name:?}"
);
}
assert!(
state.table_functions().is_empty(),
"no table function is registered"
);
for name in allowed {
assert!(
state.scalar_functions().contains_key(name)
|| state.aggregate_functions().contains_key(name)
|| state.window_functions().contains_key(name),
"allowed name {name:?} is not registered"
);
}
}
}
#[tokio::test]
async fn a_refused_function_fails_planning_on_the_per_request_session() {
let context = SessionContext::new_with_state(per_request(&composed()));
for sql in [
"SELECT repeat('x', 2)",
"SELECT REPEAT('x', 2)",
"SELECT array_repeat(1, 2)",
"SELECT list_repeat(1, 2)",
"SELECT approx_percentile_cont(x, 0.5, 1024) FROM t",
"SELECT generate_series(1, 10)",
] {
let refusal = context.sql(sql).await;
assert!(refusal.is_err(), "{sql} must fail planning: {refusal:?}");
}
}
#[test]
fn an_extra_registration_widens_the_per_request_session() {
let mut state = composed();
let before = state.scalar_functions().len();
let extra = SessionStateDefaults::default_scalar_functions()
.into_iter()
.find(|function| function.name() == "repeat")
.expect("repeat is a default function upstream");
let _ = state.register_udf(extra);
assert_eq!(state.scalar_functions().len(), before + 1);
let request = per_request(&state);
assert!(
request.scalar_functions().contains_key("repeat"),
"the drift propagates: the per-request session sees it"
);
}
#[tokio::test]
async fn the_desugared_sql_forms_still_plan() {
let context = SessionContext::new_with_state(per_request(&composed()));
for sql in [
"SELECT EXTRACT(YEAR FROM CAST('2026-01-01' AS DATE))",
"SELECT POSITION('b' IN 'abc')",
"SELECT SUBSTRING('abc' FROM 2)",
"SELECT TRIM(BOTH ' ' FROM ' a ')",
"SELECT make_array(1, 2)[1]",
"SELECT * FROM (VALUES (1)) v(x) WHERE x IN (1, 2)",
"SELECT 'abc' LIKE 'a%'",
"SELECT COALESCE(NULL, 1)",
"SELECT NULLIF(1, 2)",
] {
let planned = context.sql(sql).await;
assert!(planned.is_ok(), "{sql} must plan: {planned:?}");
}
}
}