use std::sync::Arc;
use arrow::array::{ArrayRef, StringBuilder, UInt64Builder};
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use arrow::error::ArrowError;
use arrow::record_batch::RecordBatch;
use polyc_eventlog::Event;
use polyc_proto::events_decode::decode_event_payload;
use polyc_proto::kinds;
use polyc_proto::proto::polychrome::events::v1::AttributionEvent;
#[derive(Debug, Clone)]
pub(crate) struct AttributionRow {
pub partition: String,
pub position: u64,
pub turn_id: Option<String>,
pub persona_id: String,
pub role: String,
pub identity_provider: String,
pub identity_scope: String,
pub identity_external_id: String,
pub identity_display_name: String,
pub asserting_edge_id: String,
pub signer_pk_hex: String,
pub signature_hex: String,
}
#[must_use]
pub(crate) fn schema() -> SchemaRef {
Arc::new(Schema::new(vec![
Field::new("partition", DataType::Utf8, false),
Field::new("position", DataType::UInt64, false),
Field::new("turn_id", DataType::Utf8, true),
Field::new("persona_id", DataType::Utf8, false),
Field::new("role", DataType::Utf8, false),
Field::new("identity_provider", DataType::Utf8, false),
Field::new("identity_scope", DataType::Utf8, false),
Field::new("identity_external_id", DataType::Utf8, false),
Field::new("identity_display_name", DataType::Utf8, false),
Field::new("asserting_edge_id", DataType::Utf8, false),
Field::new("signer_pk_hex", DataType::Utf8, false),
Field::new("signature_hex", DataType::Utf8, false),
]))
}
pub(crate) fn decode_attribution_batch(rows: &[AttributionRow]) -> Result<RecordBatch, ArrowError> {
let mut partition_b = StringBuilder::with_capacity(rows.len(), rows.len() * 8);
let mut position_b = UInt64Builder::with_capacity(rows.len());
let mut turn_id_b = StringBuilder::with_capacity(rows.len(), rows.len() * 36);
let mut persona_id_b = StringBuilder::with_capacity(rows.len(), rows.len() * 36);
let mut role_b = StringBuilder::with_capacity(rows.len(), rows.len() * 12);
let mut identity_provider_b = StringBuilder::with_capacity(rows.len(), rows.len() * 12);
let mut identity_scope_b = StringBuilder::with_capacity(rows.len(), rows.len() * 12);
let mut identity_external_id_b = StringBuilder::with_capacity(rows.len(), rows.len() * 24);
let mut identity_display_name_b = StringBuilder::with_capacity(rows.len(), rows.len() * 16);
let mut asserting_edge_id_b = StringBuilder::with_capacity(rows.len(), rows.len() * 12);
let mut signer_pk_hex_b = StringBuilder::with_capacity(rows.len(), rows.len() * 64);
let mut signature_hex_b = StringBuilder::with_capacity(rows.len(), rows.len() * 128);
for row in rows {
partition_b.append_value(&row.partition);
position_b.append_value(row.position);
match &row.turn_id {
Some(id) => turn_id_b.append_value(id),
None => turn_id_b.append_null(),
}
persona_id_b.append_value(&row.persona_id);
role_b.append_value(&row.role);
identity_provider_b.append_value(&row.identity_provider);
identity_scope_b.append_value(&row.identity_scope);
identity_external_id_b.append_value(&row.identity_external_id);
identity_display_name_b.append_value(&row.identity_display_name);
asserting_edge_id_b.append_value(&row.asserting_edge_id);
signer_pk_hex_b.append_value(&row.signer_pk_hex);
signature_hex_b.append_value(&row.signature_hex);
}
let columns: Vec<ArrayRef> = vec![
Arc::new(partition_b.finish()),
Arc::new(position_b.finish()),
Arc::new(turn_id_b.finish()),
Arc::new(persona_id_b.finish()),
Arc::new(role_b.finish()),
Arc::new(identity_provider_b.finish()),
Arc::new(identity_scope_b.finish()),
Arc::new(identity_external_id_b.finish()),
Arc::new(identity_display_name_b.finish()),
Arc::new(asserting_edge_id_b.finish()),
Arc::new(signer_pk_hex_b.finish()),
Arc::new(signature_hex_b.finish()),
];
RecordBatch::try_new(schema(), columns)
}
#[must_use]
pub(crate) fn decode_attribution_events(
partition: &str,
events: &[(u64, Event)],
) -> Vec<AttributionRow> {
warn_undecodable_attribution_payloads(events);
polyc_facts::attribution_events_with_positions(
events.iter().map(|(position, event)| (*position, event)),
polyc_facts::AttributionScope::Both,
)
.map(|fact| {
let identity = fact.identity.unwrap_or_default();
AttributionRow {
partition: partition.to_string(),
position: fact.position,
turn_id: fact.turn_id.map(|u| u.to_string()),
persona_id: fact.persona_id,
role: fact.role,
identity_provider: identity.provider,
identity_scope: identity.scope,
identity_external_id: identity.external_id,
identity_display_name: identity.display_name,
asserting_edge_id: fact.asserting_edge_id,
signer_pk_hex: fact.signer_pk_hex,
signature_hex: fact.signature_hex,
}
})
.collect()
}
fn warn_undecodable_attribution_payloads(events: &[(u64, Event)]) {
for (position, event) in events {
let position: u64 = *position;
let (base, _) = kinds::parse(&event.kind);
if base != kinds::CALLER && base != kinds::PARTICIPANT {
continue;
}
if event.payload.is_empty() {
continue;
}
if decode_event_payload::<AttributionEvent>(&event.payload).is_none() {
tracing::warn!(
len = event.payload.len(),
table = "attribution",
position,
"corrupt event payload; skipping row"
);
}
}
}
#[cfg(test)]
mod tests {
use arrow::array::Array as _;
use buffa::Message as _;
use polyc_proto::kinds;
use polyc_proto::proto::polychrome::events::v1::AttributionEvent;
use polyc_proto::proto::polychrome::persona::v1::ExternalIdentity;
use uuid::Uuid;
use super::*;
fn sample_attribution(
persona_id: &str,
role: &str,
identity: ExternalIdentity,
) -> AttributionEvent {
AttributionEvent {
persona_id: persona_id.to_string(),
identity: buffa::MessageField::some(identity),
role: role.to_string(),
asserting_edge_id: String::new(),
signer_pk_hex: String::new(),
signature_hex: String::new(),
__buffa_unknown_fields: buffa::UnknownFields::default(),
}
}
fn sample_identity(
provider: &str,
scope: &str,
external_id: &str,
display_name: &str,
) -> ExternalIdentity {
ExternalIdentity {
provider: provider.to_string(),
scope: scope.to_string(),
external_id: external_id.to_string(),
display_name: display_name.to_string(),
time_zone: String::new(),
__buffa_unknown_fields: buffa::UnknownFields::default(),
}
}
#[test]
fn schema_shape() {
let schema = schema();
let names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
assert_eq!(
names,
vec![
"partition",
"position",
"turn_id",
"persona_id",
"role",
"identity_provider",
"identity_scope",
"identity_external_id",
"identity_display_name",
"asserting_edge_id",
"signer_pk_hex",
"signature_hex",
]
);
let expect = [
("partition", DataType::Utf8, false),
("position", DataType::UInt64, false),
("turn_id", DataType::Utf8, true),
("persona_id", DataType::Utf8, false),
("role", DataType::Utf8, false),
("identity_provider", DataType::Utf8, false),
("identity_scope", DataType::Utf8, false),
("identity_external_id", DataType::Utf8, false),
("identity_display_name", DataType::Utf8, false),
("asserting_edge_id", DataType::Utf8, false),
("signer_pk_hex", DataType::Utf8, false),
("signature_hex", DataType::Utf8, false),
];
for (field, (name, ty, nullable)) in schema.fields().iter().zip(expect) {
assert_eq!(field.name(), name);
assert_eq!(field.data_type(), &ty);
assert_eq!(field.is_nullable(), nullable);
}
}
#[test]
fn decode_attribution_batch_round_trips() {
let rows = vec![AttributionRow {
partition: "conv-a".to_string(),
position: 5,
turn_id: Some("turn-xyz".to_string()),
persona_id: "persona-1".to_string(),
role: "initiator".to_string(),
identity_provider: "slack".to_string(),
identity_scope: "team-1".to_string(),
identity_external_id: "U123".to_string(),
identity_display_name: "Ada".to_string(),
asserting_edge_id: "slack-edge".to_string(),
signer_pk_hex: "deadbeef".to_string(),
signature_hex: "cafef00d".to_string(),
}];
let batch = decode_attribution_batch(&rows).expect("batch build");
assert_eq!(batch.num_rows(), 1);
assert_eq!(batch.schema(), schema());
let persona_id = batch
.column(3)
.as_any()
.downcast_ref::<arrow::array::StringArray>()
.unwrap();
assert_eq!(persona_id.value(0), "persona-1");
let role = batch
.column(4)
.as_any()
.downcast_ref::<arrow::array::StringArray>()
.unwrap();
assert_eq!(role.value(0), "initiator");
let identity_provider = batch
.column(5)
.as_any()
.downcast_ref::<arrow::array::StringArray>()
.unwrap();
assert_eq!(identity_provider.value(0), "slack");
let identity_external_id = batch
.column(7)
.as_any()
.downcast_ref::<arrow::array::StringArray>()
.unwrap();
assert_eq!(identity_external_id.value(0), "U123");
let asserting_edge_id = batch
.column(9)
.as_any()
.downcast_ref::<arrow::array::StringArray>()
.unwrap();
assert_eq!(asserting_edge_id.value(0), "slack-edge");
let signer_pk_hex = batch
.column(10)
.as_any()
.downcast_ref::<arrow::array::StringArray>()
.unwrap();
assert_eq!(signer_pk_hex.value(0), "deadbeef");
let signature_hex = batch
.column(11)
.as_any()
.downcast_ref::<arrow::array::StringArray>()
.unwrap();
assert_eq!(signature_hex.value(0), "cafef00d");
}
#[test]
fn decode_attribution_events_filters_and_decodes_real_buffa_bytes_for_caller() {
let turn = Uuid::from_u128(0x0195_abcd_ef01_2345_6789_abcd_ef01_2345);
let identity = sample_identity("slack", "team-9", "U999", "Grace");
let attribution = sample_attribution("persona-caller", "initiator", identity);
let bytes = attribution.encode_to_vec();
let events = vec![
(1, Event::new(kinds::TURN_START, Vec::new())),
(2, Event::new(kinds::tagged(kinds::CALLER, &turn), bytes)),
(3, Event::new(kinds::USER_MSG, b"not attribution".to_vec())),
];
let decoded = decode_attribution_events("conv-real", &events);
assert_eq!(decoded.len(), 1);
assert_eq!(decoded[0].partition, "conv-real");
assert_eq!(decoded[0].position, 2);
assert_eq!(decoded[0].turn_id, Some(turn.to_string()));
assert_eq!(decoded[0].persona_id, "persona-caller");
assert_eq!(decoded[0].role, "initiator");
assert_eq!(decoded[0].identity_provider, "slack");
assert_eq!(decoded[0].identity_scope, "team-9");
assert_eq!(decoded[0].identity_external_id, "U999");
assert_eq!(decoded[0].identity_display_name, "Grace");
}
#[test]
fn both_source_kinds_decode_with_their_own_role() {
let turn = Uuid::from_u128(0x0195_abcd_ef01_2345_6789_abcd_ef01_9999);
let caller_identity = sample_identity("slack", "team-1", "U1", "Alice");
let caller = sample_attribution("persona-alice", "initiator", caller_identity);
let participant_identity = sample_identity("telegram", "", "T2", "Bob");
let participant = sample_attribution("persona-bob", "participant", participant_identity);
let events = vec![
(
1,
Event::new(kinds::tagged(kinds::CALLER, &turn), caller.encode_to_vec()),
),
(
2,
Event::new(
kinds::tagged(kinds::PARTICIPANT, &turn),
participant.encode_to_vec(),
),
),
];
let mut decoded = decode_attribution_events("conv-both", &events);
decoded.sort_by_key(|row| row.position);
assert_eq!(decoded.len(), 2);
assert_eq!(decoded[0].persona_id, "persona-alice");
assert_eq!(decoded[0].role, "initiator");
assert_eq!(decoded[1].persona_id, "persona-bob");
assert_eq!(decoded[1].role, "participant");
}
#[test]
fn decode_attribution_events_bare_kind_has_no_turn_id() {
let events = vec![(7, Event::new(kinds::CALLER, Vec::new()))];
let decoded = decode_attribution_events("conv-bare", &events);
assert_eq!(decoded.len(), 1);
assert_eq!(decoded[0].partition, "conv-bare");
assert_eq!(decoded[0].position, 7);
assert_eq!(decoded[0].turn_id, None);
}
#[test]
fn empty_payload_attribution_event_decodes_to_defaults() {
let events = vec![(1, Event::new(kinds::PARTICIPANT, Vec::new()))];
let decoded = decode_attribution_events("conv-empty", &events);
assert_eq!(decoded.len(), 1, "an empty payload must decode, not skip");
assert_eq!(decoded[0].persona_id, "");
assert_eq!(decoded[0].role, "");
assert_eq!(decoded[0].identity_provider, "");
assert_eq!(decoded[0].identity_scope, "");
assert_eq!(decoded[0].identity_external_id, "");
assert_eq!(decoded[0].identity_display_name, "");
assert_eq!(decoded[0].asserting_edge_id, "");
assert_eq!(decoded[0].signer_pk_hex, "");
assert_eq!(decoded[0].signature_hex, "");
}
#[test]
fn decode_attribution_events_surfaces_edge_provenance_fields() {
let turn = Uuid::from_u128(0x0195_abcd_ef01_2345_6789_abcd_ef01_7777);
let attribution = AttributionEvent {
persona_id: String::new(),
identity: buffa::MessageField::none(),
role: "edge".to_string(),
asserting_edge_id: "trigger-edge".to_string(),
signer_pk_hex: "deadbeef".to_string(),
signature_hex: "cafef00d".to_string(),
__buffa_unknown_fields: buffa::UnknownFields::default(),
};
let events = vec![(
1,
Event::new(
kinds::tagged(kinds::CALLER, &turn),
attribution.encode_to_vec(),
),
)];
let decoded = decode_attribution_events("conv-edge-provenance", &events);
assert_eq!(decoded.len(), 1);
assert_eq!(decoded[0].role, "edge");
assert_eq!(decoded[0].persona_id, "");
assert_eq!(decoded[0].asserting_edge_id, "trigger-edge");
assert_eq!(decoded[0].signer_pk_hex, "deadbeef");
assert_eq!(decoded[0].signature_hex, "cafef00d");
}
#[test]
fn undecodable_non_empty_payload_is_skipped() {
let events = vec![
(1, Event::new(kinds::CALLER, vec![0xFF, 0xFE, 0xFD])),
(2, Event::new(kinds::PARTICIPANT, Vec::new())),
];
let decoded = decode_attribution_events("conv-corrupt", &events);
assert_eq!(decoded.len(), 1, "only the empty (valid) payload decodes");
assert_eq!(decoded[0].position, 2);
assert_eq!(decoded[0].persona_id, "");
}
#[test]
fn unrelated_kind_is_not_decoded_as_attribution() {
let identity = sample_identity("slack", "team-1", "U1", "Alice");
let attribution = sample_attribution("persona-alice", "initiator", identity);
let events = vec![(1, Event::new(kinds::USAGE, attribution.encode_to_vec()))];
let decoded = decode_attribution_events("conv-unrelated", &events);
assert_eq!(
decoded.len(),
0,
"a usage-kind event must never decode as attribution"
);
}
#[test]
fn attribution_event_round_trip_through_decode_and_batch() {
let turn = Uuid::from_u128(0x0195_abcd_ef01_2345_6789_abcd_ef01_1111);
let identity = sample_identity("email", "", "user@example.test", "Sam");
let attribution = sample_attribution("persona-rt", "participant", identity);
let events = vec![(
1,
Event::new(
kinds::tagged(kinds::PARTICIPANT, &turn),
attribution.encode_to_vec(),
),
)];
let decoded = decode_attribution_events("conv-rt", &events);
let batch = decode_attribution_batch(&decoded).expect("batch build");
let turn_id = batch
.column(2)
.as_any()
.downcast_ref::<arrow::array::StringArray>()
.unwrap();
assert_eq!(turn_id.value(0), turn.to_string());
let persona_id = batch
.column(3)
.as_any()
.downcast_ref::<arrow::array::StringArray>()
.unwrap();
assert_eq!(persona_id.value(0), "persona-rt");
let identity_display_name = batch
.column(8)
.as_any()
.downcast_ref::<arrow::array::StringArray>()
.unwrap();
assert_eq!(identity_display_name.value(0), "Sam");
}
}