#[cfg(feature = "tsp")]
use std::sync::Arc;
use std::time::Duration;
use affinidi_did_resolver_cache_sdk::DIDCacheClient;
use affinidi_messaging_core::{
ConnState, Inbound, InboundAck, MessageTransport, MessagingError, SendReceipt, TransportKind,
};
use affinidi_messaging_delivery::MessagingService;
use affinidi_messaging_delivery::{Delivery, OutboxState};
use affinidi_tdk::messaging::ATM;
use chrono::Utc;
use futures_util::stream::BoxStream;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tokio::sync::watch;
use tracing::{debug, info, warn};
use vta_sdk::protocol::matching::{
DIDCOMM_SERVICE_TYPE, Protocol, ServiceCapabilities, TRUST_TASK_HTTPS_SERVICE_TYPE,
};
use crate::capability_client::TRUST_TASK_ENVELOPE_TYPE;
use crate::error::AppError;
use crate::store::KeyspaceHandle;
use crate::tsp_reach::TspReachability;
#[derive(Clone, Copy)]
pub struct PushContext<'a> {
pub records: &'a KeyspaceHandle,
pub outbox: &'a KeyspaceHandle,
pub resolver: Option<&'a DIDCacheClient>,
pub messaging: Option<PushMessaging<'a>>,
pub tsp: bool,
pub learned_tsp: Option<&'a TspReachability>,
}
#[derive(Clone, Copy)]
pub struct PushMessaging<'a> {
pub service: &'a MessagingService,
pub atm: &'a ATM,
pub own_did: &'a str,
}
pub const ATTEMPT_WINDOW: Duration = Duration::from_secs(60 * 60);
const ENQUEUE_GRACE: Duration = Duration::from_secs(30);
const FINISHED_RETENTION: Duration = Duration::from_secs(7 * 24 * 60 * 60);
pub const TSP_TRANSPORT_ID: &str = "member-push-tsp";
pub const REST_TRANSPORT_ID: &str = "member-push-rest";
const RECORD_PREFIX: &str = "push:";
#[derive(Debug, Clone, Serialize, Deserialize)]
struct PushRecord {
id: String,
recipient: String,
document: Value,
remaining: Vec<Protocol>,
current: Protocol,
attempt_key: String,
attempt: u32,
#[serde(default)]
peer_tsp_mediator: Option<String>,
#[serde(default)]
rest_base: Option<String>,
deadline_ms: u64,
#[serde(default)]
queued_at_ms: u64,
#[serde(default)]
outcome: Option<PushOutcome>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
struct PushOutcome {
delivered: bool,
via: Protocol,
evidence: String,
at_ms: u64,
}
fn record_key(id: &str) -> String {
format!("{RECORD_PREFIX}{id}")
}
fn now_ms() -> u64 {
Utc::now().timestamp_millis().max(0) as u64
}
#[derive(Debug, Default, Clone, PartialEq, Eq)]
struct Reach {
tsp_mediator: Option<String>,
didcomm: bool,
rest_base: Option<String>,
}
impl Reach {
fn advertises_anything(&self) -> bool {
self.tsp_mediator.is_some() || self.didcomm || self.rest_base.is_some()
}
fn from_document(doc: &Value) -> Self {
let caps = ServiceCapabilities::from_did_document(doc);
Reach {
tsp_mediator: caps.tsp,
didcomm: caps.didcomm.is_some(),
rest_base: trust_task_https_base(doc),
}
}
}
fn trust_task_https_base(doc: &Value) -> Option<String> {
doc.get("service")?.as_array()?.iter().find_map(|svc| {
let typed = match svc.get("type")? {
Value::String(t) => t == TRUST_TASK_HTTPS_SERVICE_TYPE,
Value::Array(ts) => ts
.iter()
.any(|t| t.as_str() == Some(TRUST_TASK_HTTPS_SERVICE_TYPE)),
_ => false,
};
if !typed {
return None;
}
match svc.get("serviceEndpoint")? {
Value::String(uri) if !uri.is_empty() => Some(uri.clone()),
Value::Object(o) => o.get("uri").and_then(Value::as_str).map(str::to_string),
_ => None,
}
})
}
struct Ours {
tsp: bool,
didcomm: bool,
rest: bool,
}
fn ours(ctx: &PushContext<'_>) -> Ours {
let messaging_up = ctx.messaging.is_some();
Ours {
tsp: messaging_up && ctx.tsp,
didcomm: messaging_up,
rest: true,
}
}
async fn plan(ctx: &PushContext<'_>, recipient: &str) -> Result<(Vec<Protocol>, Reach), AppError> {
let reach = resolve_reach(ctx, recipient).await;
let ours = ours(ctx);
let learned_tsp = ctx.learned_tsp.is_some_and(|seen| seen.fresh(recipient));
let plan = choose(&reach, &ours, learned_tsp);
if plan.is_empty() {
return Err(AppError::Validation(format!(
"no matching protocol for {recipient}: it advertises {} and this node can \
send over {}",
describe_reach(&reach),
describe_ours(&ours)
)));
}
Ok((plan, reach))
}
fn choose(reach: &Reach, ours: &Ours, learned_tsp: bool) -> Vec<Protocol> {
let mut plan = Vec::new();
for p in Protocol::PREFERENCE_ORDER {
let both = match p {
Protocol::Tsp => ours.tsp && reach.tsp_mediator.is_some(),
Protocol::Didcomm => ours.didcomm && reach.didcomm,
Protocol::Rest => ours.rest && reach.rest_base.is_some(),
};
if both {
plan.push(p);
}
}
if plan.is_empty() && !reach.advertises_anything() {
if ours.tsp && learned_tsp {
plan.push(Protocol::Tsp);
}
if ours.didcomm {
plan.push(Protocol::Didcomm);
}
}
plan
}
fn describe_reach(r: &Reach) -> String {
let mut v = Vec::new();
if r.tsp_mediator.is_some() {
v.push("tsp");
}
if r.didcomm {
v.push(DIDCOMM_SERVICE_TYPE);
}
if r.rest_base.is_some() {
v.push(TRUST_TASK_HTTPS_SERVICE_TYPE);
}
if v.is_empty() {
"nothing".into()
} else {
v.join(", ")
}
}
fn describe_ours(o: &Ours) -> String {
let mut v = Vec::new();
if o.tsp {
v.push("tsp");
}
if o.didcomm {
v.push("didcomm");
}
if o.rest {
v.push("rest");
}
v.join(", ")
}
async fn resolve_reach(ctx: &PushContext<'_>, recipient: &str) -> Reach {
let Some(resolver) = ctx.resolver else {
return Reach::default();
};
match tokio::time::timeout(Duration::from_secs(15), resolver.resolve(recipient)).await {
Ok(Ok(resolved)) => serde_json::to_value(&resolved.doc)
.map(|doc| Reach::from_document(&doc))
.unwrap_or_default(),
Ok(Err(e)) => {
debug!(recipient, error = %e, "could not resolve the push recipient; using the shared mediator");
Reach::default()
}
Err(_) => {
debug!(
recipient,
"resolving the push recipient timed out; using the shared mediator"
);
Reach::default()
}
}
}
pub async fn push_trust_task(
ctx: &PushContext<'_>,
recipient: &str,
document: Value,
deliver_by: Duration,
) -> Result<String, AppError> {
let (plan, reach) = plan(ctx, recipient).await?;
let id = uuid::Uuid::new_v4().to_string();
let mut remaining = plan;
let current = remaining.remove(0);
let mut record = PushRecord {
id: id.clone(),
recipient: recipient.to_string(),
document,
remaining,
current,
attempt_key: String::new(),
attempt: 0,
peer_tsp_mediator: reach.tsp_mediator,
rest_base: reach.rest_base,
deadline_ms: now_ms().saturating_add(deliver_by.as_millis() as u64),
queued_at_ms: 0,
outcome: None,
};
loop {
match queue_attempt(ctx, &mut record).await {
Ok(()) => break,
Err(e) if !record.remaining.is_empty() => {
warn!(
recipient,
via = %record.current,
error = %e,
"could not queue a push on its preferred transport; trying the next"
);
record.current = record.remaining.remove(0);
record.attempt += 1;
}
Err(e) => return Err(e),
}
}
info!(recipient, via = %record.current, push = %id, "trust-task push queued");
Ok(id)
}
pub async fn outcome(
records: &KeyspaceHandle,
id: &str,
) -> Result<Option<(bool, Protocol, String)>, AppError> {
Ok(load_record(records, id)
.await?
.and_then(|r| r.outcome)
.map(|o| (o.delivered, o.via, o.evidence)))
}
async fn queue_attempt(ctx: &PushContext<'_>, record: &mut PushRecord) -> Result<(), AppError> {
let messaging = ctx.messaging;
let now = now_ms();
let remaining_ms = record.deadline_ms.saturating_sub(now).max(1_000);
let window_ms = if record.remaining.is_empty() {
remaining_ms
} else {
let share = remaining_ms / (record.remaining.len() as u64 + 1);
share.min(ATTEMPT_WINDOW.as_millis() as u64).max(1_000)
};
let window = Duration::from_millis(window_ms);
record.attempt_key = format!("{}:{}:{}", record.id, record.attempt, record.current);
record.queued_at_ms = now;
store_record(ctx, record).await?;
let delivery = Delivery::Guaranteed {
idempotency_key: Some(record.attempt_key.clone()),
ordering_key: None,
deliver_by: window,
};
let messaging = messaging
.ok_or_else(|| AppError::Internal("messaging not running — cannot push".into()))?;
match record.current {
Protocol::Didcomm => {
let envelope = affinidi_tdk::didcomm::Message::build(
format!("urn:uuid:{}", uuid::Uuid::new_v4()),
TRUST_TASK_ENVELOPE_TYPE.to_string(),
record.document.clone(),
)
.from(messaging.own_did.to_string())
.to(record.recipient.clone())
.finalize();
let (packed, _) = messaging
.atm
.pack_encrypted(
&envelope,
&record.recipient,
Some(messaging.own_did),
Some(messaging.own_did),
)
.await
.map_err(|e| {
AppError::Internal(format!("DIDComm pack for {} failed: {e}", record.recipient))
})?;
messaging
.service
.send(&record.recipient, packed.into_bytes(), delivery)
.await
.map_err(|e| AppError::Internal(format!("queue DIDComm push: {e}")))?;
}
Protocol::Tsp | Protocol::Rest => {
let transport = if record.current == Protocol::Tsp {
TSP_TRANSPORT_ID
} else {
REST_TRANSPORT_ID
};
messaging
.service
.send_via(
transport,
&record.recipient,
record.id.clone().into_bytes(),
delivery,
)
.await
.map_err(|e| AppError::Internal(format!("queue {} push: {e}", record.current)))?;
}
}
Ok(())
}
async fn store_record(ctx: &PushContext<'_>, record: &PushRecord) -> Result<(), AppError> {
ctx.records.insert(record_key(&record.id), record).await
}
async fn load_record(ks: &KeyspaceHandle, id: &str) -> Result<Option<PushRecord>, AppError> {
ks.get(record_key(id)).await
}
pub async fn sweep(ctx: &PushContext<'_>) -> Result<(), AppError> {
use affinidi_messaging_delivery::OutboxStore as _;
let Some(messaging) = ctx.messaging else {
return Ok(());
};
let outbox = crate::outbox_store::VtiOutboxStore::new(ctx.outbox.clone());
let now = now_ms();
for (_, bytes) in ctx
.records
.prefix_iter_raw(RECORD_PREFIX.as_bytes().to_vec())
.await?
{
let Ok(mut record) = serde_json::from_slice::<PushRecord>(&bytes) else {
continue;
};
if let Some(outcome) = &record.outcome {
if now.saturating_sub(outcome.at_ms) > FINISHED_RETENTION.as_millis() as u64 {
ctx.records.remove(record_key(&record.id)).await?;
}
continue;
}
let entry = outbox
.get(&record.attempt_key)
.await
.map_err(|e| AppError::Internal(format!("read push delivery state: {e}")))?;
let expired = entry.as_ref().is_some_and(|e| now >= e.deliver_by_ms);
let observed = entry.as_ref().is_some_and(|e| e.outbox_observed);
let entry_state = entry.map(|e| e.state);
match entry_state {
Some(OutboxState::Delivered) => {
finish(ctx, &mut record, true, "collected").await?;
}
Some(OutboxState::Sent) if record.current == Protocol::Rest => {
let _ = messaging.service.confirm(&record.attempt_key).await;
finish(ctx, &mut record, true, "reply").await?;
}
Some(OutboxState::Queued | OutboxState::Sent) if !expired => {}
None if now.saturating_sub(record.queued_at_ms) < ENQUEUE_GRACE.as_millis() as u64 => {}
None => {
warn!(
push = %record.id,
attempt = %record.attempt_key,
"push attempt was never queued; queuing it again"
);
if let Err(e) = queue_attempt(ctx, &mut record).await {
warn!(push = %record.id, error = %e, "could not re-queue the push");
escalate(ctx, &mut record).await?;
}
}
Some(OutboxState::Queued | OutboxState::Sent)
| Some(OutboxState::Failed | OutboxState::Unconfirmed) => {
debug!(
push = %record.id,
attempt = %record.attempt_key,
state = ?entry_state,
expired,
observed,
"push attempt ended without delivery evidence"
);
escalate(ctx, &mut record).await?;
}
Some(_) => {}
}
}
Ok(())
}
async fn escalate(ctx: &PushContext<'_>, record: &mut PushRecord) -> Result<(), AppError> {
if now_ms() >= record.deadline_ms {
return finish(ctx, record, false, "none").await;
}
let (fresh, reach) = plan(ctx, &record.recipient).await.unwrap_or_default();
record.peer_tsp_mediator = reach.tsp_mediator;
record.rest_base = reach.rest_base;
let tried = record.current;
let next: Vec<Protocol> = record
.remaining
.iter()
.copied()
.filter(|p| *p != tried && fresh.contains(p))
.collect();
if next.is_empty() {
return finish(ctx, record, false, "none").await;
}
warn!(
recipient = %record.recipient,
from = %tried,
to = %next[0],
"push produced no delivery evidence in its window; escalating"
);
record.remaining = next;
record.current = record.remaining.remove(0);
record.attempt += 1;
if let Err(e) = queue_attempt(ctx, record).await {
warn!(recipient = %record.recipient, error = %e, "could not queue the escalated push");
return finish(ctx, record, false, "none").await;
}
Ok(())
}
async fn finish(
ctx: &PushContext<'_>,
record: &mut PushRecord,
delivered: bool,
evidence: &str,
) -> Result<(), AppError> {
record.outcome = Some(PushOutcome {
delivered,
via: record.current,
evidence: evidence.to_string(),
at_ms: now_ms(),
});
if delivered {
info!(
recipient = %record.recipient,
via = %record.current,
evidence,
"trust-task push delivered"
);
} else {
warn!(
recipient = %record.recipient,
last_via = %record.current,
"push could not be confirmed on any transport the recipient offers"
);
}
store_record(ctx, record).await
}
async fn document_for(ks: &KeyspaceHandle, packed: &[u8]) -> Result<PushRecord, MessagingError> {
let id = std::str::from_utf8(packed)
.map_err(|_| MessagingError::Transport("push entry is not a push id".into()))?;
load_record(ks, id)
.await
.map_err(|e| MessagingError::Transport(format!("read push record: {e}")))?
.ok_or_else(|| MessagingError::Transport(format!("no push record {id}")))
}
#[cfg(feature = "tsp")]
pub struct TspPushTransport {
pub atm: Arc<affinidi_tdk::messaging::ATM>,
pub profile: Arc<affinidi_tdk::messaging::profiles::ATMProfile>,
pub mediator_did: String,
pub pushes: KeyspaceHandle,
pub conn: watch::Receiver<ConnState>,
}
#[cfg(feature = "tsp")]
#[async_trait::async_trait]
impl MessageTransport for TspPushTransport {
fn kind(&self) -> TransportKind {
TransportKind::Tsp
}
async fn send(&self, dest: &str, packed: Vec<u8>) -> Result<SendReceipt, MessagingError> {
use affinidi_tdk::messaging::protocols::tsp::{SendReadiness, invite_refusal_is_benign};
let record = document_for(&self.pushes, &packed).await?;
let doc = serde_json::to_vec(&record.document)
.map_err(|e| MessagingError::Transport(format!("serialise push document: {e}")))?;
let body = vta_sdk::tsp_binding::wrap_envelope(&doc);
let tsp = self.atm.tsp();
let err = |e: affinidi_tdk::messaging::errors::ATMError| {
MessagingError::Transport(format!("TSP send to {dest}: {e}"))
};
if matches!(
tsp.send_readiness(&self.profile, dest).await.map_err(err)?,
SendReadiness::Reestablish
) && let Err(e) = tsp.form_relationship_routed(&self.profile, dest).await
{
let after = tsp.send_readiness(&self.profile, dest).await.map_err(err)?;
if !invite_refusal_is_benign(after) {
return Err(err(e));
}
}
match record.peer_tsp_mediator.as_deref() {
Some(peer_mediator) if peer_mediator != self.mediator_did => {
tsp.send_nested_routed(
&self.profile,
&[self.mediator_did.clone(), peer_mediator.to_string()],
dest,
&body,
)
.await
.map_err(err)?;
Ok(SendReceipt {
via: TransportKind::Tsp,
hop_id: None,
})
}
_ => {
let inner = tsp.pack(&self.profile, dest, &body).await.map_err(err)?;
tsp.send_routed_opaque(
&self.profile,
&[self.mediator_did.clone(), dest.to_string()],
&inner,
)
.await
.map_err(err)?;
Ok(SendReceipt {
via: TransportKind::Tsp,
hop_id: Some(sha256_hex(tsp.encode(&inner).as_bytes())),
})
}
}
}
fn connection_state(&self) -> watch::Receiver<ConnState> {
self.conn.clone()
}
fn inbound(&self) -> BoxStream<'static, Inbound> {
Box::pin(futures_util::stream::empty())
}
async fn ack(&self, _ack: InboundAck) -> Result<(), MessagingError> {
Ok(())
}
}
#[cfg(feature = "tsp")]
fn sha256_hex(bytes: &[u8]) -> String {
use sha2::{Digest, Sha256};
hex::encode(Sha256::digest(bytes))
}
pub struct RestPushTransport {
http: reqwest::Client,
pushes: KeyspaceHandle,
_conn_tx: watch::Sender<ConnState>,
conn: watch::Receiver<ConnState>,
}
impl RestPushTransport {
pub fn new(pushes: KeyspaceHandle, http: reqwest::Client) -> Self {
let (tx, rx) = watch::channel(ConnState::Connected);
Self {
http,
pushes,
_conn_tx: tx,
conn: rx,
}
}
}
#[async_trait::async_trait]
impl MessageTransport for RestPushTransport {
fn kind(&self) -> TransportKind {
TransportKind::Rest
}
async fn send(&self, dest: &str, packed: Vec<u8>) -> Result<SendReceipt, MessagingError> {
let record = document_for(&self.pushes, &packed).await?;
let base = record.rest_base.as_deref().ok_or_else(|| {
MessagingError::Transport(format!("{dest} advertises no TrustTaskHTTPS endpoint"))
})?;
let url = format!("{}/trust-tasks", base.trim_end_matches('/'));
let resp = self
.http
.post(&url)
.header("Content-Type", "application/json")
.header("Accept", "application/json")
.json(&record.document)
.send()
.await
.map_err(|e| MessagingError::Transport(format!("POST {url}: {e}")))?;
if !resp.status().is_success() {
return Err(MessagingError::Transport(format!(
"POST {url} answered {}",
resp.status()
)));
}
Ok(SendReceipt {
via: TransportKind::Rest,
hop_id: None,
})
}
fn connection_state(&self) -> watch::Receiver<ConnState> {
self.conn.clone()
}
fn inbound(&self) -> BoxStream<'static, Inbound> {
Box::pin(futures_util::stream::empty())
}
async fn ack(&self, _ack: InboundAck) -> Result<(), MessagingError> {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn doc(services: Value) -> Value {
json!({ "id": "did:example:peer", "service": services })
}
#[test]
fn reach_matches_on_service_type() {
let d = doc(json!([
{ "id": "#whatever", "type": "TSPTransport", "serviceEndpoint": "did:example:med" },
{ "id": "#x", "type": "DIDCommMessaging", "serviceEndpoint": { "uri": "did:example:med" } },
{ "id": "#y", "type": "TrustTaskHTTPS", "serviceEndpoint": "https://peer.example/" },
]));
let r = Reach::from_document(&d);
assert_eq!(r.tsp_mediator.as_deref(), Some("did:example:med"));
assert!(r.didcomm);
assert_eq!(r.rest_base.as_deref(), Some("https://peer.example/"));
}
#[test]
fn only_trust_task_https_counts_as_rest() {
let d = doc(json!([
{ "id": "#rest", "type": "VTARest", "serviceEndpoint": "https://vta.example" },
]));
let r = Reach::from_document(&d);
assert!(r.rest_base.is_none());
assert!(!r.advertises_anything());
}
fn ours_all() -> Ours {
Ours {
tsp: true,
didcomm: true,
rest: true,
}
}
#[test]
fn a_silent_peer_seen_on_tsp_is_tried_over_tsp_then_didcomm() {
let silent = Reach::default();
assert_eq!(
choose(&silent, &ours_all(), true),
vec![Protocol::Tsp, Protocol::Didcomm]
);
assert_eq!(choose(&silent, &ours_all(), false), vec![Protocol::Didcomm]);
}
#[test]
fn learned_reach_never_overrides_an_advertised_document() {
let didcomm_only = Reach {
didcomm: true,
..Reach::default()
};
assert_eq!(
choose(&didcomm_only, &ours_all(), true),
vec![Protocol::Didcomm]
);
}
#[test]
fn a_node_without_tsp_ignores_learned_reach() {
let ours = Ours {
tsp: false,
..ours_all()
};
assert_eq!(
choose(&Reach::default(), &ours, true),
vec![Protocol::Didcomm]
);
}
#[test]
fn a_document_without_services_advertises_nothing() {
assert!(!Reach::from_document(&json!({ "id": "did:key:z6Mk" })).advertises_anything());
}
}