use super::queue::{BackChannelQueue, QueueEntry};
use crate::error::{GarrisonError, GarrisonResult};
use dbnexus::DbPool;
use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::Duration;
pub const DELIVERY_TIMEOUT: Duration = Duration::from_secs(5);
pub const DEFAULT_MAX_TTL_SECS: i64 = 24 * 60 * 60;
pub const DEFAULT_BATCH_SIZE: i64 = 32;
#[derive(Debug, Clone)]
pub struct DelivererConfig {
pub timeout: Duration,
pub batch_size: i64,
}
impl Default for DelivererConfig {
fn default() -> Self {
Self {
timeout: DELIVERY_TIMEOUT,
batch_size: DEFAULT_BATCH_SIZE,
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct DrainSummary {
pub delivered: usize,
pub retried: usize,
pub failed: usize,
}
impl DrainSummary {
pub fn has_activity(&self) -> bool {
*self != Self::default()
}
}
enum Verdict {
Delivered,
Retried,
Failed,
}
pub struct BackChannelDeliverer {
queue: Arc<BackChannelQueue>,
endpoints: BTreeMap<String, String>,
http: reqwest::Client,
config: DelivererConfig,
}
impl BackChannelDeliverer {
pub fn new(
pool: DbPool,
endpoints: impl IntoIterator<Item = (String, String)>,
config: DelivererConfig,
) -> GarrisonResult<Self> {
let queue = Arc::new(BackChannelQueue::new(pool, DEFAULT_MAX_TTL_SECS));
Self::with_queue(queue, endpoints, config)
}
pub fn with_queue(
queue: Arc<BackChannelQueue>,
endpoints: impl IntoIterator<Item = (String, String)>,
config: DelivererConfig,
) -> GarrisonResult<Self> {
let http = reqwest::Client::builder()
.timeout(config.timeout)
.redirect(reqwest::redirect::Policy::none())
.build()
.map_err(|e| GarrisonError::Config(format!("backchannel-http-client::{}", e)))?;
Ok(Self {
queue,
endpoints: endpoints.into_iter().collect(),
http,
config,
})
}
pub fn queue(&self) -> &BackChannelQueue {
&self.queue
}
pub async fn drain_once(&self) -> GarrisonResult<DrainSummary> {
let rows = self.queue.pending_batch(self.config.batch_size).await?;
let mut summary = DrainSummary::default();
for row in &rows {
match self.deliver_one(row).await? {
Verdict::Delivered => summary.delivered += 1,
Verdict::Retried => summary.retried += 1,
Verdict::Failed => summary.failed += 1,
}
}
Ok(summary)
}
pub fn spawn_drain_loop(self: Arc<Self>, interval: Duration) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut ticker = tokio::time::interval(interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
ticker.tick().await;
match self.drain_once().await {
Ok(summary) if summary.has_activity() => {
tracing::info!(
delivered = summary.delivered,
retried = summary.retried,
failed = summary.failed,
"{}",
crate::i18n::translate_detail(
"logs-backchannel-logout-drain-complete",
&[]
)
);
},
Ok(_) => {},
Err(e) => {
tracing::warn!(
"{}",
crate::i18n::translate_detail(
"logs-backchannel-logout-drain-failed",
&[("error", &crate::i18n::translate_error(&e))]
)
);
},
}
}
})
}
async fn deliver_one(&self, row: &QueueEntry) -> GarrisonResult<Verdict> {
let outcome = match self.endpoints.get(&row.client_id) {
None => Err("no-endpoint-configured".to_string()),
Some(uri) => match self.post_logout_token(uri, &row.logout_token).await {
Ok(()) => Ok(()),
Err(reason) => Err(reason),
},
};
match outcome {
Ok(()) => {
self.queue.mark_delivered(&row.id).await?;
Ok(Verdict::Delivered)
},
Err(reason) => {
let now = super::queue::unix_now_millis()?;
if now >= row.max_ttl_at {
let retries = self.queue.increment_retries(&row.id).await?;
self.queue.mark_failed(&row.id).await?;
tracing::warn!(
queue_id = %row.id,
client_id = %row.client_id,
retries,
reason = %reason,
"{}",
crate::i18n::translate_detail(
"logs-backchannel-logout-max-ttl-retired",
&[]
)
);
Ok(Verdict::Failed)
} else {
tracing::debug!(
queue_id = %row.id,
client_id = %row.client_id,
reason = %reason,
"{}",
crate::i18n::translate_detail(
"logs-backchannel-logout-delivery-retried",
&[]
)
);
self.queue.increment_retries(&row.id).await?;
Ok(Verdict::Retried)
}
},
}
}
async fn post_logout_token(&self, uri: &str, logout_token: &str) -> Result<(), String> {
let resp = self
.http
.post(uri)
.form(&[("logout_token", logout_token)])
.send()
.await
.map_err(|e| format!("transport::{}", e))?;
let status = resp.status();
if status.is_success() {
Ok(())
} else {
Err(format!("http-status-{}", status.as_u16()))
}
}
}
#[cfg(all(test, feature = "db-sqlite"))]
mod tests {
use super::*;
use crate::dao::repository::sqlite::test_support::setup_db;
use crate::oauth2_server::backchannel::queue::STATUS_PENDING;
use wiremock::matchers::{body_string_contains, method};
use wiremock::{Mock, MockServer, ResponseTemplate};
async fn setup(max_ttl_secs: i64) -> (DbPool, Arc<BackChannelQueue>) {
let pool = setup_db().await;
let queue = Arc::new(BackChannelQueue::new(pool.clone(), max_ttl_secs));
(pool, queue)
}
fn deliverer(queue: Arc<BackChannelQueue>, uri: String) -> BackChannelDeliverer {
BackChannelDeliverer::with_queue(
queue,
vec![("client-a".to_string(), uri)],
DelivererConfig {
timeout: Duration::from_millis(800),
batch_size: 32,
},
)
.expect("投递器构造应成功")
}
#[test]
fn default_config_matches_spec() {
let config = DelivererConfig::default();
assert_eq!(config.timeout, Duration::from_secs(5));
assert_eq!(DEFAULT_MAX_TTL_SECS, 24 * 60 * 60);
}
#[tokio::test]
async fn delivers_logout_token_as_form_post() {
let (_pool, queue) = setup(3600).await;
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(body_string_contains("logout_token=eyJ"))
.respond_with(ResponseTemplate::new(200))
.expect(1)
.mount(&server)
.await;
queue
.enqueue("client-a", "eyJ.header.payload")
.await
.unwrap();
let summary = deliverer(Arc::clone(&queue), server.uri())
.drain_once()
.await
.expect("drain 应成功");
assert_eq!(
summary,
DrainSummary {
delivered: 1,
retried: 0,
failed: 0
}
);
assert_eq!(queue.count_by_status(STATUS_PENDING).await.unwrap(), 0);
assert_eq!(queue.count_by_status("delivered").await.unwrap(), 1);
server.verify().await;
}
#[tokio::test]
async fn redirect_302_is_not_followed_and_counts_as_retry() {
let (_pool, queue) = setup(3600).await;
let redirect_target = MockServer::start().await;
Mock::given(method("GET"))
.respond_with(ResponseTemplate::new(200))
.expect(0)
.mount(&redirect_target)
.await;
let server = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(
ResponseTemplate::new(302)
.insert_header("Location", &format!("{}/landing", redirect_target.uri())),
)
.expect(1)
.mount(&server)
.await;
let row_id = queue
.enqueue("client-a", "eyJ.header.payload")
.await
.unwrap();
let summary = deliverer(Arc::clone(&queue), server.uri())
.drain_once()
.await
.expect("drain 应成功");
assert_eq!(
summary,
DrainSummary {
delivered: 0,
retried: 1,
failed: 0
},
"302 必须按未投递处理"
);
let entry = queue.find_by_id(&row_id).await.unwrap().expect("行应存在");
assert_eq!(entry.status, STATUS_PENDING, "行应留在 pending 等待重试");
assert_eq!(entry.pending_retries, 1, "302 应计入重试计数");
assert!(
entry.next_attempt_at > super::super::queue::unix_now_millis().unwrap(),
"失败行应被退避推后,不立即重试"
);
let second = deliverer(Arc::clone(&queue), server.uri())
.drain_once()
.await
.unwrap();
assert_eq!(
second,
DrainSummary::default(),
"退避未到期的行不得重复投递"
);
server.verify().await;
redirect_target.verify().await;
}
#[tokio::test]
async fn hanging_rp_hits_timeout_and_counts_as_retry() {
let (_pool, queue) = setup(3600).await;
let server = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(ResponseTemplate::new(200).set_delay(Duration::from_secs(5)))
.expect(1)
.mount(&server)
.await;
let row_id = queue
.enqueue("client-a", "eyJ.header.payload")
.await
.unwrap();
let start = std::time::Instant::now();
let summary = deliverer(Arc::clone(&queue), server.uri())
.drain_once()
.await
.expect("drain 应成功");
let elapsed = start.elapsed();
assert!(
elapsed < Duration::from_secs(4),
"投递应在超时封顶内返回,实际 {:?}",
elapsed
);
assert_eq!(
summary,
DrainSummary {
delivered: 0,
retried: 1,
failed: 0
},
"超时按投递失败计入重试"
);
let entry = queue.find_by_id(&row_id).await.unwrap().expect("行应存在");
assert_eq!(entry.pending_retries, 1, "超时应计入重试计数");
server.verify().await;
}
#[tokio::test]
async fn drain_does_not_redeliver_delivered_rows() {
let (_pool, queue) = setup(3600).await;
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(body_string_contains("logout_token=eyJ"))
.respond_with(ResponseTemplate::new(200))
.expect(1)
.mount(&server)
.await;
queue
.enqueue("client-a", "eyJ.header.payload")
.await
.unwrap();
let d = deliverer(Arc::clone(&queue), server.uri());
let first = d.drain_once().await.unwrap();
assert_eq!(first.delivered, 1);
let second = d.drain_once().await.unwrap();
assert_eq!(
second,
DrainSummary::default(),
"第二轮不得重复投递已 delivered 行"
);
server.verify().await;
}
#[derive(Clone)]
struct LogCapture(Arc<std::sync::Mutex<Vec<String>>>);
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for LogCapture {
type Writer = LogCaptureWriter;
fn make_writer(&'a self) -> Self::Writer {
LogCaptureWriter(self.0.clone())
}
}
struct LogCaptureWriter(Arc<std::sync::Mutex<Vec<String>>>);
impl std::io::Write for LogCaptureWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0
.lock()
.expect("日志缓冲锁应可用")
.push(String::from_utf8_lossy(buf).to_string());
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
#[tokio::test]
async fn expired_row_fails_with_warn_and_stops_being_retried() {
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
let (_pool, queue) = setup(0).await;
let server = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(ResponseTemplate::new(500))
.expect(1)
.mount(&server)
.await;
queue
.enqueue("client-a", "eyJ.header.payload")
.await
.unwrap();
let logs = LogCapture(Arc::new(std::sync::Mutex::new(Vec::new())));
let subscriber = tracing_subscriber::registry().with(
tracing_subscriber::fmt::layer()
.with_ansi(false)
.with_writer(logs.clone()),
);
let _guard = subscriber.set_default();
let summary = deliverer(Arc::clone(&queue), server.uri())
.drain_once()
.await
.expect("drain 应成功");
assert_eq!(
summary,
DrainSummary {
delivered: 0,
retried: 0,
failed: 1
},
"超窗失败行应置 failed"
);
assert_eq!(queue.count_by_status("failed").await.unwrap(), 1);
let captured = logs.0.lock().unwrap().join("\n");
assert!(
captured.contains(&crate::i18n::translate_detail(
"logs-backchannel-logout-max-ttl-retired",
&[]
)),
"超 MaxTtl 退役必须输出 warn,实际日志: {captured}"
);
let second = deliverer(Arc::clone(&queue), server.uri())
.drain_once()
.await
.unwrap();
assert_eq!(second, DrainSummary::default());
server.verify().await;
}
#[tokio::test]
async fn failure_before_max_ttl_increments_retries_only() {
let (_pool, queue) = setup(3600).await;
let server = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(ResponseTemplate::new(500))
.mount(&server)
.await;
let row_id = queue
.enqueue("client-a", "eyJ.header.payload")
.await
.unwrap();
let summary = deliverer(Arc::clone(&queue), server.uri())
.drain_once()
.await
.unwrap();
assert_eq!(
summary,
DrainSummary {
delivered: 0,
retried: 1,
failed: 0
}
);
let entry = queue.find_by_id(&row_id).await.unwrap().expect("行应存在");
assert_eq!(entry.pending_retries, 1, "失败计数应落库");
assert_eq!(entry.status, STATUS_PENDING, "未超窗失败行保持 pending");
assert!(
entry.next_attempt_at > super::super::queue::unix_now_millis().unwrap(),
"失败行应按退避推后下次投递时刻"
);
assert_eq!(queue.count_by_status("failed").await.unwrap(), 0);
}
#[tokio::test]
async fn missing_endpoint_counts_as_retry() {
let (_pool, queue) = setup(3600).await;
let deliverer = BackChannelDeliverer::with_queue(
Arc::clone(&queue),
Vec::<(String, String)>::new(),
DelivererConfig {
timeout: Duration::from_millis(800),
batch_size: 32,
},
)
.unwrap();
let row_id = queue
.enqueue("unconfigured", "eyJ.header.payload")
.await
.unwrap();
let summary = deliverer.drain_once().await.unwrap();
assert_eq!(summary.retried, 1, "缺端点按失败计入重试");
let entry = queue.find_by_id(&row_id).await.unwrap().expect("行应存在");
assert_eq!(entry.pending_retries, 1);
assert!(
entry.next_attempt_at > super::super::queue::unix_now_millis().unwrap(),
"缺端点失败行同样按退避推后"
);
}
#[tokio::test]
async fn backed_off_dead_rp_does_not_starve_healthy_rps() {
let (_pool, queue) = setup(3600).await;
let dead = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(ResponseTemplate::new(500))
.expect(1)
.mount(&dead)
.await;
let healthy = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(ResponseTemplate::new(200))
.expect(1)
.mount(&healthy)
.await;
let deliverer = BackChannelDeliverer::with_queue(
Arc::clone(&queue),
vec![
("client-dead".to_string(), dead.uri()),
("client-fresh".to_string(), healthy.uri()),
],
DelivererConfig {
timeout: Duration::from_millis(800),
batch_size: 32,
},
)
.unwrap();
queue
.enqueue("client-dead", "eyJ.header.payload")
.await
.unwrap();
let first = deliverer.drain_once().await.unwrap();
assert_eq!(first.retried, 1);
queue
.enqueue("client-fresh", "eyJ.header.payload")
.await
.unwrap();
let second = deliverer.drain_once().await.unwrap();
assert_eq!(
second,
DrainSummary {
delivered: 1,
retried: 0,
failed: 0
},
"退避中的失败行不得阻塞健康行投递"
);
dead.verify().await;
healthy.verify().await;
}
#[tokio::test]
async fn spawn_drain_loop_delivers_pending_rows() {
let (_pool, queue) = setup(3600).await;
let server = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(ResponseTemplate::new(200))
.mount(&server)
.await;
queue
.enqueue("client-a", "eyJ.header.payload")
.await
.unwrap();
let deliverer = Arc::new(deliverer(Arc::clone(&queue), server.uri()));
let handle = deliverer.spawn_drain_loop(Duration::from_millis(50));
let deadline = std::time::Instant::now() + Duration::from_secs(5);
while queue.count_by_status("delivered").await.unwrap() == 0 {
assert!(
std::time::Instant::now() < deadline,
"后台 drain 应在窗口内完成投递"
);
tokio::time::sleep(Duration::from_millis(25)).await;
}
handle.abort();
}
}