use std::collections::{HashMap, VecDeque};
use std::sync::{Mutex, MutexGuard, OnceLock};
use std::time::{Duration, Instant};
#[cfg(test)]
use std::sync::atomic::{AtomicU64, Ordering};
use teloxide::Bot;
use teloxide::prelude::Requester;
use teloxide::types::{ChatId, MessageId, ParseMode};
use crate::config::Config;
#[cfg(test)]
static CLOCK_OFFSET_MS: AtomicU64 = AtomicU64::new(0);
#[cfg(not(test))]
pub(crate) fn gate_now() -> Instant {
Instant::now()
}
#[cfg(test)]
pub(crate) fn gate_now() -> Instant {
let off = CLOCK_OFFSET_MS.load(Ordering::Relaxed);
Instant::now()
.checked_add(Duration::from_millis(off))
.expect("virtual clock offset overflow")
}
const SEND_MAX_HOLD: Duration = Duration::from_secs(30);
const FINAL_MAX_ATTEMPTS: u32 = 8;
const DRAIN_TICK: Duration = Duration::from_millis(400);
static GLOBAL_PACER: Mutex<Option<Bucket>> = Mutex::new(None);
const GLOBAL_CAPACITY: u32 = 25;
const GLOBAL_REFILL_PER_SEC: f64 = 25.0;
const GLOBAL_MAX_HOLD: Duration = Duration::from_secs(5);
pub(crate) async fn acquire_global_permit() -> bool {
super::rate_limit::wait_global_cooldown().await;
let mut total_held = Duration::ZERO;
loop {
let now = gate_now();
let wait_res = {
let mut lock = GLOBAL_PACER.lock().unwrap_or_else(|e| e.into_inner());
let bucket = lock.get_or_insert_with(|| {
let mut b = Bucket::new(GLOBAL_CAPACITY, GLOBAL_REFILL_PER_SEC);
b.last_refill = now;
b
});
bucket.take(now)
};
match wait_res {
Ok(()) => {
return true;
}
Err(delay) => {
total_held += delay;
if total_held > GLOBAL_MAX_HOLD {
tracing::warn!(
"Telegram: Global pacer held for {:?} exceeding 5s limit — failing open to avoid hanging turns",
total_held
);
return false;
}
#[cfg(test)]
test_support::advance(delay.as_millis() as u64);
tokio::time::sleep(delay).await;
}
}
}
}
struct Limits {
enabled: bool,
typing_interval: Duration,
typing_burst: u32,
typing_max_hold: Duration,
edit_rate_per_sec: f64,
edit_burst: u32,
send_interval: Duration,
send_minute_ceiling: u32,
send_burst: u32,
rich_rate_per_sec: f64,
rich_burst: u32,
spacing_floor: Duration,
summary_log_period: Duration,
}
impl Limits {
fn from_config() -> Self {
let rl = &Config::current().channels.telegram.rate_limiter;
let ceiling = rl.sends_ceiling_per_minute.max(1);
Self {
enabled: rl.enabled,
typing_interval: Duration::from_secs(rl.typing_min_interval_secs.max(1)),
typing_burst: rl.typing_burst.max(1),
typing_max_hold: Duration::from_secs(rl.typing_max_hold_secs),
edit_rate_per_sec: (rl.edits_per_minute.max(1) as f64) / 60.0,
edit_burst: rl.edit_burst.max(1),
send_interval: Duration::from_millis(rl.send_min_interval_millis.max(50)),
send_minute_ceiling: ceiling,
send_burst: rl.sends_burst.max(1),
rich_rate_per_sec: (rl.rich_per_minute.max(1) as f64) / 60.0,
rich_burst: rl.rich_burst.max(1),
spacing_floor: Duration::from_millis(rl.spacing_floor_ms),
summary_log_period: Duration::from_secs(rl.summary_log_secs.max(30)),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[allow(dead_code)]
pub(crate) enum SpacingPolicy {
Exempt,
Wait,
DropDroppable,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[allow(dead_code)]
pub(crate) enum ChromeOnDry {
Drop,
NotApplicable,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[allow(dead_code)]
pub(crate) enum ContentOnDry {
Drop,
Queue,
Wait,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[allow(dead_code)]
pub(crate) enum HoldBudget {
None,
Typing,
Send,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[allow(dead_code)]
pub(crate) enum HoldExceeded {
DropTyping,
AdmitSend,
Unreachable,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[allow(dead_code)]
pub(crate) enum PermitPolicy {
Ignore,
WarnAndProceed,
Refuse,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[allow(dead_code)]
pub(crate) enum DropReason {
SpacingFloor,
BucketDry,
HoldExceeded,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[allow(dead_code)]
pub(crate) enum PaceOutcome {
Admit,
Bypass,
Drop(DropReason),
Queue,
Defer(Duration),
FailOpen(Duration),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[allow(dead_code)]
pub(crate) struct SurfacePolicy {
pub(crate) gate: &'static str,
pub(crate) ring: &'static str,
pub(crate) spacing: SpacingPolicy,
pub(crate) chrome_on_dry: ChromeOnDry,
pub(crate) content_on_dry: ContentOnDry,
pub(crate) hold: HoldBudget,
pub(crate) hold_exceeded: HoldExceeded,
pub(crate) permit: PermitPolicy,
pub(crate) sets_forum_seen: bool,
}
#[allow(dead_code)]
impl SurfacePolicy {
pub(crate) const TYPING: Self = Self {
gate: "admit_chat_action",
ring: SURFACE_TYPING,
spacing: SpacingPolicy::Exempt,
chrome_on_dry: ChromeOnDry::NotApplicable,
content_on_dry: ContentOnDry::Drop,
hold: HoldBudget::Typing,
hold_exceeded: HoldExceeded::DropTyping,
permit: PermitPolicy::Ignore,
sets_forum_seen: true,
};
pub(crate) const EDITS: Self = Self {
gate: "edit_admission",
ring: SURFACE_EDITS,
spacing: SpacingPolicy::DropDroppable,
chrome_on_dry: ChromeOnDry::Drop,
content_on_dry: ContentOnDry::Queue,
hold: HoldBudget::None,
hold_exceeded: HoldExceeded::Unreachable,
permit: PermitPolicy::Ignore,
sets_forum_seen: false,
};
pub(crate) const SENDS: Self = Self {
gate: "pace_send",
ring: SURFACE_SENDS,
spacing: SpacingPolicy::Wait,
chrome_on_dry: ChromeOnDry::NotApplicable,
content_on_dry: ContentOnDry::Wait,
hold: HoldBudget::Send,
hold_exceeded: HoldExceeded::AdmitSend,
permit: PermitPolicy::WarnAndProceed,
sets_forum_seen: false,
};
pub(crate) const RICH: Self = Self {
gate: "pace_rich",
ring: SURFACE_RICH,
spacing: SpacingPolicy::DropDroppable,
chrome_on_dry: ChromeOnDry::Drop,
content_on_dry: ContentOnDry::Wait,
hold: HoldBudget::None,
hold_exceeded: HoldExceeded::Unreachable,
permit: PermitPolicy::Refuse,
sets_forum_seen: true,
};
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum HoldSurface {
Typing,
Sends,
Rich,
}
fn fold_hold_ms(chat_id: i64, waited: Duration, surface: HoldSurface) {
if waited.is_zero() {
return;
}
let ms = waited.as_millis() as u64;
if ms == 0 {
return;
}
if let Some(peer) = peers_guard().get_mut(&chat_id) {
match surface {
HoldSurface::Typing => peer.counters.throttled_typing_ms += ms,
HoldSurface::Sends => peer.counters.throttled_send_ms += ms,
HoldSurface::Rich => peer.counters.throttled_rich_ms += ms,
}
}
}
fn pace_engine<F>(
policy: &SurfacePolicy,
lim: &Limits,
chat_id: i64,
thread_id: Option<i32>,
waited_so_far: Duration,
admit: F,
) -> PaceOutcome
where
F: FnOnce(&mut Peer, Instant, Duration) -> PaceOutcome,
{
let mut map = peers_guard();
let peer = map.entry(chat_id).or_default();
if policy.sets_forum_seen && thread_id.is_some() {
peer.forum_seen = true;
}
if !peer.forum_seen {
return PaceOutcome::Bypass;
}
let now = gate_now();
let spacing_hold = match policy.spacing {
SpacingPolicy::Exempt => Duration::ZERO,
SpacingPolicy::Wait | SpacingPolicy::DropDroppable => {
spacing_wait(&peer.recent, now, lim.spacing_floor)
}
};
let verdict = admit(peer, now, spacing_hold);
if let PaceOutcome::Defer(wait) = verdict {
let over = match policy.hold {
HoldBudget::None => false,
HoldBudget::Typing => waited_so_far + wait > lim.typing_max_hold,
HoldBudget::Send => waited_so_far + wait > SEND_MAX_HOLD,
};
if !over {
return PaceOutcome::Defer(wait);
}
match policy.hold_exceeded {
HoldExceeded::DropTyping => {
peer.counters.dropped_typing += 1;
PaceOutcome::Drop(DropReason::HoldExceeded)
}
HoldExceeded::AdmitSend => {
peer.counters.admitted_sends += 1;
peer.recent.push(now, SURFACE_SENDS);
PaceOutcome::FailOpen(wait)
}
HoldExceeded::Unreachable => PaceOutcome::Drop(DropReason::HoldExceeded),
}
} else {
verdict
}
}
pub(crate) struct Bucket {
pub(crate) tokens: f64,
capacity: f64,
refill_per_sec: f64,
last_refill: Instant,
pause_until: Option<Instant>,
}
impl Bucket {
pub(crate) fn new(capacity: u32, refill_per_sec: f64) -> Self {
Self {
tokens: f64::from(capacity),
capacity: f64::from(capacity),
refill_per_sec,
last_refill: Instant::now(),
pause_until: None,
}
}
fn refill(&mut self, now: Instant) {
if let Some(until) = self.pause_until {
if until > now {
return;
}
self.last_refill = until;
self.pause_until = None;
}
let elapsed = now.saturating_duration_since(self.last_refill);
self.last_refill = now;
let gained = elapsed.as_secs_f64() * self.refill_per_sec;
self.tokens = (self.tokens + gained).min(self.capacity);
}
pub(crate) fn take(&mut self, now: Instant) -> Result<(), Duration> {
let wait = self.next_token_in(now);
if wait.is_zero() {
self.tokens -= 1.0;
Ok(())
} else {
Err(wait)
}
}
fn next_token_in(&mut self, now: Instant) -> Duration {
self.refill(now);
let pause = self.pause_remaining(now);
let refill = if self.tokens >= 1.0 {
Duration::ZERO
} else {
Duration::from_secs_f64((1.0 - self.tokens) / self.refill_per_sec)
};
refill.max(pause)
}
fn pause_remaining(&self, now: Instant) -> Duration {
match self.pause_until {
Some(until) if until > now => until.duration_since(now),
_ => Duration::ZERO,
}
}
pub(crate) fn paused_until(&mut self, now: Instant) -> Option<Instant> {
match self.pause_until {
Some(until) if until > now => Some(until),
Some(_) => {
self.pause_until = None;
None
}
None => None,
}
}
pub(crate) fn pause_arm(&mut self, until: Instant) {
self.pause_until = Some(match self.pause_until {
Some(existing) if existing > until => existing,
_ => until,
});
}
}
pub(crate) fn ensure_bucket(
slot: &mut Option<Bucket>,
capacity: u32,
refill_per_sec: f64,
) -> &mut Bucket {
let stale = match slot {
Some(b) => {
b.capacity != f64::from(capacity)
|| (b.refill_per_sec - refill_per_sec).abs() > f64::EPSILON
}
None => true,
};
if stale {
let carried = slot.as_ref().and_then(|b| b.pause_until);
let mut fresh = Bucket::new(capacity, refill_per_sec);
fresh.pause_until = carried;
*slot = Some(fresh);
}
slot.as_mut().expect("bucket was just built")
}
pub(crate) const RECENT_CAP: usize = 64;
pub(crate) const SURFACE_TYPING: &str = "typing";
pub(crate) const SURFACE_EDITS: &str = "edits";
pub(crate) const SURFACE_SENDS: &str = "sends";
pub(crate) const SURFACE_RICH: &str = "rich";
#[derive(Default)]
pub(crate) struct Recent {
t: VecDeque<(Instant, &'static str)>,
}
impl Recent {
pub(crate) fn push(&mut self, now: Instant, surface: &'static str) {
if self.t.len() == RECENT_CAP {
self.t.pop_front();
}
self.t.push_back((now, surface));
}
pub(crate) fn count_within(&self, now: Instant, window: Duration) -> usize {
self.t
.iter()
.filter(|(t, _)| now.saturating_duration_since(*t) < window)
.count()
}
pub(crate) fn count_surface(&self, now: Instant, window: Duration, surface: &str) -> usize {
self.t
.iter()
.filter(|(t, s)| *s == surface && now.saturating_duration_since(*t) < window)
.count()
}
pub(crate) fn gap_ms(&self, now: Instant) -> Option<u128> {
self.t
.back()
.map(|(t, _)| now.saturating_duration_since(*t).as_millis())
}
pub(crate) fn profile(&self, now: Instant) -> RecentProfile {
let minute = Duration::from_secs(60);
RecentProfile {
last_1s: self.count_within(now, Duration::from_secs(1)),
last_5s: self.count_within(now, Duration::from_secs(5)),
last_60s: self.count_within(now, minute),
typing: self.count_surface(now, minute, SURFACE_TYPING),
edits: self.count_surface(now, minute, SURFACE_EDITS),
sends: self.count_surface(now, minute, SURFACE_SENDS),
rich: self.count_surface(now, minute, SURFACE_RICH),
gap_ms: self.gap_ms(now),
}
}
}
#[derive(Debug, Default, PartialEq, Eq)]
pub(crate) struct RecentProfile {
pub(crate) last_1s: usize,
pub(crate) last_5s: usize,
pub(crate) last_60s: usize,
pub(crate) typing: usize,
pub(crate) edits: usize,
pub(crate) sends: usize,
pub(crate) rich: usize,
pub(crate) gap_ms: Option<u128>,
}
impl RecentProfile {
pub(crate) fn render(&self) -> String {
format!(
"window{{1s={},5s={},60s={}}} by_surface{{typing={},edits={},sends={},rich={}}} gap_ms={}",
self.last_1s,
self.last_5s,
self.last_60s,
self.typing,
self.edits,
self.sends,
self.rich,
self.gap_ms
.map(|g| g.to_string())
.unwrap_or_else(|| "none".to_string()),
)
}
}
pub(crate) fn recent_profile(chat: Option<i64>) -> String {
let now = gate_now();
let map = peers().lock().unwrap_or_else(|e| e.into_inner());
let mut parts: Vec<String> = Vec::new();
for (id, peer) in map.iter() {
if chat.is_some_and(|want| *id != want) {
continue;
}
parts.push(format!("chat={id} {}", peer.recent.profile(now).render()));
}
if parts.is_empty() {
"none".to_string()
} else {
parts.join("; ")
}
}
pub(crate) fn spacing_wait(recent: &Recent, now: Instant, floor: Duration) -> Duration {
if floor.is_zero() {
return Duration::ZERO;
}
match recent.gap_ms(now) {
Some(gap) => {
let gap_ms = u64::try_from(gap).unwrap_or(u64::MAX);
floor.saturating_sub(Duration::from_millis(gap_ms))
}
None => Duration::ZERO,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[allow(dead_code)]
pub(crate) enum FinalDialect {
#[default]
Html,
Markdown,
}
#[derive(Clone)]
struct PendingFinal {
bot: Bot,
html: String,
rich: bool,
dialect: FinalDialect,
media: Vec<super::rich::mermaid::MediaEntry>,
reply_markup: Option<serde_json::Value>,
attempts: u32,
}
#[derive(Default)]
pub(crate) struct Counters {
pub(crate) admitted_typing: u64,
pub(crate) admitted_edits: u64,
pub(crate) admitted_sends: u64,
pub(crate) admitted_rich: u64,
pub(crate) dropped_typing: u64,
pub(crate) dropped_spacing: u64,
pub(crate) dropped_clock: u64,
pub(crate) dropped_brain_preview: u64,
pub(crate) dropped_intermediary: u64,
pub(crate) dropped_status: u64,
pub(crate) queued_finals: u64,
pub(crate) superseded_finals: u64,
pub(crate) delivered_finals: u64,
pub(crate) failed_finals: u64,
pub(crate) throttled_typing_ms: u64,
pub(crate) throttled_send_ms: u64,
pub(crate) throttled_rich_ms: u64,
pub(crate) pause_armed_429: u64,
}
impl Counters {
pub(crate) fn note_drop(&mut self, class: EditClass) {
match class {
EditClass::Clock => self.dropped_clock += 1,
EditClass::BrainPreview => self.dropped_brain_preview += 1,
EditClass::Intermediary => self.dropped_intermediary += 1,
EditClass::Status => self.dropped_status += 1,
EditClass::Final | EditClass::Interactive => {}
}
}
pub(crate) fn note_spacing_drop(&mut self) {
self.dropped_spacing += 1;
}
fn all_zero(&self) -> bool {
self.admitted_typing == 0
&& self.admitted_edits == 0
&& self.admitted_sends == 0
&& self.dropped_typing == 0
&& self.dropped_spacing == 0
&& self.dropped_clock == 0
&& self.dropped_brain_preview == 0
&& self.dropped_intermediary == 0
&& self.dropped_status == 0
&& self.queued_finals == 0
&& self.superseded_finals == 0
&& self.delivered_finals == 0
&& self.failed_finals == 0
&& self.throttled_typing_ms == 0
&& self.throttled_send_ms == 0
&& self.pause_armed_429 == 0
}
}
#[derive(Default)]
struct Peer {
forum_seen: bool,
typing: Option<Bucket>,
edits: Option<Bucket>,
sends_sec: Option<Bucket>,
sends_min: Option<Bucket>,
rich: Option<Bucket>,
finals: HashMap<i32, PendingFinal>,
draining: bool,
counters: Counters,
recent: Recent,
}
impl Peer {
fn paused_until(&mut self, now: Instant) -> Option<Instant> {
let mut newest: Option<Instant> = None;
for slot in [
&mut self.typing,
&mut self.edits,
&mut self.sends_sec,
&mut self.sends_min,
&mut self.rich,
] {
if let Some(until) = slot.as_mut().and_then(|b| b.paused_until(now)) {
newest = Some(newest.map_or(until, |n: Instant| n.max(until)));
}
}
newest
}
}
fn peers() -> &'static Mutex<HashMap<i64, Peer>> {
static PEERS: OnceLock<Mutex<HashMap<i64, Peer>>> = OnceLock::new();
PEERS.get_or_init(|| Mutex::new(HashMap::new()))
}
fn peers_guard() -> MutexGuard<'static, HashMap<i64, Peer>> {
peers().lock().unwrap_or_else(|e| e.into_inner())
}
const MAX_429_PAUSE: Duration = Duration::from_secs(45);
pub(crate) fn note_429_pause(chat_id: i64, wait: Duration) {
super::rate_limit::record_global_429(wait);
if chat_id >= 0 {
return;
}
let wait = wait.min(MAX_429_PAUSE);
if wait.is_zero() {
return;
}
let until = gate_now() + wait + super::rate_limit::RETRY_MARGIN;
let mut map = peers_guard();
let peer = map.entry(chat_id).or_default();
for slot in [
&mut peer.typing,
&mut peer.edits,
&mut peer.sends_sec,
&mut peer.sends_min,
&mut peer.rich,
] {
let bucket = slot.get_or_insert_with(|| Bucket::new(1, 1.0));
bucket.pause_arm(until);
}
peer.counters.pause_armed_429 += 1;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct Cooldown {
pub(crate) global_until: Option<Instant>,
pub(crate) chat_until: Option<Instant>,
}
impl Cooldown {
#[allow(dead_code)]
pub(crate) fn is_global(self) -> bool {
self.global_until.is_some()
}
pub(crate) fn is_any(self) -> bool {
self.global_until.is_some() || self.chat_until.is_some()
}
}
pub(crate) fn chat_paused_until(chat_id: i64, now: Instant) -> Cooldown {
let global_until = match super::rate_limit::global_cooldown_deadline() {
Some(deadline) if deadline > now => Some(deadline),
_ => None,
};
let chat_until = peers_guard()
.get_mut(&chat_id)
.and_then(|peer| peer.paused_until(now));
Cooldown {
global_until,
chat_until,
}
}
pub(crate) fn format_summary(
chat_id: i64,
c: &Counters,
finals_pending: usize,
recent: Option<&RecentProfile>,
) -> Option<String> {
if c.all_zero() && finals_pending == 0 {
return None;
}
let recent_block = recent
.map(|p| format!(" {}", p.render()))
.unwrap_or_default();
Some(format!(
"Telegram rate-limiter chat={chat_id}: \
admitted{{typing={},edits={},sends={},rich={}}} \
dropped{{clock={},brain_preview={},intermediary={},status={},typing={},spacing={}}} \
finals{{queued={},superseded={},delivered={},failed={},pending={}}} \
throttled_ms{{typing={},send={},rich={}}} \
pause{{pause_armed={}}}{recent_block}",
c.admitted_typing,
c.admitted_edits,
c.admitted_sends,
c.admitted_rich,
c.dropped_clock,
c.dropped_brain_preview,
c.dropped_intermediary,
c.dropped_status,
c.dropped_typing,
c.dropped_spacing,
c.queued_finals,
c.superseded_finals,
c.delivered_finals,
c.failed_finals,
finals_pending,
c.throttled_typing_ms,
c.throttled_send_ms,
c.throttled_rich_ms,
c.pause_armed_429,
))
}
#[cfg(not(test))]
async fn summary_loop() {
loop {
let period = Limits::from_config().summary_log_period;
tokio::time::sleep(period).await;
let lines: Vec<String> = {
let map = peers_guard();
map.iter()
.filter_map(|(chat_id, peer)| {
let profile = peer.recent.profile(gate_now());
format_summary(*chat_id, &peer.counters, peer.finals.len(), Some(&profile))
})
.collect()
};
for line in lines {
tracing::info!("{line}");
}
}
}
fn ensure_summary_task() {
#[cfg(test)]
{
let _ = Limits::from_config().summary_log_period;
}
#[cfg(not(test))]
{
static STARTED: OnceLock<()> = OnceLock::new();
STARTED.get_or_init(|| {
tokio::spawn(summary_loop());
});
}
}
pub(crate) async fn admit_chat_action(chat: ChatId, thread_id: Option<i32>) -> bool {
if chat_paused_until(chat.0, gate_now()).is_any() {
return false;
}
let chat_id = chat.0;
if chat_id >= 0 {
return true;
}
let lim = Limits::from_config();
if !lim.enabled {
return true;
}
ensure_summary_task();
let mut waited = Duration::ZERO;
loop {
let decision = pace_engine(
&SurfacePolicy::TYPING,
&lim,
chat_id,
thread_id,
waited,
|peer, now, _spacing_hold| {
let bucket = ensure_bucket(
&mut peer.typing,
lim.typing_burst,
1.0 / lim.typing_interval.as_secs_f64(),
);
match bucket.take(now) {
Ok(()) => {
peer.counters.admitted_typing += 1;
peer.recent.push(now, SURFACE_TYPING);
PaceOutcome::Admit
}
Err(wait) => PaceOutcome::Defer(wait),
}
},
);
match decision {
PaceOutcome::Bypass | PaceOutcome::Admit => {
fold_hold_ms(chat_id, waited, HoldSurface::Typing);
return true;
}
PaceOutcome::Drop(_) => {
tracing::debug!(
"Telegram rate-limiter: typing refresh dropped for chat={chat_id} after \
holding {waited:?} (hold cap {}s)",
lim.typing_max_hold.as_secs()
);
fold_hold_ms(chat_id, waited, HoldSurface::Typing);
return false;
}
PaceOutcome::Defer(wait) => {
#[cfg(not(test))]
let start = gate_now();
tokio::time::sleep(wait).await;
#[cfg(test)]
test_support::advance(wait.as_millis() as u64);
#[cfg(test)]
let elapsed = wait;
#[cfg(not(test))]
let elapsed = gate_now().duration_since(start);
waited += elapsed;
}
PaceOutcome::Queue | PaceOutcome::FailOpen(_) => {
fold_hold_ms(chat_id, waited, HoldSurface::Typing);
return true;
}
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum EditClass {
Clock,
BrainPreview,
Intermediary,
Status,
Final,
Interactive,
}
impl EditClass {
pub(crate) fn drop_rank(self) -> u8 {
match self {
EditClass::Clock => 0,
EditClass::BrainPreview => 1,
EditClass::Intermediary => 2,
EditClass::Status => 3,
EditClass::Final => 4,
EditClass::Interactive => 5,
}
}
pub(crate) fn is_droppable(self) -> bool {
!matches!(self, EditClass::Final | EditClass::Interactive)
}
}
pub(crate) async fn edit_admission(
bot: &Bot,
chat_id: ChatId,
msg_id: MessageId,
class: EditClass,
html: String,
rich: bool,
) -> bool {
edit_admission_media_kb(
bot,
chat_id,
msg_id,
class,
html,
rich,
Vec::new(),
None,
FinalDialect::Html,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn edit_admission_media_kb(
bot: &Bot,
chat_id: ChatId,
msg_id: MessageId,
class: EditClass,
html: String,
rich: bool,
media: Vec<super::rich::mermaid::MediaEntry>,
reply_markup: Option<serde_json::Value>,
dialect: FinalDialect,
) -> bool {
if chat_paused_until(chat_id.0, gate_now()).is_any()
&& class != EditClass::Final
&& class != EditClass::Interactive
{
return false;
}
if chat_id.0 >= 0 {
return true;
}
let lim = Limits::from_config();
if !lim.enabled {
return true;
}
ensure_summary_task();
let admission = pace_engine(
&SurfacePolicy::EDITS,
&lim,
chat_id.0,
None,
Duration::ZERO,
|peer, now, spacing_hold| {
let droppable = class.is_droppable();
if droppable && !spacing_hold.is_zero() {
peer.counters.note_spacing_drop();
return PaceOutcome::Drop(DropReason::SpacingFloor);
}
let bucket = ensure_bucket(&mut peer.edits, lim.edit_burst, lim.edit_rate_per_sec);
if bucket.take(now).is_ok() {
peer.counters.admitted_edits += 1;
if !rich {
peer.recent.push(now, SURFACE_EDITS);
}
PaceOutcome::Admit
} else if class == EditClass::Interactive {
if !rich {
peer.recent.push(now, SURFACE_EDITS);
}
PaceOutcome::Admit
} else if class == EditClass::Final {
let superseded = peer
.finals
.insert(
msg_id.0,
PendingFinal {
bot: bot.clone(),
html,
rich,
dialect,
media,
reply_markup,
attempts: 0,
},
)
.is_some();
if superseded {
peer.counters.superseded_finals += 1;
}
peer.counters.queued_finals += 1;
PaceOutcome::Queue
} else {
peer.counters.note_drop(class);
PaceOutcome::Drop(DropReason::BucketDry)
}
},
);
match admission {
PaceOutcome::Bypass | PaceOutcome::Admit => true,
PaceOutcome::Drop(_) => {
tracing::debug!(
"Telegram rate-limiter: {:?} edit (rank {}) dropped for chat={} msg={} — \
next full-state refresh carries it",
class,
class.drop_rank(),
chat_id.0,
msg_id.0
);
false
}
PaceOutcome::Queue => {
tracing::debug!(
"Telegram rate-limiter: final edit queued latest-wins for chat={} msg={}",
chat_id.0,
msg_id.0
);
ensure_drain(chat_id.0);
false
}
PaceOutcome::Defer(_) | PaceOutcome::FailOpen(_) => true,
}
}
fn take_due_final(chat_id: i64, lim: &Limits) -> Option<(i32, PendingFinal)> {
let mut map = peers_guard();
let peer = map.get_mut(&chat_id)?;
if peer.finals.is_empty() {
peer.draining = false;
return None;
}
let bucket = ensure_bucket(&mut peer.edits, lim.edit_burst, lim.edit_rate_per_sec);
if bucket.take(gate_now()).is_err() {
return None;
}
let key = peer.finals.keys().next().copied()?;
peer.finals.remove(&key).map(|pending| (key, pending))
}
async fn drain_finals(chat_id: i64) {
loop {
tokio::time::sleep(DRAIN_TICK).await;
#[cfg(test)]
test_support::advance(DRAIN_TICK.as_millis() as u64);
let lim = Limits::from_config();
let Some(job) = take_due_final(chat_id, &lim) else {
let empty = peers_guard()
.get(&chat_id)
.is_none_or(|p| p.finals.is_empty());
if empty {
return;
}
continue;
};
deliver_final(chat_id, job.0, job.1).await;
}
}
fn ensure_drain(chat_id: i64) {
let should_spawn = {
let mut map = peers_guard();
let peer = map.entry(chat_id).or_default();
if peer.draining || peer.finals.is_empty() {
false
} else {
peer.draining = true;
true
}
};
if should_spawn {
tokio::spawn(drain_finals(chat_id));
}
}
pub(crate) fn is_permanent_edit_error(error: &str) -> bool {
error.contains("message to edit not found")
|| error.contains("message is not modified")
|| error.contains("message can't be edited")
|| error.contains("MESSAGE_ID_INVALID")
|| error.contains("chat not found")
}
enum Verdict {
Delivered,
Abandoned(String),
Retried,
}
async fn deliver_final(chat_id: i64, msg_id: i32, mut pending: PendingFinal) {
let result = run_final_edit(chat_id, msg_id, &pending).await;
let retry_after = match &result {
Err(e) => super::rate_limit::parse_retry_after(e),
Ok(()) => None,
};
if let Some(wait) = retry_after {
note_429_pause(chat_id, wait);
}
let verdict = {
let mut map = peers_guard();
let Some(peer) = map.get_mut(&chat_id) else {
#[cfg(test)]
test_support::notify_settled();
return;
};
match result {
Ok(()) => {
peer.counters.delivered_finals += 1;
Verdict::Delivered
}
Err(e) => {
pending.attempts += 1;
if pending.attempts >= FINAL_MAX_ATTEMPTS || is_permanent_edit_error(&e) {
peer.counters.failed_finals += 1;
Verdict::Abandoned(e)
} else {
peer.finals.insert(msg_id, pending);
Verdict::Retried
}
}
}
};
match verdict {
Verdict::Delivered => {
tracing::debug!(
"Telegram rate-limiter: queued final delivered chat={chat_id} msg={msg_id}"
);
}
Verdict::Abandoned(e) => {
tracing::warn!(
"Telegram rate-limiter: queued final abandoned after {FINAL_MAX_ATTEMPTS} \
attempts chat={chat_id} msg={msg_id}: {e}"
);
}
Verdict::Retried => {
if let Some(window) = retry_after {
let (wait, _) = super::rate_limit::clamp_inline_wait(window);
tokio::time::sleep(wait).await;
#[cfg(test)]
test_support::advance(wait.as_millis() as u64);
}
}
}
#[cfg(test)]
test_support::notify_settled();
}
async fn run_final_edit(chat_id: i64, msg_id: i32, pending: &PendingFinal) -> Result<(), String> {
match (pending.rich, pending.dialect) {
(true, FinalDialect::Markdown) => super::rich::api::edit_rich_markdown_media(
pending.bot.api_url().as_str(),
pending.bot.token(),
chat_id,
msg_id,
&pending.html,
&pending.media,
pending.reply_markup.as_ref(),
"turn",
"-",
)
.await
.map_err(|e| e.to_string()),
(true, FinalDialect::Html) => super::rich::api::edit_rich_html(
pending.bot.api_url().as_str(),
pending.bot.token(),
chat_id,
msg_id,
&pending.html,
pending.reply_markup.as_ref(),
"turn",
"-",
)
.await
.map_err(|e| e.to_string()),
(false, _) => {
use teloxide::payloads::EditMessageTextSetters;
let mut req = pending
.bot
.edit_message_text(ChatId(chat_id), MessageId(msg_id), &pending.html)
.parse_mode(ParseMode::Html);
if let Some(inline_kb) = pending.reply_markup.as_ref().and_then(|kb| {
serde_json::from_value::<teloxide::types::InlineKeyboardMarkup>(kb.clone()).ok()
}) {
req = req.reply_markup(inline_kb);
}
req.await.map(|_| ()).map_err(|e| e.to_string())
}
}
}
pub(crate) async fn pace_send(chat: ChatId) {
let chat_id = chat.0;
if chat_id >= 0 {
return;
}
acquire_global_permit().await;
let lim = Limits::from_config();
if !lim.enabled {
return;
}
let mut waited = Duration::ZERO;
loop {
let verdict = pace_engine(
&SurfacePolicy::SENDS,
&lim,
chat_id,
None,
waited,
|peer, now, spacing_hold| {
let sec = ensure_bucket(
&mut peer.sends_sec,
lim.send_burst,
1.0 / lim.send_interval.as_secs_f64(),
);
let min = ensure_bucket(
&mut peer.sends_min,
lim.send_minute_ceiling,
f64::from(lim.send_minute_ceiling) / 60.0,
);
let need = sec
.next_token_in(now)
.max(min.next_token_in(now))
.max(spacing_hold);
if need.is_zero() {
let _ = sec.take(now);
let _ = min.take(now);
peer.counters.admitted_sends += 1;
peer.recent.push(now, SURFACE_SENDS);
PaceOutcome::Admit
} else {
PaceOutcome::Defer(need)
}
},
);
match verdict {
PaceOutcome::Bypass => return,
PaceOutcome::Admit => {
fold_hold_ms(chat_id, waited, HoldSurface::Sends);
return;
}
PaceOutcome::FailOpen(next) => {
tracing::warn!(
"Telegram rate-limiter: send pacing held {waited:?} for chat={chat_id}, \
still {} from a token — failing open to the reactive backstop",
next.as_secs_f64()
);
fold_hold_ms(chat_id, waited, HoldSurface::Sends);
return;
}
PaceOutcome::Defer(delay) => {
#[cfg(not(test))]
let start = gate_now();
tokio::time::sleep(delay).await;
#[cfg(test)]
test_support::advance(delay.as_millis() as u64);
#[cfg(test)]
let elapsed = delay;
#[cfg(not(test))]
let elapsed = gate_now().duration_since(start);
waited += elapsed;
}
PaceOutcome::Drop(_) | PaceOutcome::Queue => {
fold_hold_ms(chat_id, waited, HoldSurface::Sends);
return;
}
}
}
}
pub(crate) async fn pace_rich(chat: ChatId, thread_id: Option<i32>) {
let chat_id = chat.0;
if chat_id >= 0 {
return;
}
acquire_global_permit().await;
let lim = Limits::from_config();
if !lim.enabled {
return;
}
ensure_summary_task();
let mut waited = Duration::ZERO;
loop {
let verdict = pace_engine(
&SurfacePolicy::RICH,
&lim,
chat_id,
thread_id,
waited,
|peer, now, spacing_hold| {
let bucket = ensure_bucket(&mut peer.rich, lim.rich_burst, lim.rich_rate_per_sec);
let need = bucket.next_token_in(now).max(spacing_hold);
if need.is_zero() {
let _ = bucket.take(now);
peer.counters.admitted_rich += 1;
peer.recent.push(now, SURFACE_RICH);
PaceOutcome::Admit
} else {
PaceOutcome::Defer(need)
}
},
);
match verdict {
PaceOutcome::Bypass | PaceOutcome::Admit => {
fold_hold_ms(chat_id, waited, HoldSurface::Rich);
return;
}
PaceOutcome::Defer(delay) => {
#[cfg(not(test))]
let start = gate_now();
tokio::time::sleep(delay).await;
#[cfg(test)]
test_support::advance(delay.as_millis() as u64);
#[cfg(test)]
let elapsed = delay;
#[cfg(not(test))]
let elapsed = gate_now().duration_since(start);
waited += elapsed;
}
PaceOutcome::Drop(_) | PaceOutcome::Queue | PaceOutcome::FailOpen(_) => {
fold_hold_ms(chat_id, waited, HoldSurface::Rich);
return;
}
}
}
}
#[cfg(test)]
pub(crate) mod test_support {
use super::*;
use std::sync::atomic::Ordering;
pub(crate) async fn registry_guard() -> tokio::sync::MutexGuard<'static, ()> {
static GUARD: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
GUARD.lock().await
}
pub(crate) fn reset(offset_ms: u64) {
peers_guard().clear();
*GLOBAL_PACER.lock().unwrap_or_else(|e| e.into_inner()) = None;
crate::channels::telegram::rate_limit::reset_global_cooldown();
CLOCK_OFFSET_MS.store(offset_ms, Ordering::Relaxed);
}
pub(crate) fn advance(ms: u64) {
CLOCK_OFFSET_MS.fetch_add(ms, Ordering::Relaxed);
}
pub(crate) fn settle_notifier() -> &'static tokio::sync::Notify {
static SETTLED: std::sync::OnceLock<tokio::sync::Notify> = std::sync::OnceLock::new();
SETTLED.get_or_init(tokio::sync::Notify::new)
}
pub(crate) fn notify_settled() {
settle_notifier().notify_one();
}
pub(crate) fn mark_forum(chat: ChatId) {
let chat_id = chat.0;
peers_guard().entry(chat_id).or_default().forum_seen = true;
}
pub(crate) fn burn_bucket(chat: ChatId, kind: BucketKind, capacity: u32, rate_per_sec: f64) {
let chat_id = chat.0;
let mut map = peers_guard();
let peer = map.entry(chat_id).or_default();
peer.forum_seen = true;
let slot = match kind {
BucketKind::Typing => &mut peer.typing,
BucketKind::Edits => &mut peer.edits,
BucketKind::SendsSec => &mut peer.sends_sec,
BucketKind::SendsMin => &mut peer.sends_min,
BucketKind::Rich => &mut peer.rich,
};
let bucket = ensure_bucket(slot, capacity, rate_per_sec);
for _ in 0..capacity {
let _ = bucket.take(gate_now());
}
}
#[derive(Debug, Clone, Copy)]
pub(crate) enum BucketKind {
Typing,
Edits,
SendsSec,
SendsMin,
Rich,
}
#[derive(Debug, Default, PartialEq, Eq)]
pub(crate) struct Snap {
pub forum_seen: bool,
pub typing_admitted: u64,
pub typing_dropped: u64,
pub edits_admitted: u64,
pub dropped_clock: u64,
pub dropped_brain_preview: u64,
pub dropped_intermediary: u64,
pub dropped_status: u64,
pub dropped_spacing: u64,
pub queued_finals: u64,
pub superseded_finals: u64,
pub delivered_finals: u64,
pub failed_finals: u64,
pub throttled_typing_ms: u64,
pub admitted_rich: u64,
pub admitted_sends: u64,
pub throttled_rich_ms: u64,
pub throttled_send_ms: u64,
pub finals_pending: usize,
pub pause_armed_429: u64,
}
pub(crate) fn snapshot(chat: ChatId) -> Option<Snap> {
let chat_id = chat.0;
let map = peers_guard();
map.get(&chat_id).map(|p| Snap {
forum_seen: p.forum_seen,
typing_admitted: p.counters.admitted_typing,
typing_dropped: p.counters.dropped_typing,
edits_admitted: p.counters.admitted_edits,
dropped_clock: p.counters.dropped_clock,
dropped_brain_preview: p.counters.dropped_brain_preview,
dropped_intermediary: p.counters.dropped_intermediary,
dropped_status: p.counters.dropped_status,
dropped_spacing: p.counters.dropped_spacing,
queued_finals: p.counters.queued_finals,
superseded_finals: p.counters.superseded_finals,
delivered_finals: p.counters.delivered_finals,
failed_finals: p.counters.failed_finals,
throttled_typing_ms: p.counters.throttled_typing_ms,
admitted_rich: p.counters.admitted_rich,
admitted_sends: p.counters.admitted_sends,
throttled_rich_ms: p.counters.throttled_rich_ms,
throttled_send_ms: p.counters.throttled_send_ms,
finals_pending: p.finals.len(),
pause_armed_429: p.counters.pause_armed_429,
})
}
}