use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use crate::control::ControlStore;
use crate::doc::{impl_doc, DocRef, ShardedMap};
use crate::error::Result;
pub const FLOW_REF: &str = "refs/gunnar/flow";
pub const DAY_MS: i64 = 86_400_000;
pub const RETENTION_DAYS: i64 = 400;
pub const PRINCIPAL_CEILING: usize = 10_000;
pub fn day_of(unix_ms: i64) -> i64 {
unix_ms.div_euclid(DAY_MS)
}
pub fn midnight_of(day: i64) -> i64 {
day.saturating_mul(DAY_MS)
}
pub fn iso8601_utc(unix_ms: i64) -> String {
let secs = unix_ms.div_euclid(1000);
let days = secs.div_euclid(86_400);
let rem = secs.rem_euclid(86_400);
let (hh, mm, ss) = (rem / 3600, (rem % 3600) / 60, rem % 60);
let z = days + 719_468;
let era = z.div_euclid(146_097);
let doe = z.rem_euclid(146_097);
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
let y = yoe + era * 400;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let m = if mp < 10 { mp + 3 } else { mp - 9 };
let y = if m <= 2 { y + 1 } else { y };
format!("{y:04}-{m:02}-{d:02}T{hh:02}:{mm:02}:{ss:02}Z")
}
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
pub enum Refusal {
Admission,
Cap,
Policy,
}
impl Refusal {
pub fn name(self) -> &'static str {
match self {
Refusal::Admission => "admission",
Refusal::Cap => "cap",
Refusal::Policy => "policy",
}
}
pub const ALL: [Refusal; 3] = [Refusal::Admission, Refusal::Cap, Refusal::Policy];
}
#[derive(Clone, Default, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct FlowDay {
#[serde(default)]
pub bytes_in: u64,
#[serde(default)]
pub bytes_out: u64,
#[serde(default)]
pub pushes: u64,
#[serde(default)]
pub fetches: u64,
#[serde(default)]
pub lfs_bytes_in: u64,
#[serde(default)]
pub lfs_bytes_out: u64,
#[serde(default)]
pub refused_admission: u64,
#[serde(default)]
pub refused_cap: u64,
#[serde(default)]
pub refused_policy: u64,
#[serde(default)]
pub auth_failures: u64,
#[serde(default)]
pub last_push_unix_ms: i64,
#[serde(default)]
pub last_fetch_unix_ms: i64,
#[serde(default)]
pub principals: BTreeSet<String>,
#[serde(default)]
pub principals_capped: bool,
}
impl FlowDay {
pub fn refused(&self) -> u64 {
self.refused_admission
.saturating_add(self.refused_cap)
.saturating_add(self.refused_policy)
}
pub fn refused_for(&self, reason: Refusal) -> u64 {
match reason {
Refusal::Admission => self.refused_admission,
Refusal::Cap => self.refused_cap,
Refusal::Policy => self.refused_policy,
}
}
pub fn distinct_principals(&self) -> u64 {
self.principals.len() as u64
}
pub fn is_empty(&self) -> bool {
*self == FlowDay::default()
}
pub fn last_seen_unix_ms(&self) -> i64 {
self.last_push_unix_ms.max(self.last_fetch_unix_ms)
}
pub fn add(&mut self, other: &FlowDay) {
self.bytes_in = self.bytes_in.saturating_add(other.bytes_in);
self.bytes_out = self.bytes_out.saturating_add(other.bytes_out);
self.pushes = self.pushes.saturating_add(other.pushes);
self.fetches = self.fetches.saturating_add(other.fetches);
self.lfs_bytes_in = self.lfs_bytes_in.saturating_add(other.lfs_bytes_in);
self.lfs_bytes_out = self.lfs_bytes_out.saturating_add(other.lfs_bytes_out);
self.refused_admission = self
.refused_admission
.saturating_add(other.refused_admission);
self.refused_cap = self.refused_cap.saturating_add(other.refused_cap);
self.refused_policy = self.refused_policy.saturating_add(other.refused_policy);
self.auth_failures = self.auth_failures.saturating_add(other.auth_failures);
self.last_push_unix_ms = self.last_push_unix_ms.max(other.last_push_unix_ms);
self.last_fetch_unix_ms = self.last_fetch_unix_ms.max(other.last_fetch_unix_ms);
for who in &other.principals {
if self.principals.len() >= PRINCIPAL_CEILING {
self.principals_capped = true;
break;
}
self.principals.insert(who.clone());
}
self.principals_capped |= other.principals_capped;
}
}
#[derive(Clone, Default, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct NamespaceFlow {
#[serde(default)]
pub days: BTreeMap<i64, FlowDay>,
}
#[derive(Default, Clone, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct FlowDoc {
#[serde(default)]
flows: ShardedMap<NamespaceFlow>,
}
impl_doc!(FlowDoc { flows });
#[derive(Clone, Debug)]
pub struct FlowStore {
doc: DocRef<FlowDoc>,
}
impl FlowStore {
pub fn new(store: Arc<dyn ControlStore>) -> Result<Self> {
Ok(FlowStore {
doc: DocRef::new(store, FLOW_REF)?,
})
}
pub fn ref_name(&self) -> &str {
self.doc.ref_name()
}
pub fn day(&self, namespace: &str, day: i64) -> Result<Option<FlowDay>> {
Ok(self
.doc
.load()?
.flows
.get(namespace)?
.and_then(|flow| flow.days.get(&day).cloned()))
}
pub fn series(
&self,
namespace: &str,
from_day: i64,
to_day: i64,
) -> Result<Vec<(i64, FlowDay)>> {
let Some(flow) = self.doc.load()?.flows.get(namespace)? else {
return Ok(Vec::new());
};
Ok(flow
.days
.range(from_day..=to_day)
.map(|(day, row)| (*day, row.clone()))
.collect())
}
pub fn namespaces(&self) -> Result<Vec<String>> {
let mut names: Vec<String> = self.doc.load()?.flows.keys()?;
names.sort();
Ok(names)
}
pub fn put(&self, rows: &[(String, i64, FlowDay)], today: i64) -> Result<()> {
if rows.is_empty() {
return Ok(());
}
let oldest = today.saturating_sub(RETENTION_DAYS);
let rows = rows.to_vec();
self.doc.update(|doc| {
for (namespace, day, row) in &rows {
let mut flow = doc.flows.get(namespace)?.unwrap_or_default();
if row.is_empty() {
flow.days.remove(day);
} else {
flow.days.insert(*day, row.clone());
}
flow.days.retain(|day, _| *day > oldest);
if flow.days.is_empty() {
doc.flows.remove(namespace)?;
} else {
doc.flows.insert(namespace.clone(), flow)?;
}
}
Ok(())
})
}
pub fn forget(&self, namespace: &str) -> Result<bool> {
let key = namespace.to_owned();
self.doc.update(|doc| Ok(doc.flows.remove(&key)?.is_some()))
}
}
#[derive(Clone, Default, PartialEq, Eq, Debug)]
pub struct Glance {
pub today: FlowDay,
pub last_push_unix_ms: i64,
pub last_fetch_unix_ms: i64,
}
impl Glance {
pub fn last_seen_unix_ms(&self) -> i64 {
self.last_push_unix_ms.max(self.last_fetch_unix_ms)
}
}
#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)]
pub struct Fleet {
pub bytes_in_today: u64,
pub bytes_out_today: u64,
pub refused_today: u64,
pub namespaces: u64,
pub silent: u64,
}
#[derive(Debug)]
struct Slot {
day: AtomicI64,
bytes_in: AtomicU64,
bytes_out: AtomicU64,
pushes: AtomicU64,
fetches: AtomicU64,
lfs_bytes_in: AtomicU64,
lfs_bytes_out: AtomicU64,
refused_admission: AtomicU64,
refused_cap: AtomicU64,
refused_policy: AtomicU64,
auth_failures: AtomicU64,
last_push_unix_ms: AtomicI64,
last_fetch_unix_ms: AtomicI64,
principals: Mutex<(BTreeSet<String>, bool)>,
roll: Mutex<()>,
}
impl Slot {
fn seeded(day: i64, from: FlowDay) -> Slot {
Slot {
day: AtomicI64::new(day),
bytes_in: AtomicU64::new(from.bytes_in),
bytes_out: AtomicU64::new(from.bytes_out),
pushes: AtomicU64::new(from.pushes),
fetches: AtomicU64::new(from.fetches),
lfs_bytes_in: AtomicU64::new(from.lfs_bytes_in),
lfs_bytes_out: AtomicU64::new(from.lfs_bytes_out),
refused_admission: AtomicU64::new(from.refused_admission),
refused_cap: AtomicU64::new(from.refused_cap),
refused_policy: AtomicU64::new(from.refused_policy),
auth_failures: AtomicU64::new(from.auth_failures),
last_push_unix_ms: AtomicI64::new(from.last_push_unix_ms),
last_fetch_unix_ms: AtomicI64::new(from.last_fetch_unix_ms),
principals: Mutex::new((from.principals, from.principals_capped)),
roll: Mutex::new(()),
}
}
fn read(&self) -> FlowDay {
let (principals, principals_capped) = {
let held = self.principals.lock().expect("flow principals");
(held.0.clone(), held.1)
};
FlowDay {
bytes_in: self.bytes_in.load(Ordering::Relaxed),
bytes_out: self.bytes_out.load(Ordering::Relaxed),
pushes: self.pushes.load(Ordering::Relaxed),
fetches: self.fetches.load(Ordering::Relaxed),
lfs_bytes_in: self.lfs_bytes_in.load(Ordering::Relaxed),
lfs_bytes_out: self.lfs_bytes_out.load(Ordering::Relaxed),
refused_admission: self.refused_admission.load(Ordering::Relaxed),
refused_cap: self.refused_cap.load(Ordering::Relaxed),
refused_policy: self.refused_policy.load(Ordering::Relaxed),
auth_failures: self.auth_failures.load(Ordering::Relaxed),
last_push_unix_ms: self.last_push_unix_ms.load(Ordering::Relaxed),
last_fetch_unix_ms: self.last_fetch_unix_ms.load(Ordering::Relaxed),
principals,
principals_capped,
}
}
fn reset_to(&self, day: i64) {
for counter in [
&self.bytes_in,
&self.bytes_out,
&self.pushes,
&self.fetches,
&self.lfs_bytes_in,
&self.lfs_bytes_out,
&self.refused_admission,
&self.refused_cap,
&self.refused_policy,
&self.auth_failures,
] {
counter.store(0, Ordering::Relaxed);
}
self.last_push_unix_ms.store(0, Ordering::Relaxed);
self.last_fetch_unix_ms.store(0, Ordering::Relaxed);
{
let mut held = self.principals.lock().expect("flow principals");
held.0.clear();
held.1 = false;
}
self.day.store(day, Ordering::Release);
}
fn saw(&self, principal: &str) {
if principal.is_empty() {
return;
}
let mut held = self.principals.lock().expect("flow principals");
if held.0.contains(principal) {
return;
}
if held.0.len() >= PRINCIPAL_CEILING {
held.1 = true;
return;
}
held.0.insert(principal.to_owned());
}
}
#[derive(Clone, Default, PartialEq, Eq, Debug)]
pub struct Transfer {
pub bytes_in: u64,
pub bytes_out: u64,
pub pushed: bool,
pub fetched: bool,
pub lfs_bytes_in: u64,
pub lfs_bytes_out: u64,
pub principal: String,
}
impl Transfer {
pub fn is_empty(&self) -> bool {
self.bytes_in == 0
&& self.bytes_out == 0
&& !self.pushed
&& !self.fetched
&& self.lfs_bytes_in == 0
&& self.lfs_bytes_out == 0
}
}
#[derive(Debug)]
pub struct FlowLedger {
store: FlowStore,
live: RwLock<HashMap<String, Arc<Slot>>>,
completed: Mutex<Vec<(String, i64, FlowDay)>>,
observer: RwLock<Option<FlowObserver>>,
}
#[derive(Clone)]
pub struct FlowObserver(pub Arc<dyn Fn(&str) + Send + Sync>);
impl std::fmt::Debug for FlowObserver {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("FlowObserver(..)")
}
}
impl FlowLedger {
pub fn new(store: FlowStore) -> FlowLedger {
FlowLedger {
store,
live: RwLock::new(HashMap::new()),
completed: Mutex::new(Vec::new()),
observer: RwLock::new(None),
}
}
pub fn observe(&self, observer: FlowObserver) {
*self.observer.write().expect("flow observer") = Some(observer);
}
fn tell_observer(&self, namespace: &str) {
if let Some(observer) = self.observer.read().expect("flow observer").as_ref() {
(observer.0)(namespace);
}
}
pub fn store(&self) -> &FlowStore {
&self.store
}
fn slot(&self, namespace: &str, day: i64) -> Arc<Slot> {
if let Some(slot) = self
.live
.read()
.expect("flow slots")
.get(namespace)
.map(Arc::clone)
{
self.roll_to(namespace, &slot, day);
return slot;
}
let seed = self
.store
.day(namespace, day)
.unwrap_or_default()
.unwrap_or_default();
let mut live = self.live.write().expect("flow slots");
let slot = Arc::clone(
live.entry(namespace.to_owned())
.or_insert_with(|| Arc::new(Slot::seeded(day, seed))),
);
drop(live);
self.roll_to(namespace, &slot, day);
slot
}
fn roll_to(&self, namespace: &str, slot: &Slot, day: i64) {
if slot.day.load(Ordering::Acquire) == day {
return;
}
let _held = slot.roll.lock().expect("flow roll");
let was = slot.day.load(Ordering::Acquire);
if was == day {
return;
}
let finished = slot.read();
if !finished.is_empty() {
self.completed.lock().expect("flow completed").push((
namespace.to_owned(),
was,
finished,
));
}
slot.reset_to(day);
}
pub fn record(&self, namespace: &str, at_unix_ms: i64, transfer: &Transfer) {
if namespace.is_empty() || transfer.is_empty() {
return;
}
let slot = self.slot(namespace, day_of(at_unix_ms));
add(&slot.bytes_in, transfer.bytes_in);
add(&slot.bytes_out, transfer.bytes_out);
add(&slot.lfs_bytes_in, transfer.lfs_bytes_in);
add(&slot.lfs_bytes_out, transfer.lfs_bytes_out);
if transfer.pushed {
slot.pushes.fetch_add(1, Ordering::Relaxed);
slot.last_push_unix_ms.store(at_unix_ms, Ordering::Relaxed);
}
if transfer.fetched {
slot.fetches.fetch_add(1, Ordering::Relaxed);
slot.last_fetch_unix_ms.store(at_unix_ms, Ordering::Relaxed);
}
slot.saw(&transfer.principal);
self.tell_observer(namespace);
}
pub fn refuse(&self, namespace: &str, at_unix_ms: i64, reason: Refusal) {
if namespace.is_empty() {
return;
}
let slot = self.slot(namespace, day_of(at_unix_ms));
let counter = match reason {
Refusal::Admission => &slot.refused_admission,
Refusal::Cap => &slot.refused_cap,
Refusal::Policy => &slot.refused_policy,
};
counter.fetch_add(1, Ordering::Relaxed);
self.tell_observer(namespace);
}
pub fn auth_failure(&self, namespace: &str, at_unix_ms: i64) {
if namespace.is_empty() {
return;
}
let slot = self.slot(namespace, day_of(at_unix_ms));
slot.auth_failures.fetch_add(1, Ordering::Relaxed);
self.tell_observer(namespace);
}
pub fn today(&self, namespace: &str, now_unix_ms: i64) -> Option<FlowDay> {
let day = day_of(now_unix_ms);
if let Some(slot) = self
.live
.read()
.expect("flow slots")
.get(namespace)
.map(Arc::clone)
{
if slot.day.load(Ordering::Acquire) == day {
let row = slot.read();
return (!row.is_empty()).then_some(row);
}
}
self.store.day(namespace, day).ok().flatten()
}
pub fn glance(&self, namespace: &str, now_unix_ms: i64) -> Glance {
let today_number = day_of(now_unix_ms);
let today = self.today(namespace, now_unix_ms).unwrap_or_default();
let mut glance = Glance {
last_push_unix_ms: today.last_push_unix_ms,
last_fetch_unix_ms: today.last_fetch_unix_ms,
today,
};
let kept = self
.store
.series(namespace, today_number - RETENTION_DAYS, today_number - 1)
.unwrap_or_default();
for (_, row) in kept.iter().rev() {
if glance.last_push_unix_ms == 0 {
glance.last_push_unix_ms = row.last_push_unix_ms;
}
if glance.last_fetch_unix_ms == 0 {
glance.last_fetch_unix_ms = row.last_fetch_unix_ms;
}
if glance.last_push_unix_ms != 0 && glance.last_fetch_unix_ms != 0 {
break;
}
}
glance
}
pub fn fleet(&self, now_unix_ms: i64, silent_after_days: i64) -> Fleet {
let mut names: BTreeSet<String> = self
.store
.namespaces()
.unwrap_or_default()
.into_iter()
.collect();
names.extend(self.live.read().expect("flow slots").keys().cloned());
let quiet_before = now_unix_ms.saturating_sub(silent_after_days.saturating_mul(DAY_MS));
let mut fleet = Fleet {
namespaces: names.len() as u64,
..Fleet::default()
};
for name in &names {
let glance = self.glance(name, now_unix_ms);
fleet.bytes_in_today = fleet.bytes_in_today.saturating_add(glance.today.bytes_in);
fleet.bytes_out_today = fleet.bytes_out_today.saturating_add(glance.today.bytes_out);
fleet.refused_today = fleet.refused_today.saturating_add(glance.today.refused());
if glance.last_seen_unix_ms() < quiet_before {
fleet.silent = fleet.silent.saturating_add(1);
}
}
fleet
}
pub fn flush(&self, now_unix_ms: i64) -> Result<usize> {
let today = day_of(now_unix_ms);
let mut rows: Vec<(String, i64, FlowDay)> =
std::mem::take(&mut *self.completed.lock().expect("flow completed"));
let slots: Vec<(String, Arc<Slot>)> = self
.live
.read()
.expect("flow slots")
.iter()
.map(|(name, slot)| (name.clone(), Arc::clone(slot)))
.collect();
for (namespace, slot) in slots {
self.roll_to(&namespace, &slot, today);
let _held = slot.roll.lock().expect("flow roll");
let row = slot.read();
if !row.is_empty() {
rows.push((namespace, slot.day.load(Ordering::Acquire), row));
}
}
rows.extend(std::mem::take(
&mut *self.completed.lock().expect("flow completed"),
));
let written = rows.len();
if written > 0 {
if let Err(err) = self.store.put(&rows, today) {
self.completed.lock().expect("flow completed").extend(rows);
return Err(err);
}
}
Ok(written)
}
pub fn moved(&self, namespace: &str, now_unix_ms: i64, days: u32) -> Vec<(i64, u64)> {
let today = day_of(now_unix_ms);
let live_slot = self
.live
.read()
.expect("flow slots")
.get(namespace)
.map(Arc::clone);
if let Some(slot) = live_slot {
self.roll_to(namespace, &slot, today);
}
let from = today - i64::from(days.max(1)) + 1;
let mut rows: BTreeMap<i64, u64> = if from < today {
self.store
.series(namespace, from, today - 1)
.unwrap_or_default()
.into_iter()
.map(|(day, row)| (day, row.bytes_in.saturating_add(row.bytes_out)))
.collect()
} else {
BTreeMap::new()
};
rows.extend(
self.completed
.lock()
.expect("flow completed")
.iter()
.filter(|(name, day, _)| name == namespace && *day >= from && *day < today)
.map(|(_, day, row)| (*day, row.bytes_in.saturating_add(row.bytes_out))),
);
if let Some(t) = self.today(namespace, now_unix_ms) {
rows.insert(today, t.bytes_in.saturating_add(t.bytes_out));
}
rows.into_iter().filter(|(_, b)| *b > 0).collect()
}
pub fn window_bytes(&self, namespace: &str, now_unix_ms: i64, days: u32) -> u64 {
self.moved(namespace, now_unix_ms, days)
.iter()
.fold(0u64, |acc, (_, b)| acc.saturating_add(*b))
}
pub fn forget(&self, namespace: &str) -> Result<bool> {
self.live.write().expect("flow slots").remove(namespace);
self.completed
.lock()
.expect("flow completed")
.retain(|(name, _, _)| name != namespace);
self.store.forget(namespace)
}
}
fn add(counter: &AtomicU64, by: u64) {
if by > 0 {
counter.fetch_add(by, Ordering::Relaxed);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::control::RedbControlStore;
fn ledger() -> FlowLedger {
let store = Arc::new(RedbControlStore::in_memory().unwrap()) as Arc<dyn ControlStore>;
FlowLedger::new(FlowStore::new(store).unwrap())
}
fn push(bytes: u64, who: &str) -> Transfer {
Transfer {
bytes_in: bytes,
pushed: true,
principal: who.into(),
..Transfer::default()
}
}
fn fetch(bytes: u64, who: &str) -> Transfer {
Transfer {
bytes_out: bytes,
fetched: true,
principal: who.into(),
..Transfer::default()
}
}
#[test]
fn the_window_sums_thirty_days_counting_each_day_once() {
let flow = ledger();
let day0 = 20_000 * DAY_MS + 3_600_000;
flow.record("team", day0, &push(1_000_000, "a"));
flow.record("team", day0 + DAY_MS, &push(1000, "a"));
flow.flush(day0 + DAY_MS).unwrap();
flow.record("team", day0 + 29 * DAY_MS, &fetch(200, "a"));
flow.record("team", day0 + 30 * DAY_MS, &push(30, "a"));
flow.record("team", day0 + 30 * DAY_MS, &fetch(7, "b"));
let now = day0 + 30 * DAY_MS;
assert_eq!(flow.window_bytes("team", now, 30), 1000 + 200 + 30 + 7);
let days = flow.moved("team", now, 30);
let today = day_of(now);
assert_eq!(
days,
vec![(today - 29, 1000), (today - 1, 200), (today, 37)],
"oldest first, per day"
);
assert_eq!(
flow.window_bytes("team", now, 1),
37,
"a one-day window is today"
);
assert_eq!(flow.window_bytes("other", now, 30), 0);
flow.flush(now).unwrap();
assert_eq!(flow.window_bytes("team", now, 30), 1237);
flow.record("team", now + 1, &push(3, "a"));
assert_eq!(flow.window_bytes("team", now + 1, 30), 1240);
assert_eq!(flow.window_bytes("team", now + DAY_MS, 30), 200 + 40);
}
#[test]
fn the_window_rolls_a_stale_slot_before_reading_it() {
let flow = ledger();
let day0 = 20_000 * DAY_MS;
flow.flush(day0).unwrap();
flow.record("team", day0 + 10, &push(500, "a"));
flow.flush(day0 + 20).unwrap(); flow.record("team", day0 + 30, &push(5, "a")); let next = day0 + DAY_MS + 1;
assert_eq!(flow.window_bytes("team", next, 30), 505);
assert_eq!(
flow.window_bytes("team", next, 1),
0,
"nothing moved today yet"
);
}
#[test]
fn a_push_lands_on_its_own_namespace_and_no_other() {
let flow = ledger();
let now = 1_757_000_000_000;
flow.record("team", now, &push(4096, "SHA256:aaa"));
let mine = flow.today("team", now).expect("the push was counted");
assert_eq!(mine.bytes_in, 4096);
assert_eq!(mine.pushes, 1);
assert_eq!(mine.bytes_out, 0, "a push serves no pack");
assert_eq!(mine.last_push_unix_ms, now);
assert_eq!(mine.distinct_principals(), 1);
assert_eq!(
flow.today("other", now),
None,
"a namespace with no traffic has no row, not a row of zeroes"
);
}
#[test]
fn a_fetch_is_bytes_out_and_an_advertisement_is_not_a_fetch() {
let flow = ledger();
let now = 1_757_000_000_000;
flow.record("team", now, &fetch(9000, "SHA256:bbb"));
flow.record(
"team",
now,
&Transfer {
bytes_out: 120,
..Transfer::default()
},
);
let row = flow.today("team", now).unwrap();
assert_eq!(row.bytes_out, 9120, "both wrote bytes");
assert_eq!(row.fetches, 1, "only one of them asked for objects");
assert_eq!(row.last_fetch_unix_ms, now);
}
#[test]
fn a_refusal_is_counted_by_reason_and_leaves_the_bytes_alone() {
let flow = ledger();
let now = 1_757_000_000_000;
flow.record("team", now, &push(4096, "SHA256:aaa"));
flow.refuse("team", now, Refusal::Cap);
flow.refuse("team", now, Refusal::Cap);
flow.refuse("team", now, Refusal::Admission);
let row = flow.today("team", now).unwrap();
assert_eq!(row.bytes_in, 4096, "a refused push moved nothing");
assert_eq!(row.pushes, 1);
assert_eq!(row.refused(), 3);
assert_eq!(row.refused_for(Refusal::Cap), 2);
assert_eq!(row.refused_for(Refusal::Admission), 1);
assert_eq!(row.refused_for(Refusal::Policy), 0);
}
#[test]
fn the_day_boundary_splits_the_counts() {
let flow = ledger();
let day_one = day_of(1_757_000_000_000);
let late = midnight_of(day_one) + DAY_MS - 1;
let early = midnight_of(day_one + 1) + 1;
flow.record("team", late, &push(1000, "SHA256:aaa"));
flow.record("team", early, &push(7, "SHA256:aaa"));
let today = flow.today("team", early).unwrap();
assert_eq!(today.bytes_in, 7, "the new day starts from nothing");
assert_eq!(today.pushes, 1);
flow.flush(early).unwrap();
let yesterday = flow
.store()
.day("team", day_one)
.unwrap()
.expect("yesterday was written");
assert_eq!(yesterday.bytes_in, 1000);
assert_eq!(yesterday.pushes, 1);
}
#[test]
fn a_restart_keeps_yesterday_and_resumes_today() {
let store = Arc::new(RedbControlStore::in_memory().unwrap()) as Arc<dyn ControlStore>;
let day_one = day_of(1_757_000_000_000);
let yesterday_at = midnight_of(day_one) + 3_600_000;
let today_at = midnight_of(day_one + 1) + 3_600_000;
let later_today = today_at + 60_000;
{
let flow = FlowLedger::new(FlowStore::new(Arc::clone(&store)).unwrap());
flow.record("team", yesterday_at, &push(1000, "SHA256:aaa"));
flow.flush(yesterday_at).unwrap();
flow.record("team", today_at, &push(500, "SHA256:aaa"));
flow.flush(today_at).unwrap();
}
let flow = FlowLedger::new(FlowStore::new(Arc::clone(&store)).unwrap());
assert_eq!(
flow.today("team", today_at).unwrap().bytes_in,
500,
"today survived the restart"
);
flow.record("team", later_today, &push(1, "SHA256:aaa"));
assert_eq!(
flow.today("team", later_today).unwrap().bytes_in,
501,
"counting resumed from the stored figure, not from zero"
);
assert_eq!(flow.today("team", later_today).unwrap().pushes, 2);
assert_eq!(
flow.store().day("team", day_one).unwrap().unwrap().bytes_in,
1000,
"yesterday is untouched"
);
}
#[test]
fn a_second_flush_is_not_a_second_count() {
let flow = ledger();
let now = 1_757_000_000_000;
flow.record("team", now, &push(4096, "SHA256:aaa"));
flow.flush(now).unwrap();
flow.flush(now).unwrap();
flow.flush(now).unwrap();
assert_eq!(
flow.store()
.day("team", day_of(now))
.unwrap()
.unwrap()
.bytes_in,
4096
);
}
#[test]
fn the_series_is_oldest_first_and_silent_days_are_absent() {
let flow = ledger();
let first = day_of(1_757_000_000_000);
for (offset, bytes) in [(0i64, 100u64), (1, 200), (3, 400)] {
let at = midnight_of(first + offset) + 1000;
flow.record("team", at, &push(bytes, "SHA256:aaa"));
flow.flush(at).unwrap();
}
let series = flow.store().series("team", first, first + 3).unwrap();
assert_eq!(
series.iter().map(|(day, _)| *day).collect::<Vec<_>>(),
vec![first, first + 1, first + 3],
"oldest first, and the silent day is not a row"
);
assert_eq!(
series.iter().map(|(_, r)| r.bytes_in).collect::<Vec<_>>(),
vec![100, 200, 400]
);
let narrow = flow.store().series("team", first + 1, first + 3).unwrap();
assert_eq!(narrow.len(), 2);
}
#[test]
fn nothing_older_than_the_window_survives_a_write() {
let flow = ledger();
let ancient = day_of(1_757_000_000_000);
let at = midnight_of(ancient) + 1000;
flow.record("team", at, &push(100, "SHA256:aaa"));
flow.flush(at).unwrap();
assert!(flow.store().day("team", ancient).unwrap().is_some());
let much_later = midnight_of(ancient + RETENTION_DAYS + 1) + 1000;
flow.record("team", much_later, &push(1, "SHA256:aaa"));
flow.flush(much_later).unwrap();
assert_eq!(flow.store().day("team", ancient).unwrap(), None, "aged out");
assert!(flow
.store()
.day("team", day_of(much_later))
.unwrap()
.is_some());
}
#[test]
fn principals_are_exact_and_the_ceiling_announces_itself() {
let flow = ledger();
let now = 1_757_000_000_000;
for _ in 0..5 {
flow.record("team", now, &push(1, "SHA256:aaa"));
flow.record("team", now, &fetch(1, "SHA256:bbb"));
}
let row = flow.today("team", now).unwrap();
assert_eq!(row.distinct_principals(), 2, "ten transfers, two people");
assert!(!row.principals_capped);
let mut wide = FlowDay::default();
for n in 0..PRINCIPAL_CEILING + 10 {
wide.principals.insert(format!("SHA256:{n}"));
}
let mut folded = FlowDay::default();
folded.add(&wide);
assert!(
folded.principals_capped,
"a count that stopped growing must say so"
);
assert_eq!(folded.distinct_principals(), PRINCIPAL_CEILING as u64);
}
#[test]
fn forgetting_a_namespace_forgets_its_flow() {
let flow = ledger();
let now = 1_757_000_000_000;
flow.record("team", now, &push(4096, "SHA256:aaa"));
flow.record("other", now, &push(1, "SHA256:aaa"));
flow.flush(now).unwrap();
assert!(flow.forget("team").unwrap());
assert_eq!(flow.today("team", now), None);
assert!(
flow.today("other", now).is_some(),
"the other tenant is untouched"
);
}
#[test]
fn nothing_is_never_written_as_something() {
let flow = ledger();
let now = 1_757_000_000_000;
flow.record("team", now, &Transfer::default());
flow.record("", now, &push(4096, "SHA256:aaa"));
assert_eq!(flow.today("team", now), None);
assert_eq!(flow.today("", now), None);
assert_eq!(flow.flush(now).unwrap(), 0, "nothing to write");
}
#[test]
fn the_date_renderer_is_utc_and_floors_before_the_epoch() {
assert_eq!(iso8601_utc(0), "1970-01-01T00:00:00Z");
assert_eq!(iso8601_utc(1_757_000_000_000), "2025-09-04T15:33:20Z");
assert_eq!(iso8601_utc(1_789_041_600_000), "2026-09-10T12:00:00Z");
assert_eq!(
iso8601_utc(-1),
"1969-12-31T23:59:59Z",
"one millisecond before the epoch"
);
assert_eq!(iso8601_utc(1_709_164_800_000), "2024-02-29T00:00:00Z");
}
#[test]
fn the_day_arithmetic_floors() {
assert_eq!(day_of(0), 0);
assert_eq!(day_of(DAY_MS - 1), 0);
assert_eq!(day_of(DAY_MS), 1);
assert_eq!(day_of(-1), -1, "the last millisecond of 1969 is not day 0");
assert_eq!(midnight_of(day_of(1_757_000_000_000)), 1_756_944_000_000);
}
}