use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use super::control::{self, write_atomic};
use super::state::ConflictRecord;
use super::{now_unix, LeanError, LeanResult, Syncer};
const MAX_BODY: u64 = 64 * 1024;
const MAX_NONCES: usize = 32;
pub const MAX_SCOPE_ENTRIES: usize = 64;
pub const MAX_SCOPE_ENTRY_LEN: usize = 1024;
const BUDGET: &str = "sentinel-budget.json";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Verb {
Publish,
Sync,
}
impl Verb {
pub fn as_str(&self) -> &'static str {
match self {
Verb::Publish => "publish",
Verb::Sync => "sync",
}
}
pub fn sentinel_name(&self) -> &'static str {
match self {
Verb::Publish => control::PUBLISH,
Verb::Sync => control::SYNC,
}
}
fn ack_name(&self) -> &'static str {
match self {
Verb::Publish => control::PUBLISH_ACK,
Verb::Sync => control::SYNC_ACK,
}
}
pub fn pending_name(&self) -> &'static str {
match self {
Verb::Publish => "publish.pending.json",
Verb::Sync => "sync.pending.json",
}
}
fn staging_name(&self) -> &'static str {
match self {
Verb::Publish => "publish.consumed",
Verb::Sync => "sync.consumed",
}
}
}
#[derive(Debug, Clone, Default, Deserialize)]
struct SentinelBody {
#[serde(default)]
nonce: Option<String>,
#[serde(default)]
note: Option<String>,
#[serde(default)]
scope: Option<Vec<String>>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PendingSentinel {
pub verb: String,
pub consumed_mtime_unix_ns: u128,
pub consumed_at: u64,
pub nonces: Vec<String>,
#[serde(default)]
pub note: Option<String>,
#[serde(default)]
pub scope: Option<Vec<String>>,
#[serde(default)]
pub torn: bool,
#[serde(default)]
pub whole_tree: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Ack {
pub status: String,
pub nonces: Vec<String>,
pub sentinel_mtime_unix_ns: u128,
#[serde(skip_serializing_if = "Option::is_none")]
pub seq: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub manifest_etag: Option<String>,
pub boundary: String,
pub completed_unix: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reason: Option<String>,
pub report: AckReport,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct AckReport {
pub uploaded: usize,
pub deleted: usize,
pub parked: usize,
pub consumed: usize,
pub no_change: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub applied: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub conflicts: Vec<ConflictRecord>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub scope: Option<Vec<String>>,
#[serde(default)]
pub out_of_scope_foreign: usize,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dropped: Vec<String>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct SentinelBudget {
pub charges: Vec<(u64, u64)>,
pub last_honor_unix: u64,
}
impl SentinelBudget {
fn prune(&mut self, now: u64) {
self.charges.retain(|(at, _)| now.saturating_sub(*at) < 3600);
}
pub fn spent(&self, now: u64) -> u64 {
self.charges
.iter()
.filter(|(at, _)| now.saturating_sub(*at) < 3600)
.map(|(_, u)| *u)
.sum()
}
pub fn remaining(&self, now: u64, budget: u64) -> u64 {
budget.saturating_sub(self.spent(now))
}
}
pub(crate) fn conflicts_since(
before: usize,
after: Vec<ConflictRecord>,
) -> Vec<ConflictRecord> {
if after.len() >= before {
after[before..].to_vec()
} else {
after
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Due {
Ready,
MinInterval,
BudgetDeferred,
}
fn mtime_ns(m: &std::fs::Metadata) -> u128 {
m.modified()
.ok()
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|d| d.as_nanos())
.unwrap_or(0)
}
pub(crate) fn read_bounded(path: &Path) -> std::io::Result<(Vec<u8>, bool)> {
use std::io::Read;
use std::os::unix::fs::OpenOptionsExt;
let f = std::fs::File::options()
.read(true)
.custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW)
.open(path)?;
let meta = f.metadata()?;
if !meta.is_file() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"sentinel is not a regular file",
));
}
let mut buf = Vec::new();
let mut take = f.take(MAX_BODY + 1);
take.read_to_end(&mut buf)?;
let oversize = buf.len() as u64 > MAX_BODY;
buf.truncate(MAX_BODY as usize);
Ok((buf, oversize))
}
impl Syncer {
fn pending_path(&self, verb: Verb) -> PathBuf {
self.cfg.state_dir().join(verb.pending_name())
}
fn staging_path(&self, verb: Verb) -> PathBuf {
self.cfg.state_dir().join(verb.staging_name())
}
fn budget_path(&self) -> PathBuf {
self.cfg.state_dir().join(BUDGET)
}
pub fn load_pending(&self, verb: Verb) -> LeanResult<Option<PendingSentinel>> {
let p = self.pending_path(verb);
if !p.exists() {
return Ok(None);
}
let bytes = std::fs::read(&p)?;
match serde_json::from_slice(&bytes) {
Ok(v) => Ok(Some(v)),
Err(_) => Ok(Some(PendingSentinel {
verb: verb.as_str().into(),
consumed_mtime_unix_ns: 0,
consumed_at: now_unix(),
nonces: vec![],
note: None,
scope: None,
torn: true,
whole_tree: false,
})),
}
}
fn save_pending(&self, verb: Verb, p: &PendingSentinel) -> LeanResult<()> {
let bytes = serde_json::to_vec_pretty(p)
.map_err(|e| LeanError::State(format!("pending: {e}")))?;
write_atomic(&self.pending_path(verb), &bytes)
}
fn retire_pending(&self, verb: Verb) -> LeanResult<()> {
let p = self.pending_path(verb);
if p.exists() {
std::fs::remove_file(&p)?;
}
Ok(())
}
pub fn load_budget(&self) -> LeanResult<SentinelBudget> {
let p = self.budget_path();
if !p.exists() {
return Ok(SentinelBudget::default());
}
let bytes = std::fs::read(&p)?;
Ok(serde_json::from_slice(&bytes).unwrap_or_default())
}
fn save_budget(&self, b: &SentinelBudget) -> LeanResult<()> {
let bytes =
serde_json::to_vec_pretty(b).map_err(|e| LeanError::State(format!("budget: {e}")))?;
write_atomic(&self.budget_path(), &bytes)
}
pub fn charge_budget(&self, published_bytes: u64) -> LeanResult<u64> {
let now = now_unix();
let mut b = self.load_budget()?;
b.prune(now);
let units = if published_bytes == 0 {
0
} else {
published_bytes.div_ceil(self.cfg.whole_put_max.max(1)).max(1)
};
if units > 0 {
b.charges.push((now, units));
}
b.last_honor_unix = now;
self.save_budget(&b)?;
Ok(units)
}
pub fn sentinel_due(&self) -> LeanResult<Due> {
let now = now_unix();
let b = self.load_budget()?;
if b.last_honor_unix > 0
&& now.saturating_sub(b.last_honor_unix) < self.cfg.sentinel_min_interval_secs
{
return Ok(Due::MinInterval);
}
if b.remaining(now, self.cfg.sentinel_hourly_budget) == 0 {
return Ok(Due::BudgetDeferred);
}
Ok(Due::Ready)
}
pub fn poll_sentinels(&mut self) -> LeanResult<Vec<Verb>> {
let mut consumed = vec![];
for verb in [Verb::Publish, Verb::Sync] {
if self.consume_sentinel(verb)? {
consumed.push(verb);
}
}
Ok(consumed)
}
pub fn recover_consume_staging(&mut self) -> LeanResult<()> {
for verb in [Verb::Publish, Verb::Sync] {
let staging = self.staging_path(verb);
if staging.exists() {
let meta = std::fs::metadata(&staging)?;
let (body, oversize) = read_bounded(&staging)?;
self.fold_into_pending(verb, mtime_ns(&meta), &body, oversize)?;
std::fs::remove_file(&staging)?;
}
}
Ok(())
}
pub(crate) fn consume_sentinel(&mut self, verb: Verb) -> LeanResult<bool> {
let path = self.control_path(verb.sentinel_name());
let meta = match std::fs::symlink_metadata(&path) {
Ok(m) => m,
Err(_) => return Ok(false),
};
if !meta.is_file() {
let slot = format!("{}/{}", super::CONTROL_DIR, verb.sentinel_name());
if self.noted_not_regular.insert(slot.clone()) {
self.state.append_conflict(&ConflictRecord {
path: slot,
foreign_etag: String::new(),
preserved_key: None,
kind: "sentinel-not-regular-file".into(),
at_unix: now_unix(),
})?;
}
return Ok(false);
}
let ns = mtime_ns(&meta);
let (body, oversize) = match read_bounded(&path) {
Ok(v) => v,
Err(_) => (Vec::new(), false),
};
self.noted_not_regular.remove(&format!("{}/{}", super::CONTROL_DIR, verb.sentinel_name()));
let staging = self.staging_path(verb);
std::fs::rename(&path, &staging)?;
self.fold_into_pending(verb, ns, &body, oversize)?;
let _ = std::fs::remove_file(&staging);
Ok(true)
}
fn fold_into_pending(
&mut self,
verb: Verb,
ns: u128,
body: &[u8],
oversize: bool,
) -> LeanResult<()> {
let trimmed = body.iter().all(|b| b.is_ascii_whitespace());
let (parsed, torn) = if trimmed {
(SentinelBody::default(), false) } else if oversize {
(SentinelBody::default(), true)
} else {
match serde_json::from_slice::<SentinelBody>(body) {
Ok(v) => (v, false),
Err(_) => (SentinelBody::default(), true),
}
};
if torn {
self.state.append_conflict(&ConflictRecord {
path: format!("{}/{}", super::CONTROL_DIR, verb.sentinel_name()),
foreign_etag: String::new(),
preserved_key: None,
kind: "sentinel-torn-body".into(),
at_unix: now_unix(),
})?;
}
let mut pending = self.load_pending(verb)?.unwrap_or(PendingSentinel {
verb: verb.as_str().into(),
consumed_mtime_unix_ns: 0,
consumed_at: now_unix(),
nonces: vec![],
note: None,
scope: None,
torn: false,
whole_tree: false,
});
pending.consumed_mtime_unix_ns = pending.consumed_mtime_unix_ns.max(ns);
pending.consumed_at = now_unix();
pending.torn |= torn;
if let Some(n) = parsed.nonce {
let n: String = n.chars().take(128).collect();
if !pending.nonces.contains(&n) {
pending.nonces.push(n);
}
let excess = pending.nonces.len().saturating_sub(MAX_NONCES);
if excess > 0 {
pending.nonces.drain(..excess);
}
}
if let Some(note) = parsed.note {
pending.note = Some(note.chars().take(4096).collect());
}
match parsed.scope {
None if matches!(verb, Verb::Sync) => {
pending.scope = None;
pending.whole_tree = true;
}
Some(scope) if !pending.whole_tree => {
let mut merged = pending.scope.take().unwrap_or_default();
for e in scope {
if e.len() > MAX_SCOPE_ENTRY_LEN {
continue;
}
if !merged.contains(&e) {
merged.push(e);
}
}
pending.scope = Some(merged);
}
_ => {}
}
self.save_pending(verb, &pending)?;
Ok(())
}
pub fn request_boundary(&mut self, nonce: &str, note: Option<String>) -> LeanResult<()> {
let body = serde_json::json!({ "nonce": nonce, "note": note });
let ns = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
self.fold_into_pending(Verb::Publish, ns, body.to_string().as_bytes(), false)
}
fn ack_path(&self, verb: Verb) -> PathBuf {
self.control_path(verb.ack_name())
}
pub fn read_ack(&self, verb: Verb) -> Option<Ack> {
let bytes = std::fs::read(self.ack_path(verb)).ok()?;
serde_json::from_slice(&bytes).ok()
}
fn write_ack(&self, verb: Verb, ack: &Ack) -> LeanResult<()> {
let bytes =
serde_json::to_vec_pretty(ack).map_err(|e| LeanError::State(format!("ack: {e}")))?;
write_atomic(&self.ack_path(verb), &bytes)?;
self.trace("ack", serde_json::json!({"verb": format!("{verb:?}"), "nonces": ack.nonces, "status": ack.status,
"seq": ack.seq, "dropped": ack.report.dropped, "boundary": ack.boundary}));
Ok(())
}
pub(crate) fn ack_matches(&self, verb: Verb, pending: &PendingSentinel) -> bool {
let Some(ack) = self.read_ack(verb) else { return false };
if ack.completed_unix < pending.consumed_at {
return false;
}
if ack.sentinel_mtime_unix_ns < pending.consumed_mtime_unix_ns {
return false;
}
if pending.nonces.is_empty() {
return ack.sentinel_mtime_unix_ns == pending.consumed_mtime_unix_ns;
}
pending.nonces.iter().all(|n| ack.nonces.contains(n))
}
pub async fn honor_pending(&mut self, verb: Verb, forced: bool) -> LeanResult<Option<Ack>> {
self.honor_pending_as(verb, forced, None).await
}
pub async fn honor_pending_as(
&mut self,
verb: Verb,
forced: bool,
source: Option<&str>,
) -> LeanResult<Option<Ack>> {
let Some(pending) = self.load_pending(verb)? else { return Ok(None) };
if !forced {
match self.sentinel_due()? {
Due::Ready => {}
Due::MinInterval | Due::BudgetDeferred => return Ok(None),
}
}
if self.ack_matches(verb, &pending) {
self.retire_pending(verb)?;
return Ok(None);
}
let ack = match verb {
Verb::Publish => self.honor_publish(&pending, forced, source).await,
Verb::Sync => self.honor_sync(&pending, forced).await,
};
match ack {
Ok(ack) => {
self.write_ack(verb, &ack)?;
self.retire_pending(verb)?;
Ok(Some(ack))
}
Err(e) => Err(e),
}
}
async fn honor_publish(
&mut self,
pending: &PendingSentinel,
forced: bool,
source: Option<&str>,
) -> LeanResult<Ack> {
if self.cfg.access.is_read() {
return Ok(Ack {
status: "refused-read-only".into(),
nonces: pending.nonces.clone(),
sentinel_mtime_unix_ns: pending.consumed_mtime_unix_ns,
seq: None,
manifest_etag: None,
boundary: if forced { "sentinel-deferred".into() } else { "sentinel".into() },
completed_unix: now_unix(),
reason: Some(
"this pod has read-only access to the workspace (capabilities `access: read`): \
nothing in its tree is published. `sync` still integrates the bucket's latest \
boundary"
.into(),
),
report: AckReport::default(),
});
}
let before = self.state.load_conflicts()?.len();
let report = self
.declared_barrier_as(
source.unwrap_or(if forced { "sentinel-deferred" } else { "sentinel" }),
)
.await?;
let conflicts = conflicts_since(before, self.state.load_conflicts()?);
let units = self.charge_budget(report.published_bytes)?;
let _ = units;
let baseline = self.state.load_baseline()?;
let dropped: Vec<String> =
report.parked.iter().chain(&report.outranked).chain(&report.deferred).cloned().collect();
Ok(Ack {
status: if dropped.is_empty() { "ok".into() } else { "partial".into() },
nonces: pending.nonces.clone(),
sentinel_mtime_unix_ns: pending.consumed_mtime_unix_ns,
seq: report.seq,
manifest_etag: baseline.manifest_etag.clone(),
boundary: if forced { "sentinel-deferred".into() } else { "sentinel".into() },
completed_unix: now_unix(),
reason: None,
report: AckReport {
uploaded: report.uploaded.len(),
deleted: report.deleted.len(),
parked: report.parked.len(),
consumed: report.consumed,
no_change: report.no_change,
conflicts,
dropped,
..Default::default()
},
})
}
async fn honor_sync(&mut self, pending: &PendingSentinel, forced: bool) -> LeanResult<Ack> {
if let Some(reason) = Self::scope_refusal(&pending.scope) {
return Ok(Ack {
status: "refused-scope".into(),
nonces: pending.nonces.clone(),
sentinel_mtime_unix_ns: pending.consumed_mtime_unix_ns,
seq: None,
manifest_etag: None,
boundary: if forced { "sentinel-deferred".into() } else { "sentinel".into() },
completed_unix: now_unix(),
reason: Some(reason),
report: AckReport { scope: pending.scope.clone(), ..Default::default() },
});
}
let before = self.state.load_conflicts()?.len();
let report = self.sync_scoped(pending.scope.clone()).await?;
let conflicts = conflicts_since(before, self.state.load_conflicts()?);
self.charge_budget(0)?;
Ok(Ack {
status: "ok".into(),
nonces: pending.nonces.clone(),
sentinel_mtime_unix_ns: pending.consumed_mtime_unix_ns,
seq: Some(report.seq),
manifest_etag: None,
boundary: if forced { "sentinel-deferred".into() } else { "sentinel".into() },
completed_unix: now_unix(),
reason: None,
report: AckReport {
consumed: report.applied.len(),
deleted: report.deleted.len(),
applied: report.applied.clone(),
conflicts,
scope: pending.scope.clone(),
out_of_scope_foreign: report.out_of_scope_foreign,
..Default::default()
},
})
}
fn scope_refusal(scope: &Option<Vec<String>>) -> Option<String> {
let raw = scope.as_ref()?;
if raw.len() > MAX_SCOPE_ENTRIES {
return Some(format!(
"scope names {} entries; at most {MAX_SCOPE_ENTRIES} are allowed per sync",
raw.len()
));
}
if super::sync::Scope::new(raw).is_empty() {
return Some(format!(
"scope names {} entr{} and none is valid: each must be a relative path inside the \
workspace with no `.` or `..` component (an empty scope is refused, not widened \
to the whole tree)",
raw.len(),
if raw.len() == 1 { "y" } else { "ies" }
));
}
None
}
pub async fn settle_pending_at_startup(&mut self) -> LeanResult<()> {
self.recover_consume_staging()?;
for verb in [Verb::Publish, Verb::Sync] {
let Some(pending) = self.load_pending(verb)? else { continue };
if self.ack_matches(verb, &pending) {
self.retire_pending(verb)?;
continue;
}
self.honor_pending(verb, true).await?;
}
Ok(())
}
}
#[derive(Debug, Default)]
pub struct FloorOutcome {
pub seq: Option<u64>,
pub no_change: bool,
pub acks: Vec<Ack>,
pub uploaded: usize,
pub deleted: usize,
pub consumed: usize,
pub withheld_reason: Option<String>,
pub parked: usize,
observed_etag: Option<String>,
}
impl Syncer {
fn ticker_from(&self, seq: Option<u64>, etag: Option<String>) -> LeanResult<()> {
let baseline_seq = self.state.load_baseline()?.seq;
let waiting = !self.state.load_foreign_queue()?.is_empty()
|| !self.state.load_intent()?.installed_foreign.is_empty();
let integrated = if waiting {
baseline_seq.min(self.load_remote_seq().integrated_seq)
} else {
baseline_seq
};
self.touch_remote_seq(seq, etag, integrated)
}
pub async fn sentinel_tick(&mut self) -> LeanResult<Vec<Ack>> {
let posture = self.load_posture()?;
if posture.as_ref().map(|p| !p.enabled).unwrap_or(false) {
return Ok(vec![]);
}
self.poll_sentinels()?;
let mut acks = vec![];
for verb in [Verb::Sync, Verb::Publish] {
match self.honor_pending(verb, false).await {
Ok(Some(a)) => acks.push(a),
Ok(None) => {}
Err(e) => {
eprintln!("flint-sync: {verb:?} honor failed (pending kept, retrying): {e}");
}
}
}
if !acks.is_empty() {
let last = acks.last().and_then(|a| a.seq);
self.ticker_from(last, None)?;
}
Ok(acks)
}
pub async fn floor_tick(&mut self) -> LeanResult<FloorOutcome> {
let mut out = FloorOutcome::default();
let mut published = false;
for verb in [Verb::Sync, Verb::Publish] {
if self.load_pending(verb)?.is_some() {
match self.honor_pending(verb, true).await {
Ok(Some(a)) => {
out.seq = a.seq.or(out.seq);
if verb == Verb::Publish && a.status == "ok" {
published = true;
out.uploaded = a.report.uploaded;
out.deleted = a.report.deleted;
out.consumed = a.report.consumed;
out.no_change = a.report.no_change;
}
out.acks.push(a);
}
Ok(None) => {}
Err(e) => {
eprintln!("flint-sync: {verb:?} honor failed on the floor (pending kept): {e}");
}
}
}
}
if self.cfg.access.is_read() {
return match self.reader_pull().await {
Ok(p) => {
let _ = self.clear_auth_pause();
match &p.sync {
Some(r) => {
out.seq = Some(r.seq);
out.consumed = r.applied.len();
out.deleted = r.deleted.len();
}
None => {
out.no_change = true;
out.seq = Some(self.state.load_baseline()?.seq);
}
}
self.ticker_from(p.observed_seq, p.observed_etag)?;
out.withheld_reason = self.write_gauges(None)?.withheld_reason;
Ok(out)
}
Err(e) => {
if e.is_auth() {
let _ = self.note_auth_pause();
}
Err(e)
}
};
}
if !published {
let ran = self.cadence_barrier().await.map(|r| {
out.seq = r.seq;
out.no_change = r.no_change;
out.uploaded = r.uploaded.len();
out.deleted = r.deleted.len();
out.consumed = r.consumed;
out.parked = r.parked.len();
out.observed_etag = r.observed_etag.clone();
r.observed_seq
});
match ran {
Ok(observed) => {
let _ = self.clear_auth_pause();
let etag = out.observed_etag.take();
self.ticker_from(observed, etag)?;
let forced = (out.parked > 0).then_some(super::gauges::Withheld::Parked412);
out.withheld_reason = self.write_gauges(forced)?.withheld_reason;
return Ok(out);
}
Err(e) => {
if e.is_auth() {
let _ = self.note_auth_pause();
}
return Err(e);
}
}
}
self.ticker_from(out.seq, None)?;
out.withheld_reason = self.write_gauges(None)?.withheld_reason;
Ok(out)
}
pub async fn drain(&mut self) -> LeanResult<Vec<Ack>> {
let mut acks = vec![];
let mut published = false;
for verb in [Verb::Sync, Verb::Publish] {
let is_publish = matches!(verb, Verb::Publish);
if self.load_pending(verb)?.is_some() {
match self.honor_pending_as(verb, true, Some("drain")).await {
Ok(Some(mut a)) => {
a.boundary = "drain".into();
let bytes = serde_json::to_vec_pretty(&a)
.map_err(|e| LeanError::State(format!("ack: {e}")))?;
write_atomic(&self.ack_path(verb), &bytes)?;
published |= is_publish && a.status == "ok";
acks.push(a);
}
Ok(None) => {}
Err(e) => return Err(e),
}
}
}
if !published && !self.cfg.access.is_read() {
let r = self.declared_barrier_as("drain").await?;
self.ticker_from(r.observed_seq, r.observed_etag.clone())?;
if !r.parked.is_empty() {
return Err(LeanError::State(format!(
"drain: {} path(s) could not be published (parked on a foreign version whose \
preserve failed): {:?} — not attesting",
r.parked.len(),
r.parked
)));
}
if !r.deferred.is_empty() || !r.outranked.is_empty() {
return Err(LeanError::State(format!(
"drain: not in this boundary — uploads that published nothing {:?}, deletes \
another writer's change outranked {:?} — not attesting",
r.deferred, r.outranked
)));
}
}
Ok(acks)
}
}