pub struct DecdnMetrics {Show 224 fields
pub probe_requests: Counter,
pub active_connections: Gauge,
pub lanes_open: Gauge,
pub pool_deposit_usdc: Gauge,
pub buyer_wallet_usdc: Gauge,
pub buyer_lane_seed_failures: Counter,
pub buyer_pool_adoption_failures: Counter,
pub unredeemed_usdc: Gauge,
pub client_idle_close: Counter,
pub otlp_export_failures: Counter,
pub node_uptime_seconds: Gauge,
pub rpc_healthy: Gauge,
pub dispatch_rejected_global: Counter,
pub dispatch_rejected_per_source: Counter,
pub dispatch_in_flight: Gauge,
pub dispatch_per_source_skipped_no_addr: Counter,
pub inbound_connections_direct: Counter,
pub inbound_connections_relayed: Counter,
pub node_public_address: Gauge,
pub probe_hold_slots_used: Gauge,
pub probe_hold_slots_max: Gauge,
pub dht_rate_limit_rejected_per_peer: Counter,
pub dht_rate_limit_rejected_per_ip: Counter,
pub dht_rate_limit_rejected_global: Counter,
pub dht_rate_limit_prune_sweeps_per_ip: Counter,
pub dht_rate_limit_prune_sweeps_per_peer: Counter,
pub dht_rate_limit_tracked_per_ip: Gauge,
pub dht_rate_limit_tracked_per_peer: Gauge,
pub probe_rate_limit_rejected_per_peer: Counter,
pub probe_rate_limit_rejected_per_ip: Counter,
pub probe_rate_limit_rejected_global: Counter,
pub probe_rate_limit_prune_sweeps_per_ip: Counter,
pub probe_rate_limit_prune_sweeps_per_peer: Counter,
pub probe_rate_limit_tracked_per_ip: Gauge,
pub probe_rate_limit_tracked_per_peer: Gauge,
pub dht_requests_failed: Counter,
pub dht_store_rejected_holder_mismatch: Counter,
pub dht_store_rejected_non_staked: Counter,
pub dht_store_rejected_quota: Counter,
pub dht_store_accepted: Counter,
pub dht_batch_store_received: Counter,
pub dht_batch_store_hashes_deferred_rate_limit: Counter,
pub dht_republish_lag_sweeps: Counter,
pub dht_republish_lag_sweeps_coalesced: Counter,
pub dht_republish_lag_sweep_panics: Counter,
pub dht_republish_sweep_reseeded: Counter,
pub dht_republish_seed_store_walk_failures: Counter,
pub dht_republish_seed_origin_probe_failures: Counter,
pub voucher_nonce_gaps: Counter,
pub redeem_hints_dropped: Counter,
pub redeem_hints_parked: Counter,
pub receipt_writes_dropped: Counter,
pub warming_credits_dropped: Counter,
pub warming_credits_applied: Counter,
pub redemption_failures: Counter,
pub redemption_skipped_insolvent: Counter,
pub redemption_skipped_expired: Counter,
pub redemption_reconciled_skip: Counter,
pub redemption_reconcile_ok: Counter,
pub redemption_reconcile_failures: Counter,
pub buyer_reclaim_failures: Counter,
pub buyer_pool_store_skipped_undecodable_records: Counter,
pub buyer_topup_ok: Counter,
pub buyer_topup_failure: Counter,
pub watcher_persist_failures: Counter,
pub lane_flush_failures: Counter,
pub slashes_detected: Counter,
pub slash_resync_failures: Counter,
pub slash_watcher_restarts: Counter,
pub slash_watcher_down_seconds: Gauge,
pub staker_set_watcher_restarts: Counter,
pub staker_set_watcher_resolve_failures: Counter,
pub capacity_bond_registry_resync_failures: Counter,
pub staker_set_watcher_down_seconds: Gauge,
pub staker_set_active_count: Gauge,
pub staker_set_active_unknown_region: Gauge,
pub node_address_directory_size: Gauge,
pub node_pull_attempts: Counter,
pub node_pull_first_byte_seconds: Histogram,
pub node_pull_success: Counter,
pub node_pull_no_providers: Counter,
pub probe_cache_hits: Counter,
pub probe_cache_misses: Counter,
pub probe_collection_latency_seconds: Histogram,
pub dht_lookup_round_ceiling: Counter,
pub node_pull_corruption: Counter,
pub node_pull_unreachable: Counter,
pub node_region_latency_penalty: Counter,
pub node_pull_pool_open_failures: Counter,
pub pool_open_failures_insufficient_deposit: Counter,
pub pool_open_failures_contract_revert: Counter,
pub pool_open_failures_rpc_error: Counter,
pub node_pull_too_large: Counter,
pub node_pull_rate_above_ceiling: Counter,
pub serve_economics_refused: Counter,
pub serve_economics_refused_warming: Counter,
pub serve_economics_refused_amortized: Counter,
pub node_pull_timeout: Counter,
pub node_pull_voucher_rejected: Counter,
pub node_pull_pool_wedged: Counter,
pub node_pull_refused: Counter,
pub node_pull_refused_unattributable: Counter,
pub node_upstream_rate_limited: Counter,
pub node_pull_backpressure_backoffs: Counter,
pub node_pull_backpressure_exhausted: Counter,
pub node_pull_leg_no_progress: Counter,
pub node_pull_stalled: Counter,
pub node_pull_local_fault: Counter,
pub node_pull_pool_open_pending: Counter,
pub node_pull_progress_persist_failures: Counter,
pub node_pull_progress_dropped: Counter,
pub node_pull_progress_superseded: Counter,
pub node_pull_recovery_step: Counter,
pub node_pull_recovery_step_refused: Counter,
pub node_pull_through_timeouts: Counter,
pub node_pull_through_errors: Counter,
pub node_pull_through_window_paused: Counter,
pub node_pull_through_min_draw_waits: Counter,
pub node_pull_through_wait_seconds: Histogram,
pub node_pull_through_client_abandoned: Counter,
pub node_pull_through_upstream_verify_failed: Counter,
pub local_outboard_serves: Counter,
pub serve_cache_hit: Counter,
pub serve_cache_partial_hit: Counter,
pub serve_cache_miss: Counter,
pub serve_first_byte_hit_seconds: Histogram,
pub serve_first_byte_miss_seconds: Histogram,
pub serve_admission_seconds: Histogram,
pub serve_signer_auth_cached: Counter,
pub serve_signer_auth_first_read: Counter,
pub serve_signer_auth_reread: Counter,
pub serve_response_hit_seconds: Histogram,
pub serve_response_miss_seconds: Histogram,
pub serve_first_frame_hit_seconds: Histogram,
pub serve_first_frame_miss_seconds: Histogram,
pub origin_directory_get_origins_failures: Counter,
pub origin_directory_cache_size: Gauge,
pub serve_stream_rejected_evicted_since_probe: Counter,
pub serve_stream_rejected_cache_miss: Counter,
pub serve_stream_rejected_internal_error: Counter,
pub serve_stream_rejected_unknown_lane: Counter,
pub serve_stream_rejected_owner_mismatch: Counter,
pub serve_stream_rejected_bad_binding: Counter,
pub probe_read_faults: Counter,
pub serve_stream_rejected_insufficient_deposit: Counter,
pub serve_stream_rejected_pool_unconfirmed: Counter,
pub serve_stream_rejected_pool_closing: Counter,
pub serve_stream_rejected_signer_cap_exhausted: Counter,
pub serve_stream_midstream_signer_cap_exhausted: Counter,
pub serve_stream_rejected_signer_floor_at_cap: Counter,
pub serve_stream_rejected_pull_loop_guard: Counter,
pub serve_stream_rejected_range_not_satisfiable: Counter,
pub serve_stream_rejected_blob_too_large: Counter,
pub serve_stream_rejected_hash_denied: Counter,
pub serve_stream_rejected_chain_hash_denied: Counter,
pub serve_stream_rejected_origin_denied: Counter,
pub serve_stream_rejected_foreign_declined: Counter,
pub serve_stream_rejected_chain_stale: Counter,
pub serve_stream_terminated_takedown: Counter,
pub serve_stream_rejected_load_shed_hit: Counter,
pub serve_stream_rejected_load_shed_miss: Counter,
pub serve_frame_accounting_fault: Counter,
pub serve_stream_node_fault: Counter,
pub serve_stream_proof_budget_exhausted: Counter,
pub serve_stream_rejected_stream_cap_full: Counter,
pub serve_stream_request_unreadable: Counter,
pub serve_stream_voucher_rejected: Counter,
pub serve_stream_client_declined: Counter,
pub serve_stream_client_abandoned: Counter,
pub load_shed_egress_bps: Gauge,
pub load_shed_pressure_active: Gauge,
pub load_shed_streams_in_flight: Gauge,
pub load_shed_refused_node_at_capacity: Counter,
pub load_shed_refused_egress_saturated: Counter,
pub load_shed_refused_client_at_capacity: Counter,
pub warming_speculative_blocked: Counter,
pub warming_sources_blocked: Gauge,
pub slash_watcher_last_tick_timestamp_seconds: Gauge,
pub staker_set_watcher_last_tick_timestamp_seconds: Gauge,
pub capacity_bond_registry_last_resync_timestamp_seconds: Gauge,
pub blacklist_watcher_last_tick_timestamp_seconds: Gauge,
pub settlement_watcher_last_tick_timestamp_seconds: Gauge,
pub fee_shares_watcher_last_tick_timestamp_seconds: Gauge,
pub slash_watcher_task_panicked: Counter,
pub staker_set_watcher_task_panicked: Counter,
pub blacklist_watcher_task_panicked: Counter,
pub fee_shares_watcher_task_panicked: Counter,
pub settlement_watcher_task_panicked: Counter,
pub chain_get_logs_span: Gauge,
pub chain_get_logs_range_rejections: Counter,
pub chain_get_logs_retries: Counter,
pub chain_get_logs_deferred: Counter,
pub chain_boot_read_retries: Counter,
pub blacklist_watcher_restarts: Counter,
pub blacklist_watcher_down_seconds: Gauge,
pub settlement_watcher_restarts: Counter,
pub settlement_watcher_down_seconds: Gauge,
pub blacklist_enforcement_failures: Counter,
pub blacklist_reenumeration_failures: Counter,
pub bytes_served: Counter,
pub bytes_received: Counter,
pub pool_redemptions: Counter,
pub vouchers_received: Counter,
pub preimage_reveals_received: Counter,
pub pool_grace_closes: Counter,
pub onchain_tx_landed: Counter,
pub onchain_tx_reverted: Counter,
pub onchain_tx_send_failed: Counter,
pub onchain_tx_receipt_failed: Counter,
pub onchain_tx_timeout: Counter,
pub onchain_tx_receipt_recovered: Counter,
pub dht_findvalue_queries: Counter,
pub dht_lookup_round_timeouts: Counter,
pub dht_store_published: Counter,
pub dht_routing_table_size: Gauge,
pub dht_bucket_refresh_failures: Counter,
pub dht_bootstrap_find_node_failures: Counter,
pub fee_shares_watcher_unregistered: Gauge,
pub fee_shares_watcher_poll_failures: Counter,
pub fee_shares_watcher_restarts: Counter,
pub fee_shares_watcher_down_seconds: Gauge,
pub config_reload_failures: Counter,
pub receipt_write_failures: Counter,
pub serve_stream_midstream_pool_exhausted: Counter,
/* private fields */
}Expand description
deCDN-specific counters and gauges surfaced at /metrics.
The group name becomes the metric-name prefix, so fields appear as
e.g. decdn_probe_requests_total.
Fields§
§probe_requests: CounterTotal probe requests served.
active_connections: GaugeCurrently open QUIC connections.
lanes_open: GaugeCurrently open inbound serve lanes (distinct (pool, signer, provider) keys).
pool_deposit_usdc: GaugeRaw USDC still recoverable from the pools currently paying this node:
Σ (deposit − totalRedeemed) over the distinct pools the redeemer
planned lanes against, refreshed once per redeemer self-tick beside
Self::unredeemed_usdc. Already-redeemed funds have left the pool, so
this is the ceiling those pools can still pay — not their lifetime
deposits. Operator-visible name: decdn_pool_deposit_usdc.
buyer_wallet_usdc: GaugeRaw USDC in this node’s own buyer wallet — what it can still escrow as a deposit when it opens a payment pool on its cache-miss leg.
The buyer leg is the one part of the node that spends rather than earns,
and nothing else reports on it: an operator who never funds this wallet
sees a node that serves perfectly and silently buys nothing, because
every openPool reverts on the ERC-20 transfer.
Read at bootstrap and once per reclaim sweep (RECLAIM_SWEEP_INTERVAL),
so it lags a spend — or an operator top-up, which is the reading that
matters — by up to that long.
Zero is meaningful only on a node with
cache.node_to_node_pull_through_enabled on; a cache-only node never
opens a pool and has no reason to hold USDC.
buyer_lane_seed_failures: Counterdecdn_buyer_lane_seed_failures_total: this node could not establish a
lane’s already-paid watermark, so it refused the pull rather than
resuming the lane from zero.
A lane resumed from zero is stranded permanently, not for one pull: the pull persists its own progress on every exit path, which gives the lane a local row and stops the reseed ever running for that provider again. The node refuses instead, and this counts how often. A sustained rate means the chain lane or the buyer store is unhealthy and this node is buying nothing from the affected providers.
buyer_pool_adoption_failures: Counterdecdn_buyer_pool_adoption_failures_total: bootstrap could not tell
whether this node already owns a payment pool on chain, so it left the
first cache miss to open one.
Every increment is a chance that the node escrows a second deposit beside
one it already holds. Adoption runs once per process, so this cannot
self-correct before the next restart. Pair with
decdn_buyer_wallet_usdc: the two together are what distinguishes “no
pool to adopt” from “could not look”.
unredeemed_usdc: GaugeTotal raw USDC this node holds in accepted vouchers that it has not yet
redeemed on-chain — the sum of owed − paid across every inbound lane
the redeemer plans to collect. Refreshed once per redeemer self-tick
(redeem_interval_secs), not on every voucher, so it lags live accrual
by up to one interval. Operator-visible name: decdn_unredeemed_usdc.
client_idle_close: Countercdn/client/v1 connections closed by the application-layer idle reaper
(ADR 005 §Connection lifetime): no stream for APP_IDLE_TIMEOUT after the
last one closed. A sustained rate flags peers parking streamless
keep-alive’d connections — the abuse pattern the reaper exists to reclaim,
invisible in active_connections alone. Operator-visible name:
decdn_client_idle_close_total.
otlp_export_failures: CounterOTLP span-export batches whose export call failed (connect error,
non-OK gRPC status, or timeout), counted at the batch exporter. Spans
the batch queue drops when full, and spans a collector rejects inside
an OK partial-success reply, are not counted: 0 does not mean no
traces were lost. Stays 0 when observability.otlp_endpoint is unset
or the node emits no spans. Operator-visible name:
decdn_otlp_export_failures_total.
node_uptime_seconds: GaugeSeconds since node start.
rpc_healthy: GaugeJSON-RPC endpoint reachability per the watchdog task. 1 =
reachable, 0 = unreachable. adr/appendix-observability.md names operational gauges in
the decdn_* family; this is the per-tick mirror of the startup
check_rpc_reachability probe so dashboards/alerts can fire on
a sustained outage rather than relying on a one-shot startup line.
dispatch_rejected_global: CounterConnections rejected because the global concurrency semaphore was
exhausted. Field has no _total suffix because the OpenMetrics
encoder appends it automatically; the operator-visible name is
decdn_dispatch_rejected_global_total.
dispatch_rejected_per_source: CounterConnections rejected by the per-source rate limiter. Operator-
visible name: decdn_dispatch_rejected_per_source_total.
dispatch_in_flight: GaugeCurrently in-flight QUIC handler tasks holding a dispatch permit.
dispatch_per_source_skipped_no_addr: CounterConnections accepted on a relay-only path (no resolvable peer
IP) while the per-source layer was enabled. The per-source rate
limit cannot be enforced for these — operators chasing
dispatch_rejected_per_source anomalies need this counter to
distinguish “the layer didn’t fire” from “the layer wasn’t
applicable.” Operator-visible name:
decdn_dispatch_per_source_skipped_no_addr_total.
inbound_connections_direct: CounterInbound connections (all ALPNs) that arrived with a direct IP path.
Operator-visible name: decdn_inbound_connections_direct_total.
inbound_connections_relayed: CounterInbound connections (all ALPNs) that arrived over the relay only, with
no direct IP path at accept time. Expected on a node behind NAT; on a
node with a public address it means dialers did not find or reach that
address. Operator-visible name:
decdn_inbound_connections_relayed_total.
node_public_address: Gauge1 when the host’s default route has a public source address (ADR 001
§ Node Discovery), 0 when the node is behind NAT. Sampled at
bring-up. Operator-visible name: decdn_node_public_address.
probe_hold_slots_used: Gaugedecdn_probe_hold_slots_used (registry): current active
probe-triggered eviction holds (distinct held blobs), ADR 005
§Probe-triggered eviction hold. Sampled from the cache engine at
scrape time, so it tracks hold expiry with no probe traffic; pair
with probe_hold_slots_max for a saturation ratio.
probe_hold_slots_max: Gaugedecdn_probe_hold_slots_max (registry, mandatory): the configured
max_probe_holds budget. Set once at startup. Pairs with
probe_hold_slots_used so dashboards can alert on a saturation
ratio rather than an absolute count.
dht_rate_limit_rejected_per_peer: Countercdn/dht/v1 requests rejected by the per-peer (NodeId) token
bucket (ADR 022 §DHT Rate Limiting). One Counter per layer to match
the existing dispatch_rejected_* convention: a plain counter field
carries no label dimension (a labeled series would need a Family). Operator-visible name:
decdn_dht_rate_limit_rejected_per_peer_total.
dht_rate_limit_rejected_per_ip: Countercdn/dht/v1 requests rejected by the per-IP token bucket. Sibling
to dht_rate_limit_rejected_per_peer — see its docs. Operator-
visible name: decdn_dht_rate_limit_rejected_per_ip_total.
dht_rate_limit_rejected_global: Countercdn/dht/v1 requests rejected by the global token bucket. Sibling
to dht_rate_limit_rejected_per_peer — see its docs. Operator-
visible name: decdn_dht_rate_limit_rejected_global_total.
dht_rate_limit_prune_sweeps_per_ip: Counterretain_recent sweeps of the per-IP DHT keyed-limiter map that
actually ran (single-flight CAS won, layer enabled). Bump from
both the lazy-prune path in DhtRateLimiter::check and the
periodic gc_per_ip GC task. Persistent growth without a matching
drop in decdn_dht_rate_limit_tracked_per_ip means the limiter
is observing churn faster than retain_recent can release entries
— investigate cap saturation. Operator-visible name:
decdn_dht_rate_limit_prune_sweeps_per_ip_total (#645).
dht_rate_limit_prune_sweeps_per_peer: CounterSibling of dht_rate_limit_prune_sweeps_per_ip for the per-peer
(NodeId) keyed-limiter map. Operator-visible name:
decdn_dht_rate_limit_prune_sweeps_per_peer_total (#645).
dht_rate_limit_tracked_per_ip: GaugeCurrent size of the per-IP DHT keyed-limiter map after the most
recent prune (lazy or periodic). Pair with
decdn_dht_rate_limit_prune_sweeps_per_ip_total to detect cap
saturation. Updated post-prune so the value is at most one
DHT_RATE_LIMIT_GC_INTERVAL stale on quiet nodes; reads 0
until the first successful prune (lazy or periodic) after
startup. Operator-visible name:
decdn_dht_rate_limit_tracked_per_ip (#645).
dht_rate_limit_tracked_per_peer: GaugeSibling of dht_rate_limit_tracked_per_ip for the per-peer map.
Operator-visible name: decdn_dht_rate_limit_tracked_per_peer (#645).
probe_rate_limit_rejected_per_peer: Countercdn/probe/v1 requests rejected by the per-peer (NodeId) token
bucket (ADR 005 §Probe rate limiting). One Counter per layer to match
the existing dht_rate_limit_rejected_* / dispatch_rejected_*
convention; a plain counter field carries no label dimension (a labeled
series would need a Family);
operators recover the rolled-up rate with
sum(rate({__name__=~"decdn_probe_rate_limit_rejected_(per_peer|per_ip|global)_total"}[1m])).
Operator-visible name: decdn_probe_rate_limit_rejected_per_peer_total.
probe_rate_limit_rejected_per_ip: Countercdn/probe/v1 requests rejected by the per-IP token bucket. Sibling
to probe_rate_limit_rejected_per_peer — see its docs. Operator-
visible name: decdn_probe_rate_limit_rejected_per_ip_total.
probe_rate_limit_rejected_global: Countercdn/probe/v1 requests rejected by the global token bucket. Sibling
to probe_rate_limit_rejected_per_peer — see its docs. Operator-
visible name: decdn_probe_rate_limit_rejected_global_total.
probe_rate_limit_prune_sweeps_per_ip: Counterretain_recent sweeps of the per-IP probe keyed-limiter map that
actually ran (single-flight CAS won, layer enabled). Bumped from both
the lazy-prune path in the limiter’s check and the periodic
gc_per_ip GC task. Operator-visible name:
decdn_probe_rate_limit_prune_sweeps_per_ip_total (#645).
probe_rate_limit_prune_sweeps_per_peer: CounterSibling of probe_rate_limit_prune_sweeps_per_ip for the per-peer
(NodeId) keyed-limiter map. Operator-visible name:
decdn_probe_rate_limit_prune_sweeps_per_peer_total (#645).
probe_rate_limit_tracked_per_ip: GaugeCurrent size of the per-IP probe keyed-limiter map after the most
recent prune (lazy or periodic). Pair with
decdn_probe_rate_limit_prune_sweeps_per_ip_total to detect cap
saturation. Operator-visible name:
decdn_probe_rate_limit_tracked_per_ip (#645).
probe_rate_limit_tracked_per_peer: GaugeSibling of probe_rate_limit_tracked_per_ip for the per-peer map.
Operator-visible name: decdn_probe_rate_limit_tracked_per_peer (#645).
dht_requests_failed: Countercdn/dht/v1 request handling failed after the request was admitted
by the rate limiter — frame decode error, response write error,
read timeout, etc. Tracked separately from the rate-limit
rejections so an operator running with default RUST_LOG=info can
see the rate of “I accepted this request and then it broke”
failures without scraping debug-level logs. Operator-visible name:
decdn_dht_requests_failed_total.
dht_store_rejected_holder_mismatch: CounterStore rejected because holder != authenticated NodeId (ADR
022 §STORE Flow line 140). This is the lying-holder attack
signal — a non-zero value means at least one peer is trying to
publish records on behalf of someone else’s NodeId. Operator-
visible name: decdn_dht_store_rejected_holder_mismatch_total.
dht_store_rejected_non_staked: CounterStore rejected because the holder is not in the active-staker
set (ADR 022 §STORE Flow line 140). Sustained growth from many
distinct holders without matching decdn_dht_store_accepted
growth is a Sybil-attempt indicator. Operator-visible name:
decdn_dht_store_rejected_non_staked_total.
dht_store_rejected_quota: CounterStore rejected because the publisher is at the per-publisher
quota (ADR 022 §Content Records and TTL — 200-record hard cap).
Normal load shouldn’t trip this. Operator-visible name:
decdn_dht_store_rejected_quota_total.
dht_store_accepted: CounterStore admitted to the record store (newly inserted OR
refreshed). Pair with the three dht_store_rejected_* counters
to compute admission rate. Operator-visible name:
decdn_dht_store_accepted_total.
dht_batch_store_received: CounterBatchStore requests that passed stage-1 rate limiting and the
batch-level holder == authenticated NodeId check, i.e. reached
per-hash admission (ADR 022 §STORE Flow Batched STORE, #648). The
per-hash outcomes still land in the shared dht_store_* counters
above, so this counts batches, not hashes — pair the two to see
average admitted batch size and batch adoption. Operator-visible
name: decdn_dht_batch_store_received_total.
dht_batch_store_hashes_deferred_rate_limit: CounterHashes in a BatchStore acked false purely because the
two-stage rate-limit budget was exhausted before reaching them
(the n - k tail, ADR 022 §Batch token accounting / AC 18) — NOT
counted as a per-hash Store rejection because they were never
processed. Sustained growth means publishers are sending batches
larger than the per-peer burst allows in one window; the publisher
retries the tail. Operator-visible name:
decdn_dht_batch_store_hashes_deferred_rate_limit_total.
dht_republish_lag_sweeps: CounterTimes the republisher lost its cache-commit event window — the
subscribe_inserts broadcast channel overflowed and reported
Lagged. Counts lag events, not distinct store walks: a lag
arriving while a sweep runs is folded into that sweep’s next pass,
so this can outrun the number of walks performed. The cache does not
retain the missed hashes, so without a sweep a blob committed inside
the window stays undiscoverable until the node restarts (ADR 022
§Bootstrap). A nonzero rate means commits are outrunning the
scheduler; discovery still converges, but up to one cold-start
window late. Operator-visible name:
decdn_dht_republish_lag_sweeps_total.
dht_republish_lag_sweeps_coalesced: CounterLag events folded into a sweep that was already running or already
queued, rather than starting a walk of their own. The sibling that splits
decdn_dht_republish_lag_sweeps_total into the lags that claimed an idle
slot and the lags that did not.
Not a walk count, and the difference is not one either: the slot has
a single queued position, so any number of lags arriving behind a running
worker all count here and collapse into one further pass. The ratio is
what reads: coalesced climbing at the lag rate, with the difference flat,
means the slot is never released. Operator-visible name:
decdn_dht_republish_lag_sweeps_coalesced_total.
dht_republish_lag_sweep_panics: CounterLag-sweep walks that died in a panic instead of completing. The walk
runs in its own task, so a panic releases the slot through the normal
path and a queued request still runs — but the dead pass seeded only
what it reached before it died. A rate here with
decdn_dht_republish_sweep_reseeded_total flat means every sweep dies
before repairing anything, which nothing else surfaces.
Operator-visible name:
decdn_dht_republish_lag_sweep_panics_total.
dht_republish_sweep_reseeded: CounterHashes a lag sweep newly scheduled for republish. Seeding is
idempotent, so this counts the repair, not the walk: a sweep that
finds every held hash already scheduled adds zero. Operator-visible
name: decdn_dht_republish_sweep_reseeded_total.
dht_republish_seed_store_walk_failures: CounterBulk republish seeds that could not walk the local blob store and
covered only the origin-held half — both the boot-time cold start
and the lag sweep, which share one derivation. The seed degrades
rather than failing, so nothing else surfaces this: blobs held only
in the store stay un-republished until a later sweep re-walks the
store successfully, or until a restart. A boot-path failure is not
permanent — the next lag sweep recovers it — but nothing schedules
one, so a node with no further lag stays degraded for its lifetime.
Operator-visible name:
decdn_dht_republish_seed_store_walk_failures_total.
dht_republish_seed_origin_probe_failures: CounterStored hashes a bulk republish seed could not put to its origin — the ownership test an origin-only node applies to every blob in its store, answered by neither “yours” nor “not yours”.
The origin-side twin of
decdn_dht_republish_seed_store_walk_failures_total, and it degrades the
same way: a hash that cannot be confirmed is left out of the announce set
rather than advertised, because announcing on a transport blip risks
offering content the serve gate then refuses. So an unreachable remote
origin quietly shrinks the announce set of a node that holds the content.
Distinct from decdn_cache_origin_probe_failures_total, which counts the
rescan’s probes rather than the seed’s. Operator-visible name:
decdn_dht_republish_seed_origin_probe_failures_total.
voucher_nonce_gaps: CounterAccepted vouchers whose nonce skipped one or more values past the
previously-accepted nonce (voucher.nonce > last_nonce + 1), counted
once per gapped voucher (#747). The voucher is still accepted —
vouchers are cumulative, so on-chain settlement is unaffected — but a
non-zero rate flags dropped vouchers (per-voucher deliveries the node
never billed) or a client resetting/forking its counter (a replay-probe
signal). The precise skipped-count rides the paired tracing::warn! in
ChannelState::apply_voucher. Operator-visible name:
decdn_voucher_nonce_gaps_total.
redeem_hints_dropped: CounterRedeem hints the voucher-accept path could not enqueue because the
bounded advisory channel was full (try_send → Full), counted once
per dropped hint (#751). A hint is advisory — the next voucher re-hints
and the redeemer self-tick / shutdown close still redeem — so a few drops
are benign, but a sustained non-zero rate means the redeemer is not
keeping up with channel fan-out and threshold redemption is leaning on
the slow self-tick. Operator-visible name:
decdn_redeem_hints_dropped_total.
redeem_hints_parked: CounterRedeem hints the redeemer dropped because their lane is parked (#2340).
A lane parks when its hint-path redemption hits a chain fault — a
failed pre-redeem watermark read or a redeemMany that did not land,
an RPC fault or a revert alike. Each sweep (self-tick or serve cutoff)
retries the parked lanes with every other lane, and only a sweep that
finishes without a chain fault releases them. So an RPC outage costs one
failed attempt per lane per interval: the sweep’s batched retry. Hints
for a parked lane stay dropped until a sweep finishes without a chain
fault. A non-zero rate tracks chain faults on the settlement path; read it beside
decdn_onchain_tx_send_failed_total and
decdn_redemption_reconcile_failures_total. Operator-visible name:
decdn_redeem_hints_parked_total.
receipt_writes_dropped: CounterDownload-receipt audit writes dropped because the bounded writer queue
was full (try_send → Full, #803), counted once per dropped receipt.
The receipt log is audit-only and the payment already advanced the lane
watermark, so a drop never affects settlement — but a sustained non-zero
rate means the receipt writer cannot keep up with disk I/O (a slow or full
data_dir, the end-state of #802) and audit/dispute records are being
lost. Operator-visible name:
decdn_receipt_writes_dropped_total.
warming_credits_dropped: CounterADR 041 warming serve credits dropped before they reached the ledger,
counted once per dropped credit. Either the bounded aggregator queue was
full (try_send → Full) or the aggregator was gone (Closed). A drop
is conservative — it leaves the source’s ledger lower than reality,
never higher — so it can only block speculative warming,
never over-fund it. But a sustained non-zero rate means warming is being
throttled by lost bookkeeping rather than by real losses, and a rate that
tracks the serve rate exactly means the aggregator task is gone and every
source will drift to blocked. Operator-visible name:
decdn_warming_credits_dropped_total.
warming_credits_applied: CounterADR 041 warming serve credits the aggregator applied to a source ledger.
The success sibling of warming_credits_dropped, and the reason a zero
drop count means something: zero drops beside zero applies is a node
whose serve path never reached the ledger at all — an unwired
crate::warming_allowance::WarmingCreditSink leaves the inert
NoopWarmingCreditSink in place, which drops nothing because it enqueues
nothing. A node serving tagged blobs must show this climbing.
Operator-visible name: decdn_warming_credits_applied_total.
redemption_failures: CounterRedemption steps that failed: a redeemMany chunk that reverted
on-chain or that the RPC refused at send, a lane whose plan failed,
or a hint-path lane that could not be read from the lane store. A
chunk whose receipt stays unconfirmed after the by-hash fetch does not
count here; it counts into decdn_onchain_tx_receipt_failed_total or
decdn_onchain_tx_timeout_total. Each failure is otherwise only a
single warn!; a sustained rate means accrued earnings are not being
redeemed and warrants investigating the RPC / wallet. Operator-visible
name: decdn_redemption_failures_total.
redemption_skipped_insolvent: CounterLanes the redeemer held or dropped for the pool’s chain-observed
solvency (pool_is_redeemable, ADR 003): a drained Open pool is held
for a top-up, and a drained Closing pool, or one whose close deadline
falls within the redeemer’s landing slack, is dropped.
Counted once per skipped lane per planning pass. A sustained non-zero
rate means real unredeemed value is stuck behind pools this node can no
longer collect from — worth checking against pool_deposit_usdc for
which pools are dry. Operator-visible name:
decdn_redemption_skipped_insolvent_total.
redemption_skipped_expired: CounterLanes with value owed that the redeemer skipped because the signer’s
capability has expired, or expires within the redeemer’s landing slack
(ADR 003 §Revocation). The contract pays 0 for an expired capability, so
the redemption would only spend gas. Counted once per skipped lane per
planning pass. A lane with nothing owed is not counted. The serve path
stops accepting vouchers one redeem interval plus that slack before
expiry, so a sustained rate means a lane earned near its capability’s
expiry and no sweep redeemed it in time: check for failed or
floor-deferred redemptions. Operator-visible name:
decdn_redemption_skipped_expired_total.
redemption_reconciled_skip: CounterLanes dropped from a redeem batch by the pre-submit on-chain watermark
reconciliation because the chain already shows them settled to their
claim value — a redeemMany the contract would silently no-op, caught
before it costs gas. Counted once per dropped lane. A sustained non-zero
rate means the event-fed paid cache is lagging the chain (or a second
actor is redeeming the same lanes); a burst right after a restart is the
expected catch-up. Operator-visible name:
decdn_redemption_reconciled_skip_total.
redemption_reconcile_ok: CounterPre-submit on-chain watermark batches that landed and were reconciled
against the chain. Counted once per getWatermarks batch that decoded,
so a redeem sweep over more lanes than the read batch caps contributes
one per batch. This is the attempt signal
redemption_reconciled_skip cannot give: that counter is legitimately
zero on a healthy sweep with nothing to drop, so zero alone cannot
separate “the reconciliation ran and found nothing” from “the
reconciliation never ran”. Read the two together — a redeemer planning
lanes with this flat at zero is not reconciling at all. Operator-visible
name: decdn_redemption_reconcile_ok_total.
redemption_reconcile_failures: CounterPre-submit on-chain watermark batches that did not land — an RPC error,
a timed timeout, or a return whose length did not match the batch.
Counted once per failed batch. The read is fail-open (the contract’s own
claimed <= w.amount guard is the backstop), so a sustained rate costs
gas rather than correctness: the seller submits redeemMany batches it
never checked against the chain. A sustained non-zero rate means the
chain RPC is rejecting or timing out the batch read — check the
endpoint’s eth_call response-size and gas ceilings against the read
batch size. Operator-visible name:
decdn_redemption_reconcile_failures_total.
buyer_reclaim_failures: CounterBuyer-side reclaim-sweep attempts (try_reclaim) that failed — a failed
getChannel/reclaimExpired RPC, a receipt wait, an on-chain revert, or
a failed store write when clearing the local record after a reclaim/drop
(#906). Each is otherwise only a single warn! per hourly sweep; a
sustained rate means an expired channel’s refundable deposit is not being
recovered (check the gas wallet / RPC). Pairs with the error!
escalation once the same channel fails RECLAIM_ESCALATION_THRESHOLD
consecutive sweeps. Operator-visible name:
decdn_buyer_reclaim_failures_total.
buyer_pool_store_skipped_undecodable_records: CounterBuyer-pool rows omitted from a successful store hydration because
their persisted values could not be decoded. Counted once per skipped
row per load attempt, so a persistent malformed row keeps the alert
active while healthy rows continue through buyer maintenance. Operator-
visible name:
decdn_buyer_pool_store_skipped_undecodable_records_total.
buyer_topup_ok: CounterBackground low-water top-ups (#1146) that landed: a reused buyer channel
whose remaining deposit had fallen below its 20% low-water mark was
re-funded to the working deposit, so sustained miss pulls to that provider
keep flowing instead of silently stranding on a spent-down channel. The
healthy signal of the auto-refill path. Operator-visible name:
decdn_buyer_topup_ok_total.
buyer_topup_failure: CounterBackground low-water top-ups (#1146) that did NOT cleanly land: the standing
allowance re-approval failed, the topUp submission/receipt errored or
reverted, OR the topUp landed on-chain but the local channel row vanished
or rotated during the RPC, or the credit itself faulted (escrowed-but-untracked
— fund_pool logs the tx at error! for reconcile). Folding the untracked case in here — rather than
counting it as buyer_topup_ok — means an operator alerting on this metric
sees stranded deposits. The refill is best-effort (the channel is simply left
un-topped), but a sustained rate means reused channels
are not being refilled and pull-through to busy providers will degrade as
their deposits drain (check the gas wallet / RPC / USDC balance). Operator-
visible name: decdn_buyer_topup_failure_total.
watcher_persist_failures: CounterLane-store reconciliation the settlement watcher could not apply from a
poll tick: a forget_lane store write that failed while dropping a
reclaimed pool’s lanes on a PoolReclaimed event. The failure is
swallowed (logged, and the cursor advances past it), so a non-zero rate
flags a struggling lane store or RPC rather than a stuck settlement.
Operator-visible name: decdn_watcher_persist_failures_total.
lane_flush_failures: CounterLane-store background flushes that failed (the dirty set is retained and
retried next tick). A sustained non-zero rate means the store cannot be
fsynced — the frontier-only replay surface is widening. Operator-visible
name: decdn_lane_flush_failures_total.
slashes_detected: CounterSlashes detected against this node’s operator by the slash watcher
(SlashJudge.Slashed), counting each distinct slashId once across the
bring-up backfill and the live stream (#1032). A non-zero value means the
operator was slashed and should consider decdn appeal slash within the
30-day window. Operator-visible name: decdn_slashes_detected_total.
slash_resync_failures: Counterdecdn_slash_resync_failures_total: times the slash watcher’s periodic
re-enumeration could not read chain state and kept the detected-slash
set it already had. The resync reports success upward so the event tail
keeps running, so a persistently failing repair moves nothing else.
Pairs with the warn! in slash_watcher’s SlashSink::on_tick_complete.
Field has no _total suffix because the OpenMetrics encoder appends it.
slash_watcher_restarts: Counterdecdn_slash_watcher_restarts_total (#1032): distinct drift windows the
slash-detection watcher has entered, bumped once on the edge into the
error/backoff state. Pairs with slash_watcher_down_seconds to tell one
long outage from repeated flapping. Mirrors staker_set_watcher_restarts.
slash_watcher_down_seconds: Gaugedecdn_slash_watcher_down_seconds (#1032): seconds the slash-detection
watcher has been stuck in its per-tick backoff loop (0 on a healthy
cycle), recomputed at scrape from slash_watcher_down_since. Mirrors the
other chain watchers — since admin_v1_slashes is always wired, this is
the only signal that a wedged watcher could be silently missing slashes.
staker_set_watcher_restarts: Counterdecdn_staker_set_watcher_restarts_total (#783): distinct drift
windows the crate::dht::chain_staker_set watcher has entered —
bumped once on the transition from a healthy cycle into the
error/backoff state, NOT on every backoff iteration of one continuous
outage. Each increment therefore brackets exactly one window in which
the cached active-staker set can drift from chain state — the module’s
documented mid-run degradation. Because the stake-lane probe
reservation (#757) and the DHT Store admission path both read that
cached set, sustained restarts are revenue-impacting, not just a
discovery-health blip. The loop-level tracing::warn! in
multiplexed_poller::run ("watcher RPC error; restarting after backoff")
is emitted in the same failed-tick Err arm that fires the on_backoff
hook this counter hangs off, and carries the underlying error; this
counter is the alertable rate. An idle poll tick (head has not advanced /
no new logs) is NOT an error and does not bump this. Field has no
_total suffix because the OpenMetrics encoder appends it.
staker_set_watcher_resolve_failures: Counterdecdn_staker_set_watcher_resolve_failures_total (#788): times an
operator-indexed event (Reinstated / UnbondingRequested) was
dropped because the follow-up nodeIdOf(operator) RPC failed. A dropped
resolution leaves the cached active set out of sync with chain state for
that operator until a later event, or until the watcher’s cadence-gated
getRegisteredNodes re-enumeration corrects it — exactly the silent
drift these #783/#788 metrics exist to surface. That resync shortens the
window rather than bounding it: it is skipped while the route is errored,
and a failed read defers another interval.
Unlike a stream-level error, this does NOT trip a backoff/restart, so it
would otherwise move no metric at all. Pairs with the per-failure
warn! in crate::dht::capacity_bond_registry’s
RegistrySink::on_operator_change. Field has no _total suffix because
the OpenMetrics encoder appends it.
capacity_bond_registry_resync_failures: Counterdecdn_capacity_bond_registry_resync_failures_total: times the
capacity-bond registry’s periodic re-enumeration of getRegisteredNodes
could not read chain state and kept the projections it already had.
That re-enumeration is the only systematic repair for an active-staker
set that has drifted from chain state, and it deliberately reports
success upward — returning an error would mark the watcher route errored
and stall event pickup, trading a stale set for no updates at all. So a
persistently failing resync moves nothing else. Two behaviours compound
it, both correlated with the RPC fault that caused the drift: the resync
is skipped entirely while the route is errored, and a failed read stamps
the cadence clock anyway, deferring the next attempt a further interval.
Pairs with the liveness gauge below, which is what catches the skipped
case. Field has no _total suffix because the OpenMetrics encoder
appends it.
staker_set_watcher_down_seconds: Gaugedecdn_staker_set_watcher_down_seconds (#783, semantics corrected
#788): true downtime — seconds the staker-set watcher has been in the
error/backoff state, i.e. failing its eth_getLogs poll tick. Reads 0
for the entire life of any established cycle, however long or quiet (a
healthy poll loop persists indefinitely, so this must NOT measure cycle
age), and climbs only while the loop is between a failed cycle and the
next successful re-establishment. Reads 0 until the first cycle is
established after bootstrap. Recomputed at scrape time from a monotonic
down_since timestamp. A poisoned lock reports i64::MAX (not 0) so
it trips the alert rather than masking an in-progress outage — the
conservative direction for a downtime gauge. This is the drift-window
depth gauge that decdn_rpc_healthy (a different watchdog) does not
cover.
staker_set_active_count: Gaugedecdn_staker_set_active_count (#783): current cached active-staker
set size, sampled on every membership change the watcher applies. Pairs
with decdn_staker_set_watcher_down_seconds to spot a collapse —
e.g. the count holding flat while down-seconds climbs means the cache
is frozen, not that the network genuinely lost operators.
staker_set_active_unknown_region: Gaugedecdn_staker_set_active_unknown_region: active stakers whose
regionHint is empty or not an accepted region code. With the
per-region family it sums to decdn_staker_set_active_count after each
completed watcher tick. The count moves on each event and the region
split at the end of the tick, so the two can differ between ticks and
while the watcher is in backoff.
node_address_directory_size: Gaugedecdn_node_address_directory_size (#831): current count of cached
NodeId → operator address bindings the node-to-node pull path resolves
against (crate::dht::node_address::ChainNodeAddressDirectory).
Sampled on every NodeRegistered / NodeDeregistered the watcher
applies. A binding the watcher misses (drift window) means the
orchestrator cannot pay that provider and skips it, so a collapsed or
frozen count here directly caps reachable upstream providers.
node_pull_attempts: Counterdecdn_node_pull_attempts_total (#831): node-to-node cache-miss pull
orchestrations that found ≥1 candidate provider to try. Denominator for
the success/corruption/unreachable rates below.
node_pull_first_byte_seconds: Histogramdecdn_node_pull_first_byte_seconds: time to first byte of one paid
node-to-node pull leg, from the start of its open (a dial included, when
the node holds no warm connection to the peer) to the first bao bytes
read off the upstream stream. One observation per leg. An adopted header
handshake keeps the start of the handshake open, so its window includes
the handshake. A leg that reads no bytes records nothing.
node_pull_success: Counterdecdn_node_pull_success_total (#831): pulls that delivered verified
bytes from an upstream node.
node_pull_no_providers: Counterdecdn_node_pull_no_providers_total (#831): misses where discovery
(DHT plus the origin-directory fallback) surfaced no provider — the blob
is unavailable on the network, not a pull failure.
probe_cache_hits: Counterdecdn_probe_cache_hits_total (#1165): cache-miss pulls whose candidate
walk STARTED from a live ADR 001 §Probe cache entry with at least one
still-selectable provider. The DHT lookup and probe fanout are skipped —
unless every cached provider fails, in which case the same fetch either
falls through to a fresh lookup + probe (if attempt budget remains) or
returns a clean miss (if the cached providers exhausted the budget first,
the common case since an entry holds up to 10 providers but the budget is
3) — either way it stays counted here (the hit is “the cache had
something worth trying”, not “the cache delivered”). Field has no
_total suffix because the
OpenMetrics encoder appends it. With probe_cache_misses this is the
hit ratio the TTL exists to buy; a ratio near zero means the TTL is
shorter than the request inter-arrival time for hot blobs and the cache
is pure overhead.
probe_cache_misses: Counterdecdn_probe_cache_misses_total (#1165): cache-miss pulls that had to run
a fresh DHT lookup + probe. Counts an entry that was absent, expired, OR
fully suppressed (every cached provider negative-cached, wedged, no longer
an active staker, or otherwise unselectable), and on the ranged pull leg an
entry whose providers do not cover every block of the blob (#2195) — all
cost the same network work, which is what this measures.
probe_collection_latency_seconds: Histogramdecdn_probe_collection_latency_seconds (ADR 001 §Probe response
collection): the time from the start of the concurrent probes, dials
included, to the end of collection — the early stop once enough holders
answer, or the drain of the probe set, which PROBE_TIMEOUT bounds per
probe. A round that sends no probe records nothing. Each probe round
records one sample, so a ranged miss whose holders do not span the blob,
and that then probes origin-directory candidates, records two (#2195).
dht_lookup_round_ceiling: Counterdecdn_dht_lookup_round_ceiling_total (#1145 review): a find_providers lookup was
TRUNCATED at MAX_LOOKUP_ROUNDS while still finding closer nodes.
Not an error — the providers already found are usable and the pull proceeds with them
— but not nothing either, and a bare debug! would be invisible at the
project’s default RUST_LOG=info, so it is a counter. The ceiling exists because
discovery’s worst case
has to be FINITE for PULL_THROUGH_OUTER_SLACK to budget for it at all; the cost is
that a lookup which needs more rounds silently returns a smaller candidate set.
So a sustained rate means this node’s round ceiling is too low for its network size:
every lookup is cut short, and MAX_PROVIDER_ATTEMPTS is choosing from a worse set of
providers than the DHT could have offered. Kademlia converges in O(log n) rounds, so
this is the metric that says “your network outgrew the constant”.
node_pull_corruption: Counterdecdn_node_pull_corruption_total (#831): an upstream served
hash-mismatched bytes for a paid pull (Byzantine / buggy provider). A
sustained nonzero rate means a peer is taking payment for wrong content;
the cache engine rejects the bytes, but the USDC was still spent.
node_pull_unreachable: Counterdecdn_node_pull_unreachable_total (#831): a probe or pull to a
candidate failed at the transport (scored Outcome::Unreachable).
node_region_latency_penalty: Counterdecdn_node_region_latency_penalty_total (#1177): a probed peer
self-attested this node’s own region yet answered slower than the ADR 030
latency ceiling, so it was penalized in local reputation
(Outcome::RegionLatencyMismatch). A sustained rate points at
region-spoofing peers (or a genuinely mis-set local identity.region).
node_pull_pool_open_failures: Counterdecdn_node_pull_pool_open_failures_total (#831): a buyer
open_or_reuse_pool failed before a pull could start. This is the
node’s own payment-side fault (gas, RPC, expired pool), NOT the
provider’s — a sustained rate means node→node buying is wedged. This is
the unlabeled total across all causes; the
pool_open_failures_*_total family below (#966) breaks the
openPool-tx failures out by cause so an operator can tell a
misconfiguration (insufficient_deposit) from infrastructure
(rpc_error). It also covers store/expired-reclaim causes the by-reason
family does not, so the two are not expected to sum equal.
One failure moves this counter once. A site increments it if and only
if it marks the error OpenReported, which is what stops the classifier
in node_origin from restating a failure the open task already counted.
Against node_pull_attempts_total this reads above 1.0 legitimately: a
pull orchestration meters one attempt and may open a lane per candidate
and per assembled run. A clean 2× ratio is the signature of a leg that
meters without marking (#2072).
pool_open_failures_insufficient_deposit: Counterdecdn_pool_open_failures_insufficient_deposit_total (#966): a buyer
openPool tx reverted because the node’s USDC balance/allowance could
not cover the deposit, or the deposit was zero — either as requested, or
as the balance delta actually received under a fee-on-transfer token.
Both zero cases revert the same argument-less ZeroAmount, so this
counter cannot separate them; the wallet balance is what distinguishes a
misconfigured deposit from a token that shaved it. A token that reverts
in the older Error(string) style lands here too, matched on its message
rather than a custom-error selector. A misconfiguration
signal either way — the fix is operator-side (fund the wallet, raise the
configured deposit), not infrastructure. A plain counter
field carries no label dimension (a labeled series would need a Family),
so the issue’s {reason=…} split is realized as
three sibling counters (mirroring dht_rate_limit_rejected_*); the
reason value is the field-name token. The OpenMetrics encoder appends
the _total suffix.
pool_open_failures_contract_revert: Counterdecdn_pool_open_failures_contract_revert_total (#966): a buyer
openPool tx reverted on-chain for a reason other than insufficient
deposit (provider not active, a paused contract, a mined revert whose
reason is not recoverable from the receipt). The deposit was not
escrowed; the cause is on-chain state, not this node’s wallet or RPC.
pool_open_failures_rpc_error: Counterdecdn_pool_open_failures_rpc_error_total (#966): a buyer
openPool submit or receipt wait failed at the transport layer (no
revert data) — connectivity, a timed-out receipt, a nonce blip. A
transient infrastructure signal; retrying typically clears it. Pair
with the two reverting counters above to tell “operator under-funded the
wallet” from “the RPC endpoint is flaky”.
node_pull_too_large: Counterdecdn_node_pull_too_large_total (#840): the bytes received from a
selected upstream crossed this node’s max_blob_size ceiling, so the buyer
aborted the pull (#1895). The upstream’s total_bytes claim plays no part.
Like a channel-open failure this is a buyer-side policy decision, NOT
necessarily provider misbehavior (the provider may legitimately serve
larger blobs to nodes with a higher ceiling), so it does not tar the
provider’s reputation. A sustained rate
means this node’s ceiling is below the content it is trying to warm.
node_pull_rate_above_ceiling: Counterdecdn_node_pull_rate_above_ceiling_total (#1375): a selected upstream
signed an open-stage StreamResponse quoting a per-MB rate above this
node’s effective buyer ceiling (the lower of the candidate’s probe rate and
the configured cache.max_rate_per_mb), so the buyer refused before paying
any voucher. Sibling of Self::node_pull_too_large: a buyer-side policy
decision that does NOT tar the provider (an over-config quote is our tight
policy; an over-probe quote is a possible bait-and-switch we do not
adjudicate here). A sustained rate is the operator’s signal that nodes are
being probed for rate manipulation — the one refusal on this path that
guards a slashable offense, so it earns a counter of its own.
serve_economics_refused: Counterdecdn_serve_economics_refused_total (ADR 041): candidates existed for a
cache-miss buy but every one quoted above this node’s serve-economics buy
ceiling, so the buyer declined an unprofitable relay rather than paying for
bytes it cannot resell for a margin. Sibling of
Self::node_pull_rate_above_ceiling: a buyer-side policy decision, so it
does not tar the provider’s reputation. The node still signs the client a
NotFound for this — the pricing floor never reaches the wire — so a
sustained rate is visible only here, and means this node’s serve economics
(or the market it is buying into) are too tight to relay profitably.
The aggregate over both regime siblings below, mirroring the
pool_open_failures_* shape: bumped on every refusal beside the regime
counter, so a dashboard can rate the total and drill into the split.
serve_economics_refused_warming: Counterdecdn_serve_economics_refused_warming_total (ADR 041): the subset of
Self::serve_economics_refused where the source still had warming
allowance, so the ceiling was the market price max(sell, amortized) and
the market itself is above it. Remedy: the market is too hot to relay
into — there is no local config fix. See ServeEconomicsRegime.
serve_economics_refused_amortized: Counterdecdn_serve_economics_refused_amortized_total (ADR 041): the subset of
Self::serve_economics_refused where the source’s warming allowance was
spent (or warming is off), so the ceiling was already the amortized floor.
Remedy: this node’s own sell rate or margin config is too tight. A rising
split toward this regime is the operator-actionable one. See
ServeEconomicsRegime.
node_pull_timeout: Counterdecdn_node_pull_timeout_total (#857): a buyer→upstream pull hit one of this node’s
own deadlines. Like a channel-open failure this is a buyer-side condition (a possibly
mis-sized local budget), NOT evidence the provider is unreachable, so it does NOT tar
the provider’s local reputation. Distinct from node_pull_through_timeouts (the
delivery handler’s own serving deadline).
It fires on two budgets, and they have different remedies (#1145 review):
- the STREAM-OPEN stage exceeding
node_pull_timeout_sec; and - a pull that has received no first byte within
node_pull_stall_window_sec. Before the first chunk, the throughput floor is measuring the server’s time-to-FIRST-byte, which scales with blob size (the serve path materialises the whole bao encoding before it can emit chunk #1) — so it is our deadline, not the peer’s fault, and it lands here rather than onnode_pull_stalled_total.
A sustained rate therefore means one of those two is too tight for the upstreams this
node selects — and for the large-blob case it is node_pull_stall_window_sec, not
node_pull_timeout_sec, that wants raising.
The peer is not scored, but it IS suppressed briefly: see REFUSAL_SUPPRESSION_TTL.
Exonerating a peer and ignoring it are different things, and a peer that accepts a
stream and then says nothing lands here.
node_pull_voucher_rejected: Counterdecdn_node_pull_voucher_rejected_total (#857): an upstream rejected a
voucher this node presented mid-pull (an amount or bytes regression, a
spent capability cap, or a voucher addressed to the wrong pool or
provider). This is the node’s own payment-side fault, NOT the provider’s,
so it does NOT tar the provider’s reputation. A sustained rate means this
node’s buyer lanes are drifting out of sync with what upstreams accept.
node_pull_pool_wedged: Counterdecdn_node_pull_pool_wedged_total (#1145 review): an upstream rejected our
voucher on terms this lane cannot recover from, while the pool’s deposit is STILL
ESCROWED — an amount or bytes regression, a spent capability cap, an expired
capability, or a voucher addressed to the wrong pool or provider.
The pool row is KEPT (it is the only thing that can still reclaim the deposit) and the PROVIDER is suppressed for a fixed one-hour window. The deposit is not stranded: it is shared across every lane, so it stays available to every other provider throughout, and the pool has no expiry of its own to wait on.
A sustained rate is a provider this node keeps failing to pay. A spike means buyer watermarks are drifting out of sync with what upstreams have committed — the desync tracked in #1122 — and the deposit sizing and pool count should be reviewed alongside it. One tick costs an hour of that provider’s capacity, not the deposit.
node_pull_refused: Counterdecdn_node_pull_refused_total (#1144): a selected upstream refused
delivery up front (a StreamResponse with ok == false). Counts every
wire class, and none of them tars the provider’s reputation. It is not
split by class (a labeled series would need a Family). Most refusals
are honest and benign: a NotFound is simply a healthy-but-empty node, so
a sustained rate here usually means content discovery is steering this node
at upstreams that do not hold the blob — not that the upstreams are bad.
node_pull_refused_unattributable: Counterdecdn_node_pull_refused_unattributable_total (#1520): the subset of
Self::node_pull_refused whose wire code this node cannot pin on the
upstream — NotFound and Unfunded, i.e. RefusalVerdict::Transient.
Those briefly suppress the (peer, hash) pair without touching reputation,
except an open-stage NotFound on a node-origin leg, which is backpressure
and suppresses nothing unless it outlasts the wait budget (#2178).
Split out because it is the shape a buyer-side misconfiguration takes:
Unfunded is what a seller signs when it refuses OUR pool for its
deposit, and NotFound when our signer sits at its floor cap, so a node
whose own deposit is too small to buy anything sees 100% of its pulls
refused here.
Read it with one caveat: NotFound also carries the PEER’s load shed,
which the code respects rather than punishes. So a network-wide load event
drives this ratio to ~1 for a reason no local change fixes. A rate
approaching node_pull_refused therefore means “nobody is serving me”,
which is usually local (deposit, binding) but is worth confirming against
peer health first; a small fraction is the healthy “that peer did not
have it”.
node_upstream_rate_limited: Counterdecdn_node_upstream_rate_limited_total (#1986): a probe or pull to a
candidate was shed at the transport with APP_ERR_RATE_LIMITED (0x10,
ADR 013 §Application Error Codes) — the upstream’s connection limiter,
probe limiter, or per-connection stream cap refused the work before any
signed message existed. The transport-level twin of a load-shed NotFound
refusal, treated the same way: NO reputation outcome is recorded, and the
(peer, hash) pair is suppressed for the short refusal TTL except on a
node-origin leg, where the shed is backpressure (#2178). So this is the
only trace the event leaves at the default RUST_LOG=info.
A sustained rate is peer-side load, not a local fault — but a rate that
tracks node_pull_unreachable is worth a look: a global-full shed skips
the upstream’s close ack-wait and can still arrive as a bare drop, which
lands in the unreachable counter instead of here.
node_pull_backpressure_backoffs: Counterdecdn_node_pull_backpressure_backoffs_total (#2178): a ranged run’s
upstream refused the stream open for backpressure (NotFound or a
transport rate-limit), no other candidate covered the
still-missing gap, and the assembly waited to ask the same upstream again
instead of dropping it. The common cause is this node’s own other pulls
filling the upstream’s per-signer live cap: the refusal clears as those
pulls pay.
Each tick is one completed wait, and each wait follows one refusal already
counted in node_pull_refused_unattributable or
node_upstream_rate_limited. A sustained rate most often means an
upstream’s cap is too small for the concurrency this node’s clients drive
through it.
node_pull_backpressure_exhausted: Counterdecdn_node_pull_backpressure_exhausted_total (#2178): the only holder of
a still-missing range kept refusing for backpressure through the whole
wait budget, so the assembly ended short and every serve stream attached
to it was truncated. The holder is then suppressed for the hash for the
short refusal TTL, and a warn! names it.
Zero is the expected value. A non-zero rate means a refusal that waiting
does not clear: a holder whose cap never releases, or one of the refusals
the seller collapses onto NotFound that is not a cap at all.
node_pull_leg_no_progress: Counterdecdn_node_pull_leg_no_progress_total (#2194): a pull leg streamed and
finished cleanly, yet moved neither the gap’s paid frontier nor the store’s
delivered frontier. The drive ends the gap rather than re-open and re-pay the
same range. An upstream leg is not scored: its (peer, hash) pair is
suppressed briefly and the run re-plans onto another source. A warn! names
the range, both frontiers and the channel’s paid wire since the leg opened.
Zero is the expected value. A tick means this node’s store did not keep a leg’s bytes, its ledger did not record the leg’s payment, or the upstream ended the stream without taking the leg’s final proof.
node_pull_stalled: Counterdecdn_node_pull_stalled_total (#1797): an upstream’s throughput fell below the
floor mid-stream — the bytes across node_pull_stall_window_sec dropped under
node_pull_min_throughput_bps — so the pull was abandoned after at least one byte had
arrived. Split from node_pull_timeout (the same abort before the first byte) only as
a metric: BOTH are requester-local policy and neither tars the provider’s reputation,
because a sub-floor stream may be slow for reasons the peer cannot be blamed for and a
throughput signal is spoofable. A sustained rate points at flaky upstreams or a
node_pull_min_throughput_bps too high (or node_pull_stall_window_sec too tight)
for the network.
node_pull_local_fault: Counterdecdn_node_pull_local_fault_total (#1145 review): a pull failed for a reason
that is OURS — a broken signer, an encode fault, a bad range computation, an
unusable deadline config, a buyer channel open this node’s own state defeated
(an unreadable or unwritable channel store, a poisoned open lock, a panicked open
task, a wallet that cannot fund a deposit; #1560), or a cache store that cannot
take the pulled bytes (a full or read-only disk, a cache code bug; #2286) — and
the upstream was exonerated.
The only counter here that says nothing about the network. Any sustained rate is an emergency: a node that cannot sign a voucher or store a byte cannot complete a pull, so every pull it attempts will fail. These failures land here, not on the honest providers the node tries: scored against them, they would show as a node steadily blaming a healthy network in its own reputation scores.
Counted PER CANDIDATE, not per request: one node-wide fault moves this by up to
MAX_PROVIDER_ATTEMPTS for a single client request, because the walk tries each
candidate and fails identically on all of them. Read the rate, not the absolute,
and do not infer the number of affected requests from it.
node_pull_pool_open_pending: Counterdecdn_node_pull_pool_open_pending_total (#1143): a buyer channel open
was still in flight when the per-candidate budget expired, so the pull moved
to the next candidate while the open continued in the background.
NOT a failure — kept separate from node_pull_pool_open_failures because
the diagnosis is different: a failure says the tx reverted or the wallet is
under-funded, whereas this says the open is simply not done yet. It scores no
reputation: a wedged open is our lane, not evidence about the peer.
§Two distinct causes, one counter
- the node’s own chain lane (a slow L2, a stuck nonce) is slower than
CHANNEL_OPEN_CALLER_BUDGET— the interesting one; and - a boot or idle reconcile holds the provider’s open slot.
The verdict is the same for both — try the next candidate, score nothing — which is why they share a counter. But the diagnosis is not: reconcile runs at every boot, so a restart produces a burst here that means nothing is wrong. Read a sustained rate as a chain-lane signal only once it outlives a restart.
node_pull_progress_persist_failures: Counterdecdn_node_pull_progress_persist_failures_total (#852): a pull paid ≥1
voucher but persisting the buyer channel’s resume watermark
(record_progress) failed. The bytes were delivered, but the channel’s
stored nonce/bytes/amount now lags what the upstream accepted — the
next reuse of this channel will re-sign a stale voucher and be rejected. A
non-zero count means a provider is at risk of becoming unusable until
channel rotation.
node_pull_progress_dropped: Counterdecdn_node_pull_progress_dropped_total (#1145 review): a pull paid ≥1
voucher, but by the time the watermark was written the provider’s slot had
been replaced by a newer open — so the write was skipped rather than clobber
the replacement, and that voucher’s progress is gone.
Sibling of node_pull_progress_persist_failures_total, which only counts the
Err path; this is the Ok-but-dropped path, which is otherwise invisible.
The rare benign case is a channel rotating mid-pull. A sustained rate means
a channel_id-plumbing bug, and every tick is real USDC whose watermark was
discarded — so this is the counter to alert on, not just to look at.
node_pull_progress_superseded: Counterdecdn_node_pull_progress_superseded_total (#1145 review): a pull’s watermark write was
REGRESSED because a CONCURRENT pull on the same shared channel ledger had already
persisted a higher (correct) watermark. Benign and EXPECTED under the shared
BuyerLedgers — concurrent pulls on one channel are routine — because the monotonic
store keeps the winner’s higher value, so no voucher is lost. Split from
node_pull_progress_persist_failures_total (a real store-write failure that leaves the
watermark lagging) so ordinary settle races do not drown out a genuine persist fault.
node_pull_recovery_step: Counterdecdn_node_pull_recovery_step_total (#1530): a node→node miss fill ran a
funding recovery step (ADR 003 § Funding recovery) that funds the next
pass: every candidate refused this node’s funding, so the step topped the
buyer pool up, opened a new pool in place of one that accepts no more
funds, or settled a pool that already holds its working deposit while the
upstreams catch up.
Expected to be RARE once buyer_working_deposit_micro_usdc is sized for the
blobs this node pulls: the low-water refill should refill a
pool long before a fill runs it dry. A sustained rate means the working
deposit is too small for the blob sizes in play, and every tick is a
transaction plus a settlement wait a client sat through.
Distinct from decdn_buyer_topup_ok_total, which counts on-chain top-ups from
BOTH legs: this one isolates the recovery step, so the low-water refill’s
routine traffic cannot hide it.
node_pull_recovery_step_refused: Counterdecdn_node_pull_recovery_step_refused_total (#1530): a fill’s funding
recovery step had no way to fund the next pass: the working deposit is
zero (funding off), the top-up landed nothing, or the funding
transaction failed (allowance, revert, RPC, or a mined topUp the local
pool row could not credit).
A failed funding transaction also ticks decdn_buyer_topup_failure_total.
node_pull_through_timeouts: Counterdecdn_node_pull_through_timeouts_total (#831): cache-miss pull-through
attempts the delivery handler abandoned at its deadline. Distinguishes a
slow/wedged upstream from a genuine miss (both otherwise return
NotFound). Covers BOTH reactive fill tiers — node→node (#831) and the
local-origin populate (#1116) — so it is not solely a node→node-health
signal; a cache-only operator’s own wedged origin bumps it too.
node_pull_through_errors: Counterdecdn_node_pull_through_errors_total (#831): cache-engine errors (not
clean misses) hit while filling a miss on the delivery path — a real
store/pull fault, surfaced as NotFound to the client but logged + bumped
here so it isn’t silent. Covers BOTH reactive fill tiers: node→node (#831)
and the local-origin populate (#1116).
node_pull_through_window_paused: Counterdecdn_node_pull_through_window_paused_total (#856): times the
window-paced serve loop paused the upstream pull because the per-request
unrecouped frontier (bytes pulled − bytes paid) reached the ramped
credit window (ADR 003 §Credit window, #1669) and it waited for the
downstream voucher to clear. A high rate is benign (the window is
doing its job pacing speculation); a flat zero under real pull-through
traffic means the window never binds.
node_pull_through_min_draw_waits: Counterdecdn_node_pull_through_min_draw_waits_total (#2061): times the
window-paced serve loop paused the upstream pull because the ramped
window had room, but less than the minimum draw of half the window, and
no serve leg was parked at the pull’s frontier. Each draw opens a new
upstream request, so the pause batches the room into fewer, larger draws.
A high rate is benign; it grows with the number of vouchers per draw.
node_pull_through_wait_seconds: Histogramdecdn_node_pull_through_wait_seconds: how long one window-paced pause
(window full or minimum draw) held the upstream pull before a downstream
payment or a parked serve leg released it, or its leg cancelled it. One
observation per pause.
Most pauses end within a voucher round trip; a tail near the top bucket
means a payer that stalls or a pacing regression that parks the pull
with no serve leg left to wake it. A pause past PULL_WAIT_WARN_AFTER (45 s)
also logs a warning.
node_pull_through_client_abandoned: Counterdecdn_node_pull_through_client_abandoned_total (#856): window-paced
serves the requesting client dropped or underpaid mid-pull, so the node
aborted the upstream pull and abandoned the partial fill. The per-request
loss is bounded to the ramped credit window; a sustained rate flags a leech.
A node-side fault on the serve leg is not an abandon: it counts on
decdn_serve_stream_node_fault_total only. Nor is an owed chunk that passes
its proof-wait ceiling while its connection still sends: a slow path or a
busy sibling stream can cause that, and the stream ends on the declined or
abandoned serve reason counter only. Every abandon here also ends the
inbound stream on one serve reason counter: a spent per-chunk proof budget
on decdn_serve_stream_proof_budget_exhausted_total, a rejected voucher on
decdn_serve_stream_voucher_rejected_total, and a client that drops on
decdn_serve_stream_client_declined_total before it pays or
decdn_serve_stream_client_abandoned_total after.
node_pull_through_upstream_verify_failed: Counterdecdn_node_pull_through_upstream_verify_failed_total (#856/#915): a
serve-miss pull ingested upstream bytes that failed bao verification against
the content root (ADR 038) — a chunk group rejected mid-stream (the
bait-and-switch case), a whole-blob root mismatch, or an over-delivery past
the promised wire. The partial fill is never promoted, the
upstream is scored Corruption, and the
client’s own bao decoder rejects the forwarded bytes. Distinct from
node_pull_corruption (the buffered orchestration’s own check). A sustained
rate means clients are being served
corrupt-upstream bytes through this node. Field has no _total suffix
because the OpenMetrics encoder appends it.
local_outboard_serves: Counterdecdn_local_outboard_serves_total (#1130): times the node served a
whole-blob cache miss by streaming straight from its own configured
origin (http/s3/fs) into the paying client while filling
the local cache store beside it — the stream-while-store path, entered only when
the origin publishes a {H}.obao4 pre-order outboard alongside the
object. Incremented once per entry into serve_via_backend_origin,
before any admission guard, so it also counts requests this tier later
rejects (deposit/leech/size) — it marks which SERVE TIER fired, not
whether the fill ultimately succeeded. A miss that instead falls back to
the buffered populate_local path (no outboard, or this tier declined)
never bumps this counter, which is what lets a test or dashboard tell
the two fill strategies apart deterministically instead of by timing.
serve_cache_hit: Counterdecdn_serve_cache_hit_total: paid serves the blob-availability gate
classified as servable from a COMPLETE locally-held blob, before any fill
tier runs. One bump per request the gate classifies, in
handlers::client::dispatch.
Counted BEFORE the load-shed admission call, so a shed refusal still records the availability decision it refused. That keeps the hit rate a property of the store rather than of the node’s current pressure — a node shedding hard would otherwise report a hit rate that collapses exactly when an operator most needs to read it.
This is the serve-path twin of decdn_cache_hits_total, and the two are
not interchangeable: the cache-crate counter meters
CacheEngine::get(), the whole-blob buffered read, which the paid serve
path never calls — it streams through export_bao_range_stream instead.
A node that serves only paying clients therefore holds
decdn_cache_hits_total at zero however full its store is. Its partner
decdn_cache_misses_total is NOT confined to get — the fill tiers bump
it too — so that pair reads as a populated, permanently-0% ratio rather
than as an empty one, which is why the serve path needs its own family.
Sibling of serve_cache_partial_hit and serve_cache_miss. At most one
of the three is bumped per request, and two requests that reach
serve_audit bump none: a withdrawn hash (operator evict or corruption
quarantine) is refused for evicted_since_probe, and a serve_audit store
fault is refused for internal_error. Neither is an availability class — one
is a refusal, the other is the store failing to answer — so the ratio’s
denominator deliberately excludes both. Field has no _total suffix
because the OpenMetrics encoder appends it.
serve_cache_partial_hit: Counterdecdn_serve_cache_partial_hit_total: paid serves admitted from a blob
that is not Complete but whose held chunk groups already cover the
requested span (#1506). Counted apart from serve_cache_hit because the
two answer different questions — this one is the payoff of the
range-keyed partial-holder advertisement a node makes over
cdn/probe/v1 and the DHT, and folding it into the plain hit would hide
whether that mechanism carries any traffic. Both are hits for hit-rate
purposes.
serve_cache_miss: Counterdecdn_serve_cache_miss_total: paid serves the gate could not satisfy
from locally-held bytes. Counts the ADMISSION decision and nothing
downstream of it — a fill tier may never run at all, because the bump
precedes the load-shed gate, the pre-spend floor reservation, and the
pull_authorized check that every tier is gated on. A shed refusal, a
floor refusal and an unbound request therefore all count here having
asked no origin and no peer. Where a tier does run and satisfies the
blob, the request still counts here; the refusal siblings
(serve_stream_rejected_*) say whether it ended in a refusal.
Also absorbs one non-absence: partial_hit_size resolves the local
bitfield with .ok()?, so a store fault while answering “do my held
ranges cover this span?” reads as “they do not” and lands here. That is
the conflation the serve_audit Err arm exists to avoid one branch
earlier, and it bounds how clean a reading of this counter can be.
serve_first_byte_hit_seconds: Histogramdecdn_serve_first_byte_hit_seconds: time to first byte of a paid serve
the availability gate classed as a hit (complete or partial), from the
decoded request to the first ChunkData frame written. The first frame
rides the opening credit window, so no client payment round trip is in
the window; one getPool chain read is, when the pool view has no entry
for the pool. A stream that ends before any frame records nothing.
Sibling of serve_first_byte_miss_seconds on the cache-class axis.
serve_first_byte_miss_seconds: Histogramdecdn_serve_first_byte_miss_seconds: time to first byte of a paid serve
the availability gate classed as a miss — the same window as
serve_first_byte_hit_seconds, which here also covers discovery and the
upstream or origin fill that produces the first frame. A window-paced
fill streams its first frame early; a buffered fill completes the whole
blob first, so on that tier the value grows with blob size.
serve_admission_seconds: Histogramdecdn_serve_admission_seconds: the admission phase of a paid serve’s
time to first byte, from the decoded request to the load-shed admit. It
covers the binding check, the deny gates, the getPool and
getAuthorization chain reads on a view miss, and the availability
audit. The class is unknown until the audit ends, so this phase has no
hit/miss siblings. Records only for a stream that writes its first frame,
so the three phase histograms and the total share one population.
serve_signer_auth_cached: Counterdecdn_serve_signer_auth_cached_total: admit-path signer confirms the
signer-auth cache answered, with no getAuthorization. A Registered
read answers for good; an Unregistered one for a short window.
serve_signer_auth_first_read: Counterdecdn_serve_signer_auth_first_read_total: admit-path signer confirms for
a (pool, signer) the node holds no read of: a new signer, or one
evicted from the bounded cache. Each waits on a getAuthorization, its
own or one already in flight for the pair.
serve_signer_auth_reread: Counterdecdn_serve_signer_auth_reread_total: admit-path signer confirms that
re-read an Unregistered signer, because the read aged out or the
projection has since folded a redemption by it. Each waits on a
getAuthorization, its own or one already in flight for the pair.
serve_response_hit_seconds: Histogramdecdn_serve_response_hit_seconds: the response phase of a paid hit,
from the load-shed admit to the signed StreamResponse written.
serve_response_miss_seconds: Histogramdecdn_serve_response_miss_seconds: the response phase of a paid miss,
from the load-shed admit to the signed StreamResponse written. It
covers the floor reservation, the origin size and range probes, and on a
node-to-node fill the upstream discovery, pool open and handshake. A
buffered fill imports the whole blob in this phase.
serve_first_frame_hit_seconds: Histogramdecdn_serve_first_frame_hit_seconds: the first-frame phase of a paid
hit, from the StreamResponse written to the first ChunkData frame
written. It covers the read and encode of the whole first frame.
serve_first_frame_miss_seconds: Histogramdecdn_serve_first_frame_miss_seconds: the first-frame phase of a paid
miss, from the StreamResponse written to the first ChunkData frame
written. On a window-paced fill it covers the pull leg filling the whole
first frame from upstream or origin.
origin_directory_get_origins_failures: Counterdecdn_origin_directory_get_origins_failures_total: getOrigins
lookups that failed on a cold-namespace cache miss. The directory fails
closed on each (resolves no origins for that request, does not cache
the failure), so this is the drift signal to alert on — sustained
failures mean real requests are silently losing their origin fallback.
origin_directory_cache_size: Gaugedecdn_origin_directory_cache_size: current namespace count held in
the lazy origin directory cache (positive + negative entries).
Bounded by the configured cache capacity (LRU eviction).
serve_stream_rejected_evicted_since_probe: CounterPaid-delivery (serve_stream) requests refused because the blob was
deliberately evicted between probe and stream (#279). One Counter per
reason — like the dispatch_rejected_* convention — because a plain counter
field carries no label dimension, and the wire StreamError deliberately
conflates the three NotFound reasons (cache_miss, unknown_lane,
owner_mismatch) (#876). Visible name:
decdn_serve_stream_rejected_evicted_since_probe_total.
serve_stream_rejected_cache_miss: Counterserve_stream requests refused because the blob is absent and
pull-through was unavailable or failed to fill it. Visible name:
decdn_serve_stream_rejected_cache_miss_total.
serve_stream_rejected_internal_error: Counterserve_stream requests refused by a local store fault (has/inspect
error, or a present blob reporting no size) — signed to the client as
Declined, not a signed absence. Visible name:
decdn_serve_stream_rejected_internal_error_total.
serve_stream_rejected_unknown_lane: Counterserve_stream requests refused on an unknown / never-opened lane
(#848). Wire-indistinguishable from cache_miss/owner_mismatch (all
signed as NotFound to avoid leaking lane existence), so this
server-side counter is the only place the distinction lives — a rising
value isolates an unknown-lane abuse campaign. Visible name:
decdn_serve_stream_rejected_unknown_lane_total.
serve_stream_rejected_owner_mismatch: Counterserve_stream requests refused because a verified client binding does
not authorize the named channel (#327). Visible name:
decdn_serve_stream_rejected_owner_mismatch_total.
serve_stream_rejected_bad_binding: Counterserve_stream requests reset because the client binding failed to
verify: the signature is invalid, or it recovers a different address
than the one claimed. No signed response is sent. Visible name:
decdn_serve_stream_rejected_bad_binding_total.
probe_read_faults: CounterProbe requests this node could not read: the read timed out, the frame
or the message failed to decode, or the peer sent a response on the
server stream. The connection closes with the matching ADR 013 code.
Visible name: decdn_probe_read_faults_total.
serve_stream_rejected_insufficient_deposit: Counterserve_stream requests refused before any bytes are served because the
requesting channel’s remaining deposit could not cover what the node
would front at its rate. (The refusal itself is signed — as an ok: false
response; what is never signed is an ok: true.) Three guards bump
this. Not a sequence any one request walks: a cache HIT reaches only (3),
and the window tier returns out of serve_stream, so (2) and (3) are
mutually exclusive. A miss passes (1) and then at most one of (2)/(3):
- the cache-miss floor (#1519) — one credit window, applied above every fill tier so none of them fronts origin egress or upstream USDC for a channel that cannot pay for a single interval;
- the window tier’s speculative ceiling (#856) — the whole-blob cost
when
max_blob_size_bytesis finite, else the window cost; - the direct-serve ceiling (#1516) —
min(credit window, chunk-group-aligned request span).
It does not distinguish them, so a spike cannot be attributed to a layer
from this series alone — the log line at the refusal site is what says
which. Guards (1) and (3) also split by CAP: both run the two-level floor
check, and its per-signer arm bumps serve_stream_rejected_signer_floor_at_cap
instead, so this counter is the pool-ceiling half of them.
Signed as Unfunded: every guard runs past the lane-ownership proof, so
its audience is the proven requester (ADR 005 §Open-time refusal classes).
Visible name: decdn_serve_stream_rejected_insufficient_deposit_total.
serve_stream_rejected_pool_unconfirmed: CounterDelivery refused at admission because a wired PoolView could not confirm
the pool on-chain — the pool has no on-chain record, is closed/reclaimed, or
the admit-path getPool faulted — or could not confirm the voucher signer:
the admit-path getAuthorization faulted and the node holds no cached
registered read of the signer. Signed as NotFound, like a cache miss,
so this counter is the only place the reason lives;
kept distinct from insufficient_deposit so a real drained-pool refusal is
not conflated with an unconfirmed or unreachable pool. Visible name:
decdn_serve_stream_rejected_pool_unconfirmed_total.
serve_stream_rejected_pool_closing: CounterDelivery refused at admission because the request’s pool is Closing
on-chain: it takes no top-up, and its redemption ends at the dispute
deadline (ADR 003 §Pool solvency). Signed as Unfunded to a requester
that holds a lane on the pool, so its owner opens a new pool, and as
NotFound to anyone else. Visible name:
decdn_serve_stream_rejected_pool_closing_total.
serve_stream_rejected_signer_cap_exhausted: CounterDelivery refused at admission because the request’s voucher signer
cannot pay: it is registered on-chain with cap − spent below a serve
floor or with an expired registration (a “spent” capability the node
could never cash), or its lane’s capability is inside the node’s expiry
margin. A getAuthorization fault never counts here (see
pool_unconfirmed). Signed as Unfunded: the requester holds a verified
binding for a registered signer or a lane, so it is proven (ADR 005
§Open-time refusal classes). A rising value flags capabilities presented
whose signer has drained its shared cap at other nodes. Visible name:
decdn_serve_stream_rejected_signer_cap_exhausted_total.
serve_stream_midstream_signer_cap_exhausted: CounterA live serve was stopped MID-STREAM because the request’s voucher signer
drained its shared on-chain cap at other nodes AFTER admission — the
event-fed projection’s signer_spent rose until the cap − spent headroom
the node holds on the lane no longer covers a serve floor (ADR 003 §Pool
solvency). The mid-stream twin of serve_stream_rejected_signer_cap_exhausted
(which is the admit-time refusal), kept distinct so a running value flags
cross-node cap drain on long streams — the large-blob over-delivery this
re-check bounds — rather than an open-time refusal. The stream stops in-band
with a VoucherRejected { SignerCapExhausted }, which the requester treats
as Unfunded from this node. Visible name:
decdn_serve_stream_midstream_signer_cap_exhausted_total.
serve_stream_rejected_signer_floor_at_cap: CounterDelivery refused because ONE capability signer hit its per-signer live
concurrency cap of un-vouchered reservation, while the pool itself can still
pay (ADR 003 §Pool solvency, per-signer floor isolation).
Signed as NotFound, because it clears by waiting rather than by funding
(ADR 005 §Open-time refusal classes), so this counter is the only place
the reason survives — a rising value means one signer runs
more concurrent un-vouchered streams than its share covers while the pool as a
whole is solvent. The two remedies differ: a pool shortfall clears with a
top-up, a signer at its share does not. Visible name:
decdn_serve_stream_rejected_signer_floor_at_cap_total.
serve_stream_rejected_pull_loop_guard: CounterWhole-blob requests from an active staker refused while this node’s own
upstream open for the blob was in progress (#2224): the guard that stops
partial holders pulling a blob from each other in a loop. Signed as
NotFound, so this counter is the only place the reason lives. A rate
that tracks node-to-node pulls of one blob shows the guard breaking loops;
the debug line refusing a whole-blob request from a node names the
requester. Visible name: decdn_serve_stream_rejected_pull_loop_guard_total.
serve_stream_rejected_range_not_satisfiable: CounterDelivery refused because the requested bounded range starts at or past
this node’s size claim for the blob (ADR 005 §Bounded byte ranges).
Signed as Declined: the size is this node’s claim, and another node
can disagree. A rising value flags clients issuing ranges past the end,
or nodes that disagree on a blob’s size. Visible name:
decdn_serve_stream_rejected_range_not_satisfiable_total.
serve_stream_rejected_blob_too_large: CounterDelivery refused because a pull-through request starts at or past this
node’s max_blob_size ceiling (ADR 005 §max_blob_size enforcement).
The node refuses before it commits to the stream, so no byte is bought
upstream. Signed as Declined. A rising value means clients ask
this node for blobs above its ceiling. Visible name:
decdn_serve_stream_rejected_blob_too_large_total.
serve_stream_rejected_hash_denied: CounterDelivery refused because the blob is on this operator’s local denylist
(ADR 011 §Local Denylist). Signed as Declined. Deliberately
counts ONLY the local list; the governance blacklist has its own
counter, Self::serve_stream_rejected_chain_hash_denied. Splitting
them here is safe where the wire code must not: this is the operator’s
own gauge of their own denylist, not something a client can probe. A
rising value after a takedown is the confirmation the order is being
discharged. Visible name:
decdn_serve_stream_rejected_hash_denied_total.
serve_stream_rejected_chain_hash_denied: CounterDelivery refused because the blob is on the governance blacklist (ADR
011 §On Blacklist Event). Also signed as Declined — identically
to the local list, which is the ADR’s requirement — so this counter is
the only place the two are distinguishable, and it is readable by the
operator alone. Visible name:
decdn_serve_stream_rejected_chain_hash_denied_total.
Not to be confused with serve_stream_rejected_evicted_since_probe,
which sees only evictions with no blacklist entry behind them (corruption
recovery, a manual decdn node evict).
serve_stream_rejected_origin_denied: CounterDelivery refused because the channel’s funding address is blacklisted as
an origin — local denied_origins or the on-chain ContentBlacklist
(ADR 011 §On Blacklist Event). Signed as Declined. Visible
name: decdn_serve_stream_rejected_origin_denied_total.
serve_stream_rejected_foreign_declined: CounterDelivery declined by the origin-only policy (#1759,
cache.relay_foreign_namespaces = false): a memoized live probe of this
node’s own backend genuinely does not hold the hash. Signed as
Declined: this node will not relay the hash, and another node may
(ADR 005 §Open-time refusal classes). Distinct from a backend FAULT
during that same probe, which is never counted here: a fault is not an
absence and lands in
Self::serve_stream_rejected_internal_error instead. Visible name:
decdn_serve_stream_rejected_foreign_declined_total.
serve_stream_rejected_chain_stale: CounterA serve refused because the node has been unable to reach the chain for
longer than blockchain.chain_staleness_grace_sec (ADR 011 § Serving
while chain-stale): its deny-set, pool-solvency, and signer-cap guards
are all reading stale state, so it refuses rather than sign a serve it
cannot vouch for. Signed as NotFound, identically to
Self::serve_stream_rejected_cache_miss — the client should re-route
to a peer whose chain reads are live — so this counter is the only place
an operator sees the staleness refusals. Visible name:
decdn_serve_stream_rejected_chain_stale_total.
serve_stream_terminated_takedown: CounterAn ALREADY-RUNNING delivery cut off at an MB boundary because a takedown
landed after the stream opened (ADR 011 §On Blacklist Event). Visible
name: decdn_serve_stream_terminated_takedown_total.
Distinct from the rejected_* family above, which counts refusals at
stream open. This one is the operator’s evidence that the compliance
window was honored for traffic already in flight — the case that would
otherwise keep a multi-GB blob flowing for minutes after the order took
effect, which is the slashable one.
serve_stream_rejected_load_shed_hit: Counterdecdn_serve_stream_rejected_load_shed_hit_total: cache-hit serves shed
under node overload (egress saturation).
serve_stream_rejected_load_shed_miss: Counterdecdn_serve_stream_rejected_load_shed_miss_total: cache-miss serves shed
under node overload (concurrency pressure or per-client fairness).
serve_frame_accounting_fault: Counterdecdn_serve_frame_accounting_fault_total: a serve leg refused to cut or
frame a ChunkData because its own byte accounting did not add up — a zero
frame target, a queued/queue desync, a payload whose chunks disagree with
the length the header would declare, or a header the encoder rejected.
Every one of those is a node-side bug, never peer behaviour, and each aborts
the delivery without StreamEnd so the client neither receives a mislabelled
frame nor pays the closing voucher. The refusals are correct; this counter
exists because they are otherwise invisible — the stream simply ends, which
reads to an operator as a client that hung up. Any nonzero value is a
latent-bug report, not a degradation: alert on > 0 and file it rather
than tuning anything.
serve_stream_node_fault: Counterdecdn_serve_stream_node_fault_total: a cdn/client/v1 delivery ended on a
fault this node caused — an encode fault, an alignment error, a store fault,
a framing fault — rather than on a peer hang-up or a client payment fault.
The client sees only a short stream, so without this counter the whole class
is visible solely in the error! log line beside it. Any nonzero value is a
latent-bug report, not a degradation: alert on > 0 and file it. It is a
superset of decdn_serve_frame_accounting_fault_total, which meters one of
the four classes on its own.
serve_stream_proof_budget_exhausted: Counterdecdn_serve_stream_proof_budget_exhausted_total: a cdn/client/v1
delivery ended because the payer sent MAX_PROOFS_PER_CHUNK proofs for one
chunk and none of them settled it. Counted for both the cache-hit and the
miss serve loop; on the miss loop the same exit also counts on
decdn_node_pull_through_client_abandoned_total.
Each such proof credits part of the chunk or nothing, so a payer that
answers one chunk with zero-credit vouchers or partial payments ends here.
This budget is what stops such a payer from holding a stream open (#2132).
It is the payer’s fault, not a node fault, so the serve loop logs it only
at debug! and ends the stream as a stop; this counter is what makes the
rate visible at the default log level. Each bump ends exactly one inbound
stream as failed.
serve_stream_rejected_stream_cap_full: Counterdecdn_serve_stream_rejected_stream_cap_full_total: cdn/client/v1
streams reset with RATE_LIMITED because the connection already had
max_concurrent_streams streams in flight. No signed response is sent.
A client that opens more concurrent streams than the cap drives it.
serve_stream_request_unreadable: Counterdecdn_serve_stream_request_unreadable_total: cdn/client/v1 streams
reset because the node could not read the first request: the read timed
out, the peer reset the stream or dropped the connection, or the frame or
its extensions failed to decode. Peer-side, logged at debug!.
serve_stream_voucher_rejected: Counterdecdn_serve_stream_voucher_rejected_total: paying cdn/client/v1
streams the node stopped because it rejected a voucher or a chunk
preimage: for example a wrong pool or signer, a bad signature, an
underpayment, a regression, an expired capability, or a voucher that fails
the rate check. Counted for both the cache-hit and the miss serve loop; on
the miss loop the same exit also counts on
decdn_node_pull_through_client_abandoned_total. A rejection counts here
even when the peer leaves before it reads the reject frame.
serve_stream_client_declined: Counterdecdn_serve_stream_client_declined_total: cdn/client/v1 streams the
requester left before any voucher credited a byte. The routine shapes are a
requester that reads the signed StreamResponse and does not take the
quote, and the header handshake a downstream node’s miss pull opens and
closes without adopting. A peer that sends a malformed proof before paying
counts here too. The reference requester closes every stream with code 0,
and the node does not read the code, so it sees only that the peer left,
not why. Peer-side, logged at debug!.
serve_stream_client_abandoned: Counterdecdn_serve_stream_client_abandoned_total: paying cdn/client/v1
streams the requester left or broke the protocol on after at least one
voucher credited bytes: it stopped reading, reset the stream, dropped the
connection, stopped sending proofs, or sent a malformed proof. A fully paid
stream whose StreamEnd write fails counts here too. Peer-side, logged at debug!. A sustained rise against
decdn_streams_completed_total{direction="inbound"} flags clients that
give up mid-delivery.
load_shed_egress_bps: Gaugedecdn_load_shed_egress_bps: current measured egress EWMA, bytes/sec.
load_shed_pressure_active: Gaugedecdn_load_shed_pressure_active: 1 while the load-shed policy considers
the node pressured, else 0.
load_shed_streams_in_flight: Gaugedecdn_load_shed_streams_in_flight: node-wide serves in flight, sampled
from the shed controller’s live counter beside the egress EWMA. Read it
against the configured high-water mark to see how close the node is to
shedding on concurrency — the pressure_active gauge only says whether
the mark is crossed, not the headroom.
load_shed_refused_node_at_capacity: Counterdecdn_load_shed_refused_node_at_capacity_total: new serves shed because
node-wide concurrency was above the high-water mark. The
crate::load_shed::ShedReason::NodeAtCapacity sibling — per
adr/appendix-observability.md § Reason splits each shed reason is a
sibling counter, because the remedy differs: this one says add capacity or
raise max_concurrent_serves_high. Orthogonal to the hit/miss split in
serve_stream_rejected_load_shed_{hit,miss}, which counts the same
refusals by cache class.
load_shed_refused_egress_saturated: Counterdecdn_load_shed_refused_egress_saturated_total: new serves shed because
measured egress reached the configured budget
(crate::load_shed::ShedReason::EgressSaturated). Remedy: raise
egress_budget_mbps or add bandwidth. See
Self::load_shed_refused_node_at_capacity.
load_shed_refused_client_at_capacity: Counterdecdn_load_shed_refused_client_at_capacity_total: new serves shed
because one client already held its fair share while the node was
pressured (crate::load_shed::ShedReason::ClientAtCapacity). This is
fairness working as designed, not node distress — remedy is usually none.
See Self::load_shed_refused_node_at_capacity.
warming_speculative_blocked: Counterdecdn_warming_speculative_blocked_total (ADR 041): times an above-floor
(speculative) warming buy was downgraded to the amortized floor because
the upstream source’s per-source allowance was spent. Counted only when
warming is enabled (warming_budget > 0), so a node with warming off does
not report every source as blocked. A sustained rate means one or more
sources are being griefed down — pair it with
Self::warming_sources_blocked for how many.
warming_sources_blocked: Gaugedecdn_warming_sources_blocked (ADR 041): how many upstream sources
currently hold no warming allowance (spent ledger, refill projected
forward), sampled beside the egress EWMA. A rising gauge with a rising
Self::warming_speculative_blocked is warming being throttled by real
per-source losses; either at zero means warming is unconstrained.
slash_watcher_last_tick_timestamp_seconds: Gaugedecdn_slash_watcher_last_tick_timestamp_seconds (#1316): Unix time of
the slash watcher’s last successful poll tick, stamped every tick (not
edge-triggered). Alert on staleness — time() - value > 3 × poll_interval — to catch a panicked, wedged, or cleanly-exited task that
slash_watcher_down_seconds (error-triggered) cannot. 0 until the
first successful tick.
staker_set_watcher_last_tick_timestamp_seconds: Gaugedecdn_staker_set_watcher_last_tick_timestamp_seconds (#1316): Unix time
of the staker-set watcher’s last successful poll tick. Staleness
semantics as slash_watcher_last_tick_timestamp_seconds.
capacity_bond_registry_last_resync_timestamp_seconds: Gaugedecdn_capacity_bond_registry_last_resync_timestamp_seconds: Unix time
of the capacity-bond registry’s last successful re-enumeration of
getRegisteredNodes.
Distinct from the tick gauge above: a tick is the event tail, this is the
authoritative re-read that repairs a drifted set. It is the only signal
that catches a resync being skipped — the reconcile does not run at all
while the route is errored, which emits nothing, not even the failure
counter. Stamped by the bootstrap enumeration too, which is the same read
against the same contract; without that a node whose every later resync
fails would hold the gauge at 0 and never trip a > 0-guarded alert.
The guard is still needed for the window before bootstrap completes, and
the threshold must exceed REGISTRY_RESYNC_INTERVAL.
blacklist_watcher_last_tick_timestamp_seconds: Gaugedecdn_blacklist_watcher_last_tick_timestamp_seconds (#1316, #1320): Unix
time of the blacklist watcher’s last successful poll tick. This is the
signal that distinguishes a live blacklist loop from a dead one — a dead
loop keeps serving content blacklisted after the failure (slashable via
SlashJudge.submitBlacklistChallenge) while blacklist_watcher_down_seconds
reads 0.
settlement_watcher_last_tick_timestamp_seconds: Gaugedecdn_settlement_watcher_last_tick_timestamp_seconds (#1316): Unix time
of the payment-settlement watcher’s last successful poll tick.
decdn_fee_shares_watcher_last_tick_timestamp_seconds: Unix time of the
fee-shares watcher’s last successful poll tick. Load-bearing: the
watcher’s getShares() poll failure and undecodable-log paths both
return Ok by design, so a watcher stuck in RPC backoff — or dead — is
otherwise indistinguishable from a healthy one while the node keeps
signing quotes against a stale operator fee share. Alert on this gauge
going stale.
slash_watcher_task_panicked: Counterdecdn_slash_watcher_task_panicked_total (#1316): the slash watcher task
unwound on a panic. Bumped from a Drop guard in multiplexed_poller::run
— the only thing that still runs on the unwind, since nothing awaits the
detached task. Any non-zero value is a bug in this node.
staker_set_watcher_task_panicked: Counterdecdn_staker_set_watcher_task_panicked_total (#1316): the staker-set
watcher task unwound on a panic. See slash_watcher_task_panicked.
blacklist_watcher_task_panicked: Counterdecdn_blacklist_watcher_task_panicked_total (#1316, #1283): the
blacklist watcher task unwound on a panic.
decdn_fee_shares_watcher_task_panicked_total: the fee-shares watcher
task unwound on a panic. Any non-zero value is a bug in this node.
settlement_watcher_task_panicked: Counterdecdn_settlement_watcher_task_panicked_total (#1316): the
payment-settlement watcher task unwound on a panic.
chain_get_logs_span: Gaugedecdn_chain_get_logs_span: block span of the chain-event poller’s next
eth_getLogs window. It starts at blockchain.get_logs_max_block_span,
drops to half a window the RPC provider rejects for its range, and
doubles back after 32 accepted windows, up to the ceiling. A caught-up
node climbs back to the ceiling after a transient rejection, so a low
value is a recent rejection, not proof of a cap: read the rejection
counter for that. The lowest values over a range with rejections show
the provider’s cap.
chain_get_logs_range_rejections: Counterdecdn_chain_get_logs_range_rejections_total: eth_getLogs windows the
RPC provider rejected for their block range or result count. Each one
costs one wasted request. Rejections that keep coming mean the provider
caps eth_getLogs; set blockchain.get_logs_max_block_span to the
lowest span the gauge reaches while they come, and they stop.
chain_get_logs_retries: Counterdecdn_chain_get_logs_retries_total (#2161): in-tick retries of
eth_getLogs windows after a transient provider error. One window adds
up to two, including a window that fails every retry and is deferred
(decdn_chain_get_logs_deferred_total). A retry that succeeds does not
count as watcher downtime, so a provider that the retries absorb shows
here and not in the watchers’ down-family. Rate limits and permanent
errors are not retried.
chain_get_logs_deferred: Counterdecdn_chain_get_logs_deferred_total (#2253): eth_getLogs windows
that failed every in-tick retry, so the poller ended the tick there and
resumed the window on the next tick. A deferral is not watcher downtime;
the tick fails, and the watchers go down, only when no tick has reached
head for the stall budget (2 min).
chain_boot_read_retries: Counterdecdn_chain_boot_read_retries_total (#2159): retries of a boot-time
chain read (the registry, slash, usdc() self-check and blacklist
bootstraps, and the best-effort fee-share reads) after a transient
error; one read can retry many times. The metrics
listener binds after these reads, so the value becomes visible once boot
completes. A rise after a restart points at the RPC provider.
blacklist_watcher_restarts: Counterdecdn_blacklist_watcher_restarts_total (#1283): distinct drift windows
the blacklist watcher entered, bumped once on the edge into the
error/backoff state. Mirrors slash_watcher_restarts.
blacklist_watcher_down_seconds: Gaugedecdn_blacklist_watcher_down_seconds (#1283): seconds the blacklist
watcher has been failing its chain read (0 on a healthy cycle),
recomputed at scrape from blacklist_watcher_down_since. Tracks
chain-read outages only — an enforcement failure that leaves a blacklisted
blob servable is a separate signal (blacklist_enforcement_failures).
settlement_watcher_restarts: Counterdecdn_settlement_watcher_restarts_total (#1316): distinct drift windows
the payment-settlement watcher entered. Mirrors slash_watcher_restarts.
settlement_watcher_down_seconds: Gaugedecdn_settlement_watcher_down_seconds (#1316): seconds the settlement
watcher has been failing its chain read, recomputed at scrape from
settlement_watcher_down_since.
blacklist_enforcement_failures: Counterdecdn_blacklist_enforcement_failures_total (#1319): distinct hashes a
batched re-scope could NOT re-verify or evict this pass (Recheck::Failed
— a disk error or a scope eth_call failure). An increase means a pass
did not fully enforce the deny-set, so a blacklisted blob may still be
servable and slashable, even while blacklist_watcher_down_seconds reads
0. Answers a different question than the down-family, which tracks
chain-read outages only. A boot that retries its enforcement pass also
adds to it; the node serves only after a clean pass, so that increase does
not mean a served blob. Pairs with the aggregate warn! in rescan.
blacklist_reenumeration_failures: Counterdecdn_blacklist_reenumeration_failures_total: times the blacklist
watcher’s periodic re-enumeration could not read chain state and kept the
deny-set it already had. That re-enumeration is the backstop for a lost
tail event and for a region/ripening transition, which emits no event.
It reports success upward so the event tail keeps running, so a
persistently failing backstop moves nothing else. Pairs with the warn!
in blacklist_watcher’s BlacklistSink::on_tick_complete.
bytes_served: Counterdecdn_bytes_served_total: payload bytes this node wrote to clients and
downstream nodes on cdn/client/v1, counted per frame as it is written.
bytes_received: Counterdecdn_bytes_received_total: payload bytes this node admitted from
upstream nodes in paid node-to-node pulls, counted per verified range
(chunk-group aligned, so it can exceed the requested bytes).
pool_redemptions: Counterdecdn_pool_redemptions_total: lane claims this node redeemed on-chain in
a landed redeemMany — one cumulative voucher per lane, however many
vouchers the lane accepted.
vouchers_received: Counterdecdn_vouchers_received_total: signed vouchers this node accepted as
the payee on an inbound stream. PayWord preimage reveals are not
vouchers and do not count; a PayWord stream counts its anchor voucher.
preimage_reveals_received: Counterdecdn_preimage_reveals_received_total: PayWord hash-chain preimage
reveals this node accepted as the payee on an inbound stream, counted
when a reveal advances the lane’s chain frontier. Each one also hints the
redeemer, so this rate bounds the redeemer’s hint rate beside
decdn_vouchers_received_total.
pool_grace_closes: Counterdecdn_pool_grace_closes_total: pools that entered the owner-close grace
window while this node still held unredeemed vouchers on them, read from
the flushed lane store when the close event arrives. Each one is revenue
that must redeem before the grace window ends. A pool already closing at
startup is not counted.
onchain_tx_landed: Counterdecdn_onchain_tx_landed_total: settlement transactions (redeemMany)
mined and succeeded. The outcome family — landed, reverted,
send_failed, receipt_failed, timeout — counts every transaction
sent through send_and_await_receipt exactly once;
onchain_tx_receipt_recovered sits beside it and is not an outcome.
Buyer-side pool transactions go through decdn-client and are not
counted here.
onchain_tx_reverted: Counterdecdn_onchain_tx_reverted_total: node transactions mined and reverted.
onchain_tx_send_failed: Counterdecdn_onchain_tx_send_failed_total: node transactions the RPC refused at
send — none was issued. An oversize redeemMany the redeemer then
halves and retries counts here once too.
onchain_tx_receipt_failed: Counterdecdn_onchain_tx_receipt_failed_total: issued node transactions whose
receipt wait failed and whose receipt no by-hash fetch found. The
transaction is unconfirmed and may still mine.
onchain_tx_timeout: Counterdecdn_onchain_tx_timeout_total: issued node transactions whose receipt
did not arrive inside the caller’s bound and whose receipt no by-hash
fetch found. The transaction is unconfirmed and may still mine.
onchain_tx_receipt_recovered: Counterdecdn_onchain_tx_receipt_recovered_total: node transactions whose
receipt wait failed or timed out but whose receipt a by-hash fetch then
found. Each one also counts once as landed or reverted, so this sits
beside the outcome family, not inside it. A steady rate points at a
lagging or load-balanced RPC provider.
dht_findvalue_queries: Counterdecdn_dht_findvalue_queries_total: FIND_VALUE lookups this node ran
to discover providers.
dht_lookup_round_timeouts: Counterdecdn_dht_lookup_round_timeouts_total: lookup rounds that hit the round
timeout and aborted their in-flight RPCs.
dht_store_published: Counterdecdn_dht_store_published_total: DHT STORE records a peer accepted
from this node.
dht_routing_table_size: Gaugedecdn_dht_routing_table_size: distinct entries in the local Kademlia
routing table, set after bootstrap and on every bucket-refresh tick.
dht_bucket_refresh_failures: Counterdecdn_dht_bucket_refresh_failures_total: bucket refreshes that failed:
a FIND_NODE RPC error, a routing-table peer that is not a valid public
key, or a panicked refresh task.
dht_bootstrap_find_node_failures: Counterdecdn_dht_bootstrap_find_node_failures_total: bootstrap FIND_NODE
RPCs against a seed that failed.
decdn_fee_shares_watcher_unregistered: 1 when the startup
PaymentPool.feeRouter() read failed, so the fee-shares watcher is not
running and the operator share stays at the floor for the life of the
process; 0 otherwise. The watcher’s own down gauges read healthy in
that state, since it never started.
decdn_fee_shares_watcher_poll_failures_total: authoritative
getShares() re-reads that failed. The watcher keeps the current share and
its tick still succeeds, so this is the only signal that the safety-net
re-read is not landing.
decdn_fee_shares_watcher_restarts_total: distinct drift windows the
fee-shares watcher entered. Mirrors slash_watcher_restarts.
decdn_fee_shares_watcher_down_seconds: seconds the fee-shares watcher
has been failing its chain read, recomputed at scrape from
fee_shares_watcher_down_since.
config_reload_failures: Counterdecdn_config_reload_failures_total: SIGHUP or admin config reloads that
failed. Sections committed before the failure stay applied; the rest keep
their previous values.
receipt_write_failures: Counterdecdn_receipt_write_failures_total: download-receipt audit records the
writer failed to persist (an I/O error or a panicked write task). Audit
only; settlement is unaffected.
serve_stream_midstream_pool_exhausted: Counterdecdn_serve_stream_midstream_pool_exhausted_total: paying streams this
node stopped mid-delivery because the pool could no longer fund the
floor credit committed across its signers. The pool-level sibling of
serve_stream_midstream_signer_cap_exhausted.
Trait Implementations§
Source§impl Debug for DecdnMetrics
impl Debug for DecdnMetrics
Source§impl Default for DecdnMetrics
impl Default for DecdnMetrics
Source§impl<'de> Deserialize<'de> for DecdnMetrics
impl<'de> Deserialize<'de> for DecdnMetrics
Source§fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where
__D: Deserializer<'de>,
fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where
__D: Deserializer<'de>,
Source§impl Iterable for DecdnMetrics
impl Iterable for DecdnMetrics
Source§fn metric_field_count(&self) -> usize
fn metric_field_count(&self) -> usize
Source§fn metric_field_ref(&self, n: usize) -> Option<MetricItem<'_>>
fn metric_field_ref(&self, n: usize) -> Option<MetricItem<'_>>
Source§fn family_field_count(&self) -> usize
fn family_field_count(&self) -> usize
Family fields in the struct.Source§fn family_field_ref(&self, n: usize) -> Option<FamilyItem<'_>>
fn family_field_ref(&self, n: usize) -> Option<FamilyItem<'_>>
Family field at the given index.Source§impl MetricsGroup for DecdnMetrics
impl MetricsGroup for DecdnMetrics
Auto Trait Implementations§
impl !Freeze for DecdnMetrics
impl RefUnwindSafe for DecdnMetrics
impl Send for DecdnMetrics
impl Sync for DecdnMetrics
impl Unpin for DecdnMetrics
impl UnsafeUnpin for DecdnMetrics
impl UnwindSafe for DecdnMetrics
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
impl<T> DeserializeOwned for Twhere
T: for<'de> Deserialize<'de>,
Source§impl<T> FutureExt for T
impl<T> FutureExt for T
Source§fn with_context(self, otel_cx: Context) -> WithContext<Self> ⓘ
fn with_context(self, otel_cx: Context) -> WithContext<Self> ⓘ
Source§fn with_current_context(self) -> WithContext<Self> ⓘ
fn with_current_context(self) -> WithContext<Self> ⓘ
Source§impl<T> Instrument for T
impl<T> Instrument for T
Source§fn instrument(self, span: Span) -> Instrumented<Self> ⓘ
fn instrument(self, span: Span) -> Instrumented<Self> ⓘ
Source§fn in_current_span(self) -> Instrumented<Self> ⓘ
fn in_current_span(self) -> Instrumented<Self> ⓘ
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§impl<T> IntoIterable for Twhere
T: Iterable,
impl<T> IntoIterable for Twhere
T: Iterable,
Source§fn as_iterable(&self) -> &dyn Iterable
fn as_iterable(&self) -> &dyn Iterable
self as dyn IterableSource§fn field_iter(&self) -> FieldIter<'_> ⓘ
fn field_iter(&self) -> FieldIter<'_> ⓘ
Source§fn family_iter(&self) -> FamilyIter<'_> ⓘ
fn family_iter(&self) -> FamilyIter<'_> ⓘ
Source§impl<T> IntoRequest<T> for T
impl<T> IntoRequest<T> for T
Source§fn into_request(self) -> Request<T>
fn into_request(self) -> Request<T>
T in a tonic::Request