<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Later Dashboard</title>
<style>
:root { color-scheme: light; font-family: ui-sans-serif, system-ui, sans-serif; color: #172033; background: #f4f6fa; }
* { box-sizing: border-box; }
body { margin: 0; }
header { padding: 1rem 1.5rem 0; color: white; background: #3157d5; box-shadow: 0 2px 8px #16295b33; }
header strong { font-size: 1.2rem; }
.header-title { padding-bottom: 1rem; }
main { display: grid; grid-template-columns: minmax(10rem, 14rem) minmax(0, 1fr); gap: 1.5rem; max-width: 90rem; margin: 1.5rem auto; padding: 0 1.5rem; }
#performance { grid-column: 1 / -1; display: grid; grid-template-columns: repeat(auto-fit, minmax(9rem, 1fr)); gap: .75rem; }
.metric { min-height: 6rem; padding: 1rem; border: 1px solid #dce1ec; border-radius: .65rem; background: white; box-shadow: 0 2px 10px #1720330d; }
.metric-label { display: block; color: #59657a; font-size: .76rem; text-transform: uppercase; letter-spacing: .04em; }
.metric-value { display: block; margin-top: .35rem; font-size: 1.65rem; font-weight: 650; }
.metric-note { display: block; margin-top: .2rem; color: #6f7a8e; font-size: .75rem; }
nav, section { background: white; border: 1px solid #dce1ec; border-radius: .65rem; box-shadow: 0 2px 10px #1720330d; }
nav { padding: .5rem; align-self: start; }
button { font: inherit; cursor: pointer; }
#tabs { display: flex; gap: .25rem; align-items: center; }
#tabs[hidden] { display: none; }
.tab { padding: .6rem 1.1rem; border: 0; border-bottom: 3px solid transparent; border-radius: .4rem .4rem 0 0; color: #c8d3f0; background: transparent; }
.tab:hover { color: white; }
.tab[aria-current="true"] { color: white; border-bottom-color: white; }
#topics-menu { position: relative; }
#topics-menu summary { list-style: none; }
#topics-menu summary::-webkit-details-marker { display: none; }
#topics-menu summary::after { content: "▾"; margin-left: .35rem; font-size: .75rem; }
#topic-list { position: absolute; z-index: 1; top: calc(100% + .25rem); left: 0; display: grid; min-width: 12rem; max-height: 16rem; overflow-y: auto; padding: .35rem; border: 1px solid #dce1ec; border-radius: .4rem; background: white; box-shadow: 0 6px 18px #17203333; }
.topic-option { display: flex; align-items: center; justify-content: space-between; gap: .6rem; padding: .55rem .7rem; border: 0; border-radius: .3rem; color: #30394c; background: transparent; text-align: left; }
.topic-option:hover, .topic-option[aria-current="true"] { color: #2145bd; background: #edf1ff; }
.lag-badge { min-width: 1.6rem; padding: .05rem .45rem; border-radius: 1rem; font-size: .74rem; color: white; background: #68748d; text-align: center; }
.lag-badge[data-zero="true"] { background: #8b95a8; opacity: .55; }
.lag-badge[hidden] { display: none; }
#partition-picker { display: flex; flex-direction: column; gap: .35rem; padding: .5rem .8rem .9rem; border-bottom: 1px solid #e5e8ef; margin-bottom: .5rem; }
#partition-picker[hidden] { display: none; }
#partition-picker label { color: #59657a; font-size: .76rem; text-transform: uppercase; letter-spacing: .04em; }
#partition-picker select { padding: .45rem; border: 1px solid #bdc5d6; border-radius: .35rem; font: inherit; }
.stage { display: flex; width: 100%; justify-content: space-between; padding: .7rem .8rem; border: 0; border-radius: .4rem; color: #30394c; background: transparent; text-align: left; }
.stage:hover, .stage[aria-current="true"] { color: #2145bd; background: #edf1ff; }
.count { min-width: 2rem; padding: .1rem .5rem; border-radius: 1rem; color: white; background: #68748d; text-align: center; }
section { min-height: 16rem; overflow: hidden; }
#notice { margin: 1rem; padding: .8rem 1rem; border-radius: .4rem; background: #edf1ff; }
#notice.error { color: #851f2a; background: #ffeaed; }
.table-scroll { overflow-x: auto; overscroll-behavior-x: contain; -webkit-overflow-scrolling: touch; }
table { width: 100%; min-width: 42rem; border-collapse: collapse; }
th, td { padding: .75rem 1rem; border-bottom: 1px solid #e5e8ef; text-align: left; vertical-align: top; }
th { color: #59657a; background: #fafbfc; font-size: .8rem; text-transform: uppercase; letter-spacing: .04em; }
code { overflow-wrap: anywhere; }
.job-link { padding: 0; border: 0; color: #2145bd; background: none; text-align: left; }
footer { display: flex; justify-content: space-between; align-items: center; padding: 1rem; }
footer button, dialog button { padding: .45rem .8rem; border: 1px solid #bdc5d6; border-radius: .35rem; background: white; }
footer button:disabled { cursor: default; opacity: .45; }
dialog { width: min(46rem, calc(100% - 2rem)); border: 0; border-radius: .7rem; box-shadow: 0 18px 60px #17203355; }
dialog::backdrop { background: #17203388; }
.dialog-head { display: flex; align-items: start; justify-content: space-between; gap: 1rem; }
.detail-navigation { display: flex; align-items: center; gap: .6rem; margin: 1rem 0; }
.detail-navigation span { color: #59657a; font-size: .85rem; }
.history { padding-left: 1.25rem; }
.relations { display: grid; gap: .5rem; }
.relations p { margin: 0; }
.payload { background: #f4f6fa; border: 1px solid #e5e8ef; border-radius: .35rem; padding: .6rem .8rem; font-size: .8rem; overflow-x: auto; white-space: pre-wrap; word-break: break-word; }
.payload-note { color: #59657a; font-size: .76rem; margin: .3rem 0 0; }
.relation-links { display: flex; flex-wrap: wrap; gap: .4rem; }
#monitoring { grid-column: 1 / -1; overflow: visible; }
#monitoring[hidden] { display: none; }
.monitoring-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(19rem, 1fr)); gap: 1rem; padding: 1.1rem; }
.chart-card h3 { margin: 0 0 .1rem; font-size: .85rem; color: #30394c; }
.chart-note { margin: 0 0 .6rem; color: #8b95a8; font-size: .74rem; }
.chart-card canvas { display: block; width: 100%; height: 9.5rem; }
.chart-legend { display: flex; flex-wrap: wrap; gap: .4rem .9rem; margin-top: .5rem; font-size: .76rem; color: #59657a; }
.chart-legend .swatch { display: inline-block; width: .6rem; height: .6rem; margin-right: .35rem; border-radius: 50%; vertical-align: middle; }
@media (max-width: 700px) { main { grid-template-columns: 1fr; padding: 0 .75rem; } }
</style>
</head>
<body>
<header>
<div class="header-title"><strong>later</strong> job dashboard</div>
<div id="tabs" aria-label="Job views" hidden>
<button id="all-jobs" class="tab" type="button">All</button>
<details id="topics-menu">
<summary class="tab">Topics</summary>
<div id="topic-list" role="menu" aria-label="Topics"></div>
</details>
<button id="monitoring-tab" class="tab" type="button" hidden>Monitoring</button>
</div>
</header>
<main>
<div id="performance" aria-label="Performance over the last 60 seconds">
<div class="metric"><span class="metric-label">Completed / sec</span><span class="metric-value" id="jobs-per-second">0.00</span><span class="metric-note">60 second average</span></div>
<div class="metric"><span class="metric-label">Active workers</span><span class="metric-value" id="active-workers">0</span><span class="metric-note">heartbeat in last 15s</span></div>
<div class="metric"><span class="metric-label">Busy workers</span><span class="metric-value" id="busy-workers">0</span><span class="metric-note">jobs currently running</span></div>
<div class="metric"><span class="metric-label">Started</span><span class="metric-value" id="started">0</span><span class="metric-note">last 60 seconds</span></div>
<div class="metric"><span class="metric-label">Succeeded</span><span class="metric-value" id="succeeded">0</span><span class="metric-note">last 60 seconds</span></div>
<div class="metric"><span class="metric-label">Retried</span><span class="metric-value" id="retried">0</span><span class="metric-note">last 60 seconds</span></div>
<div class="metric"><span class="metric-label">Failed</span><span class="metric-value" id="failed">0</span><span class="metric-note">last 60 seconds</span></div>
<div class="metric"><span class="metric-label">Regular queue wait</span><span class="metric-value" id="wait-regular">-</span><span class="metric-note" id="wait-regular-note">avg, last 60s</span></div>
<div class="metric"><span class="metric-label">Sequential queue wait</span><span class="metric-value" id="wait-sequential">-</span><span class="metric-note" id="wait-sequential-note">avg, last 60s</span></div>
<div class="metric"><span class="metric-label">Database size</span><span class="metric-value" id="db-size">-</span><span class="metric-note">entire database, not just Later's tables</span></div>
</div>
<nav id="stages" aria-label="Job stages">
<div id="partition-picker" hidden>
<label for="partition-select">Partition</label>
<select id="partition-select"></select>
</div>
<div id="stage-buttons"></div>
</nav>
<section id="jobs-section" aria-live="polite">
<div id="notice">Loading jobs...</div>
<div id="job-table" class="table-scroll" tabindex="0" aria-label="Jobs table. Scroll horizontally to see all columns." hidden>
<table id="jobs">
<thead><tr><th>ID</th><th class="type">Type</th><th class="seq">Seq</th><th class="date">Date</th><th>Duration</th><th>Stage</th></tr></thead>
<tbody></tbody>
</table>
</div>
<footer id="paging" hidden>
<button id="previous" type="button">Previous</button>
<span id="page-label"></span>
<button id="next" type="button">Next</button>
</footer>
</section>
<section id="monitoring" aria-label="Monitoring" hidden>
<div id="monitoring-grid" class="monitoring-grid"></div>
</section>
</main>
<dialog id="details">
<div class="dialog-head"><h2>Job details</h2><button id="close" type="button">Close</button></div>
<div class="detail-navigation" aria-label="Jobs in the current view">
<button id="detail-previous" type="button">Previous job</button>
<span id="detail-position"></span>
<button id="detail-next" type="button">Next job</button>
</div>
<p><strong>ID:</strong> <code id="detail-id"></code></p>
<p><strong>Type:</strong> <code id="detail-type"></code></p>
<p><strong>Current stage:</strong> <code id="detail-stage"></code></p>
<p><strong>Total duration:</strong> <code id="detail-total-duration"></code></p>
<p><strong>Processing:</strong> <code id="detail-processing-duration"></code></p>
<p><strong>Waiting:</strong> <code id="detail-waiting-duration"></code></p>
<p id="detail-partition-row" hidden><strong>Partition:</strong> <code id="detail-partition"></code></p>
<h3>Related jobs</h3>
<div id="relations" class="relations"></div>
<h3>Payload</h3>
<pre id="detail-payload" class="payload"></pre>
<p id="detail-payload-note" class="payload-note" hidden>
Shown as a positional array: fields are in declaration order, not
named, because payloads are stored without field name metadata.
</p>
<h3>Previous stages</h3>
<ol id="history" class="history"></ol>
</dialog>
<script>
const BASE_APP = __LATER_DASHBOARD_BASE__;
const TOPICS = __LATER_DASHBOARD_TOPICS__;
const LAG_TOPICS = __LATER_DASHBOARD_LAG_TOPICS__;
const METRICS_ENABLED = __LATER_DASHBOARD_METRICS_ENABLED__;
const MENU_TOPICS = TOPICS.map(t => t.name)
.concat(LAG_TOPICS.filter(name => !TOPICS.some(t => t.name === name)));
const STAGE_ORDER = ["delayed", "waiting", "enqueued", "running", "requeued", "success", "failed"];
const state = {
mode: "stages", counts: {}, stage: "", topic: null, partition: 0,
cursor: null, nextCursor: null, history: [], visibleJobs: [],
selectedJobIndex: null, loading: false,
consumerLag: {},
queueDepthTotals: {},
queueDepthByPartition: {}
};
const elements = {
tabs: document.querySelector("#tabs"), allJobs: document.querySelector("#all-jobs"),
topicsMenu: document.querySelector("#topics-menu"), topicList: document.querySelector("#topic-list"),
stages: document.querySelector("#stage-buttons"),
partitionPicker: document.querySelector("#partition-picker"),
partitionSelect: document.querySelector("#partition-select"),
notice: document.querySelector("#notice"),
jobTable: document.querySelector("#job-table"), body: document.querySelector("#jobs tbody"),
paging: document.querySelector("#paging"), previous: document.querySelector("#previous"),
next: document.querySelector("#next"), pageLabel: document.querySelector("#page-label"),
details: document.querySelector("#details"),
detailPrevious: document.querySelector("#detail-previous"),
detailNext: document.querySelector("#detail-next"), detailPosition: document.querySelector("#detail-position"),
performance: document.querySelector("#performance"), jobsSection: document.querySelector("#jobs-section"),
monitoringTab: document.querySelector("#monitoring-tab"), monitoring: document.querySelector("#monitoring"),
monitoringGrid: document.querySelector("#monitoring-grid")
};
const CHART_COLORS = ["#2145bd", "#dc6803", "#12b76a", "#7839ee", "#e11d48", "#0891b2", "#65a30d", "#c026d3"];
const MAX_METRIC_POINTS = 150; const CHART_WINDOW_MS = 2 * 60 * 1000;
let monitoringAnimationFrame = null;
const CHART_DEFS = [
{ key: "queueLength", title: "Queue length", unit: "", note: "Jobs waiting for a worker (sampled each minute, kept on the server).", fixedLabels: ["queued", "partitioned"], server: true, windowMs: 6 * 60 * 60 * 1000 },
{ key: "throughput", title: "Job throughput", unit: "/s", note: "Completed stage transitions per second.", fixedLabels: ["success", "failed", "requeued"] },
{ key: "workers", title: "Workers", unit: "", note: "Worker tasks in this process, by state.", fixedLabels: ["active", "busy"] },
{ key: "queueDepth", title: "Durable queue depth", unit: "", note: "Not-yet-completed jobs, by state." },
{ key: "handlerLatency", title: "Handler latency", unit: "ms", note: "Average time spent in job handlers.", fixedLabels: ["success", "error"] },
{ key: "partitionBacklog", title: "Partition backlog", unit: "", note: "Ready, unleased partition heads, by topic." },
{ key: "commands", title: "Worker commands", unit: "/s", note: "Delivery commands handled per second, by outcome.", fixedLabels: ["success", "error"] }
];
const metricsState = {
series: {}, prevCounter: {}, prevHistogram: {} };
function uri(command) {
return `${BASE_APP}?${new URLSearchParams({ query: JSON.stringify(command) })}`;
}
const REQUEST_TIMEOUT_MS = 20000;
async function request(command) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
let response;
try {
response = await fetch(uri(command), { signal: controller.signal });
} catch (error) {
if (error.name === "AbortError") throw new Error("Dashboard request timed out");
throw error;
} finally {
clearTimeout(timer);
}
if (!response.ok) {
let message = `Dashboard request failed (${response.status})`;
try { message = (await response.json()).error || message; } catch (_) {}
throw new Error(message);
}
return response.json();
}
function showNotice(message, error = false) {
elements.notice.textContent = message;
elements.notice.classList.toggle("error", error);
elements.notice.hidden = false;
elements.jobTable.hidden = true;
elements.paging.hidden = true;
}
function renderStages() {
const existing = elements.stages.children.length === STAGE_ORDER.length;
STAGE_ORDER.forEach((name, index) => {
const isCurrent = String(name === state.stage);
const displayCount = state.counts[name] ?? 0;
if (existing) {
const button = elements.stages.children[index];
button.setAttribute("aria-current", isCurrent);
button.querySelector(".count").textContent = displayCount;
return;
}
const button = document.createElement("button");
button.type = "button";
button.className = "stage";
button.dataset.stage = name;
button.setAttribute("aria-current", isCurrent);
const label = document.createElement("span");
label.textContent = name;
const count = document.createElement("span");
count.className = "count";
count.textContent = displayCount;
button.append(label, count);
button.addEventListener("click", () => loadStage(name, null, true));
elements.stages.append(button);
});
}
function lagBadge(name) {
const badge = document.createElement("span");
badge.className = "lag-badge";
badge.title = "Backlog: unread retained-log records plus not-yet-completed queued jobs";
badge.hidden = true;
return badge;
}
function renderTabs() {
elements.tabs.hidden = false;
const viewingAll = state.mode === "stages";
elements.allJobs.setAttribute("aria-current", String(viewingAll));
elements.topicsMenu.hidden = MENU_TOPICS.length === 0;
elements.topicsMenu.querySelector("summary").setAttribute("aria-current", String(state.mode === "partition"));
elements.monitoringTab.hidden = !METRICS_ENABLED;
elements.monitoringTab.setAttribute("aria-current", String(state.mode === "monitoring"));
const existing = elements.topicList.children.length === MENU_TOPICS.length;
MENU_TOPICS.forEach((name, index) => {
const isJobQueueTopic = TOPICS.some(t => t.name === name);
const isCurrent = String(isJobQueueTopic && name === state.topic);
if (existing) {
const row = elements.topicList.children[index];
if (isJobQueueTopic) row.setAttribute("aria-current", isCurrent);
return;
}
const label = document.createElement("span");
label.textContent = name;
if (isJobQueueTopic) {
const button = document.createElement("button");
button.type = "button";
button.className = "topic-option";
button.setAttribute("role", "menuitem");
button.setAttribute("aria-current", isCurrent);
button.append(label, lagBadge(name));
button.addEventListener("click", () => {
elements.topicsMenu.open = false;
switchToTopic(name);
});
elements.topicList.append(button);
} else {
const row = document.createElement("div");
row.className = "topic-option";
row.append(label, lagBadge(name));
elements.topicList.append(row);
}
});
}
function renderTopicBadges() {
MENU_TOPICS.forEach((name, index) => {
const row = elements.topicList.children[index];
if (!row) return;
const badge = row.querySelector(".lag-badge");
const lag = state.consumerLag[name];
const depth = state.queueDepthTotals[name];
if (lag === undefined && depth === undefined) return;
const total = (lag || 0) + (depth || 0);
badge.hidden = false;
badge.dataset.zero = String(total === 0);
badge.textContent = total;
});
}
function switchToStages() {
if (state.mode === "stages") return;
stopMonitoringAnimation();
state.mode = "stages";
state.topic = null;
elements.performance.hidden = false;
elements.jobsSection.hidden = false;
elements.monitoring.hidden = true;
elements.stages.hidden = false;
elements.partitionPicker.hidden = true;
elements.topicsMenu.open = false;
renderTabs();
loadStage(state.stage || "enqueued", null, true);
}
function switchToTopic(name) {
stopMonitoringAnimation();
state.mode = "partition";
state.topic = name;
state.partition = 0;
elements.performance.hidden = false;
elements.jobsSection.hidden = false;
elements.monitoring.hidden = true;
elements.stages.hidden = true;
renderTabs();
renderPartitionPicker();
loadPartition(name, 0, null, true);
}
function switchToMonitoring() {
if (!METRICS_ENABLED || state.mode === "monitoring") return;
state.mode = "monitoring";
elements.topicsMenu.open = false;
elements.performance.hidden = true;
elements.jobsSection.hidden = true;
elements.stages.hidden = true;
elements.partitionPicker.hidden = true;
elements.monitoring.hidden = false;
renderTabs();
CHART_DEFS.forEach(renderChart);
startMonitoringAnimation();
}
function startMonitoringAnimation() {
if (monitoringAnimationFrame !== null) return;
const tick = () => {
if (state.mode !== "monitoring") {
monitoringAnimationFrame = null;
return;
}
CHART_DEFS.forEach(def => drawChartCanvas(def));
monitoringAnimationFrame = requestAnimationFrame(tick);
};
monitoringAnimationFrame = requestAnimationFrame(tick);
}
function stopMonitoringAnimation() {
if (monitoringAnimationFrame !== null) {
cancelAnimationFrame(monitoringAnimationFrame);
monitoringAnimationFrame = null;
}
}
function renderMonitoringCards() {
elements.monitoringGrid.replaceChildren(...CHART_DEFS.map(def => {
const card = document.createElement("div");
card.className = "chart-card";
const heading = document.createElement("h3");
heading.textContent = def.title;
const note = document.createElement("p");
note.className = "chart-note";
note.textContent = def.note;
const canvas = document.createElement("canvas");
canvas.id = `chart-${def.key}`;
canvas.setAttribute("role", "img");
canvas.setAttribute("aria-label", `${def.title} chart`);
const legend = document.createElement("div");
legend.className = "chart-legend";
legend.id = `legend-${def.key}`;
if (def.server) {
const picker = document.createElement("select");
picker.setAttribute("aria-label", `${def.title} time range`);
[["1 hour", 60], ["6 hours", 360], ["24 hours", 1440], ["7 days", 10080]].forEach(([label, minutes]) => {
const option = document.createElement("option");
option.value = String(minutes);
option.textContent = `Last ${label}`;
picker.append(option);
});
picker.value = String(Math.round(def.windowMs / 60000));
picker.addEventListener("change", () => {
def.windowMs = Number(picker.value) * 60000;
refreshQueueLength();
});
card.append(heading, note, picker, canvas, legend);
return card;
}
card.append(heading, note, canvas, legend);
return card;
}));
}
function colorForLabel(label) {
let hash = 0;
for (let i = 0; i < label.length; i += 1) hash = (hash * 31 + label.charCodeAt(i)) >>> 0;
return CHART_COLORS[hash % CHART_COLORS.length];
}
function formatMetricValue(value, unit) {
if (!Number.isFinite(value)) return "-";
const rounded = value >= 100 ? Math.round(value) : Math.round(value * 100) / 100;
const text = value >= 1000 ? rounded.toLocaleString() : String(rounded);
return unit ? `${text}${unit}` : text;
}
function familyIndex(families) {
const byName = {};
(families || []).forEach(family => { byName[family.name] = family; });
return byName;
}
function sumByLabel(byName, metricName, groupLabel) {
const family = byName[metricName];
const totals = {};
if (!family) return totals;
family.metrics.forEach(metric => {
const key = groupLabel ? (metric.labels[groupLabel] ?? "unknown") : "total";
totals[key] = (totals[key] || 0) + (metric.value || 0);
});
return totals;
}
function sumHistogramByLabel(byName, metricName, groupLabel) {
const family = byName[metricName];
const totals = {};
if (!family) return totals;
family.metrics.forEach(metric => {
if (!metric.histogram) return;
const key = groupLabel ? (metric.labels[groupLabel] ?? "unknown") : "total";
const bucket = totals[key] || { count: 0, sum: 0 };
bucket.count += metric.histogram.sample_count || 0;
bucket.sum += metric.histogram.sample_sum || 0;
totals[key] = bucket;
});
return totals;
}
function pushMetricPoint(chartKey, label, t, v) {
const series = (metricsState.series[chartKey] ??= {});
const points = (series[label] ??= []);
points.push({ t, v });
if (points.length > MAX_METRIC_POINTS) points.shift();
}
function recordGauge(chartKey, totals, now) {
Object.entries(totals).forEach(([label, value]) => pushMetricPoint(chartKey, label, now, value));
}
function recordCounterRate(chartKey, totals, now) {
Object.entries(totals).forEach(([label, value]) => {
const key = `${chartKey}:${label}`;
const previous = metricsState.prevCounter[key];
metricsState.prevCounter[key] = { t: now, value };
if (!previous) return;
const deltaSeconds = (now - previous.t) / 1000;
if (deltaSeconds <= 0) return;
pushMetricPoint(chartKey, label, now, Math.max(0, (value - previous.value) / deltaSeconds));
});
}
function recordHistogramRate(chartKey, totals, now) {
Object.entries(totals).forEach(([label, { count, sum }]) => {
const key = `${chartKey}:${label}`;
const previous = metricsState.prevHistogram[key];
metricsState.prevHistogram[key] = { t: now, count, sum };
if (!previous) return;
const deltaCount = count - previous.count;
if (deltaCount <= 0) return;
pushMetricPoint(chartKey, label, now, Math.max(0, ((sum - previous.sum) / deltaCount) * 1000));
});
}
async function refreshQueueLength() {
const def = CHART_DEFS.find(candidate => candidate.key === "queueLength");
if (!def) return;
let history;
try { history = await request({ cmd: "queue_history", minutes: Math.round(def.windowMs / 60000) }); }
catch (_) { return; }
metricsState.series.queueLength = {
queued: history.points.map(point => ({ t: point.t, v: point.queued })),
partitioned: history.points.map(point => ({ t: point.t, v: point.partitioned }))
};
if (state.mode === "monitoring") renderChart(def);
}
async function pollMetrics() {
if (!METRICS_ENABLED) return;
let families;
try { families = await request({ cmd: "metrics" }); }
catch (_) { return; } const byName = familyIndex(families);
const now = Date.now();
recordCounterRate("throughput", sumByLabel(byName, "later_job_transitions_total", "stage"), now);
recordGauge("workers", sumByLabel(byName, "later_workers", "state"), now);
recordGauge("queueDepth", sumByLabel(byName, "later_queue_jobs", "state"), now);
recordHistogramRate("handlerLatency", sumHistogramByLabel(byName, "later_job_handler_duration_seconds", "outcome"), now);
recordGauge("partitionBacklog", sumByLabel(byName, "later_partition_backlog", "topic"), now);
recordCounterRate("commands", sumByLabel(byName, "later_worker_commands_total", "outcome"), now);
if (state.mode === "monitoring") CHART_DEFS.forEach(renderChart);
}
function chartLabels(def) {
const series = metricsState.series[def.key] || {};
return def.fixedLabels
? def.fixedLabels.filter(label => series[label] && series[label].length > 0)
: Object.keys(series).filter(label => series[label].length > 0);
}
function renderChart(def) {
updateLegend(def);
drawChartCanvas(def);
}
function updateLegend(def) {
const legend = document.querySelector(`#legend-${def.key}`);
if (!legend) return;
const series = metricsState.series[def.key] || {};
const labels = chartLabels(def);
if (labels.length === 0) {
legend.replaceChildren(document.createTextNode("Collecting data..."));
return;
}
legend.replaceChildren(...labels.map(label => {
const points = series[label];
const latest = points[points.length - 1].v;
const item = document.createElement("span");
const swatch = document.createElement("i");
swatch.className = "swatch";
swatch.style.background = colorForLabel(label);
item.append(swatch, document.createTextNode(`${label}: ${formatMetricValue(latest, def.unit)}`));
return item;
}));
}
function drawChartCanvas(def) {
const canvas = document.querySelector(`#chart-${def.key}`);
if (!canvas) return;
const series = metricsState.series[def.key] || {};
const labels = chartLabels(def);
drawChart(canvas, labels.map(label => ({ color: colorForLabel(label), points: series[label] })), def.windowMs || CHART_WINDOW_MS);
}
function drawChart(canvas, series, windowMs = CHART_WINDOW_MS) {
const ctx = canvas.getContext("2d");
const dpr = window.devicePixelRatio || 1;
const rect = canvas.getBoundingClientRect();
const width = Math.max(rect.width, 1);
const height = Math.max(rect.height, 1);
const pixelWidth = Math.round(width * dpr);
const pixelHeight = Math.round(height * dpr);
if (canvas.width !== pixelWidth || canvas.height !== pixelHeight) {
canvas.width = pixelWidth;
canvas.height = pixelHeight;
}
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
ctx.clearRect(0, 0, width, height);
const now = Date.now();
const windowStart = now - windowMs;
const visibleSeries = series
.map(({ color, points }) => ({ color, points: points.filter(point => point.t >= windowStart) }))
.filter(entry => entry.points.length >= 2);
if (visibleSeries.length === 0) {
ctx.fillStyle = "#9aa4b8";
ctx.font = "12px ui-sans-serif, system-ui, sans-serif";
ctx.fillText("Collecting data...", 10, height / 2);
return;
}
const padding = 8;
const plotWidth = width - padding * 2;
const plotHeight = height - padding * 2;
const maxV = Math.max(1, ...visibleSeries.flatMap(entry => entry.points.map(point => point.v)));
const x = t => padding + ((t - windowStart) / windowMs) * plotWidth;
const y = v => padding + plotHeight - (v / maxV) * plotHeight;
ctx.strokeStyle = "#e5e8ef";
ctx.lineWidth = 1;
for (let i = 0; i <= 2; i += 1) {
const gy = padding + (plotHeight / 2) * i;
ctx.beginPath();
ctx.moveTo(padding, gy);
ctx.lineTo(width - padding, gy);
ctx.stroke();
}
ctx.fillStyle = "#9aa4b8";
ctx.font = "11px ui-sans-serif, system-ui, sans-serif";
ctx.textBaseline = "top";
ctx.textAlign = "right";
ctx.fillText(Math.round(maxV).toLocaleString(), width - padding - 2, padding + 2);
ctx.textAlign = "left";
visibleSeries.forEach(({ color, points }) => {
ctx.strokeStyle = color;
ctx.lineWidth = 1.75;
ctx.lineJoin = "round";
ctx.lineCap = "round";
ctx.beginPath();
ctx.moveTo(x(points[0].t), y(points[0].v));
for (let i = 1; i < points.length - 1; i += 1) {
const px = x(points[i].t);
const py = y(points[i].v);
const midX = (px + x(points[i + 1].t)) / 2;
const midY = (py + y(points[i + 1].v)) / 2;
ctx.quadraticCurveTo(px, py, midX, midY);
}
const last = points[points.length - 1];
ctx.lineTo(x(last.t), y(last.v));
ctx.stroke();
});
}
function renderPartitionPicker() {
const topic = TOPICS.find(candidate => candidate.name === state.topic);
if (!topic) { elements.partitionPicker.hidden = true; return; }
elements.partitionPicker.hidden = false;
const depths = state.queueDepthByPartition[topic.name] || {};
const options = [];
for (let partition = 0; partition < topic.partition_count; partition += 1) options.push(partition);
elements.partitionSelect.replaceChildren(...options.map(partition => {
const option = document.createElement("option");
option.value = String(partition);
const depth = depths[partition];
option.textContent = depth === undefined
? `Partition ${partition}`
: `Partition ${partition} (${depth} queued)`;
return option;
}));
elements.partitionSelect.value = String(state.partition);
}
function renderPerformance(performance, dbSizeBytes) {
document.querySelector("#jobs-per-second").textContent = Number(performance.jobs_per_second || 0).toFixed(2);
for (const name of ["active_workers", "busy_workers", "started", "succeeded", "retried", "failed"]) {
document.querySelector(`#${name.replaceAll("_", "-")}`).textContent = performance[name] ?? 0;
}
for (const mode of ["regular", "sequential"]) {
const wait = (performance.wait_time || {})[mode] || { count: 0, avg_ms: 0, max_ms: 0 };
document.querySelector(`#wait-${mode}`).textContent = wait.count > 0 ? formatDuration(wait.avg_ms) : "-";
document.querySelector(`#wait-${mode}-note`).textContent = wait.count > 0
? `avg of ${wait.count}, max ${formatDuration(wait.max_ms)}`
: "no recent dispatches";
}
document.querySelector("#db-size").textContent = dbSizeBytes == null ? "-" : formatBytes(dbSizeBytes);
}
function formatBytes(bytes) {
const value = Number(bytes);
if (!Number.isFinite(value) || value < 0) return "-";
if (value < 1024) return `${value} B`;
if (value < 1024 ** 2) return `${(value / 1024).toFixed(1)} KB`;
if (value < 1024 ** 3) return `${(value / 1024 ** 2).toFixed(1)} MB`;
return `${(value / 1024 ** 3).toFixed(2)} GB`;
}
function stageDate(stage) {
if (!stage || !stage.date) return "";
const date = new Date(stage.date);
if (Number.isNaN(date.valueOf())) return stage.date;
const match = /\.(\d+)/.exec(stage.date);
const fraction = match
? match[1].slice(0, 6).padEnd(3, "0")
: String(date.getMilliseconds()).padStart(3, "0");
return `${date.toLocaleString()}.${fraction}`;
}
function formatDuration(milliseconds) {
const value = Number(milliseconds);
if (!Number.isFinite(value) || value < 0) return "-";
if (value < 1) return "<1 ms";
if (value < 1000) return `${Math.round(value)} ms`;
if (value < 60000) return `${(value / 1000).toFixed(2)} s`;
if (value < 3600000) return `${(value / 60000).toFixed(2)} min`;
return `${(value / 3600000).toFixed(2)} h`;
}
function renderJobs(response) {
const rows = response.result.map(job => {
const row = document.createElement("tr");
const idCell = document.createElement("td");
const link = document.createElement("button");
link.type = "button";
link.className = "job-link";
link.textContent = job.id.slice(-6);
link.title = job.id;
link.addEventListener("click", () => loadJob(job.id));
idCell.append(link);
const type = document.createElement("td");
type.className = "type";
type.textContent = job.info.payload_type;
const seq = document.createElement("td");
seq.className = "seq";
seq.textContent = job.info.sequence ?? "-";
const date = document.createElement("td");
date.className = "date";
date.textContent = stageDate(job.info.stage);
const duration = document.createElement("td");
duration.textContent = formatDuration(job.timing.total_ms);
const stage = document.createElement("td");
stage.textContent = job.info.stage.type;
row.append(idCell, type, seq, date, duration, stage);
return row;
});
elements.body.replaceChildren(...rows);
elements.notice.hidden = response.result.length > 0;
elements.notice.classList.remove("error");
elements.notice.textContent = "Nothing to show here";
const selectedId = state.selectedJobIndex === null
? null
: state.visibleJobs[state.selectedJobIndex]?.id;
state.visibleJobs = response.result;
state.selectedJobIndex = selectedId === null
? null
: state.visibleJobs.findIndex(job => job.id === selectedId);
if (state.selectedJobIndex === -1) state.selectedJobIndex = null;
updateDetailNavigation();
elements.jobTable.hidden = response.result.length === 0;
state.nextCursor = response.paging.next_cursor;
elements.pageLabel.textContent = `Page ${state.history.length + 1}`;
elements.previous.disabled = state.history.length === 0;
elements.next.disabled = state.nextCursor === null;
elements.paging.hidden = state.history.length === 0 && state.nextCursor === null;
}
async function loadStage(stage, cursor, reset = false, silent = false) {
if (state.loading) return;
state.loading = true;
if (reset || state.stage !== stage) state.history = [];
state.stage = stage;
state.cursor = cursor;
renderStages();
if (!silent) showNotice("Loading jobs...");
try { renderJobs(await request({ cmd: "jobs_in_stage", stage, cursor })); }
catch (error) { showNotice(error.message, true); }
finally { state.loading = false; }
}
async function loadPartition(topic, partition, cursor, reset = false, silent = false) {
if (state.loading) return;
state.loading = true;
if (reset || state.topic !== topic || state.partition !== partition) state.history = [];
state.mode = "partition";
state.topic = topic;
state.partition = partition;
state.cursor = cursor;
if (!silent) showNotice("Loading jobs...");
try { renderJobs(await request({ cmd: "partition_jobs", topic, partition, cursor })); }
catch (error) { showNotice(error.message, true); }
finally { state.loading = false; }
}
function showJob(job) {
const info = job.info;
location.hash = encodeURIComponent(info.id);
const index = state.visibleJobs.findIndex(candidate => candidate.id === info.id);
state.selectedJobIndex = index === -1 ? null : index;
updateDetailNavigation();
document.querySelector("#detail-id").textContent = info.id;
document.querySelector("#detail-type").textContent = info.payload_type;
document.querySelector("#detail-stage").textContent = info.stage.type;
document.querySelector("#detail-total-duration").textContent = formatDuration(job.timing.total_ms);
document.querySelector("#detail-processing-duration").textContent = formatDuration(job.timing.processing_ms);
document.querySelector("#detail-waiting-duration").textContent = formatDuration(job.timing.waiting_ms);
const relations = job.relations || {};
const partition = relations.partition;
const partitionRow = document.querySelector("#detail-partition-row");
if (partition) {
document.querySelector("#detail-partition").textContent =
`${partition.topic} / partition ${partition.partition} / sequence ${partition.sequence}`;
partitionRow.hidden = false;
} else {
partitionRow.hidden = true;
}
const relationRows = [];
if (relations.parent_job_id) {
relationRows.push(relationRow("Parent", [relations.parent_job_id]));
}
if ((relations.continuation_job_ids || []).length > 0) {
relationRows.push(relationRow("Continues with", relations.continuation_job_ids));
}
if (relations.continuations_truncated) {
const note = document.createElement("p");
note.textContent = "Only the first 25 continuation jobs are shown.";
relationRows.push(note);
}
if (partition && partition.previous_job_id) {
relationRows.push(relationRow("Previous in partition", [partition.previous_job_id]));
}
if (partition && partition.next_job_id) {
relationRows.push(relationRow("Next in partition", [partition.next_job_id]));
}
if (relationRows.length === 0) {
const empty = document.createElement("p");
empty.textContent = "No related jobs";
relationRows.push(empty);
}
document.querySelector("#relations").replaceChildren(...relationRows);
const payload = job.payload;
const payloadNote = document.querySelector("#detail-payload-note");
const payloadEl = document.querySelector("#detail-payload");
if (payload && payload.format === "msgpack") {
payloadEl.textContent = JSON.stringify(payload.value, null, 2);
payloadNote.hidden = !Array.isArray(payload.value);
} else if (payload && payload.format === "raw") {
payloadEl.textContent = payload.value;
payloadNote.hidden = true;
} else {
payloadEl.textContent = "(unavailable)";
payloadNote.hidden = true;
}
document.querySelector("#history").replaceChildren(...info.previous_stages.slice().reverse().map(stage => {
const item = document.createElement("li");
item.textContent = `${stage.type} - ${stageDate(stage)}`;
return item;
}));
if (!elements.details.open) elements.details.showModal();
}
function updateDetailNavigation() {
const index = state.selectedJobIndex;
const jobCount = state.visibleJobs.length;
const isInView = Number.isInteger(index) && index >= 0 && index < jobCount;
elements.detailPrevious.disabled = !isInView || index === 0;
elements.detailNext.disabled = !isInView || index === jobCount - 1;
elements.detailPosition.textContent = isInView
? `Job ${index + 1} of ${jobCount}`
: "Not in the current view";
}
function relationRow(label, jobIds) {
const row = document.createElement("p");
const title = document.createElement("strong");
title.textContent = `${label}: `;
const links = document.createElement("span");
links.className = "relation-links";
links.replaceChildren(...jobIds.map(id => {
const link = document.createElement("button");
link.type = "button";
link.className = "job-link";
link.textContent = id;
link.addEventListener("click", () => loadJob(id));
return link;
}));
row.append(title, links);
return row;
}
async function loadJob(id) {
try { showJob(await request({ cmd: "job", id })); }
catch (error) { showNotice(error.message, true); }
}
function loadAdjacentJob(offset) {
if (state.selectedJobIndex === null) return;
const job = state.visibleJobs[state.selectedJobIndex + offset];
if (job) loadJob(job.id);
}
function reload(cursor) {
if (state.mode === "partition") loadPartition(state.topic, state.partition, cursor);
else loadStage(state.stage, cursor);
}
async function refreshConsumerLag() {
if (LAG_TOPICS.length === 0) return;
try {
const readings = await request({ cmd: "consumer_lag" });
const totals = {};
readings.forEach(reading => {
totals[reading.topic] = (totals[reading.topic] || 0) + reading.lag;
});
state.consumerLag = totals;
} catch (_) {
}
}
async function refreshQueueDepth() {
if (TOPICS.length === 0) return;
try {
const readings = await request({ cmd: "partition_queue_depth" });
const totals = {};
TOPICS.forEach(topic => { totals[topic.name] = 0; });
const byPartition = {};
readings.forEach(reading => {
totals[reading.topic] = (totals[reading.topic] || 0) + reading.depth;
(byPartition[reading.topic] ??= {})[reading.partition] = reading.depth;
});
state.queueDepthTotals = totals;
state.queueDepthByPartition = byPartition;
} catch (_) {
}
}
async function refresh() {
try {
const response = await request({ cmd: "count" });
const previousCount = state.counts[state.stage];
state.counts = response.stages;
renderPerformance(response.performance, response.db_size_bytes);
if (state.mode === "stages") {
renderStages();
if (!state.stage) await loadStage("enqueued", null, true);
else if (previousCount !== state.counts[state.stage]) await loadStage(state.stage, state.cursor, false, true);
} else if (state.mode === "partition") {
await loadPartition(state.topic, state.partition, state.cursor, false, true);
}
} catch (error) { showNotice(error.message, true); }
await Promise.all([refreshConsumerLag(), refreshQueueDepth()]);
renderTopicBadges();
if (state.mode === "partition") renderPartitionPicker();
}
elements.previous.addEventListener("click", () => {
if (state.history.length === 0) return;
reload(state.history.pop());
});
elements.next.addEventListener("click", () => {
if (state.nextCursor === null) return;
state.history.push(state.cursor);
reload(state.nextCursor);
});
elements.partitionSelect.addEventListener("change", () => {
loadPartition(state.topic, Number(elements.partitionSelect.value), null, true);
});
elements.allJobs.addEventListener("click", switchToStages);
elements.monitoringTab.addEventListener("click", switchToMonitoring);
elements.detailPrevious.addEventListener("click", () => loadAdjacentJob(-1));
elements.detailNext.addEventListener("click", () => loadAdjacentJob(1));
document.querySelector("#close").addEventListener("click", () => { elements.details.close(); location.hash = ""; });
window.addEventListener("hashchange", () => {
const id = decodeURIComponent(location.hash.slice(1));
if (id) loadJob(id);
});
renderMonitoringCards();
renderTabs();
if (location.hash.length > 1) loadJob(decodeURIComponent(location.hash.slice(1)));
function pollEvery(task, delayMs) {
const tick = async () => {
try { await task(); } finally { setTimeout(tick, delayMs); }
};
tick();
}
pollEvery(refresh, 2500);
if (METRICS_ENABLED) {
pollEvery(pollMetrics, 2500);
pollEvery(refreshQueueLength, 15000);
}
</script>
</body>
</html>