{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "green", "value": null}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
"id": 10,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"expr": "sum(llm_requests_total{provider=~\"$provider\", model=~\"$model\"})",
"legendFormat": "Total Requests",
"refId": "A"
}
],
"title": "Total Requests",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "blue", "value": null}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
"id": 11,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"expr": "sum(llm_tokens_prompt_total{provider=~\"$provider\", model=~\"$model\"}) + sum(llm_tokens_completion_total{provider=~\"$provider\", model=~\"$model\"})",
"legendFormat": "Total Tokens",
"refId": "A"
}
],
"title": "Total Tokens",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "green", "value": null},
{"color": "red", "value": 1}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
"id": 12,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"expr": "sum(llm_errors_total{provider=~\"$provider\", model=~\"$model\"}) or vector(0)",
"legendFormat": "Total Errors",
"refId": "A"
}
],
"title": "Total Errors",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "purple", "value": null}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
"id": 13,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"expr": "sum(llm_request_duration_seconds_sum{provider=~\"$provider\", model=~\"$model\"}) / sum(llm_request_duration_seconds_count{provider=~\"$provider\", model=~\"$model\"})",
"legendFormat": "Avg Latency",
"refId": "A"
}
],
"title": "Avg Latency",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {"mode": "palette-classic"},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {"legend": false, "tooltip": false, "viz": false},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {"type": "linear"},
"showPoints": "auto",
"spanNulls": false,
"stacking": {"group": "A", "mode": "none"},
"thresholdsStyle": {"mode": "off"}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{"color": "green", "value": null}]
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
"id": 1,
"options": {
"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true},
"tooltip": {"mode": "multi", "sort": "desc"}
},
"targets": [
{
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"expr": "sum(rate(llm_requests_total{provider=~\"$provider\", model=~\"$model\"}[$__rate_interval])) by (provider, model)",
"legendFormat": "{{provider}} - {{model}}",
"refId": "A"
}
],
"title": "Request Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {"mode": "palette-classic"},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {"legend": false, "tooltip": false, "viz": false},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {"type": "linear"},
"showPoints": "auto",
"spanNulls": false,
"stacking": {"group": "A", "mode": "none"},
"thresholdsStyle": {"mode": "off"}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{"color": "green", "value": null}]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 4},
"id": 2,
"options": {
"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true},
"tooltip": {"mode": "multi", "sort": "desc"}
},
"targets": [
{
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"expr": "llm_request_duration_seconds{provider=~\"$provider\", model=~\"$model\", quantile=\"0.95\"}",
"legendFormat": "p95 {{provider}} - {{model}}",
"refId": "A"
},
{
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"expr": "llm_request_duration_seconds{provider=~\"$provider\", model=~\"$model\", quantile=\"0.5\"}",
"legendFormat": "p50 {{provider}} - {{model}}",
"refId": "B"
}
],
"title": "Request Latency (p50/p95)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {"mode": "palette-classic"},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 30,
"gradientMode": "none",
"hideFrom": {"legend": false, "tooltip": false, "viz": false},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {"type": "linear"},
"showPoints": "auto",
"spanNulls": false,
"stacking": {"group": "A", "mode": "normal"},
"thresholdsStyle": {"mode": "off"}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{"color": "green", "value": null}]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
"id": 3,
"options": {
"legend": {"calcs": ["sum"], "displayMode": "table", "placement": "bottom", "showLegend": true},
"tooltip": {"mode": "multi", "sort": "desc"}
},
"targets": [
{
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"expr": "sum(increase(llm_tokens_prompt_total{provider=~\"$provider\", model=~\"$model\"}[$__rate_interval])) by (provider, model)",
"legendFormat": "Prompt - {{provider}} {{model}}",
"refId": "A"
},
{
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"expr": "sum(increase(llm_tokens_completion_total{provider=~\"$provider\", model=~\"$model\"}[$__rate_interval])) by (provider, model)",
"legendFormat": "Completion - {{provider}} {{model}}",
"refId": "B"
}
],
"title": "Token Usage",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {"mode": "palette-classic"},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {"legend": false, "tooltip": false, "viz": false},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {"type": "linear"},
"showPoints": "auto",
"spanNulls": false,
"stacking": {"group": "A", "mode": "none"},
"thresholdsStyle": {"mode": "off"}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{"color": "red", "value": null}]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 12},
"id": 4,
"options": {
"legend": {"calcs": ["sum"], "displayMode": "table", "placement": "bottom", "showLegend": true},
"tooltip": {"mode": "multi", "sort": "desc"}
},
"targets": [
{
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"expr": "sum(rate(llm_errors_total{provider=~\"$provider\", model=~\"$model\"}[$__rate_interval])) by (provider, error_type)",
"legendFormat": "{{provider}} - {{error_type}}",
"refId": "A"
}
],
"title": "Error Rate by Type",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {"mode": "palette-classic"},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {"legend": false, "tooltip": false, "viz": false},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {"type": "linear"},
"showPoints": "auto",
"spanNulls": false,
"stacking": {"group": "A", "mode": "none"},
"thresholdsStyle": {"mode": "off"}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{"color": "orange", "value": null}]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 20},
"id": 6,
"options": {
"legend": {"calcs": ["sum"], "displayMode": "table", "placement": "bottom", "showLegend": true},
"tooltip": {"mode": "multi", "sort": "desc"}
},
"targets": [
{
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"expr": "sum(rate(llm_retries_total{provider=~\"$provider\"}[$__rate_interval])) by (provider)",
"legendFormat": "{{provider}}",
"refId": "A"
}
],
"title": "Retry Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {"mode": "palette-classic"},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {"legend": false, "tooltip": false, "viz": false},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {"type": "linear"},
"showPoints": "auto",
"spanNulls": false,
"stacking": {"group": "A", "mode": "none"},
"thresholdsStyle": {"mode": "off"}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{"color": "yellow", "value": null}]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 20},
"id": 7,
"options": {
"legend": {"calcs": ["sum"], "displayMode": "table", "placement": "bottom", "showLegend": true},
"tooltip": {"mode": "multi", "sort": "desc"}
},
"targets": [
{
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"expr": "sum(rate(llm_rate_limits_total{provider=~\"$provider\"}[$__rate_interval])) by (provider)",
"legendFormat": "{{provider}}",
"refId": "A"
}
],
"title": "Rate Limit Events",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {"mode": "palette-classic"},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {"legend": false, "tooltip": false, "viz": false},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {"type": "linear"},
"showPoints": "auto",
"spanNulls": false,
"stacking": {"group": "A", "mode": "none"},
"thresholdsStyle": {"mode": "off"}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{"color": "green", "value": null}]
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 28},
"id": 8,
"options": {
"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true},
"tooltip": {"mode": "multi", "sort": "desc"}
},
"targets": [
{
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"expr": "sum(rate(llm_requests_total{task=~\"$task\"}[$__rate_interval])) by (task)",
"legendFormat": "{{task}}",
"refId": "A"
}
],
"title": "Request Rate by Task",
"type": "timeseries"
}
],
"refresh": "10s",
"schemaVersion": 39,
"tags": ["llm", "flyllm", "ai"],
"templating": {
"list": [
{
"current": {},
"hide": 0,
"includeAll": false,
"label": "Datasource",
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".*",
"current": {},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"definition": "label_values(llm_requests_total, provider)",
"hide": 0,
"includeAll": true,
"label": "Provider",
"multi": true,
"name": "provider",
"options": [],
"query": {"qryType": 1, "query": "label_values(llm_requests_total, provider)", "refId": "PrometheusVariableQueryEditor-VariableQuery"},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
},
{
"allValue": ".*",
"current": {},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"definition": "label_values(llm_requests_total{provider=~\"$provider\"}, model)",
"hide": 0,
"includeAll": true,
"label": "Model",
"multi": true,
"name": "model",
"options": [],
"query": {"qryType": 1, "query": "label_values(llm_requests_total{provider=~\"$provider\"}, model)", "refId": "PrometheusVariableQueryEditor-VariableQuery"},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
},
{
"allValue": ".*",
"current": {},
"datasource": {"type": "prometheus", "uid": "${datasource}"},
"definition": "label_values(llm_requests_total, task)",
"hide": 0,
"includeAll": true,
"label": "Task",
"multi": true,
"name": "task",
"options": [],
"query": {"qryType": 1, "query": "label_values(llm_requests_total, task)", "refId": "PrometheusVariableQueryEditor-VariableQuery"},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "FlyLLM Dashboard",
"uid": "flyllm-dashboard",
"version": 1,
"weekStart": ""
}