26Y28c4
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 400,
"title": "🚨 1. TARGET REPLICATION VITALS & BACKLOG LOGIC",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "orange",
"value": 107374182400
},
{
"color": "red",
"value": 536870912000
}
]
},
"unit": "decbytes"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 6,
"x": 0,
"y": 1
},
"id": 401,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(max by (cluster) (minio_cluster_replication_pending_bytes{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"} or minio_replication_pending_bytes{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"}))",
"legendFormat": "Pending Bytes",
"refId": "A"
}
],
"title": "Replication Pending Backlog Size (Bytes)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "orange",
"value": 10000
},
{
"color": "red",
"value": 50000
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 6,
"x": 6,
"y": 1
},
"id": 402,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(max by (cluster) (minio_cluster_replication_pending_count{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"} or minio_replication_pending_count{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"}))",
"legendFormat": "Pending Objects",
"refId": "A"
}
],
"title": "Replication Pending Objects Count",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
},
"unit": "pps"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 6,
"x": 12,
"y": 1
},
"id": 403,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(rate(minio_cluster_replication_failed_count{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"}[$interval]) or rate(minio_replication_failed_count{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"}[$interval]))",
"legendFormat": "Failed Rate",
"refId": "A"
}
],
"title": "Replication Failed Transfer Speed (Objects/s)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "orange",
"value": 0.2
},
{
"color": "red",
"value": 0.5
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 6,
"x": 18,
"y": 1
},
"id": 404,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "max(minio_cluster_replication_link_latency_seconds{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"} or minio_replication_link_latency_seconds{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"})",
"legendFormat": "Max Latency",
"refId": "A"
}
],
"title": "Worst Cross-Site Link Latency (Seconds)",
"type": "stat"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 6
},
"id": 500,
"title": "📈 2. CROSS-SITE BANDWIDTH THROUGHPUT & LATENCY TRENDS (TOP N NODES)",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 10,
"lineInterpolation": "smooth",
"lineWidth": 2,
"showPoints": "none"
},
"mappings": [],
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 7
},
"id": 501,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "topk($topn, sum by (server) (rate(minio_cluster_replication_sent_bytes_total{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"}[$interval]) or rate(minio_replication_sent_bytes_total{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"}[$interval])))",
"legendFormat": "Sent from {{server}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "topk($topn, sum by (server) (rate(minio_cluster_replication_received_bytes_total{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"}[$interval]) or rate(minio_replication_received_bytes_total{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"}[$interval])))",
"legendFormat": "Received by {{server}}",
"refId": "B"
}
],
"title": "Cross-Site Replication Bandwidth Velocity (Sent/Received Bps per Node)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 5,
"lineInterpolation": "smooth",
"lineWidth": 2,
"showPoints": "none"
},
"mappings": [],
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 7
},
"id": 502,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "topk($topn, minio_cluster_replication_link_latency_seconds{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"} or minio_replication_link_latency_seconds{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"})",
"legendFormat": "Link: {{server}} -> Target",
"refId": "A"
}
],
"title": "Cross-Site Replication Link Latency Timeline (Top N Nodes)",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 600,
"title": "📋 3. DETAILED BACKLOG ACCUMULATION TRENDS (TOP N NODES)",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 5,
"lineInterpolation": "smooth",
"lineWidth": 2,
"showPoints": "none"
},
"mappings": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 15
},
"id": 601,
"options": {
"legend": {
"calcs": [
"max",
"lastNotNull"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "topk($topn, minio_cluster_replication_pending_count{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"} or minio_replication_pending_count{cluster=~\"${cluster:regex}\", namespace=~\"${namespace:regex}\"})",
"legendFormat": "Pending: {{server}}",
"refId": "A"
}
],
"title": "Replication Pending Backlog Count Profile per Node Server (Top N)",
"type": "timeseries"
}
],
"refresh": "10s",
"schemaVersion": 38,
"style": "dark",
"tags": [
"AIStor",
"Metrics-V3",
"Replication-Only",
"Multi-Cluster-WAN"
],
"templating": {
"list": [
{
"current": {},
"hide": 0,
"includeAll": false,
"label": "Data Source",
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"current": {},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values(cluster)",
"hide": 0,
"includeAll": true,
"multi": true,
"allValue": ".*",
"label": "AIStor Cluster",
"name": "cluster",
"options": [],
"query": "label_values(cluster)",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
},
{
"current": {},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values({cluster=~\"${cluster:regex}\"}, namespace)",
"hide": 0,
"includeAll": true,
"multi": true,
"allValue": ".*",
"label": "Namespace",
"name": "namespace",
"options": [],
"query": "label_values({cluster=~\"${cluster:regex}\"}, namespace)",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
},
{
"name": "topn",
"type": "custom",
"query": "10,15,20,30",
"label": "Top N Nodes",
"hide": 0,
"current": {
"text": "15",
"value": "15"
},
"options": [
{
"text": "10",
"value": "10",
"selected": false
},
{
"text": "15",
"value": "15",
"selected": true
},
{
"text": "20",
"value": "20",
"selected": false
},
{
"text": "30",
"value": "30",
"selected": false
}
]
},
{
"current": {
"selected": true,
"text": "2m",
"value": "2m"
},
"hide": 0,
"label": "Interval",
"name": "interval",
"options": [
{
"selected": false,
"text": "30s",
"value": "30s"
},
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": true,
"text": "2m",
"value": "2m"
},
{
"selected": false,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "10m",
"value": "10m"
}
],
"query": "30s,1m,2m,5m,10m",
"type": "interval"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s"
]
},
"timezone": "browser",
"title": "AIStor Operations - 4. Replication (Multi-Cluster v3 - Fixed)",
"uid": "aistor-operations-4-replication-fixed",
"version": 1
}
engineer