26Z12n

{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "Prometheus data source scraping MinIO AIStor's /minio/metrics/v3 endpoint",
      "type": "datasource",
      "pluginId": "prometheus"
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": { "type": "grafana", "uid": "-- Grafana --" },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
      "id": 100,
      "panels": [],
      "title": "Cluster Overview — Throughput / Inflight / IOPS / Goroutines",
      "type": "row"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
      "description": "Cluster-wide S3 API request rate (TPS), split by request type (S3 / admin / internal).",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "lineWidth": 1,
            "pointSize": 5,
            "stacking": { "mode": "none" }
          },
          "unit": "reqps"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 6, "x": 0, "y": 1 },
      "id": 1,
      "options": {
        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
        "tooltip": { "mode": "multi", "sort": "desc" }
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "sum(rate(minio_api_requests_total{type=\"s3\"}[$__rate_interval]))",
          "legendFormat": "Cluster TPS (S3)",
          "refId": "A"
        },
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "sum(rate(minio_api_requests_total{type=\"s3\"}[$__rate_interval])) by (server)",
          "legendFormat": "{{server}}",
          "refId": "B"
        }
      ],
      "title": "Cluster TPS (S3 Requests/sec)",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
      "description": "Requests currently being actively processed (in-flight) vs. waiting in queue, cluster-wide.",
      "fieldConfig": {
        "defaults": {
          "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 6, "x": 6, "y": 1 },
      "id": 2,
      "options": {
        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
        "tooltip": { "mode": "multi", "sort": "desc" }
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "sum(minio_api_requests_inflight_total{type=\"s3\"})",
          "legendFormat": "Inflight (processing)",
          "refId": "A"
        },
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "sum(minio_api_requests_waiting_total{type=\"s3\"})",
          "legendFormat": "Waiting in queue",
          "refId": "B"
        },
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "sum(minio_api_requests_incoming_total{type=\"s3\"})",
          "legendFormat": "Incoming",
          "refId": "C"
        }
      ],
      "title": "Inflight / Waiting Requests",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
      "description": "Cluster-wide drive IOPS derived from iostat-style per-drive read/write rates (minio_system_drive_reads_per_sec / writes_per_sec).",
      "fieldConfig": {
        "defaults": {
          "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "normal" } },
          "unit": "iops"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 6, "x": 12, "y": 1 },
      "id": 3,
      "options": {
        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
        "tooltip": { "mode": "multi", "sort": "desc" }
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "sum(minio_system_drive_reads_per_sec)",
          "legendFormat": "Read IOPS",
          "refId": "A"
        },
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "sum(minio_system_drive_writes_per_sec)",
          "legendFormat": "Write IOPS",
          "refId": "B"
        }
      ],
      "title": "Cluster Drive IOPS (Read/Write)",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
      "description": "Go runtime goroutine count per node, exposed via the standard Prometheus Go client collector on the same v3 endpoint.",
      "fieldConfig": {
        "defaults": {
          "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 6, "x": 18, "y": 1 },
      "id": 4,
      "options": {
        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
        "tooltip": { "mode": "multi", "sort": "desc" }
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "go_goroutines{job=~\"$job\"}",
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "title": "Goroutines per Node",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
      "id": 200,
      "panels": [],
      "title": "Latency — Overall / Per-API / Read TTFB",
      "type": "row"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
      "description": "Cluster-wide average request duration = sum(duration)/count(requests). minio_api_requests_duration_total is a cumulative counter in milliseconds.",
      "fieldConfig": {
        "defaults": {
          "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } },
          "unit": "ms"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 10 },
      "id": 10,
      "options": {
        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
        "tooltip": { "mode": "multi", "sort": "desc" }
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "sum(rate(minio_api_requests_duration_total{type=\"s3\"}[$__rate_interval])) / sum(rate(minio_api_requests_total{type=\"s3\"}[$__rate_interval]))",
          "legendFormat": "Avg request latency",
          "refId": "A"
        }
      ],
      "title": "Overall Average Latency",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
      "description": "p99 TTFB latency broken down by API operation (name label), top 10 by current value. Source: minio_api_requests_ttfb_seconds_distribution histogram.",
      "fieldConfig": {
        "defaults": {
          "custom": { "drawStyle": "line", "fillOpacity": 5, "lineWidth": 1, "stacking": { "mode": "none" } },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 10 },
      "id": 11,
      "options": {
        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
        "tooltip": { "mode": "multi", "sort": "desc" }
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "topk(10, histogram_quantile(0.99, sum(rate(minio_api_requests_ttfb_seconds_distribution{type=\"s3\"}[$__rate_interval])) by (le, name)))",
          "legendFormat": "{{name}}",
          "refId": "A"
        }
      ],
      "title": "Latency by API (p99 TTFB, Top 10)",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
      "description": "p95/p99 Time-To-First-Byte for read operations (GetObject/HeadObject family). Adjust the name regex to match your workload's read APIs.",
      "fieldConfig": {
        "defaults": {
          "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 10 },
      "id": 12,
      "options": {
        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
        "tooltip": { "mode": "multi", "sort": "desc" }
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "histogram_quantile(0.99, sum(rate(minio_api_requests_ttfb_seconds_distribution{type=\"s3\", name=~\"$read_apis\"}[$__rate_interval])) by (le))",
          "legendFormat": "Read TTFB p99",
          "refId": "A"
        },
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "histogram_quantile(0.95, sum(rate(minio_api_requests_ttfb_seconds_distribution{type=\"s3\", name=~\"$read_apis\"}[$__rate_interval])) by (le))",
          "legendFormat": "Read TTFB p95",
          "refId": "B"
        }
      ],
      "title": "Read TTFB p99 / p95",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
      "id": 300,
      "panels": [],
      "title": "Network — Cluster Total / Per-Interface (bond0, bond1)",
      "type": "row"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
      "description": "Cluster-wide network Send/Receive throughput, summed across all reported interfaces on all nodes. minio_system_network_interface_{rx,tx}_bytes report bytes seen in the last 60s, so the value is divided by 60 to approximate Bytes/sec.",
      "fieldConfig": {
        "defaults": {
          "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } },
          "unit": "Bps"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
      "id": 20,
      "options": {
        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
        "tooltip": { "mode": "multi", "sort": "desc" }
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "sum(minio_system_network_interface_rx_bytes) / 60",
          "legendFormat": "Cluster Receive",
          "refId": "A"
        },
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "sum(minio_system_network_interface_tx_bytes) / 60",
          "legendFormat": "Cluster Send",
          "refId": "B"
        }
      ],
      "title": "Cluster Network Total (Send / Receive)",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
      "description": "Per-node, per-interface Send/Receive throughput, filtered by the $interface variable (e.g. bond0, bond1).",
      "fieldConfig": {
        "defaults": {
          "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "none" } },
          "unit": "Bps"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 },
      "id": 21,
      "options": {
        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
        "tooltip": { "mode": "multi", "sort": "desc" }
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "minio_system_network_interface_rx_bytes{interface=~\"$interface\"} / 60",
          "legendFormat": "{{server}} {{interface}} RX",
          "refId": "A"
        },
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "minio_system_network_interface_tx_bytes{interface=~\"$interface\"} / 60",
          "legendFormat": "{{server}} {{interface}} TX",
          "refId": "B"
        }
      ],
      "title": "Network by Interface (bond0 / bond1)",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 },
      "id": 400,
      "panels": [],
      "title": "Drive Latency — Top / Bottom per Node·Drive",
      "type": "row"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
      "description": "Slowest 10 node/drive/API combinations by average last-minute storage-API latency (µs). Source: minio_system_drive_api_latency_micros{drive,pool_index,set_index,drive_index,api,server}.",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": "auto",
            "cellOptions": { "type": "color-background" },
            "filterable": true
          },
          "unit": "µs"
        },
        "overrides": []
      },
      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 28 },
      "id": 30,
      "options": {
        "showHeader": true,
        "sortBy": [{ "desc": true, "displayName": "Value" }]
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "topk(10, minio_system_drive_api_latency_micros{server=~\"$server\"})",
          "format": "table",
          "instant": true,
          "legendFormat": "__auto",
          "refId": "A"
        }
      ],
      "title": "Drive Latency — Top 10 (Slowest)",
      "type": "table"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
      "description": "Fastest 10 node/drive/API combinations by average last-minute storage-API latency (µs), excluding zero/idle series.",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": "auto",
            "cellOptions": { "type": "color-background" },
            "filterable": true
          },
          "unit": "µs"
        },
        "overrides": []
      },
      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 28 },
      "id": 31,
      "options": {
        "showHeader": true,
        "sortBy": [{ "desc": false, "displayName": "Value" }]
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "expr": "bottomk(10, minio_system_drive_api_latency_micros{server=~\"$server\"} > 0)",
          "format": "table",
          "instant": true,
          "legendFormat": "__auto",
          "refId": "A"
        }
      ],
      "title": "Drive Latency — Bottom 10 (Fastest, excl. idle)",
      "type": "table"
    },
    {
      "collapsed": true,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 },
      "id": 500,
      "panels": [
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "description": "Per-node, per-drive read/write await time and utilization, useful for spotting a single slow disk.",
          "fieldConfig": {
            "defaults": {
              "custom": { "align": "auto", "cellOptions": { "type": "color-background" }, "filterable": true },
              "unit": "ms"
            },
            "overrides": []
          },
          "gridPos": { "h": 9, "w": 12, "x": 0, "y": 38 },
          "id": 40,
          "options": { "showHeader": true, "sortBy": [{ "desc": true, "displayName": "Value" }] },
          "targets": [
            {
              "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
              "expr": "topk(10, minio_system_drive_reads_await{server=~\"$server\"})",
              "format": "table",
              "instant": true,
              "legendFormat": "__auto",
              "refId": "A"
            }
          ],
          "title": "Drive Read Await — Top 10",
          "type": "table"
        },
        {
          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
          "description": "Per-node, per-drive write await time, top 10.",
          "fieldConfig": {
            "defaults": {
              "custom": { "align": "auto", "cellOptions": { "type": "color-background" }, "filterable": true },
              "unit": "ms"
            },
            "overrides": []
          },
          "gridPos": { "h": 9, "w": 12, "x": 12, "y": 38 },
          "id": 41,
          "options": { "showHeader": true, "sortBy": [{ "desc": true, "displayName": "Value" }] },
          "targets": [
            {
              "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
              "expr": "topk(10, minio_system_drive_writes_await{server=~\"$server\"})",
              "format": "table",
              "instant": true,
              "legendFormat": "__auto",
              "refId": "A"
            }
          ],
          "title": "Drive Write Await — Top 10",
          "type": "table"
        }
      ],
      "title": "Drive Detail (optional) — Await Time by Drive",
      "type": "row"
    }
  ],
  "refresh": "30s",
  "schemaVersion": 39,
  "tags": ["minio", "aistor", "prometheus", "v3"],
  "templating": {
    "list": [
      {
        "current": { "text": "Prometheus", "value": "Prometheus" },
        "hide": 0,
        "label": "Data source",
        "name": "DS_PROMETHEUS",
        "query": "prometheus",
        "refresh": 1,
        "type": "datasource"
      },
      {
        "current": { "text": ".*", "value": ".*" },
        "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
        "definition": "label_values(up{job=~\".*minio.*|.*aistor.*\"}, job)",
        "hide": 0,
        "includeAll": false,
        "label": "Scrape job",
        "multi": false,
        "name": "job",
        "options": [],
        "query": { "query": "label_values(up{job=~\".*minio.*|.*aistor.*\"}, job)", "refId": "StandardVariableQuery" },
        "refresh": 2,
        "regex": "",
        "sort": 1,
        "type": "query"
      },
      {
        "current": { "text": "All", "value": "$__all" },
        "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
        "definition": "label_values(minio_system_drive_count, server)",
        "hide": 0,
        "includeAll": true,
        "label": "Node (server)",
        "multi": true,
        "name": "server",
        "options": [],
        "query": { "query": "label_values(minio_system_drive_count, server)", "refId": "StandardVariableQuery" },
        "refresh": 2,
        "regex": "",
        "sort": 1,
        "type": "query"
      },
      {
        "current": { "text": "All", "value": "$__all" },
        "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
        "definition": "label_values(minio_system_network_interface_rx_bytes, interface)",
        "hide": 0,
        "includeAll": true,
        "label": "Interface",
        "multi": true,
        "name": "interface",
        "options": [],
        "query": { "query": "label_values(minio_system_network_interface_rx_bytes, interface)", "refId": "StandardVariableQuery" },
        "refresh": 2,
        "regex": "",
        "sort": 1,
        "type": "query"
      },
      {
        "current": { "text": "GetObject|HeadObject|GetObjectTagging|SelectObjectContent", "value": "GetObject|HeadObject|GetObjectTagging|SelectObjectContent" },
        "hide": 0,
        "label": "Read API name regex",
        "name": "read_apis",
        "query": "GetObject|HeadObject|GetObjectTagging|SelectObjectContent",
        "skipUrlSync": false,
        "type": "textbox"
      }
    ]
  },
  "time": { "from": "now-6h", "to": "now" },
  "timepicker": {},
  "timezone": "",
  "title": "MinIO AIStor — Cluster Performance (Prometheus Metrics v3)",
  "uid": "aistor-cluster-v3",
  "version": 1,
  "weekStart": ""
}