26Y27x3

{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 100,
      "title": "Drive Health & IO Errors (Across 2,560 Drives)",
      "type": "row"
    },
    {
      "id": 1,
      "title": "Drive Timeout Errors Rate",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 1
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line"
          },
          "unit": "errorsps"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "topk($topn, sum by (server, drive_id) (rate(minio_system_drive_timeout_errors_total{cluster=~\"$cluster\", server=~\"$server\"}[$interval])))",
          "legendFormat": "{{server}} - Drive {{drive_id}}",
          "refId": "A"
        }
      ]
    },
    {
      "id": 2,
      "title": "Drive IO Errors Rate",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 1
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line"
          },
          "unit": "errorsps"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "topk($topn, sum by (server, drive_id) (rate(minio_system_drive_io_errors_total{cluster=~\"$cluster\", server=~\"$server\"}[$interval])))",
          "legendFormat": "{{server}} - Drive {{drive_id}}",
          "refId": "A"
        }
      ]
    },
    {
      "id": 3,
      "title": "Waiting IO Heatmap / Outliers",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 9
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line"
          },
          "unit": "short"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "topk($topn, minio_system_drive_waiting_io{cluster=~\"$cluster\", server=~\"$server\"})",
          "legendFormat": "{{server}} - Drive {{drive_id}}",
          "refId": "A"
        }
      ]
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 17
      },
      "id": 200,
      "title": "Internode Network & Lock Performance",
      "type": "row"
    },
    {
      "id": 4,
      "title": "Internode Failed Calls Rate",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 18
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line"
          },
          "unit": "short"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "topk($topn, sum by (server) (rate(minio_internode_failed_calls_total{cluster=~\"$cluster\", server=~\"$server\"}[$interval])))",
          "legendFormat": "Failed RPC - {{server}}",
          "refId": "A"
        }
      ]
    },
    {
      "id": 5,
      "title": "Internode Traffic Throughput (Rx / Tx)",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 18
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line"
          },
          "unit": "Bps"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (server) (rate(minio_internode_rx_bytes_total{cluster=~\"$cluster\", server=~\"$server\"}[$interval]))",
          "legendFormat": "Rx - {{server}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (server) (rate(minio_internode_tx_bytes_total{cluster=~\"$cluster\", server=~\"$server\"}[$interval]))",
          "legendFormat": "Tx - {{server}}",
          "refId": "B"
        }
      ]
    },
    {
      "id": 6,
      "title": "Internode Distributed READ / WRITE Active Locks",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 26
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line"
          },
          "unit": "short"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (server) (minio_internode_locks_read_total{cluster=~\"$cluster\", server=~\"$server\"})",
          "legendFormat": "Read Locks - {{server}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (server) (minio_internode_locks_write_total{cluster=~\"$cluster\", server=~\"$server\"})",
          "legendFormat": "Write Locks - {{server}}",
          "refId": "B"
        }
      ]
    }
  ],
  "refresh": "30s",
  "schemaVersion": 38,
  "style": "dark",
  "tags": [
    "minio",
    "aistor",
    "drive",
    "network"
  ],
  "templating": {
    "list": [
      {
        "name": "datasource",
        "type": "datasource",
        "query": "prometheus",
        "hide": 0,
        "label": "Prometheus",
        "current": {}
      },
      {
        "name": "cluster",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "query": "label_values(minio_cluster_health_nodes_online_count, cluster)",
        "label": "Cluster",
        "hide": 0,
        "includeAll": true,
        "multi": true,
        "allValue": ".*",
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "refresh": 2,
        "sort": 1
      },
      {
        "name": "server",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "query": "label_values(minio_system_process_go_routine_total{cluster=~\"$cluster\"}, server)",
        "label": "Node (server)",
        "hide": 0,
        "includeAll": true,
        "multi": true,
        "allValue": ".*",
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "refresh": 2,
        "sort": 1
      },
      {
        "name": "topn",
        "type": "custom",
        "query": "10,15,20,30",
        "label": "Top N",
        "hide": 0,
        "current": {
          "text": "15",
          "value": "15"
        },
        "options": [
          {
            "text": "10",
            "value": "10",
            "selected": false
          },
          {
            "text": "15",
            "value": "15",
            "selected": true
          },
          {
            "text": "20",
            "value": "20",
            "selected": false
          },
          {
            "text": "30",
            "value": "30",
            "selected": false
          }
        ]
      },
      {
        "current": {
          "selected": true,
          "text": "2m",
          "value": "2m"
        },
        "hide": 0,
        "label": "Interval",
        "name": "interval",
        "options": [
          {
            "selected": false,
            "text": "30s",
            "value": "30s"
          },
          {
            "selected": false,
            "text": "1m",
            "value": "1m"
          },
          {
            "selected": true,
            "text": "2m",
            "value": "2m"
          },
          {
            "selected": false,
            "text": "5m",
            "value": "5m"
          },
          {
            "selected": false,
            "text": "10m",
            "value": "10m"
          }
        ],
        "query": "30s,1m,2m,5m,10m",
        "type": "interval"
      }
    ]
  },
  "time": {
    "from": "now-3h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m"
    ]
  },
  "timezone": "browser",
  "title": "2. MinIO AIStor - Drive & Internode Performance",
  "uid": "minio-aistor-2-drive-internode",
  "version": 1
}