26Z20b

{
  "annotations": { "list": [] },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
      "id": 100,
      "title": "1. Distributed Lock & Peer Health (v3)",
      "type": "row"
    },
    {
      "title": "Active Locks per Peer (minio_locks_total)",
      "type": "timeseries",
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
      "id": 1,
      "fieldConfig": {
        "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineInterpolation": "smooth" } }
      },
      "targets": [
        {
          "expr": "sum(minio_locks_total{server=~\"$server\"}) by (server)",
          "legendFormat": "{{server}} (Locks)"
        }
      ]
    },
    {
      "title": "Lock Timeouts & Dropped RPCs (Stall / Flapping Indicator)",
      "type": "timeseries",
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
      "id": 2,
      "fieldConfig": {
        "defaults": { "unit": "ops", "custom": { "drawStyle": "line" } }
      },
      "targets": [
        {
          "expr": "sum(rate(minio_locks_timeout{server=~\"$server\"}[5m])) by (server)",
          "legendFormat": "Timeout - {{server}}"
        },
        {
          "expr": "sum(rate(minio_locks_dropped{server=~\"$server\"}[5m])) by (server)",
          "legendFormat": "Dropped - {{server}}"
        }
      ]
    },
    {
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
      "id": 200,
      "title": "2. API Requests & Hot Object Correlation (v3)",
      "type": "row"
    },
    {
      "title": "S3 Mutation Operations (Put/Copy/Delete) by Peer",
      "type": "timeseries",
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 },
      "id": 3,
      "fieldConfig": {
        "defaults": { "unit": "reqps", "custom": { "drawStyle": "line" } }
      },
      "targets": [
        {
          "expr": "sum(rate(minio_api_requests_total{name=~\"PutObject|CopyObject|PutObjectPart|DeleteObject.*\", server=~\"$server\"}[5m])) by (name, server)",
          "legendFormat": "{{server}} - {{name}}"
        }
      ]
    },
    {
      "title": "S3 API TTFB Latency (P95 / P99)",
      "type": "timeseries",
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 },
      "id": 4,
      "fieldConfig": {
        "defaults": { "unit": "s", "custom": { "drawStyle": "line" } }
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(minio_api_requests_ttfb_seconds_distribution{name=~\"PutObject|CopyObject\", server=~\"$server\"}[5m])) by (le, name))",
          "legendFormat": "P95 TTFB - {{name}}"
        },
        {
          "expr": "histogram_quantile(0.99, sum(rate(minio_api_requests_ttfb_seconds_distribution{name=~\"PutObject|CopyObject\", server=~\"$server\"}[5m])) by (le, name))",
          "legendFormat": "P99 TTFB - {{name}}"
        }
      ]
    },
    {
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
      "id": 300,
      "title": "3. Waiting Queues & Internode Latency",
      "type": "row"
    },
    {
      "title": "API Inflight & Waiting Requests Queue",
      "type": "timeseries",
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
      "id": 5,
      "fieldConfig": {
        "defaults": { "unit": "short", "custom": { "drawStyle": "line" } }
      },
      "targets": [
        {
          "expr": "sum(minio_api_requests_waiting_total{server=~\"$server\"}) by (server)",
          "legendFormat": "Waiting Queue - {{server}}"
        },
        {
          "expr": "sum(minio_api_requests_inflight_total{server=~\"$server\"}) by (server)",
          "legendFormat": "Inflight - {{server}}"
        }
      ]
    },
    {
      "title": "Go Goroutines & Internode Dial Latency (ms)",
      "type": "timeseries",
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 },
      "id": 6,
      "fieldConfig": {
        "defaults": { "unit": "short", "custom": { "drawStyle": "line" } }
      },
      "targets": [
        {
          "expr": "go_goroutines{job=~\"minio.*\", instance=~\"$server\"}",
          "legendFormat": "Goroutines - {{instance}}"
        },
        {
          "expr": "minio_system_network_internode_dial_avg_duration_nanoseconds{server=~\"$server\"} / 1000000",
          "legendFormat": "Internode Dial (ms) - {{server}}"
        }
      ]
    }
  ],
  "refresh": "10s",
  "schemaVersion": 38,
  "style": "dark",
  "tags": ["minio", "aistor", "v3", "lock", "sre"],
  "templating": {
    "list": [
      {
        "current": { "selected": true, "text": "default", "value": "default" },
        "name": "datasource",
        "type": "datasource",
        "query": "prometheus"
      },
      {
        "allValue": ".*",
        "includeAll": true,
        "multi": true,
        "name": "server",
        "type": "query",
        "datasource": "${datasource}",
        "query": "label_values(minio_api_requests_total, server)"
      }
    ]
  },
  "time": { "from": "now-1h", "to": "now" },
  "title": "MinIO AIStor - Lock & RPC Correlation (v3)"
}