26Z20a

{
  "annotations": { "list": [] },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
      "id": 100,
      "title": "Lock Overview & Peer Stall Detection",
      "type": "row"
    },
    {
      "title": "Lock Latency: Local vs Distributed (P99 & Max)",
      "type": "timeseries",
      "gridPos": { "h": 8, "w": 16, "x": 0, "y": 1 },
      "id": 1,
      "fieldConfig": {
        "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineInterpolation": "smooth" } }
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.99, sum(rate(minio_locks_dist_waiting_duration_seconds_bucket{instance=~\"$pod\"}[5m])) by (le))",
          "legendFormat": "Dist Lock P99"
        },
        {
          "expr": "max(minio_locks_dist_max_latency_seconds{instance=~\"$pod\"})",
          "legendFormat": "Dist Lock MAX (Peer Stall Indicator)"
        },
        {
          "expr": "histogram_quantile(0.99, sum(rate(minio_locks_local_waiting_duration_seconds_bucket{instance=~\"$pod\"}[5m])) by (le))",
          "legendFormat": "Local Lock P99"
        }
      ]
    },
    {
      "title": "Top Pods by Dist Max Latency",
      "type": "bargauge",
      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 },
      "id": 2,
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "green", "value": null },
              { "color": "orange", "value": 1 },
              { "color": "red", "value": 10 }
            ]
          }
        }
      },
      "targets": [
        {
          "expr": "topk(10, max_over_time(minio_locks_dist_max_latency_seconds[5m]))",
          "legendFormat": "{{instance}}"
        }
      ]
    },
    {
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
      "id": 200,
      "title": "Hot Shard & Erasure Set Correlation",
      "type": "row"
    },
    {
      "title": "Erasure Set Lock Request Rate (Hot Shard Detection)",
      "type": "timeseries",
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 },
      "id": 3,
      "fieldConfig": {
        "defaults": { "unit": "reqps", "custom": { "drawStyle": "line" } }
      },
      "targets": [
        {
          "expr": "sum(rate(minio_locks_dist_waiting_duration_seconds_count[5m])) by (set_index)",
          "legendFormat": "Set {{set_index}}"
        }
      ]
    },
    {
      "title": "S3 Mutations (Put/Copy) by Erasure Set",
      "type": "timeseries",
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 },
      "id": 4,
      "fieldConfig": {
        "defaults": { "unit": "reqps", "custom": { "drawStyle": "line" } }
      },
      "targets": [
        {
          "expr": "sum(rate(minio_api_requests_total{api=~\"PutObject|CopyObject|PutObjectPart|DeleteObject.*\", instance=~\"$pod\"}[5m])) by (set_index, api)",
          "legendFormat": "Set {{set_index}} - {{api}}"
        }
      ]
    },
    {
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
      "id": 300,
      "title": "Peer Scheduling & Process Diagnostics",
      "type": "row"
    },
    {
      "title": "MinIO Process Goroutines (Lock Contention / Waiting Queues)",
      "type": "timeseries",
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
      "id": 5,
      "fieldConfig": {
        "defaults": { "unit": "short", "custom": { "drawStyle": "line" } }
      },
      "targets": [
        {
          "expr": "go_goroutines{job=~\"minio.*\", instance=~\"$pod\"}",
          "legendFormat": "{{instance}}"
        }
      ]
    },
    {
      "title": "API Time vs Lock Waiting Duration (P95)",
      "type": "timeseries",
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 },
      "id": 6,
      "fieldConfig": {
        "defaults": { "unit": "s", "custom": { "drawStyle": "line" } }
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(minio_api_time_seconds_bucket{api=~\"PutObject|CopyObject\", instance=~\"$pod\"}[5m])) by (le))",
          "legendFormat": "API Latency (Put/Copy)"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(minio_locks_dist_waiting_duration_seconds_bucket{instance=~\"$pod\"}[5m])) by (le))",
          "legendFormat": "Lock Waiting Latency"
        }
      ]
    }
  ],
  "refresh": "10s",
  "schemaVersion": 38,
  "style": "dark",
  "tags": ["minio", "aistor", "lock", "sre"],
  "templating": {
    "list": [
      {
        "current": { "selected": true, "text": "default", "value": "default" },
        "name": "datasource",
        "type": "datasource",
        "query": "prometheus"
      },
      {
        "allValue": ".*",
        "includeAll": true,
        "multi": true,
        "name": "pod",
        "type": "query",
        "datasource": "${datasource}",
        "query": "label_values(minio_locks_dist_waiting_duration_seconds_bucket, instance)"
      }
    ]
  },
  "time": { "from": "now-1h", "to": "now" },
  "title": "MinIO AIStor - Distributed Lock & Hot Shard Analysis"
}