26Y27x6

{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 100,
      "title": "Goroutine Outliers & Runtime Health",
      "type": "row"
    },
    {
      "id": 1,
      "title": "Goroutine Distribution Heatmap",
      "type": "heatmap",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 1
      },
      "options": {
        "calculate": false,
        "color": {
          "mode": "scheme",
          "scheme": "Spectral"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "minio_system_process_go_routine_total{cluster=~\"$cluster\", server=~\"$server\"}",
          "format": "heatmap",
          "legendFormat": "{{server}}",
          "refId": "A"
        }
      ]
    },
    {
      "id": 2,
      "title": "Top N Goroutine Spike Servers",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 1
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line"
          },
          "unit": "short"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "topk($topn, minio_system_process_go_routine_total{cluster=~\"$cluster\", server=~\"$server\"})",
          "legendFormat": "{{server}}",
          "refId": "A"
        }
      ]
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 9
      },
      "id": 200,
      "title": "Correlation Analysis (Background Tasks & Internode RPC)",
      "type": "row"
    },
    {
      "id": 3,
      "title": "Goroutines vs Inflight Requests",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 10
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line"
          },
          "unit": "short"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(minio_system_process_go_routine_total{cluster=~\"$cluster\"})",
          "legendFormat": "Total Goroutines",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(minio_api_requests_inflight_total{cluster=~\"$cluster\"})",
          "legendFormat": "Inflight Requests",
          "refId": "B"
        }
      ]
    },
    {
      "id": 4,
      "title": "Internode RPC Failed Calls vs Lock Waiters",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 10
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line"
          },
          "unit": "short"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(minio_internode_failed_calls_total{cluster=~\"$cluster\"}[$interval]))",
          "legendFormat": "Internode RPC Failures",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(minio_api_requests_waiting_total{cluster=~\"$cluster\"})",
          "legendFormat": "Waiting Requests",
          "refId": "B"
        }
      ]
    },
    {
      "id": 5,
      "title": "Background Job Active Workers (Replication / Scanner / ILM)",
      "type": "timeseries",
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 18
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "drawStyle": "line"
          },
          "unit": "short"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(minio_cluster_replication_active_workers{cluster=~\"$cluster\"})",
          "legendFormat": "Replication Active Workers",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(minio_ilm_transition_active_tasks{cluster=~\"$cluster\"})",
          "legendFormat": "ILM Active Transition Tasks",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(minio_batch_jobs_in_progress{cluster=~\"$cluster\"})",
          "legendFormat": "Batch Jobs In Progress",
          "refId": "C"
        }
      ]
    }
  ],
  "refresh": "30s",
  "schemaVersion": 38,
  "style": "dark",
  "tags": [
    "minio",
    "aistor",
    "goroutine"
  ],
  "templating": {
    "list": [
      {
        "name": "datasource",
        "type": "datasource",
        "query": "prometheus",
        "hide": 0,
        "label": "Prometheus",
        "current": {}
      },
      {
        "name": "cluster",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "query": "label_values(minio_cluster_health_nodes_online_count, cluster)",
        "label": "Cluster",
        "hide": 0,
        "includeAll": true,
        "multi": true,
        "allValue": ".*",
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "refresh": 2,
        "sort": 1
      },
      {
        "name": "server",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "query": "label_values(minio_system_process_go_routine_total{cluster=~\"$cluster\"}, server)",
        "label": "Node (server)",
        "hide": 0,
        "includeAll": true,
        "multi": true,
        "allValue": ".*",
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "refresh": 2,
        "sort": 1
      },
      {
        "name": "topn",
        "type": "custom",
        "query": "10,15,20,30",
        "label": "Top N",
        "hide": 0,
        "current": {
          "text": "15",
          "value": "15"
        },
        "options": [
          {
            "text": "10",
            "value": "10",
            "selected": false
          },
          {
            "text": "15",
            "value": "15",
            "selected": true
          },
          {
            "text": "20",
            "value": "20",
            "selected": false
          },
          {
            "text": "30",
            "value": "30",
            "selected": false
          }
        ]
      },
      {
        "current": {
          "selected": true,
          "text": "2m",
          "value": "2m"
        },
        "hide": 0,
        "label": "Interval",
        "name": "interval",
        "options": [
          {
            "selected": false,
            "text": "30s",
            "value": "30s"
          },
          {
            "selected": false,
            "text": "1m",
            "value": "1m"
          },
          {
            "selected": true,
            "text": "2m",
            "value": "2m"
          },
          {
            "selected": false,
            "text": "5m",
            "value": "5m"
          },
          {
            "selected": false,
            "text": "10m",
            "value": "10m"
          }
        ],
        "query": "30s,1m,2m,5m,10m",
        "type": "interval"
      }
    ]
  },
  "time": {
    "from": "now-3h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m"
    ]
  },
  "timezone": "browser",
  "title": "5. MinIO AIStor - Goroutine Correlation Analytics",
  "uid": "minio-aistor-5-goroutine-correlation",
  "version": 1
}