26Y28p2
{
"__inputs": [
{
"name": "DS_PROMETHEUS",
"label": "Prometheus",
"description": "MinIO AIStor 메트릭을 스크레이핑하는 Prometheus 데이터소스",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus"
}
],
"__requires": [
{ "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.4.0" },
{ "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" },
{ "type": "panel", "id": "timeseries", "name": "Time series", "version": "" },
{ "type": "panel", "id": "stat", "name": "Stat", "version": "" },
{ "type": "panel", "id": "table", "name": "Table", "version": "" },
{ "type": "panel", "id": "heatmap", "name": "Heatmap", "version": "" },
{ "type": "panel", "id": "text", "name": "Text", "version": "" },
{ "type": "panel", "id": "row", "name": "Row", "version": "" }
],
"id": null,
"uid": "minio-aistor-perf-02",
"title": "MinIO AIStor Performance (2/2) - CPU/Memory, TTFB, Goroutine",
"description": "3) 노드별 CPU/Memory, 4) TTFB 분포(전체+버킷별), 5) Goroutine 급증과 영향 범위. 128노드 규모를 고려해 트렌드(avg/max)+Top-N 테이블 조합으로 구성. 버킷별 TTFB는 /minio/metrics/v3/bucket/api 서브패스 스크레이프가 활성화되어 있어야 데이터가 채워집니다.",
"tags": ["minio", "aistor", "performance", "s3", "ttfb", "goroutine"],
"timezone": "browser",
"editable": true,
"graphTooltip": 1,
"refresh": "30s",
"schemaVersion": 39,
"version": 1,
"time": { "from": "now-6h", "to": "now" },
"timepicker": { "refresh_intervals": ["10s", "30s", "1m", "5m", "15m", "30m", "1h"] },
"templating": {
"list": [
{ "name": "datasource", "type": "datasource", "query": "prometheus", "hide": 0, "label": "Prometheus", "current": {} },
{
"name": "cluster",
"type": "query",
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"query": "label_values(minio_cluster_health_nodes_online_count, cluster)",
"label": "Cluster",
"hide": 0,
"includeAll": false,
"multi": false,
"current": {},
"refresh": 2,
"sort": 1
},
{
"name": "server",
"type": "query",
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"query": "label_values(minio_system_drive_used_bytes{cluster=~\"$cluster\"}, server)",
"label": "Node (drill-down, optional)",
"hide": 0,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": { "text": "All", "value": "$__all" },
"refresh": 2,
"sort": 1
},
{
"name": "verb",
"type": "query",
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"query": "label_values(minio_api_requests_ttfb_seconds_distribution{cluster=~\"$cluster\"}, name)",
"label": "API Verb (TTFB)",
"hide": 0,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": { "text": "All", "value": "$__all" },
"refresh": 2,
"sort": 1
},
{
"name": "bucket",
"type": "query",
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"query": "label_values(minio_bucket_api_ttfb_seconds_distribution{cluster=~\"$cluster\"}, bucket)",
"label": "Bucket (TTFB drill-down)",
"hide": 0,
"includeAll": false,
"multi": false,
"current": {},
"refresh": 2,
"sort": 1
},
{
"name": "topn",
"type": "custom",
"query": "10,15,20,30",
"label": "Top N",
"hide": 0,
"current": { "text": "15", "value": "15" },
"options": [
{ "text": "10", "value": "10", "selected": false },
{ "text": "15", "value": "15", "selected": true },
{ "text": "20", "value": "20", "selected": false },
{ "text": "30", "value": "30", "selected": false }
]
}
]
},
"panels": [
{ "id": 200, "type": "row", "title": "3. CPU / Memory - 노드별 (128 노드)", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "collapsed": false, "panels": [] },
{
"id": 30, "type": "stat", "title": "Cluster Avg CPU Load %",
"description": "minio_system_cpu_load_perc 클러스터 평균",
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "avg(minio_system_cpu_load_perc{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "avg load %", "refId": "A" }],
"fieldConfig": { "defaults": { "unit": "percent", "decimals": 1, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] } }, "overrides": [] },
"options": { "graphMode": "area", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "auto" }
},
{
"id": 31, "type": "stat", "title": "Cluster Avg IOWait %",
"description": "minio_system_cpu_avg_iowait - 스토리지 병목으로 인한 CPU 대기. 높으면 디스크 latency 패널과 상관관계 확인",
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "avg(minio_system_cpu_avg_iowait{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "avg iowait %", "refId": "A" }],
"fieldConfig": { "defaults": { "unit": "percent", "decimals": 1, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 10 }, { "color": "red", "value": 25 }] } }, "overrides": [] },
"options": { "graphMode": "area", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "auto" }
},
{
"id": 32, "type": "stat", "title": "Cluster Avg Memory Used %",
"description": "minio_system_memory_used_perc 클러스터 평균",
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "avg(minio_system_memory_used_perc{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "avg mem used %", "refId": "A" }],
"fieldConfig": { "defaults": { "unit": "percent", "decimals": 1, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 90 }] } }, "overrides": [] },
"options": { "graphMode": "area", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "auto" }
},
{
"id": 33, "type": "stat", "title": "Min Memory Available (노드 중 최소)",
"description": "minio_system_memory_available 최소값 - 메모리 여유가 가장 적은 노드",
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "min(minio_system_memory_available{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "min available", "refId": "A" }],
"fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } }, "overrides": [] },
"options": { "graphMode": "area", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "auto" }
},
{
"id": 34, "type": "timeseries", "title": "CPU Load % - Avg / Max (클러스터 추이)",
"description": "avg는 전반적 추세, max는 가장 부하가 높은 노드 1개. 128노드 전체를 나열하지 않고 추이+이상치만 추적",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "avg(minio_system_cpu_load_perc{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "avg", "refId": "A" },
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(minio_system_cpu_load_perc{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "max (worst node)", "refId": "B" }
],
"fieldConfig": { "defaults": { "unit": "percent", "custom": { "drawStyle": "line", "fillOpacity": 10, "showPoints": "never" } }, "overrides": [{ "matcher": { "id": "byName", "options": "max (worst node)" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] }] },
"options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }
},
{
"id": 35, "type": "timeseries", "title": "CPU IOWait % - Avg / Max (스토리지 병목 신호)",
"description": "높은 iowait는 디스크가 CPU 대비 병목임을 의미. minio_system_drive_api_latency_micros와 함께 확인",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "avg(minio_system_cpu_avg_iowait{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "avg", "refId": "A" },
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(minio_system_cpu_avg_iowait{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "max (worst node)", "refId": "B" }
],
"fieldConfig": { "defaults": { "unit": "percent", "custom": { "drawStyle": "line", "fillOpacity": 10, "showPoints": "never" } }, "overrides": [{ "matcher": { "id": "byName", "options": "max (worst node)" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] }] },
"options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }
},
{
"id": 36, "type": "timeseries", "title": "Memory Used % - Avg / Max",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "avg(minio_system_memory_used_perc{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "avg", "refId": "A" },
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(minio_system_memory_used_perc{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "max (worst node)", "refId": "B" }
],
"fieldConfig": { "defaults": { "unit": "percent", "custom": { "drawStyle": "line", "fillOpacity": 10, "showPoints": "never" } }, "overrides": [{ "matcher": { "id": "byName", "options": "max (worst node)" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] }] },
"options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }
},
{
"id": 37, "type": "timeseries", "title": "Memory Available - Avg / Min",
"description": "minio_system_memory_available. min은 메모리가 가장 부족한 노드 - OOM 위험 조기 감지",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "avg(minio_system_memory_available{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "avg", "refId": "A" },
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "min(minio_system_memory_available{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "min (worst node)", "refId": "B" }
],
"fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "fillOpacity": 10, "showPoints": "never" } }, "overrides": [{ "matcher": { "id": "byName", "options": "min (worst node)" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] }] },
"options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }
},
{
"id": 38, "type": "table", "title": "Top-N 노드 - CPU Load %",
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 21 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "topk($topn, minio_system_cpu_load_perc{cluster=~\"$cluster\", server=~\"$server\"})", "format": "table", "instant": true, "refId": "A" }],
"transformations": [
{ "id": "organize", "options": { "excludeByName": { "Time": true }, "renameByName": { "Value": "cpu_load_%" } } },
{ "id": "sortBy", "options": { "fields": {}, "sort": [{ "field": "cpu_load_%", "desc": true }] } }
],
"fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "color-background" } }, "unit": "percent" }, "overrides": [] },
"options": { "showHeader": true }
},
{
"id": 39, "type": "table", "title": "Top-N 노드 - IOWait %",
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 21 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "topk($topn, minio_system_cpu_avg_iowait{cluster=~\"$cluster\", server=~\"$server\"})", "format": "table", "instant": true, "refId": "A" }],
"transformations": [
{ "id": "organize", "options": { "excludeByName": { "Time": true }, "renameByName": { "Value": "iowait_%" } } },
{ "id": "sortBy", "options": { "fields": {}, "sort": [{ "field": "iowait_%", "desc": true }] } }
],
"fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "color-background" } }, "unit": "percent" }, "overrides": [] },
"options": { "showHeader": true }
},
{
"id": 40, "type": "table", "title": "Top-N 노드 - Memory Used %",
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 21 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "topk($topn, minio_system_memory_used_perc{cluster=~\"$cluster\", server=~\"$server\"})", "format": "table", "instant": true, "refId": "A" }],
"transformations": [
{ "id": "organize", "options": { "excludeByName": { "Time": true }, "renameByName": { "Value": "mem_used_%" } } },
{ "id": "sortBy", "options": { "fields": {}, "sort": [{ "field": "mem_used_%", "desc": true }] } }
],
"fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "color-background" } }, "unit": "percent" }, "overrides": [] },
"options": { "showHeader": true }
},
{
"id": 41, "type": "timeseries", "title": "MinIO Process RSS (Resident Memory) - Avg / Max",
"description": "minio_system_process_resident_memory_bytes - MinIO 프로세스 자체 메모리 사용량 (OS 메모리와 별개로 프로세스 레벨 확인)",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "avg(minio_system_process_resident_memory_bytes{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "avg", "refId": "A" },
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(minio_system_process_resident_memory_bytes{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "max (worst node)", "refId": "B" }
],
"fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "fillOpacity": 10, "showPoints": "never" } }, "overrides": [{ "matcher": { "id": "byName", "options": "max (worst node)" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] }] },
"options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }
},
{
"id": 42, "type": "timeseries", "title": "MinIO Process CPU 사용률 - Avg / Max (코어 환산)",
"description": "rate(minio_system_process_cpu_total_seconds[5m]) - 프로세스가 실제 사용 중인 CPU 코어 수(1.0 = 코어 1개 풀 사용)",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "avg(rate(minio_system_process_cpu_total_seconds{cluster=~\"$cluster\", server=~\"$server\"}[5m]))", "legendFormat": "avg cores", "refId": "A" },
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(rate(minio_system_process_cpu_total_seconds{cluster=~\"$cluster\", server=~\"$server\"}[5m]))", "legendFormat": "max (worst node)", "refId": "B" }
],
"fieldConfig": { "defaults": { "unit": "short", "decimals": 2, "custom": { "drawStyle": "line", "fillOpacity": 10, "showPoints": "never" } }, "overrides": [{ "matcher": { "id": "byName", "options": "max (worst node)" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] }] },
"options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }
},
{
"id": 43, "type": "stat", "title": "File Descriptor 사용률 - 최대 (클러스터)",
"description": "minio_system_process_file_descriptor_open_total / _limit_total 최대값. 0.9 초과 시 ulimit 상향 필요",
"gridPos": { "h": 6, "w": 6, "x": 0, "y": 37 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(minio_system_process_file_descriptor_open_total{cluster=~\"$cluster\", server=~\"$server\"} / minio_system_process_file_descriptor_limit_total{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "max fd ratio", "refId": "A" }],
"fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 0.9 }] } }, "overrides": [] },
"options": { "graphMode": "area", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "auto" }
},
{
"id": 44, "type": "table", "title": "Top-N 노드 - File Descriptor 사용률",
"gridPos": { "h": 6, "w": 18, "x": 6, "y": 37 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "topk($topn, minio_system_process_file_descriptor_open_total{cluster=~\"$cluster\", server=~\"$server\"} / minio_system_process_file_descriptor_limit_total{cluster=~\"$cluster\", server=~\"$server\"})", "format": "table", "instant": true, "refId": "A" }],
"transformations": [
{ "id": "organize", "options": { "excludeByName": { "Time": true }, "renameByName": { "Value": "fd_ratio" } } },
{ "id": "sortBy", "options": { "fields": {}, "sort": [{ "field": "fd_ratio", "desc": true }] } }
],
"fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "color-background" } }, "unit": "percentunit" }, "overrides": [] },
"options": { "showHeader": true }
},
{ "id": 201, "type": "row", "title": "4. TTFB 분포 - 전체 및 버킷별", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, "collapsed": false, "panels": [] },
{
"id": 50, "type": "timeseries", "title": "전체 TTFB p50/p90/p99 (선택 Verb 기준)",
"description": "minio_api_requests_ttfb_seconds_distribution 기반. 상단 $verb 변수로 오퍼레이션 선택 (기본 All이면 verb 혼합 - GetObject/PutObject를 나눠서 보는 것을 권장)",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le, name) (rate(minio_api_requests_ttfb_seconds_distribution{cluster=~\"$cluster\", server=~\"$server\", name=~\"$verb\"}[5m])))", "legendFormat": "p50 {{name}}", "refId": "A" },
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.90, sum by (le, name) (rate(minio_api_requests_ttfb_seconds_distribution{cluster=~\"$cluster\", server=~\"$server\", name=~\"$verb\"}[5m])))", "legendFormat": "p90 {{name}}", "refId": "B" },
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le, name) (rate(minio_api_requests_ttfb_seconds_distribution{cluster=~\"$cluster\", server=~\"$server\", name=~\"$verb\"}[5m])))", "legendFormat": "p99 {{name}}", "refId": "C" }
],
"fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "fillOpacity": 5, "showPoints": "never" } }, "overrides": [] },
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }, "tooltip": { "mode": "multi" } }
},
{
"id": 51, "type": "heatmap", "title": "전체 TTFB 분포 히트맵 (선택 Verb)",
"description": "minio_api_requests_ttfb_seconds_distribution 히스토그램 버킷(le)을 시간에 따른 히트맵으로 표시 - 분포 형태(쌍봉/꼬리)를 한눈에 확인",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (le) (rate(minio_api_requests_ttfb_seconds_distribution{cluster=~\"$cluster\", server=~\"$server\", name=~\"$verb\"}[5m]))", "format": "heatmap", "legendFormat": "{{le}}", "refId": "A" }],
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
"options": { "calculate": false, "yAxis": { "unit": "s" }, "color": { "mode": "scheme", "scheme": "Spectral", "reverse": true }, "tooltip": { "show": true, "yHistogram": true } }
},
{
"id": 52, "type": "table", "title": "Top-N 버킷 - p99 TTFB",
"description": "minio_bucket_api_ttfb_seconds_distribution 기준 버킷별 p99 TTFB 상위. /minio/metrics/v3/bucket/api 스크레이프 필요",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "topk($topn, histogram_quantile(0.99, sum by (le, bucket) (rate(minio_bucket_api_ttfb_seconds_distribution{cluster=~\"$cluster\", server=~\"$server\"}[5m]))))", "format": "table", "instant": true, "refId": "A" }],
"transformations": [
{ "id": "organize", "options": { "excludeByName": { "Time": true }, "renameByName": { "Value": "p99_ttfb_s" } } },
{ "id": "sortBy", "options": { "fields": {}, "sort": [{ "field": "p99_ttfb_s", "desc": true }] } }
],
"fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "color-background" } }, "unit": "s" }, "overrides": [] },
"options": { "showHeader": true }
},
{
"id": 53, "type": "heatmap", "title": "버킷별 TTFB 분포 히트맵 ($bucket 선택)",
"description": "상단 $bucket 변수로 특정 버킷을 선택해 해당 버킷의 TTFB 분포를 드릴다운",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 52 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (le) (rate(minio_bucket_api_ttfb_seconds_distribution{cluster=~\"$cluster\", server=~\"$server\", bucket=\"$bucket\"}[5m]))", "format": "heatmap", "legendFormat": "{{le}}", "refId": "A" }],
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
"options": { "calculate": false, "yAxis": { "unit": "s" }, "color": { "mode": "scheme", "scheme": "Spectral", "reverse": true }, "tooltip": { "show": true, "yHistogram": true } }
},
{
"id": 54, "type": "timeseries", "title": "Top-10 버킷 p99 TTFB 추이",
"description": "요청량 상위가 아니라 지연 상위 버킷의 시계열 추이 - 특정 버킷만 서서히 느려지는 회귀(regression) 조기 감지용",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 60 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "topk(10, histogram_quantile(0.99, sum by (le, bucket) (rate(minio_bucket_api_ttfb_seconds_distribution{cluster=~\"$cluster\", server=~\"$server\"}[5m]))))", "legendFormat": "{{bucket}}", "refId": "A" }],
"fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "fillOpacity": 5, "showPoints": "never" } }, "overrides": [] },
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } }
},
{ "id": 202, "type": "row", "title": "5. Goroutine 급증 및 영향 범위", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 68 }, "collapsed": false, "panels": [] },
{
"id": 60, "type": "stat", "title": "Goroutine 합계 (현재)",
"description": "minio_system_process_go_routine_total 클러스터 합계",
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 69 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(minio_system_process_go_routine_total{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "total goroutines", "refId": "A" }],
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } }, "overrides": [] },
"options": { "graphMode": "area", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "auto" }
},
{
"id": 61, "type": "stat", "title": "Goroutine 최대 노드 (현재)",
"description": "특정 노드에 goroutine이 쏠려있는지 확인",
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 69 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(minio_system_process_go_routine_total{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "max node", "refId": "A" }],
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 5000 }, { "color": "red", "value": 20000 }] } }, "overrides": [] },
"options": { "graphMode": "area", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "auto" }
},
{
"id": 62, "type": "stat", "title": "Requests Inflight (현재)",
"description": "goroutine과 상관관계 확인용 - 참고: 1번 대시보드와 동일 지표",
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 69 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(minio_api_requests_inflight_total{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "inflight", "refId": "A" }],
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } }, "overrides": [] },
"options": { "graphMode": "area", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "auto" }
},
{
"id": 63, "type": "stat", "title": "Internode Error Rate (현재)",
"description": "goroutine 급증이 노드 간 통신 문제와 겹치는지 확인용",
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 69 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(minio_system_network_internode_errors_total{cluster=~\"$cluster\", server=~\"$server\"}[5m]))", "legendFormat": "internode err/s", "refId": "A" }],
"fieldConfig": { "defaults": { "unit": "short", "decimals": 2, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }] } }, "overrides": [] },
"options": { "graphMode": "area", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "auto" }
},
{
"id": 64, "type": "timeseries", "title": "Goroutine 수 - Avg / Max / Sum 추이",
"description": "급증 시점을 먼저 이 패널에서 찾고, 오른쪽/아래 상관관계 패널로 원인을 좁혀나가는 방식 권장",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 73 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "avg(minio_system_process_go_routine_total{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "avg", "refId": "A" },
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(minio_system_process_go_routine_total{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "max (worst node)", "refId": "B" },
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(minio_system_process_go_routine_total{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "sum (cluster)", "refId": "C" }
],
"fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "fillOpacity": 8, "showPoints": "never" } }, "overrides": [
{ "matcher": { "id": "byName", "options": "max (worst node)" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] },
{ "matcher": { "id": "byName", "options": "sum (cluster)" }, "properties": [{ "id": "custom.axisPlacement", "value": "right" }, { "id": "color", "value": { "mode": "fixed", "fixedColor": "purple" } }] }
] },
"options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }
},
{
"id": 65, "type": "timeseries", "title": "Goroutine(Sum) vs Inflight/Waiting Requests",
"description": "goroutine 급증이 특정 요청 급증(fan-out)과 동시에 발생하는지 오버레이로 확인. 오른쪽 축=goroutine, 왼쪽 축=요청 수",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 73 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(minio_api_requests_inflight_total{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "inflight", "refId": "A" },
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(minio_api_requests_waiting_total{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "waiting", "refId": "B" },
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(minio_system_process_go_routine_total{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "goroutines (sum)", "refId": "C" }
],
"fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "fillOpacity": 8, "showPoints": "never" } }, "overrides": [
{ "matcher": { "id": "byName", "options": "goroutines (sum)" }, "properties": [{ "id": "custom.axisPlacement", "value": "right" }, { "id": "color", "value": { "mode": "fixed", "fixedColor": "purple" } }] }
] },
"options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }
},
{
"id": 66, "type": "timeseries", "title": "Goroutine(Sum) vs Internode Error Rate",
"description": "goroutine 급증이 노드 간 통신 실패와 동시에 나타나면 분산 락/erasure 쓰기 fan-out 지연이 원인일 가능성",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 81 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(minio_system_network_internode_errors_total{cluster=~\"$cluster\", server=~\"$server\"}[5m]))", "legendFormat": "internode err/s", "refId": "A" },
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(minio_system_process_go_routine_total{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "goroutines (sum)", "refId": "B" }
],
"fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "fillOpacity": 8, "showPoints": "never" } }, "overrides": [
{ "matcher": { "id": "byName", "options": "goroutines (sum)" }, "properties": [{ "id": "custom.axisPlacement", "value": "right" }, { "id": "color", "value": { "mode": "fixed", "fixedColor": "purple" } }] }
] },
"options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }
},
{
"id": 67, "type": "timeseries", "title": "Goroutine(Sum) vs 백그라운드 스캐너 활동",
"description": "minio_scanner_bucket_scans_started/_finished (이름 접미사는 배포에서 /minio/metrics/v3/scanner?list 로 검증 권장). 스캐너/ILM/복제 등 백그라운드 작업과 겹치는 시점 확인",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 81 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(minio_scanner_bucket_scans_started{cluster=~\"$cluster\", server=~\"$server\"}[5m]))", "legendFormat": "scans started/s", "refId": "A" },
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(minio_system_process_go_routine_total{cluster=~\"$cluster\", server=~\"$server\"})", "legendFormat": "goroutines (sum)", "refId": "B" }
],
"fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "fillOpacity": 8, "showPoints": "never" } }, "overrides": [
{ "matcher": { "id": "byName", "options": "goroutines (sum)" }, "properties": [{ "id": "custom.axisPlacement", "value": "right" }, { "id": "color", "value": { "mode": "fixed", "fixedColor": "purple" } }] }
] },
"options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }
},
{
"id": 68, "type": "table", "title": "Top-N 노드 - Goroutine 수 (현재)",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 89 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "topk($topn, minio_system_process_go_routine_total{cluster=~\"$cluster\", server=~\"$server\"})", "format": "table", "instant": true, "refId": "A" }],
"transformations": [
{ "id": "organize", "options": { "excludeByName": { "Time": true }, "renameByName": { "Value": "goroutines" } } },
{ "id": "sortBy", "options": { "fields": {}, "sort": [{ "field": "goroutines", "desc": true }] } }
],
"fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "color-background" } }, "unit": "short" }, "overrides": [] },
"options": { "showHeader": true }
},
{
"id": 69, "type": "text", "title": "Goroutine 급증 대응 가이드",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 89 },
"options": {
"mode": "markdown",
"content": "**Goroutine 급증 시 조사 순서**\n\n1. 위 패널들로 급증 '시점'과 '노드'를 특정한다 (Top-N 테이블 활용)\n2. 같은 시점에 inflight/waiting, internode error, 스캐너/복제/ILM 활동이 동반 상승했는지 확인한다\n3. 원인 후보가 좁혀지면 해당 노드/시간대에 대해 `mc support profile` 로 goroutine/CPU/메모리 프로파일을 캡처하고 `go tool pprof` 로 분석한다\n4. 실시간으로 재현 중이면 `mc admin trace`로 어떤 API 호출이 몰리는지 즉시 확인한다\n\n메트릭만으로는 '왜'까지는 알기 어렵습니다 - 지표는 언제/어디서를 좁히는 용도이고, 근본 원인은 프로파일링으로 확인하는 2단계 접근을 권장합니다."
}
}
]
}
engineer