26S26h2

{
  "schema_version": "v1.2",
  "collector_source": "robusta-headless-standalone",
  "incident_meta": {
    "incident_id": "INC-PROD-20260927150709",
    "alarm_name": "MinIODriveLatencyHigh",
    "severity": "P1-CRITICAL",
    "timestamp": "2026-09-27T15:07:09.441736+00:00",
    "cluster_name": "prod-kr01-k8s",
    "target": {
      "namespace": "storage",
      "node_name": "kr01-aistor-dn03",
      "pod_name": "minio-aistor-pool-0-2",
      "mount_path": "/data/nvme02"
    }
  },
  "symptom_summary": "Client S3 PutObject timeout (HTTP 503 Slow Down) and I/O wait spike on drive /data/nvme02 on kr01-aistor-dn03",
  "evidence": {
    "k8s_events": [
      {
        "type": "Warning",
        "reason": "StorageLatencySpike",
        "message": "Persistent volume attached to minio-aistor-pool-0-2 via /data/nvme02 exceeded SLO response time (>3000ms)",
        "source": "directpv-driver-kr01-aistor-dn03",
        "count": 14
      },
      {
        "type": "Warning",
        "reason": "Unhealthy",
        "message": "Readiness probe failed on minio-aistor-pool-0-2: HTTP probe failed with statuscode: 503",
        "source": "kubelet",
        "count": 3
      }
    ],
    "pod_logs_tail": [
      "15:02:09 [INFO] (api) S3 API request received: PutObject bucket=corp-ai-training-data size=12400192B",
      "15:02:24 [WARN] (storage) Drive /data/nvme02 on kr01-aistor-dn03 responded slowly: 4.821s latency threshold exceeded",
      "15:02:31 [ERROR] (dsync) dsync: timeout while acquiring write lock on peer kr01-aistor-dn03:9000 for path corp-ai-training-data/weights_v2.bin",
      "15:02:32 [ERROR] (api) PutObject(bucket=corp-ai-training-data) err: context deadline exceeded while waiting on /data/nvme02",
      "15:02:37 [WARN] (http) Returned HTTP 503 Slow Down to client ai-training-worker-6789f",
      "15:02:44 [WARN] (storage) Drive /data/nvme02 marked as degraded due to multiple I/O timeouts"
    ],
    "prometheus_metrics": {
      "minio_node_drive_total_duration_microseconds": {
        "query": "minio_node_drive_total_duration_microseconds{pod=\"minio-aistor-pool-0-2\", drive=\"/data/nvme02\"}",
        "baseline": "15,000 µs (15ms)",
        "values": [
          {
            "timestamp": 1790520729,
            "value": "15682.42"
          },
          {
            "timestamp": 1790520789,
            "value": "14700.00"
          },
          {
            "timestamp": 1790520849,
            "value": "15087.14"
          },
          {
            "timestamp": 1790520909,
            "value": "14343.30"
          },
          {
            "timestamp": 1790520969,
            "value": "15239.86"
          },
          {
            "timestamp": 1790521029,
            "value": "15544.86"
          },
          {
            "timestamp": 1790521089,
            "value": "15600.79"
          },
          {
            "timestamp": 1790521149,
            "value": "15660.46"
          },
          {
            "timestamp": 1790521209,
            "value": "2003380.92"
          },
          {
            "timestamp": 1790521269,
            "value": "2830121.62"
          },
          {
            "timestamp": 1790521329,
            "value": "3701179.28"
          },
          {
            "timestamp": 1790521389,
            "value": "4864505.04"
          },
          {
            "timestamp": 1790521449,
            "value": "4587462.44"
          },
          {
            "timestamp": 1790521509,
            "value": "4586162.11"
          },
          {
            "timestamp": 1790521569,
            "value": "4838179.99"
          }
        ]
      },
      "go_goroutines": {
        "query": "go_goroutines{pod=\"minio-aistor-pool-0-2\"}",
        "baseline": "2,200",
        "values": [
          {
            "timestamp": 1790520729,
            "value": "2126.38"
          },
          {
            "timestamp": 1790520789,
            "value": "2173.46"
          },
          {
            "timestamp": 1790520849,
            "value": "2224.84"
          },
          {
            "timestamp": 1790520909,
            "value": "2211.98"
          },
          {
            "timestamp": 1790520969,
            "value": "2259.82"
          },
          {
            "timestamp": 1790521029,
            "value": "2203.74"
          },
          {
            "timestamp": 1790521089,
            "value": "2192.50"
          },
          {
            "timestamp": 1790521149,
            "value": "2160.47"
          },
          {
            "timestamp": 1790521209,
            "value": "8866.92"
          },
          {
            "timestamp": 1790521269,
            "value": "12624.61"
          },
          {
            "timestamp": 1790521329,
            "value": "16020.59"
          },
          {
            "timestamp": 1790521389,
            "value": "19571.95"
          },
          {
            "timestamp": 1790521449,
            "value": "18172.18"
          },
          {
            "timestamp": 1790521509,
            "value": "19160.16"
          },
          {
            "timestamp": 1790521569,
            "value": "18456.95"
          }
        ]
      },
      "minio_s3_requests_waiting_total": {
        "query": "minio_s3_requests_waiting_total{pod=\"minio-aistor-pool-0-2\"}",
        "baseline": "12",
        "values": [
          {
            "timestamp": 1790520729,
            "value": "11.48"
          },
          {
            "timestamp": 1790520789,
            "value": "11.82"
          },
          {
            "timestamp": 1790520849,
            "value": "11.72"
          },
          {
            "timestamp": 1790520909,
            "value": "12.42"
          },
          {
            "timestamp": 1790520969,
            "value": "12.07"
          },
          {
            "timestamp": 1790521029,
            "value": "11.44"
          },
          {
            "timestamp": 1790521089,
            "value": "11.52"
          },
          {
            "timestamp": 1790521149,
            "value": "12.41"
          },
          {
            "timestamp": 1790521209,
            "value": "12.50"
          },
          {
            "timestamp": 1790521269,
            "value": "568.72"
          },
          {
            "timestamp": 1790521329,
            "value": "863.41"
          },
          {
            "timestamp": 1790521389,
            "value": "1147.96"
          },
          {
            "timestamp": 1790521449,
            "value": "1449.43"
          },
          {
            "timestamp": 1790521509,
            "value": "1384.89"
          },
          {
            "timestamp": 1790521569,
            "value": "1429.09"
          }
        ]
      },
      "node_disk_io_time_seconds_total": {
        "query": "rate(node_disk_io_time_seconds_total{instance=\"kr01-aistor-dn03\"}[1m])",
        "baseline": "0.15",
        "values": [
          {
            "timestamp": 1790520729,
            "value": "0.15"
          },
          {
            "timestamp": 1790520789,
            "value": "0.14"
          },
          {
            "timestamp": 1790520849,
            "value": "0.14"
          },
          {
            "timestamp": 1790520909,
            "value": "0.14"
          },
          {
            "timestamp": 1790520969,
            "value": "0.16"
          },
          {
            "timestamp": 1790521029,
            "value": "0.16"
          },
          {
            "timestamp": 1790521089,
            "value": "0.14"
          },
          {
            "timestamp": 1790521149,
            "value": "0.15"
          },
          {
            "timestamp": 1790521209,
            "value": "0.47"
          },
          {
            "timestamp": 1790521269,
            "value": "0.67"
          },
          {
            "timestamp": 1790521329,
            "value": "0.83"
          },
          {
            "timestamp": 1790521389,
            "value": "1.00"
          },
          {
            "timestamp": 1790521449,
            "value": "0.95"
          },
          {
            "timestamp": 1790521509,
            "value": "0.96"
          },
          {
            "timestamp": 1790521569,
            "value": "0.97"
          }
        ]
      }
    }
  },
  "system_context": {
    "component": "minio-aistor",
    "storage_driver": "directpv",
    "k8s_version": "v1.28.10"
  }
}