26S26h3

{
  "schema_version": "v1.2",
  "collector_source": "robusta-headless-standalone",
  "incident_meta": {
    "incident_id": "INC-PROD-20260927150710",
    "alarm_name": "MinIONodePeerUnreachable",
    "severity": "P1-CRITICAL",
    "timestamp": "2026-09-27T15:07:10.365159+00:00",
    "cluster_name": "prod-kr01-k8s",
    "target": {
      "namespace": "storage",
      "node_name": "kr01-aistor-dn03",
      "pod_name": "minio-aistor-pool-0-2",
      "mount_path": "/data/nvme02"
    }
  },
  "symptom_summary": "Peer interconnect dropped between kr01-aistor-dn03 and other cluster nodes, causing dsync quorum loss",
  "evidence": {
    "k8s_events": [
      {
        "type": "Warning",
        "reason": "NodeNetworkFlapping",
        "message": "Node kr01-aistor-dn03 NIC dropped BGP routes / interface packet drop spike",
        "source": "cilium-agent",
        "count": 8
      }
    ],
    "pod_logs_tail": [
      "[ERROR] (peer-rpc) Failed to dial peer node kr01-aistor-dn03:9000: dial tcp kr01-aistor-dn03:9000: i/o timeout",
      "[ERROR] (dsync) Lost quorum on locker dsync for distributed pool: expected 8 votes, received 4",
      "[WARN] (cluster) Node kr01-aistor-dn03 temporarily marked as offline in peer table"
    ],
    "prometheus_metrics": {
      "minio_node_interconnect_rx_errors_total": {
        "query": "rate(minio_node_interconnect_rx_errors_total{instance=\"kr01-aistor-dn03\"}[1m])",
        "baseline": "0.0",
        "values": [
          {
            "timestamp": 1790520730,
            "value": "0.00"
          },
          {
            "timestamp": 1790520790,
            "value": "0.00"
          },
          {
            "timestamp": 1790520850,
            "value": "0.00"
          },
          {
            "timestamp": 1790520910,
            "value": "0.00"
          },
          {
            "timestamp": 1790520970,
            "value": "0.00"
          },
          {
            "timestamp": 1790521030,
            "value": "0.00"
          },
          {
            "timestamp": 1790521090,
            "value": "0.00"
          },
          {
            "timestamp": 1790521150,
            "value": "160.66"
          },
          {
            "timestamp": 1790521210,
            "value": "243.82"
          },
          {
            "timestamp": 1790521270,
            "value": "350.29"
          },
          {
            "timestamp": 1790521330,
            "value": "416.20"
          },
          {
            "timestamp": 1790521390,
            "value": "413.71"
          },
          {
            "timestamp": 1790521450,
            "value": "427.36"
          },
          {
            "timestamp": 1790521510,
            "value": "431.97"
          },
          {
            "timestamp": 1790521570,
            "value": "412.54"
          }
        ]
      },
      "minio_cluster_nodes_offline_total": {
        "query": "minio_cluster_nodes_offline_total",
        "baseline": "0",
        "values": [
          {
            "timestamp": 1790520730,
            "value": "0.00"
          },
          {
            "timestamp": 1790520790,
            "value": "0.00"
          },
          {
            "timestamp": 1790520850,
            "value": "0.00"
          },
          {
            "timestamp": 1790520910,
            "value": "0.00"
          },
          {
            "timestamp": 1790520970,
            "value": "0.00"
          },
          {
            "timestamp": 1790521030,
            "value": "0.00"
          },
          {
            "timestamp": 1790521090,
            "value": "0.00"
          },
          {
            "timestamp": 1790521150,
            "value": "0.00"
          },
          {
            "timestamp": 1790521210,
            "value": "0.41"
          },
          {
            "timestamp": 1790521270,
            "value": "0.60"
          },
          {
            "timestamp": 1790521330,
            "value": "0.79"
          },
          {
            "timestamp": 1790521390,
            "value": "0.97"
          },
          {
            "timestamp": 1790521450,
            "value": "0.97"
          },
          {
            "timestamp": 1790521510,
            "value": "1.02"
          },
          {
            "timestamp": 1790521570,
            "value": "0.99"
          }
        ]
      }
    }
  },
  "system_context": {
    "component": "minio-aistor",
    "storage_driver": "directpv",
    "k8s_version": "v1.28.10"
  }
}