poc1028b
#!/usr/bin/env python3
"""
MinIO Node Failure Impact Simulator
This script simulates node failures and visualizes the impact on MinIO erasure sets.
It shows which erasure sets are affected, their health status, and generates
an interactive HTML report.
"""
import argparse
import json
import re
import subprocess
import sys
from collections import defaultdict, Counter
from typing import Dict, List, Set, Tuple
def natural_sort_key(text):
"""Generate a key for natural sorting that handles numbers correctly."""
def atoi(text):
return int(text) if text.isdigit() else text
return [atoi(c) for c in re.split(r'(\d+)', text)]
def run_mc_command(alias: str, insecure: bool = False) -> Dict:
"""Run mc admin info command and return JSON output"""
cmd = f"mc admin info {alias} --json"
if insecure:
cmd += " --insecure"
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
if result.returncode != 0:
print(f"Error running mc command: {result.stderr}", file=sys.stderr)
sys.exit(1)
try:
return json.loads(result.stdout)
except json.JSONDecodeError as e:
print(f"Error parsing mc output as JSON: {e}", file=sys.stderr)
print(f"Output was: {result.stdout}", file=sys.stderr)
sys.exit(1)
def get_pod_node_mapping(namespace: str, objectstore_name: str = None) -> Dict[str, str]:
"""Get mapping of pod names to node names using kubectl"""
pod_to_node = {}
label_selectors = [
f'v1.min.io/tenant={objectstore_name}' if objectstore_name else None,
'app=minio',
'app.kubernetes.io/name=minio',
]
for selector in label_selectors:
if not selector:
continue
try:
result = subprocess.run(
['kubectl', '-n', namespace, 'get', 'pods', '-l', selector, '-o', 'json'],
capture_output=True,
text=True,
check=True
)
pods_info = json.loads(result.stdout)
items = pods_info.get('items', [])
if items:
for pod in items:
pod_name = pod['metadata']['name']
node_name = pod['spec'].get('nodeName', 'unknown')
if objectstore_name:
if objectstore_name in pod_name or 'minio' in pod_name.lower():
pod_to_node[pod_name] = node_name
else:
if 'minio' in pod_name.lower():
pod_to_node[pod_name] = node_name
if pod_to_node:
break
except (subprocess.CalledProcessError, json.JSONDecodeError):
continue
# Fallback: get all pods and filter by name
if not pod_to_node:
try:
result = subprocess.run(
['kubectl', '-n', namespace, 'get', 'pods', '-o', 'json'],
capture_output=True,
text=True,
check=True
)
pods_info = json.loads(result.stdout)
for pod in pods_info.get('items', []):
pod_name = pod['metadata']['name']
if 'minio' in pod_name.lower() or (objectstore_name and objectstore_name in pod_name):
node_name = pod['spec'].get('nodeName', 'unknown')
pod_to_node[pod_name] = node_name
except (subprocess.CalledProcessError, json.JSONDecodeError) as e:
print(f"Warning: Could not get pod information: {e}", file=sys.stderr)
return pod_to_node
def analyze_erasure_sets(mc_info: Dict, pod_to_node: Dict[str, str] = None) -> Dict:
"""Analyze erasure set distribution from mc admin info output"""
servers = mc_info.get('info', {}).get('servers', [])
if not servers:
print("Error: No servers found in mc admin info output", file=sys.stderr)
return {}
erasure_sets = defaultdict(list)
for i, server in enumerate(servers):
endpoint = server.get('endpoint', '')
# Extract pod/node name
pod_or_node_name = endpoint.split(':')[0] if ':' in endpoint else endpoint
pod_or_node_name = pod_or_node_name.split('.')[0] if '.' in pod_or_node_name else pod_or_node_name
# Determine actual node
actual_node = 'unknown'
if pod_to_node and pod_or_node_name in pod_to_node:
actual_node = pod_to_node[pod_or_node_name]
else:
# Try fuzzy matching
if pod_to_node:
for pod_name, node_name in pod_to_node.items():
if (pod_name == pod_or_node_name or
pod_or_node_name in pod_name or
pod_name in pod_or_node_name):
actual_node = node_name
break
if actual_node == 'unknown':
actual_node = pod_or_node_name
# Process drives
drives = server.get('drives', [])
for drive in drives:
erasure_set_id = drive.get('set_index')
if erasure_set_id is None:
erasure_set_id = drive.get('set')
if erasure_set_id is not None:
erasure_sets[erasure_set_id].append({
'node': actual_node,
'drive': drive.get('path', ''),
'state': drive.get('state', 'unknown'),
'endpoint': endpoint,
'pod_or_node': pod_or_node_name,
'used_bytes': drive.get('used_bytes', 0),
'total_bytes': drive.get('total_bytes', 0),
})
return erasure_sets
def simulate_node_failures(erasure_sets: Dict, failed_nodes: List[str]) -> Dict:
"""Simulate node failures and calculate impact on erasure sets"""
failed_nodes_set = set(failed_nodes)
impact_analysis = {}
for set_id, drives in erasure_sets.items():
total_drives = len(drives)
affected_drives = [d for d in drives if d['node'] in failed_nodes_set]
affected_count = len(affected_drives)
healthy_count = total_drives - affected_count
# Calculate health status based on erasure coding
# Typical MinIO uses EC:4 (can lose 4 drives) or EC:2 (can lose 2 drives)
# We'll calculate the parity drives based on total drives
if total_drives >= 16:
parity = total_drives // 2 # EC:8 for 16 drives
elif total_drives >= 8:
parity = total_drives // 2 # EC:4 for 8 drives
else:
parity = total_drives // 2 # At least half
data_drives = total_drives - parity
# Determine health status
if affected_count == 0:
status = 'healthy'
status_level = 0
elif affected_count <= parity:
status = 'degraded'
status_level = 1
elif healthy_count >= data_drives:
status = 'critical'
status_level = 2
else:
status = 'failed'
status_level = 3
# Calculate risk percentage
risk_percentage = (affected_count / total_drives) * 100
# Calculate remaining fault tolerance
remaining_tolerance = max(0, parity - affected_count)
impact_analysis[set_id] = {
'total_drives': total_drives,
'affected_drives': affected_count,
'healthy_drives': healthy_count,
'affected_drive_list': affected_drives,
'parity_drives': parity,
'data_drives': data_drives,
'status': status,
'status_level': status_level,
'risk_percentage': risk_percentage,
'remaining_tolerance': remaining_tolerance,
'all_drives': drives
}
return impact_analysis
def generate_html_report(impact_analysis: Dict, failed_nodes: List[str],
all_nodes: Set[str], objectstore_name: str = "MinIO") -> str:
"""Generate interactive HTML report for node failure simulation"""
total_sets = len(impact_analysis)
status_counts = Counter(info['status'] for info in impact_analysis.values())
# Get all unique nodes from erasure sets
nodes_in_sets = set()
for info in impact_analysis.values():
for drive in info['all_drives']:
nodes_in_sets.add(drive['node'])
html = f"""<!DOCTYPE html>
<html>
<head>
<title>MinIO Node Failure Impact Analysis - {objectstore_name}</title>
<meta charset="UTF-8">
<style>
* {{
margin: 0;
padding: 0;
box-sizing: border-box;
}}
body {{
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
min-height: 100vh;
padding: 20px;
}}
.container {{
max-width: 1400px;
margin: 0 auto;
background: white;
border-radius: 20px;
box-shadow: 0 20px 60px rgba(0,0,0,0.3);
overflow: hidden;
}}
.header {{
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
color: white;
padding: 40px;
text-align: center;
}}
.header h1 {{
font-size: 2.5em;
margin-bottom: 10px;
text-shadow: 2px 2px 4px rgba(0,0,0,0.2);
}}
.header .subtitle {{
font-size: 1.2em;
opacity: 0.9;
}}
.summary {{
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 20px;
padding: 30px;
background: #f8f9fa;
}}
.summary-card {{
background: white;
padding: 25px;
border-radius: 15px;
box-shadow: 0 4px 6px rgba(0,0,0,0.1);
transition: transform 0.3s ease, box-shadow 0.3s ease;
}}
.summary-card:hover {{
transform: translateY(-5px);
box-shadow: 0 8px 12px rgba(0,0,0,0.15);
}}
.summary-card .label {{
font-size: 0.9em;
color: #666;
text-transform: uppercase;
letter-spacing: 1px;
margin-bottom: 10px;
}}
.summary-card .value {{
font-size: 2.5em;
font-weight: bold;
color: #333;
}}
.summary-card.failed-nodes {{
background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);
color: white;
}}
.summary-card.failed-nodes .label,
.summary-card.failed-nodes .value {{
color: white;
}}
.summary-card.healthy {{ border-left: 5px solid #28a745; }}
.summary-card.degraded {{ border-left: 5px solid #ffc107; }}
.summary-card.critical {{ border-left: 5px solid #fd7e14; }}
.summary-card.failed {{ border-left: 5px solid #dc3545; }}
.failed-nodes-list {{
background: #fff3cd;
border-left: 4px solid #ffc107;
padding: 20px;
margin: 20px 30px;
border-radius: 10px;
}}
.failed-nodes-list h3 {{
color: #856404;
margin-bottom: 15px;
font-size: 1.3em;
}}
.failed-nodes-list ul {{
list-style: none;
display: grid;
grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
gap: 10px;
}}
.failed-nodes-list li {{
background: white;
padding: 10px 15px;
border-radius: 8px;
border: 2px solid #ffc107;
font-weight: 500;
color: #856404;
}}
.failed-nodes-list li::before {{
content: '⚠️ ';
margin-right: 5px;
}}
.erasure-sets {{
padding: 30px;
}}
.erasure-sets h2 {{
font-size: 2em;
margin-bottom: 25px;
color: #333;
border-bottom: 3px solid #667eea;
padding-bottom: 10px;
}}
.filters {{
margin-bottom: 25px;
display: flex;
gap: 15px;
flex-wrap: wrap;
}}
.filter-btn {{
padding: 10px 20px;
border: 2px solid #667eea;
background: white;
color: #667eea;
border-radius: 25px;
cursor: pointer;
font-weight: 600;
transition: all 0.3s ease;
}}
.filter-btn:hover {{
background: #667eea;
color: white;
transform: scale(1.05);
}}
.filter-btn.active {{
background: #667eea;
color: white;
}}
.erasure-set {{
background: white;
border: 2px solid #e0e0e0;
border-radius: 15px;
margin-bottom: 20px;
overflow: hidden;
transition: all 0.3s ease;
}}
.erasure-set:hover {{
box-shadow: 0 8px 16px rgba(0,0,0,0.1);
transform: translateY(-2px);
}}
.erasure-set.status-healthy {{
border-left: 6px solid #28a745;
}}
.erasure-set.status-degraded {{
border-left: 6px solid #ffc107;
}}
.erasure-set.status-critical {{
border-left: 6px solid #fd7e14;
}}
.erasure-set.status-failed {{
border-left: 6px solid #dc3545;
}}
.erasure-set-header {{
padding: 20px;
background: #f8f9fa;
border-bottom: 1px solid #e0e0e0;
cursor: pointer;
display: flex;
justify-content: space-between;
align-items: center;
}}
.erasure-set-header:hover {{
background: #e9ecef;
}}
.set-title {{
font-size: 1.3em;
font-weight: bold;
color: #333;
}}
.status-badge {{
padding: 8px 16px;
border-radius: 20px;
font-weight: 600;
text-transform: uppercase;
font-size: 0.85em;
letter-spacing: 1px;
}}
.status-badge.healthy {{
background: #d4edda;
color: #155724;
}}
.status-badge.degraded {{
background: #fff3cd;
color: #856404;
}}
.status-badge.critical {{
background: #ffe5d0;
color: #8b3a00;
}}
.status-badge.failed {{
background: #f8d7da;
color: #721c24;
}}
.erasure-set-body {{
padding: 25px;
display: none;
}}
.erasure-set-body.expanded {{
display: block;
}}
.set-stats {{
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 15px;
margin-bottom: 25px;
}}
.stat-item {{
background: #f8f9fa;
padding: 15px;
border-radius: 10px;
border-left: 4px solid #667eea;
}}
.stat-item .stat-label {{
font-size: 0.85em;
color: #666;
margin-bottom: 5px;
}}
.stat-item .stat-value {{
font-size: 1.5em;
font-weight: bold;
color: #333;
}}
.progress-bar {{
width: 100%;
height: 30px;
background: #e0e0e0;
border-radius: 15px;
overflow: hidden;
margin: 20px 0;
position: relative;
}}
.progress-fill {{
height: 100%;
transition: width 0.5s ease;
display: flex;
align-items: center;
justify-content: center;
color: white;
font-weight: bold;
font-size: 0.9em;
}}
.progress-fill.healthy {{ background: linear-gradient(90deg, #28a745, #20c997); }}
.progress-fill.degraded {{ background: linear-gradient(90deg, #ffc107, #ffb300); }}
.progress-fill.critical {{ background: linear-gradient(90deg, #fd7e14, #ff6b00); }}
.progress-fill.failed {{ background: linear-gradient(90deg, #dc3545, #c82333); }}
.drives-grid {{
display: grid;
grid-template-columns: repeat(auto-fill, minmax(300px, 1fr));
gap: 15px;
margin-top: 20px;
}}
.drive-card {{
background: #f8f9fa;
padding: 15px;
border-radius: 10px;
border: 2px solid #e0e0e0;
transition: all 0.3s ease;
}}
.drive-card.affected {{
background: #fff5f5;
border-color: #dc3545;
}}
.drive-card:hover {{
box-shadow: 0 4px 8px rgba(0,0,0,0.1);
transform: translateY(-2px);
}}
.drive-node {{
font-weight: bold;
color: #333;
margin-bottom: 8px;
font-size: 1.1em;
}}
.drive-card.affected .drive-node {{
color: #dc3545;
}}
.drive-card.affected .drive-node::before {{
content: '❌ ';
}}
.drive-card:not(.affected) .drive-node::before {{
content: '✅ ';
}}
.drive-path {{
font-size: 0.9em;
color: #666;
font-family: 'Courier New', monospace;
margin-top: 5px;
}}
.drive-state {{
display: inline-block;
padding: 4px 8px;
border-radius: 5px;
font-size: 0.8em;
margin-top: 8px;
font-weight: 600;
}}
.drive-state.online {{
background: #d4edda;
color: #155724;
}}
.drive-state.offline {{
background: #f8d7da;
color: #721c24;
}}
.legend {{
background: #f8f9fa;
padding: 20px;
margin: 30px;
border-radius: 15px;
border: 2px solid #e0e0e0;
}}
.legend h3 {{
margin-bottom: 15px;
color: #333;
}}
.legend-items {{
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 15px;
}}
.legend-item {{
display: flex;
align-items: center;
gap: 10px;
}}
.legend-color {{
width: 30px;
height: 30px;
border-radius: 8px;
flex-shrink: 0;
}}
.legend-text {{
font-size: 0.95em;
}}
.legend-text strong {{
display: block;
margin-bottom: 3px;
}}
.legend-text small {{
color: #666;
}}
@media (max-width: 768px) {{
.summary {{
grid-template-columns: 1fr;
}}
.drives-grid {{
grid-template-columns: 1fr;
}}
.set-stats {{
grid-template-columns: 1fr;
}}
}}
.expand-icon {{
transition: transform 0.3s ease;
}}
.expanded .expand-icon {{
transform: rotate(180deg);
}}
</style>
</head>
<body>
<div class="container">
<div class="header">
<h1>🔍 Node Failure Impact Analysis</h1>
<div class="subtitle">{objectstore_name} - Erasure Set Health Status</div>
</div>
<div class="summary">
<div class="summary-card failed-nodes">
<div class="label">Failed Nodes</div>
<div class="value">{len(failed_nodes)}</div>
</div>
<div class="summary-card">
<div class="label">Total Erasure Sets</div>
<div class="value">{total_sets}</div>
</div>
<div class="summary-card healthy">
<div class="label">Healthy Sets</div>
<div class="value">{status_counts.get('healthy', 0)}</div>
</div>
<div class="summary-card degraded">
<div class="label">Degraded Sets</div>
<div class="value">{status_counts.get('degraded', 0)}</div>
</div>
<div class="summary-card critical">
<div class="label">Critical Sets</div>
<div class="value">{status_counts.get('critical', 0)}</div>
</div>
<div class="summary-card failed">
<div class="label">Failed Sets</div>
<div class="value">{status_counts.get('failed', 0)}</div>
</div>
</div>
"""
# Failed nodes list
if failed_nodes:
html += """
<div class="failed-nodes-list">
<h3>⚠️ Simulated Failed Nodes</h3>
<ul>
"""
for node in sorted(failed_nodes, key=natural_sort_key):
html += f" <li>{node}</li>\n"
html += """
</ul>
</div>
"""
# Erasure sets section
html += """
<div class="erasure-sets">
<h2>Erasure Sets Analysis</h2>
<div class="filters">
<button class="filter-btn active">All Sets</button>
<button class="filter-btn">Healthy Only</button>
<button class="filter-btn">Degraded Only</button>
<button class="filter-btn">Critical Only</button>
<button class="filter-btn">Failed Only</button>
<button class="filter-btn">All Affected</button>
</div>
"""
# Generate erasure set cards
for set_id in sorted(impact_analysis.keys()):
info = impact_analysis[set_id]
status = info['status']
health_percentage = (info['healthy_drives'] / info['total_drives']) * 100
html += f"""
<div class="erasure-set status-{status}" data-status="{status}">
<div class="erasure-set-header">
<div class="set-title">
Erasure Set {set_id}
<span class="expand-icon">▼</span>
</div>
<span class="status-badge {status}">{status.upper()}</span>
</div>
<div class="erasure-set-body">
<div class="progress-bar">
<div class="progress-fill {status}" style="width: {health_percentage}%">
{info['healthy_drives']}/{info['total_drives']} drives healthy
</div>
</div>
<div class="set-stats">
<div class="stat-item">
<div class="stat-label">Total Drives</div>
<div class="stat-value">{info['total_drives']}</div>
</div>
<div class="stat-item">
<div class="stat-label">Affected Drives</div>
<div class="stat-value" style="color: #dc3545;">{info['affected_drives']}</div>
</div>
<div class="stat-item">
<div class="stat-label">Data Drives</div>
<div class="stat-value">{info['data_drives']}</div>
</div>
<div class="stat-item">
<div class="stat-label">Parity Drives</div>
<div class="stat-value">{info['parity_drives']}</div>
</div>
<div class="stat-item">
<div class="stat-label">Remaining Tolerance</div>
<div class="stat-value" style="color: {'#28a745' if info['remaining_tolerance'] > 0 else '#dc3545'};">
{info['remaining_tolerance']}
</div>
</div>
<div class="stat-item">
<div class="stat-label">Risk Level</div>
<div class="stat-value">{info['risk_percentage']:.1f}%</div>
</div>
</div>
<h4 style="margin: 20px 0 10px 0; color: #333;">Drive Details</h4>
<div class="drives-grid">
"""
# Add drive cards
for drive in sorted(info['all_drives'], key=lambda d: (d['node'], d['drive'])):
is_affected = drive['node'] in failed_nodes
affected_class = 'affected' if is_affected else ''
state_class = drive['state'].lower()
html += f"""
<div class="drive-card {affected_class}">
<div class="drive-node">{drive['node']}</div>
<div class="drive-path">{drive['drive']}</div>
<span class="drive-state {state_class}">{drive['state'].upper()}</span>
</div>
"""
html += """
</div>
</div>
</div>
"""
html += """
</div>
<div class="legend">
<h3>Status Legend</h3>
<div class="legend-items">
<div class="legend-item">
<div class="legend-color" style="background: #28a745;"></div>
<div class="legend-text">
<strong>Healthy</strong>
<small>All drives operational</small>
</div>
</div>
<div class="legend-item">
<div class="legend-color" style="background: #ffc107;"></div>
<div class="legend-text">
<strong>Degraded</strong>
<small>Some drives down, within tolerance</small>
</div>
</div>
<div class="legend-item">
<div class="legend-color" style="background: #fd7e14;"></div>
<div class="legend-text">
<strong>Critical</strong>
<small>Near failure threshold</small>
</div>
</div>
<div class="legend-item">
<div class="legend-color" style="background: #dc3545;"></div>
<div class="legend-text">
<strong>Failed</strong>
<small>Cannot maintain quorum</small>
</div>
</div>
</div>
</div>
</div>
<script>
function toggleSet(header) {
const body = header.nextElementSibling;
const icon = header.querySelector('.expand-icon');
body.classList.toggle('expanded');
icon.classList.toggle('expanded');
}
function filterSets(status) {
const sets = document.querySelectorAll('.erasure-set');
const buttons = document.querySelectorAll('.filter-btn');
// Update active button
buttons.forEach(btn => btn.classList.remove('active'));
event.target.classList.add('active');
// Filter sets
sets.forEach(set => {
if (status === 'all') {
set.style.display = 'block';
} else if (status === 'affected') {
// Show degraded, critical, and failed
const setStatus = set.dataset.status;
if (setStatus !== 'healthy') {
set.style.display = 'block';
} else {
set.style.display = 'none';
}
} else {
if (set.dataset.status === status) {
set.style.display = 'block';
} else {
set.style.display = 'none';
}
}
});
}
// Expand first affected set by default
document.addEventListener('DOMContentLoaded', function() {
const affectedSets = document.querySelectorAll('.erasure-set:not(.status-healthy)');
if (affectedSets.length > 0) {
affectedSets[0].querySelector('.erasure-set-header').click();
}
});
</script>
</body>
</html>
"""
return html
def generate_text_report(impact_analysis: Dict, failed_nodes: List[str]) -> str:
"""Generate text-based report for console output"""
report = []
report.append("=" * 80)
report.append("MinIO Node Failure Impact Simulation")
report.append("=" * 80)
report.append("")
report.append(f"🔴 SIMULATED FAILED NODES ({len(failed_nodes)}):")
for node in sorted(failed_nodes, key=natural_sort_key):
report.append(f" • {node}")
report.append("")
# Summary statistics
total_sets = len(impact_analysis)
status_counts = Counter(info['status'] for info in impact_analysis.values())
report.append("📊 SUMMARY STATISTICS:")
report.append(f" Total Erasure Sets: {total_sets}")
report.append(f" ✅ Healthy: {status_counts.get('healthy', 0)}")
report.append(f" ⚠️ Degraded: {status_counts.get('degraded', 0)}")
report.append(f" 🔥 Critical: {status_counts.get('critical', 0)}")
report.append(f" ❌ Failed: {status_counts.get('failed', 0)}")
report.append("")
# Detailed analysis
report.append("📋 DETAILED ERASURE SET ANALYSIS:")
report.append("-" * 80)
for set_id in sorted(impact_analysis.keys()):
info = impact_analysis[set_id]
# Status icon
status_icons = {
'healthy': '✅',
'degraded': '⚠️',
'critical': '🔥',
'failed': '❌'
}
icon = status_icons.get(info['status'], '❓')
report.append(f"\n{icon} Erasure Set {set_id} - {info['status'].upper()}")
report.append(f" Drives: {info['healthy_drives']}/{info['total_drives']} healthy")
report.append(f" Configuration: {info['data_drives']} data + {info['parity_drives']} parity")
report.append(f" Affected: {info['affected_drives']} drives")
report.append(f" Remaining Tolerance: {info['remaining_tolerance']} drive(s)")
report.append(f" Risk Level: {info['risk_percentage']:.1f}%")
if info['affected_drives'] > 0:
report.append(f" Affected Drives:")
for drive in sorted(info['affected_drive_list'], key=lambda d: (d['node'], d['drive'])):
report.append(f" ❌ {drive['node']}: {drive['drive']}")
report.append("")
report.append("=" * 80)
# Risk assessment
critical_or_failed = sum(1 for info in impact_analysis.values()
if info['status'] in ['critical', 'failed'])
if critical_or_failed > 0:
report.append("⚠️ RISK ASSESSMENT:")
report.append(f" {critical_or_failed} erasure set(s) are in CRITICAL or FAILED state!")
report.append(" Immediate action recommended to prevent data loss.")
else:
degraded = status_counts.get('degraded', 0)
if degraded > 0:
report.append("⚠️ RISK ASSESSMENT:")
report.append(f" {degraded} erasure set(s) are DEGRADED but operational.")
report.append(" Monitor closely and plan for node recovery.")
else:
report.append("✅ RISK ASSESSMENT:")
report.append(" All erasure sets are HEALTHY. No immediate risk.")
report.append("=" * 80)
return "\n".join(report)
def get_all_nodes_from_mc(mc_info: Dict) -> Set[str]:
"""Extract all unique node names from mc admin info"""
nodes = set()
servers = mc_info.get('info', {}).get('servers', [])
for server in servers:
endpoint = server.get('endpoint', '')
node_name = endpoint.split(':')[0] if ':' in endpoint else endpoint
node_name = node_name.split('.')[0] if '.' in node_name else node_name
if node_name and node_name != 'unknown':
nodes.add(node_name)
return nodes
def interactive_node_selection(all_nodes: List[str]) -> List[str]:
"""Interactive CLI for selecting nodes to fail"""
print("\n" + "=" * 80)
print("INTERACTIVE NODE SELECTION")
print("=" * 80)
print("\nAvailable nodes:")
for i, node in enumerate(all_nodes, 1):
print(f" {i}. {node}")
print("\nEnter node numbers to simulate failure (comma-separated, e.g., 1,3,5)")
print("Or enter node names (comma-separated)")
print("Press Enter without input to cancel.")
user_input = input("\n> ").strip()
if not user_input:
return []
failed_nodes = []
# Try to parse as numbers or names
for item in user_input.split(','):
item = item.strip()
# Try as number (1-indexed)
if item.isdigit():
idx = int(item) - 1
if 0 <= idx < len(all_nodes):
failed_nodes.append(all_nodes[idx])
else:
print(f"Warning: Invalid node number: {item}")
else:
# Try as node name
if item in all_nodes:
failed_nodes.append(item)
else:
print(f"Warning: Node not found: {item}")
return failed_nodes
def main():
parser = argparse.ArgumentParser(
description='Simulate MinIO node failures and analyze erasure set impact',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
# Simulate failure of specific nodes
%(prog)s --mc-alias myminio --failed-nodes node-1,node-2,node-3
# Interactive mode - select nodes interactively
%(prog)s --mc-alias myminio --interactive
# Generate HTML report
%(prog)s --mc-alias myminio --failed-nodes node-1 --html-output report.html
# With namespace and objectstore name for better pod mapping
%(prog)s --mc-alias myminio --namespace minio-tenant --name minio \\
--failed-nodes node-1,node-2
"""
)
parser.add_argument('--mc-alias', required=True,
help='MinIO mc alias for admin info command')
parser.add_argument('--failed-nodes',
help='Comma-separated list of node names to simulate failure')
parser.add_argument('--interactive', '-i', action='store_true',
help='Interactive mode to select nodes')
parser.add_argument('--namespace', '-n',
help='Kubernetes namespace (for better pod-to-node mapping)')
parser.add_argument('--name',
help='ObjectStore name (for better pod-to-node mapping)')
parser.add_argument('--html-output',
help='Generate HTML report to specified file')
parser.add_argument('--insecure', action='store_true',
help='Use --insecure flag with mc command')
parser.add_argument('--json-output',
help='Save analysis results as JSON')
args = parser.parse_args()
# Get MinIO server info
print("🔍 Fetching MinIO server information...")
mc_info = run_mc_command(args.mc_alias, args.insecure)
# Get pod-to-node mapping if namespace provided
pod_to_node = None
if args.namespace:
print(f"📍 Fetching pod-to-node mapping from namespace '{args.namespace}'...")
pod_to_node = get_pod_node_mapping(args.namespace, args.name)
if pod_to_node:
print(f" Found {len(pod_to_node)} pods")
# Analyze erasure sets
print("📊 Analyzing erasure set configuration...")
erasure_sets = analyze_erasure_sets(mc_info, pod_to_node)
if not erasure_sets:
print("Error: No erasure sets found", file=sys.stderr)
sys.exit(1)
print(f" Found {len(erasure_sets)} erasure sets")
# Get all nodes
all_nodes_in_sets = set()
for drives in erasure_sets.values():
for drive in drives:
all_nodes_in_sets.add(drive['node'])
all_nodes = sorted(all_nodes_in_sets, key=natural_sort_key)
print(f" Total nodes in cluster: {len(all_nodes)}")
# Determine which nodes to fail
failed_nodes = []
if args.interactive:
failed_nodes = interactive_node_selection(all_nodes)
if not failed_nodes:
print("No nodes selected. Exiting.")
return
elif args.failed_nodes:
failed_nodes = [n.strip() for n in args.failed_nodes.split(',')]
# Validate nodes
invalid_nodes = [n for n in failed_nodes if n not in all_nodes]
if invalid_nodes:
print(f"Error: Invalid node names: {', '.join(invalid_nodes)}", file=sys.stderr)
print(f"Available nodes: {', '.join(all_nodes)}", file=sys.stderr)
sys.exit(1)
else:
print("Error: Either --failed-nodes or --interactive must be specified", file=sys.stderr)
sys.exit(1)
# Simulate failure
print(f"\n⚡ Simulating failure of {len(failed_nodes)} node(s)...")
impact_analysis = simulate_node_failures(erasure_sets, failed_nodes)
# Generate text report (always to console)
text_report = generate_text_report(impact_analysis, failed_nodes)
print("\n" + text_report)
# Generate HTML report if requested
if args.html_output:
print(f"\n📄 Generating HTML report...")
objectstore_name = args.name or "MinIO Cluster"
html_report = generate_html_report(impact_analysis, failed_nodes,
all_nodes_in_sets, objectstore_name)
with open(args.html_output, 'w', encoding='utf-8') as f:
f.write(html_report)
print(f" ✅ HTML report saved to: {args.html_output}")
# Save JSON if requested
if args.json_output:
print(f"\n💾 Saving analysis as JSON...")
json_data = {
'failed_nodes': failed_nodes,
'total_erasure_sets': len(impact_analysis),
'summary': {
'healthy': sum(1 for i in impact_analysis.values() if i['status'] == 'healthy'),
'degraded': sum(1 for i in impact_analysis.values() if i['status'] == 'degraded'),
'critical': sum(1 for i in impact_analysis.values() if i['status'] == 'critical'),
'failed': sum(1 for i in impact_analysis.values() if i['status'] == 'failed'),
},
'erasure_sets': {
str(set_id): {
'status': info['status'],
'total_drives': info['total_drives'],
'affected_drives': info['affected_drives'],
'healthy_drives': info['healthy_drives'],
'parity_drives': info['parity_drives'],
'data_drives': info['data_drives'],
'remaining_tolerance': info['remaining_tolerance'],
'risk_percentage': info['risk_percentage'],
}
for set_id, info in impact_analysis.items()
}
}
with open(args.json_output, 'w') as f:
json.dump(json_data, f, indent=2)
print(f" ✅ JSON data saved to: {args.json_output}")
if __name__ == "__main__":
main()
engineer