26S05b

앞서 정리한 1~4단계 점검 및 테스트 과정을 현장에서 바로 실행할 수 있도록 단계별 Bash 스크립트로 분리하여 작성했습니다.

실행 환경에 맞게 스크립트 상단의 변수(CX6_IFACE, E810_IFACE, STORAGE_IP 등)만 지정해 사용하시면 됩니다.

---

Phase 1: 하드웨어 및 OS 무결성 점검 스크립트 (phase1_hw_check.sh)

#!/usr/bin/env bash
set -euo pipefail

# ================= Configuration =================
CX6_IFACE="${CX6_IFACE:-eth0}"      # ConnectX-6 외부망 인터페이스명
E810_IFACE="${E810_IFACE:-eth1}"    # Intel E810 내부망 인터페이스명
NVME_DEV="${NVME_DEV:-/dev/nvme0n1}" # 테스트할 로컬 NVMe 디바이스
STRESS_TIME="10m"                   # 초기 인수 점검용 (전수 검사 시 1h 권장)
# =================================================

echo "=================================================="
echo " [Phase 1] Hardware & System Integrity Check"
echo "=================================================="

echo ">>> 1. PCIe Device Enumeration (GPU & NIC)"
lspci -nn | grep -iE "3d|vga|mellanox|ethernet controller.*810" || true

echo -e "\n>>> 2. PCIe Link Speed & Width Verification"
# NVIDIA (10de) & Mellanox (15b3) 디바이스의 LnkCap vs LnkSta 비교
for bdf in $(lspci -d 10de: -d 15b3: | awk '{print $1}'); do
    echo "--- Device: $bdf ---"
    lspci -s "$bdf" -vvv | grep -E "LnkCap:|LnkSta:"
done

echo -e "\n>>> 3. NUMA Topology Check"
numactl -H || true
for dev in $(lspci -d 10de: -d 15b3: | awk '{print $1}'); do
    echo "Device $dev -> NUMA Node: $(cat /sys/bus/pci/devices/0000:$dev/numa_node 2>/dev/null || echo 'N/A')"
done

echo -e "\n>>> 4. Kernel Hardware Error Check (AER / MCE / EDAC)"
dmesg -T | grep -iE "mce|edac|aer|pcie.*error|corrupted" | tail -n 20 || echo "No critical hardware errors found in dmesg."

echo -e "\n>>> 5. NIC Link & Firmware Status"
echo "--- ConnectX-6 ($CX6_IFACE) ---"
ethtool -i "$CX6_IFACE" || true
ethtool "$CX6_IFACE" | grep -E "Speed:|Duplex:|Link detected:" || true

echo "--- Intel E810 ($E810_IFACE) ---"
ethtool -i "$E810_IFACE" || true
ethtool "$E810_IFACE" | grep -E "Speed:|Duplex:|Link detected:" || true

echo -e "\n>>> 6. Local NVMe SMART Health & FIO Benchmark"
if command -v nvme >/dev/null 2>&1; then
    nvme smart-log "$NVME_DEV" || true
fi
fio --name=nvme_bench --filename=/tmp/fio_test_tmp --size=5G --rw=write --bs=1M --direct=1 --ioengine=libaio --runtime=15 --group_reporting
rm -f /tmp/fio_test_tmp

echo -e "\n>>> 7. CPU & RAM Stress Test ($STRESS_TIME)"
stress-ng --cpu "$(nproc)" --vm 4 --vm-bytes 80% --verify --timeout "$STRESS_TIME" --metrics-brief

echo -e "\n[Phase 1] Completed successfully."

---

Phase 2: GPU 드라이버 설치 및 번인 테스트 (phase2_gpu_burn.sh)

#!/usr/bin/env bash
set -euo pipefail

# ================= Configuration =================
BURN_DURATION="1800" # gpu-burn 실행 시간 (초 단위, 권장: 3600)
# =================================================

echo "=================================================="
echo " [Phase 2] GPU Stack Setup & Burn-In Test"
echo "=================================================="

echo ">>> 1. Checking Driver, Fabric Manager & Persistence Mode"
nvidia-smi
nvidia-smi -pm 1

if systemctl is-active --quiet nvidia-fabricmanager; then
    echo "nvidia-fabricmanager is running."
else
    echo "Checking Fabric Manager status:"
    systemctl status nvidia-fabricmanager || true
fi

echo -e "\n>>> 2. Topology & NVLink Status"
nvidia-smi topo -m
nvidia-smi nvlink -s || true
nvidia-smi nvlink -e || true

echo -e "\n>>> 3. P2P Bandwidth & Latency Test (via Container)"
docker run --rm --gpus all nvcr.io/nvidia/k8s/cuda-sample:vectorAdd-cuda12.5.0 \
    p2pBandwidthLatencyTest

echo -e "\n>>> 4. DCGM Diagnostic Level 3 (HW & Memory Stress)"
if command -v dcgmi >/dev/null 2>&1; then
    dcgmi diag -r 3
else
    echo "dcgmi not found. Skipping DCGM diag."
fi

echo -e "\n>>> 5. GPU-Burn Full Load Test (${BURN_DURATION}s)"
docker run --rm --gpus all \
    wilic/gpu-burn:latest \
    "$BURN_DURATION"

echo -e "\n[Phase 2] Completed successfully."

---

Phase 3: vLLM 서빙 및 연동 테스트 (phase3_vllm_verify.sh)

#!/usr/bin/env bash
set -euo pipefail

# ================= Configuration =================
GPU_HOST_IP="${GPU_HOST_IP:-127.0.0.1}" # 외부망 통신 시 GPU의 ConnectX-6 IP
MODEL_PATH="/data/models/Llama-3.1-8B-Instruct"
SERVED_NAME="llama-3.1-8b"
PORT="8000"
# =================================================

echo "=================================================="
echo " [Phase 3] vLLM Deployment & Ingestion Test"
echo "=================================================="

echo ">>> 1. Kernel Network Parameter Adjustment (PBR / Asymmetric Routing Safe)"
sysctl -w net.ipv4.conf.all.rp_filter=2
sysctl -w net.ipv4.conf.default.rp_filter=2

echo ">>> 2. Launching vLLM Test Container (Single GPU / Eager Mode)"
docker rm -f vllm-test >/dev/null 2>&1 || true
docker run -d --name vllm-test \
    --runtime nvidia \
    --gpus '"device=0"' \
    -v /data/models:/models \
    -p "${PORT}:8000" \
    --ipc=host \
    vllm/vllm-openai:latest \
    --model "/models/$(basename "$MODEL_PATH")" \
    --served-model-name "$SERVED_NAME" \
    --port 8000 \
    --max-model-len 4096 \
    --gpu-memory-utilization 0.90 \
    --enforce-eager

echo "Waiting for vLLM server to become healthy..."
until curl -s "http://localhost:${PORT}/health" >/dev/null 2>&1; do
    sleep 3
    echo -n "."
done
echo -e "\nvLLM is ready."

echo -e "\n>>> 3. Local API Completion Test"
curl -s -X POST "http://localhost:${PORT}/v1/chat/completions" \
    -H "Content-Type: application/json" \
    -d "{
      \"model\": \"$SERVED_NAME\",
      \"messages\": [{\"role\": \"user\", \"content\": \"Ping test from GPU Node local.\"}],
      \"max_tokens\": 30
    }" | grep -o '"content":"[^"]*"' || true

echo -e "\n>>> 4. Remote Test Command (Execute on Compute Cluster Node)"
cat <<EOF
[Compute Cluster Node Execution Command]
curl -X POST http://${GPU_HOST_IP}:${PORT}/v1/chat/completions \\
  -H "Content-Type: application/json" \\
  -d '{
    "model": "$SERVED_NAME",
    "messages": [{"role": "user", "content": "Hello from Compute Cluster"}],
    "max_tokens": 50
  }'
EOF

echo -e "\n[Phase 3] Completed successfully."

---

Phase 4: 스토리지 네트워크 & I/O 벤치마크 (phase4_storage_test.sh)

#!/usr/bin/env bash
set -euo pipefail

# ================= Configuration =================
E810_IFACE="${E810_IFACE:-eth1}"
STORAGE_IP="${STORAGE_IP:-10.100.0.10}"     # 스토리지 클러스터 타깃 IP
STORAGE_SUBNET="${STORAGE_SUBNET:-10.100.0.0/16}"
MOUNT_DIR="${MOUNT_DIR:-/mnt/storage}"
# =================================================

echo "=================================================="
echo " [Phase 4] Storage Interconnect & I/O Benchmark"
echo "=================================================="

echo ">>> 1. Static Route & Jumbo Frame (MTU 9000) Setup"
ip link set dev "$E810_IFACE" mtu 9000
ip route replace "$STORAGE_SUBNET" dev "$E810_IFACE" proto static || true

echo "Checking Path MTU with DF bit set (ICMP payload 8972 + 28 = 9000 bytes):"
ping -c 3 -M do -s 8972 "$STORAGE_IP"

echo -e "\n>>> 2. Network Layer Throughput Benchmark (iperf3)"
echo "Target Storage Server must be running: 'iperf3 -s'"
iperf3 -c "$STORAGE_IP" -P 8 -t 15 -O 2

echo -e "\n>>> 3. Storage I/O Benchmark (POSIX / NFS Mount)"
if mountpoint -q "$MOUNT_DIR"; then
    echo "Running FIO Sequential Read on mounted storage ($MOUNT_DIR)..."
    fio --name=storage_seq_read \
        --directory="$MOUNT_DIR" \
        --rw=read \
        --bs=1M \
        --size=20G \
        --numjobs=8 \
        --iodepth=16 \
        --ioengine=libaio \
        --direct=1 \
        --group_reporting
else
    echo "Directory $MOUNT_DIR is not mounted. Skipping POSIX FIO."
    echo "If S3 Object Storage is used, run MinIO Warp or s3-benchmark instead:"
    echo "  warp get --host=<S3_ENDPOINT> --access-key=<KEY> --secret-key=<SECRET> --bucket=<BUCKET> --concurrent=8"
fi

echo -e "\n[Phase 4] Completed successfully."

===