26S29c
#!/usr/bin/env bash
# ==========================================================================
# STAGE 3a-8: Prefix Caching A/B 검증
#
# 목적:
# 1) Prefix Cache가 실제로 hit 되는지 확인
# 2) Cached token 수 확인
# 3) KV cache capacity / usage 확인
# 4) APC ON/OFF의 TTFT 차이 측정
# 5) benchmark가 동일 prefix를 사용하는지 검증
#
# 기본 workload:
# 32K fixed prefix + 32K random suffix + 256 output
# 총 input ~= 64K tokens/request
#
# 주의:
# 이 테스트에서는 64K 전체가 cache되는 것이 아니라
# 동일한 32K prefix 부분만 APC 대상이다.
# ==========================================================================
set -Eeuo pipefail
export HF_HUB_OFFLINE=1
export TRANSFORMERS_OFFLINE=1
export HF_DATASETS_OFFLINE=1
# --------------------------------------------------------------------------
# Configuration
# --------------------------------------------------------------------------
LOCAL_MODEL_ROOT="${LOCAL_MODEL_ROOT:-/mnt/local-nvme-cache/models}"
MODEL_70B="${MODEL_70B:-${LOCAL_MODEL_ROOT}/Meta-Llama-3.1-70B-Instruct}"
[ -d "${MODEL_70B}" ] || {
echo "ERROR: 모델 없음: ${MODEL_70B}"
exit 1
}
TP_SIZE="${TP_SIZE:-8}"
VLLM_PORT="${VLLM_PORT:-8000}"
SERVER_HOST="${SERVER_HOST:-127.0.0.1}"
# --------------------------------------------------------------------------
# Benchmark configuration
# --------------------------------------------------------------------------
DATASET_NAME="${DATASET_NAME:-random}"
# Fixed prefix
PREFIX_LEN="${PREFIX_LEN:-32000}"
# Random suffix
SUFFIX_LEN="${SUFFIX_LEN:-32768}"
# Output
OUTPUT_LEN="${OUTPUT_LEN:-256}"
NUM_PROMPTS="${NUM_PROMPTS:-30}"
MAX_CONCURRENCY="${MAX_CONCURRENCY:-8}"
# --------------------------------------------------------------------------
# Logging
# --------------------------------------------------------------------------
LOG_DIR="/var/log/b300_validation/stage3a8_prefixab_$(date +%Y%m%d_%H%M%S)"
mkdir -p "${LOG_DIR}"
source ~/vllm-bench-env/bin/activate
log() {
echo -e "\n\033[1;32m[$(date '+%Y-%m-%d %H:%M:%S')] $1\033[0m"
}
warn() {
echo -e "\033[1;33mWARNING: $1\033[0m"
}
error() {
echo -e "\033[1;31mERROR: $1\033[0m"
}
# --------------------------------------------------------------------------
# Cleanup
# --------------------------------------------------------------------------
SERVER_PID=""
METRICS_PID=""
cleanup() {
if [[ -n "${METRICS_PID}" ]]; then
kill "${METRICS_PID}" 2>/dev/null || true
METRICS_PID=""
fi
if [[ -n "${SERVER_PID}" ]]; then
kill -TERM "${SERVER_PID}" 2>/dev/null || true
wait "${SERVER_PID}" 2>/dev/null || true
SERVER_PID=""
fi
}
trap cleanup EXIT INT TERM
# --------------------------------------------------------------------------
# Check vLLM version
# --------------------------------------------------------------------------
log "vLLM version 확인"
python3 - <<'PYEOF' | tee "${LOG_DIR}/vllm_version.txt"
import vllm
print(vllm.__version__)
PYEOF
# --------------------------------------------------------------------------
# Check supported CLI options
# --------------------------------------------------------------------------
log "Prefix Cache 관련 CLI 확인"
python3 -m vllm.entrypoints.openai.api_server --help 2>&1 |
grep -E \
'enable-prefix-caching|prefix-match-unit|gpu-memory-utilization|max-model-len' \
| tee "${LOG_DIR}/prefix_cache_cli_options.txt" || true
# --------------------------------------------------------------------------
# Start server
# --------------------------------------------------------------------------
start_server() {
local tag="$1"
local cache_flag="$2"
SERVER_LOG="${LOG_DIR}/server_${tag}.log"
METRICS_LOG="${LOG_DIR}/metrics_${tag}.log"
log "============================================================"
log "서버 기동: ${tag}"
log "Prefix Cache: ${cache_flag}"
log "============================================================"
python3 -m vllm.entrypoints.openai.api_server \
--model "${MODEL_70B}" \
--served-model-name "llama-3.1-70b" \
--tensor-parallel-size "${TP_SIZE}" \
--gpu-memory-utilization 0.90 \
--max-model-len 65536 \
${cache_flag} \
--port "${VLLM_PORT}" \
> "${SERVER_LOG}" 2>&1 &
SERVER_PID=$!
log "vLLM PID=${SERVER_PID}"
# ----------------------------------------------------------------------
# Wait for server
# ----------------------------------------------------------------------
log "vLLM API readiness 대기"
local ready=0
for i in $(seq 1 120); do
if curl -fsS \
"http://${SERVER_HOST}:${VLLM_PORT}/v1/models" \
> /dev/null 2>&1; then
ready=1
break
fi
if ! kill -0 "${SERVER_PID}" 2>/dev/null; then
error "vLLM 서버가 종료되었습니다."
echo
echo "===== SERVER LOG ====="
tail -100 "${SERVER_LOG}"
exit 1
fi
sleep 5
done
if [[ "${ready}" != "1" ]]; then
error "vLLM 서버 readiness timeout"
echo
echo "===== SERVER LOG ====="
tail -100 "${SERVER_LOG}"
exit 1
fi
log "vLLM API ready"
# ----------------------------------------------------------------------
# Save model information
# ----------------------------------------------------------------------
curl -fsS \
"http://${SERVER_HOST}:${VLLM_PORT}/v1/models" \
> "${LOG_DIR}/models_${tag}.json"
# ----------------------------------------------------------------------
# Save initial metrics
# ----------------------------------------------------------------------
curl -fsS \
"http://${SERVER_HOST}:${VLLM_PORT}/metrics" \
> "${LOG_DIR}/metrics_${tag}_before.txt"
# ----------------------------------------------------------------------
# Start metrics collector
# ----------------------------------------------------------------------
log "Runtime metrics 수집 시작"
(
while true; do
{
echo
echo "============================================================"
echo "TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')"
echo "============================================================"
curl -fsS \
"http://${SERVER_HOST}:${VLLM_PORT}/metrics" 2>/dev/null |
grep -E \
'prefix_cache_(queries|hits)|prompt_tokens_cached|kv_cache_usage_perc|gpu_cache_usage_perc' \
|| true
} >> "${METRICS_LOG}"
sleep 1
done
) &
METRICS_PID=$!
# ----------------------------------------------------------------------
# Show KV cache related startup information
# ----------------------------------------------------------------------
log "KV cache startup 정보"
grep -Ei \
'KV cache|GPU KV|num_gpu_blocks|gpu.*block|cache size|block' \
"${SERVER_LOG}" \
| tail -30 \
| tee "${LOG_DIR}/kv_cache_startup_${tag}.txt" \
|| true
}
# --------------------------------------------------------------------------
# Run benchmark
# --------------------------------------------------------------------------
run_benchmark() {
local tag="$1"
local RESULT_FILE="${LOG_DIR}/${tag}.json"
log "============================================================"
log "Benchmark 시작: ${tag}"
log "============================================================"
log "Workload:"
log " Dataset = ${DATASET_NAME}"
log " Fixed prefix = ${PREFIX_LEN}"
log " Random suffix = ${SUFFIX_LEN}"
log " Output = ${OUTPUT_LEN}"
log " Prompts = ${NUM_PROMPTS}"
log " Concurrency = ${MAX_CONCURRENCY}"
log " Approx input = $((PREFIX_LEN + SUFFIX_LEN)) tokens"
log "============================================================"
# ----------------------------------------------------------------------
# Benchmark
# ----------------------------------------------------------------------
vllm bench serve \
--backend openai-chat \
--endpoint /v1/chat/completions \
--host "${SERVER_HOST}" \
--port "${VLLM_PORT}" \
--model "llama-3.1-70b" \
--tokenizer "${MODEL_70B}" \
--dataset-name "${DATASET_NAME}" \
--random-input-len "${SUFFIX_LEN}" \
--random-output-len "${OUTPUT_LEN}" \
--random-prefix-len "${PREFIX_LEN}" \
--num-prompts "${NUM_PROMPTS}" \
--max-concurrency "${MAX_CONCURRENCY}" \
--request-rate inf \
--save-result \
--result-dir "${LOG_DIR}" \
--result-filename "${tag}.json"
# ----------------------------------------------------------------------
# Verify result
# ----------------------------------------------------------------------
if [[ ! -s "${RESULT_FILE}" ]]; then
error "Benchmark result 없음: ${RESULT_FILE}"
exit 1
fi
# ----------------------------------------------------------------------
# Save final metrics
# ----------------------------------------------------------------------
curl -fsS \
"http://${SERVER_HOST}:${VLLM_PORT}/metrics" \
> "${LOG_DIR}/metrics_${tag}_after.txt"
# ----------------------------------------------------------------------
# Stop metrics collector
# ----------------------------------------------------------------------
if [[ -n "${METRICS_PID}" ]]; then
kill "${METRICS_PID}" 2>/dev/null || true
METRICS_PID=""
fi
log "Benchmark 완료: ${tag}"
# ----------------------------------------------------------------------
# Show cache metrics
# ----------------------------------------------------------------------
echo
echo "===== ${tag}: Prefix Cache Metrics ====="
grep -E \
'prefix_cache_(queries|hits)|prompt_tokens_cached|kv_cache_usage_perc|gpu_cache_usage_perc' \
"${LOG_DIR}/metrics_${tag}_after.txt" \
|| true
}
# --------------------------------------------------------------------------
# Stop server
# --------------------------------------------------------------------------
stop_server() {
log "vLLM 서버 종료"
if [[ -n "${METRICS_PID}" ]]; then
kill "${METRICS_PID}" 2>/dev/null || true
METRICS_PID=""
fi
if [[ -n "${SERVER_PID}" ]]; then
kill -TERM "${SERVER_PID}" 2>/dev/null || true
wait "${SERVER_PID}" 2>/dev/null || true
SERVER_PID=""
fi
sleep 5
}
# ==========================================================================
# CASE 1: Prefix Cache ON
# ==========================================================================
run_case_on() {
start_server \
"prefix_cache_on" \
"--enable-prefix-caching"
run_benchmark \
"prefix_cache_on"
stop_server
}
# ==========================================================================
# CASE 2: Prefix Cache OFF
# ==========================================================================
run_case_off() {
start_server \
"prefix_cache_off" \
"--no-enable-prefix-caching"
run_benchmark \
"prefix_cache_off"
stop_server
}
# ==========================================================================
# Execute A/B
# ==========================================================================
log "============================================================"
log "STAGE 3a-8 Prefix Caching A/B"
log "============================================================"
log "Model : ${MODEL_70B}"
log "TP : ${TP_SIZE}"
log "Prefix : ${PREFIX_LEN}"
log "Suffix : ${SUFFIX_LEN}"
log "Output : ${OUTPUT_LEN}"
log "Prompts : ${NUM_PROMPTS}"
log "Concurrency : ${MAX_CONCURRENCY}"
log "Log Dir : ${LOG_DIR}"
run_case_on
run_case_off
# ==========================================================================
# Result Analysis
# ==========================================================================
log "=== 결과 분석 ==="
python3 - "${LOG_DIR}" <<'PYEOF'
import json
import os
import re
import statistics
import sys
log_dir = sys.argv[1]
# --------------------------------------------------------------------------
# JSON loader
# --------------------------------------------------------------------------
def load_json(tag):
path = os.path.join(log_dir, f"{tag}.json")
if not os.path.exists(path):
return None
try:
with open(path) as f:
return json.load(f)
except Exception as e:
print(f"WARNING: JSON parse failed: {path}: {e}")
return None
# --------------------------------------------------------------------------
# Metrics parser
# --------------------------------------------------------------------------
def parse_metrics(path):
result = {
"prefix_cache_queries": None,
"prefix_cache_hits": None,
"prompt_tokens_cached": None,
"kv_cache_usage_perc": None,
}
if not os.path.exists(path):
return result
with open(path, errors="ignore") as f:
text = f.read()
patterns = {
"prefix_cache_queries":
r'(?:vllm:)?prefix_cache_queries(?:\{[^}]*\})?\s+([0-9.eE+-]+)',
"prefix_cache_hits":
r'(?:vllm:)?prefix_cache_hits(?:\{[^}]*\})?\s+([0-9.eE+-]+)',
"prompt_tokens_cached":
r'(?:vllm:)?prompt_tokens_cached(?:\{[^}]*\})?\s+([0-9.eE+-]+)',
"kv_cache_usage_perc":
r'(?:vllm:)?kv_cache_usage_perc(?:\{[^}]*\})?\s+([0-9.eE+-]+)',
}
for key, pattern in patterns.items():
matches = re.findall(pattern, text)
if matches:
try:
result[key] = float(matches[-1])
except ValueError:
pass
return result
# --------------------------------------------------------------------------
# Startup log parser
# --------------------------------------------------------------------------
def parse_startup(tag):
path = os.path.join(
log_dir,
f"kv_cache_startup_{tag}.txt"
)
if not os.path.exists(path):
return []
with open(path, errors="ignore") as f:
return [
line.strip()
for line in f
if line.strip()
]
# --------------------------------------------------------------------------
# Benchmark JSON
# --------------------------------------------------------------------------
on = load_json("prefix_cache_on")
off = load_json("prefix_cache_off")
metrics_on = parse_metrics(
os.path.join(
log_dir,
"metrics_prefix_cache_on_after.txt"
)
)
metrics_off = parse_metrics(
os.path.join(
log_dir,
"metrics_prefix_cache_off_after.txt"
)
)
startup_on = parse_startup("prefix_cache_on")
startup_off = parse_startup("prefix_cache_off")
# --------------------------------------------------------------------------
# Extract benchmark values
# --------------------------------------------------------------------------
def value(data, key):
if not data:
return None
value = data.get(key)
if isinstance(value, (int, float)):
return value
return None
ttft_on = value(on, "median_ttft_ms")
ttft_off = value(off, "median_ttft_ms")
e2e_on = value(on, "median_e2e_latency_ms")
e2e_off = value(off, "median_e2e_latency_ms")
tpot_on = value(on, "median_tpot_ms")
tpot_off = value(off, "median_tpot_ms")
input_tps_on = value(on, "input_throughput")
input_tps_off = value(off, "input_throughput")
output_tps_on = value(on, "output_throughput")
output_tps_off = value(off, "output_throughput")
# --------------------------------------------------------------------------
# Calculate cache hit ratio
# --------------------------------------------------------------------------
queries = metrics_on["prefix_cache_queries"]
hits = metrics_on["prefix_cache_hits"]
hit_ratio = None
if queries is not None and queries > 0 and hits is not None:
hit_ratio = hits / queries * 100.0
# --------------------------------------------------------------------------
# Cached tokens / request
# --------------------------------------------------------------------------
cached_tokens_per_request = None
cached_tokens = metrics_on["prompt_tokens_cached"]
if cached_tokens is not None:
# This is an approximate interpretation because
# prompt_tokens_cached is a cumulative metric.
cached_tokens_per_request = (
cached_tokens / 30.0
)
# --------------------------------------------------------------------------
# Speedup
# --------------------------------------------------------------------------
ttft_speedup = None
if ttft_on and ttft_on > 0 and ttft_off:
ttft_speedup = ttft_off / ttft_on
e2e_speedup = None
if e2e_on and e2e_on > 0 and e2e_off:
e2e_speedup = e2e_off / e2e_on
# --------------------------------------------------------------------------
# Markdown report
# --------------------------------------------------------------------------
report = []
report.append(
"# Stage 3a-8 Prefix Caching A/B 결과"
)
report.append("")
report.append(
"## Workload"
)
report.append("")
report.append(
"| 항목 | 값 |"
)
report.append("|---|---:|")
report.append(
"| Fixed Prefix | 32,000 tokens |"
)
report.append(
"| Random Suffix | 32,768 tokens |"
)
report.append(
"| Approx Input | ~64,768 tokens |"
)
report.append(
"| Output | 256 tokens |"
)
report.append(
"| Requests | 30 |"
)
report.append(
"| Concurrency | 8 |"
)
report.append("")
report.append(
"## Latency"
)
report.append("")
report.append(
"| Metric | APC OFF | APC ON | Improvement |"
)
report.append("|---|---:|---:|---:|")
def fmt(x):
if x is None:
return "N/A"
return f"{x:.2f}"
def ratio(off_value, on_value):
if (
off_value is None
or on_value is None
or on_value == 0
):
return "N/A"
return f"{off_value / on_value:.2f}x"
report.append(
f"| Median TTFT | {fmt(ttft_off)} ms | "
f"{fmt(ttft_on)} ms | "
f"{ratio(ttft_off, ttft_on)} |"
)
report.append(
f"| Median E2E | {fmt(e2e_off)} ms | "
f"{fmt(e2e_on)} ms | "
f"{ratio(e2e_off, e2e_on)} |"
)
report.append(
f"| Median TPOT | {fmt(tpot_off)} ms | "
f"{fmt(tpot_on)} ms | "
f"{ratio(tpot_off, tpot_on)} |"
)
report.append(
f"| Input throughput | {fmt(input_tps_off)} | "
f"{fmt(input_tps_on)} | "
f"{ratio(input_tps_on, input_tps_off)} |"
)
report.append(
f"| Output throughput | {fmt(output_tps_off)} | "
f"{fmt(output_tps_on)} | "
f"{ratio(output_tps_on, output_tps_off)} |"
)
report.append("")
report.append(
"## Prefix Cache Evidence"
)
report.append("")
report.append(
"| Metric | APC OFF | APC ON |"
)
report.append("|---|---:|---:|")
report.append(
f"| Prefix cache queries | "
f"{fmt(metrics_off['prefix_cache_queries'])} | "
f"{fmt(metrics_on['prefix_cache_queries'])} |"
)
report.append(
f"| Prefix cache hits | "
f"{fmt(metrics_off['prefix_cache_hits'])} | "
f"{fmt(metrics_on['prefix_cache_hits'])} |"
)
report.append(
f"| Cached prompt tokens | "
f"{fmt(metrics_off['prompt_tokens_cached'])} | "
f"{fmt(metrics_on['prompt_tokens_cached'])} |"
)
report.append(
f"| KV cache usage | "
f"{fmt(metrics_off['kv_cache_usage_perc'])} | "
f"{fmt(metrics_on['kv_cache_usage_perc'])} |"
)
report.append("")
report.append(
"## APC ON 해석"
)
report.append("")
if queries is None or hits is None:
report.append(
"⚠️ Prefix cache hit/query metric을 확인하지 못했습니다. "
"vLLM version 또는 metric name을 확인해야 합니다."
)
else:
report.append(
f"- Prefix cache queries: {queries:,.0f}"
)
report.append(
f"- Prefix cache hits: {hits:,.0f}"
)
report.append(
f"- Cache hit ratio: {hit_ratio:.2f}%"
)
if cached_tokens is not None:
report.append(
f"- Cached prompt tokens: {cached_tokens:,.0f}"
)
if hit_ratio is not None and hit_ratio > 0:
report.append(
"- Prefix Cache가 실제로 hit된 evidence가 확인됩니다."
)
else:
report.append(
"- Prefix Cache hit evidence가 확인되지 않습니다."
)
report.append("")
report.append(
"## 중요 해석"
)
report.append("")
report.append(
"이 테스트는 약 64.8K input 중 약 32K fixed prefix만 "
"Prefix Cache 대상입니다. 따라서 APC ON에서 5~10x TTFT "
"개선을 사전에 정상 기준으로 가정하지 않습니다."
)
report.append("")
report.append(
"APC 효과는 반드시 cache hit/cached tokens와 TTFT를 함께 "
"판단해야 합니다."
)
report.append("")
report.append(
"## KV Cache Startup Information"
)
report.append("")
for line in startup_on:
report.append(f"`{line}`")
report.append("")
report.append(
"## Files"
)
report.append("")
report.append(
"### Server logs"
)
report.append(
f"- `{os.path.join(log_dir, 'server_prefix_cache_on.log')}`"
)
report.append(
f"- `{os.path.join(log_dir, 'server_prefix_cache_off.log')}`"
)
report.append("")
report.append(
"### Metrics"
)
report.append(
f"- `{os.path.join(log_dir, 'metrics_prefix_cache_on_after.txt')}`"
)
report.append(
f"- `{os.path.join(log_dir, 'metrics_prefix_cache_off_after.txt')}`"
)
report_text = "\n".join(report)
report_path = os.path.join(
log_dir,
"prefix_cache_ab_report.md"
)
with open(report_path, "w") as f:
f.write(report_text)
print()
print(report_text)
print()
print(f"Report: {report_path}")
PYEOF
log "============================================================"
log "Prefix Caching A/B 완료"
log "============================================================"
log "결과 디렉터리:"
echo "${LOG_DIR}"
log "Report:"
echo "${LOG_DIR}/prefix_cache_ab_report.md"
engineer