"""
config.py — FinOps 파이프라인 전사 통합 환경설정 및 정규식 일원화 엔진
"""
import os
import re
from pathlib import Path
os.environ.setdefault("MINIO_ENDPOINT", "http://minio-service.internal.zone:9000")
os.environ.setdefault("MINIO_ACCESS_KEY", "admin")
os.environ.setdefault("MINIO_SECRET_KEY", "password")
os.environ.setdefault("MINIO_RAW_BUCKET", "devops-test")
os.environ.setdefault("MINIO_REPORT_BUCKET", "devops-test")
BASE_DATA_DIR = Path("./data")
RAW_DIR = BASE_DATA_DIR / "raw"
MERGED_DIR = BASE_DATA_DIR / "merged"
OUT_DIR = BASE_DATA_DIR / "output"
PLOT_DIR = OUT_DIR / "plots"
RAW_DIR.mkdir(parents=True, exist_ok=True)
MERGED_DIR.mkdir(parents=True, exist_ok=True)
OUT_DIR.mkdir(parents=True, exist_ok=True)
PLOT_DIR.mkdir(parents=True, exist_ok=True)
CLUSTER_NODE_PATTERNS = {
"COMPUTE": r"^(name1wk\d+|icdlh-prod-wk\d+|stg-lh-wk\d+)",
"STORAGE": r"^(minio-storage-node\d+|aistor-prod-wk\d+|lake-pool-\d+)"
}
def classify_cluster_infrastructure(node_name):
"""
하드코딩 키워드 대신 상단의 CLUSTER_NODE_PATTERNS 정규식을 사용하여
소속 클러스터의 실명과 속성을 역추적합니다.
"""
n = str(node_name)
if re.search(CLUSTER_NODE_PATTERNS["STORAGE"], n, re.IGNORECASE):
return "prod-storage-cluster", "STORAGE"
elif re.search(CLUSTER_NODE_PATTERNS["COMPUTE"], n, re.IGNORECASE):
return "prod-compute-cluster", "COMPUTE"
else:
return "unclassified-cluster", "COMPUTE"
def get_finops_promql_queries(selector):
pv_selector = selector.replace('container!="", ', '').replace(', container!=""', '')
return {
"cpu_request": f'kube_pod_container_resource_requests{selector.replace("container!=", "resource=\"cpu\", container!=")}',
"cpu_limit": f'kube_pod_container_resource_limits{selector.replace("container!=", "resource=\"cpu\", container!=")}',
"cpu_usage": f'rate(container_cpu_usage_seconds_total{selector}[1m])',
"cpu_throttled": f'rate(container_cpu_cfs_throttled_periods_total{selector}[1m])',
"mem_request": f'kube_pod_container_resource_requests{selector.replace("container!=", "resource=\"memory\", container!=")}',
"mem_limit": f'kube_pod_container_resource_limits{selector.replace("container!=", "resource=\"memory\", container!=")}',
"mem_usage": f'container_memory_working_set_bytes{selector}',
"mem_rss": f'container_memory_rss{selector}',
"oom_event": f'kube_pod_container_status_terminated_reason{selector.replace("container!=", "reason=\"OOMKilled\", container!=")}',
"pv_capacity": f'kubelet_volume_stats_capacity_bytes{pv_selector}',
"pv_used": f'kubelet_volume_stats_used_bytes{pv_selector}'
}
def get_workload_type(pod_name):
p = str(pod_name).lower()
if "spark" in p or "-exec-" in p or "-driver" in p:
if "executor" in p or "-exec-" in p: return "SPARK_EXECUTOR"
if "driver" in p or "-driver" in p: return "SPARK_DRIVER"
return "SPARK_SYSTEM"
if "airflow" in p or "statsd" in p:
if "worker" in p: return "AIRFLOW_WORKER"
if "scheduler" in p: return "AIRFLOW_SCHEDULER"
return "AIRFLOW_SYSTEM"
if "minio" in p or "aistor" in p or "lake-pool" in p: return "MINIO_STORAGE"
return "GENERAL_APPS"