"""
config.py (글로벌 파이프라인 설정 및 도메인 규칙 원부 - 데이터 레이크 고도화 버전)
"""
import re
from pathlib import Path
BASE_DIR = Path(__file__).resolve().parent
RAW_DIR = BASE_DIR / "data" / "raw"
MERGED_DIR = BASE_DIR / "data" / "merged"
OUT_DIR = BASE_DIR / "data" / "output"
for d in [RAW_DIR, MERGED_DIR, OUT_DIR, OUT_DIR / "plots"]:
d.mkdir(parents=True, exist_ok=True)
DEFAULT_THANOS_URL = "http://thanos-query.internal.zone:9090"
MINIO_RAW_BUCKET = "enterprise-finops-raw-lake"
MINIO_REPORT_BUCKET = "enterprise-finops-reports"
NODE_PREFIX_PATTERN = r"name1wk\d+"
def get_finops_promql_queries(selector):
return {
"cpu_request": f"kube_pod_container_resource_requests{selector.replace('container!=', 'resource="cpu", container!=')}",
"cpu_limit": f"kube_pod_container_resource_limits{selector.replace('container!=', 'resource="cpu", container!=')}",
"cpu_usage": f"rate(container_cpu_usage_seconds_total{selector}[1m])",
"mem_request": f"kube_pod_container_resource_requests{selector.replace('container!=', 'resource="memory", container!=')}",
"mem_limit": f"kube_pod_container_resource_limits{selector.replace('container!=', 'resource="memory", container!=')}",
"mem_usage": f"container_memory_working_set_bytes{selector}",
"oom_event": f"kube_pod_container_status_terminated_reason{selector.replace('container!=', 'reason="OOMKilled", container!=')}"
}
def get_workload_type(pod_name):
p = str(pod_name).lower()
if "spark" in p or "-exec-" in p or "-driver" in p:
if "executor" in p or "-exec-" in p: return "SPARK_EXECUTOR"
if "driver" in p or "-driver" in p: return "SPARK_DRIVER"
return "SPARK_SYSTEM"
if "airflow" in p or "statsd" in p:
if "worker" in p: return "AIRFLOW_WORKER"
if "scheduler" in p: return "AIRFLOW_SCHEDULER"
if "dag-processor" in p: return "AIRFLOW_DAG_PROC"
if "triggerer" in p: return "AIRFLOW_TRIGGERER"
return "AIRFLOW_SYSTEM"
if "starrocks" in p or "-be-" in p or "-fe-" in p or "-cn-" in p:
if "-be" in p: return "STARROCKS_BE"
if "-fe" in p: return "STARROCKS_FE"
if "-cn" in p: return "STARROCKS_CN"
return "STARROCKS_SYSTEM"
if "postgres" in p or "pgpool" in p:
if "postgresql-backup" in p: return "POSTGRES_BACKUP"
if "pgpool" in p: return "POSTGRES_POOL"
return "POSTGRESQL"
if "hyperset" in p or "superset" in p or re.search(r"hyperset-\d{7,8}-redis", p):
return "HYPERSET"
if "jupyter" in p or "notebook" in p:
return "JUPYTERLAB"
if "minio" in p or "aistor" in p or "lake-pool" in p:
return "MINIO_STORAGE"
if "cilium" in p:
return "CILIUM_CNI"
return "GENERAL_APPS"