kubernetes(influxdb HA)

1. haproxy 설정 파일 변경(각각의 서비스 구분을 위한 header 변경)

frontend influx_front
    bind *:8086
    mode http

    # [핵심] 외부에서 IP로 들어와도 Ingress가 알아먹을 수 있게 이름표를 갈아끼웁니다.
    http-request set-header Host influxdb.local

    default_backend ingress_nginx

frontend longhorn_front
    bind *:80
    mode http

    # [핵심] 80포트로 들어오면 롱혼 이름표를 붙입니다.
    http-request set-header Host longhorn.local

    default_backend ingress_nginx
backend ingress_nginx
    mode http
    balance roundrobin
    # SERVERID라는 쿠키를 삽입하여 한 번 접속한 노드로 계속 고정합니다.
    cookie SERVERID insert indirect nocache
    # 각 노드(VM)의 실제 IP와 확인한 NodePort를 적습니다.
    server node3 10.0.2.4:32142 check
    server node2 10.0.2.6:32142 check
    server node1 10.0.2.5:32142 check

frontend k8s-api
    bind 10.0.2.10:6444
    default_backend k8s-masters

backend k8s-masters
    balance roundrobin
    server node3 10.0.2.4:6443 check
    server node1 10.0.2.5:6443 check
    server node2 10.0.2.6:6443 check

2. influxdb(telegraf)

apiVersion: v1
kind: ConfigMap
metadata:
  name: telegraf-config
  labels:
    app: influxdb-ha
data:
  telegraf.conf: |
    [agent]
      interval = "10s"
      round_interval = true

    [[inputs.influxdb_v2_listener]]
      ## 외부에서 데이터를 받을 포트 (Relay 역할)
      service_address = ":8080"

    [[outputs.influxdb_v2]]
      ## 첫 번째 InfluxDB 노드 (StatefulSet 0번)
      urls = ["http://influxdb-0.influxdb-service:8086"]
      token = "${INFLUX_TOKEN}"
      organization = "my-org"
      bucket = "my-bucket"

    [[outputs.influxdb_v2]]
      ## 두 번째 InfluxDB 노드 (StatefulSet 1번)
      urls = ["http://influxdb-1.influxdb-service:8086"]
      token = "${INFLUX_TOKEN}"
      organization = "my-org"
      bucket = "my-bucket"

    [[outputs.influxdb_v2]]
      ## 두 번째 InfluxDB 노드 (StatefulSet 1번)
      urls = ["http://influxdb-2.influxdb-service:8086"]
      token = "${INFLUX_TOKEN}"
      organization = "my-org"
      bucket = "my-bucket"


---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: longhorn-influxdb
provisioner: driver.longhorn.io
allowVolumeExpansion: true
parameters:
  # Longhorn에서 데이터를 몇 군데 복제할지 결정 (보통 3 권장)
  numberOfReplicas: "3"
  # 데이터 기록이 완료되었다고 판단하는 기준
  staleReplicaTimeout: "30"
  # 노드 장애 시 데이터 자동 재균형
  fromBackup: ""
---
apiVersion: v1
kind: Service
metadata:
  name: influxdb-service
spec:
  # StatefulSet의 개별 Pod에 접근하기 위해 Headless Service 권장
  # clusterIP: None #내부에서만 ㅅ쓸때 설정
  type: NodePort #외부 포트를 열떄 필요.
  selector:
    app: influxdb
  ports:
    - name: influxdb
      port: 8086
      targetPort: 8086
      nodePort: 30086
---
apiVersion: v1
kind: Service
metadata:
  name: influxdb-write-proxy
spec:
  # 쓰기 전용 엔드포인트 (Telegraf Relay로 연결)
  selector:
    app: telegraf
  ports:
    - name: write
      port: 8080
      targetPort: 8080
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: influxdb
spec:
  serviceName: "influxdb-service"
  replicas: 1 #두개이상시 충돌
  selector:
    matchLabels:
      app: influxdb
  template:
    metadata:
      labels:
        app: influxdb
    spec:    
      affinity:
        podAntiAffinity:
          # required... 는 "무조건 지켜라"라는 뜻입니다. (Hard anti-affinity)
          # 만약 노드가 2개뿐인데 replicas가 3이면, 1개 Pod는 배포되지 않고 Pending 상태가 됩니다.
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: app
                    operator: In
                    values:
                      - influxdb
              topologyKey: "kubernetes.io/hostname"              
      containers:
        - name: influxdb
          image: influxdb:2.7 # 2.x 최신 안정 버전
          ports:
            - containerPort: 8086
          env:
            # 초기 설정 자동화 (최초 실행 시에만 적용)
            - name: DOCKER_INFLUXDB_INIT_MODE
              value: setup
            - name: DOCKER_INFLUXDB_INIT_USERNAME
              value: <ID>
            - name: DOCKER_INFLUXDB_INIT_PASSWORD
              value: <password>
            - name: DOCKER_INFLUXDB_INIT_ORG
              value: my-org
            - name: DOCKER_INFLUXDB_INIT_BUCKET
              value: my-bucket
            - name: DOCKER_INFLUXDB_INIT_ADMIN_TOKEN
              value: my-super-secret-token
          volumeMounts:
            - name: influxdb-data
              mountPath: /var/lib/influxdb2
  volumeClaimTemplates:
    - metadata:
        name: influxdb-data
      spec:
        accessModes: [ "ReadWriteOnce" ]
        storageClassName: "longhorn-influxdb"
        resources:
          requests:
            storage: 2Gi # 스토리지 용량 설정
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: telegraf-relay
spec:
  replicas: 3
  selector:
    matchLabels:
      app: telegraf
  template:
    metadata:
      labels:
        app: telegraf
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - telegraf
              topologyKey: "kubernetes.io/hostname"
      containers:
        - name: telegraf
          image: telegraf:latest
          ports:
            - containerPort: 8080
          env:
            # ConfigMap에서 사용할 토큰 (InfluxDB와 동일하게 설정)
            - name: INFLUX_TOKEN
              value: my-super-secret-token
          volumeMounts:
            - name: config
              mountPath: /etc/telegraf/telegraf.conf
              subPath: telegraf.conf
      volumes:
        - name: config
          configMap:
            name: telegraf-config

3. ingress 설정 파일

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: longhorn-ingress
  namespace: longhorn-system
  annotations:
    kubernetes.io/ingress.class: nginx
spec:
  ingressClassName: nginx
  rules:
  - host: longhorn.local
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: longhorn-frontend
            port:
              number: 80
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: default-ingress
  namespace: default
  annotations:
    kubernetes.io/ingress.class: nginx
    #influxdb 대시보드 접속시 세션의 유실 문제를 해결하기 위해 아래 3줄 추가
    nginx.ingress.kubernetes.io/affinity: "cookie"
    nginx.ingress.kubernetes.io/session-cookie-name: "route"
    nginx.ingress.kubernetes.io/session-cookie-hash: "sha1"
spec:
  ingressClassName: nginx
  rules:
  - host: influxdb.local
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: influxdb-service # 이전에 만든 서비스 이름
            port:
              number: 8086

문제발생

• influxdb가 떠있는 노드를 강제로 종료시켰을때, Pod가 끝나지 않고 계속 terminating 상태이다.

# influxdb statefulset spec 내부에 추가
tolerations:
- key: "node.kubernetes.io/unreachable"
  operator: "Exists"
  effect: "NoExecute"
  tolerationSeconds: 30  # 30초만 응답 없으면 바로 다른 노드로 이사 준비
- key: "node.kubernetes.io/not-ready"
  operator: "Exists"
  effect: "NoExecute"
  tolerationSeconds: 30 

• 정상 작동 setup yml 파일

apiVersion: v1
kind: ConfigMap
metadata:
  name: telegraf-config
  labels:
    app: influxdb-ha
data:
  telegraf.conf: |
    [agent]
      interval = "10s"
      round_interval = true

    [[inputs.influxdb_v2_listener]]
      ## 외부에서 데이터를 받을 포트 (Relay 역할)
      service_address = ":8080"

    [[outputs.influxdb_v2]]
      ## 첫 번째 InfluxDB 노드 (StatefulSet 0번)
      urls = ["http://influxdb-0.influxdb-service:8086"]
      token = "${INFLUX_TOKEN}"
      organization = "my-org"
      bucket = "my-bucket"

    [[outputs.influxdb_v2]]
      ## 두 번째 InfluxDB 노드 (StatefulSet 1번)
      urls = ["http://influxdb-1.influxdb-service:8086"]
      token = "${INFLUX_TOKEN}"
      organization = "my-org"
      bucket = "my-bucket"

    [[outputs.influxdb_v2]]
      ## 두 번째 InfluxDB 노드 (StatefulSet 1번)
      urls = ["http://influxdb-2.influxdb-service:8086"]
      token = "${INFLUX_TOKEN}"
      organization = "my-org"
      bucket = "my-bucket"


---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: longhorn-influxdb
provisioner: driver.longhorn.io
allowVolumeExpansion: true
parameters:
  # Longhorn에서 데이터를 몇 군데 복제할지 결정 (보통 3 권장)
  numberOfReplicas: "3"
  # 데이터 기록이 완료되었다고 판단하는 기준
  staleReplicaTimeout: "30"
  # 노드 장애 시 데이터 자동 재균형
  fromBackup: ""
---
apiVersion: v1
kind: Service
metadata:
  name: influxdb-service
spec:
  # StatefulSet의 개별 Pod에 접근하기 위해 Headless Service 권장
  # clusterIP: None #내부에서만 ㅅ쓸때 설정
  type: NodePort #외부 포트를 열떄 필요.
  selector:
    app: influxdb
  ports:
    - name: influxdb
      port: 8086
      targetPort: 8086
      nodePort: 30086
---
apiVersion: v1
kind: Service
metadata:
  name: influxdb-write-proxy
spec:
  # 쓰기 전용 엔드포인트 (Telegraf Relay로 연결)
  selector:
    app: telegraf
  ports:
    - name: write
      port: 8080
      targetPort: 8080
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: influxdb
spec:
  serviceName: "influxdb-service"
  replicas: 1 # 2개의 노드를 유지 (이중화)
  selector:
    matchLabels:
      app: influxdb
  template:
    metadata:
      labels:
        app: influxdb
    spec:   
      tolerations:
      - key: "node.kubernetes.io/unreachable"
        operator: "Exists"
        effect: "NoExecute"
        tolerationSeconds: 30  # 10초만 응답 없으면 바로 다른 노드로 이사 준비
      - key: "node.kubernetes.io/not-ready"
        operator: "Exists"
        effect: "NoExecute"
        tolerationSeconds: 30
      affinity:
        podAntiAffinity:
          # required... 는 "무조건 지켜라"라는 뜻입니다. (Hard anti-affinity)
          # 만약 노드가 2개뿐인데 replicas가 3이면, 1개 Pod는 배포되지 않고 Pending 상태가 됩니다.
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: app
                    operator: In
                    values:
                      - influxdb
              topologyKey: "kubernetes.io/hostname"              
      containers:
        - name: influxdb
          image: influxdb:2.7 # 2.x 최신 안정 버전
          ports:
            - containerPort: 8086
          env:
            # 초기 설정 자동화 (최초 실행 시에만 적용)
            - name: DOCKER_INFLUXDB_INIT_MODE
              value: setup
            - name: DOCKER_INFLUXDB_INIT_USERNAME
              value: <ID> # 아무거나
            - name: DOCKER_INFLUXDB_INIT_PASSWORD
              value: <password> # 아무거나
            - name: DOCKER_INFLUXDB_INIT_ORG
              value: my-org
            - name: DOCKER_INFLUXDB_INIT_BUCKET
              value: my-bucket
            - name: DOCKER_INFLUXDB_INIT_ADMIN_TOKEN
              value: my-super-secret-token
          volumeMounts:
            - name: influxdb-data
              mountPath: /var/lib/influxdb2
  volumeClaimTemplates:
    - metadata:
        name: influxdb-data
      spec:
        accessModes: [ "ReadWriteOnce" ]
        storageClassName: "longhorn-influxdb"
        resources:
          requests:
            storage: 2Gi # 스토리지 용량 설정
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: telegraf-relay
spec:
  replicas: 3
  selector:
    matchLabels:
      app: telegraf
  template:
    metadata:
      labels:
        app: telegraf
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - telegraf
              topologyKey: "kubernetes.io/hostname"
      containers:
        - name: telegraf
          image: telegraf:latest
          ports:
            - containerPort: 8080
          env:
            # ConfigMap에서 사용할 토큰 (InfluxDB와 동일하게 설정)
            - name: INFLUX_TOKEN
              value: my-super-secret-token
          volumeMounts:
            - name: config
              mountPath: /etc/telegraf/telegraf.conf
              subPath: telegraf.conf
      volumes:
        - name: config
          configMap:
            name: telegraf-config