mqtt HA 설정(kubernetes)

1. mqtt setup.yml

# 1. Longhorn StorageClass 정의 (레플리카 3개 설정)
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: emqx-longhorn-sc
provisioner: driver.longhorn.io
allowVolumeExpansion: true
parameters:
  numberOfReplicas: "3"      # 레플리카 수를 3으로 명시 (노드가 3개이므로 최적)
  staleReplicaTimeout: "30"  # 비정상 레플리카 제거 대기 시간(분)
  fromBackup: ""
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: tcp-services
  namespace: ingress-nginx
data:
  "1883": "default/emqx-service:1883"
---
# 2. RBAC (EMQX 클러스터 검색용 권한)
apiVersion: v1
kind: ServiceAccount
metadata:
  name: emqx-sa
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: emqx-role
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: emqx-rb
subjects:
  - kind: ServiceAccount
    name: emqx-sa
roleRef:
  kind: Role
  name: emqx-role
  apiGroup: rbac.authorization.k8s.io
---
# 3. Service (Discovery 및 통신용)
apiVersion: v1
kind: Service
metadata:
  name: emqx-headless
spec:
  clusterIP: None
  ports:
    - name: cluster-rpc
      port: 5369
  selector:
    app: emqx
---
apiVersion: v1
kind: Service
metadata:
  name: emqx-service
spec:
  ports:
    - name: mqtt
      port: 1883
    - name: dashboard
      port: 18083
  selector:
    app: emqx
---
# 4. StatefulSet (EMQX + 정의한 StorageClass 사용)
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: emqx
spec:
  serviceName: "emqx-headless"
  replicas: 3
  selector:
    matchLabels:
      app: emqx
  template:
    metadata:
      labels:
        app: emqx
    spec:
      # --- 이 부분을 반드시 추가하세요 ---
      securityContext:
        runAsUser: 1000
        runAsGroup: 1000
        fsGroup: 1000   # Longhorn 볼륨의 소유권을 emqx 유저 권한으로 강제 조정
      # -----------------------------
      serviceAccountName: emqx-sa
      containers:
      - name: emqx
        image: emqx/emqx:5.3.0
        env:
          - name: EMQX_CLUSTER__DISCOVERY_STRATEGY
            value: "k8s"
          - name: EMQX_CLUSTER__K8S__ADDRESS_TYPE
            value: "hostname"
          - name: EMQX_CLUSTER__K8S__NAMESPACE
            value: "default"
          - name: EMQX_CLUSTER__K8S__SERVICE_NAME
            value: "emqx-headless"
          - name: EMQX_CLUSTER__K8S__SUFFIX
            value: "svc.cluster.local"
        volumeMounts:
          - name: emqx-data
            mountPath: /opt/emqx/data
  volumeClaimTemplates:
  - metadata:
      name: emqx-data
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: "emqx-longhorn-sc" # 위에서 정의한 SC 사용
      resources:
        requests:
          storage: 1Gi

2. 기존 ingress 파일 변경

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: longhorn-ingress
  namespace: longhorn-system
  annotations:
    kubernetes.io/ingress.class: nginx
spec:
  ingressClassName: nginx
  rules:
  - host: longhorn.local
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: longhorn-frontend
            port:
              number: 80
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: default-ingress
  namespace: default
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/affinity: "cookie"
    nginx.ingress.kubernetes.io/session-cookie-name: "route"
    nginx.ingress.kubernetes.io/session-cookie-hash: "sha1"
spec:
  ingressClassName: nginx
  rules:
  - host: influxdb.local
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: influxdb-service # ▒▒▒▒ ▒▒▒▒ ▒▒▒▒ ▒̸▒
            port:
              number: 8086
  - host: mqtt.local
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: emqx-service # ▒▒▒▒ ▒▒▒▒ ▒▒▒▒ ▒̸▒
            port:
              number: 18083

kubectl apply -f 파일.yml 하면 적용됨.
3. ingress-nginx-controller 서비스 파일 백업 및 수정
kubectl get svc ingress-nginx-controller -n ingress-nginx -o yaml > ingress-svc-update.yaml

apiVersion: v1
kind: Service
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"controller","app.kubernetes.io/instance":"ingress-nginx","app.kubernetes.io/name":"ingress-nginx","app.kubernetes.io/part-of":"ingress-nginx","app.kubernetes.io/version":"1.8.1"},"name":"ingress-nginx-controller","namespace":"ingress-nginx"},"spec":{"ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"appProtocol":"http","name":"http","port":80,"protocol":"TCP","targetPort":"http"},{"appProtocol":"https","name":"https","port":443,"protocol":"TCP","targetPort":"https"}],"selector":{"app.kubernetes.io/component":"controller","app.kubernetes.io/instance":"ingress-nginx","app.kubernetes.io/name":"ingress-nginx"},"type":"NodePort"}}
  creationTimestamp: "2026-01-24T11:27:50Z"
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.8.1
  name: ingress-nginx-controller
  namespace: ingress-nginx
  resourceVersion: "55756"
  uid: cacbc931-e466-4a19-a1cf-d6a59e5f3053
spec:
  clusterIP: 10.100.46.190
  clusterIPs:
  - 10.100.46.190
  externalTrafficPolicy: Cluster
  internalTrafficPolicy: Cluster
  ipFamilies:
  - IPv4
  ipFamilyPolicy: SingleStack
  ports:
  - appProtocol: http
    name: http
    nodePort: 32142
    port: 80
    protocol: TCP
    targetPort: http
  - appProtocol: https
    name: https
    nodePort: 31171
    port: 443
    protocol: TCP
    targetPort: https
    # 추가
  - name: mqtt 
    port: 1883
    targetPort: 1883
    protocol: TCP
    nodePort: 32257 #외부 노출 포트
    ### 추가 끝
  selector:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
  sessionAffinity: None
  type: NodePort
status:
  loadBalancer: {}

# 3. 수정된 설정 반영
kubectl apply -f 파일이름.yaml

# 4. 포트가 정상적으로 열렸는지 확인
kubectl get svc ingress-nginx-controller -n ingress-nginx

4. haproxy 접속 설정.

frontend mqtt_dash
    bind *:18083
    mode http
    http-request set-header Host mqtt.local

    default_backend ingress_nginx

frontend mqtt_front
    bind *:1883
    mode tcp

    default_backend mqtt_backend

# 2. MQTT 전용 백엔드 (포트 번호: 32257)
backend mqtt_backend
    mode tcp
    balance roundrobin
    # kubectl에서 확인한 NodePort 32257을 사용합니다.
    server node3 10.0.2.4:32257 check #위에서 설정한 node port
    server node2 10.0.2.6:32257 check
    server node1 10.0.2.5:32257 check

frontend influx_front
    bind *:8086
    mode http

    # [핵심] 외부에서 IP로 들어와도 Ingress가 알아먹을 수 있게 이름표를 갈아끼웁니다.
    http-request set-header Host influxdb.local

    default_backend ingress_nginx

frontend longhorn_front
    bind *:80
    mode http

    # [핵심] 80포트로 들어오면 롱혼 이름표를 붙입니다.
    http-request set-header Host longhorn.local

    default_backend ingress_nginx
backend ingress_nginx
    mode http
    balance roundrobin
    # 각 노드(VM)의 실제 IP와 확인한 NodePort를 적습니다.
    server node3 10.0.2.4:32142 check
    server node2 10.0.2.6:32142 check
    server node1 10.0.2.5:32142 check

frontend k8s-api
    bind 10.0.2.10:6444
    default_backend k8s-masters

backend k8s-masters
    balance roundrobin
    server node3 10.0.2.4:6443 check
    server node1 10.0.2.5:6443 check
    server node2 10.0.2.6:6443 check

6. TCP 설정 옵션 주입

• 명령을 통한 주입
  kubectl patch deployment ingress-nginx-controller -n ingress-nginx --type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--tcp-services-configmap=ingress-nginx/tcp-services"}]'
• 파일을 통한 주입
  kubectl get deployment -n ingress-nginx ingress-nginx-controller -o yaml > ingress-controller.yaml
  여기에서 뽑아낸 ingress-controller.yml 파일을 수정후 apply를 통해 시작

apiVersion: apps/v1
kind: Deployment
metadata:
  name: ingress-nginx-controller
  namespace: ingress-nginx
spec:
  # ... (생략) ...
  template:
    spec:
      containers:
      - name: controller
        image: ...
        args: # <--- 바로 여기입니다!
        - /nginx-ingress-controller
        - --election-id=ingress-nginx-leader
        - --controller-class=k8s.io/ingress-nginx
        - --ingress-class=nginx
        - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
        # 아래 줄을 리스트의 맨 마지막이나 적당한 곳에 추가하세요.
        - --tcp-services-configmap=ingress-nginx/tcp-services

• 적용 확인
  kubectl describe deploy -n ingress-nginx ingress-nginx-controller | grep -A 7 "Args"
• 결과확인

# 인그레스 컨트롤러 포드 이름 다시 가져오기
INGRESS_POD=$(kubectl get pods -n ingress-nginx -l app.kubernetes.io/name=ingress-nginx -o jsonpath='{.items[0].metadata.name}')

# 포드 내부에서 1883 리스닝 확인
kubectl exec -n ingress-nginx $INGRESS_POD -- netstat -tuln | grep 1883

• 1883에 대한 정보가 떠야한다.