[k8s] ETCD외부 설치 & 인증서 갱신

Nam_JU·2025년 6월 11일

kubernetes

k8s

목록 보기

11/14

외부 ETCD를 설치하는 이유

안정성 향상: etcd를 Kubernetes 마스터와 분리해 장애 확산 방지
성능 개선: etcd에 독립 리소스 할당으로 부하 분산
운영·보안 용이: 별도 관리 및 보안 정책 적용 가능
확장성 확보: 클러스터와 독립적으로 etcd 확장 가능
유연한 환경 대응: 온프레·클라우드 혼합 환경에서 활용 편리

VM 환경

rocky 8.8 minimal

k8s version: 1.29.7v

etcd 3.5.9버전

외부 ETCD 설치

1. 외부 etcd설치 및 TLS 구성

사용자 생성

[root@k8s-master ~]# useradd -r -s /sbin/nologin etcd
[root@k8s-master ~]# mkdir -p /etc/etcd /var/lib/etcd
[root@k8s-master ~]# chown -R etcd:etcd /etc/etcd /var/lib/etcd
[root@k8s-master ~]# ls -ld /etc/etcd
drwxr-xr-x 2 etcd etcd 6  6월 11 14:37 /etc/etcd
[root@k8s-master ~]# ls -ld /var/lib/etcd
drwxr-xr-x 2 etcd etcd 6  6월 11 14:37 /var/lib/etcd

etcd 설치

ETCD_VER=v3.5.9
curl -LO https://github.com/etcd-io/etcd/releases/download/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz
tar xzvf etcd-${ETCD_VER}-linux-amd64.tar.gz
mv etcd-${ETCD_VER}-linux-amd64/etcd* /usr/local/bin/

서비스 등록

[root@k8s-master kubernetes]# cat /etc/systemd/system/etcd.service
[Unit]
Description=etcd
After=network.target

[Service]
User=etcd
Type=notify
ExecStart=/usr/local/bin/etcd \
  --name etcd \
  --data-dir /var/lib/etcd \
  --listen-client-urls https://0.0.0.0:2379 \
  --advertise-client-urls https://192.168.0.66:2379 \
  --cert-file=/etc/etcd/pki/etcd-server.crt \
  --key-file=/etc/etcd/pki/etcd-server.key \
  --client-cert-auth=true \
  --trusted-ca-file=/etc/etcd/pki/ca.crt

Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target

[root@k8s-master ~]# vi /etc/systemd/system/etcd.service
[root@k8s-master ~]# systemctl daemon-reexec
[root@k8s-master ~]# systemctl daemon-reload
[root@k8s-master ~]# systemctl enable --now etcd
Created symlink /etc/systemd/system/multi-user.target.wants/etcd.service → /etc/systemd/system/etcd.service.

[root@k8s-master ~]# systemctl status etcd
● etcd.service - etcd
   Loaded: loaded (/etc/systemd/system/etcd.service; enabled; vendor preset: disabled)
   Active: activating (start) since Wed 2025-06-11 14:40:11 KST; 1s ago
 Main PID: 32502 (etcd)
    Tasks: 6 (limit: 50014)
   Memory: 10.7M
   CGroup: /system.slice/etcd.service
           └─32502 /usr/local/bin/etcd --name etcd --data-dir /var/lib/etcd --listen-client-urls https://0.0.0.0:2379 --advertise->

configmap 생성

apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
kubernetesVersion: v1.29.7
controlPlaneEndpoint: "192.168.0.66:6443"

etcd:
  external:
    endpoints:
    - https://192.168.0.66:2379
    caFile: /etc/etcd/pki/ca.crt
    certFile: /etc/etcd/pki/etcd-client.crt
    keyFile: /etc/etcd/pki/etcd-client.key

networking:
  podSubnet: 10.244.0.0/16

인증서 생성

mkdir -p /root/etcd-certs
cd /root/etcd-certs

# CA 생성
openssl genrsa -out ca.key 2048
openssl req -x509 -new -nodes -key ca.key -subj "/CN=etcd-ca" -days 3650 -out ca.crt

# etcd 서버용 인증서 생성
openssl genrsa -out etcd-server.key 2048
openssl req -new -key etcd-server.key -subj "/CN=etcd-server" -out etcd-server.csr

openssl x509 -req -in etcd-server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out etcd-server.crt -days 3650 -extensions v3_req -extfile <(printf "[v3_req]\nsubjectAltName=IP:192.168.0.66")

# 클라이언트용 인증서도 비슷하게 생성

# 작업 끝나면 인증서들을 실제 위치로 복사
mkdir -p /etc/etcd/pki
cp *.crt *.key ca.key /etc/etcd/pki/
chown -R etcd:etcd /etc/etcd/pki
chmod 600 /etc/etcd/pki/*.key
chmod 644 /etc/etcd/pki/*.crt

======================================================================
[root@k8s-master etcd-certs]# mkdir -p /etc/etcd/pki
[root@k8s-master etcd-certs]# cp *.crt *.key ca.key /etc/etcd/pki/
cp: warning: source file 'ca.key' specified more than once
[root@k8s-master etcd-certs]# chown -R etcd:etcd /etc/etcd/pki
[root@k8s-master etcd-certs]# chmod 644 /etc/etcd/pki/*.crt
[root@k8s-master etcd-certs]# cd /etc/etcd
[root@k8s-master etcd]# ls
pki
[root@k8s-master etcd]# cd pki/
[root@k8s-master pki]# ls
ca.crt  ca.key  etcd-server.crt  etcd-server.key
[root@k8s-master pki]# ls -ld /etc/etcd/pki
drwxr-xr-x 2 etcd etcd 80  6월 11 14:52 /etc/etcd/pki

인증서 생성2 : etcd-client.crt, etcd-client.key

[root@k8s-master kubernetes]# cd /etc/etcd/pki
[root@k8s-master pki]# ls
ca.crt  ca.key  etcd-server.crt  etcd-server.key
[root@k8s-master pki]#
[root@k8s-master pki]#
[root@k8s-master pki]# openssl genrsa -out /etc/etcd/pki/etcd-client.key 2048
Generating RSA private key, 2048 bit long modulus (2 primes)
..........................................................+++++
.............................................................+++++
e is 65537 (0x010001)
[root@k8s-master pki]# ls
ca.crt  ca.key  etcd-client.key  etcd-server.crt  etcd-server.key
[root@k8s-master pki]# openssl req -new -key /etc/etcd/pki/etcd-client.key -out /etc/etcd/pki/etcd-client.csr -subj "/CN=etcd-client"
[root@k8s-master pki]# openssl x509 -req -in /etc/etcd/pki/etcd-client.csr -CA /etc/etcd/pki/ca.crt -CAkey /etc/etcd/pki/ca.key -CAcreateserial -out /etc/etcd/pki/etcd-client.crt -days 3650 -sha256
Signature ok
subject=CN = etcd-client
Getting CA Private Key
[root@k8s-master pki]# ls
ca.crt  ca.key  ca.srl  etcd-client.crt  etcd-client.csr  etcd-client.key  etcd-server.crt  etcd-server.key

k8s설치

시스템 설정

os version

[root@k8s-master ~]# cat /etc/os-release
NAME="Rocky Linux"
VERSION="8.8 (Green Obsidian)"
ID="rocky"
ID_LIKE="rhel centos fedora"
VERSION_ID="8.8"
PLATFORM_ID="platform:el8"
PRETTY_NAME="Rocky Linux 8.8 (Green Obsidian)"
ANSI_COLOR="0;32"
LOGO="fedora-logo-icon"
CPE_NAME="cpe:/o:rocky:rocky:8:GA"
HOME_URL="https://rockylinux.org/"
BUG_REPORT_URL="https://bugs.rockylinux.org/"
SUPPORT_END="2029-05-31"
ROCKY_SUPPORT_PRODUCT="Rocky-Linux-8"
ROCKY_SUPPORT_PRODUCT_VERSION="8.8"
REDHAT_SUPPORT_PRODUCT="Rocky Linux"
REDHAT_SUPPORT_PRODUCT_VERSION="8.8"

[root@k8s-master ~]# free -m
              total        used        free      shared  buff/cache   available
Mem:           7853         139        7538          12         175        7482
Swap:             0           0           0

[root@k8s-master ~]# systemctl disable firewalld && systemctl stop firewalld
[root@k8s-master ~]# systemctl status firewalld
● firewalld.service - firewalld - dynamic firewall daemon
   Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)
   Active: inactive (dead)
     Docs: man:firewalld(1)
     
[root@k8s-master ~]# findmnt /sys/fs/cgroup
TARGET         SOURCE FSTYPE OPTIONS
/sys/fs/cgroup tmpfs  tmpfs  ro,nosuid,nodev,noexec,mode=755

[root@k8s-master ~]# stat -fc %T /sys/fs/cgroup
tmpfs

## cgroup2로 변경
[root@k8s-master ~]# cat /etc/default/grub
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=saved
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL_OUTPUT="console"
GRUB_CMDLINE_LINUX="resume=/dev/mapper/rl-swap rd.lvm.lv=rl/root rd.lvm.lv=rl/swap selinux=0 systemd.unified_cgroup_hierarchy=1"
GRUB_DISABLE_RECOVERY="true"
GRUB_ENABLE_BLSCFG=true

## 부팅시 옵션 적용
[root@k8s-master ~]# sudo grub2-mkconfig -o /boot/grub2/grub.cfg
Generating grub configuration file ...
Adding boot menu entry for EFI firmware configuration
done

## 현재 커널에 cgroup v2 옵션 추가
sudo grubby --args="systemd.unified_cgroup_hierarchy=1" --update-kernel=ALL
sudo reboot

## 적용 확인
[root@k8s-master ~]# stat -fc %T /sys/fs/cgroup
cgroup2fs
[root@k8s-master ~]# mount | grep cgroup
cgroup2 on /sys/fs/cgroup type cgroup2 (rw,nosuid,nodev,noexec,relatime,nsdelegate)

## 커널 모듈 설정
[root@k8s-worker ~]# sudo vi /etc/modules-load.d/k8s.conf
[root@k8s-worker ~]# cat /etc/modules-load.d/k8s.conf
overlay
br_netfilter

[root@k8s-master ~]# sudo modprobe br_netfilter
[root@k8s-master ~]# lsmod | grep br_netfilter
br_netfilter           24576  0
bridge                290816  1 br_netfilter

[root@k8s-worker ~]# vi /etc/sysctl.d/k8s.conf
[root@k8s-worker ~]# cat /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward=1

# 변경사항 적용
sudo sysctl --system

## 방화벽 
[root@k8s-master ~]# iptables -L
Chain INPUT (policy ACCEPT)
target     prot opt source               destination

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination

## 파일 스크립터 늘이기
cat << EOF | sudo tee -a /etc/security/limits.conf
* soft nofile 65536
* hard nofile 65536
root soft nofile 65536
root hard nofile 65536
EOF

필요한 패키지 설치

## runC  ======================================
curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc

## containerd 1.7.13  ======================================
sudo dnf install -y wget
wget https://github.com/containerd/containerd/releases/download/v1.7.13/containerd-1.7.13-linux-amd64.tar.gz
tar -C /usr/local -xzf containerd-1.7.13-linux-amd64.tar.gz

sudo mkdir -p /usr/lib/systemd/system

cat <<EOF | sudo tee /usr/lib/systemd/system/containerd.service
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target

[Service]
ExecStart=/usr/local/bin/containerd
Restart=always
Delegate=yes
KillMode=process
OOMScoreAdjust=-999
LimitNOFILE=1048576
LimitNPROC=infinity
LimitCORE=infinity

[Install]
WantedBy=multi-user.target
EOF

# 서비스 시작
sudo systemctl daemon-reexec
sudo systemctl daemon-reload
sudo systemctl enable --now containerd

## containerd cgroup2적용
# 기본 설정 생성
sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml

# systemd cgroup 드라이버 설정
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml

# 재시작
sudo systemctl restart containerd
 
## CNI설치 ======================================
# 기본 CNI 플러그인 다운로드 (필요 시)
sudo mkdir -p /opt/cni/bin
curl -LO https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz
sudo tar -C /opt/cni/bin -xzf cni-plugins-linux-amd64-v1.3.0.tgz

[root@k8s-master ~]# containerd --version
containerd github.com/containerd/containerd v1.7.13 7c3aca7a610df76212171d200ca3811ff6096eb8
[root@k8s-master ~]# runc --version
runc version 1.1.12
commit: v1.1.12-0-g51d5e946
spec: 1.0.2-dev
go: go1.20.13
libseccomp: 2.5.4
[root@k8s-master ~]# which runc

## nerdctl설치 =====================================
cd /root
curl -LO https://github.com/containerd/nerdctl/releases/download/v1.7.7/nerdctl-full-1.7.7-linux-amd64.tar.gz
tar -xvf nerdctl-full-1.7.7-linux-amd64.tar.gz
nerdctl --version

sudo yum install -y iproute-tc

k8s 패키지 설치

rocky8용으로 설정

cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/repodata/repomd.xml.key
EOF

sudo dnf clean all
sudo dnf install -y kubelet-1.29.7 kubeadm-1.29.7 kubectl-1.29.7
sudo systemctl enable --now kubelet

[root@k8s-master ~]# rpm -qa | grep kube
kubernetes-cni-1.3.0-150500.1.1.x86_64
kubelet-1.29.7-150500.1.1.x86_64
kubectl-1.29.7-150500.1.1.x86_64
kubeadm-1.29.7-150500.1.1.x86_64

[root@k8s-master ~]# kubectl version --client
Client Version: v1.29.7
Kustomize Version: v5.0.4-0.20230601165947-6ce0bf390ce3
[root@k8s-master ~]# kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"29", GitVersion:"v1.29.7", GitCommit:"4e4a18878ce330fefda1dc46acca88ba355e9ce7", GitTreeState:"clean", BuildDate:"2024-07-17T00:04:38Z", GoVersion:"go1.22.5", Compiler:"gc", Platform:"linux/amd64"}
[root@k8s-master ~]# kubelet --version
Kubernetes v1.29.7

## cgroup kubelet설치 
[root@k8s-master ~]# mkdir -p /etc/default
[root@k8s-master ~]# echo 'KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"' | sudo tee /etc/default/kubelet
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"
[root@k8s-master ~]# cd /etc/default
[root@k8s-master default]# ls
grub  kubelet  useradd

init

kubeadm init --config=kubeadm-config.yaml --v=5

[root@k8s-master pki]# kubectl get po -A
NAMESPACE     NAME                                 READY   STATUS              RESTARTS   AGE
kube-system   coredns-76f75df574-97g57             0/1     ContainerCreating   0          4m25s
kube-system   coredns-76f75df574-x9jk5             0/1     ContainerCreating   0          4m25s
kube-system   kube-apiserver-k8s-master            1/1     Running             0          4m32s
kube-system   kube-controller-manager-k8s-master   1/1     Running             0          4m32s
kube-system   kube-proxy-47ghm                     1/1     Running             0          4m25s
kube-system   kube-proxy-855m2                     1/1     Running             0          23s
kube-system   kube-scheduler-k8s-master            1/1     Running             0          4m32s

ETCD 인증서 갱신

인증서 체크

etcd 외부 설치 된 경우 (/etc/etcd 경로)

인증서 체크를 해보면 etcd가 누락되어있는 것이 보임

[root@k8s-master pki]# kubeadm certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'

CERTIFICATE                EXPIRES                  RESIDUAL TIME   CERTIFICATE AUTHORITY   EXTERNALLY MANAGED
admin.conf                 Jun 11, 2026 06:06 UTC   364d            ca                      no
apiserver                  Jun 11, 2026 06:06 UTC   364d            ca                      no
apiserver-kubelet-client   Jun 11, 2026 06:06 UTC   364d            ca                      no
controller-manager.conf    Jun 11, 2026 06:06 UTC   364d            ca                      no
front-proxy-client         Jun 11, 2026 06:06 UTC   364d            front-proxy-ca          no
scheduler.conf             Jun 11, 2026 06:07 UTC   364d            ca                      no
super-admin.conf           Jun 11, 2026 06:06 UTC   364d            ca                      no

CERTIFICATE AUTHORITY   EXPIRES                  RESIDUAL TIME   EXTERNALLY MANAGED
ca                      Jun 09, 2035 06:06 UTC   9y              no
front-proxy-ca          Jun 09, 2035 06:06 UTC   9y              no

인증서 적용 (etcd제외 모든 인증서 갱신)

클러스터 인증서 갱신
https://somaz.tistory.com/148

./update-kubeadm-cert.sh  master # etcd 제외

etcd 인증서 날짜 확인

[root@k8s-master pki]# ls
ca.crt  ca.key  ca.srl  etcd-client.crt  etcd-client.csr  etcd-client.key  etcd-server.crt  etcd-server.key

[root@k8s-master pki]# openssl x509 -in etcd-server.crt -noout -dates
notBefore=Jun 11 05:52:15 2025 GMT
notAfter=Jun  9 05:52:15 2035 GMT
[root@k8s-master pki]# openssl x509 -in etcd-client.crt -noout -dates
notBefore=Jun 11 05:58:56 2025 GMT
notAfter=Jun  9 05:58:56 2035 GMT
[root@k8s-master pki]#

파일명	용도	설명	확인 명령어
`ca.crt`	CA 인증서	인증서 서명자(issuer) 역할. 모든 etcd 관련 인증서의 신뢰 루트.	`openssl x509 -in ca.crt -noout -subject -dates`
`ca.key`	CA 비밀키	인증서 서명에 사용되는 비공개 키. 매우 중요. 외부 노출 금지.	🔒 노출 금지
`ca.srl`	CA 시리얼 추적	인증서 일련번호 관리 파일. 자동 생성. 삭제 X.	-
`etcd-server.crt`	etcd 서버 인증서	etcd 자신이 클라이언트(API 서버 등)와 통신할 때 사용하는 인증서. SAN에 IP 들어가야 함.	`openssl x509 -in etcd-server.crt -noout -subject -issuer -dates -text
`etcd-server.key`	etcd 서버 비밀키	위 서버 인증서에 대응하는 개인 키.	🔒 노출 금지
`etcd-client.crt`	etcd 클라이언트 인증서	Kubernetes API 서버가 etcd에 접근할 때 사용하는 인증서. CN은 일반적으로 `kube-apiserver`.	`openssl x509 -in etcd-client.crt -noout -subject -dates`
`etcd-client.key`	클라이언트 비밀키	위 인증서에 대응하는 개인 키.	🔒 노출 금지
`etcd-client.csr`	인증서 서명 요청	클라이언트 인증서를 만들 때 사용했던 요청 파일 (재사용 가능).	`openssl req -in etcd-client.csr -noout -subject`

SAN포함 여부 확인

CN 외에 인증서가 유효한 도메인 이름이나 IP 주소 등을 여러 개 넣을 수 있는 확장 필드

SAN은 접속 가능한 여러 IP/DNS를 명시해서 인증서가 여러 이름을 커버하도록 함

openssl x509 -in etcd-server.crt -noout -text | grep -A1 "Subject Alternative Name"

[root@k8s-master pki]# openssl x509 -in etcd-server.crt -noout -text | grep -A1 "Subject Alternative Name"
            X509v3 Subject Alternative Name:
                IP Address:192.168.0.66 ### 출력

CN 확인 : 인증서의 주체 이름

[root@k8s-master pki]# openssl x509 -in etcd-server.crt -noout -subject
subject=CN = etcd-server
[root@k8s-master pki]# openssl x509 -in etcd-client.crt -noout -subject
subject=CN = etcd-client

etcd 외부 인증서 갱신

조건

인증서가 셀프사인이고,
기존 CA (etcd-ca.key, etcd-ca.crt)가 보존되어 있으며,
기존 SAN, CN 정보를 유지하면서 유효기간만 연장하고 싶을 경우

config파일

[root@k8s-master pki]# cat openssl-etcd.cnf
[ req ]
default_bits       = 2048
prompt             = no
default_md         = sha256
distinguished_name = req_distinguished_name
req_extensions     = v3_ext

[ req_distinguished_name ]
CN = etcd-server  ## 수정CN

[ v3_ext ]
subjectAltName = @alt_names

[ alt_names ]
IP.1 = 192.168.0.66 ## 수정 SAN

config 적용

# etcd-server
openssl req -new -key etcd-server.key -out etcd-server.csr -config openssl-etcd.cnf
# etcd-client
openssl req -new -key etcd-client.key -out etcd-client.csr -config openssl-etcd.cnf

CSR을 사용하여 새 인증서 생성

[root@k8s-master pki]# openssl x509 -req -in etcd-client.csr \
>   -CA ca.crt -CAkey ca.key -CAcreateserial \
>   -out etcd-client.crt -days 7300 -extensions v3_ext \
>   -extfile openssl-etcd.cnf
Signature ok
subject=CN = etcd-client
Getting CA Private Key

[root@k8s-master pki]# openssl x509 -req -in etcd-server.csr \
>   -CA ca.crt -CAkey ca.key -CAcreateserial \
>   -out etcd-server.crt -days 7300 -extensions v3_ext \
>   -extfile openssl-etcd.cnf
Signature ok
subject=CN = etcd-server
Getting CA Private Key

생성된 인증서 etcd-server.crt etcd-client.crt

systemctl restart etcd

## 적용 확인
[root@k8s-master pki]# openssl x509 -in etcd-server.crt -noout -dates
notBefore=Jun 11 07:31:43 2025 GMT
notAfter=Jun  6 07:31:43 2045 GMT
[root@k8s-master pki]# openssl x509 -in etcd-client.crt -noout -dates
notBefore=Jun 11 07:31:16 2025 GMT
notAfter=Jun  6 07:31:16 2045 GMT