k8s deploy 4주차

dev_suha·2026년 1월 31일

kubespray

Ansible 기반으로 Kubernetes 클러스터를 자동 설치·구성하는 오픈소스 프로젝트

cycle in kubespray

  • Release Cycle in Kubespray : ‘26.01.24
    KubesprayKube Version (N-2)
    2.29.x (master-branch)1.31 ~ 1.33 (1.34)
    2.28.x1.30 ~ 1.32
    2.27.x1.29 ~ 1.31
    • Kubespray 한 버전당 Kubernetes 3개 minor 지원
    • 운영 시 버전 추천
      • Dev 환경 : Kubespray 최신 + K8s N-1
      • Prd 환경 : Kubespray 최신-1 + K8s N-2

kubespray 설치 사전 요건

  • Ansible v2.14+, Jinja 2.11+
  • Ansible 2.17.3 이상 & Python 3.10 ~ 3.12
  • Spec : Control Plane (Memory 2 GB), Worker Node (Memory 1 GB)
  • Linux Kernel Requirements : 5.8+ 이상 권장
  • Rocky Linux 9, 10 (experimental in 10: see Rocky Linux 10 notes)

실습 환경 배포

실습 환경 배포 수행

# 실습용 디렉터리 생성
mkdir k8s-kubespary
cd k8s-kubespary

# 파일 다운로드
https://raw.githubusercontent.com/gasida/vagrant-lab/refs/heads/main/k8s-kubespary/Vagrantfile
https://raw.githubusercontent.com/gasida/vagrant-lab/refs/heads/main/k8s-kubespary/init_cfg.sh

# 실습 환경 배포
vagrant up
vagrant status

Vagrantfile

# Base Image  https://portal.cloud.hashicorp.com/vagrant/discover/bento/rockylinux-10.0
BOX_IMAGE = "bento/rockylinux-10.0"
BOX_VERSION = "202510.26.0"

Vagrant.configure("2") do |config|

# ControlPlane Nodes 
    config.vm.define "k8s-ctr" do |subconfig|
      subconfig.vm.box = BOX_IMAGE
      subconfig.vm.box_version = BOX_VERSION
      subconfig.vm.provider "virtualbox" do |vb|
        vb.customize ["modifyvm", :id, "--groups", "/Kubespray-Lab"]
        vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
        vb.name = "k8s-ctr"
        vb.cpus = 4
        vb.memory = 4096
        vb.linked_clone = true
      end
      subconfig.vm.host_name = "k8s-ctr"
      subconfig.vm.network "private_network", ip: "192.168.10.10"
      subconfig.vm.network "forwarded_port", guest: 22, host: "60100", auto_correct: true, id: "ssh"
      subconfig.vm.synced_folder "./", "/vagrant", disabled: true
      subconfig.vm.provision "shell", path: "init_cfg.sh"
    end

end

init_cfg.sh

#!/usr/bin/env bash

echo ">>>> Initial Config Start <<<<"

echo "[TASK 1] Change Timezone and Enable NTP"
timedatectl set-local-rtc 0
timedatectl set-timezone Asia/Seoul

echo "[TASK 2] Disable firewalld and selinux"
systemctl disable --now firewalld >/dev/null 2>&1
setenforce 0
sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config

echo "[TASK 3] Disable and turn off SWAP & Delete swap partitions"
swapoff -a
sed -i '/swap/d' /etc/fstab
sfdisk --delete /dev/sda 2 >/dev/null 2>&1
partprobe /dev/sda >/dev/null 2>&1

echo "[TASK 4] Config kernel & module"
cat << EOF > /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF
modprobe overlay >/dev/null 2>&1
modprobe br_netfilter >/dev/null 2>&1

cat << EOF >/etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF
sysctl --system >/dev/null 2>&1

echo "[TASK 5] Setting Local DNS Using Hosts file"
sed -i '/^127\.0\.\(1\|2\)\.1/d' /etc/hosts
cat << EOF >> /etc/hosts
192.168.10.10 k8s-ctr
EOF

echo "[TASK 6] Delete default routing - enp0s9 NIC" # setenforce 0 설정 필요
nmcli connection modify enp0s9 ipv4.never-default yes
nmcli connection up enp0s9 >/dev/null 2>&1

echo "sudo su -" >> /home/vagrant/.bashrc

echo ">>>> Initial Config End <<<<"

사전 설정 수행

which python3 && python3 -V
/usr/bin/python
Python 3.12.9
/usr/bin/python3
Python 3.12.9

**dnf install -y python3-pip git**
which pip  && pip -V
which pip3 && pip3 -V

# /etc/hosts 확인
ip -br -c -4 addr

lo               UNKNOWN        127.0.0.1/8 
enp0s3           UP             10.0.2.15/24
enp0s8           UP             192.168.10.10/24

cat /etc/hosts

# Loopback entries; do not change.
# For historical reasons, localhost precedes localhost.localdomain:
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
# See hosts(5) for proper format and other examples:
# 192.168.1.10 foo.example.org foo
# 192.168.1.13 bar.example.org bar
192.168.10.10 k8s-ctr

ping -c 1 k8s-ctr

--- k8s-ctr ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.181/0.181/0.181/0.000 ms

# ssh 접속 설정 

**echo "root:qwe123" | chpasswd**

cat << EOF >> /etc/ssh/sshd_config
**PermitRootLogin yes
PasswordAuthentication yes**
EOF
**systemctl restart sshd

ssh-keygen -t rsa -N "" -f /root/.ssh/id_rsa
ls -l ~/.ssh

The key's randomart image is:
+---[RSA 3072]----+
|         ..=++...|
|          = o+ . |
|        .+  . o  |
|        *o.  .   |
|     o +SXo   o .|
|      B B==. o o |
|     * *..Eoo    |
|    . = o+ o.    |
|     ..+. ..o    |
+----[SHA256]-----+

-rw-------. 1 root root 2602 Jan 26 20:12 id_rsa
-rw-r--r--. 1 root root  566 Jan 26 20:12 id_rsa.pub
root@k8s-ctr:~# ssh-copy-id -o StrictHostKeyChecking=no root@192.168.10.10
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/.ssh/id_rsa.pub"
/usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed
/usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys**

root@k8s-ctr:~# cat /root/.ssh/authorized_keys
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC7KmkUKZduzT1X1o5kvQK0kPQ1Nne5EO+stlmSx9w/f1ClXrLkwjxAQ3LFzYF6l7waamfksPh8lsKMwK9sX1LtOvGfyVr42gKCymdhnxSJstIGrdETlVbjcZTFdA/0GykydJyeG1wKsGQxuuorr1OAesKWssRQ/zWvFRjfjGWNSWGa1iwBGraaKubh12AIyUuKQMeIL6wx9QTX4R4MxsoEzaOCFyye/z4qwUF6hb0fPjcf2bdKFYbHq550unN7HCbdLINeh45n+rAxiOw7fK2Ud2JmV1sVaU5rXOEgOnG3ZTST8+y0RXWOctgy3Ancx9MyXBC8rd8RY/28nlbsRGO42ZkHDY5w9e9SfqEHc4vUiHaiMpNt+q1kjsFftpvaFDlx5UgBdfgAU0tiZo8pQ6eF1gAnubuCS6V7iCyFx/QMkWXaMq/H1AIWXFLNAjrs75SHXO7H0nj/xxzEC1f+mCeZcQ8z6otEO9MvEnKoRkAXCpXiNXsTQL10g9og4o7LKTU= root@k8s-ctr
root@k8s-ctr:~# ssh root@192.168.10.10 hostname
ssh -o StrictHostKeyChecking=no root@k8s-ctr hostname
ssh root@k8s-ctr hostname
k8s-ctr
Warning: Permanently added 'k8s-ctr' (ED25519) to the list of known hosts.
k8s-ctr
k8s-ctr

# Clone Kubespray Repository
git clone **-b v2.29.1** https://github.com/kubernetes-sigs/kubespray.git /root/kubespray
**cd /root/kubespray

## vs-code vm ssh 접속 
HOST admin-vm
    HostName 192.168.10.10
    User root

ls -l *.yml
-rw-r--r--. 1 root root  88 Jan 26 20:14 cluster.yml
-rw-r--r--. 1 root root  30 Jan 26 20:14 _config.yml
-rw-r--r--. 1 root root 747 Jan 26 20:14 galaxy.yml
-rw-r--r--. 1 root root 105 Jan 26 20:14 recover-control-plane.yml
-rw-r--r--. 1 root root  85 Jan 26 20:14 remove-node.yml
-rw-r--r--. 1 root root  85 Jan 26 20:14 remove_node.yml
-rw-r--r--. 1 root root  85 Jan 26 20:14 reset.yml
-rw-r--r--. 1 root root  85 Jan 26 20:14 scale.yml
-rw-r--r--. 1 root root  93 Jan 26 20:14 upgrade-cluster.yml
-rw-r--r--. 1 root root  93 Jan 26 20:14 upgrade_cluster.yml

cat requirements.txt
ansible==10.7.0
# Needed for community.crypto module
cryptography==46.0.2
# Needed for jinja2 json_query templating
jmespath==1.0.1
# Needed for ansible.utils.ipaddr
netaddr==1.3.0

pip3 install -r /root/kubespray/requirements.txt

root@k8s-ctr:~/kubespray# which ansible
/usr/local/bin/ansible

# ansible-playbook 실행 시 적용되는 ansible.cfg 
cat ansible.cfg 
[ssh_connection]
pipelining=True # 하나의 세션에서 여러 명령 진행 
ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null #한 번 연결된 커넥션 30분 유지 네트워크 연결 시도 100
control_path = ~/.ssh/ansible-%%r@%%h:%%p
[defaults]
# https://github.com/ansible/ansible/issues/56930 (to ignore group names with - and .)
force_valid_group_names = ignore #이름 규칙 무시 

host_key_checking=False
gathering = smart # 대상 fact를 한번만 수집후 /tmp에 JSON 파일로 저장 
fact_caching = jsonfile
fact_caching_connection = /tmp
fact_caching_timeout = 86400
timeout = 300
stdout_callback = default
display_skipped_hosts = no
library = ./library
callbacks_enabled = profile_tasks
roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles
deprecation_warnings=False
inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds, .gpg
[inventory]
ignore_patterns = artifacts, credentials**

Kubespary 를 통한 k8s 배포

**cat << EOF > /root/kubespray/inventory/mycluster/inventory.ini
k8s-ctr ansible_host=192.168.10.10 ip=192.168.10.10

[kube_control_plane]
k8s-ctr

[etcd:children]
kube_control_plane

[kube_node]
k8s-ctr
EOF

root@k8s-ctr:~/kubespray# grep -iE 'kube_network_plugin:|kube_proxy_mode|enable_nodelocaldns:|^auto_renew_certificates' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
kube_network_plugin: calico
kube_proxy_mode: ipvs
enable_nodelocaldns: true
auto_renew_certificates: false
root@k8s-ctr:~/kubespray# sed -i 's|kube_network_plugin: calico|kube_network_plugin: flannel|g' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
sed -i 's|kube_proxy_mode: ipvs|kube_proxy_mode: iptables|g' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
sed -i 's|enable_nodelocaldns: true|enable_nodelocaldns: false|g' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
sed -i 's|auto_renew_certificates: false|auto_renew_certificates: true|g' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
sed -i 's|# auto_renew_certificates_systemd_calendar|auto_renew_certificates_systemd_calendar|g' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
root@k8s-ctr:~/kubespray# grep -iE 'kube_network_plugin:|kube_proxy_mode|enable_nodelocaldns:|^auto_renew_certificates' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
kube_network_plugin: flannel
kube_proxy_mode: iptables
enable_nodelocaldns: false
auto_renew_certificates: true
auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:00:00"

root@k8s-ctr:~/kubespray# echo "flannel_interface: enp0s8" >> inventory/mycluster/group_vars/k8s_cluster/k8s-net-flannel.yml
root@k8s-ctr:~/kubespray# grep "^[^#]" inventory/mycluster/group_vars/k8s_cluster/k8s-net-flannel.yml
flannel_interface: enp0s8

root@k8s-ctr:~/kubespray# sed -i 's|helm_enabled: false|helm_enabled: true|g' inventory/mycluster/group_vars/k8s_cluster/addons.yml
sed -i 's|metrics_server_enabled: false|metrics_server_enabled: true|g' inventory/mycluster/group_vars/k8s_cluster/addons.yml
sed -i 's|node_feature_discovery_enabled: false|node_feature_discovery_enabled: true|g' inventory/mycluster/group_vars/k8s_cluster/addons.yml
root@k8s-ctr:~/kubespray# grep -iE 'helm_enabled:|metrics_server_enabled:|node_feature_discovery_enabled:' inventory/mycluster/group_vars/k8s_cluster/addons.yml
helm_enabled: true
metrics_server_enabled: true
node_feature_discovery_enabled: true**

# 기본 환경 정보 출력 저장

ip addr | tee -a ip_addr-1.txt 
ss -tnlp | tee -a ss-1.txt
df -hT | tee -a df-1.txt
findmnt | tee -a findmnt-1.txt
sysctl -a | tee -a sysctl-1.txt

# 배포   
ANSIBLE_FORCE_COLOR=true **ansible-playbook -i inventory/mycluster/inventory.ini -v cluster.yml -e kube_version="1.33.3" | tee kubespray_install.log**

**kubectl get node -v=6
NAME      STATUS   ROLES           AGE     VERSION
k8s-ctr   Ready    control-plane   7m43s   v1.33.3

kubectl get node -owide
NAME      STATUS   ROLES           AGE    VERSION   INTERNAL-IP     EXTERNAL-IP   OS-IMAGE                        KERNEL-VERSION                 CONTAINER-RUNTIME
k8s-ctr   Ready    control-plane   8m7s   v1.33.3   192.168.10.10   <none>        Rocky Linux 10.0 (Red Quartz)   6.12.0-55.39.1.el10_0.x86_64   containerd://2.1.5

## 기본 환경 정보 출력 저장** 
ip addr | tee -a ip_addr-2.txt 
ss -tnlp | tee -a ss-2.txt
df -hT | tee -a df-2.txt
findmnt | tee -a findmnt-2.txt
sysctl -a | tee -a sysctl-2.txt

# 파일 출력 비교 : 빠져나오기 ':q' -> ':q' => 변경된 부분이 어떤 동작과 역할인지 조사해보기! , ctrl + f / b
vi -d ip_addr-1.txt ip_addr-2.txt
vi -d ss-1.txt ss-2.txt
vi -d df-1.txt df-2.txt
vi -d findmnt-1.txt findmnt-2.txt
vi -d sysctl-1.txt sysctl-2.txt
****

alias, 자동완성 및 k9s

source <(kubectl completion bash)
source <(kubeadm completion bash)

# Alias kubectl to k
alias k=kubectl
complete -o default -F __start_kubectl k

# k9s 설치 : https://github.com/derailed/k9s
CLI_ARCH=amd64
if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
wget https://github.com/derailed/k9s/releases/latest/download/k9s_linux_${CLI_ARCH}.tar.gz
tar -xzf k9s_linux_*.tar.gz
ls -al k9s
chown root:root k9s
mv k9s /usr/local/bin/
chmod +x /usr/local/bin/k9s
**k9s**

image.png

ansible playbook & Role 분석

사전 검증 → 접근/부트스트랩 → etcd → node → control-plane → CNI → 애드온

[공통 준비]
  ├─ boilerplate.yml
  └─ internal_facts.yml

[사전 준비 단계]
  └─ Preinstall + Container Runtime + Download

[etcd]
  └─ etcd Cluster 설치

[Kubernetes 노드 기반 구성]
  ├─ Node 설정
  ├─ Control Plane 설정
  └─ Client / Cluster Role 구성

[클러스터 초기화]
  └─ kubeadm + CNI(Network)

[네트워크 확장]
  └─ Calico Route Reflector (미사용 skip)

[플랫폼 패치]
  └─ Windows Node 대응 패치 (미사용 skip)

[애플리케이션 계층]
  └─ Ingress / Storage / Policy / Apps

[DNS 후처리]
  └─ resolv.conf 재적용

1. 공통 준비

[Common tasks]
 └─ boilerplate.yml
     ├─ Python / OS 기본 설정
     ├─ SSH, sudo, 패키지 준비
     └─ Kubespray 공통 변수 초기화

[Gather facts]
 └─ internal_facts.yml
     ├─ 노드 OS / 커널
     ├─ NIC / IP
     └─ CPU, Memory

2. 사전 준비 단계

[Prepare for etcd install]
 (hosts: k8s_cluster + etcd)

 ├─ kubespray_defaults
 ├─ kubernetes/preinstall
 │   ├─ swap off
 │   ├─ sysctl (net.ipv4.ip_forward 등)
 │   └─ 커널 모듈
 ├─ container-engine
 │   └─ containerd / cri-o
 └─ download
     └─ kube / etcd 바이너리

3. etcd 클러스터 설치

[Install etcd]
 (import_playbook)

 ├─ etcd_cluster_setup = true
 ├─ etcd member 생성
 ├─ TLS 인증서 생성
 └─ etcd HA 구성

4. Kubernetes 노드 기본 구성

[Install Kubernetes nodes]
 (hosts: k8s_cluster)

 ├─ kubelet 설치
 ├─ kubelet 설정
 └─ 노드 공통 설정

5. control plane 설치

[Install the control plane]
 (hosts: kube_control_plane)

 ├─ API Server
 ├─ Controller Manager
 ├─ Scheduler
 ├─ kubectl (client)
 └─ RBAC / ClusterRoles

6. kubeadm 실행 + CNI 설치 (핵심)

[Invoke kubeadm and install a CNI]
 (hosts: k8s_cluster)

 ├─ kubeadm init / join
 ├─ node-label
 ├─ node-taint
 ├─ CRD 등록
 └─ network_plugin
     └─ Calico / Cilium / Flannel
    1. Calico Route Reflector (skip)
    1. Windows 노드 패치 (skip)

9. Kubernetes 애플리케이션 설치

[Install Kubernetes apps]
 (hosts: kube_control_plane)

 ├─ External Cloud Controller
 ├─ Policy Controller
 ├─ Ingress Controller
 ├─ External Provisioner
 └─ kubernetes-apps (기본 애드온)

10. DNS 후처리

[Apply resolv.conf changes]
 (hosts: k8s_cluster)

 └─ resolv.conf 재설정
     (CoreDNS 정상 기동 이후)

play 과정

cat kubespray_install.log | grep -E 'PLAY'
PLAY [Check Ansible version] ***************************************************
PLAY [Inventory setup and validation] ******************************************
PLAY [Install bastion ssh config] **********************************************
PLAY [Bootstrap hosts for Ansible] *********************************************
PLAY [Gather facts] ************************************************************
PLAY [Prepare for etcd install] ************************************************
PLAY [Add worker nodes to the etcd play if needed] *****************************
PLAY [Install etcd] ************************************************************
PLAY [Install Kubernetes nodes] ************************************************
PLAY [Install the control plane] ***********************************************
PLAY [Invoke kubeadm and install a CNI] ****************************************
PLAY [Install Calico Route Reflector] ******************************************
PLAY [Patch Kubernetes for Windows] ********************************************
PLAY [Install Kubernetes apps] *************************************************
PLAY [Apply resolv.conf changes now that cluster DNS is up] ********************
PLAY RECAP *********************************************************************
  1. [Check Ansible version] : kubespray가 지원하는 ansible version 확인
  2. [Inventory setup and validation] : inventory 설정 정합성 검증
  3. [Install bastion ssh config] : bastion 환경 지원
  4. [Bootstrap hosts for Ansible] : Ansible 실행 가능한 상태로 만듦 : Python 설치, sudo 권한 확보, 기본 패키지 설치, /usr/bin/python 보장
  5. [Gather facts]: fact 수집
  6. [Prepare for etcd install] : etcd 설치전 사전준비
  7. [Add worker nodes to the etcd play if needed] : kube_node: + etcd: 둘 다 포함된 노드를 etcd PLAY에 추가
  8. [Install etcd] : etcd 설치, tls 인증서 생성, systemd 등록
  9. [Install Kubernetes nodes] : 모든 노드에 공통 K8s 컴포넌트 설치, 아직 클러스터 join은 안 함
  10. [Install the control plane] : control-plane 노드 구성
  11. [Invoke kubeadm and install a CNI] : 네트워크 cni 설치
  12. [Install Calico Route Reflector] : 미사용 skip
  13. [Patch Kubernetes for Windows] : 미사용 skip
  14. [Install Kubernetes apps] : 기본 add-on 설치
  15. [Apply resolv.conf changes now that cluster DNS is up] : coreDNS 설치후 노드의 DNS 설정 최종 정리

실습 배포 환경 분석

다운로드 파일 경로 확인

root@k8s-ctr:~# tree -ug /tmp/releases/
[root     root    ]  /tmp/releases/
├── [root     root    ]  cni-plugins-linux-amd64-1.8.0.tgz
├── [root     root    ]  containerd-2.1.5-linux-amd64.tar.gz
├── [root     root    ]  containerd-rootless-setuptool.sh
├── [root     root    ]  containerd-rootless.sh
├── [root     118     ]  crictl
├── [root     root    ]  crictl-1.33.0-linux-amd64.tar.gz
├── [root     root    ]  etcd-3.5.25-linux-amd64.tar.gz
├── [vagrant  vagrant ]  etcd-v3.5.25-linux-amd64
│   ├── [vagrant  vagrant ]  Documentation
│   │   ├── [vagrant  vagrant ]  dev-guide
│   │   │   └── [vagrant  vagrant ]  apispec
│   │   │       └── [vagrant  vagrant ]  swagger
│   │   │           ├── [vagrant  vagrant ]  rpc.swagger.json
│   │   │           ├── [vagrant  vagrant ]  v3election.swagger.json
│   │   │           └── [vagrant  vagrant ]  v3lock.swagger.json
│   │   └── [vagrant  vagrant ]  README.md
│   ├── [vagrant  vagrant ]  etcd
│   ├── [vagrant  vagrant ]  etcdctl
│   ├── [vagrant  vagrant ]  etcdutl
│   ├── [vagrant  vagrant ]  README-etcdctl.md
│   ├── [vagrant  vagrant ]  README-etcdutl.md
│   ├── [vagrant  vagrant ]  README.md
│   └── [vagrant  vagrant ]  READMEv2-etcdctl.md
├── [root     root    ]  helm-3.18.4
│   ├── [root     root    ]  helm-3.18.4-linux-amd64.tar.gz
│   └── [root     118     ]  linux-amd64
│       ├── [root     118     ]  helm
│       ├── [root     118     ]  LICENSE
│       └── [root     118     ]  README.md
├── [root     root    ]  images
├── [root     root    ]  kubeadm-1.33.3-amd64
├── [root     root    ]  kubectl-1.33.3-amd64
├── [root     root    ]  kubelet-1.33.3-amd64
├── [root     root    ]  nerdctl
├── [root     root    ]  nerdctl-2.1.6-linux-amd64.tar.gz
└── [root     root    ]  runc-1.3.4.amd64

설치된 바이너리 확인

/usr/local/bin/
├── ansible
├── ansible-community
├── ansible-config
├── ansible-connection
├── ansible-console
├── ansible-doc
├── ansible-galaxy
├── ansible-inventory
├── ansible-playbook
├── ansible-pull
├── ansible-test
├── ansible-vault
├── containerd
├── containerd-shim-runc-v2
├── containerd-stress
├── crictl
├── ctr
├── etcd
├── etcdctl
├── etcdctl.sh
├── etcd-scripts
│   └── make-ssl-etcd.sh
├── etcdutl
├── helm
├── jp.py
├── k8s-certs-renew.sh
├── k9s
├── kubeadm
├── kubectl
├── kubelet
├── kubernetes-scripts
├── nerdctl
├── netaddr
├── __pycache__
│   └── jp.cpython-312.pyc
└── runc
  • 바이너리 설치된 경로 /usr/local/bin

설치된 cni 관련 파일 확인

root@k8s-ctr:~# find / -user kube 2>/dev/null
/etc/cni
/etc/cni/net.d
/etc/kubernetes
/etc/kubernetes/manifests
/usr/libexec/kubernetes
/usr/libexec/kubernetes/kubelet-plugins
/usr/libexec/kubernetes/kubelet-plugins/volume
/usr/libexec/kubernetes/kubelet-plugins/volume/exec
/usr/local/bin/kubernetes-scripts
/opt/cni
/opt/cni/bin
/opt/cni/bin/README.md
/opt/cni/bin/static
/opt/cni/bin/host-device
/opt/cni/bin/ipvlan
/opt/cni/bin/dhcp
/opt/cni/bin/LICENSE
/opt/cni/bin/portmap
/opt/cni/bin/tap
/opt/cni/bin/host-local
/opt/cni/bin/vlan
/opt/cni/bin/loopback
/opt/cni/bin/sbr
/opt/cni/bin/firewall
/opt/cni/bin/bandwidth
/opt/cni/bin/bridge
/opt/cni/bin/vrf
/opt/cni/bin/macvlan
/opt/cni/bin/tuning
/opt/cni/bin/dummy
/opt/cni/bin/ptp

ls -l /opt
total 0
drwxr-xr-x. 3 kube root  17 Jan 26 21:57 cni
drwx--x--x. 4 root root  28 Jan 26 22:10 containerd
drwxr-xr-x. 7 root root 119 Oct 24 06:42 VBoxGuestAdditions-7.2.4

ls -l /etc | grep cni
drwxr-xr-x. 3 kube root     19 Jan 26 21:57 cni

cat /etc/cni/net.d/10-flannel.conflist
{
  "name": "cbr0",
  "cniVersion": "0.3.1",
  "plugins": [
    {
      "type": "flannel",
      "delegate": {
        "hairpinMode": true,
        "isDefaultGateway": true
      }
    },
    {
      "type": "portmap",
      "capabilities": {
        "portMappings": true
      }
    }
  ]
}
  • kube user로 만들어지는 내역 확인

k8s-certs-renew.sh 스크립트 확인

systemctl list-timers --all --no-pager
NEXT                            LEFT LAST                              PASSED UNIT                         ACTIVATES                     
Tue 2026-01-27 22:35:15 KST    56min Tue 2026-01-27 21:11:23 KST    27min ago dnf-makecache.timer          dnf-makecache.service
Wed 2026-01-28 00:04:46 KST 2h 26min Tue 2026-01-27 21:11:23 KST    27min ago logrotate.timer              logrotate.service
Wed 2026-01-28 00:23:05 KST 2h 44min Tue 2026-01-27 21:11:23 KST    27min ago plocate-updatedb.timer       plocate-updatedb.service
Wed 2026-01-28 19:47:00 KST      22h Mon 2026-01-26 22:02:16 KST 1h 51min ago systemd-tmpfiles-clean.timer systemd-tmpfiles-clean.service
Sun 2026-02-01 01:00:00 KST   4 days Mon 2026-01-26 21:47:57 KST  2h 6min ago raid-check.timer             raid-check.service
Mon 2026-02-02 01:05:21 KST   5 days Mon 2026-01-26 21:57:50 KST 1h 56min ago fstrim.timer                 fstrim.service
Mon 2026-02-02 03:01:57 KST   5 days -                                      - k8s-certs-renew.timer        k8s-certs-renew.service

cat /etc/systemd/system/k8s-certs-renew.timer
[Unit]
Description=Timer to renew K8S control plane certificates

[Timer]
OnCalendar=Mon *-*-1,2,3,4,5,6,7 03:00:00
RandomizedDelaySec=10min
FixedRandomDelay=yes
Persistent=yes

[Install]
WantedBy=multi-user.target

cat /etc/systemd/system/k8s-certs-renew.service
[Unit]
Description=Renew K8S control plane certificates

[Service]
Type=oneshot
ExecStart=/usr/local/bin/k8s-certs-renew.sh

# 스크립트 확인 
cat /usr/local/bin/k8s-certs-renew.sh
#!/bin/bash

echo "## Check Expiration before renewal ##"

/usr/local/bin/kubeadm certs check-expiration

days_buffer=7 # set a time margin, because we should not renew at the last moment
calendar=Mon *-*-1,2,3,4,5,6,7 03:00:00
next_time=$(systemctl show k8s-certs-renew.timer  -p NextElapseUSecRealtime --value)

if [ "${next_time}" == "" ]; then
        echo "## Skip expiry comparison due to fail to parse next elapse from systemd calendar,do renewal directly ##"
else
        current_time=$(date +%s)
        target_time=$(date -d "${next_time} + ${days_buffer} days" +%s) # $next_time - $days_buffer days
        expiry_threshold=$(( ${target_time} - ${current_time} ))
        expired_certs=$(/usr/local/bin/kubeadm certs check-expiration -o jsonpath="{.certificates[?(@.residualTime<${expiry_threshold}.0)]}")
    if [ "${expired_certs}" == "" ];then
                echo "## Skip cert renew and K8S container restart, since all residualTimes are beyond threshold ##"
                exit 0
        fi
fi

echo "## Renewing certificates managed by kubeadm ##"
/usr/local/bin/kubeadm certs renew all

echo "## Restarting control plane pods managed by kubeadm ##"
/usr/local/bin/crictl pods --namespace kube-system --name 'kube-scheduler-*|kube-controller-manager-*|kube-apiserver-*|etcd-*' -q | /usr/bin/xargs /usr/local/bin/crictl rmp -f

echo "## Updating /root/.kube/config ##"
cp /etc/kubernetes/admin.conf /root/.kube/config

echo "## Waiting for apiserver to be up again ##"
until printf "" 2>>/dev/null >>/dev/tcp/127.0.0.1/6443; do sleep 1; done

echo "## Expiration after renewal ##"
/usr/local/bin/kubeadm certs check-expiration
  • OnCalendar=Mon --1,2,3,4,5,6,7 03:00:00 : 매달 1~7일 중에서 월요일인 경우 3 am 작동
  • /usr/local/bin/crictl pods --namespace kube-system --name 'kube-scheduler-|kube-controller-manager-|kube-apiserver-|etcd-' -q | /usr/bin/xargs /usr/local/bin/crictl rmp -f : kube-system 네임스페이스에 있는 Kubernetes 컨트롤 플레인(static pod)들을 강제로 삭제

TASK [bootstrap_os : Gather facts]

/tmp
├── code-8933ce89-5e06-4ce7-b3fd-d884ab82ff7c
├── k8s-ctr
├── releases
│   ├── cni-plugins-linux-amd64-1.8.0.tgz
│   ├── containerd-2.1.5-linux-amd64.tar.gz
│   ├── containerd-rootless-setuptool.sh
│   ├── containerd-rootless.sh
│   ├── crictl
│   ├── crictl-1.33.0-linux-amd64.tar.gz
│   ├── etcd-3.5.25-linux-amd64.tar.gz
│   ├── etcd-v3.5.25-linux-amd64
│   │   ├── Documentation
│   │   │   ├── dev-guide
│   │   │   │   └── apispec
│   │   │   │       └── swagger
│   │   │   │           ├── rpc.swagger.json
│   │   │   │           ├── v3election.swagger.json
│   │   │   │           └── v3lock.swagger.json
│   │   │   └── README.md
│   │   ├── etcd
│   │   ├── etcdctl
│   │   ├── etcdutl
│   │   ├── README-etcdctl.md
│   │   ├── README-etcdutl.md
│   │   ├── README.md
│   │   └── READMEv2-etcdctl.md
│   ├── helm-3.18.4
│   │   ├── helm-3.18.4-linux-amd64.tar.gz
│   │   └── linux-amd64
│   │       ├── helm
│   │       ├── LICENSE
│   │       └── README.md
│   ├── images
│   ├── kubeadm-1.33.3-amd64
│   ├── kubectl-1.33.3-amd64
│   ├── kubelet-1.33.3-amd64
│   ├── nerdctl
│   ├── nerdctl-2.1.6-linux-amd64.tar.gz
│   └── runc-1.3.4.amd64
├── systemd-private-9582513fa74f47a0abeae3d5a906ab59-chronyd.service-IDTYld
│   └── tmp
├── systemd-private-9582513fa74f47a0abeae3d5a906ab59-dbus-broker.service-5PBCgS
│   └── tmp
├── systemd-private-9582513fa74f47a0abeae3d5a906ab59-irqbalance.service-79Kfxz
│   └── tmp
├── systemd-private-9582513fa74f47a0abeae3d5a906ab59-polkit.service-02MeCp
│   └── tmp
├── systemd-private-9582513fa74f47a0abeae3d5a906ab59-systemd-logind.service-juOmt3
│   └── tmp
└── vagrant-shell
  • fact 확인 가능 more /tmp/k8s-ctr

PLAY [Prepare for etcd install] → TASK [adduser : User | Create User/Group]

# kube, etcd 생성 
root@k8s-ctr:~/kubespray# cat /etc/passwd | tail -n 3
vboxadd:x:991:1::/var/run/vboxadd:/bin/false
kube:x:990:988:Kubernetes user:/home/kube:/sbin/nologin
etcd:x:989:987:Etcd user:/home/etcd:/sbin/nologin

root@k8s-ctr:~/kubespray# cat /etc/group | tail -n 3
vboxdrmipc:x:989:
kube-cert:x:988:
etcd:x:987:

root@k8s-ctr:~/kubespray# find / -user etcd 2>/dev/null
/etc/ssl/etcd
/etc/ssl/etcd/ssl
/etc/ssl/etcd/ssl/admin-k8s-ctr-key.pem
/etc/ssl/etcd/ssl/admin-k8s-ctr.pem
/etc/ssl/etcd/ssl/ca-key.pem
/etc/ssl/etcd/ssl/ca.pem
/etc/ssl/etcd/ssl/member-k8s-ctr-key.pem
/etc/ssl/etcd/ssl/member-k8s-ctr.pem
/etc/ssl/etcd/ssl/node-k8s-ctr-key.pem
/etc/ssl/etcd/ssl/node-k8s-ctr.pem

sysctl 관련 작업

root@k8s-ctr:~/kubespray# grep "^[^#]" /etc/sysctl.conf
net.ipv4.ip_forward=1
kernel.keys.root_maxbytes=25000000
kernel.keys.root_maxkeys=1000000
kernel.panic=10
kernel.panic_on_oops=1
vm.overcommit_memory=1
vm.panic_on_oom=0
net.ipv4.ip_local_reserved_ports=30000-32767
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-arptables=1
net.bridge.bridge-nf-call-ip6tables=1

kubernetes/preinstall

roles/kubernetes/preinstall/tasks/
├── 0010-swapoff.yml
├── 0020-set_facts.yml
├── 0040-verify-settings.yml
├── 0050-create_directories.yml
├── 0060-resolvconf.yml
├── 0061-systemd-resolved.yml
├── 0062-networkmanager-unmanaged-devices.yml
├── 0063-networkmanager-dns.yml
├── 0080-system-configurations.yml
├── 0081-ntp-configurations.yml
├── 0100-dhclient-hooks.yml
├── 0110-dhclient-hooks-undo.yml
└── main.yml

**cat roles/kubernetes/preinstall/defaults/main.yml**

# Set to true to allow pre-checks to fail and continue deployment
ignore_assert_errors: false
# Set to false to disable the backup parameter, set to true to accumulate backups of config files.
leave_etc_backup_files: true
nameservers: []
cloud_resolver: []
disable_host_nameservers: false
# Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf
dns_late: false

# Set to true if your network does not support IPv6
# This may be necessary for pulling Docker images from
# GCE docker repository
disable_ipv6_dns: false

# Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``).
remove_default_searchdomains: false

kube_owner: kube
kube_cert_group: kube-cert
kube_config_dir: /etc/kubernetes
kube_cert_dir: "{{ kube_config_dir }}/ssl"
kube_cert_compat_dir: /etc/kubernetes/pki
kubelet_flexvolumes_plugins_dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec

# Flatcar Container Linux by Kinvolk cloud init config file to define /etc/resolv.conf content
# for hostnet pods and infra needs
resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf

# sysctl_file_path to add sysctl conf to
sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"

# Minimal memory requirement in MB for safety checks
minimal_node_memory_mb: 1024
minimal_master_memory_mb: 1500

## NTP Settings

# Manage the NTP configuration file.
ntp_manage_config: false
# Specify the NTP servers
# Only takes effect when ntp_manage_config is true.
ntp_servers:
  - "0.pool.ntp.org iburst"
  - "1.pool.ntp.org iburst"
  - "2.pool.ntp.org iburst"
  - "3.pool.ntp.org iburst"
# Restrict NTP access to these hosts.
# Only takes effect when ntp_manage_config is true.
ntp_restrict:
  - "127.0.0.1"
  - "::1"
# Specify whether to filter interfaces
ntp_filter_interface: false
# Specify the interfaces
# Only takes effect when ntp_filter_interface is true
# ntp_interfaces:
#   - ignore wildcard
#   - listen xxx
# The NTP driftfile path
# Only takes effect when ntp_manage_config is true.
# Default value is `/var/lib/ntp/ntp.drift`, for ntpsec use '/var/lib/ntpsec/ntp.drift'
ntp_driftfile: >-
      {% if ntp_package == "ntpsec" -%}
      /var/lib/ntpsec/ntp.drift
      {%- else -%}
      /var/lib/ntp/ntp.drift
      {%- endif -%}
# Only takes effect when ntp_manage_config is true.
ntp_tinker_panic: false

# Force sync time immediately after the ntp installed, which is useful in a newly installed system.
ntp_force_sync_immediately: false

# Set the timezone for your server.  eg: "Etc/UTC","Etc/GMT-8". If not set, the timezone will not change.
ntp_timezone: ""

# Currently known os distributions
# 제공하는 os 확인 가능 
supported_os_distributions:
  - 'RedHat'
  - 'CentOS'
  - 'Fedora'
  - 'Ubuntu'
  - 'Debian'
  - 'Flatcar'
  - 'Flatcar Container Linux by Kinvolk'
  - 'Suse'
  - 'openSUSE Leap'
  - 'openSUSE Tumbleweed'
  - 'ClearLinux'
  - 'OracleLinux'
  - 'AlmaLinux'
  - 'Rocky'
  - 'Amazon'
  - 'Kylin Linux Advanced Server'
  - 'UnionTech'
  - 'UniontechOS'
  - 'openEuler'

# Extending some distributions into the redhat os family
redhat_os_family_extensions:
  - "UnionTech"
  - "UniontechOS"

# Sets DNSStubListener=no, useful if you get "0.0.0.0:53: bind: address already in use"
systemd_resolved_disable_stub_listener: "{{ ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk'] }}"

# Used to disable File Access Policy Daemon service.
# If service is enabled, the CNI plugin installation will fail
disable_fapolicyd: true
  • kubelet이 안정적으로 기동되고, kubeadm이 실패하지 않도록 OS 상태를 Kubernetes 친화적으로 만드는 단계

TASK [kubernetes/preinstall : Check if host has NetworkManager , Check

root@k8s-ctr:~/kubespray# nmcli device status
DEVICE        TYPE      STATE                   CONNECTION 
enp0s3        ethernet  connected               enp0s3
enp0s8        ethernet  connected               enp0s8
cni0          bridge    connected (externally)  cni0
lo            loopback  connected (externally)  lo
flannel.1     vxlan     connected (externally)  flannel.1
veth04a8a213  ethernet  unmanaged               --
veth26a27da8  ethernet  unmanaged               --
veth4ad74ac5  ethernet  unmanaged               --
vethb04f55ee  ethernet  unmanaged               --
vethc6f2f2c5  ethernet  unmanaged               --
vethca7489fd  ethernet  unmanaged               --
root@k8s-ctr:~/kubespray# nmcli connection show
NAME       UUID                                  TYPE      DEVICE    
enp0s3     74d64498-c95a-435b-9340-81b2e4a7d1f2  ethernet  enp0s3
enp0s8     551e548c-cf41-4a28-96b9-8f4d95fd25de  ethernet  enp0s8
cni0       22fb3157-8d39-4456-b03a-9f6f3517c439  bridge    cni0
flannel.1  2e96d003-77ec-4636-af55-1bd632727aa6  vxlan     flannel.1
lo         f5b8330c-64e6-4878-8114-7e04dd2c1a19  loopback  lo

TASK [kubernetes/preinstall : 관련 디렉터리 생성] : 각각 kube , root 유저

cat roles/kubernetes/preinstall/tasks/0050-create_directories.yml
---
- name: Create kubernetes directories
  file:
    path: "{{ item }}"
    state: directory
    owner: "{{ kube_owner }}"
    mode: "0755"
  when: ('k8s_cluster' in group_names)
  become: true
  tags:
    - kubelet
    - kube-controller-manager
    - kube-apiserver
    - bootstrap_os
    - apps
    - network
    - control-plane
    - node
  with_items:
    - "{{ kube_config_dir }}"
    - "{{ kube_manifest_dir }}"
    - "{{ kube_script_dir }}"
    - "{{ kubelet_flexvolumes_plugins_dir }}"

- name: Create other directories of root owner
  file:
    path: "{{ item }}"
    state: directory
    owner: root
    mode: "0755"
  when: ('k8s_cluster' in group_names)
  become: true
  tags:
    - kubelet
    - kube-controller-manager
    - kube-apiserver
    - bootstrap_os
    - apps
    - network
    - control-plane
    - node
  with_items:
    - "{{ kube_cert_dir }}"
    - "{{ bin_dir }}"

- name: Check if kubernetes kubeadm compat cert dir exists
  stat:
    path: "{{ kube_cert_compat_dir }}"
    get_attributes: false
    get_checksum: false
    get_mime: false
  register: kube_cert_compat_dir_check
  when:
    - ('k8s_cluster' in group_names)
    - kube_cert_dir != kube_cert_compat_dir

- name: Create kubernetes kubeadm compat cert dir (kubernetes/kubeadm issue 1498)
  file:
    src: "{{ kube_cert_dir }}"
    dest: "{{ kube_cert_compat_dir }}"
    state: link
    mode: "0755"
  when:
    - ('k8s_cluster' in group_names)
    - kube_cert_dir != kube_cert_compat_dir
    - not kube_cert_compat_dir_check.stat.exists

- name: Create cni directories
  file:
    path: "{{ item }}"
    state: directory
    owner: "{{ kube_owner }}"
    mode: "0755"
  with_items:
    - "/etc/cni/net.d"
    - "/opt/cni/bin"
  when:
    - kube_network_plugin in ["calico", "flannel", "cilium", "kube-ovn", "kube-router", "macvlan"]
    - ('k8s_cluster' in group_names)
  tags:
    - network
    - cilium
    - calico
    - kube-ovn
    - kube-router
    - bootstrap_os

- name: Create calico cni directories
  file:
    path: "{{ item }}"
    state: directory
    owner: "{{ kube_owner }}"
    mode: "0755"
  with_items:
    - "/var/lib/calico"
  when:
    - kube_network_plugin == "calico"
    - ('k8s_cluster' in group_names)
  tags:
    - network
    - calico
    - bootstrap_os

- name: Create local volume provisioner directories
  file:
    path: "{{ local_volume_provisioner_storage_classes[item].host_dir }}"
    state: directory
    owner: root
    group: root
    mode: "{{ local_volume_provisioner_directory_mode }}"
  with_items: "{{ local_volume_provisioner_storage_classes.keys() | list }}"
  when:
    - ('k8s_cluster' in group_names)
    - local_volume_provisioner_enabled
  tags:
    - persistent_volumes

RUNNING HANDLER [kubernetes/preinstall : Preinstall | reload NetworkManager]

---
- name: Preinstall | apply resolvconf cloud-init
  command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }}
  when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
  listen: Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk

- name: Preinstall | reload NetworkManager
  service:
    name: NetworkManager.service
    state: restarted
  listen: Preinstall | update resolvconf for networkmanager

- name: Preinstall | reload kubelet
  service:
    name: kubelet
    state: restarted
  notify:
    - Preinstall | kube-controller configured
    - Preinstall | kube-apiserver configured
    - Preinstall | restart kube-controller-manager docker
    - Preinstall | restart kube-controller-manager crio/containerd
    - Preinstall | restart kube-apiserver docker
    - Preinstall | restart kube-apiserver crio/containerd
  when: not dns_early | bool
  listen:
    - Preinstall | propagate resolvconf to k8s components
    - Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk
    - Preinstall | update resolvconf for networkmanager

# FIXME(mattymo): Also restart for kubeadm mode
- name: Preinstall | kube-apiserver configured
  stat:
    path: "{{ kube_manifest_dir }}/kube-apiserver.yaml"
    get_attributes: false
    get_checksum: false
    get_mime: false
  register: kube_apiserver_set
  when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'
  listen: Preinstall | propagate resolvconf to k8s components

# FIXME(mattymo): Also restart for kubeadm mode
- name: Preinstall | kube-controller configured
  stat:
    path: "{{ kube_manifest_dir }}/kube-controller-manager.yaml"
    get_attributes: false
    get_checksum: false
    get_mime: false
  register: kube_controller_set
  when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'
  listen: Preinstall | propagate resolvconf to k8s components

- name: Preinstall | restart kube-controller-manager docker
  shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f"
  args:
    executable: /bin/bash
  when:
    - container_manager == "docker"
    - ('kube_control_plane' in group_names)
    - dns_mode != 'none'
    - resolvconf_mode == 'host_resolvconf'
    - kube_controller_set.stat.exists
  listen: Preinstall | propagate resolvconf to k8s components

- name: Preinstall | restart kube-controller-manager crio/containerd
  shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-controller-manager* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
  args:
    executable: /bin/bash
  register: preinstall_restart_controller_manager
  retries: 10
  delay: 1
  until: preinstall_restart_controller_manager.rc == 0
  when:
    - container_manager in ['crio', 'containerd']
    - ('kube_control_plane' in group_names)
    - dns_mode != 'none'
    - resolvconf_mode == 'host_resolvconf'
    - kube_controller_set.stat.exists
  listen: Preinstall | propagate resolvconf to k8s components

- name: Preinstall | restart kube-apiserver docker
  shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-apiserver* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f"     
  args:
    executable: /bin/bash
  when:
    - container_manager == "docker"
    - ('kube_control_plane' in group_names)
    - dns_mode != 'none'
    - resolvconf_mode == 'host_resolvconf'
    - kube_apiserver_set.stat.exists
  listen: Preinstall | propagate resolvconf to k8s components

- name: Preinstall | restart kube-apiserver crio/containerd
  shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-apiserver* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
  args:
    executable: /bin/bash
  register: preinstall_restart_apiserver
  retries: 10
  until: preinstall_restart_apiserver.rc == 0
  delay: 1
  when:
    - container_manager in ['crio', 'containerd']
    - ('kube_control_plane' in group_names)
    - dns_mode != 'none'
    - resolvconf_mode == 'host_resolvconf'
    - kube_apiserver_set.stat.exists
  listen: Preinstall | propagate resolvconf to k8s components

# When running this as the last phase ensure we wait for kube-apiserver to come up
- name: Preinstall | wait for the apiserver to be running
  uri:
    url: "{{ kube_apiserver_endpoint }}/healthz"
    validate_certs: false
  register: result
  until: result.status == 200
  retries: 60
  delay: 1
  when:
    - dns_late
    - ('kube_control_plane' in group_names)
    - dns_mode != 'none'
    - resolvconf_mode == 'host_resolvconf'
    - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and not is_fedora_coreos
  listen: Preinstall | propagate resolvconf to k8s components

- name: Preinstall | Restart systemd-resolved
  service:
    name: systemd-resolved
    state: restarted

- name: Preinstall | restart ntp
  service:
    name: "{{ ntp_service_name }}"
    state: restarted
  when: ntp_enabled

role container-engine 컨테이너 런타임(containerd/crio/docker) + runc + CNI 실행 환경을 설치/검증

roles/container-engine/
├── containerd
│   ├── defaults
│   ├── handlers
│   ├── meta
│   ├── molecule
│   ├── tasks
│   └── templates
├── containerd-common
│   ├── defaults
│   ├── meta
│   ├── tasks
│   └── vars
├── crictl
│   ├── handlers
│   ├── tasks
│   └── templates
├── cri-dockerd
│   ├── defaults
│   ├── handlers
│   ├── meta
│   ├── molecule
│   ├── tasks
│   └── templates
├── cri-o
│   ├── defaults
│   ├── handlers
│   ├── meta
│   ├── molecule
│   ├── tasks
│   ├── templates
│   └── vars
├── crun
│   └── tasks
├── docker
│   ├── defaults
│   ├── files
│   ├── handlers
│   ├── meta
│   ├── tasks
│   ├── templates
│   └── vars
├── gvisor
│   ├── molecule
│   └── tasks
├── kata-containers
│   ├── defaults
│   ├── molecule
│   ├── tasks
│   └── templates
├── meta
│   └── main.yml
├── molecule
│   ├── files
│   ├── prepare.yml
│   ├── templates
│   ├── test_cri.yml
│   └── test_runtime.yml
├── nerdctl
│   ├── handlers
│   ├── tasks
│   └── templates
├── runc
│   ├── defaults
│   └── tasks
├── skopeo
│   └── tasks
├── validate-container-engine
│   └── tasks
└── youki
    ├── defaults
    ├── molecule
    └── tasks

69 directories, 4 files

기본 순서

container-engine
 ├─ **validate-container-engine**
 ├─ **containerd**
 │   ├─ download containerd
 │   ├─ config.toml 생성
 │   ├─ systemd 등록
 │   └─ 기동
 ├─ **runc**
 │   ├─ 다운로드
 │   ├─ 검증
 │   └─ 설치
 ├─ **nerdctl**
 ├─ **crictl**
 └─ **cleanup**

validate-container-engine

---
- name: Validate-container-engine | check if fedora coreos
  stat:
    path: /run/ostree-booted
    get_attributes: false
    get_checksum: false
    get_mime: false
  register: ostree
  tags:
    - facts

- name: Validate-container-engine | set is_ostree
  set_fact:
    is_ostree: "{{ ostree.stat.exists }}"
  tags:
    - facts

- name: Ensure kubelet systemd unit exists
  stat:
    path: "/etc/systemd/system/kubelet.service"
  register: kubelet_systemd_unit_exists
  tags:
    - facts

- name: Populate service facts
  service_facts:
  tags:
    - facts

- name: Check if containerd is installed
  find:
    file_type: file
    recurse: true
    use_regex: true
    patterns:
      - containerd.service$
    paths:
      - /lib/systemd
      - /etc/systemd
      - /run/systemd
  register: containerd_installed
  tags:
    - facts

- name: Check if docker is installed
  find:
    file_type: file
    recurse: true
    use_regex: true
    patterns:
      - docker.service$
    paths:
      - /lib/systemd
      - /etc/systemd
      - /run/systemd
  register: docker_installed
  tags:
    - facts

- name: Check if crio is installed
  find:
    file_type: file
    recurse: true
    use_regex: true
    patterns:
      - crio.service$
    paths:
      - /lib/systemd
      - /etc/systemd
      - /run/systemd
  register: crio_installed
  tags:
    - facts

- name: Uninstall containerd
  vars:
    service_name: containerd.service
  when:
    - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar"))
    - container_manager != "containerd"
    - docker_installed.matched == 0
    - containerd_installed.matched > 0
    - ansible_facts.services[service_name]['state'] == 'running'
  block:
    - name: Drain node
      include_role:
        name: remove_node/pre_remove
        apply:
          tags:
            - pre-remove
      when: kubelet_systemd_unit_exists.stat.exists
    - name: Stop kubelet
      service:
        name: kubelet
        state: stopped
      when: kubelet_systemd_unit_exists.stat.exists
    - name: Remove Containerd
      import_role:
        name: container-engine/containerd
        tasks_from: reset
        handlers_from: reset

- name: Uninstall docker
  vars:
    service_name: docker.service
  when:
    - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar"))
    - container_manager != "docker"
    - docker_installed.matched > 0
    - ansible_facts.services[service_name]['state'] == 'running'
  block:
    - name: Drain node
      include_role:
        name: remove_node/pre_remove
        apply:
          tags:
            - pre-remove
      when: kubelet_systemd_unit_exists.stat.exists
    - name: Stop kubelet
      service:
        name: kubelet
        state: stopped
      when: kubelet_systemd_unit_exists.stat.exists
    - name: Remove Docker
      import_role:
        name: container-engine/docker
        tasks_from: reset

- name: Uninstall crio
  vars:
    service_name: crio.service
  when:
    - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar"))
    - container_manager != "crio"
    - crio_installed.matched > 0
    - ansible_facts.services[service_name]['state'] == 'running'
  block:
    - name: Drain node
      include_role:
        name: remove_node/pre_remove
        apply:
          tags:
            - pre-remove
      when: kubelet_systemd_unit_exists.stat.exists
    - name: Stop kubelet
      service:
        name: kubelet
        state: stopped
      when: kubelet_systemd_unit_exists.stat.exists
    - name: Remove CRI-O
      import_role:
        name: container-engine/cri-o
        tasks_from: reset

runc

---
- name: Runc | check if fedora coreos
  stat:
    path: /run/ostree-booted
    get_attributes: false
    get_checksum: false
    get_mime: false
  register: ostree

- name: Runc | set is_ostree
  set_fact:
    is_ostree: "{{ ostree.stat.exists }}"

- name: Runc | Uninstall runc package managed by package manager
  package:
    name: "{{ runc_package_name }}"
    state: absent
  when:
    - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar"))

- name: Runc | Download runc binary
  include_tasks: "../../../download/tasks/download_file.yml"
  vars:
    download: "{{ download_defaults | combine(downloads.runc) }}"

- name: Copy runc binary from download dir
  copy:
    src: "{{ downloads.runc.dest }}"
    dest: "{{ runc_bin_dir }}/runc"
    mode: "0755"
    remote_src: true

- name: Runc | Remove orphaned binary
  file:
    path: /usr/bin/runc
    state: absent
  when: runc_bin_dir != "/usr/bin"
  ignore_errors: true  # noqa ignore-errors

containerd

tree roles/container-engine/containerd/
roles/container-engine/containerd/
├── defaults
│   └── main.yml
├── handlers
│   ├── main.yml
│   └── reset.yml
├── meta
│   └── main.yml
├── molecule
│   └── default
│       ├── converge.yml
│       ├── molecule.yml
│       └── verify.yml
├── tasks
│   ├── main.yml
│   └── reset.yml
└── templates
    ├── config.toml.j2
    ├── config-v1.toml.j2
    ├── containerd.service.j2
    ├── hosts.toml.j2
    └── http-proxy.conf.j2

cat roles/container-engine/containerd/tasks/main.yml
---
- name: Containerd | Download containerd
  include_tasks: "../../../download/tasks/download_file.yml"
  vars:
    download: "{{ download_defaults | combine(downloads.containerd) }}"

- name: Containerd | Unpack containerd archive
  unarchive:
    src: "{{ downloads.containerd.dest }}"
    dest: "{{ containerd_bin_dir }}"
    mode: "0755"
    remote_src: true
    extra_opts:
      - --strip-components=1
  notify: Restart containerd

- name: Containerd | Generate systemd service for containerd
  template:
    src: containerd.service.j2
    dest: /etc/systemd/system/containerd.service
    mode: "0644"
    validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:containerd.service'"
    # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release)
    # Remove once we drop support for systemd < 250
  notify: Restart containerd

- name: Containerd | Ensure containerd directories exist
  file:
    dest: "{{ item }}"
    state: directory
    mode: "0755"
    owner: root
    group: root
  with_items:
    - "{{ containerd_systemd_dir }}"
    - "{{ containerd_cfg_dir }}"

- name: Containerd | Write containerd proxy drop-in
  template:
    src: http-proxy.conf.j2
    dest: "{{ containerd_systemd_dir }}/http-proxy.conf"
    mode: "0644"
  notify: Restart containerd
  when: http_proxy is defined or https_proxy is defined

- name: Containerd | Generate default base_runtime_spec
  register: ctr_oci_spec
  command: "{{ containerd_bin_dir }}/ctr oci spec"
  check_mode: false
  changed_when: false

- name: Containerd | Store generated default base_runtime_spec
  set_fact:
    containerd_default_base_runtime_spec: "{{ ctr_oci_spec.stdout | from_json }}"

- name: Containerd | Write base_runtime_specs
  copy:
    content: "{{ item.value }}"
    dest: "{{ containerd_cfg_dir }}/{{ item.key }}"
    owner: "root"
    mode: "0644"
  with_dict: "{{ containerd_base_runtime_specs | default({}) }}"
  notify: Restart containerd

- name: Containerd | Copy containerd config file
  template:
    src: "{{ 'config.toml.j2' if containerd_version is version('2.0.0', '>=') else 'config-v1.toml.j2' }}"
    dest: "{{ containerd_cfg_dir }}/config.toml"
    owner: "root"
    mode: "0640"
  notify: Restart containerd

- name: Containerd | Configure containerd registries
  # mirror configuration can contain sensitive information on headers configuration
  no_log: "{{ not (unsafe_show_logs | bool) }}"
  block:
    - name: Containerd | Create registry directories
      file:
        path: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}"
        state: directory
        mode: "0755"
      loop: "{{ containerd_registries_mirrors }}"
    - name: Containerd | Write hosts.toml file
      template:
        src: hosts.toml.j2
        dest: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}/hosts.toml"
        mode: "0640"
      loop: "{{ containerd_registries_mirrors }}"

# you can sometimes end up in a state where everything is installed
# but containerd was not started / enabled
- name: Containerd | Flush handlers
  meta: flush_handlers

- name: Containerd | Ensure containerd is started and enabled
  systemd_service:
    name: containerd
    daemon_reload: true
    enabled: true
    state: started

다운로드의 경우 공통적으로 download role 를 활용

tree roles/download/
roles/download/
├── meta
│   └── main.yml
├── tasks
│   ├── check_pull_required.yml
│   ├── download_container.yml
│   ├── download_file.yml
│   ├── extract_file.yml
│   ├── main.yml
│   ├── prep_download.yml
│   ├── prep_kubeadm_images.yml
│   └── set_container_facts.yml
└── templates
    └── kubeadm-images.yaml.j2

4 directories, 10 files

PLAY Install etcd

cat playbooks/install_etcd.yml
---
- name: Add worker nodes to the etcd play if needed
  hosts: kube_node
  roles:
    - { role: kubespray_defaults }
  tasks:
    - name: Check if nodes needs etcd client certs (depends on network_plugin)
      group_by:
        key: "_kubespray_needs_etcd"
      when:
        - kube_network_plugin in ["flannel", "canal", "cilium"] or
          (cilium_deploy_additionally | default(false)) or
          (kube_network_plugin == "calico" and calico_datastore == "etcd")
        - etcd_deployment_type != "kubeadm"
      tags: etcd

- name: Install etcd
  hosts: etcd:kube_control_plane:_kubespray_needs_etcd
  gather_facts: false
  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
  environment: "{{ proxy_disable_env }}"
  roles:
    - { role: kubespray_defaults }
    - role: etcd
      tags: etcd
      when: etcd_deployment_type != "kubeadm"

tree ~/kubespray/roles/etcd
/root/kubespray/roles/etcd
├── handlers
│   ├── backup_cleanup.yml
│   ├── backup.yml
│   └── main.yml
├── meta
│   └── main.yml
├── tasks
│   ├── check_certs.yml
│   ├── configure.yml
│   ├── gen_certs_script.yml
│   ├── gen_nodes_certs_script.yml
│   ├── install_docker.yml
│   ├── install_host.yml
│   ├── join_etcd-events_member.yml
│   ├── join_etcd_member.yml
│   ├── main.yml
│   ├── refresh_config.yml
│   └── upd_ca_trust.yml
└── templates
    ├── etcd-docker.service.j2
    ├── etcd.env.j2
    ├── etcd-events-docker.service.j2
    ├── etcd-events.env.j2
    ├── etcd-events-host.service.j2
    ├── etcd-events.j2
    ├── etcd-host.service.j2
    ├── etcd.j2
    ├── make-ssl-etcd.sh.j2
    └── openssl.conf.j2

cat /etc/systemd/system/etcd.service
[Unit]
Description=etcd
After=network.target

[Service]
Type=notify
User=root
EnvironmentFile=/etc/etcd.env
ExecStart=/usr/local/bin/etcd
NotifyAccess=all
Restart=always
RestartSec=10s
LimitNOFILE=40000

[Install]
WantedBy=multi-user.target

cat /etc/etcd.env 
# Environment file for etcd 3.5.25
ETCD_DATA_DIR=/var/lib/etcd
ETCD_ADVERTISE_CLIENT_URLS=https://192.168.10.10:2379
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://192.168.10.10:2380
ETCD_INITIAL_CLUSTER_STATE=existing
ETCD_METRICS=basic
ETCD_LISTEN_CLIENT_URLS=https://192.168.10.10:2379,https://127.0.0.1:2379
ETCD_ELECTION_TIMEOUT=5000
ETCD_HEARTBEAT_INTERVAL=250
ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
ETCD_LISTEN_PEER_URLS=https://192.168.10.10:2380
ETCD_NAME=etcd1
ETCD_PROXY=off
ETCD_INITIAL_CLUSTER=etcd1=https://192.168.10.10:2380
ETCD_AUTO_COMPACTION_RETENTION=8
ETCD_SNAPSHOT_COUNT=100000
ETCD_QUOTA_BACKEND_BYTES=2147483648
ETCD_MAX_REQUEST_BYTES=1572864
ETCD_LOG_LEVEL=info
ETCD_MAX_SNAPSHOTS=5
ETCD_MAX_WALS=5
# Flannel need etcd v2 API
ETCD_ENABLE_V2=true

# TLS settings
ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem
ETCD_CERT_FILE=/etc/ssl/etcd/ssl/member-k8s-ctr.pem
ETCD_KEY_FILE=/etc/ssl/etcd/ssl/member-k8s-ctr-key.pem
ETCD_CLIENT_CERT_AUTH=true

ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem
ETCD_PEER_CERT_FILE=/etc/ssl/etcd/ssl/member-k8s-ctr.pem
ETCD_PEER_KEY_FILE=/etc/ssl/etcd/ssl/member-k8s-ctr-key.pem
ETCD_PEER_CLIENT_CERT_AUTH=True

# CLI settings
ETCDCTL_ENDPOINTS=https://127.0.0.1:2379
ETCDCTL_CACERT=/etc/ssl/etcd/ssl/ca.pem
ETCDCTL_KEY=/etc/ssl/etcd/ssl/admin-k8s-ctr-key.pem
ETCDCTL_CERT=/etc/ssl/etcd/ssl/admin-k8s-ctr.pem

# ETCD 3.5.x issue
# https://groups.google.com/a/kubernetes.io/g/dev/c/B7gJs88XtQc/m/rSgNOzV2BwAJ?utm_medium=email&utm_source=footer
ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK=True

ETCD_EXPERIMENTAL_WATCH_PROGRESS_NOTIFY_INTERVAL=5s

# etcd 설치 후 핸들러에 의해 백업 수행 결과
tree /var/backups/
/var/backups/
└── etcd-2026-01-26_22:19:27
    ├── member
    │   ├── snap
    │   │   └── db
    │   └── wal
    │       └── 0000000000000000-0000000000000000.wal
    └── snapshot.db

5 directories, 3 files

root@k8s-ctr:~/kubespray# ss -tnlp | grep etcd
LISTEN 0      4096   192.168.10.10:2380       0.0.0.0:*    users:(("etcd",pid=26903,fd=6))
LISTEN 0      4096   192.168.10.10:2379       0.0.0.0:*    users:(("etcd",pid=26903,fd=8))
LISTEN 0      4096       127.0.0.1:2379       0.0.0.0:*    users:(("etcd",pid=26903,fd=7))
root@k8s-ctr:~/kubespray# etcdctl.sh member list -w table
+------------------+---------+-------+----------------------------+----------------------------+------------+
|        ID        | STATUS  | NAME  |         PEER ADDRS         |        CLIENT ADDRS        | IS LEARNER |
+------------------+---------+-------+----------------------------+----------------------------+------------+
| a997582217e26c7f | started | etcd1 | https://192.168.10.10:2380 | https://192.168.10.10:2379 |      false |
+------------------+---------+-------+----------------------------+----------------------------+------------+

etcdctl.sh 스크립트 확인

root@k8s-ctr:~/kubespray# cat /usr/local/bin/etcdctl.sh#!/bin/bash
# Ansible managed
# example invocation: etcdctl.sh get --keys-only --from-key ""

etcdctl \
  --cacert /etc/ssl/etcd/ssl/ca.pem \
  --cert /etc/ssl/etcd/ssl/admin-k8s-ctr.pem \
  --key /etc/ssl/etcd/ssl/admin-k8s-ctr-key.pem "$@"

root@k8s-ctr:~/kubespray# tree /etc/ssl/etcd
/etc/ssl/etcd
├── openssl.conf
└── ssl
    ├── admin-k8s-ctr-key.pem
    ├── admin-k8s-ctr.pem
    ├── ca-key.pem
    ├── ca.pem
    ├── member-k8s-ctr-key.pem
    ├── member-k8s-ctr.pem
    ├── node-k8s-ctr-key.pem
    └── node-k8s-ctr.pem

root@k8s-ctr:~/kubespray# cat /etc/ssl/etcd/openssl.conf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name

[req_distinguished_name]

[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names

[ ssl_client ]
extendedKeyUsage = clientAuth, serverAuth
basicConstraints = CA:FALSE
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
subjectAltName = @alt_names

[ v3_ca ]
basicConstraints = CA:TRUE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
authorityKeyIdentifier=keyid:always,issuer

[alt_names]
DNS.1 = localhost
DNS.2 = k8s-ctr
DNS.3 = lb-apiserver.kubernetes.local
DNS.4 = etcd.kube-system.svc.cluster.local
DNS.5 = etcd.kube-system.svc
DNS.6 = etcd.kube-system
DNS.7 = etcd
IP.1 = 192.168.10.10
IP.2 = 127.0.0.1
IP.3 = ::1

# 스크립트 실행 
**etcdctl.sh get --keys-only --from-key ""**
root@k8s-ctr:~/kubespray# etcdctl.sh endpoint status -w table
+----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|    ENDPOINT    |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| 127.0.0.1:2379 | a997582217e26c7f |  3.5.25 |  5.2 MB |      true |      false |         3 |      10285 |              10285 |        |
+----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
root@k8s-ctr:~/kubespray# etcdctl.sh member list -w table
+------------------+---------+-------+----------------------------+----------------------------+------------+
|        ID        | STATUS  | NAME  |         PEER ADDRS         |        CLIENT ADDRS        | IS LEARNER |
+------------------+---------+-------+----------------------------+----------------------------+------------+
| a997582217e26c7f | started | etcd1 | https://192.168.10.10:2380 | https://192.168.10.10:2379 |      false |
+------------------+---------+-------+----------------------------+----------------------------+------------+

PLAY [Install Kubernetes nodes]

root@k8s-ctr:~/kubespray# tree roles/kubernetes/node
roles/kubernetes/node
├── defaults
│   └── main.yml
├── handlers
│   └── main.yml
├── tasks
│   ├── facts.yml
│   ├── install.yml
│   ├── kubelet.yml
│   ├── loadbalancer
│   │   ├── haproxy.yml
│   │   ├── kube-vip.yml
│   │   └── nginx-proxy.yml
│   └── main.yml
├── templates
│   ├── http-proxy.conf.j2
│   ├── kubelet-config.v1beta1.yaml.j2
│   ├── kubelet.env.v1beta1.j2
│   ├── kubelet.service.j2
│   ├── loadbalancer
│   │   ├── haproxy.cfg.j2
│   │   └── nginx.conf.j2
│   ├── manifests
│   │   ├── haproxy.manifest.j2
│   │   ├── kube-vip.manifest.j2
│   │   └── nginx-proxy.manifest.j2
│   └── node-kubeconfig.yaml.j2
└── vars
    ├── fedora.yml
    ├── ubuntu-18.yml
    ├── ubuntu-20.yml
    ├── ubuntu-22.yml
    └── ubuntu-24.yml

root@k8s-ctr:~/kubespray# cat roles/kubernetes/node/tasks/main.yml
---
- name: Fetch facts
  import_tasks: facts.yml
  tags:
    - facts
    - kubelet

- name: Ensure /var/lib/cni exists
  file:
    path: /var/lib/cni
    state: directory
    mode: "0755"

- name: Install kubelet binary
  import_tasks: install.yml
  tags:
    - kubelet

- name: Install kube-vip
  import_tasks: loadbalancer/kube-vip.yml
  when:
    - ('kube_control_plane' in group_names)
    - kube_vip_enabled
  tags:
    - kube-vip

- name: Install nginx-proxy
  import_tasks: loadbalancer/nginx-proxy.yml
  when:
    - ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '::')
    - loadbalancer_apiserver_localhost
    - loadbalancer_apiserver_type == 'nginx'
  tags:
    - nginx

- name: Install haproxy
  import_tasks: loadbalancer/haproxy.yml
  when:
    - ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '::')
    - loadbalancer_apiserver_localhost
    - loadbalancer_apiserver_type == 'haproxy'
  tags:
    - haproxy

- name: Ensure nodePort range is reserved
  ansible.posix.sysctl:
    name: net.ipv4.ip_local_reserved_ports
    value: "{{ kube_apiserver_node_port_range }}"
    sysctl_set: true
    sysctl_file: "{{ sysctl_file_path }}"
    state: present
    reload: true
    ignoreerrors: "{{ sysctl_ignore_unknown_keys }}"
  when: kube_apiserver_node_port_range is defined
  tags:
    - kube-proxy

- name: Verify if br_netfilter module exists
  command: "modinfo br_netfilter"
  environment:
    PATH: "{{ ansible_env.PATH }}:/sbin"  # Make sure we can workaround RH's conservative path management
  register: modinfo_br_netfilter
  failed_when: modinfo_br_netfilter.rc not in [0, 1]
  changed_when: false
  check_mode: false

- name: Verify br_netfilter module path exists
  file:
    path: "{{ item }}"
    state: directory
    mode: "0755"
  loop:
    - /etc/modules-load.d
    - /etc/modprobe.d

- name: Enable br_netfilter module
  community.general.modprobe:
    name: br_netfilter
    state: present
  when: modinfo_br_netfilter.rc == 0

- name: Persist br_netfilter module
  copy:
    dest: /etc/modules-load.d/kubespray-br_netfilter.conf
    content: br_netfilter
    mode: "0644"
  when: modinfo_br_netfilter.rc == 0

# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled when found if br_netfilter is not a module
- name: Check if bridge-nf-call-iptables key exists
  command: "sysctl net.bridge.bridge-nf-call-iptables"
  failed_when: false
  changed_when: false
  check_mode: false
  register: sysctl_bridge_nf_call_iptables

- name: Enable bridge-nf-call tables
  ansible.posix.sysctl:
    name: "{{ item }}"
    state: present
    sysctl_file: "{{ sysctl_file_path }}"
    value: "1"
    reload: true
    ignoreerrors: "{{ sysctl_ignore_unknown_keys }}"
  when: sysctl_bridge_nf_call_iptables.rc == 0
  with_items:
    - net.bridge.bridge-nf-call-iptables
    - net.bridge.bridge-nf-call-arptables
    - net.bridge.bridge-nf-call-ip6tables

- name: Modprobe Kernel Module for IPVS
  community.general.modprobe:
    name: "{{ item }}"
    state: present
    persistent: present
  loop: "{{ kube_proxy_ipvs_modules }}"
  when: kube_proxy_mode == 'ipvs'
  tags:
    - kube-proxy

- name: Modprobe conntrack module
  community.general.modprobe:
    name: "{{ item }}"
    state: present
    persistent: present
  register: modprobe_conntrack_module
  ignore_errors: true  # noqa ignore-errors
  loop:
    - nf_conntrack
    - nf_conntrack_ipv4
  when:
    - kube_proxy_mode == 'ipvs'
    - modprobe_conntrack_module is not defined or modprobe_conntrack_module is ansible.builtin.failed  # loop until first success
  tags:
    - kube-proxy

- name: Modprobe Kernel Module for nftables
  community.general.modprobe:
    name: "nf_tables"
    state: present
    persistent: present
  when: kube_proxy_mode == 'nftables'
  tags:
    - kube-proxy

- name: Install kubelet
  import_tasks: kubelet.yml
  tags:
    - kubelet
    - kubeadm

설치된 정보 확인

root@k8s-ctr:~/kubespray# cat sysctl-1.txt | grep net.ipv4.ip_local_reserved_ports
net.ipv4.ip_local_reserved_ports =
root@k8s-ctr:~/kubespray# cat sysctl-2.txt | grep net.ipv4.ip_local_reserved_ports
net.ipv4.ip_local_reserved_ports = 30000-32767
root@k8s-ctr:~/kubespray# sysctl net.ipv4.ip_local_reserved_ports
net.ipv4.ip_local_reserved_ports = 30000-32767
root@k8s-ctr:~/kubespray# tree /etc/kubernetes/
/etc/kubernetes/
├── addons
│   ├── metrics_server
│   │   ├── auth-delegator.yaml
│   │   ├── auth-reader.yaml
│   │   ├── metrics-apiservice.yaml
│   │   ├── metrics-server-deployment.yaml
│   │   ├── metrics-server-sa.yaml
│   │   ├── metrics-server-service.yaml
│   │   ├── resource-reader-clusterrolebinding.yaml
│   │   └── resource-reader.yaml
│   └── node_feature_discovery
│       ├── nfd-api-crds.yaml
│       ├── nfd-clusterrolebinding.yaml
│       ├── nfd-clusterrole.yaml
│       ├── nfd-gc.yaml
│       ├── nfd-master-conf.yaml
│       ├── nfd-master.yaml
│       ├── nfd-ns.yaml
│       ├── nfd-rolebinding.yaml
│       ├── nfd-role.yaml
│       ├── nfd-serviceaccount.yaml
│       ├── nfd-service.yaml
│       ├── nfd-topologyupdater-conf.yaml
│       ├── nfd-worker-conf.yaml
│       └── nfd-worker.yaml
├── admin.conf
├── admin.conf.30181.2026-01-26@22:21:41~
├── cni-flannel-rbac.yml
├── cni-flannel.yml
├── controller-manager.conf
├── controller-manager.conf.30206.2026-01-26@22:21:42~
├── k8s-cluster-critical-pc.yml
├── kubeadm-config.yaml
├── kubeadm-images.yaml
├── kubelet.conf
├── kubelet.conf.30241.2026-01-26@22:21:43~
├── kubelet-config.yaml
├── kubelet.env
├── kubescheduler-config.yaml
├── manifests
│   ├── kube-apiserver.yaml
│   ├── kube-controller-manager.yaml
│   └── kube-scheduler.yaml
├── node-crb.yml
├── node-feature-discovery
│   ├── features.d
│   └── source.d
├── pki -> /etc/kubernetes/ssl
├── scheduler.conf
├── scheduler.conf.30281.2026-01-26@22:21:44~
├── ssl
│   ├── apiserver.crt
│   ├── apiserver.key
│   ├── apiserver-kubelet-client.crt
│   ├── apiserver-kubelet-client.key
│   ├── ca.crt
│   ├── ca.key
│   ├── front-proxy-ca.crt
│   ├── front-proxy-ca.key
│   ├── front-proxy-client.crt
│   ├── front-proxy-client.key
│   ├── sa.key
│   └── sa.pub
└── super-admin.conf

10 directories, 55 files

root@k8s-ctr:~/kubespray# kubectl describe node | grep pods
  pods:               110
  pods:               110

PLAY [Install the control plane] tags: control-plane , tags: client , tags: cluster-roles - kubeadm init , kubectl 접근 환경, rbac 설정

roles/kubernetes/control-plane/
├── defaults
│   └── main
│       ├── etcd.yml
│       ├── kube-proxy.yml
│       ├── kube-scheduler.yml
│       └── main.yml
├── handlers
│   └── main.yml
├── meta
│   └── main.yml
├── tasks
│   ├── check-api.yml
│   ├── define-first-kube-control.yml
│   ├── encrypt-at-rest.yml
│   ├── kubeadm-backup.yml
│   ├── kubeadm-etcd.yml
│   ├── kubeadm-fix-apiserver.yml
│   ├── kubeadm-secondary.yml
│   ├── kubeadm-setup.yml
│   ├── kubeadm-upgrade.yml
│   ├── kubelet-fix-client-cert-rotation.yml
│   ├── main.yml
│   └── pre-upgrade.yml
├── templates
│   ├── admission-controls.yaml.j2
│   ├── apiserver-audit-policy.yaml.j2
│   ├── apiserver-audit-webhook-config.yaml.j2
│   ├── apiserver-tracing.yaml.j2
│   ├── eventratelimit.yaml.j2
│   ├── k8s-certs-renew.service.j2
│   ├── k8s-certs-renew.sh.j2
│   ├── k8s-certs-renew.timer.j2
│   ├── kubeadm-config.v1beta3.yaml.j2
│   ├── kubeadm-config.v1beta4.yaml.j2
│   ├── kubeadm-controlplane.yaml.j2
│   ├── kubescheduler-config.yaml.j2
│   ├── podnodeselector.yaml.j2
│   ├── podsecurity.yaml.j2
│   ├── resourcequota.yaml.j2
│   ├── secrets_encryption.yaml.j2
│   ├── webhook-authorization-config.yaml.j2
│   └── webhook-token-auth-config.yaml.j2
└── vars
    └── main.yaml

8 directories, 37 files

kubernetes/client : kubectl 접근 환경, admin kubeconfig 생성

root@k8s-ctr:~/kubespray# tree roles/kubernetes/client/
roles/kubernetes/client/
├── defaults
│   └── main.yml
└── tasks
    └── main.yml

3 directories, 2 files
root@k8s-ctr:~/kubespray# cat roles/kubernetes/client/defaults/main.yml 
---
kubeconfig_localhost: false
kubeconfig_localhost_ansible_host: false
kubectl_localhost: false
artifacts_dir: "{{ inventory_dir }}/artifacts"

kube_config_dir: "/etc/kubernetes"
kube_apiserver_port: "6443"
root@k8s-ctr:~/kubespray# cat roles/kubernetes/client/tasks/main.yml
---
- name: Set external kube-apiserver endpoint
  set_fact:
    # noqa: jinja[spacing]
    external_apiserver_address: >-
      {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined -%}
      {{ loadbalancer_apiserver.address }}
      {%- elif kubeconfig_localhost_ansible_host is defined and kubeconfig_localhost_ansible_host -%}
      {{ hostvars[groups['kube_control_plane'][0]].ansible_host }}
      {%- else -%}
      {{ kube_apiserver_access_address }}
      {%- endif -%}
    # noqa: jinja[spacing]
    external_apiserver_port: >-
      {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined and loadbalancer_apiserver.port is defined -%}
      {{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
      {%- else -%}
      {{ kube_apiserver_port }}
      {%- endif -%}
  tags:
    - facts

- name: Create kube config dir for current/ansible become user
  file:
    path: "{{ ansible_env.HOME | default('/root') }}/.kube"
    mode: "0700"
    state: directory

- name: Copy admin kubeconfig to current/ansible become user home
  copy:
    src: "{{ kube_config_dir }}/admin.conf"
    dest: "{{ ansible_env.HOME | default('/root') }}/.kube/config"
    remote_src: true
    mode: "0600"
    backup: true

- name: Create kube artifacts dir
  file:
    path: "{{ artifacts_dir }}"
    mode: "0750"
    state: directory
  delegate_to: localhost
  connection: local
  become: false
  run_once: true
  when: kubeconfig_localhost

- name: Wait for k8s apiserver
  wait_for:
    host: "{{ kube_apiserver_access_address }}"
    port: "{{ kube_apiserver_port }}"
    timeout: 180

- name: Get admin kubeconfig from remote host
  slurp:
    src: "{{ kube_config_dir }}/admin.conf"
  run_once: true
  register: raw_admin_kubeconfig
  when: kubeconfig_localhost

- name: Convert kubeconfig to YAML
  set_fact:
    admin_kubeconfig: "{{ raw_admin_kubeconfig.content | b64decode | from_yaml }}"
  when: kubeconfig_localhost

- name: Override username in kubeconfig
  set_fact:
    final_admin_kubeconfig: "{{ admin_kubeconfig | combine(override_cluster_name, recursive=true) | combine(override_context, recursive=true) | combine(override_user, recursive=true) }}"
  vars:
    cluster_infos: "{{ admin_kubeconfig['clusters'][0]['cluster'] }}"
    user_certs: "{{ admin_kubeconfig['users'][0]['user'] }}"
    username: "kubernetes-admin-{{ cluster_name }}"
    context: "kubernetes-admin-{{ cluster_name }}@{{ cluster_name }}"
    override_cluster_name: "{{ {'clusters': [{'cluster': (cluster_infos | combine({'server': 'https://' + (external_apiserver_address | ansible.utils.ipwrap) + ':' + (external_apiserver_port | string)})), 'name': cluster_name}]} }}"
    override_context: "{{ {'contexts': [{'context': {'user': username, 'cluster': cluster_name}, 'name': context}], 'current-context': context} }}"
    override_user: "{{ {'users': [{'name': username, 'user': user_certs}]} }}"
  when: kubeconfig_localhost

- name: Write admin kubeconfig on ansible host
  copy:
    content: "{{ final_admin_kubeconfig | to_nice_yaml(indent=2) }}"
    dest: "{{ artifacts_dir }}/admin.conf"
    mode: "0600"
  delegate_to: localhost
  connection: local
  become: false
  run_once: true
  when: kubeconfig_localhost

- name: Copy kubectl binary to ansible host
  fetch:
    src: "{{ bin_dir }}/kubectl"
    dest: "{{ artifacts_dir }}/kubectl"
    flat: true
    validate_checksum: false
  register: copy_binary_result
  until: copy_binary_result is not failed
  retries: 20
  become: false
  run_once: true
  when: kubectl_localhost

- name: Create helper script kubectl.sh on ansible host
  copy:
    content: |
      #!/bin/bash
      ${BASH_SOURCE%/*}/kubectl --kubeconfig=${BASH_SOURCE%/*}/admin.conf "$@"
    dest: "{{ artifacts_dir }}/kubectl.sh"
    mode: "0755"
  become: false
  run_once: true
  delegate_to: localhost
  connection: local
  when: kubectl_localhost and kubeconfig_localhost

kubernetes-apps/cluster_roles

root@k8s-ctr:~/kubespray# tree roles/kubernetes-apps/cluster_roles/
roles/kubernetes-apps/cluster_roles/
├── files
│   └── k8s-cluster-critical-pc.yml
├── tasks
│   └── main.yml
└── templates
    ├── namespace.j2
    ├── node-crb.yml.j2
    └── vsphere-rbac.yml.j2

4 directories, 5 files
root@k8s-ctr:~/kubespray# cat roles/kubernetes-apps/cluster_roles/tasks/main.yml
---
- name: Kubernetes Apps | Wait for kube-apiserver
  uri:
    url: "{{ kube_apiserver_endpoint }}/healthz"
    validate_certs: false
    client_cert: "{{ kube_apiserver_client_cert }}"
    client_key: "{{ kube_apiserver_client_key }}"
  register: result
  until: result.status == 200
  retries: 10
  delay: 6
  when: inventory_hostname == groups['kube_control_plane'][0]

- name: Kubernetes Apps | Add ClusterRoleBinding to admit nodes
  template:
    src: "node-crb.yml.j2"
    dest: "{{ kube_config_dir }}/node-crb.yml"
    mode: "0640"
  register: node_crb_manifest
  when:
    - rbac_enabled
    - inventory_hostname == groups['kube_control_plane'][0]

- name: Apply workaround to allow all nodes with cert O=system:nodes to register
  kube:
    name: "kubespray:system:node"
    kubectl: "{{ bin_dir }}/kubectl"
    resource: "clusterrolebinding"
    filename: "{{ kube_config_dir }}/node-crb.yml"
    state: latest
  register: result
  until: result is succeeded
  retries: 10
  delay: 6
  when:
    - rbac_enabled
    - node_crb_manifest.changed
    - inventory_hostname == groups['kube_control_plane'][0]

- name: Kubernetes Apps | Remove old webhook ClusterRole
  kube:
    name: "system:node-webhook"
    kubectl: "{{ bin_dir }}/kubectl"
    resource: "clusterrole"
    state: absent
  when:
    - rbac_enabled
    - inventory_hostname == groups['kube_control_plane'][0]
  tags: node-webhook

- name: Kubernetes Apps | Remove old webhook ClusterRoleBinding
  kube:
    name: "system:node-webhook"
    kubectl: "{{ bin_dir }}/kubectl"
    resource: "clusterrolebinding"
    state: absent
  when:
    - rbac_enabled
    - inventory_hostname == groups['kube_control_plane'][0]
  tags: node-webhook

- name: PriorityClass | Copy k8s-cluster-critical-pc.yml file
  copy:
    src: k8s-cluster-critical-pc.yml
    dest: "{{ kube_config_dir }}/k8s-cluster-critical-pc.yml"
    mode: "0640"
  when: inventory_hostname == groups['kube_control_plane'] | last

- name: PriorityClass | Create k8s-cluster-critical
  kube:
    name: k8s-cluster-critical
    kubectl: "{{ bin_dir }}/kubectl"
    resource: "PriorityClass"
    filename: "{{ kube_config_dir }}/k8s-cluster-critical-pc.yml"
    state: latest
  register: result
  until: result is succeeded
  retries: 10
  delay: 6
  when: inventory_hostname == groups['kube_control_plane'] | last
root@k8s-ctr:~/kubespray#

Invoke kubeadm and install a CNI → role: network_plugin, tags: network

root@k8s-ctr:~/kubespray# tree roles/network_plugin/ -L 1
roles/network_plugin/
├── calico
├── calico_defaults
├── cilium
├── cni
├── custom_cni
├── flannel
├── kube-ovn
├── kube-router
├── macvlan
├── meta
├── multus
└── ovn4nfv

13 directories, 0 files
root@k8s-ctr:~/kubespray# 
tree roles/network_plugin/cni/
roles/network_plugin/cni/
├── defaults
│   └── main.yml
└── tasks
    └── main.yml

3 directories, 2 files
root@k8s-ctr:~/kubespray# tree roles/network_plugin/flannel/
roles/network_plugin/flannel/
├── defaults
│   └── main.yml
├── meta
│   └── main.yml
├── tasks
│   ├── main.yml
│   └── reset.yml
└── templates
    ├── cni-flannel-rbac.yml.j2
    └── cni-flannel.yml.j2

5 directories, 6 files

PLAY [Install Kubernetes apps]

root@k8s-ctr:~/kubespray# tree roles/kubernetes-apps/external_provisioner/
roles/kubernetes-apps/external_provisioner/
├── local_path_provisioner
│   ├── defaults
│   │   └── main.yml
│   ├── tasks
│   │   └── main.yml
│   └── templates
│       ├── local-path-storage-clusterrolebinding.yml.j2
│       ├── local-path-storage-cm.yml.j2
│       ├── local-path-storage-cr.yml.j2
│       ├── local-path-storage-deployment.yml.j2
│       ├── local-path-storage-ns.yml.j2
│       ├── local-path-storage-sa.yml.j2
│       └── local-path-storage-sc.yml.j2
├── local_volume_provisioner
│   ├── defaults
│   │   └── main.yml
│   ├── tasks
│   │   ├── basedirs.yml
│   │   └── main.yml
│   └── templates
│       ├── local-volume-provisioner-clusterrolebinding.yml.j2
│       ├── local-volume-provisioner-clusterrole.yml.j2
│       ├── local-volume-provisioner-cm.yml.j2
│       ├── local-volume-provisioner-ds.yml.j2
│       ├── local-volume-provisioner-ns.yml.j2
│       ├── local-volume-provisioner-sa.yml.j2
│       └── local-volume-provisioner-sc.yml.j2
└── meta
    └── main.yml

10 directories, 20 files

root@k8s-ctr:~/kubespray# tree roles/kubernetes-apps/ -L 1
roles/kubernetes-apps/
├── ansible
├── argocd
├── cluster_roles
├── common_crds
├── container_engine_accelerator
├── container_runtimes
├── csi_driver
├── defaults
├── external_cloud_controller
├── external_provisioner
├── helm
├── ingress_controller
├── kubelet-csr-approver
├── meta
├── metallb
├── metrics_server
├── node_feature_discovery
├── persistent_volumes
├── policy_controller
├── registry
├── scheduler_plugins
├── snapshots
└── vars
profile
dev_suha
이전 포스트

k8s deploy 3주차-2

다음 포스트

k8s deploy 5주차 Kubespary HA & Upgrade

0개의 댓글