centos에서 kubeadm으로 k8s 설치하기

1. Pre-requisites

swapoff -a

vi /etc/fstab
UUID=8ac075e3-1124-4bb6-bef7-a6811bf8b870 /                       xfs     defaults        0 0
#/swapfile none swap defaults 0 0

setenforce 0
sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo   https://download.docker.com/linux/centos/docker-ce.repo
yum update -y && yum install -y   containerd.io-1.2.13   docker-ce-19.03.8   docker-ce-cli-19.03.8

mkdir /etc/docker
cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2",
  "storage-opts": [
    "overlay2.override_kernel_check=true"
  ]
}
EOF

mkdir -p /etc/systemd/system/docker.service.d
systemctl daemon-reload
systemctl restart docker

yum list installed | grep docker
containerd.io.x86_64                1.2.13-3.2.el7             @docker-ce-stable
docker-ce.x86_64                    3:19.03.8-3.el7            @docker-ce-stable
docker-ce-cli.x86_64                1:19.03.8-3.el7            @docker-ce-stable

cat > /etc/modules-load.d/containerd.conf <<EOF
overlay
br_netfilter
EOF

modprobe overlay
modprobe br_netfilter

cat > /etc/sysctl.d/99-kubernetes-cri.conf <<EOF
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

sysctl --system

mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml

vi /etc/containerd/config.toml
plugins.cri.systemd_cgroup = true
systemctl restart containerd

2. kubeadm init

cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-\$basearch
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
exclude=kubelet kubeadm kubectl
EOF

yum install -y kubelet-1.17.4 kubeadm-1.17.4 kubectl-1.17.4 --disableexcludes=kubernetes

--container-runtime=remote \
--container-runtime-endpoint={{ cri_socket }} \

K8s HA cluster를 구성하기 위해서는 kubeadm의 설정파일인 kubeadm-config.yaml파일을 환경에 맞게 먼저 작성한다.

$ vi /etc/kubernetes/kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta1
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.1.141
  bindPort: 6443
---
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.32.0.0/12
kubernetesVersion: 1.13.5
controlPlaneEndpoint: 192.168.1.141:6443
certificatesDir: /etc/kubernetes/pki

master1번 장비에서 아래 실행.

kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=all

나머지 mater node에서 cluster에 join하기 위해 아래 파일을 master1에서 나머지 master node로 복사함.

scp /etc/kubernetes/pki/ca.* root@192.168.1.142:/etc/kubernetes/pki
scp /etc/kubernetes/pki/sa.* root@192.168.1.142:/etc/kubernetes/pki
scp /etc/kubernetes/pki/front-proxy-ca* root@192.168.1.142:/etc/kubernetes/pki
scp /etc/kubernetes/pki/etcd/ca* root@192.168.1.142:/etc/kubernetes/pki/etcd
scp /etc/kubernetes/pki/ca.* root@192.168.1.143:/etc/kubernetes/pki
scp /etc/kubernetes/pki/sa.* root@192.168.1.143:/etc/kubernetes/pki
scp /etc/kubernetes/pki/front-proxy-ca* root@192.168.1.143:/etc/kubernetes/pki
scp /etc/kubernetes/pki/etcd/ca* root@192.168.1.143:/etc/kubernetes/pki/etcd

master1번에서 kubeadm init 명령의 결과로 출력되는 join 명령을 copy하여 나머지 master node에서 실행.

kubeadm join 192.168.1.141:6443 --token tfuzvi.pc63qr4u1q99o27m --discovery-token-ca-cert-hash sha256:08a81818812483b6432d689f20a068d021d900a810d4a98d77f246f5f84c22a5 --experimental-control-plane

master1번에서 kubeadm init 명령의 결과로 출력되는 join 명령을 copy하여 나머지 master node에서 실행.

kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"

kubeadm join 192.168.1.141:6443 --token tfuzvi.pc63qr4u1q99o27m --discovery-token-ca-cert-hash sha256:08a81818812483b6432d689f20a068d021d900a810d4a98d77f246f5f84c22a5 --ignore-preflight-errors=all

---

kubeadm alpha certs renew all
rm -f /etc/kubernetes/admin.conf /etc/kubernetes/controller-manager.conf /etc/kubernetes/kubelet.conf /etc/kubernetes/scheduler.conf
kubeadm init phase kubeconfig all

// control-plane을 재시작
systemctl stop kubelet docker
systemctl start docker kubelet


kubeadm token create --print-join-command

rm -f /etc/kubernetes/bootstrap-kubelet.conf /etc/kubernetes/kubelet.conf /etc/kubernetes/pki/ca.crt
rm -f /var/lib/kubelet/pki/*

kubeadm join 192.168.1.141:6443 --token iqiytm.dyjiivd9f4vp6f80 --discovery-token-ca-cert-hash sha256:08a81818812483b6432d689f20a068d021d900a810d4a98d77f246f5f84c22a5 --ignore-preflight-errors=all

3. Troubleshooting

• etcd 가 systemd가 아닌 pod로 기동될 경우 etcd pod가 정상기동하지 않으면 kube-apiserver는 crashloopback 발생하여 주기적으로 재시작된다.
• kube-apiserver에 log 관련 설정을 추가하여 생성되는 log를 확인한다.

$ vi /etc/kubernetes/manifests/kube-apiserver.yaml
apiVersion: v1
kind: Pod
metadata:
  annotations:
    scheduler.alpha.kubernetes.io/critical-pod: ""
  creationTimestamp: null
  labels:
    component: kube-apiserver
    tier: control-plane
  name: kube-apiserver
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-apiserver
    - --v=4
    - --logtostderr=false
    - --log-dir=/root/work/log
    - --authorization-mode=Node,RBAC
    - --advertise-address=192.168.1.141
    - --allow-privileged=true
    - --client-ca-file=/etc/kubernetes/pki/ca.crt
    - --enable-admission-plugins=NodeRestriction
    - --enable-bootstrap-token-auth=true
    - --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt
    - --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt
    - --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
    - --etcd-servers=https://127.0.0.1:2379
    - --insecure-port=0
    - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
    - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
    - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
    - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
    - --requestheader-allowed-names=front-proxy-client
    - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
    - --requestheader-extra-headers-prefix=X-Remote-Extra-
    - --requestheader-group-headers=X-Remote-Group
    - --requestheader-username-headers=X-Remote-User
    - --secure-port=6443
    - --service-account-key-file=/etc/kubernetes/pki/sa.pub
    - --service-cluster-ip-range=10.96.0.0/12
    - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
    - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
    image: k8s.gcr.io/kube-apiserver:v1.13.5
    imagePullPolicy: IfNotPresent
    livenessProbe:
      failureThreshold: 8
      httpGet:
        host: 192.168.1.141
        path: /healthz
        port: 6443
        scheme: HTTPS
      initialDelaySeconds: 15
      timeoutSeconds: 15
    name: kube-apiserver
    resources:
      requests:
        cpu: 250m
    volumeMounts:
    - mountPath: /etc/ssl/certs
      name: ca-certs
      readOnly: true
    - mountPath: /etc/pki
      name: etc-pki
      readOnly: true
    - mountPath: /etc/kubernetes/pki
      name: k8s-certs
      readOnly: true
    - mountPath: /root/work/log
      name: log
  hostNetwork: true
  priorityClassName: system-cluster-critical
  volumes:
  - hostPath:
      path: /etc/ssl/certs
      type: DirectoryOrCreate
    name: ca-certs
  - hostPath:
      path: /etc/pki
      type: DirectoryOrCreate
    name: etc-pki
  - hostPath:
      path: /etc/kubernetes/pki
      type: DirectoryOrCreate
    name: k8s-certs
  - hostPath:
      path: /root/work/log
      type: DirectoryOrCreate
    name: log
status: {}

• kubelet service에 log관련 설정을 추가하여 log확인

$ vi /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS="--log-dir=/root/work/log \
--logtostderr=false \
--v=2"

• etcd pod에 접속하여 health check

export ETCDCTL_API=3
etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt --key=/etc/kubernetes/pki/etcd/healthcheck-client.key --endpoints https://192.168.1.141:2379,https://192.168.1.143:2379,https://192.168.1.142:2379  endpoint health

etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt --key=/etc/kubernetes/pki/etcd/healthcheck-client.key --e
ndpoints https://192.168.1.141:2379,https://192.168.1.143:2379,https://192.168.1.142:2379 get --prefix / --keys-only | sed '/^\s*$/d'

/root$ kubeadm alpha certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'

CERTIFICATE                EXPIRES                  RESIDUAL TIME   CERTIFICATE AUTHORITY   EXTERNALLY MANAGED
admin.conf                 Mar 24, 2030 05:27 UTC   9y                                      no
apiserver                  Mar 24, 2030 05:27 UTC   9y              ca                      no
apiserver-kubelet-client   Mar 24, 2030 05:27 UTC   9y              ca                      no
controller-manager.conf    Mar 24, 2030 05:27 UTC   9y                                      no
front-proxy-client         Mar 24, 2030 05:27 UTC   9y              front-proxy-ca          no
scheduler.conf             Mar 24, 2030 05:27 UTC   9y                                      no

CERTIFICATE AUTHORITY   EXPIRES                  RESIDUAL TIME   EXTERNALLY MANAGED
ca                      Mar 24, 2030 05:26 UTC   9y              no
front-proxy-ca          Mar 24, 2030 05:26 UTC   9y              no