[Kubernetes] Kubernetes the Hard Way (w/o Installer)

gununoo·2022년 10월 11일
0

Kuberbetes

목록 보기
9/10
post-thumbnail
  • 인스턴스 준비
ubuntu@ip-172-31-1-65:~$ sudo hostnamectl set-hostname gunwoo-k8s-hardway-master-1
ubuntu@ip-172-31-1-65:~$ sudo -i 
root@gunwoo-k8s-hardway-master-1:~# cat << EOF >> /etc/hosts
> 127.0.0.1 localhost
> 172.31.1.65  gunwoo-k8s-hardway-master-1
> 172.31.7.230  gunwoo-k8s-hardway-master-2
> 172.31.15.197 gunwoo-k8s-hardway-master-3
> 172.31.3.25   gunwoo-k8s-hardway-worker-1
> 172.31.2.183  gunwoo-k8s-hardway-worker-2
> 172.31.11.91  gunwoo-k8s-hardway-worker-3
> EOF
root@gunwoo-k8s-hardway-master-1:~# cat /etc/hosts
127.0.0.1 localhost

# The following lines are desirable for IPv6 capable hosts
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts
127.0.0.1 localhost
172.31.1.65  gunwoo-k8s-hardway-master-1
172.31.7.230  gunwoo-k8s-hardway-master-2
172.31.15.197 gunwoo-k8s-hardway-master-3
172.31.3.25   gunwoo-k8s-hardway-worker-1
172.31.2.183  gunwoo-k8s-hardway-worker-2
172.31.11.91  gunwoo-k8s-hardway-worker-3
root@gunwoo-auth-basic:~# mkdir hardway
root@gunwoo-auth-basic:~# cd hardway/
root@gunwoo-auth-basic:~/hardway# mkdir cert 
root@gunwoo-auth-basic:~/hardway# cd cert/
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# openssl genrsa -out ca.key 2048
Generating RSA private key, 2048 bit long modulus (2 primes)
.................................................................................................................+++++
....................+++++
e is 65537 (0x010001)


root@gunwoo-auth-basic:~/hardway/cert# # Create CSR using the private key
root@gunwoo-auth-basic:~/hardway/cert# openssl req -new -key ca.key -subj "/CN=KUBERNETES-CA" -out ca.csr


root@gunwoo-auth-basic:~/hardway/cert# # Self sign the csr using its own private key
root@gunwoo-auth-basic:~/hardway/cert# openssl x509 -req -in ca.csr -signkey ca.key -CAcreateserial  -out ca.crt -days 3650
Signature ok
subject=CN = KUBERNETES-CA
Getting Private key
root@gunwoo-auth-basic:~/hardway/cert# # Generate private key for admin user
root@gunwoo-auth-basic:~/hardway/cert# openssl genrsa -out admin.key 2048
Generating RSA private key, 2048 bit long modulus (2 primes)
.......................+++++
................................................................................................................+++++
e is 65537 (0x010001)


root@gunwoo-auth-basic:~/hardway/cert# # Generate CSR for admin user. Note the OU.
root@gunwoo-auth-basic:~/hardway/cert# openssl req -new -key admin.key -subj "/CN=admin/O=system:masters" -out admin.csr


root@gunwoo-auth-basic:~/hardway/cert# # Sign certificate for admin user using CA servers private key
root@gunwoo-auth-basic:~/hardway/cert# openssl x509 -req -in admin.csr -CA ca.crt -CAkey ca.key -CAcreateserial  -out admin.crt -days 365
Signature ok
subject=CN = admin, O = system:masters
Getting CA Private Key
root@gunwoo-auth-basic:~/hardway/cert# # kube-controller-manager
root@gunwoo-auth-basic:~/hardway/cert# openssl genrsa -out kube-controller-manager.key 2048
Generating RSA private key, 2048 bit long modulus (2 primes)
..........+++++
................................................................................................+++++
e is 65537 (0x010001)
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# openssl req -new -key kube-controller-manager.key -subj "/CN=system:kube-controller-manager" -out kube-controller-manager.csr
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# openssl x509 -req -in kube-controller-manager.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out kube-controller-manager.crt -days 365
Signature ok
subject=CN = system:kube-controller-manager
Getting CA Private Key
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# # kube-proxy
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# openssl genrsa -out kube-proxy.key 2048
Generating RSA private key, 2048 bit long modulus (2 primes)
.............+++++
.....................................................................................................+++++
e is 65537 (0x010001)
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# openssl req -new -key kube-proxy.key -subj "/CN=system:kube-proxy" -out kube-proxy.csr
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# openssl x509 -req -in kube-proxy.csr -CA ca.crt -CAkey ca.key -CAcreateserial  -out kube-proxy.crt -days 365
Signature ok
subject=CN = system:kube-proxy
Getting CA Private Key
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# # kube-scheduler
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# openssl genrsa -out kube-scheduler.key 2048
Generating RSA private key, 2048 bit long modulus (2 primes)
................................................+++++
...................................+++++
e is 65537 (0x010001)
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# openssl req -new -key kube-scheduler.key -subj "/CN=system:kube-scheduler" -out kube-scheduler.csr
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# openssl x509 -req -in kube-scheduler.csr -CA ca.crt -CAkey ca.key -CAcreateserial  -out kube-scheduler.crt -days 365
Signature ok
subject=CN = system:kube-scheduler
Getting CA Private Key
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# ll
total 72
drwxr-xr-x 2 root root 4096 Oct 11 01:09 ./
drwxr-xr-x 3 root root 4096 Oct 11 01:06 ../
-rw-r--r-- 1 root root 1025 Oct 11 01:08 admin.crt
-rw-r--r-- 1 root root  920 Oct 11 01:08 admin.csr
-rw------- 1 root root 1679 Oct 11 01:08 admin.key
-rw-r--r-- 1 root root 1001 Oct 11 01:07 ca.crt
-rw-r--r-- 1 root root  895 Oct 11 01:07 ca.csr
-rw------- 1 root root 1679 Oct 11 01:07 ca.key
-rw-r--r-- 1 root root   41 Oct 11 01:09 ca.srl
-rw-r--r-- 1 root root 1025 Oct 11 01:09 kube-controller-manager.crt
-rw-r--r-- 1 root root  920 Oct 11 01:09 kube-controller-manager.csr
-rw------- 1 root root 1675 Oct 11 01:09 kube-controller-manager.key
-rw-r--r-- 1 root root 1009 Oct 11 01:09 kube-proxy.crt
-rw-r--r-- 1 root root  903 Oct 11 01:09 kube-proxy.csr
-rw------- 1 root root 1675 Oct 11 01:09 kube-proxy.key
-rw-r--r-- 1 root root 1013 Oct 11 01:09 kube-scheduler.crt
-rw-r--r-- 1 root root  907 Oct 11 01:09 kube-scheduler.csr
-rw------- 1 root root 1679 Oct 11 01:09 kube-scheduler.key

인증서 10개 생성됨 

root@gunwoo-auth-basic:~/hardway/cert# # 1번 마스터의 INTERNAL IP를 사용
root@gunwoo-auth-basic:~/hardway/cert# export KUBE_API_SERVER_ADDRESS=172.31.1.65
root@gunwoo-auth-basic:~/hardway/cert# export KUBE_CLUSTER_NAME=gunwoo-k8s-hardway
root@gunwoo-auth-basic:~/hardway/cert# # kubectl 바이너리 다운로드
root@gunwoo-auth-basic:~/hardway/cert# wget -q --show-progress --https-only --timestamping  "https://storage.googleapis.com/kubernetes-release/release/v1.19.16/bin/linux/amd64/kubectl"
kubectl                                   100%[=====================================================================================>]  40.96M  48.8MB/s    in 0.8s    
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# {
>   chmod +x kubectl
>   sudo mv kubectl /usr/local/bin/
> }

Client 인증서와 key를 이용해 kubeconfig를 생성

root@gunwoo-auth-basic:~/hardway/cert# {
>   kubectl config set-cluster ${KUBE_CLUSTER_NAME} \
>     --certificate-authority=ca.crt \
>     --embed-certs=true \
>     --server=https://${KUBE_API_SERVER_ADDRESS}:6443 \
>     --kubeconfig=kube-proxy.kubeconfig
>  
>   kubectl config set-credentials system:kube-proxy \
>     --client-certificate=kube-proxy.crt \
>     --client-key=kube-proxy.key \
>     --embed-certs=true \
>     --kubeconfig=kube-proxy.kubeconfig
>  
>   kubectl config set-context default \
>     --cluster=${KUBE_CLUSTER_NAME} \
>     --user=system:kube-proxy \
>     --kubeconfig=kube-proxy.kubeconfig
>  
>   kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
> }
Cluster "gunwoo-k8s-hardway" set.
User "system:kube-proxy" set.
Context "default" created.
Switched to context "default".
root@gunwoo-auth-basic:~/hardway/cert# {
>   kubectl config set-cluster ${KUBE_CLUSTER_NAME} \
>     --certificate-authority=ca.crt \
>     --embed-certs=true \
>     --server=https://${KUBE_API_SERVER_ADDRESS}:6443 \
>     --kubeconfig=kube-controller-manager.kubeconfig
>  
>   kubectl config set-credentials system:kube-controller-manager \
>     --client-certificate=kube-controller-manager.crt \
>     --client-key=kube-controller-manager.key \
>     --embed-certs=true \
>     --kubeconfig=kube-controller-manager.kubeconfig
>  
>   kubectl config set-context default \
>     --cluster=${KUBE_CLUSTER_NAME} \
>     --user=system:kube-controller-manager \
>     --kubeconfig=kube-controller-manager.kubeconfig
>  
>   kubectl config use-context default --kubeconfig=kube-controller-manager.kubeconfig
> }
Cluster "gunwoo-k8s-hardway" set.
User "system:kube-controller-manager" set.
Context "default" created.
Switched to context "default".
root@gunwoo-auth-basic:~/hardway/cert# {
>   kubectl config set-cluster ${KUBE_CLUSTER_NAME} \
>     --certificate-authority=ca.crt \
>     --embed-certs=true \
>     --server=https://${KUBE_API_SERVER_ADDRESS}:6443 \
>     --kubeconfig=kube-scheduler.kubeconfig
>  
>   kubectl config set-credentials system:kube-scheduler \
>     --client-certificate=kube-scheduler.crt \
>     --client-key=kube-scheduler.key \
>     --embed-certs=true \
>     --kubeconfig=kube-scheduler.kubeconfig
>  
>   kubectl config set-context default \
>     --cluster=${KUBE_CLUSTER_NAME} \
>     --user=system:kube-scheduler \
>     --kubeconfig=kube-scheduler.kubeconfig
>  
>   kubectl config use-context default --kubeconfig=kube-scheduler.kubeconfig
> }

Cluster "gunwoo-k8s-hardway" set.
User "system:kube-scheduler" set.
Context "default" created.
Switched to context "default".
root@gunwoo-auth-basic:~/hardway/cert# {
>   kubectl config set-cluster ${KUBE_CLUSTER_NAME} \
>     --certificate-authority=ca.crt \
>     --embed-certs=true \
>     --server=https://${KUBE_API_SERVER_ADDRESS}:6443 \
>     --kubeconfig=admin.kubeconfig
>  
>   kubectl config set-credentials admin \
>     --client-certificate=admin.crt \
>     --client-key=admin.key \
>     --embed-certs=true \
>     --kubeconfig=admin.kubeconfig
>  
>   kubectl config set-context default \
>     --cluster=${KUBE_CLUSTER_NAME} \
>     --user=admin \
>     --kubeconfig=admin.kubeconfig
>  
>   kubectl config use-context default --kubeconfig=admin.kubeconfig
> }
Cluster "gunwoo-k8s-hardway" set.
User "admin" set.
Context "default" created.
Switched to context "default".
root@gunwoo-auth-basic:~/hardway/cert# mkdir -p ~/.kube
root@gunwoo-auth-basic:~/hardway/cert# cp -rf admin.kubeconfig ~/.kube/config
  • SANs 생성 
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# cat > openssl-kube-apiserver.cnf <<EOF
> [req]
> req_extensions = v3_req
> distinguished_name = req_distinguished_name
> [req_distinguished_name]
> [ v3_req ]
> basicConstraints = CA:FALSE
> keyUsage = nonRepudiation, digitalSignature, keyEncipherment
> subjectAltName = @alt_names
> [alt_names]
> DNS.1 = kubernetes
> DNS.2 = kubernetes.default
> DNS.3 = kubernetes.default.svc
> DNS.4 = kubernetes.default.svc.cluster.local
> IP.1 = 10.231.0.1
> IP.2 = 172.31.1.65
> IP.3 = 172.31.7.230
> IP.4 = 172.31.15.197
> IP.5 = 127.0.0.1
> EOF

master 1, 2, 3의 IP를 넣음

  • API 인증서 
root@gunwoo-auth-basic:~/hardway/cert# openssl genrsa -out kube-apiserver.key 2048
Generating RSA private key, 2048 bit long modulus (2 primes)
..............................+++++
.............+++++
e is 65537 (0x010001)
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# openssl req -new -key kube-apiserver.key -subj "/CN=kube-apiserver" -out kube-apiserver.csr -config openssl-kube-apiserver.cnf
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# openssl x509 -req -in kube-apiserver.csr -CA ca.crt -CAkey ca.key -CAcreateserial  -out kube-apiserver.crt -extensions v3_req -extfile openssl-kube-apiserver.cnf -days 365
Signature ok
subject=CN = kube-apiserver
Getting CA Private Key
  • 생성된 인증서(kube-apiserver.crt) SANs 확인 
root@gunwoo-auth-basic:~/hardway/cert# openssl x509 -in kube-apiserver.crt -noout -text
  • etcd 인증서 생성 
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# cat > openssl-etcd.cnf <<EOF
> [req]
> req_extensions = v3_req
> distinguished_name = req_distinguished_name
> [req_distinguished_name]
> [ v3_req ]
> basicConstraints = CA:FALSE
> keyUsage = nonRepudiation, digitalSignature, keyEncipherment
> subjectAltName = @alt_names
> [alt_names]
> IP.1 = 172.31.1.65
> IP.2 = 172.31.7.230
> IP.3 = 172.31.15.197
> IP.4 = 127.0.0.1
> EOF
root@gunwoo-auth-basic:~/hardway/cert# openssl genrsa -out etcd-server.key 2048
Generating RSA private key, 2048 bit long modulus (2 primes)
........................................................+++++
......................................................................+++++
e is 65537 (0x010001)

root@gunwoo-auth-basic:~/hardway/cert# openssl req -new -key etcd-server.key -subj "/CN=etcd-server" -out etcd-server.csr -config openssl-etcd.cnf

root@gunwoo-auth-basic:~/hardway/cert# openssl x509 -req -in etcd-server.csr -CA ca.crt -CAkey ca.key -CAcreateserial  -out etcd-server.crt -extensions v3_req -extfile openssl-etcd.cnf -days 3650
Signature ok
subject=CN = etcd-server
Getting CA Private Key
  • 생성된 인증서(etcd-server.crt) SANs 확인 
root@gunwoo-auth-basic:~/hardway/cert# openssl x509 -in etcd-server.crt -noout -text
  • Service Account Key Pair 생성 
root@gunwoo-auth-basic:~/hardway/cert# openssl genrsa -out service-account.key 2048
Generating RSA private key, 2048 bit long modulus (2 primes)
......................+++++
..........+++++
e is 65537 (0x010001)
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# openssl req -new -key service-account.key -subj "/CN=service-accounts" -out service-account.csr
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# openssl x509 -req -in service-account.csr -CA ca.crt -CAkey ca.key -CAcreateserial  -out service-account.crt -days 1
Signature ok
subject=CN = service-accounts
Getting CA Private Key
root@gunwoo-auth-basic:~/hardway/cert# vi gunwoo.pem
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEAr2Yd6FIs9MboX9mvj6+mdc99HwF2XirPjzWj4yckbSywrUC9
MXBpLQxtl7RRYwW6Hwv0zhPmxLyL0dT4Z7gWiN8vULdFCvliFyuy6bkBRuOK2ZQt
5jdow0Nntu5PVsJE70H7ogUI62bPdAdVYlsbkx8LKxtYsEP27xdjy34+wnvWdvHv
s5sPDP/Y/gZPBOpX7misSeKMGqKSNFD1mBsC2x8tgj4af44xaYpa2AlO+QKQoQ0Z
SJnFKh3Rl1ENoWL9Do2KV7+xlaVheuwrZMxvhl43tA2RPD2U7GIPnYCGTazOipyW
vTeOz971C2otCYU89Oww+w14ekqsWWEkr87f2QIDAQABAoIBAFlYE+3uRfYdh+qj
MRlo6Mrj+SEdFpVbC0Uwlp2zp4txE0H9UyHm17xZs01uTXK6TRdZeMasPoWCYVpu
OGZk0B3CShC8eg/f/kY9PTJuyaa5a1Xvc/3rxZKJOEBnXcScyo5xNuNN20BFq3C3
95dVP1OOoZrAoEmRJSye5neWhBN+Nf4hyJNapc6+6PRyY0UhHwH1gW42/GDEh8h1
aURV8cPgQuKb4b48Nq5VYnkWCpGvO0ASYotlKyK8F3ElRCqtNdW0SS+L2U22VxRJ
YtM4/A0Ay1MqZiy50GZcy6dH1YxrlEfD5oeNlSaC1Zjz+bLVNxAbO32hDjYSjBsE
q29g8r0CgYEA7SYtbDto6B1nM0NQqtIlIPl4XrvOZMF0r/hJynGUfhYBmcAXx+7q
qUuQGPQKXkEulpN8mijocOJjlIu68zkE1r6akN/gnWOlKA9teQT3cJMPNs420tXe
5WxpluYPxE6ZRe8mIBAqWeXtcgQAGcKwvoXMYY3AHFgLUlvyK2AHixMCgYEAvVdd
RCBELf7ju5ETADOwBn2kdTA2UgC7CPIOMfBqFTGQdkAvOw9W3KivrKA5h+YAhiXY
j6TCjnJyhzyk+lCtOVyX+uNicVVg9RCqLT5Eo/xjruH52wSeX0Er0AHiG7lzzK22
as8pGX2EfpsXY0LpfcG8TFtUnZd8iadLS9M6+uMCgYEAyjN+EUWKijbf0ma39h5d
yALNrkCFl4UXgB68lB9J1EPhM34FCNTeQmEKSSxzVrBPD9ZLLIpqvi1ZaMPHMDDS
vmMgGRy6QJhArqqQ1dn1PIg3NayYNCZk+cr5MM+Sa/f0Whuxry75XVNdpJRinZRR
7PDaNeeOANzVMnDxSCiF5f8CgYAOa1vX91kZ46J2Q1seFdGe/OyMXCqW3iVgTgbU
uMlracXlq4etSMLFtTcDv4QCHHsHvFQcxGBotsCQGgc6ZPB8QNf2LGDv3p+uqiE7
ptVVY8A3rk/35Qcmm/D5O1t73wi5mrUct0AbB9sSRT9nuM3NXPa5He94vPOoDAyI
7sI8HwKBgD41tUNQz2l193OtoICRiZZfbyJpp/pWgUnQtHDW1z5oif7xVvFVhCgC
Udbjn4nq8dwF6uL0FwtkvLTK0+eIaxFSFW6Bm6z2LwSr60jUMKDqY+55A/i2ZKVF
kVGRts8zxH9KkO/NLeVOQJTpssqu1xn+A+WOwfd++B1E8S1gfc2q
-----END RSA PRIVATE KEY-----
root@gunwoo-auth-basic:~/hardway/cert# chmod 400 gunwoo.pem 
root@gunwoo-auth-basic:~/hardway/cert# for instance in 172.31.1.65 172.31.7.230 172.31.15.197; do
>   scp  -i gunwoo.pem ca.crt ca.key kube-apiserver.key kube-apiserver.crt \
>     service-account.key service-account.crt \
>     etcd-server.key etcd-server.crt \
>     ubuntu@${instance}:~/
> done
The authenticity of host '172.31.1.65 (172.31.1.65)' can't be established.
ECDSA key fingerprint is SHA256:27gsQ8TuotxQnWbSQm/JSxYm87Zfm5JwTPT+iawkQco.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added '172.31.1.65' (ECDSA) to the list of known hosts.
ca.crt                                                                                                                                100% 1001     1.3MB/s   00:00    
ca.key                                                                                                                                100% 1679     2.3MB/s   00:00    
kube-apiserver.key                                                                                                                    100% 1679     1.3MB/s   00:00    
kube-apiserver.crt                                                                                                                    100% 1237   932.3KB/s   00:00    
service-account.key                                                                                                                   100% 1679     1.3MB/s   00:00    
service-account.crt                                                                                                                   100% 1005   210.8KB/s   00:00    
etcd-server.key                                                                                                                       100% 1679     1.3MB/s   00:00    
etcd-server.crt                                                                                                                       100% 1090   617.3KB/s   00:00    
The authenticity of host '172.31.7.230 (172.31.7.230)' can't be established.
ECDSA key fingerprint is SHA256:g22Qvv90n+lEr99d7I8cpwq7DomAjEimUdYjasCQ558.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added '172.31.7.230' (ECDSA) to the list of known hosts.
ca.crt                                                                                                                                100% 1001     1.4MB/s   00:00    
ca.key                                                                                                                                100% 1679     2.2MB/s   00:00    
kube-apiserver.key                                                                                                                    100% 1679     1.2MB/s   00:00    
kube-apiserver.crt                                                                                                                    100% 1237     1.7MB/s   00:00    
service-account.key                                                                                                                   100% 1679     2.5MB/s   00:00    
service-account.crt                                                                                                                   100% 1005     1.5MB/s   00:00    
etcd-server.key                                                                                                                       100% 1679     1.2MB/s   00:00    
etcd-server.crt                                                                                                                       100% 1090     1.7MB/s   00:00    
The authenticity of host '172.31.15.197 (172.31.15.197)' can't be established.
ECDSA key fingerprint is SHA256:lT7GEml7+xgAlYMax0+M83QdomeGFuzkeggLIaLlIyo.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added '172.31.15.197' (ECDSA) to the list of known hosts.
ca.crt                                                                                                                                100% 1001     1.0MB/s   00:00    
ca.key                                                                                                                                100% 1679     2.2MB/s   00:00    
kube-apiserver.key                                                                                                                    100% 1679     2.0MB/s   00:00    
kube-apiserver.crt                                                                                                                    100% 1237     1.7MB/s   00:00    
service-account.key                                                                                                                   100% 1679     2.5MB/s   00:00    
service-account.crt                                                                                                                   100% 1005     1.5MB/s   00:00    
etcd-server.key                                                                                                                       100% 1679     2.1MB/s   00:00    
etcd-server.crt

파일을 마스터 서버에게 옮김

  • 모든 노드에 배포
  • ca.crt
  • kube-proxy.kubeconfig 
root@gunwoo-auth-basic:~/hardway/cert# for instance in 172.31.1.65 172.31.7.230 172.31.15.197 172.31.3.25 172.31.2.183 172.31.11.91; do
>   scp -i gunwoo.pem ca.crt kube-proxy.kubeconfig ubuntu@${instance}:~/
> done
ca.crt                                                                                                                                100% 1001     1.3MB/s   00:00    
kube-proxy.kubeconfig                                                                                                                 100% 5297     2.7MB/s   00:00    
ca.crt                                                                                                                                100% 1001     1.2MB/s   00:00    
kube-proxy.kubeconfig                                                                                                                 100% 5297     3.0MB/s   00:00    
ca.crt                                                                                                                                100% 1001     1.2MB/s   00:00    
kube-proxy.kubeconfig                                                                                                                 100% 5297     6.7MB/s   00:00    
The authenticity of host '172.31.3.25 (172.31.3.25)' can't be established.
ECDSA key fingerprint is SHA256:rKULwNOhZd6vswinZslwdpU2rMXLfm5KDMHLAEOCxQE.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added '172.31.3.25' (ECDSA) to the list of known hosts.
ca.crt                                                                                                                                100% 1001     1.2MB/s   00:00    
kube-proxy.kubeconfig                                                                                                                 100% 5297     6.9MB/s   00:00    
The authenticity of host '172.31.2.183 (172.31.2.183)' can't be established.
ECDSA key fingerprint is SHA256:PDD3iwMmlQWXLfWx5XPmqzuHmNjkH0oxtYJm5Y/LgjM.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added '172.31.2.183' (ECDSA) to the list of known hosts.
ca.crt                                                                                                                                100% 1001     1.1MB/s   00:00    
kube-proxy.kubeconfig                                                                                                                 100% 5297     6.3MB/s   00:00    
The authenticity of host '172.31.11.91 (172.31.11.91)' can't be established.
ECDSA key fingerprint is SHA256:qUN4Mizh1EJb6jSTYIoyeEr2ZYHg8YI9kaRB0FfQIQU.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added '172.31.11.91' (ECDSA) to the list of known hosts.
ca.crt                                                                                                                                100% 1001     1.3MB/s   00:00    
kube-proxy.kubeconfig                                                                                                                 100% 5297     6.6MB/s   00:00    
root@gunwoo-auth-basic:~/hardway/cert#
  • 마스터 노드에 배포
  • admin.kubeconfig
  • kube-controller-manager.kubeconfig
  • kube-scheduler.kubeconfig
root@gunwoo-auth-basic:~/hardway/cert# for instance in 172.31.1.65 172.31.7.230 172.31.15.197; do
>   scp -i gunwoo.pem admin.kubeconfig kube-controller-manager.kubeconfig kube-scheduler.kubeconfig ubuntu@${instance}:~/
> done
admin.kubeconfig                                                                                                                      100% 5297     3.3MB/s   00:00    
kube-controller-manager.kubeconfig                                                                                                    100% 5343     3.5MB/s   00:00    
kube-scheduler.kubeconfig                                                                                                             100% 5313     3.8MB/s   00:00    
admin.kubeconfig                                                                                                                      100% 5297     5.7MB/s   00:00    
kube-controller-manager.kubeconfig                                                                                                    100% 5343     6.2MB/s   00:00    
kube-scheduler.kubeconfig                                                                                                             100% 5313     1.8MB/s   00:00    
admin.kubeconfig                                                                                                                      100% 5297     5.9MB/s   00:00    
kube-controller-manager.kubeconfig                                                                                                    100% 5343     6.6MB/s   00:00    
kube-scheduler.kubeconfig                                                                                                             100% 5313     6.8MB/s   00:00

etcd 클러스터 생성 (마스터 3대에서 모두 실행)

wget https://github.com/coreos/etcd/releases/download/v3.4.13/etcd-v3.4.13-linux-amd64.tar.gz

{
  tar -xvf etcd-v3.4.13-linux-amd64.tar.gz
  sudo mv etcd-v3.4.13-linux-amd64/etcd* /usr/local/bin/
  sudo mkdir -p /etc/ssl/etcd/ssl /var/lib/etcd  sudo chmod 700 /var/lib/etcd
  sudo cp /home/ubuntu/ca.crt /home/ubuntu/etcd-server.key /home/ubuntu/etcd-server.crt /etc/ssl/etcd/ssl
}

export INTERNAL_IP=$(ip addr show eth0|grep -w "inet"|awk '{print $2}'|cut -d / -f 1)
export ETCD_NAME=$(hostname -s)
  • systemd etcd service 등록
cat <<EOF | sudo tee /etc/systemd/system/etcd.service
[Unit]
Description=etcd
Documentation=https://github.com/coreos
 
[Service]
ExecStart=/usr/local/bin/etcd \\
  --name ${ETCD_NAME} --cert-file=/etc/ssl/etcd/ssl/etcd-server.crt --key-file=/etc/ssl/etcd/ssl/etcd-server.key --peer-cert-file=/etc/ssl/etcd/ssl/etcd-server.crt \\
  --peer-key-file=/etc/ssl/etcd/ssl/etcd-server.key --trusted-ca-file=/etc/ssl/etcd/ssl/ca.crt --peer-trusted-ca-file=/etc/ssl/etcd/ssl/ca.crt --peer-client-cert-auth \\
  --client-cert-auth --initial-advertise-peer-urls https://${INTERNAL_IP}:2380 --listen-peer-urls https://${INTERNAL_IP}:2380 \\
  --listen-client-urls https://${INTERNAL_IP}:2379,https://127.0.0.1:2379 --advertise-client-urls https://${INTERNAL_IP}:2379 \\
  --initial-cluster-token etcd-cluster-0 --initial-cluster-state new --data-dir=/var/lib/etcd \\
 --initial-cluster gunwoo-k8s-hardway-master-1=https://172.31.1.65:2380,gunwoo-k8s-hardway-master-2=https://172.31.7.230:2380,gunwoo-k8s-hardway-master-3=https://172.31.15.197:2380
Restart=on-failure
RestartSec=5
 
[Install]
WantedBy=multi-user.target
EOF
  • etcd 권한 700 확인 
root@gunwoo-k8s-hardway-master-1:~# ls -al /var/lib | grep etcd 
drwxr-xr-x  2 root      root      4096 Oct 11 02:03 etcd
root@gunwoo-k8s-hardway-master-1:~# sudo chmod 700 /var/lib/etcd
root@gunwoo-k8s-hardway-master-1:~# ls -al /var/lib | grep etcd 
drwx------  2 root      root      4096 Oct 11 02:03 etcd
  • etcd service 시작 
root@gunwoo-k8s-hardway-master-1:~# {
>   sudo systemctl daemon-reload
>   sudo systemctl enable etcd
>   sudo systemctl start etcd
> }
Created symlink /etc/systemd/system/multi-user.target.wants/etcd.service → /etc/systemd/system/etcd.service.
  • etcd cluster health check 
root@gunwoo-k8s-hardway-master-1:~# export ETCDCTL_API=3
root@gunwoo-k8s-hardway-master-1:~# export ETCDCTL_CERT=/etc/ssl/etcd/ssl/etcd-server.crt
root@gunwoo-k8s-hardway-master-1:~# export ETCDCTL_KEY=/etc/ssl/etcd/ssl/etcd-server.key
root@gunwoo-k8s-hardway-master-1:~# export ETCDCTL_CACERT=/etc/ssl/etcd/ssl/ca.crt


root@gunwoo-k8s-hardway-master-1:~# etcdctl endpoint status --cluster --write-out=table
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|          ENDPOINT          |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://172.31.15.197:2379 | 9b0457963e93d934 |  3.4.13 |   20 kB |     false |      false |         5 |          9 |                  9 |        |
|  https://172.31.7.230:2379 | d40c5ebf706021ad |  3.4.13 |   20 kB |     false |      false |         5 |          9 |                  9 |        |
|   https://172.31.1.65:2379 | f7eeb6f0f7c3540c |  3.4.13 |   20 kB |      true |      false |         5 |          9 |                  9 |        |
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

bootstrap k8s control plane

  • Binary 다운로드 (마스터 노드 3대에서 각각 실행)
root@gunwoo-k8s-hardway-master-1:~# wget -q --show-progress --https-only --timestamping \
>   https://storage.googleapis.com/kubernetes-release/release/v1.19.16/bin/linux/amd64/kubectl \
>   "https://storage.googleapis.com/kubernetes-release/release/v1.19.16/bin/linux/amd64/kube-apiserver" \
>   "https://storage.googleapis.com/kubernetes-release/release/v1.19.16/bin/linux/amd64/kube-controller-manager" \
>   "https://storage.googleapis.com/kubernetes-release/release/v1.19.16/bin/linux/amd64/kube-scheduler"
kubectl                                   100%[=====================================================================================>]  40.96M  34.8MB/s    in 1.2s    
kube-apiserver                            100%[=====================================================================================>] 110.00M  42.7MB/s    in 2.6s    
kube-controller-manager                   100%[=====================================================================================>] 102.35M  88.4MB/s    in 1.2s    
kube-scheduler                            100%[=====================================================================================>]  40.95M  41.6MB/s    in 1.0s    
root@gunwoo-k8s-hardway-master-1:~# {
>   chmod +x kubectl kube-apiserver kube-controller-manager kube-scheduler
>   sudo mv kubectl kube-apiserver kube-controller-manager kube-scheduler /usr/local/bin/
> }
root@gunwoo-k8s-hardway-master-1:~# mkdir -p ~/.kube
root@gunwoo-k8s-hardway-master-1:~# cp -rf /home/ubuntu/admin.kubeconfig ~/.kube/config
  • k8s dir를 생성하고 인증서 복사
root@gunwoo-k8s-hardway-master-1:~# {
>    sudo mkdir -p /etc/kubernetes/ssl
>  
>    sudo cp /home/ubuntu/ca.crt /home/ubuntu/ca.key /home/ubuntu/kube-apiserver.crt /home/ubuntu/kube-apiserver.key \
>     /home/ubuntu/service-account.key /home/ubuntu/service-account.crt /etc/kubernetes/ssl
> }
root@gunwoo-k8s-hardway-master-1:~# ls /etc/kubernetes/ssl
ca.crt  ca.key  kube-apiserver.crt  kube-apiserver.key  service-account.crt  service-account.key
  • control plane과 admin이 사용할 kubeconfig 복사
root@gunwoo-k8s-hardway-master-1:~# {
>    sudo cp /home/ubuntu/kube-controller-manager.kubeconfig /etc/kubernetes
>    sudo cp /home/ubuntu/kube-scheduler.kubeconfig /etc/kubernetes
>    sudo cp /home/ubuntu/admin.kubeconfig /etc/kubernetes
> }
root@gunwoo-k8s-hardway-master-1:~# 
root@gunwoo-k8s-hardway-master-1:~# export INTERNAL_IP=$(ip addr show eth0|grep -w "inet"|awk '{print $2}'|cut -d / -f 1)
root@gunwoo-k8s-hardway-master-1:~# echo $INTERNAL_IP 
172.31.1.65
root@gunwoo-k8s-hardway-master-1:~# ls /etc/kubernetes/
admin.kubeconfig  kube-controller-manager.kubeconfig  kube-scheduler.kubeconfig  ssl
  • systemd service 등록 - kube-apiserver
cat <<EOF | sudo tee /etc/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
 
[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
  --advertise-address=${INTERNAL_IP} --allow-privileged=true --apiserver-count=3  --service-cluster-ip-range=10.231.0.0/18 --service-node-port-range=30000-32767 \\
  --authorization-mode=Node,RBAC --bind-address=0.0.0.0  --v=2 \\
  --client-ca-file=/etc/kubernetes/ssl/ca.crt --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-swagger-ui=true \\
  --enable-bootstrap-token-auth=true --etcd-cafile=/etc/ssl/etcd/ssl/ca.crt --etcd-certfile=/etc/ssl/etcd/ssl/etcd-server.crt --etcd-keyfile=/etc/ssl/etcd/ssl/etcd-server.key \\
  --etcd-servers=https://172.31.1.65:2379,https://172.31.7.230:2379,https://172.31.15.197:2379 \\
  --event-ttl=1h --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.crt \\
  --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver.key --kubelet-https=true --runtime-config= --service-account-key-file=/etc/kubernetes/ssl/service-account.crt \\
  --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.crt --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver.key
Restart=on-failure
RestartSec=5
 
[Install]
WantedBy=multi-user.target
EOF
  • systemd service 등록 - kube-controller-manager
cat <<EOF | sudo tee /etc/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
 
[Service]
ExecStart=/usr/local/bin/kube-controller-manager \\
  --address=0.0.0.0 --cluster-cidr=10.240.0.0/13 --allocate-node-cidrs=true --cluster-name=gunwoo-k8s-hardway \\
  --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.crt --cluster-signing-key-file=/etc/kubernetes/ssl/ca.key \\
  --kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \\
  --leader-elect=true --root-ca-file=/etc/kubernetes/ssl/ca.crt --service-account-private-key-file=/etc/kubernetes/ssl/service-account.key \\
  --service-cluster-ip-range=10.231.0.0/18 --use-service-account-credentials=true --v=2
Restart=on-failure
RestartSec=5
 
[Install]
WantedBy=multi-user.target
EOF
  • systemd service 등록 - kube-scheduler
cat <<EOF | sudo tee /etc/systemd/system/kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
 
[Service]
ExecStart=/usr/local/bin/kube-scheduler \\
  --kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \\
  --address=127.0.0.1 \\
  --leader-elect=true \\
  --v=2
Restart=on-failure
RestartSec=5
 
[Install]
WantedBy=multi-user.target
EOF
  • systemd service 시작 
root@gunwoo-k8s-hardway-master-1:~# {
>   sudo systemctl daemon-reload
>   sudo systemctl enable kube-apiserver kube-controller-manager kube-scheduler
>   sudo systemctl start kube-apiserver kube-controller-manager kube-scheduler
> }
Created symlink /etc/systemd/system/multi-user.target.wants/kube-apiserver.service → /etc/systemd/system/kube-apiserver.service.
Created symlink /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service → /etc/systemd/system/kube-controller-manager.service.
Created symlink /etc/systemd/system/multi-user.target.wants/kube-scheduler.service → /etc/systemd/system/kube-scheduler.service.
  • kubectl 명령어 실행 
root@gunwoo-k8s-hardway-master-1:~# kubectl get nodes 
No resources found
root@gunwoo-k8s-hardway-master-1:~# kubectl version 
Client Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.16", GitCommit:"e37e4ab4cc8dcda84f1344dda47a97bb1927d074", GitTreeState:"clean", BuildDate:"2021-10-27T16:25:59Z", GoVersion:"go1.15.15", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.16", GitCommit:"e37e4ab4cc8dcda84f1344dda47a97bb1927d074", GitTreeState:"clean", BuildDate:"2021-10-27T16:20:18Z", GoVersion:"go1.15.15", Compiler:"gc", Platform:"linux/amd64"}
root@gunwoo-k8s-hardway-master-1:~# kubectl get svc 
NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE
kubernetes   ClusterIP   10.231.0.1   <none>        443/TCP   25s
root@gunwoo-k8s-hardway-master-1:~# curl https://localhost:6443/version -k
{
  "major": "1",
  "minor": "19",
  "gitVersion": "v1.19.16",
  "gitCommit": "e37e4ab4cc8dcda84f1344dda47a97bb1927d074",
  "gitTreeState": "clean",
  "buildDate": "2021-10-27T16:20:18Z",
  "goVersion": "go1.15.15",
  "compiler": "gc",
  "platform": "linux/amd64"
}
  • kubelet client 인증서 생성 (gunwoo-auth-basic 인스턴스에서)
root@gunwoo-auth-basic:~/hardway/cert# cat > openssl-gunwoo-k8s-hardway-worker-1.cnf <<EOF
> [req]
> req_extensions = v3_req
> distinguished_name = req_distinguished_name
> [req_distinguished_name]
> [ v3_req ]
> basicConstraints = CA:FALSE
> keyUsage = nonRepudiation, digitalSignature, keyEncipherment
> subjectAltName = @alt_names
> [alt_names] 
> IP.1 = 172.31.3.25
> DNS.1 = gunwoo-k8s-hardway-worker-1
> EOF
root@gunwoo-auth-basic:~/hardway/cert#  
root@gunwoo-auth-basic:~/hardway/cert# openssl genrsa -out gunwoo-k8s-hardway-worker-1.key 2048
Generating RSA private key, 2048 bit long modulus (2 primes)
..................................+++++
..............................+++++
e is 65537 (0x010001)
root@gunwoo-auth-basic:~/hardway/cert# openssl req -new -key gunwoo-k8s-hardway-worker-1.key -subj "/CN=system:node:gunwoo-k8s-hardway-worker-1/O=system:nodes" -out gunwoo-k8s-hardway-worker-1.csr -config openssl-gunwoo-k8s-hardway-worker-1.cnf
root@gunwoo-auth-basic:~/hardway/cert# openssl x509 -req -in gunwoo-k8s-hardway-worker-1.csr -CA ca.crt -CAkey ca.key -CAcreateserial  -out gunwoo-k8s-hardway-worker-1.crt -extensions v3_req -extfile openssl-gunwoo-k8s-hardway-worker-1.cnf -days 365
Signature ok
subject=CN = system:node:gunwoo-k8s-hardway-worker-1, O = system:nodes
Getting CA Private Key

IP.1 = 172.31.3.25
worker node 1의 ip 주소

  • kubelet kubeconfig 생성 (gunwoo-auth-basic 인스턴스에서)
root@gunwoo-auth-basic:~/hardway/cert# export KUBE_API_SERVER_ADDRESS=172.31.1.65
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# {
>   kubectl config set-cluster gunwoo-k8s-hardway \
>     --certificate-authority=ca.crt --embed-certs=true \
>     --server=https://${KUBE_API_SERVER_ADDRESS}:6443 --kubeconfig=gunwoo-k8s-hardway-worker-1.kubeconfig
>  
>   kubectl config set-credentials system:node:gunwoo-k8s-hardway-worker-1 \
>     --client-certificate=gunwoo-k8s-hardway-worker-1.crt --client-key=gunwoo-k8s-hardway-worker-1.key \
>     --embed-certs=true --kubeconfig=gunwoo-k8s-hardway-worker-1.kubeconfig
>  
>   kubectl config set-context default \
>     --cluster=gunwoo-k8s-hardway --user=system:node:gunwoo-k8s-hardway-worker-1 --kubeconfig=gunwoo-k8s-hardway-worker-1.kubeconfig
>  
>   kubectl config use-context default --kubeconfig=gunwoo-k8s-hardway-worker-1.kubeconfig
> }
Cluster "gunwoo-k8s-hardway" set.
User "system:node:gunwoo-k8s-hardway-worker-1" set.
Context "default" created.
Switched to context "default".

export KUBE_API_SERVER_ADDRESS=172.31.1.65
master node 1의 ip 주소

kubelet server/client cert/key 배포

root@gunwoo-auth-basic:~/hardway/cert# for instance in 172.31.3.25; do
>   scp -i gunwoo.pem gunwoo-k8s-hardway-worker-1.crt gunwoo-k8s-hardway-worker-1.key gunwoo-k8s-hardway-worker-1.kubeconfig \
>   ubuntu@${instance}:~/
> done
gunwoo-k8s-hardway-worker-1.crt                                                                                                       100% 1176     1.6MB/s   00:00    
gunwoo-k8s-hardway-worker-1.key                                                                                                       100% 1675     2.6MB/s   00:00    
gunwoo-k8s-hardway-worker-1.kubeconfig                                                                                                100% 5561     7.9MB/s   00:00    
root@gunwoo-auth-basic:~/hardway/cert#

worker1에게 인증서 전송

docker 설치

  • docker 설치 (worker-1) 
sudo apt-get update
sudo apt-get install -y docker.io
sudo swapoff -a
  • Docker 서비스 시작 및 테스트 (worker-1)
root@gunwoo-k8s-hardway-worker-1:~# {
> sudo systemctl daemon-reload
> sudo systemctl enable docker
> sudo systemctl restart docker
> }
root@gunwoo-k8s-hardway-worker-1:~# 
root@gunwoo-k8s-hardway-worker-1:~# sudo docker run --rm -it nicolaka/netshoot
Unable to find image 'nicolaka/netshoot:latest' locally
latest: Pulling from nicolaka/netshoot
2408cc74d12b: Pull complete 
3cf03e9fd7e4: Pull complete 
d3299a042ee2: Pull complete 
e00dbf0b44c3: Pull complete 
1cb6452f30ec: Pull complete 
cc37e878581f: Pull complete 
4f4fb700ef54: Pull complete 
b01a5ec78db0: Pull complete 
123e924da2d1: Pull complete 
367cf0ba0067: Pull complete 
5a3733011111: Pull complete 
6adf3d98c1c3: Pull complete 
77c79836780d: Pull complete 
4aafb6d72b2b: Pull complete 
Digest: sha256:aeafd567d7f7f1edb5127ec311599bb2b8a9c0fb31d7a53e9cff26af6d29fd4e
Status: Downloaded newer image for nicolaka/netshoot:latest
                    dP            dP                           dP   
                    88            88                           88   
88d888b. .d8888b. d8888P .d8888b. 88d888b. .d8888b. .d8888b. d8888P 
88'  `88 88ooood8   88   Y8ooooo. 88'  `88 88'  `88 88'  `88   88   
88    88 88.  ...   88         88 88    88 88.  .88 88.  .88   88   
dP    dP `88888P'   dP   `88888P' dP    dP `88888P' `88888P'   dP   
                                                                    
Welcome to Netshoot! (github.com/nicolaka/netshoot)
                                                                


 360c61f9da2c  ~ 

Binary 다운로드

  • kubectl, kube-proxy, kubelet (worker-1)
wget -q --show-progress --https-only --timestamping \
  https://storage.googleapis.com/kubernetes-release/release/v1.19.16/bin/linux/amd64/kubectl \
  https://storage.googleapis.com/kubernetes-release/release/v1.19.16/bin/linux/amd64/kube-proxy \
  https://storage.googleapis.com/kubernetes-release/release/v1.19.16/bin/linux/amd64/kubelet

sudo mkdir -p /etc/cni/net.d /opt/cni/bin /var/lib/kubelet /var/lib/kube-proxy /etc/kubernetes/ssl /var/run/kubernetes
 
{
  chmod +x kubectl kube-proxy kubelet
  sudo mv kubectl kube-proxy kubelet /usr/local/bin/
}

{
  sudo mkdir -p /opt/cni/bin
  wget https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz
  sudo tar -xzvf cni-plugins-linux-amd64-v0.8.6.tgz --directory /opt/cni/bin/
}
  • 인증서 옮기기 
root@gunwoo-k8s-hardway-worker-1:~# export HOSTNAME=$(hostname)
root@gunwoo-k8s-hardway-worker-1:~# {
>   mkdir -p /etc/kubernetes/ssl
>   sudo cp -rf /home/ubuntu/${HOSTNAME}.key /home/ubuntu/${HOSTNAME}.crt /home/ubuntu/ca.crt /etc/kubernetes/ssl
>   sudo cp -rf /home/ubuntu/${HOSTNAME}.kubeconfig /etc/kubernetes
> }
  • kubelet-config.yaml
root@gunwoo-k8s-hardway-worker-1:~# cat << EOF | sudo tee /etc/kubernetes/kubelet-config.yaml
> kind: KubeletConfiguration
> apiVersion: kubelet.config.k8s.io/v1beta1
> authentication:
>   anonymous:
>     enabled: false
>   webhook:
>     enabled: true
>   x509:
>     clientCAFile: "/etc/kubernetes/ssl/ca.crt"
> authorization:
>   mode: Webhook
> clusterDomain: "cluster.local"
> clusterDNS:
> - 10.231.0.10
> resolvConf: "/run/systemd/resolve/resolv.conf"
> runtimeRequestTimeout: "15m"
> EOF
  • kubelet.service
root@gunwoo-k8s-hardway-worker-1:~# cat <<EOF | sudo tee /etc/systemd/system/kubelet.service
> [Unit]
> Description=Kubernetes Kubelet
> Documentation=https://github.com/kubernetes/kubernetes
> After=docker.service
> Requires=docker.service
>  
> [Service]
> ExecStart=/usr/local/bin/kubelet \\
>   --config=/etc/kubernetes/kubelet-config.yaml --image-pull-progress-deadline=2m \\
>   --kubeconfig=/etc/kubernetes/${HOSTNAME}.kubeconfig --tls-cert-file=/etc/kubernetes/ssl/${HOSTNAME}.crt \\
>   --tls-private-key-file=/etc/kubernetes/ssl/${HOSTNAME}.key --network-plugin=cni --register-node=true --v=2
> Restart=on-failure
> RestartSec=5
>  
> [Install]
> WantedBy=multi-user.target
> EOF
root@gunwoo-k8s-hardway-worker-1:~# {
>   sudo cp /home/ubuntu/kube-proxy.kubeconfig /etc/kubernetes
> }

root@gunwoo-k8s-hardway-worker-1:~# cd /etc/kubernetes/
root@gunwoo-k8s-hardway-worker-1:/etc/kubernetes# ls
gunwoo-k8s-hardway-worker-1.kubeconfig  kube-proxy.kubeconfig  kubelet-config.yaml  ssl
sudo apt-get install ipset -y

cat <<EOF | sudo tee /etc/kubernetes/kube-proxy-config.yaml
kind: KubeProxyConfiguration
apiVersion: kubeproxy.config.k8s.io/v1alpha1
clientConnection:
  kubeconfig: "/etc/kubernetes/kube-proxy.kubeconfig"
mode: "ipvs"
clusterCIDR: "10.240.0.0/13"
EOF
cat <<EOF | sudo tee /etc/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube Proxy
Documentation=https://github.com/kubernetes/kubernetes
 
[Service]
ExecStart=/usr/local/bin/kube-proxy \\
  --config=/etc/kubernetes/kube-proxy-config.yaml
  Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF
{
  sudo systemctl daemon-reload
  sudo systemctl enable kubelet kube-proxy
  sudo systemctl start kubelet kube-proxy
}


systemctl status kubelet
systemctl status kube-proxy
sudo journalctl -xef -u kubelet
sudo journalctl -xef -u kube-proxy
  • auth-basic에서 worker 노드 확인 
root@gunwoo-auth-basic:~/hardway/cert# kubectl get nodes 
NAME                          STATUS     ROLES    AGE   VERSION
gunwoo-k8s-hardway-worker-1   NotReady   <none>   50s   v1.19.16

overlay network가 없기 떄문에 NotReady 상태이다.

  • auth-basic에서 CNI plugin 설치 
{
  wget https://raw.githubusercontent.com/cilium/cilium/v1.7/install/kubernetes/quick-install.yaml
  kubectl apply -f quick-install.yaml
}
  • 아직 로그 볼 권한이 없음 
root@gunwoo-auth-basic:~/hardway/cert# kubectl logs -f cilium-mtx66 -n kube-system 
Error from server (Forbidden): Forbidden (user=kube-apiserver, verb=get, resource=nodes, subresource=proxy) ( pods/log cilium-mtx66)
  • clusterrole 바인딩 
cat << EOF > kubelet-clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:kube-apiserver-to-kubelet
rules:
- apiGroups:
  - ""
  resources:
    - nodes/proxy
    - nodes/stats
    - nodes/log
    - nodes/spec
    - nodes/metrics
  verbs:
    - "*"
EOF

kubectl apply -f kubelet-clusterrole.yaml

root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# cat << EOF > kubelet-clusterrolebinding.yaml
> apiVersion: rbac.authorization.k8s.io/v1beta1
> kind: ClusterRoleBinding
> metadata:
>   name: system:kube-apiserver
>   namespace: ""
> roleRef:
>   apiGroup: rbac.authorization.k8s.io
>   kind: ClusterRole
>   name: system:kube-apiserver-to-kubelet
> subjects:
>   - apiGroup: rbac.authorization.k8s.io
>     kind: User
>     name: kube-apiserver
> EOF
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# kubectl apply -f kubelet-clusterrolebinding.yaml
Warning: rbac.authorization.k8s.io/v1beta1 ClusterRoleBinding is deprecated in v1.17+, unavailable in v1.22+; use rbac.authorization.k8s.io/v1 ClusterRoleBinding
clusterrolebinding.rbac.authorization.k8s.io/system:kube-apiserver created
  • 이제 로그 볼 수 있음 
root@gunwoo-auth-basic:~/hardway/cert# kubectl logs -f cilium-mtx66 -n kube-system

CoreDNS

vi coredns.yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
  labels:
      addonmanager.kubernetes.io/mode: EnsureExists
data:
  Corefile: |
    .:53 {
        errors
        health {
            lameduck 5s
        }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
          pods insecure
          fallthrough in-addr.arpa ip6.arpa
        }
        prometheus :9153
        forward . /etc/resolv.conf {
          prefer_udp
        }
        cache 30
        loop
        reload
        loadbalance
    }
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
    addonmanager.kubernetes.io/mode: EnsureExists
  name: system:coredns
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:coredns
subjects:
  - kind: ServiceAccount
    name: coredns
    namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
    addonmanager.kubernetes.io/mode: Reconcile
  name: system:coredns
rules:
  - apiGroups:
      - ""
    resources:
      - endpoints
      - services
      - pods
      - namespaces
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - get
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: "coredns"
  namespace: kube-system
  labels:
    k8s-app: "kube-dns"
    addonmanager.kubernetes.io/mode: Reconcile
    kubernetes.io/name: "coredns"
spec:
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 0
      maxSurge: 10%
  selector:
    matchLabels:
      k8s-app: kube-dns
  template:
    metadata:
      labels:
        k8s-app: kube-dns
      annotations:
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
    spec:
      priorityClassName: system-cluster-critical
      nodeSelector:
        kubernetes.io/os: linux
      serviceAccountName: coredns
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule
        - key: "CriticalAddonsOnly"
          operator: "Exists"
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - topologyKey: "kubernetes.io/hostname"
            labelSelector:
              matchLabels:
                k8s-app: kube-dns
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            preference:
              matchExpressions:
              - key: node-role.kubernetes.io/master
                operator: In
                values:
                - ""
      containers:
      - name: coredns
        image: "coredns/coredns:1.6.5"
        imagePullPolicy: IfNotPresent
        resources:
          # TODO: Set memory limits when we've profiled the container for large
          # clusters, then set request = limit to keep this container in
          # guaranteed class. Currently, this container falls into the
          # "burstable" category so the kubelet doesn't backoff from restarting it.
          limits:
            memory: 170Mi
          requests:
            cpu: 100m
            memory: 70Mi
        args: [ "-conf", "/etc/coredns/Corefile" ]
        volumeMounts:
        - name: config-volume
          mountPath: /etc/coredns
        - name: tz-config
          mountPath: /etc/localtime
          readOnly: true
        ports:
        - containerPort: 53
          name: dns
          protocol: UDP
        - containerPort: 53
          name: dns-tcp
          protocol: TCP
        - containerPort: 9153
          name: metrics
          protocol: TCP
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            add:
            - NET_BIND_SERVICE
            drop:
            - all
          readOnlyRootFilesystem: true
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
            scheme: HTTP
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 10
        readinessProbe:
          httpGet:
            path: /ready
            port: 8181
            scheme: HTTP
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 10
      dnsPolicy: Default
      volumes:
        - name: config-volume
          configMap:
            name: coredns
            items:
            - key: Corefile
              path: Corefile
        - name: tz-config
          hostPath:
            path: /etc/localtime
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: coredns
  namespace: kube-system
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: v1
kind: Service
metadata:
  name: coredns
  namespace: kube-system
  labels:
    k8s-app: kube-dns
    kubernetes.io/name: "coredns"
    addonmanager.kubernetes.io/mode: Reconcile
  annotations:
    prometheus.io/port: "9153"
    prometheus.io/scrape: "true"
spec:
  selector:
    k8s-app: kube-dns
  clusterIP: 10.231.0.10
  ports:
    - name: dns
      port: 53
      protocol: UDP
    - name: dns-tcp
      port: 53
      protocol: TCP
    - name: metrics
      port: 9153
      protocol: TCP
root@gunwoo-auth-basic:~/hardway/cert# kubectl apply -f coredns.yaml 
configmap/coredns created
clusterrolebinding.rbac.authorization.k8s.io/system:coredns created
clusterrole.rbac.authorization.k8s.io/system:coredns created
deployment.apps/coredns created
serviceaccount/coredns created
service/coredns created

netshoot

root@gunwoo-auth-basic:~/hardway/cert# vi netshoot.yaml 
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: netshoot-cluster
  labels:
    app: netshoot-cluster
spec:
  selector:
    matchLabels:
      name: netshoot-cluster
  template:
    metadata:
      labels:
        name: netshoot-cluster
    spec:
      tolerations:
      - operator: Exists
      containers:
      - name: netshoot-cluster
        tty: true
        stdin: true
        args:
        - bash
        image: nicolaka/netshoot
root@gunwoo-auth-basic:~/hardway/cert# kubectl apply -f netshoot.yaml
daemonset.apps/netshoot-cluster created
  • netshoot 사용 
root@gunwoo-auth-basic:~/hardway/cert# kubectl get pod 
NAME                     READY   STATUS    RESTARTS   AGE
netshoot-cluster-jj48b   1/1     Running   0          39s
root@gunwoo-auth-basic:~/hardway/cert# kubectl exec -it netshoot-cluster-jj48b /bin/bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
bash-5.1# hostname
netshoot-cluster-jj48b
bash-5.1# nslookup kubernetes 
Server:         10.231.0.10
Address:        10.231.0.10#53

Name:   kubernetes.default.svc.cluster.local
Address: 10.231.0.1

bash-5.1# nslookup naver.com 
Server:         10.231.0.10
Address:        10.231.0.10#53

Non-authoritative answer:
Name:   naver.com
Address: 223.130.200.107
Name:   naver.com
Address: 223.130.195.200
Name:   naver.com
Address: 223.130.200.104
Name:   naver.com
Address: 223.130.195.95
  • echo-server 
root@gunwoo-auth-basic:~/hardway/cert# vi echo-server.yaml
kind: Deployment
apiVersion: apps/v1
metadata:
  name: service-test
spec:
  replicas: 2
  selector:
    matchLabels:
      app: service_test_pod
  template:
    metadata:
      labels:
        app: service_test_pod
    spec:
      containers:
      - name: simple-http
        image: python:2.7
        imagePullPolicy: IfNotPresent
        command: ["/bin/bash"]
        args: ["-c", "echo \"<p>Hello from $(hostname)</p>\" > index.html; python -m SimpleHTTPServer 8080"]
        ports:
        - name: http
          containerPort: 8080
root@gunwoo-auth-basic:~/hardway/cert# kubectl apply -f echo-server.yaml 
deployment.apps/service-test created
root@gunwoo-auth-basic:~/hardway/cert# vi echo-service.yaml
apiVersion: v1
kind: Service
metadata:
  labels:
    app: service_test_pod
  name: echo
spec:
  ports:
    - name: 8080-8080
      port: 8080
      protocol: TCP
      targetPort: 8080
  selector:
    app: service_test_pod
  type: ClusterIP
root@gunwoo-auth-basic:~/hardway/cert# kubectl apply -f echo-service.yaml 
service/echo created
root@gunwoo-auth-basic:~/hardway/cert# kubectl get pod -o wide 
NAME                           READY   STATUS    RESTARTS   AGE    IP             NODE                          NOMINATED NODE   READINESS GATES
netshoot-cluster-jj48b         1/1     Running   0          7m2s   10.240.0.165   gunwoo-k8s-hardway-worker-1   <none>           <none>
service-test-db6d56f8f-8sqv2   1/1     Running   0          119s   10.240.0.179   gunwoo-k8s-hardway-worker-1   <none>           <none>
service-test-db6d56f8f-jwwpn   1/1     Running   0          119s   10.240.0.208   gunwoo-k8s-hardway-worker-1   <none>           <none>

10.240.0.179

root@gunwoo-auth-basic:~/hardway/cert# kubectl exec -it netshoot-cluster-jj48b /bin/bash 
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.

bash-5.1# curl 10.240.0.179:8080 
<p>Hello from service-test-db6d56f8f-8sqv2</p>

bash-5.1# curl 10.240.0.208:8080 
<p>Hello from service-test-db6d56f8f-jwwpn</p>
bash-5.1# nslookup echo 
Server:         10.231.0.10
Address:        10.231.0.10#53

Name:   echo.default.svc.cluster.local
Address: 10.231.31.28

bash-5.1# curl echo:8080 
<p>Hello from service-test-db6d56f8f-8sqv2</p>
bash-5.1# curl echo:8080 
<p>Hello from service-test-db6d56f8f-jwwpn</p>
bash-5.1# curl echo:8080 
<p>Hello from service-test-db6d56f8f-8sqv2</p>
bash-5.1# curl echo:8080 
<p>Hello from service-test-db6d56f8f-jwwpn</p>

echo는 로드밸런서 역할을 하여 라운드 로빈으로 접속 


root@gunwoo-k8s-hardway-worker-1:~# ipvsadm -ln 
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.231.0.1:443 rr
  -> 172.31.1.65:6443             Masq    1      2          0         
  -> 172.31.7.230:6443            Masq    1      2          0         
  -> 172.31.15.197:6443           Masq    1      1          0         
TCP  10.231.0.10:53 rr
  -> 10.240.0.77:53               Masq    1      0          0         
TCP  10.231.0.10:9153 rr
  -> 10.240.0.77:9153             Masq    1      0          0         
TCP  10.231.31.28:8080 rr
  -> 10.240.0.179:8080            Masq    1      0          0         
  -> 10.240.0.208:8080            Masq    1      0          0         
UDP  10.231.0.10:53 rr
  -> 10.240.0.77:53               Masq    1      0          0

TLS Bootstraping

root@gunwoo-auth-basic:~/hardway/cert# cat > bootstrap-token-07401b.yaml <<EOF
> apiVersion: v1
> kind: Secret
> metadata:
>   # Name MUST be of form "bootstrap-token-<token id>"
>   name: bootstrap-token-07401b
>   namespace: kube-system
>  
> # Type MUST be 'bootstrap.kubernetes.io/token'
> type: bootstrap.kubernetes.io/token
> stringData:
>   # Human readable description. Optional.
>   description: "The bootstrap token generated by kdt kakao"
>  
>   # Token ID and secret. Required.
>   token-id: 07401b
>   token-secret: f395accd246ae52d
>  
>   # Expiration. Optional.
>   expiration: 2121-09-01T00:00:00Z
>  
>   # Allowed usages.
>   usage-bootstrap-authentication: "true"
>   usage-bootstrap-signing: "true"
>  
>   # Extra groups to authenticate the token as. Must start with "system:bootstrappers:"
>   auth-extra-groups: system:bootstrappers:worker
> EOF
root@gunwoo-auth-basic:~/hardway/cert# kubectl apply -f bootstrap-token-07401b.yaml 
secret/bootstrap-token-07401b created
root@gunwoo-auth-basic:~/hardway/cert# kubectl get secret -n kube-system | grep bootstrap 
bootstrap-token-07401b                           bootstrap.kubernetes.io/token         7      17s
  • csrs-for-bootstrapping.yaml
root@gunwoo-auth-basic:~/hardway/cert# cat > csrs-for-bootstrapping.yaml <<EOF
> # enable bootstrapping nodes to create CSR
> kind: ClusterRoleBinding
> apiVersion: rbac.authorization.k8s.io/v1
> metadata:
>   name: create-csrs-for-bootstrapping
> subjects:
> - kind: Group
>   name: system:bootstrappers
>   apiGroup: rbac.authorization.k8s.io
> roleRef:
>   kind: ClusterRole
>   name: system:node-bootstrapper
>   apiGroup: rbac.authorization.k8s.io
> EOF
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# kubectl create -f csrs-for-bootstrapping.yaml
  • auto-approve-csrs-for-group.yaml 
cat > auto-approve-csrs-for-group.yaml <<EOF
# Approve all CSRs  for  the group  "system:bootstrappers"
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: auto-approve-csrs-for-group
subjects:
- kind: Group
  name: system:bootstrappers
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
  apiGroup: rbac.authorization.k8s.io
EOF

kubectl create -f auto-approve-csrs-for-group.yaml
  • 확인 
root@gunwoo-auth-basic:~/hardway/cert# kubectl get clusterrolebinding | grep csr 
create-csrs-for-bootstrapping                          ClusterRole/system:node-bootstrapper                               2m52s
root@gunwoo-auth-basic:~/hardway/cert# kubectl get secret -n kube-system | grep bootstrap 
bootstrap-token-07401b                           bootstrap.kubernetes.io/token         7      5m42s
  • auto-approve-renewals-for-nodes.yaml 
root@gunwoo-auth-basic:~/hardway/cert# cat > auto-approve-renewals-for-nodes.yaml <<EOF
> # Approve renewal CSRs  for  the group  "system:nodes"
> kind: ClusterRoleBinding
> apiVersion: rbac.authorization.k8s.io/v1
> metadata:
>   name: auto-approve-renewals-for-nodes
> subjects:
> - kind: Group
>   name: system:nodes
>   apiGroup: rbac.authorization.k8s.io
> roleRef:
>   kind: ClusterRole
>   name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
>   apiGroup: rbac.authorization.k8s.io
> EOF
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# kubectl create -f auto-approve-renewals-for-nodes.yaml
clusterrolebinding.rbac.authorization.k8s.io/auto-approve-renewals-for-nodes created
root@gunwoo-auth-basic:~/hardway/cert#

worker 2

root@gunwoo-auth-basic:~/hardway/cert# 
root@gunwoo-auth-basic:~/hardway/cert# for instance in 172.31.2.183; do
>   scp -i gunwoo.pem ca.crt kube-proxy.kubeconfig ubuntu@${instance}:~/
> done
ca.crt                                                                                                                                100% 1001     1.4MB/s   00:00    
kube-proxy.kubeconfig                                                                                                                 100% 5297     6.7MB/s   00:00    
root@gunwoo-auth-basic:~/hardway/cert#
  • 도커 설치 
root@gunwoo-k8s-hardway-worker2:~# 
root@gunwoo-k8s-hardway-worker2:~# {
>   sudo apt-get update
>   sudo apt-get install -y docker.io
>   sudo swapoff -a
> }
root@gunwoo-k8s-hardway-worker2:~# 
root@gunwoo-k8s-hardway-worker2:~# {
> sudo systemctl daemon-reload
> sudo systemctl enable docker
> sudo systemctl restart docker
> }
root@gunwoo-k8s-hardway-worker2:~# 
root@gunwoo-k8s-hardway-worker2:~# sudo docker run --rm -it nicolaka/netshoot
Unable to find image 'nicolaka/netshoot:latest' locally
latest: Pulling from nicolaka/netshoot
2408cc74d12b: Pull complete 
3cf03e9fd7e4: Pull complete 
d3299a042ee2: Pull complete 
e00dbf0b44c3: Pull complete 
1cb6452f30ec: Pull complete 
cc37e878581f: Pull complete 
4f4fb700ef54: Pull complete 
b01a5ec78db0: Pull complete 
123e924da2d1: Pull complete 
367cf0ba0067: Pull complete 
5a3733011111: Pull complete 
6adf3d98c1c3: Pull complete 
77c79836780d: Pull complete 
4aafb6d72b2b: Pull complete 
Digest: sha256:aeafd567d7f7f1edb5127ec311599bb2b8a9c0fb31d7a53e9cff26af6d29fd4e
Status: Downloaded newer image for nicolaka/netshoot:latest
                    dP            dP                           dP   
                    88            88                           88   
88d888b. .d8888b. d8888P .d8888b. 88d888b. .d8888b. .d8888b. d8888P 
88'  `88 88ooood8   88   Y8ooooo. 88'  `88 88'  `88 88'  `88   88   
88    88 88.  ...   88         88 88    88 88.  .88 88.  .88   88   
dP    dP `88888P'   dP   `88888P' dP    dP `88888P' `88888P'   dP   
                                                                    
Welcome to Netshoot! (github.com/nicolaka/netshoot)
                                                                


 e3b01919af15  ~  
root@gunwoo-k8s-hardway-worker2:~# wget -q --show-progress --https-only --timestamping \
>   https://storage.googleapis.com/kubernetes-release/release/v1.19.16/bin/linux/amd64/kubectl \
>   https://storage.googleapis.com/kubernetes-release/release/v1.19.16/bin/linux/amd64/kube-proxy \
>   https://storage.googleapis.com/kubernetes-release/release/v1.19.16/bin/linux/amd64/kubelet
kubectl                                   100%[=====================================================================================>]  40.96M  47.1MB/s    in 0.9s    
kube-proxy                                100%[=====================================================================================>]  36.96M  43.4MB/s    in 0.9s    
kubelet                                   100%[=====================================================================================>] 105.01M  49.4MB/s    in 2.1s    
root@gunwoo-k8s-hardway-worker2:~# 
root@gunwoo-k8s-hardway-worker2:~# sudo mkdir -p /etc/cni/net.d /opt/cni/bin /var/lib/kubelet /var/lib/kube-proxy /etc/kubernetes/ssl /var/run/kubernetes
root@gunwoo-k8s-hardway-worker2:~#  
root@gunwoo-k8s-hardway-worker2:~# {
>   chmod +x kubectl kube-proxy kubelet
>   sudo mv kubectl kube-proxy kubelet /usr/local/bin/
> }
root@gunwoo-k8s-hardway-worker2:~# 

root@gunwoo-k8s-hardway-worker2:~# {
>   sudo cp -rf /home/ubuntu/ca.crt /etc/kubernetes/ssl
> }
root@gunwoo-k8s-hardway-worker2:~# 
root@gunwoo-k8s-hardway-worker2:~# sudo mkdir -p /var/lib/kubelet/ssl
root@gunwoo-k8s-hardway-worker2:~# 
root@gunwoo-k8s-hardway-worker2:~# cat <<EOF | sudo tee /etc/kubernetes/bootstrap-kubeconfig
> apiVersion: v1
> kind: Config
> clusters:
> - cluster:
>     certificate-authority: /etc/kubernetes/ssl/ca.crt
>     server: https://172.31.1.65:6443
>   name: bootstrap
> contexts:
> - context:
>     cluster: bootstrap
>     user: kubelet-bootstrap
>   name: bootstrap
> current-context: bootstrap
> users:
> - name: kubelet-bootstrap
>   user:
>     token: 07401b.f395accd246ae52d
> preferences: {}
> EOF

master 1 ip 주소 입력 

root@gunwoo-k8s-hardway-worker2:~# 
root@gunwoo-k8s-hardway-worker2:~# cat << EOF | sudo tee /etc/kubernetes/kubelet-config.yaml
> kind: KubeletConfiguration
> apiVersion: kubelet.config.k8s.io/v1beta1
> authentication:
>   anonymous:
>     enabled: false
>   webhook:
>     enabled: true
>   x509:
>     clientCAFile: "/etc/kubernetes/ssl/ca.crt"
>     authorization:
>   mode: Webhook
> clusterDomain: "cluster.local"
> clusterDNS:
>   - "10.231.0.10"
> resolvConf: "/run/systemd/resolve/resolv.conf"
> runtimeRequestTimeout: "15m"
> rotateCertificates: true
> EOF
root@gunwoo-k8s-hardway-worker2:~# ls /etc/kubernetes/
bootstrap-kubeconfig  kubelet-config.yaml  ssl
root@gunwoo-k8s-hardway-worker2:~# ls /etc/kubernetes/ssl/
ca.crt
root@gunwoo-k8s-hardway-worker2:~# 
root@gunwoo-k8s-hardway-worker2:~# cat <<EOF | sudo tee /etc/systemd/system/kubelet.service
> [Unit]
> Description=Kubernetes Kubelet
> Documentation=https://github.com/kubernetes/kubernetes
> After=docker.service
> Requires=docker.service
>  
> [Service]
> ExecStart=/usr/local/bin/kubelet \\
>   --config=/etc/kubernetes/kubelet-config.yaml --image-pull-progress-deadline=2m \\
>   --network-plugin=cni --register-node=true --v=2 \\
>   --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/var/lib/kubelet/ssl/ 
> Restart=on-failure
> RestartSec=5
>  
> [Install]
> WantedBy=multi-user.target
> EOF
root@gunwoo-k8s-hardway-worker2:~# {
>   sudo cp /home/ubuntu/kube-proxy.kubeconfig /etc/kubernetes
> }
root@gunwoo-k8s-hardway-worker2:~# 
root@gunwoo-k8s-hardway-worker2:~# {
> sudo apt-get install ipset -y
> 
> cat <<EOF | sudo tee /etc/kubernetes/kube-proxy-config.yaml
> kind: KubeProxyConfiguration
> apiVersion: kubeproxy.config.k8s.io/v1alpha1
> clientConnection:
>   kubeconfig: "/etc/kubernetes/kube-proxy.kubeconfig"
> mode: "ipvs"
> clusterCIDR: "10.240.0.0/13"
> EOF
> }
root@gunwoo-k8s-hardway-worker2:~# cat <<EOF | sudo tee /etc/systemd/system/kube-proxy.service
> [Unit]
> Description=Kubernetes Kube Proxy
> Documentation=https://github.com/kubernetes/kubernetes
> 
> [Service]
> ExecStart=/usr/local/bin/kube-proxy \\
>   --config=/etc/kubernetes/kube-proxy-config.yaml
> Restart=on-failure
> RestartSec=5
> 
> [Install]
> WantedBy=multi-user.target
> EOF
  • systemd service 시작 
root@gunwoo-k8s-hardway-worker2:~# {
>   sudo systemctl daemon-reload
>   sudo systemctl enable kubelet kube-proxy
>   sudo systemctl start kubelet kube-proxy
> }
Created symlink /etc/systemd/system/multi-user.target.wants/kubelet.service → /etc/systemd/system/kubelet.service.
Created symlink /etc/systemd/system/multi-user.target.wants/kube-proxy.service → /etc/systemd/system/kube-proxy.service.
root@gunwoo-k8s-hardway-worker2:~# 
root@gunwoo-auth-basic:~/hardway/cert# kubectl get node
NAME                          STATUS   ROLES    AGE     VERSION
gunwoo-k8s-hardway-worker-1   Ready    <none>   95m     v1.19.16
gunwoo-k8s-hardway-worker2    Ready    <none>   2m35s   v1.19.16

worker2 확인

네트워크 공부: https://ssup2.github.io

profile
gununoo
take a look
이전 포스트

[Kubernetes] etcd

다음 포스트

[Kubernetes] Monitoring

0개의 댓글