[k8s] 트러블 슈팅 기본 프로세스

일반적인 쿠버네티스 트러블 슈팅 작업 순서

Apply -> Get -> Describe -> Logs -> Get Event

1. yaml 파일을 이용해 오브젝트 생성(apply)하고 생성한 오브젝트 리스트는 get 명령어로 확인
2. 파드가 정상적으로 생성되지 않은면 상세한 설정 정보를 describe 명령어로 확인
3. 애플리케이션 관련 에러는 logs 로 확인하고 클러스터 관련 메시지는 get event로 확인

실습

잘못된 이미지 nginx 파드 배포 후 에러 확인

[root@k8s-master01 ~]# cat nginx-error-pod.yml

apiVersion: v1
kind: Pod
metadata:
  name: nginx-19
spec:
  containers:
  - name: nginx-pod
    image: nginx:1.19.19  # 존재하지 않는 이미지 버전입니다.

apply / get

[root@k8s-master01 ~]# kubectl apply -f nginx-error-pod.yml 
pod/nginx-19 created
[root@k8s-master01 ~]# kubectl get pod -o wide
NAME       READY   STATUS         RESTARTS   AGE   IP            NODE           NOMINATED NODE   READINESS GATES
busybox    1/1     Running        0          77m   10.1.79.74    k8s-worker01   <none>           <none>
nginx-19   0/1     ErrImagePull   0          9s    10.1.69.204   k8s-worker02   <none>           <none>

describe

[root@k8s-master01 ~]# kubectl describe pod nginx-19
Name:             nginx-19
Namespace:        default
Priority:         0
Service Account:  default
Node:             k8s-worker02/172.16.3.101
Start Time:       Mon, 17 Jun 2024 15:01:47 +0900
Labels:           <none>
Annotations:      cni.projectcalico.org/containerID: dedb0f03b21495d2414ded3cba9e4f453fd91224afa3e69b11ff8c97b30241f6
                  cni.projectcalico.org/podIP: 10.1.69.204/32
                  cni.projectcalico.org/podIPs: 10.1.69.204/32
Status:           Pending
IP:               10.1.69.204
IPs:
  IP:  10.1.69.204
Containers:
  nginx-pod:
    Container ID:   
    Image:          nginx:1.19.19
    Image ID:       
    Port:           <none>
    Host Port:      <none>
    State:          Waiting
      Reason:       ErrImagePull
    Ready:          False
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-hdwc8 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  kube-api-access-hdwc8:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason     Age                From               Message
  ----     ------     ----               ----               -------
  Normal   Scheduled  64s                default-scheduler  Successfully assigned default/nginx-19 to k8s-worker02
  Normal   Pulling    15s (x3 over 63s)  kubelet            Pulling image "nginx:1.19.19"
  Warning  Failed     14s (x3 over 62s)  kubelet            Failed to pull image "nginx:1.19.19": rpc error: code = NotFound desc = failed to pull and unpack image "docker.io/library/nginx:1.19.19": failed to resolve reference "docker.io/library/nginx:1.19.19": docker.io/library/nginx:1.19.19: not found
  Warning  Failed     14s (x3 over 62s)  kubelet            Error: ErrImagePull
  Normal   BackOff    2s (x3 over 61s)   kubelet            Back-off pulling image "nginx:1.19.19"
  Warning  Failed     2s (x3 over 61s)   kubelet            Error: ImagePullBackOff

logs

[root@k8s-master01 ~]# kubectl logs -f nginx-19 
Error from server (BadRequest): container "nginx-pod" in pod "nginx-19" is waiting to start: trying and failing to pull image

get event
클러스터 레벨에서 이벤트 확인

[root@k8s-master01 ~]# kubectl get events
LAST SEEN   TYPE      REASON      OBJECT         MESSAGE
11m         Normal    Scheduled   pod/nginx-19   Successfully assigned default/nginx-19 to k8s-worker02
10m         Normal    Pulling     pod/nginx-19   Pulling image "nginx:1.19.19"
10m         Warning   Failed      pod/nginx-19   Failed to pull image "nginx:1.19.19": rpc error: code = NotFound desc = failed to pull and unpack image "docker.io/library/nginx:1.19.19": failed to resolve reference "docker.io/library/nginx:1.19.19": docker.io/library/nginx:1.19.19: not found
10m         Warning   Failed      pod/nginx-19   Error: ErrImagePull
95s         Normal    BackOff     pod/nginx-19   Back-off pulling image "nginx:1.19.19"
9m47s       Warning   Failed      pod/nginx-19   Error: ImagePullBackOff

네임스페이스별 이벤트

• -n 옵션으로 네임스페이스 지정
• -A 옵션으로 전체 네임스페이스

[root@k8s-master01 ~]# kubectl get events -n kube-system
No resources found in kube-system namespace.

[root@k8s-master01 ~]# kubectl get events -A
NAMESPACE   LAST SEEN   TYPE      REASON      OBJECT         MESSAGE
default     13m         Normal    Scheduled   pod/nginx-19   Successfully assigned default/nginx-19 to k8s-worker02
default     11m         Normal    Pulling     pod/nginx-19   Pulling image "nginx:1.19.19"
default     11m         Warning   Failed      pod/nginx-19   Failed to pull image "nginx:1.19.19": rpc error: code = NotFound desc = failed to pull and unpack image "docker.io/library/nginx:1.19.19": failed to resolve reference "docker.io/library/nginx:1.19.19": docker.io/library/nginx:1.19.19: not found
default     11m         Warning   Failed      pod/nginx-19   Error: ErrImagePull
default     3m25s       Normal    BackOff     pod/nginx-19   Back-off pulling image "nginx:1.19.19"
default     11m         Warning   Failed      pod/nginx-19   Error: ImagePullBackOff