ClusterIP
사용자가 특정 IP를 할당하지 않으면, 자동으로 available range에서 IP 할당됨
available range : 10.233.0.0 ~ 20.233.63.255
▶ 서비스 할당 IP 대역과 Pod IP 대역 확인
kubespray 설정 값 확인
[root@master ~/kube/08]# cd ~/kubespray/inventory/mycluster/group_vars/k8s_cluster/
[root@master ~/kubespray/inventory/mycluster/group_vars/k8s_cluster]# ls
addons.yml k8s-net-calico.yml k8s-net-flannel.yml k8s-net-kube-router.yml k8s-net-weave.yml
k8s-cluster.yml k8s-net-cilium.yml k8s-net-kube-ovn.yml k8s-net-macvlan.yml
[root@master ~/kubespray/inventory/mycluster/group_vars/k8s_cluster]# cat k8s-cluster.yml | egrep "kube_service|kube_pods"
kube_service_addresses: 10.233.0.0/18
kube_pods_subnet: 10.233.64.0/18
# - kube_pods_subnet: 10.233.64.0/18
# - kube_pods_subnet: 10.233.64.0/18
kube_service_addresses_ipv6: fd85:ee78:d8a6:8607::1000/116
kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112
kube_apiserver_ip: "{{ kube_service_addresses | ipaddr('net') | ipaddr(1) | ipaddr('address') }}"
skydns_server: "{{ kube_service_addresses | ipaddr('net') | ipaddr(3) | ipaddr('address') }}"
skydns_server_secondary: "{{ kube_service_addresses | ipaddr('net') | ipaddr(4) | ipaddr('address') }}"
# kubelet_runtime_cgroups: "/{{ kube_service_cgroups }}/{{ container_manager }}.service"
# kubelet_kubelet_cgroups: "/{{ kube_service_cgroups }}/kubelet.service"
[root@master ~]# ip address
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:54:ee:2e brd ff:ff:ff:ff:ff:ff
altname enp2s1
inet 192.168.2.60/24 brd 192.168.2.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe54:ee2e/64 scope link noprefixroute
valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
link/ether b2:5c:bc:72:b4:f0 brd ff:ff:ff:ff:ff:ff
inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
valid_lft forever preferred_lft forever
4: nodelocaldns: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN group default
link/ether 26:44:dd:45:bb:7b brd ff:ff:ff:ff:ff:ff
inet 169.254.25.10/32 scope global nodelocaldns
valid_lft forever preferred_lft forever
5: kube-ipvs0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN group default
link/ether de:ec:49:cd:4e:de brd ff:ff:ff:ff:ff:ff
inet 10.233.0.3/32 scope global kube-ipvs0
valid_lft forever preferred_lft forever
inet 10.233.0.1/32 scope global kube-ipvs0
valid_lft forever preferred_lft forever
6: calidcc8a760e41@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet6 fe80::ecee:eeff:feee:eeee/64 scope link
valid_lft forever preferred_lft forever
7: calia42ce326ad9@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
inet6 fe80::ecee:eeff:feee:eeee/64 scope link
valid_lft forever preferred_lft forever
10: vxlan.calico: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default
link/ether 66:4f:26:ae:af:db brd ff:ff:ff:ff:ff:ff
inet 10.233.97.128/32 scope global vxlan.calico
valid_lft forever preferred_lft forever
inet6 fe80::644f:26ff:feae:afdb/64 scope link
valid_lft forever preferred_lft forever
[root@master ~]# kubectl get svc -A
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default kubernetes ClusterIP 10.233.0.1 <none> 443/TCP 4h22m
kube-system coredns ClusterIP 10.233.0.3 <none> 53/UDP,53/TCP,9153/TCP 29h
[root@master ~]# kubectl describe svc kubernetes
Name: kubernetes
Namespace: default
Labels: component=apiserver
provider=kubernetes
Annotations: <none>
Selector: <none>
Type: ClusterIP
IP Family Policy: SingleStack
IP Families: IPv4
IP: 10.233.0.1
IPs: 10.233.0.1
Port: https 443/TCP
TargetPort: 6443/TCP
Endpoints: 192.168.2.60:6443
Session Affinity: None
Events: <none>
[root@master ~]# kubectl get pod -o wide -A | grep 192.168.2.60
kube-system calico-node-hbv94 1/1 Running 1 (6h35m ago) 29h 192.168.2.60 master <none> <none>
kube-system kube-apiserver-master 1/1 Running 2 (6h35m ago) 29h 192.168.2.60 master <none> <none>
kube-system kube-controller-manager-master 1/1 Running 3 (6h35m ago) 29h 192.168.2.60 master <none> <none>
kube-system kube-proxy-trxht 1/1 Running 1 (6h35m ago) 29h 192.168.2.60 master <none> <none>
kube-system kube-scheduler-master 1/1 Running 3 (6h35m ago) 29h 192.168.2.60 master <none> <none>
kube-system nodelocaldns-7lp84 1/1 Running 2 (6h35m ago) 29h 192.168.2.60 master <none> <none>▶ vxlan.calico :: pod와 네트워크 통신하는 장치
▶ kube-ipvs0 :: 서비스가 사용하는 IP 주소 → kube-system/coredns와 default/kubernetes가 사용중
ClusterIP Service 생성
[root@master ~/kube/08/clusterip]# vi clusterip-test.yaml
[root@master ~/kube/08/clusterip]# cat clusterip-test.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: nginx-sts
spec:
replicas: 3
selector:
matchLabels:
app: webui
template:
metadata:
name: nginx-pod
labels:
app: webui
spec:
containers:
- name: nginx-container
image: nginx:1.14
---
apiVersion: v1
kind: Service
metadata:
name: nginx-clusterip
spec:
selector:
app: webui
ports:
- protocol: TCP
port: 80
targetPort: 80
[root@master ~/kube/08/clusterip]# kubectl apply -f clusterip-test.yaml
statefulset.apps/nginx-sts created
service/nginx-clusterip created
[root@master ~/kube/08/clusterip]# kubectl get all
NAME READY STATUS RESTARTS AGE
pod/nginx-sts-0 1/1 Running 0 5s
pod/nginx-sts-1 1/1 Running 0 3s
pod/nginx-sts-2 0/1 ContainerCreating 0 1s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes ClusterIP 10.233.0.1 <none> 443/TCP 4h37m
service/nginx-clusterip ClusterIP 10.233.41.245 <none> 80/TCP 5s
NAME READY AGE
statefulset.apps/nginx-sts 2/3 5s
[root@master ~/kube/08/clusterip]# kubectl get svc -o wide
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
kubernetes ClusterIP 10.233.0.1 <none> 443/TCP 4h39m <none>
nginx-clusterip ClusterIP 10.233.41.245 <none> 80/TCP 73s app=webui
[root@master ~/kube/08/clusterip]# ip address show kube-ipvs0
[root@master ~/kube/08/clusterip]# ip address show kube-ipvs0
5: kube-ipvs0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN group default
link/ether de:ec:49:cd:4e:de brd ff:ff:ff:ff:ff:ff
inet 10.233.0.3/32 scope global kube-ipvs0
valid_lft forever preferred_lft forever
inet 10.233.0.1/32 scope global kube-ipvs0
valid_lft forever preferred_lft forever
inet 10.233.41.245/32 scope global kube-ipvs0
valid_lft forever preferred_lft forevernode1에서 확인
[root@node1 /etc/kubernetes/manifests]# ip address show kube-ipvs0
5: kube-ipvs0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN group default
link/ether f6:6c:1c:e0:4e:1c brd ff:ff:ff:ff:ff:ff
inet 10.233.0.3/32 scope global kube-ipvs0
valid_lft forever preferred_lft forever
inet 10.233.0.1/32 scope global kube-ipvs0
valid_lft forever preferred_lft forever
inet 10.233.41.245/32 scope global kube-ipvs0
valid_lft forever preferred_lft forever▶ master에서 확인한 바와 동일
Deploy 생성
[root@master ~/kube/08/clusterip]# vi nginx-deploy.yaml
[root@master ~/kube/08/clusterip]# cat nginx-deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deploy
spec:
replicas: 3
selector:
matchLabels:
app: webui
template:
metadata:
name: nginx-pod
labels:
app: webui
spec:
containers:
- name: nginx-container
image: nginx:1.14
[root@master ~/kube/08/clusterip]# kubectl get all
NAME READY STATUS RESTARTS AGE
pod/nginx-deploy-9cc457697-f2dc2 1/1 Running 0 19s
pod/nginx-deploy-9cc457697-gvrnn 1/1 Running 0 19s
pod/nginx-deploy-9cc457697-s4tgm 1/1 Running 0 19s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes ClusterIP 10.233.0.1 <none> 443/TCP 4h41m
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/nginx-deploy 3/3 3 3 19s
NAME DESIRED CURRENT READY AGE
replicaset.apps/nginx-deploy-9cc457697 3 3 3 19s
Service 생성
[root@master ~/kube/08/clusterip]# vi clusterip-svc.yaml
[root@master ~/kube/08/clusterip]# cat clusterip-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx-clusterip
spec:
selector:
app: webui
ports:
- protocol: TCP
port: 80
targetPort: 80
[root@master ~/kube/08/clusterip]# kubectl apply -f clusterip-svc.yaml
service/nginx-clusterip created
[root@master ~/kube/08/clusterip]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.233.0.1 <none> 443/TCP 4h44m
nginx-clusterip ClusterIP 10.233.5.174 <none> 80/TCP 2s
[root@master ~/kube/08/clusterip]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.233.0.1 <none> 443/TCP 4h44m
nginx-clusterip ClusterIP 10.233.5.174 <none> 80/TCP 2s
[root@master ~/kube/08/clusterip]# kubectl describe svc nginx-clusterip
Name: nginx-clusterip
Namespace: default
Labels: <none>
Annotations: <none>
Selector: app=webui
Type: ClusterIP
IP Family Policy: SingleStack
IP Families: IPv4
IP: 10.233.5.174
IPs: 10.233.5.174
Port: <unset> 80/TCP
TargetPort: 80/TCP
Endpoints: 10.233.102.170:80,10.233.71.37:80,10.233.75.59:80
Session Affinity: None
Events: <none>
[root@master ~/kube/08/clusterip]# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-deploy-9cc457697-f2dc2 1/1 Running 0 4m9s 10.233.102.170 node1 <none> <none>
nginx-deploy-9cc457697-gvrnn 1/1 Running 0 4m9s 10.233.71.37 node3 <none> <none>
nginx-deploy-9cc457697-s4tgm 1/1 Running 0 4m9s 10.233.75.59 node2 <none> <none>
▶ Endpoints가 위 생성된 Deploy의 Pod로 되어있음을 확인할 수 있음
각 pod에 개별적 작업
[root@master ~/kube/08/clusterip]# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-deploy-9cc457697-f2dc2 1/1 Running 0 4m59s 10.233.102.170 node1 <none> <none>
nginx-deploy-9cc457697-gvrnn 1/1 Running 0 4m59s 10.233.71.37 node3 <none> <none>
nginx-deploy-9cc457697-s4tgm 1/1 Running 0 4m59s 10.233.75.59 node2 <none> <none>
[root@master ~/kube/08/clusterip]# kubectl exec nginx-deploy-9cc457697-f2dc2 -it -- bash
root@nginx-deploy-9cc457697-f2dc2:/# echo "<h1>#1# Test Page</h1>" > /usr/share/nginx/html/index.html
root@nginx-deploy-9cc457697-f2dc2:/# exit
exit
[root@master ~/kube/08/clusterip]# kubectl exec nginx-deploy-9cc457697-gvrnn -it -- bash
root@nginx-deploy-9cc457697-gvrnn:/# echo "<h1>#2# Test Page</h1>" > /usr/share/nginx/html/index.html
root@nginx-deploy-9cc457697-gvrnn:/# exit
exit
[root@master ~/kube/08/clusterip]# kubectl exec nginx-deploy-9cc457697-s4tgm -it -- bash
root@nginx-deploy-9cc457697-s4tgm:/# echo "<h1>#3# Test Page</h1>" > /usr/share/nginx/html/index.html
root@nginx-deploy-9cc457697-s4tgm:/# exit
exit
replicas scale up 3 to 5
[root@master ~/kube/08/clusterip]# kubectl scale deployment nginx-deploy --replicas=5
deployment.apps/nginx-deploy scaled
[root@master ~/kube/08/clusterip]# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-deploy-9cc457697-d5hjg 1/1 Running 0 77s 10.233.75.60 node2 <none> <none>
nginx-deploy-9cc457697-f2dc2 1/1 Running 0 13m 10.233.102.170 node1 <none> <none>
nginx-deploy-9cc457697-gvrnn 1/1 Running 0 13m 10.233.71.37 node3 <none> <none>
nginx-deploy-9cc457697-pb5tm 1/1 Running 0 77s 10.233.102.171 node1 <none> <none>
nginx-deploy-9cc457697-s4tgm 1/1 Running 0 13m 10.233.75.59 node2 <none> <none>
[root@master ~/kube/08/clusterip]# kubectl get ep
NAME ENDPOINTS AGE
kubernetes 192.168.2.60:6443 4h54m
nginx-clusterip 10.233.102.170:80,10.233.102.171:80,10.233.71.37:80 + 2 more... 10m새로 생긴 pod에도 동일한 작업 수행
[root@master ~/kube/08/clusterip]# kubectl exec nginx-deploy-9cc457697-pb5tm -it -- bash
root@nginx-deploy-9cc457697-pb5tm:/# echo "<h1>#4# Test Page</h1>" > /usr/share/nginx/html/index.html
root@nginx-deploy-9cc457697-pb5tm:/# exit
exit
[root@master ~/kube/08/clusterip]# kubectl exec nginx-deploy-9cc457697-d5hjg -it -- bash
root@nginx-deploy-9cc457697-d5hjg:/# echo "<h1>#5# Test Page</h1>" > /usr/share/nginx/html/index.html
root@nginx-deploy-9cc457697-d5hjg:/# exit
exit실제로 서비스가 원활히 가는지 확인
[root@master ~/kube/08/clusterip]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.233.0.1 <none> 443/TCP 4h56m
nginx-clusterip ClusterIP 10.233.5.174 <none> 80/TCP 11m
[root@master ~/kube/08/clusterip]# curl <http://10.233.5.174>
<h1>#4# Test Page</h1>
[root@master ~/kube/08/clusterip]# curl <http://10.233.5.174>
<h1>#5# Test Page</h1>
[root@master ~/kube/08/clusterip]# curl <http://10.233.5.174>
<h1>#3# Test Page</h1>
[root@master ~/kube/08/clusterip]# curl <http://10.233.5.174>
<h1>#2# Test Page</h1>
[root@master ~/kube/08/clusterip]# curl <http://10.233.5.174>
<h1>#1# Test Page</h1>replicas scale down 5 to 3
[root@master ~/kube/08/clusterip]# kubectl scale deployment nginx-deploy --replicas=3
deployment.apps/nginx-deploy scaled