개요
rook ceph를 K8s 에서 Ceph RBD, CephFS 로 사용해보자.
rook ceph 에서는 Ceph RBD, CephFS에 대한 예제 yaml 파일을 제공한다.
이 예제 파일을 이용하여 테스트 해보자.
Block 볼륨 생성 테스트
- pool 생성
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl apply -f pool.yaml
cephblockpool.ceph.rook.io/replicapool created
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl get CephBlockPool -n=rook-ceph
NAME PHASE TYPE FAILUREDOMAIN AGE
replicapool Ready Replicated host 2m37s
- 스토리지 클래스 생성
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl apply -f csi/rbd/storageclass.yaml
cephblockpool.ceph.rook.io/replicapool created
storageclass.storage.k8s.io/rook-ceph-block created
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
rook-ceph-block rook-ceph.rbd.csi.ceph.com Delete Immediate true 15s
- pvc 생성
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl apply -f csi/rbd/pvc.yaml
persistentvolumeclaim/rbd-pvc created
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS VOLUMEATTRIBUTESCLASS AGE
rbd-pvc Bound pvc-37f75c66-9351-4d1c-9a98-0a608cc4932a 1Gi RWO rook-ceph-block <unset> 11s
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl describe pvc rbd-pvc
Name: rbd-pvc
Namespace: default
StorageClass: rook-ceph-block
Status: Bound
Volume: pvc-37f75c66-9351-4d1c-9a98-0a608cc4932a
Labels: <none>
Annotations: pv.kubernetes.io/bind-completed: yes
pv.kubernetes.io/bound-by-controller: yes
volume.beta.kubernetes.io/storage-provisioner: rook-ceph.rbd.csi.ceph.com
volume.kubernetes.io/storage-provisioner: rook-ceph.rbd.csi.ceph.com
Finalizers: [kubernetes.io/pvc-protection]
Capacity: 1Gi
Access Modes: RWO
VolumeMode: Filesystem
Used By: <none>
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Provisioning 58s rook-ceph.rbd.csi.ceph.com_csi-rbdplugin-provisioner-56bf9c67dd-trgl2_f093a004-4512-4ad7-b0b2-d79dc6d87c15 External provisioner is provisioning volume for claim "default/rbd-pvc"
Normal ExternalProvisioning 58s (x2 over 58s) persistentvolume-controller Waiting for a volume to be created either by the external provisioner 'rook-ceph.rbd.csi.ceph.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered.
Normal ProvisioningSucceeded 58s rook-ceph.rbd.csi.ceph.com_csi-rbdplugin-provisioner-56bf9c67dd-trgl2_f093a004-4512-4ad7-b0b2-d79dc6d87c15 Successfully provisioned volume pvc-37f75c66-9351-4d1c-9a98-0a608cc4932a
- pod 생성
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# cat csi/rbd/pod.yaml
---
apiVersion: v1
kind: Pod
metadata:
name: csirbd-demo-pod
spec:
containers:
- name: web-server
image: nginx
volumeMounts:
- name: mypvc
mountPath: /var/lib/www/html
volumes:
- name: mypvc
persistentVolumeClaim:
claimName: rbd-pvc
readOnly: false
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl apply -f csi/rbd/pod.yaml
pod/csirbd-demo-pod created
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl get pod
NAME READY STATUS RESTARTS AGE
csirbd-demo-pod 1/1 Running 0 60s
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl describe pod csirbd-demo-pod |tail -3
Normal Pulled 2m38s kubelet Successfully pulled image "nginx" in 1.893s (1.893s including waiting)
Normal Created 2m38s kubelet Created container web-server
Normal Started 2m37s kubelet Started container web-server
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl exec -it csirbd-demo-pod -- df -h /var/lib/www/html
Filesystem Size Used Avail Use% Mounted on
/dev/rbd0 974M 24K 958M 1% /var/lib/www/html
CephFS 볼륨 생성 테스트
- filesystem pool 생성
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl apply -f filesystem.yaml
cephfilesystem.ceph.rook.io/myfs created
cephfilesystemsubvolumegroup.ceph.rook.io/myfs-csi created
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl get -n=rook-ceph cephfilesystem
NAME ACTIVEMDS AGE PHASE
myfs 1 20s Progressing
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl get -n=rook-ceph cephfilesystemsubvolumegroup
NAME PHASE FILESYSTEM QUOTA AGE
myfs-csi Progressing myfs 30s- storage class 생성
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl apply -f csi/cephfs/storageclass.yaml
storageclass.storage.k8s.io/rook-cephfs created
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
rook-ceph-block rook-ceph.rbd.csi.ceph.com Delete Immediate true 2d1h
rook-cephfs rook-ceph.cephfs.csi.ceph.com Delete Immediate true 110s- pvc 생성
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# cd csi/cephfs/
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples/csi/cephfs# kubectl apply -f pvc.yaml
persistentvolumeclaim/cephfs-pvc created
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples/csi/cephfs# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS VOLUMEATTRIBUTESCLASS AGE
cephfs-pvc Pending rook-cephfs <unset> 43s
rbd-pvc Bound pvc-37f75c66-9351-4d1c-9a98-0a608cc4932a 1Gi RWO rook-ceph-block <unset> 2d1h
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples/csi/cephfs# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS VOLUMEATTRIBUTESCLASS AGE
cephfs-pvc Bound pvc-6530511b-d710-4cba-b80b-5ed31c3c20db 1Gi RWO rook-cephfs <unset> 38m
rbd-pvc Bound pvc-37f75c66-9351-4d1c-9a98-0a608cc4932a 1Gi RWO rook-ceph-block <unset> 2d1h
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples/csi/cephfs# kubectl get pv |grep rook-cephfs
pvc-6530511b-d710-4cba-b80b-5ed31c3c20db 1Gi RWO Delete Bound default/cephfs-pvc rook-cephfs <unset> 2d1h
- pod 생성
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples/csi/cephfs# cat pod.yaml
---
apiVersion: v1
kind: Pod
metadata:
name: csicephfs-demo-pod
spec:
containers:
- name: web-server
image: nginx
volumeMounts:
- name: mypvc
mountPath: /var/lib/www/html
volumes:
- name: mypvc
persistentVolumeClaim:
claimName: cephfs-pvc
readOnly: false
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples/csi/cephfs# kubectl apply -f pod.yaml
pod/csicephfs-demo-pod created
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples/csi/cephfs# kubectl get pod
NAME READY STATUS RESTARTS AGE
csicephfs-demo-pod 1/1 Running 0 38s
csirbd-demo-pod 1/1 Running 0 4d2h
root@kjh-m1:/opt/rook-ceph/rook/deploy/examples/csi/cephfs# kubectl exec -it csicephfs-demo-pod -- df -Th
Filesystem Type Size Used Avail Use% Mounted on
overlay overlay 49G 20G 29G 42% /
tmpfs tmpfs 64M 0 64M 0% /dev
/dev/vda1 ext4 49G 20G 29G 42% /etc/hosts
shm tmpfs 64M 0 64M 0% /dev/shm
172.21.1.79:6789,172.21.160.204:6789,172.21.254.71:6789:/volumes/csi/csi-vol-9a23dc4d-c4c8-4c19-8e5b-78bad9265d9e/cc8a3685-b0e5-459f-9f02-cfa34d363e72 ceph 1.0G 0 1.0G 0% /var/lib/www/html
tmpfs tmpfs 7.3G 12K 7.3G 1% /run/secrets/kubernetes.io/serviceaccount
tmpfs tmpfs 3.9G 0 3.9G 0% /proc/acpi
tmpfs tmpfs 3.9G 0 3.9G 0% /proc/scsi
tmpfs tmpfs 3.9G 0 3.9G 0% /sys/firmware
