Rook Ceph 설치하기

Kaite.Kang·2025년 3월 30일

환경

root@kjh-m1:~# kubectl get nodes
NAME     STATUS   ROLES           AGE    VERSION
kjh-m1   Ready    control-plane   137d   v1.29.7
kjh-w1   Ready    <none>          137d   v1.29.7
kjh-w2   Ready    <none>          137d   v1.29.7

root@kjh-m1:~# kubectl describe node kjh-m1 |grep -i taint
Taints:             <none>

노드별 스펙

kjh-m1	control-plane	4 core	8 GiB	Ubuntu 22.04.05 LTS
kjh-w1	worker	4 core	8 GiB	Ubuntu 22.04.05 LTS
kjh-w2	worker	4 core	8 GiB	Ubuntu 22.04.05 LTS

rook ceph 버전: v1.16

작업 준비

vm에 디스크 추가

# ansible all -m shell -a "lsblk /dev/vdb" -i inventory/bee/inventory.ini
[WARNING]: Skipping callback plugin 'ara_default', unable to load
kjh-w1 | CHANGED | rc=0 >>
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
vdb  252:16   0  20G  0 disk
kjh-m1 | CHANGED | rc=0 >>
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
vdb  252:16   0  10G  0 disk
kjh-w2 | CHANGED | rc=0 >>
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
vdb  252:16   0  10G  0 disk

rook-ceph 설치

1. 파일 다운로드

# mkdir /opt/rook-ceph
# cd /opt/rook-ceph
# git clone https://github.com/rook/rook.git -b release-1.16
Cloning into 'rook'...
remote: Enumerating objects: 141628, done.
remote: Counting objects: 100% (29/29), done.
remote: Compressing objects: 100% (20/20), done.
remote: Total 141628 (delta 15), reused 9 (delta 9), pack-reused 141599 (from 2)
Receiving objects: 100% (141628/141628), 65.45 MiB | 18.20 MiB/s, done.
Resolving deltas: 100% (98501/98501), done.

2. rook-ceph cluster 설치

사전 설치

# cd rook/deploy/examples/
# kubectl create -f crds.yaml -f common.yaml -f operator.yaml
customresourcedefinition.apiextensions.k8s.io/cephblockpoolradosnamespaces.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephblockpools.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephbucketnotifications.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephbuckettopics.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephclients.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephclusters.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephcosidrivers.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephfilesystemmirrors.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephfilesystems.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephfilesystemsubvolumegroups.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephnfses.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephobjectrealms.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephobjectstores.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephobjectstoreusers.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephobjectzonegroups.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephobjectzones.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephrbdmirrors.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/objectbucketclaims.objectbucket.io created
customresourcedefinition.apiextensions.k8s.io/objectbuckets.objectbucket.io created
namespace/rook-ceph created
clusterrole.rbac.authorization.k8s.io/cephfs-csi-nodeplugin created
clusterrole.rbac.authorization.k8s.io/cephfs-external-provisioner-runner created
clusterrole.rbac.authorization.k8s.io/objectstorage-provisioner-role created
clusterrole.rbac.authorization.k8s.io/rbd-csi-nodeplugin created
clusterrole.rbac.authorization.k8s.io/rbd-external-provisioner-runner created
clusterrole.rbac.authorization.k8s.io/rook-ceph-cluster-mgmt created
clusterrole.rbac.authorization.k8s.io/rook-ceph-global created
clusterrole.rbac.authorization.k8s.io/rook-ceph-mgr-cluster created
clusterrole.rbac.authorization.k8s.io/rook-ceph-mgr-system created
clusterrole.rbac.authorization.k8s.io/rook-ceph-object-bucket created
clusterrole.rbac.authorization.k8s.io/rook-ceph-osd created
clusterrole.rbac.authorization.k8s.io/rook-ceph-system created
clusterrolebinding.rbac.authorization.k8s.io/cephfs-csi-nodeplugin-role created
clusterrolebinding.rbac.authorization.k8s.io/cephfs-csi-provisioner-role created
clusterrolebinding.rbac.authorization.k8s.io/objectstorage-provisioner-role-binding created
clusterrolebinding.rbac.authorization.k8s.io/rbd-csi-nodeplugin created
clusterrolebinding.rbac.authorization.k8s.io/rbd-csi-provisioner-role created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-global created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-mgr-cluster created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-object-bucket created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-osd created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-system created
role.rbac.authorization.k8s.io/cephfs-external-provisioner-cfg created
role.rbac.authorization.k8s.io/rbd-csi-nodeplugin created
role.rbac.authorization.k8s.io/rbd-external-provisioner-cfg created
role.rbac.authorization.k8s.io/rook-ceph-cmd-reporter created
role.rbac.authorization.k8s.io/rook-ceph-mgr created
role.rbac.authorization.k8s.io/rook-ceph-osd created
role.rbac.authorization.k8s.io/rook-ceph-purge-osd created
role.rbac.authorization.k8s.io/rook-ceph-system created
rolebinding.rbac.authorization.k8s.io/cephfs-csi-provisioner-role-cfg created
rolebinding.rbac.authorization.k8s.io/rbd-csi-nodeplugin-role-cfg created
rolebinding.rbac.authorization.k8s.io/rbd-csi-provisioner-role-cfg created
rolebinding.rbac.authorization.k8s.io/rook-ceph-cluster-mgmt created
rolebinding.rbac.authorization.k8s.io/rook-ceph-cmd-reporter created
rolebinding.rbac.authorization.k8s.io/rook-ceph-mgr created
rolebinding.rbac.authorization.k8s.io/rook-ceph-mgr-system created
rolebinding.rbac.authorization.k8s.io/rook-ceph-osd created
rolebinding.rbac.authorization.k8s.io/rook-ceph-purge-osd created
rolebinding.rbac.authorization.k8s.io/rook-ceph-system created
serviceaccount/objectstorage-provisioner created
serviceaccount/rook-ceph-cmd-reporter created
serviceaccount/rook-ceph-default created
serviceaccount/rook-ceph-mgr created
serviceaccount/rook-ceph-osd created
serviceaccount/rook-ceph-purge-osd created
serviceaccount/rook-ceph-rgw created
serviceaccount/rook-ceph-system created
serviceaccount/rook-csi-cephfs-plugin-sa created
serviceaccount/rook-csi-cephfs-provisioner-sa created
serviceaccount/rook-csi-rbd-plugin-sa created
serviceaccount/rook-csi-rbd-provisioner-sa created
configmap/rook-ceph-operator-config created
deployment.apps/rook-ceph-operator created

실행 결과
- rook-ceph 네임스페이스 아래 리소스가 생성됨.

root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl get ns rook-ceph
NAME        STATUS   AGE
rook-ceph   Active   15s

root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl get all -n=rook-ceph
NAME                                      READY   STATUS    RESTARTS   AGE
pod/rook-ceph-operator-544b5d7d6c-87ck5   1/1     Running   0          20s

NAME                                 READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/rook-ceph-operator   1/1     1            1           20s

NAME                                            DESIRED   CURRENT   READY   AGE
replicaset.apps/rook-ceph-operator-544b5d7d6c   1         1         1       20s

cluster.yaml 배포
- cluster.yaml 기본값으로 모든 노드에서 사용 가능한 모든 디바이스를 사용하도록 설정되어 있다. 파일을 수정하여 특정 노드의 특정 디바이스만 사용하도록 변경 한다.
- storage.nodes.name 필드의 값은 'kubernetes.io/hostname' 라벨의 값을 넣어주면 된다.

## 각 노드의 [kubernetes.io/hostname](http://kubernetes.io/hostname) 라벨 값 확인
# kubectl get nodes --show-labels |grep --color kubernetes.io/hostname
kjh-m1   Ready    control-plane   137d   v1.29.7   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=kjh-m1,kubernetes.io/os=linux,node-role.kubernetes.io/control-plane=,node.kubernetes.io/exclude-from-external-load-balancers=
kjh-w1   Ready    <none>          137d   v1.29.7   beesys=enabled,beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=kjh-w1,kubernetes.io/os=linux
kjh-w2   Ready    <none>          137d   v1.29.7   beesys=enabled,beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=kjh-w2,kubernetes.io/os=linux

## 원본 백업
# cp cluster.yaml cluster.yaml.org

## ceph 클러스터에 사용할 노드 및 디스크 설정
# vi cluster.yaml
  storage: # cluster level storage configuration and selection
    useAllNodes: false #true -> false 변경
    #useAllDevices: true #주석처리
    nodes:
      - name: kjh-m1
        devices:
          - name: "vdb"
      - name: kjh-w1
        devices:
          - name: "vdb"
      - name: kjh-w2
        devices:
          - name: "vdb"

# kubectl create -f cluster.yaml
cephcluster.ceph.rook.io/rook-ceph created

실행 결과 확인

root@kjh-m1:/opt/rook-ceph/rook/deploy/examples# kubectl get pod -n=rook-ceph
NAME                                            READY   STATUS    RESTARTS   AGE
csi-cephfsplugin-289hw                          3/3     Running   0          37s
csi-cephfsplugin-b7f47                          3/3     Running   0          37s
csi-cephfsplugin-provisioner-76fc854ddb-92v7t   6/6     Running   0          37s
csi-cephfsplugin-provisioner-76fc854ddb-jjhxg   6/6     Running   0          37s
csi-cephfsplugin-xpt5n                          3/3     Running   0          37s
csi-rbdplugin-drsmc                             3/3     Running   0          37s
csi-rbdplugin-nt44k                             3/3     Running   0          37s
csi-rbdplugin-pfx4h                             3/3     Running   0          37s
csi-rbdplugin-provisioner-56bf9c67dd-lpvk8      6/6     Running   0          37s
csi-rbdplugin-provisioner-56bf9c67dd-trgl2      6/6     Running   0          37s
rook-ceph-mon-a-75f4548f6d-529jf                2/2     Running   0          27s
rook-ceph-operator-544b5d7d6c-87ck5             1/1     Running   0          26m

3. rook ceph 명령어 도구 설치

# pwd
/opt/rook-ceph/rook/deploy/examples

# kubectl apply -f toolbox.yaml
deployment.apps/rook-ceph-tools created

확인

# kubectl get all -n=rook-ceph -l app=rook-ceph-tools
NAME                                   READY   STATUS    RESTARTS   AGE
pod/rook-ceph-tools-56fbc74755-t75c4   1/1     Running   0          5m58s

NAME                              READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/rook-ceph-tools   1/1     1            1           5m58s

NAME                                         DESIRED   CURRENT   READY   AGE
replicaset.apps/rook-ceph-tools-56fbc74755   1         1         1       5m58s

Kaite.Kang

이전 포스트

Sparse File: 파일 크기와 디스크 사용량이 다른 이유

다음 포스트

Rook Ceph 설치하기

환경

작업 준비

vm에 디스크 추가

rook-ceph 설치

1. 파일 다운로드

2. rook-ceph cluster 설치

3. rook ceph 명령어 도구 설치

Sparse File: 파일 크기와 디스크 사용량이 다른 이유

Kubernetes에서 Rook-Ceph로 Ceph RBD와 CephFS 활용하기

0개의 댓글