Kubernetes
Jenkins를 이용하기 위해 쿠버네티스 환경을 구축한다.
kubeadm을 이용한 쿠버네티스 클러스터링 설치
- swap 비활성화
swapoff -avi /etc/fstab가장 아래 /swapfile 부분 해시 처리
- 도커 설치 위한 GPG 키 다운
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -- 도커 리포지토리 추가
sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"- 쿠버네티스 설치 위한 GPG 키 다운
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -- k8s 저장소 추가
cat << EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
deb https://apt.kubernetes.io/ kubernetes-xenial main
EOF- 저장소 업데이트
sudo apt-get update- docker-ce, k8s 1.21 버전 설치
sudo apt-get install 0y docker-ce=5:20.10.7~3-0~ubuntu-$(lsb_release -cs) kubelet=1.21.1-00 kubeadm=1.21.1-00 kubectl=1.21.1-00- 업그레이드로 인한 버전업 방지
sudo apt-mark hold docker-ce kubelet kubeadm kubectl- master 노드에서 클러스터 시작하기
kubeadm init- 현재 사용자에게 클러스터 어드민 권한 부여
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
export KUBECONFIG=/etc/kubernetes/admin.conf- calico 클러스터 네트워크 배포
kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml- worker 노드 클러스터 조인
위 9번에서 출력된 토큰을 붙여넣는다
- 설치 후 확인
kubectl get no, nskubectl get pod -n kube-system모두 Running이 되어 있어야 한다.
Metal LB 사용
- metallb-system 네임스페이스 생성
kubectl create ns metallb-system- metallb apply
kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/metallb.yaml- metallb-config
apiVersion: v1
kind: ConfigMap
metadata:
namespace: metallb-system
name: config
data:
config: |
address-pools:
- name: default
protocol: layer2
addresses:
- 192.168.8.201-192.168.8.239- 배포하기
kubectl apply -f metallb-config.yaml- 확인
kubectl get deploy controller -n metallb-system -o wide- lb-nginx
apiVersion: v1
kind: Service
metadata:
name: nginxlb
labels:
app: nginx
spec:
externalTrafficPolicy: Local
ports:
- name: http
port: 80
protocol: TCP
targetPort: 80
selector:
app: nginx
type: LoadBalancer- lb nginx 배포
kubectl apply -f lb-nginx.yamlJenkins
젠킨스 컨트롤러-에이전트 구조
컨트롤러 : 젠킨스 자체의 관리 및 CI/CD 관련 설정 담당
에이전트 : build/deploy 담당 (이후 사라짐)
커스터마이즈로 MetalLB 한번에 만들기
파일 clone
git clone https://github.com/beomtaek/cicd_samplecode.git- kustomize-install.sh
#!/usr/bin/env bash
curl -L \
https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv3.6.1/kustomize_v3.6.1_linux_amd64.tar.gz -o /tmp/kustomize.tar.gz
tar -xzf /tmp/kustomize.tar.gz -C /usr/local/bin
echo "kustomize install successfully"Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 12.4M 100 12.4M 0 0 20.2M 0 --:--:-- --:--:-- --:--:-- 20.2M- clone받은 metallb.yaml, metallb-l2config.yaml namespace.yaml
-
namespace.yaml
apiVersion: v1 kind: Namespace metadata: name: metallb-system labels: app: metallb -
metallb-l2config.yaml
apiVersion: v1 kind: ConfigMap metadata: namespace: metallb-system name: config data: config: | address-pools: - name: metallb-ip-range protocol: layer2 addresses: - 192.168.8.201-192.168.8.239 -
metallb.yaml
apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: labels: app: metallb name: speaker namespace: metallb-system spec: allowPrivilegeEscalation: false allowedCapabilities: - NET_ADMIN - NET_RAW - SYS_ADMIN fsGroup: rule: RunAsAny hostNetwork: true hostPorts: - max: 7472 min: 7472 privileged: true runAsUser: rule: RunAsAny seLinux: rule: RunAsAny supplementalGroups: rule: RunAsAny volumes: - '*' --- apiVersion: v1 kind: ServiceAccount metadata: labels: app: metallb name: controller namespace: metallb-system --- apiVersion: v1 kind: ServiceAccount metadata: labels: app: metallb name: speaker namespace: metallb-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app: metallb name: metallb-system:controller rules: -
apiGroups:
- ''
resources: - services
verbs: - get
- list
- watch
- update
- ''
-
apiGroups:
- ''
resources: - services/status
verbs: - update
- ''
-
apiGroups:
- ''
resources: - events
verbs: - create
- patch
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app: metallb
name: metallb-system:speaker
rules: - ''
-
apiGroups:
- ''
resources: - services
- endpoints
- nodes
verbs: - get
- list
- watch
- ''
-
apiGroups:
- ''
resources: - events
verbs: - create
- patch
- ''
-
apiGroups:
- extensions
resourceNames: - speaker
resources: - podsecuritypolicies
verbs: - use
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
app: metallb
name: config-watcher
namespace: metallb-system
rules: - extensions
-
apiGroups:
- ''
resources: - configmaps
verbs: - get
- list
- watch
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app: metallb
name: metallb-system:controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: metallb-system:controller
subjects: - ''
-
kind: ServiceAccount
name: controllernamespace: metallb-system
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app: metallb
name: metallb-system:speaker
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: metallb-system:speaker
subjects: -
kind: ServiceAccount
name: speakernamespace: metallb-system
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app: metallb
name: config-watcher
namespace: metallb-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: config-watcher
subjects: -
kind: ServiceAccount
name: controller -
kind: ServiceAccount
name: speaker
apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
app: metallb
component: speaker
name: speaker
namespace: metallb-system
spec:
selector:
matchLabels:
app: metallb
component: speaker
template:
metadata:
annotations:
prometheus.io/port: '7472'
prometheus.io/scrape: 'true'
labels:
app: metallb
component: speaker
spec:
containers:
- args:
- --port=7472
- --config=config
env:
- name: METALLB_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: METALLB_HOST
valueFrom:
fieldRef:
fieldPath: status.hostIP
image: metallb/speaker:v0.8.2
imagePullPolicy: IfNotPresent
name: speaker
ports:
- containerPort: 7472
name: monitoring
resources:
limits:
cpu: 100m
memory: 100Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_ADMIN
- NET_RAW
- SYS_ADMIN
drop:
- ALL
readOnlyRootFilesystem: true
hostNetwork: true
nodeSelector:
beta.kubernetes.io/os: linux
serviceAccountName: speaker
terminationGracePeriodSeconds: 0
tolerations:
- effect: NoSchedulekey: node-role.kubernetes.io/master
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: metallb
component: controller
name: controller
namespace: metallb-system
spec:
revisionHistoryLimit: 3
selector:
matchLabels:
app: metallb
component: controller
template:
metadata:
annotations:
prometheus.io/port: '7472'
prometheus.io/scrape: 'true'
labels:
app: metallb
component: controller
spec:
containers:
- args:
- --port=7472
- --config=config
image: metallb/controller:v0.8.2
imagePullPolicy: IfNotPresent
name: controller
ports:
- containerPort: 7472
name: monitoring
resources:
limits:
cpu: 100m
memory: 100Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- all
readOnlyRootFilesystem: true
nodeSelector:
beta.kubernetes.io/os: linux
securityContext:
runAsNonRoot: true
runAsUser: 65534
serviceAccountName: controller
terminationGracePeriodSeconds: 0
-
- create하여 kustomization.yaml 파일 생성
root@master:~/lab2/cicd_samplecode/metallb# kustomize create --namespace=metallb-system --resources namespace.yaml,metallb.yaml,metallb-l2config.yaml
root@master:~/lab2/cicd_samplecode/metallb# ls
kustomization.yaml metallb-l2config.yaml namespace.yaml
kustomize-install.sh metallb.yaml- kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- metallb.yaml
- metallb-l2config.yaml
namespace: metallb-system- kustomize build 내용을 apply
kustomize build | kubectl apply -f -만약 인터넷이 느려서 controller, speaker 이미지가 잘 안받아 와진다면?
worker에서 각각
docker pull metallb/controller:v0.8.2docker pull metallb/speaker:v0.8.2로 수동으로 이미지를 다운받아준다.
- 다음 실습을 위해 삭제
kustomize build | kubectl delete -f -헬름으로 MetalLB 한번에 만들기
- 헬름 설치
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
chmod 700 get_helm.sh
export DESIRED_VERSION=v3.2.1
./get_helm.sh- metallb 설치 위해 주소 확인
metallb를 검색해 주소를 확인할 수 있다.
metallb 아이템을 클릭해 정보를 확인할 수 있다.
- 저장소 등록
helm repo add edu https://iac-source.github.io/helm-charts- 저장소 목록 확인
helm repo list- repo update
helm repo update- 차트 설치
helm install metallb edu/metallb \
--namespace=metallb-system
--create-namespace \
--set controller.tag=v0.8.3 \
--set speaker.tag=v0.8.3 \
--set configmap.ipRange=192.168.8.201-192.168.8.239헬름으로 젠킨스 설치
사설 저장소 구축하기 (registry)
- registry 설치
docker container run -d --restart=always --name registry -p 5000:5000 registry- web을 통해 확인할 수 있도록 web UI 설치
docker run -d -p 8888:8080 --restart always --name registry-web --link registry -e REGISTRY_URL=http://192.168.8.100:5000/v2 -e REGISTRY_NAME=192.168.8.100:5000 hyper/docker-registry-web8888 포트에서 노출되도록
현재 접속은 https가 아닌 http 접속이므로 이를 허용해 주어야 한다.
모든 노드에서 적용한다.
vi /etc/docker/daemon.json{
"insecure-registries" : ["192.168.8.100"]
}systemctl restart dockerworker에서 이미지 올리기
docker pull brian24/testweb:blue
docker pull brian24/testweb:green
docker image ls | grep brian24
docker image tag brian24/testweb:green 192.168.8.100:5000/green:1.0
docker image tag brian24/testweb:blue 192.168.8.100:5000/blue:1.0
docker push 192.168.8.100:5000/green:1.0
docker push 192.168.8.100:5000/blue:1.0NFS 설치
모든 노드에서
apt install -y nfs-commonmaster에서
apt install -y nfs-server
systemctl enable nfs-server
systemctl start nfs-server- nfs-exporter.sh
#!/usr/bin/env bash
nfsdir=/nfs_shared/$1
if [ $# -eq 0 ]; then
echo "usage: nfs-exporter.sh <name>"; exit 0
fi
if [[ ! -d $nfsdir ]]; then
mkdir -p $nfsdir
echo "$nfsdir 192.168.8.0/24(rw,sync,no_root_squash)" >> /etc/exports
if [[ $(systemctl is-enabled nfs) -eq "disabled" ]]; then
systemctl enable nfs-server
fi
systemctl restart nfs-server
fi헬름으로 설치되는 젠킨시는 파드에서 동작하기 때문에 PV를 마운트하지않으면 다시 시작될때 내부 볼륨 데이터가 삭제된다. 이를 방지하기 위해 NFS 디렉터리를 /nfs_shared/jenkins에 만든다.
root@master:~/lab2/cicd_samplecode# chmod +x nfs-exporter.sh
root@master:~/lab2/cicd_samplecode# ./nfs-exporter.sh jenkins
Failed to get unit file state for nfs.service: No such file or directory
root@master:~/lab2/cicd_samplecode# ./nfs-exporter.sh jenkins
root@master:~/lab2/cicd_samplecode# ls -l /nfs_shared/
total 4
drwxr-xr-x 2 root root 4096 10월 27 16:23 jenkins- 젠킨스 컨트롤러 이미지에서 기본적으로 사용하는 유저ID와 그룹ID가 1000번이기 때문에 접근 ID를 1000번으로 설정
chown 1000:1000 /nfs_shared/jenkins/- jenkins-volume.yaml
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: jenkins
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Retain
nfs:
server: 192.168.8.100
path: /nfs_shared/jenkins
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: jenkins
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 5Gikubectl apply -f jenkins-volume.yamlroot@master:~/lab2/cicd_samplecode# kubectl get pv,pvc
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
persistentvolume/jenkins 5Gi RWX Retain Bound default/jenkins 12s
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
persistentvolumeclaim/jenkins Bound jenkins 5Gi RWX 12s바운드 된 것을 볼 수 있다.
젠킨스는 사용자가 배포를 위해 생성한 내용, 사용자 계정 정보, 플러그인 같은 데이터를 저장하기 위해 PV, PVC 구성이 필요하다.
- jenkins-install.sh
#!/usr/bin/env bash
jkopt1="--sessionTimeout=1440"
jkopt2="--sessionEviction=86400"
jvopt1="-Duser.timezone=Asia/Seoul"
jvopt2="-Dcasc.jenkins.config=https://raw.githubusercontent.com/beomtaek/cicd_samplecode/4b32b6f7a3ab3cb11fa02847fa0aca6b7f2309fc/jenkins-config.yaml"
jvopt3="-Dhudson.model.DownloadService.noSignatureCheck=true"
helm install jenkins edu/jenkins \
--set persistence.existingClaim=jenkins \
--set master.adminPassword=admin \
--set master.nodeSelector."kubernetes\.io/hostname"=master \
--set master.tolerations[0].key=node-role.kubernetes.io/master \
--set master.tolerations[0].effect=NoSchedule \
--set master.tolerations[0].operator=Exists \
--set master.runAsUser=1000 \
--set master.runAsGroup=1000 \
--set master.tag=2.249.3-lts-centos7 \
--set master.serviceType=LoadBalancer \
--set master.servicePort=80 \
--set master.jenkinsOpts="$jkopt1 $jkopt2" \
--set master.javaOpts="$jvopt1 $jvopt2 $jvopt3"- 젠킨스를 설치한다.
root@master:~/lab2/cicd_samplecode# chmod +x jenkins-install.sh
root@master:~/lab2/cicd_samplecode# ./jenkins-install.sh
NAME: jenkins
LAST DEPLOYED: Thu Oct 27 16:38:19 2022
NAMESPACE: default
STATUS: deployed
REVISION: 1
NOTES:
1. Get your 'admin' user password by running:
printf $(kubectl get secret --namespace default jenkins -o jsonpath="{.data.jenkins-admin-password}" | base64 --decode);echo
2. Get the Jenkins URL to visit by running these commands in the same shell:
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status of by running 'kubectl get svc --namespace default -w jenkins'
export SERVICE_IP=$(kubectl get svc --namespace default jenkins --template "{{ range (index .status.loadBalancer.ingress 0) }}{{ . }}{{ end }}")
echo http://$SERVICE_IP:80/login
3. Login with the password from step 1 and the username: admin
4. Use Jenkins Configuration as Code by specifying configScripts in your values.yaml file, see documentation: http:///configuration-as-code and examples: https://github.com/jenkinsci/configuration-as-code-plugin/tree/master/demos
For more information on running Jenkins on Kubernetes, visit:
https://cloud.google.com/solutions/jenkins-on-container-engine
For more information about Jenkins Configuration as Code, visit:
https://jenkins.io/projects/jcasc/root@master:~/lab2/cicd_samplecode# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
jenkins LoadBalancer 10.104.1.100 192.168.8.201 80:32440/TCP 39s
jenkins-agent ClusterIP 10.105.41.140 <none> 50000/TCP 39s
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 5h23m이미지가 잘 다운이 되지 않으면 수동으로 다운받자
docker pull jenkins/inbound-agent:4.3-4
docker pull jenkins/jenkins:2.249.3-lts-centos7
docker pull kiwigrid/k8s-sidecar:0.1.193root@master:~/lab2/cicd_samplecode# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
jenkins-6f46869d7b-t6rrs 2/2 Running 0 20m 172.18.219.69 master <none> <none>처음 아이디, 비밀번호는 admin, admin이다.
Jenkins에 들어가면 jenkins관리 > 플러그인 관리 > 맨밑 컴패러블 선택 > 지금다운로드 체크하고 재시작하기 클릭 설치가 끝나고 실행중인 작업이 없으면 Jenkisn재시작
Jenkins > Jenkins관리 > 노드 관리 > 왼쪽 configure clouds > pod templates > pod detail templates
환경변수 부분에 192.168.8.201로 변경, /usr/bin/docker로 Host Path Volume 변경 Apply 후 저장
