로그 라이브러리 종류
- java.util.logging
- Apache Commons logging
- Log4j
- Logback
ELFK를 같이 쓰기도 하며 EL+Kafka+K를 쓰기도 합니다.
7.10 버젼 이후로는 License 부분에 있어서 SSPL라이센스를 차용.
관계된 소프트웨어에 대한 소스코드 공개가 필요하므로 Elastic Search를 쓰기 위해선 7.10버젼을 사용해야함.
https://www.elastic.co/kr/pricing/faq/licensing
if [type] == "end" {
elasticsearch {
hosts => ["es-server"]
query => "type:start AND operation:%{[opid]}"
fields => { "@timestamp" => "started" }
}
date {
match => ["[started]", "ISO8601"]
target => "[started]"
}
ruby {
code => 'event.set("duration_hrs", (event.get("@timestamp") - event.get("started")) / 3600) rescue nil'
}
}
<source>
@type tail
tag system.logs
# ...
</source>
<filter app.**>
@type record_transformer
<record>
hostname "#{Socket.gethostname}"
</record>
</filter>
<match {app.**,system.logs}>
@type file
# ...
</match>
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: es-cluster
namespace: kube-logging
spec:
serviceName: elasticsearch
replicas: 3
selector:
matchLabels:
app: elasticsearch
template:
metadata:
labels:
app: elasticsearch
spec:
containers:
- name: elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0
resources:
limits:
cpu: 1000m
requests:
cpu: 100m
ports:
- containerPort: 9200
name: rest
protocol: TCP
- containerPort: 9300
name: inter-node
protocol: TCP
volumeMounts:
- name: data
mountPath: /usr/share/elasticsearch/data
env:
- name: cluster.name
value: k8s-logs
- name: node.name
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: discovery.seed_hosts
value: "es-cluster-0.elasticsearch,es-cluster-1.elasticsearch,es-cluster-2.elasticsearch"
- name: cluster.initial_master_nodes
value: "es-cluster-0,es-cluster-1,es-cluster-2"
- name: ES_JAVA_OPTS
value: "-Xms512m -Xmx512m"
initContainers:
- name: fix-permissions
image: busybox
command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
securityContext:
privileged: true
volumeMounts:
- name: data
mountPath: /usr/share/elasticsearch/data
- name: increase-vm-max-map
image: busybox
command: ["sysctl", "-w", "vm.max_map_count=262144"]
securityContext:
privileged: true
- name: increase-fd-ulimit
image: busybox
command: ["sh", "-c", "ulimit -n 65536"]
securityContext:
privileged: true
volumeClaimTemplates:
- metadata:
name: data
labels:
app: elasticsearch
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: local-storage
resources:
requests:
storage: 50Gi
apiVersion: v1
kind: ServiceAccount
metadata:
name: fluentd
namespace: kube-logging
labels:
app: fluentd
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: fluentd
labels:
app: fluentd
rules:
- apiGroups:
- ""
resources:
- pods
- namespaces
verbs:
- get
- list
- watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: fluentd
roleRef:
kind: ClusterRole
name: fluentd
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: fluentd
namespace: kube-logging
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: fluentd
namespace: kube-logging
labels:
app: fluentd
spec:
selector:
matchLabels:
app: fluentd
template:
metadata:
labels:
app: fluentd
spec:
serviceAccount: fluentd
serviceAccountName: fluentd
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
containers:
- name: fluentd
image: fluent/fluentd-kubernetes-daemonset:v1.4.2-debian-elasticsearch-1.1
env:
- name: FLUENT_ELASTICSEARCH_HOST
value: "elasticsearch.kube-logging.svc.cluster.local"
- name: FLUENT_ELASTICSEARCH_PORT
value: "9200"
- name: FLUENT_ELASTICSEARCH_SCHEME
value: "http"
- name: FLUENTD_SYSTEMD_CONF
value: disable
resources:
limits:
memory: 512Mi
requests:
cpu: 100m
memory: 200Mi
volumeMounts:
- name: varlog
mountPath: /var/log
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- name: varlog
hostPath:
path: /var/log
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
apiVersion: v1
kind: Service
metadata:
name: kibana
namespace: kube-logging
labels:
kubernetes.io/cluster-service: 'true'
kubernetes.io/name: kibana
spec:
ports:
- port: 5601
targetPort: 5601
nodePort: 30050
type: NodePort
selector:
app: kibana
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: kibana
namespace: kube-logging
labels:
app: kibana
spec:
replicas: 1
selector:
matchLabels:
app: kibana
template:
metadata:
labels:
app: kibana
spec:
containers:
- name: kibana
image: docker.elastic.co/kibana/kibana:7.2.0
resources:
limits:
cpu: 1000m
requests:
cpu: 100m
env:
- name: ELASTICSEARCH_URL
value: http://elasticsearch:9200
ports:
- containerPort: 5601
Promtail + Loki + Grafana 스택.
Promtail : Collector가 출력하는 로그 데이터를 Loki에게 전달
Loki : 시계열 로그(string) 데이터를 저장하는 DB
Grafana : Loki의 Data를 시각화
Collector : 모니터링 대상 시스템이 생성하는 데이터를 Prometheus와 Promtail이 원하는 형식으로 제공
모두 AGPLv3 License
kube-loki-statefulset.yaml
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: loki
labels:
app: loki
spec:
revisionHistoryLimit: 1
podManagementPolicy: OrderedReady
replicas: 1
selector:
matchLabels:
app: loki
serviceName: loki-headless
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
app: loki
annotations:
prometheus.io/port: http-metrics
prometheus.io/scrape: "true"
spec:
automountServiceAccountToken: false
containers:
- name: loki
image: docker.io/grafana/loki:latest
imagePullPolicy: IfNotPresent
args:
- "-config.file=/etc/loki/loki.yaml"
volumeMounts:
- name: config
mountPath: "/etc/loki"
- name: storage
mountPath: "/data:z"
subPath: data
- name: storage
mountPath: "/wal:z"
subPath: wal
ports:
- name: http-metrics
containerPort: 3100
protocol: TCP
livenessProbe:
httpGet:
path: /ready
port: http-metrics
initialDelaySeconds: 45
readinessProbe:
httpGet:
path: /ready
port: http-metrics
initialDelaySeconds: 45
securityContext:
allowPrivilegeEscalation: false
privileged: false
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsGroup: 10001
runAsUser: 10001
seccompProfile:
type: RuntimeDefault
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
terminationGracePeriodSeconds: 60
volumes:
- name: config
configMap:
name: loki
defaultMode: 0755
- name: storage
emptyDir:
sizeLimit: "2Gi"
---
# Daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: promtail-daemonset
spec:
selector:
matchLabels:
name: promtail
template:
metadata:
labels:
name: promtail
spec:
serviceAccount: promtail-serviceaccount
containers:
- name: promtail-container
image: grafana/promtail
args:
- -config.file=/etc/promtail/promtail.yaml
env:
- name: 'HOSTNAME' # needed when using kubernetes_sd_configs
valueFrom:
fieldRef:
fieldPath: 'spec.nodeName'
volumeMounts:
- name: logs
mountPath: /var/log
- name: promtail-config
mountPath: /etc/promtail
- mountPath: /var/lib/docker/containers
name: varlibdockercontainers
readOnly: true
volumes:
- name: logs
hostPath:
path: /var/log
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: promtail-config
configMap:
name: promtail-config
--- # configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: promtail-config
data:
promtail.yaml: |
server:
http_listen_port: 9080
grpc_listen_port: 0
clients:
- url: https://{YOUR_LOKI_ENDPOINT}/loki/api/v1/push
positions:
filename: /tmp/positions.yaml
target_config:
sync_period: 10s
scrape_configs:
- job_name: pod-logs
kubernetes_sd_configs:
- role: pod
pipeline_stages:
- docker: {}
relabel_configs:
- source_labels:
- __meta_kubernetes_pod_node_name
target_label: __host__
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- action: replace
replacement: $1
separator: /
source_labels:
- __meta_kubernetes_namespace
- __meta_kubernetes_pod_name
target_label: job
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- action: replace
source_labels:
- __meta_kubernetes_pod_name
target_label: pod
- action: replace
source_labels:
- __meta_kubernetes_pod_container_name
target_label: container
- replacement: /var/log/pods/*$1/*.log
separator: /
source_labels:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
--- # Clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: promtail-clusterrole
rules:
- apiGroups: [""]
resources:
- nodes
- services
- pods
verbs:
- get
- watch
- list
--- # ServiceAccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: promtail-serviceaccount
--- # Rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: promtail-clusterrolebinding
subjects:
- kind: ServiceAccount
name: promtail-serviceaccount
namespace: default
roleRef:
kind: ClusterRole
name: promtail-clusterrole
apiGroup: rbac.authorization.k8s.io
https://github.com/developer-guy/kubernetes-audit-logs-collect-with-PLG-stack/blob/main/auditing.md
기본적으로 auidt 로그는 event라는 객체의 구조로 아래와 같은 정보들을 담을 수 있다.
콜을 날린 클라이언트가 누구인지
언제 날렸는지
어떤 리소스에 대한 것인지
HTTP의 성공 여부
verb는 무엇인지
URI
request body, reponse body
RequestReceived : audit 핸들러가 request를 받자마자
ResponseStarted : response 헤더만 보내지고, reponse body는 아직 안보내졌을 때. long-running request의 경우에만 발생한다 (예: watch)
ResponseComplete : response body까지 전부 보내진 후
Panic : panic이 발생 했을 때
어느정도 수준의 정보를 남길 것인지도 선택할 수 있다.
아래로 갈 수록 더 많은 양의 정보를 담고 있다.
None : 남기지 않는다
Metadata : request의 metadata (request한 유저, timestamp, resource, verb, 등) (body는 포함하지 않음)
Request : Metadata + request body (response body는 포함하지 않음) (non-resource request에는 적용 안 됨)
RequestResponse : Request + response body (non-resource request에는 적용 안 됨)
audit을 저장하는 방식 2가지가 있다.
파일시스템으로 작성
웹훅을 이용해 외부 서버에 HTTP 콜 전송
요약하자면
audit이란 kube-apiserver를 오가는 HTTP 콜을 로깅하는 것이고,
audit을 언제, 얼마만큼의 정보로, 어떤 방식으로 저장할지 커스터마이징 할 수 있다.
https://www.infracloud.io/blogs/kubernetes-auditing/
https://grafana.com/docs/grafana/latest/setup-grafana/configure-security/audit-grafana/
https://github.com/grafana/loki
Auditing allows you to track important changes to your Grafana instance. By default, audit logs are logged to file but the auditing feature also supports sending logs directly to Loki.
kube-apiserver.yaml
--audit-policy-file=”<AUDIT_POLICY_FILE_PATH>”
--audit-log-path=”<WHERE_TO_WRITE_LOGS>
kube-apiserver-policy.yaml
- level: Request
verbs: ["get", "list", "watch"]
resources:
- group: "" # core
- group: "apps"
- group: "certificates.k8s.io"
scrape_configs:
- job_name: system
static_configs:
- targets:
- localhost
labels:
job: varlogs
host: MASTER_NODE_ADDR
agent: promtail
__path__: /var/log/kubernetes/audit/audit.log
이런식으로 scrape config를 주어서 Promtail이 Kubernetes의 audit을 긁어모으면 된다.
server:
....
..
..
clients:
- url: http://LOKI_HOST_ADDR:3100/loki/api/v1/push