Tempo & Opentelemetry 도입하기

DevOps Engineer·2023년 10월 16일

OpenTelemetry TRACE Tempo collector node.js nodejs operator

도입을 고민하게 된 이유
기존 ECK를 통해 log 및 trace를 모니터링이 남도록 할 수 있었지만 alert 설정이 무료배포판에서는 어려워 Grafana와 연동하여 사용할 수 있는 Loki와 Tempo를 사용해보려고 시도했고 그 과정에서 Tempo가 sample이 많지않아 글을 작성함.

Tempo를 통해 조회하려면 trace ID가 로그로 남아있어야 하고 그 역할을 해줄 Opentelemetry Auto instrumentation 기능을 통해 조회가 되도록 만들 수 있었음

작업순서

1) Cert-manager 설치
2) Opentelemetry Operator
3) Tempo 설치
4) Opentelemetry Collector
5) node.js 코드 수정

Cert-manager

#yaml로 설치하기
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml 

#helm으로 설치 시
helm repo add jetstack https://charts.jetstack.io
helm repo update
k create ns cert-manager

vi cert-manager-values.yaml
prometheus:
  enabled: false

helm install \
  cert-manager jetstack/cert-manager \
  --namespace cert-manager \
  --create-namespace \
  -f cert-manager-values.yaml --version v1.12.1

cert-manager-values.yaml

prometheus:
  enabled: false

Opentelemetry Operator

helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
helm repo update

kubectl create ns opentelemetry-operator
helm install opentelemetry-operator open-telemetry/opentelemetry-operator -n opentelemetry-operator

Tempo

kubectl create ns tempo
helm pull grafana/tempo-distributed
tar xvfz tempo-distributed-1.6.10.tgz
mv tempo-distributed tempo-distributed-1.6.10
rm –rf tempo-distributed-1.6.10.tgz
cd tempo-distributed-1.6.10
helm install tempo -f my-values.yaml .
# helm update 시 버전이 다를 수 있음
k get pod -n tempo -o wide
k get svc tempo-distributor-discovery -n tempo

values.yaml
value값 수정

ingester:
  persistence:
    enabled: true
    size: 10Gi
    storageClass: local-storage
traces:
  otlp:
    http:
      enabled: true
      receiverConfig: {}
    grpc:
      enabled: true
      receiverConfig: {}

storage:
  trace:
    block:
      version: vParquet
    backend: local
  admin:
    backend: filesystem

Grafana에 데이터 소스 추가하기
url: http://tempo-query-frontend-discovery.tempo:3100

opentelemetry Collector

helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
helm pull open-telemetry/opentelemetry-collector
tar xvfz opentelemetry-collector-0.72.0.tgz 
rm -rf opentelemetry-collector-0.72.0.tgz
mv opentelemetry-collector opentelemetry-collector-0.72.0
cd opentelemetry-collector-0.72.0

kubectl create ns otel-collector
helm install otel-collector -f values.yaml . -n opentelemetry-operator

#지울 시 
helm uninstall otel-collector -n opentelemetry-operator

Auto-instrumentation.yaml
이 yaml은 파드와 같은 네임스페이스 내에 있어야 함

apiVersion: opentelemetry.io/v1alpha1
kind: Instrumentation
metadata:
  name: otel-instrumentation
spec:
  exporter:
    endpoint: http://otel-collector-opentelemetry-collector.opentelemetry-operator:4317
  propagators:
  - tracecontext
  - baggage
  sampler:
    type: parentbased_traceidratio
    argument: "1"

kubectl apply -f Auto-instrumentation.yaml
kubectl get instrumentations.opentelemetry.io

instrumentation을 생성하면 operator에 log가 찍힘

deployment.yaml annotation 값 추가

spec:
  template:
    metadata:
      annotations:
        instrumentation.opentelemetry.io/inject-nodejs: "true"

자동으로 템포 로그 찍히게 yaml 만들기 (추가)

apiVersion: apps/v1
kind: Deployment
metadata:
  annotations:
    instrumentation.opentelemetry.io/inject-java: "kw-dev/otel-instrumentation"
	# ex) instrumentation.opentelemetry.io/inject-java: "true"
    # ex) instrumentation.opentelemetry.io/inject-java: "instrumentation-name"
spec:
  template:
    metadata:
      annotations:
        instrumentation.opentelemetry.io/inject-java: kw-dev/otel-instrumentation

다른 네임스페이스에 있는 instrumentation을 사용할 경우 명시를 해줘야 함
어떤 instrumentation을 사용할지 특정 instrumentation을 명시해줘도 됨
3개의 양식 중 하나만 작성하면 됨!

정상적으로 trace, metric이 찍힐 경우

Node.js 코드 수정

위 내용은 공식문서를 참고함 이를 적용 안할 시 tempo에서 조회가 안됨

# instrumentation.js
const opentelemetry = require("@opentelemetry/sdk-node");
const {
  getNodeAutoInstrumentations,
} = require("@opentelemetry/auto-instrumentations-node");
const {
  OTLPTraceExporter,
} = require("@opentelemetry/exporter-trace-otlp-proto");
const {
  OTLPMetricExporter,
} = require("@opentelemetry/exporter-metrics-otlp-proto");
const { PeriodicExportingMetricReader } = require("@opentelemetry/sdk-metrics");
const { Resource } = require("@opentelemetry/resources");
const {
  SemamticResourceAttributes,
  SemanticAttributes,
  SemanticResourceAttributes,
} = require("@opentelemetry/semantic-conventions");

const sdk = new opentelemetry.NodeSDK({
  resource: new Resource({
    [SemanticResourceAttributes.SERVICE_NAME]: "test",
    [SemanticResourceAttributes.SERVICE_NAMESPACE]: "dev",
    [SemanticResourceAttributes.SERVICE_VERSION]: "1.0",
  }),

  traceExporter: new OTLPTraceExporter({
    // optional - default url is http://localhost:4318/v1/traces
    url: "http://tempo-distributor-discovery.tempo:4318/v1/traces",
    // optional - collection of custom headers to be sent with each request, empty by default
    headers: {},
  }),
  metricReader: new PeriodicExportingMetricReader({
    exporter: new OTLPMetricExporter({
      url: "http://tempo-distributor-discovery.tempo:4318/v1/metrics", // url is optional and can be omitted - default is http://localhost:4318/v1/metrics
      headers: {}, // an optional object containing custom headers to be sent with each request
      concurrencyLimit: 1, // an optional limit on pending requests
    }),
  }),
  instrumentations: [getNodeAutoInstrumentations()],
});
sdk.start();

# 설치 modules
npm install --save @opentelemetry/exporter-trace-otlp-proto \
  @opentelemetry/exporter-metrics-otlp-proto \
  @opentelemetry/api @opentelemetry/resources \
  @opentelemetry/semantic-conventions \
  @opentelemetry/resources \
  @opentelemetry/semantic-conventions
# 실행명령어
node --require ./instrumentation.js app.js

apiVersion: opentelemetry.io/v1alpha1
kind: Instrumentation
metadata:
  labels:
    app.kubernetes.io/managed-by: opentelemetry-operator
      manager: kubectl-client-side-apply
      operation: Update
      time: '2023-10-19T06:30:04Z'
  name: otel-instrumentation
  namespace: kw-dev
  resourceVersion: '14588001'
  uid: 3106d6d9-8718-477d-9d2a-eda024453c82
spec:
  nodejs:
  	image:	ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-nodejs:0.40.0
    #0.43.0 버전 Usage empDir 150Mi 초과 에러발생