grafana-agent-flow 설치

rpm -ivh grafana-agent-flow-0.41.1-1.amd64.rpm

/etc/grafana-agent-flow.river

1. PromIP 환경 변수 설정

PromIP="<Prometheus IP>"

2. river파일 수정

sh -c 'cat <<EOF > /etc/grafana-agent-flow.river
logging {
        level = "warn"
}
prometheus.exporter.unix "default" {
        include_exporter_metrics = true
        disable_collectors       = ["mdadm"]
        filesystem {
                fs_types_exclude     = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"
                mount_points_exclude = "^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)"
                mount_timeout        = "5s"
        }
        netclass {
                ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$"
        }
        netdev {
                device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$"
        }
}
prometheus.scrape "default" {
        targets = concat(
                prometheus.exporter.unix.default.targets,
                prometheus.exporter.blackbox.default.targets,
        )
        forward_to = [ prometheus.remote_write.default.receiver ]
}
prometheus.remote_write "default" {
        endpoint {
                url = "http://"'"$PromIP"':9090/api/v1/write"      //# Prometheus IP
        }
}
EOF'

blackbox exporter

logging {
        level = "warn"
}

prometheus.exporter.unix "default" {
        include_exporter_metrics = true
        disable_collectors       = ["mdadm"]
        filesystem {
                fs_types_exclude     = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"
                mount_points_exclude = "^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)"
                mount_timeout        = "5s"
        }
        netclass {
                ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$"
        }
        netdev {
                device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$"
        }
}

//##################################################################
//# blackbox 추가 (대상 Port, 서비스명 수정)
//##################################################################
prometheus.exporter.blackbox "default" {           
        config = "{ modules: { tcp_probe: {prober: tcp, timeout: 5s }, http_probe: { prober: http, timeout: 5s, http: { method: GET, valid_status_codes: [200, 201, 202, 203, 204, 205, 206, 207, 208, 226, 301, 302, 303, 304, 305, 307, 308], tls_config: {insecure_skip_verify: true}, preferred_ip_protocol: ip4, no_follow_redirects: true } } } }"

        target {
                name    = "sshd"
                address = "127.0.0.1:22"
                module  = "tcp_probe"

                labels = {
                        "port" = "22",
                        "service" = "sshd",
                }
        }
        target {
                name    = "httpd"
                address = "127.0.0.1"
                module  = "http_probe"

                labels = {
                        "port" = "80",
                        "service" = "httpd",
                }
        }
        target {
                name    = "grafana"
                address = "10.0.136.43:3000"
                module  = "http_probe"

                labels = {
                        "port" = "grafana",
                        "service" = "web",
                }
        }
}

prometheus.scrape "default" {
        targets = concat(
                prometheus.exporter.unix.default.targets,
                prometheus.exporter.blackbox.default.targets,       //# blackbox 추가
        )
        forward_to = [ prometheus.remote_write.default.receiver ]
}

prometheus.remote_write "default" {
        endpoint {
                url = "http://10.0.136.43:9090/api/v1/write"
        }
}

node exporter label 변경

• relabel부분에 forward_to를 remote_write
• scrape부분에 forward_to를 relabel 블럭

logging {
        level = "warn"
}

prometheus.exporter.unix "default" {
        include_exporter_metrics = true
}

prometheus.scrape "default" {
        targets = prometheus.exporter.unix.default.targets
        forward_to = [ prometheus.relabel.example.receiver, ]
}

prometheus.remote_write "default" {
        endpoint {
                url = "http://10.0.24.241:9090/api/v1/write"      //# Prometheus IP
        }
}

prometheus.relabel "example" {
  forward_to = [prometheus.remote_write.default.receiver]

  rule {
    action        = "replace"
    source_labels = ["instance"]
    regex = "ip.*"
    replacement = "prom"
    target_label = "instance"
  }
}

• replacement : 바꾸고 싶은 값 (optional)
• regex : 정규식 (optional)
• target_label : relabel 하고난 후 label 명을 어떻게 설정할 것인지

/usr/lib/systemd/system/grafana-agent-flow.service

1.  서비스 파일 조회

cat /usr/lib/systemd/system/grafana-agent-flow.service

2.  CUSTOM_ARGS 수정

• -server.http.listen-addr=0.0.0.0:12345 --disable-reporting

sudo sed -i 's/CUSTOM_ARGS=\"\"/CUSTOM_ARGS=\"--server.http.listen-addr=0.0.0.0:12345 --disable-reporting\"/g' /etc/sysconfig/grafana-agent-flow

참고

• 위의 명령으로 CUSTOM_ARGS를 수정했으나 동작하지 않아 하드코딩으로 넣었음 → 해당 부분이 자동으로 잡히지 않아서 생긴 문제인지 원인 불명

서비스 실행

systemctl daemon-reload
systemctl enable --now grafana-agent-flow 
systemctl status -l grafana-agent-flow

GET /metrics 체크

curl 127.0.0.1:12345/metrics

• Prometheus와 연동이 잘 되었는지 확인