[Grafana] Webhook 실습

alertmanger config 수정

vim /alertmanager/config/alertmanager.yml

route:
  group_by: ['alertname']
  group_wait: 3s
  group_interval: 5s
  repeat_interval: 20s
  receiver: 'engineer'

receivers:
- name: 'webhook'
  webhook_configs:
  - url: http://localhost:8080

- name: 'engineer'
  email_configs:
  - to: ekdma1403@gmail.com
    from: ekdma1403@gmail.com
    smarthost: smtp.gmail.com:587
    auth_username: ekdma1403@gmail.com
    auth_password: "google_app 입력하기"

inhibit_rules:
- source_matchers:
  - severity = critical
  target_matchers:
  - severity = warning
  equal: ['alertname']

docker restart alertmanager

서버에 리스닝 포트 열기

 nc -l -k -w 1 0.0.0.0 8080
 
 

swagger editor 수정

swagger: '2.0'

[추가]
host: localhost:9093
basePath: /api/v2
schemes:
- http

[깃허브 참고]
https://github.com/prometheus/alertmanager/blob/main/api/v2/openapi.yaml

• swagger 양식 https://editor.swagger.io/ 에 복사 붙여넣기

• [POST] /alerts
  작성 후 execute 후 curl 아래 내용 복사 하여 서버에 붙여넣기

curl -X 'POST' \
  'http://localhost:9093/api/v2/alerts' \
  -H 'accept: application/json' \
  -H 'Content-Type: application/json' \
  -d '[
  {
    "startsAt": "2022-11-15T01:45:18.000Z",
    "endsAt": "2022-12-15T01:45:18.000Z",
    "annotations": {
      "summary": "sehun",
      "description": "zzang"
    },
    "labels": {
      "alertname": "This is Alert Test"
    }
  }
]'

• 데이터 잘 나오는지 확인

{"receiver":"webhook","status":"firing","alerts":[{"status":"firing","labels":{"alertname":"test"},"annotations":{"description":"description","summary":"summary"},"startsAt":"2022-11-11T07:37:55.239Z","endsAt":"0001-01-01T00:00:00Z","generatorURL":"","fingerprint":"1f56cb8f931d1bd4"}],"groupLabels":{"alertname":"test"},"commonLabels":{"alertname":"test"},"commonAnnotations":{"description":"description","summary":"summary"},"externalURL":"http://prometheus:9093","version":"4","groupKey":"{}:{alertname=\"test\"}","truncatedAlerts":0}
POST / HTTP/1.1
Host: localhost:8080
User-Agent: Alertmanager/0.24.0
Content-Length: 538
Content-Type: application/json

node.js 설정

js 파일 생성.

data.js에 module.export 선언 후 , 더미파일 복사하여 붙여넣기

module.exports = {
    receiver: 'webhook',
    status: 'firing',
    alerts: [
        {
            status: 'firing',
            labels: { alertname: 'test' },
            annotations: { description: 'description', summary: 'summary' },
            startsAt: '2022-11-14T01:24:17.119Z',
            endsAt: '0001-01-01T00:00:00Z',
            generatorURL: '',
            fingerprint: '1f56cb8f931d1bd4',
        },
    ],
    groupLabels: { alertname: 'test' },
    commonLabels: { alertname: 'test' },
    commonAnnotations: { description: 'description', summary: 'summary' },
    externalURL: 'http://prometheus:9093',
    version: '4',
    groupKey: '{}:{alertname="test"}',
    truncatedAlerts: 0,
};

• 경로가서 npm i -g express-generator 설치

• express 엔터 -> "y"

• 파일 생성된 것 확인
  

• view, public 폴더 삭제 ( 웹은 사용 안하기 때문에)

npm i -g nodemon 설치
npm i (npm install)
npm i axios
npm i nodemailer
nodemon ./bin/www

[다른터미널]
curl localhost:3000 -XPOST
data 내용 잘 받아오는지 확인.

[실패 뜰 때], console.log(e) 찍어보기 TLS 에러일 ㄸ ㅐ,, (간혹 네트워크 환경에 따라)
self-signed certificate in certificate chain 발생 할 때

// 환경변수로 node에서 허가되지 않은 인증TLS통신을 거부하지 않겠다고 설정
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';

success 확인 후 메일 확인

Silence 설정 (알람 끄기)

index.js 설정

var express = require('express');
var router = express.Router();
var axios = require('axios');
var nodemailer = require('nodemailer');

process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';

/* GET home page. */
router.post('/', function (req, res, next) {
    var data = require('../data');
    const transporter = nodemailer.createTransport({
        service: 'gmail',
        host: 'smtp.gmail.com',
        port: 587,
        auth: {
            user: 'ekdma1403@gmail.com',
            pass: 'xqmccweihrjzbdga',
        },
    });
    data.alerts.forEach(alert => {
        const labels = alert.labels;
        var qs = '?';
        Object.keys(labels).forEach(k => {
            qs += `${k}=${labels[k]}&`;
        });
        transporter.sendMail(
            {
                from: 'Alertmanager <ekdma1403@gmail.com>',
                to: 'ekdma1403@gmail.com',
                subject: alert.annotations.summary,
                html:
                    `<div>${alert.annotations.description}</div>` +
                    `<a href="http://localhost:3000/silence/3${qs.slice(0, -1)}"> Turn off</a>`,
            },
            e => {
                if (e) res.status(500).json({ result: 'fail' });
                else res.status(200).json({ result: 'success' });
            }
        );
    });
});

router.get('/silence/:hour', async (req, res, next) => {
    const hour = req.params.hour;
    var labels = req.query;

    var matchers = [];
    Object.keys(labels).forEach(k => {
        matchers.push({
            name: k,
            value: labels[k],
            isRegex: false,
            isEqual: true,
        });
    });
    try {
        await axios({
            method: 'POST',
            url: 'http://alertmanager:9093/api/v2/silences',
            headers: {
                'Content-Type': 'application/json',
            },
            data: {
                matchers,
                startsAt: new Date(),
                endsAt: new Date(Date.now() + 60 * 60 * 1000 * hour),
                createdBy: 'Webhook_sehun',
                comment: 'This is silence is Created by Webhook_sehun',
            },
        });
        res.send(`<script>alert("Success"); window.close();</script>`);
    } catch (e) {
        res.send(`<script>alert("Fail"); window.close();</script>`);
        console.log(e);
    }
});

module.exports = router;

curl localhost:3000 -XPOST 실행 후 메일확인, Link 클릭 "success" 확인

index.js 수정

/* GET home page. */
router.post('/', function (req, res, next) {
    //var data = require('../data');

    var data = req.body;

    const transporter = nodemailer.createTransport({
        service: 'gmail',
        host: 'smtp.gmail.com',
        port: 587,
        auth: {
            user: 'ekdma1403@gmail.com',
            pass: '앱 비밀번호',
        },
    });
    data.alerts.forEach(alert => {
        const labels = alert.labels;
        var qs = '?';
        Object.keys(labels).forEach(k => {
            qs += `${k}=${labels[k]}&`;
        });
        transporter.sendMail(
            {
                from: 'Alertmanager <ekdma1403@gmail.com>',
                to: 'ekdma1403@gmail.com',
                subject: alert.annotations.summary,
                html:
                    `<div>${alert.annotations.description}</div>` +
                    `<a href="http://alertmanager:8080/silence/3${qs.slice(