Zabbix - Slack 연동 # 2.Zabbix item&trigger설정

Zabbix 5.4 설정

1.Administration > General > Macros

{$SLACK_BOT_TOKEN} --> xoxb-~~~
{$SNMP_COMMUNITY} --> public
{$ZABBIX.URL}  --> https://sre.mysite.io

2.Administration > Media types > Slack

3.Configuration > Hosts > 모니터링 타겟 Host 클릭

3.1. Items 탭 클릭 후 CREATE ITEM 클릭

3.2 item은 모니터링하는 rule을 생성하는 개념이다

item실행 interval을 시간대별로 설정할 수 있다.

3.3 Key부분에 select 버튼을 클릭한다.
proc.num으로 특정 프로세스를 카운트 결과를 기준으로 모니터링 하기로 한다.
proc.num[,,,,] 으로 user와 cmdline을 이용해서 모니터링 한다.
프로세스 user가 ec2-user이고 cmdline에 mysvc-core 가 들어간 프로세스를 모니터링 하기로 한다면

proc.num[,ec2-user,,mysvc-core]

3.4 중요한 프로세스이기에 custom intervals로 중요 이용시간대별로 모니터링 시간 텀을 분리할 수 있다.

3.5 Update 클릭으로 저장한다.

4.Triggers 생성

4.1 생성한 item 메뉴 옆에 Triggers 메뉴를 클릭한다

4.2 CREATE TRIGGER 를 클릭한다

4.3 Expression 항목에서 Add 를 클릭한다.

4.4 Select를 클릭하여 생성한 item을 선택한다.

4.5 해당 프로세스는 항상 1개만 실행이 되어야 하므로 1이 아닌 경우에 트리거를 발생하기로 설정한다.

4.6 생성하는 트리거의 심각성(Severity)을 설정한다. 프로세스 다운시 서비스 실패이므로 ‘Disaster’로 설정한다.
추후 사용자마다 메시지를 받을 긴급도를 설정할 수 있다.

# Problem expression
last(/wheet-sre-c-01/proc.num[,ec2-user,,wheet])<>1

# Recovery expression
last(/wheet-sre-c-01/proc.num[,ec2-user,,wheet])>0

5.대시보드에서 확인

프로세스 오류등으로 item rule에 따라 감지되면 trigger 로 인해 대시보드에서 확인가능하다.

6.yaml import (필요시)

slack media type 변경 작업 중에 초기화가 필요시 아래 yaml 파일을 import 해야 한다.
media_slack.yaml

6.1 Scirpt 내용 ( media_slack.yaml 에 포함됨 )

var SEVERITY_COLORS = [
    '#97AAB3', '#7499FF', '#FFC859',
    '#FFA059', '#E97659', '#E45959'
];

var RESOLVE_COLOR = '#009900';

var SLACK_MODE_HANDLERS = {
    alarm: handlerAlarm,
    event: handlerEvent
};


if (!String.prototype.format) {
    String.prototype.format = function() {
        var args = arguments;

        return this.replace(/{(\d+)}/g, function(match, number) {
            return number in args
                ? args[number]
                : match
            ;
        });
    };
}

function isEventProblem(params) {
    return params.event_value == 1
        && params.event_update_status == 0
    ;
}

function isEventUpdate(params) {
    return params.event_value == 1
        && params.event_update_status == 1
    ;
}

function isEventResolve(params) {
    return params.event_value == 0;
}

function getPermalink(channelId, messageTimestamp) {
    var req = new HttpRequest();

    if (typeof params.HTTPProxy === 'string' && params.HTTPProxy.trim() !== '') {
        req.setProxy(params.HTTPProxy);
    }

    req.addHeader('Content-Type: application/x-www-form-urlencoded; charset=utf-8');
    req.addHeader('Authorization: Bearer ' + params.bot_token);

    var query = '{0}?channel={1}&message_ts={2}'.format(
            Slack.getPermalink,
            encodeURIComponent(channelId),
            encodeURIComponent(messageTimestamp)),
        resp = JSON.parse(req.get(query));

    if (req.getStatus() != 200 || !resp.ok || resp.ok === 'false') {
        throw 'message was created, but getting message link was failed with reason "' + resp.error + '"';
    }

    return resp.permalink;
}

function createProblemURL(zabbix_url, triggerid, eventid, event_source) {
    var problem_url = '';
    if (event_source === '0') {
        problem_url = '{0}/tr_events.php?triggerid={1}&eventid={2}'
            .format(
                zabbix_url,
                triggerid,
                eventid
            );
    }
    else {
        problem_url = zabbix_url;
    }

    return problem_url;
}

function handlerAlarm(params) {
    var fields = {
        channel: params.channel,
        as_user: params.slack_as_user,
    };

    if (isEventProblem(params)) {
        fields.attachments = [
            createMessage(
                SEVERITY_COLORS[params.event_nseverity] || 0,
                params.event_date,
                params.event_time,
                createProblemURL(params.zabbix_url, params.trigger_id, params.event_id, params.event_source)
            )
        ];

        var resp = JSON.parse(req.post(Slack.postMessage, JSON.stringify(fields)));

        if (req.getStatus() != 200 || !resp.ok || resp.ok === 'false') {
            throw resp.error;
        }

        result.tags = {
            ['__message_ts_' + params.channel]: resp.ts,
            ['__channel_id_' + params.channel]: resp.channel,
            ['__message_link_' + params.channel]: getPermalink(resp.channel, resp.ts),
        };

    }
    else if (isEventUpdate(params)) {
        try {
            var channel_event_tags = JSON.parse(params.event_tags);
        } catch (error) {
            throw 'Cannot process event tags: ' + error;
        }

        if (Array.isArray(channel_event_tags)) {
            for (i in channel_event_tags) {
                if (channel_event_tags[i].tag.includes('__message_ts_' + params.channel)) {
                    fields.thread_ts = channel_event_tags[i].value;
                    break;
                }
            }
        }

        fields.attachments = [
            createMessage(
                SEVERITY_COLORS[params.event_nseverity] || 0,
                params.event_update_date,
                params.event_update_time,
                createProblemURL(params.zabbix_url, params.trigger_id, params.event_id, params.event_source),
                true
            )
        ];

        resp = JSON.parse(req.post(Slack.postMessage, JSON.stringify(fields)));

        if (req.getStatus() != 200 || !resp.ok || resp.ok === 'false') {
            throw resp.error;
        }

    }
    else if (isEventResolve(params)) {

        fields.text = '';

        try {
            var channel_event_tags = JSON.parse(params.event_tags);
        } catch (error) {
            throw 'Cannot process event tags: ' + error;
        }

        if (Array.isArray(channel_event_tags)) {
            for (i in channel_event_tags) {
                if (channel_event_tags[i].tag.includes('__channel_id_' + params.channel)) {
                    fields.channel = channel_event_tags[i].value;
                    continue;
                }
                if (channel_event_tags[i].tag.includes('__message_ts_' + params.channel)) {
                    fields.ts = channel_event_tags[i].value;
                }
            }
        }

        fields.attachments = [
            createMessage(
                RESOLVE_COLOR,
                params.event_date,
                params.event_time,
                createProblemURL(params.zabbix_url, params.trigger_id, params.event_id, params.event_source)
            )
        ];

        resp = JSON.parse(req.post(Slack.chatUpdate, JSON.stringify(fields)));
        if (req.getStatus() != 200 || !resp.ok || resp.ok === 'false') {
            throw resp.error;
        }
    }
}

function handlerEvent(params) {
    var fields = {
        channel: params.channel,
        as_user: params.slack_as_user
    };

    if (isEventProblem(params)) {
        fields.attachments = [
            createMessage(
                SEVERITY_COLORS[params.event_nseverity] || 0,
                params.event_date,
                params.event_time,
                createProblemURL(params.zabbix_url, params.trigger_id, params.event_id, params.event_source)
            )
        ];

        var resp = JSON.parse(req.post(Slack.postMessage, JSON.stringify(fields)));

        if (req.getStatus() != 200 || !resp.ok || resp.ok === 'false') {
            throw resp.error;
        }

        result.tags = {
            ['__message_link_' + params.channel]: getPermalink(resp.channel, resp.ts)
        }

    }
    else if (isEventUpdate(params)) {
        fields.attachments = [
            createMessage(
                SEVERITY_COLORS[params.event_nseverity] || 0,
                params.event_update_date,
                params.event_update_time,
                createProblemURL(params.zabbix_url, params.trigger_id, params.event_id, params.event_source),
                false
            )
        ];

        resp = JSON.parse(req.post(Slack.postMessage, JSON.stringify(fields)));

        if (req.getStatus() != 200 || !resp.ok || resp.ok === 'false') {
            throw resp.error;
        }

    }
    else if (isEventResolve(params)) {
        fields.attachments = [
            createMessage(
                RESOLVE_COLOR,
                params.event_recovery_date,
                params.event_recovery_time,
                createProblemURL(params.zabbix_url, params.trigger_id, params.event_id, params.event_source)
            )
        ];

        resp = JSON.parse(req.post(Slack.postMessage, JSON.stringify(fields)));

        if (req.getStatus() != 200 || !resp.ok || resp.ok === 'false') {
            throw resp.error;
        }
    }
}

function createMessage(
    event_severity_color,
    event_date,
    event_time,
    problem_url,
    isShort,
    messageText
) {
    var message = {
        fallback: params.alert_subject,
        title: params.alert_subject,
        color: event_severity_color,
        title_link: problem_url,
        pretext: messageText || '',

        fields: [
            {
                title: 'Host',
                value: '{0} [{1}]'.format(params.host_name, params.host_conn),
                short: true
            },
            {
                title: 'Event time',
                value: '{0} {1}'.format(event_date, event_time),
                short: true
            }
        ],
    };

    if (params.event_source === '0') {
        message.fields.push(
            {
                title: 'Severity',
                value: params.event_severity,
                short: true
            },
            {
                title: 'Opdata',
                value: params.event_opdata,
                short: true
            }
        );
    }

    if (!isShort  && params.event_source === '0') {
        message['actions'] = [
            {
                type: 'button',
                text: 'Open in Zabbix',
                url: problem_url
            }
        ];

        message.fields.push(
            {
                title: 'Event tags',
                value: JSON.parse(params.event_tags).filter(function (e) { return !e.tag.includes('__') }).map(function (e) { return e.tag + ': ' + e.value }).join('\n') || 'None',
                short: true
            },
            {
                title: 'Trigger description',
                value: params.trigger_description,
                short: true
            }
        );
    }

    if (params.event_source !== '0' || params.event_update_status === '1') {
        message.fields.push(
            {
                title: 'Details',
                value: params.alert_message,
                short: false
            }
        );
    }

    return message;
}

function validateParams(params) {
    if (typeof params.bot_token !== 'string' || params.bot_token.trim() === '') {
        throw 'Field "bot_token" cannot be empty';
    }

    if (typeof params.channel !== 'string' || params.channel.trim() === '') {
        throw 'Field "channel" cannot be empty';
    }

    if (isNaN(params.event_id)) {
        throw 'Field "event_id" is not a number';
    }

    if ([0, 1, 2, 3].indexOf(parseInt(params.event_source)) === -1) {
        throw 'Incorrect "event_source" parameter given: "' + params.event_source + '".\nMust be 0-3.';
    }

    if (params.event_source !== '0') {
        params.event_nseverity = '0';
        params.event_severity = 'Not classified';
        params.event_update_status = '0';
        params.slack_mode = 'event';
    }

    if (params.event_source === '1' || params.event_source === '2') {
        params.event_value = '1';
    }

    if (params.event_source === '1') {
        params.host_name = params.discovery_host_dns;
        params.host_ip = params.discovery_host_ip;
    }

    if (!~[0, 1, 2, 3, 4, 5].indexOf(parseInt(params.event_nseverity))) {
        throw 'Incorrect "event_nseverity" parameter given: ' + params.event_nseverity + '\nMust be 0-5.';
    }

    if (typeof params.event_severity !== 'string' || params.event_severity.trim() === '') {
        throw 'Field "event_severity" cannot be empty';
    }

    if (params.event_update_status !== '0' && params.event_update_status !== '1') {
        throw 'Incorrect "event_update_status" parameter given: ' + params.event_update_status + '\nMust be 0 or 1.';
    }

    if (params.event_value !== '0' && params.event_value !== '1') {
        throw 'Incorrect "event_value" parameter given: ' + params.event_value + '\nMust be 0 or 1.';
    }

    if (typeof params.host_conn !== 'string' || params.host_conn.trim() === '') {
        throw 'Field "host_conn" cannot be empty';
    }

    if (typeof params.host_name !== 'string' || params.host_name.trim() === '') {
        throw 'Field "host_name" cannot be empty';
    }

    if (!~['true', 'false'].indexOf(params.slack_as_user.toLowerCase())) {
        throw 'Incorrect "slack_as_user" parameter given: ' + params.slack_as_user + '\nMust be "true" or "false".';
    }

    if (!~['alarm', 'event'].indexOf(params.slack_mode)) {
        throw 'Incorrect "slack_mode" parameter given: ' + params.slack_mode + '\nMust be "alarm" or "event".';
    }

    if (isNaN(params.trigger_id) && params.event_source === '0') {
        throw 'field "trigger_id" is not a number';
    }

    if (typeof params.zabbix_url !== 'string' || params.zabbix_url.trim() === '') {
        throw 'Field "zabbix_url" cannot be empty';
    }

    if (!/^(http|https):\/\/.+/.test(params.zabbix_url)) {
        throw 'Field "zabbix_url" must contain a schema';
    }
}

try {
    var params = JSON.parse(value);

    validateParams(params);

    var req = new HttpRequest(),
        result = {tags: {}};

    if (typeof params.HTTPProxy === 'string' && params.HTTPProxy.trim() !== '') {
        req.setProxy(params.HTTPProxy);
    }

    req.addHeader('Content-Type: application/json; charset=utf-8');
    req.addHeader('Authorization: Bearer ' + params.bot_token);

    var slack_endpoint = 'https://slack.com/api/';

    var Slack = {
        postMessage: slack_endpoint + 'chat.postMessage',
        getPermalink: slack_endpoint + 'chat.getPermalink',
        chatUpdate: slack_endpoint + 'chat.update'
    };

    params.slack_mode = params.slack_mode.toLowerCase();
    params.slack_mode = params.slack_mode in SLACK_MODE_HANDLERS
        ? params.slack_mode
        : 'alarm';

    SLACK_MODE_HANDLERS[params.slack_mode](params);

    if (params.event_source === '0') {
        return JSON.stringify(result);
    }
    else {
        return 'OK';
    }
}
catch (error) {
    Zabbix.log(4, '[ Slack Webhook ] Slack notification failed : ' + error);
    throw 'Slack notification failed : ' + error;
}