# OpenStack : CloudBand Application Manager Health check script

このスクリプトは同じチームのメンバーがベースを作ってそれを元に修正したものです。

メンバーが作った機能:

▸ UndercloudからTargetVMへコマンドがスムーズに実行できる
▸ Pingが通らないVMはSkipする

自分が作った機能:

自分がやったことは可読性よくしたくらい(^^)v
▸ VIMで編集しなくてもConsole上で直接入力して使えるように変更

作成理由

: それぞれのVMに入ってHealthcheckしなければならない

Undercloud上でスクリプトを使って楽に確認できるようにしたかった

#!/bin/bash
# chmod +x cbam-status-check.sh
# ./cbam-status-check.sh
# 20210917 Add : skip inactive server by.Shibata

echo ""

echo ">>> CBAM21 Status check"
date

echo ""&&echo ""

check_filepath="/tmp/cbam21_node_list.txt"

if test -e ${check_filepath}; then
        echo "# CBAM21"
        cat ${check_filepath}
else
        touch ${check_filepath}
        echo "Enter CBAM21 AP01's IP : "
        read AP01
        echo "AP01=${AP01}" >> ${check_filepath}
        echo "Enter CBAM21 AP02's IP : "
        read AP02
        echo "AP02=${AP02}" >> ${check_filepath}
        echo "Enter CBAM21 DB01's IP : "
        read DB01
        echo "DB01=${DB01}" >> ${check_filepath}
        echo "Enter CBAM21 DB02's IP : "
        read DB02
        echo "DB02=${DB02}" >> ${check_filepath}
        echo "Enter CBAM21 DB03's IP : "
        read DB03
        echo "DB03=${DB03}" >> ${check_filepath}
fi

source $check_filepath

echo ""&&echo ""

source ~/cbam.rc

echo "Command : openstack stack list"
openstack stack list

echo ""&&echo ""

echo "Command : openstack server list --fit"
openstack server list --fit

### AP01
function check_AP01(){
ssh -i ~/.ssh/cbam_key.pem -q cbam@${AP01} <<\EOC
echo ">>> CBAM21 APP01 Check"

echo "Command : ectlm get /locks/master"
ectlm get /locks/master

echo ""&&echo ""

echo "Command : hostname"
hostname

echo ""&&echo ""

echo "Command : cbam-status"
cbam-status

echo ""&&echo ""

echo "Command : ntpq -pn"
ntpq -pn

echo ""&&echo ""

EOC
}

### AP02
function check_AP02(){
ssh -i ~/.ssh/cbam_key.pem -q cbam@${AP02} <<\EOC
echo ">>> CBAM21 APP02 Check"

echo "Command : ectlm get /locks/master"
ectlm get /locks/master

echo ""&&echo ""

echo "Command : hostname"
hostname

echo ""&&echo ""

echo "Command : cbam-status"
cbam-status

echo ""&&echo ""

echo "Command : ntpq -pn"
ntpq -pn

echo ""&&echo ""

EOC
}

### DB01
function check_DB01(){
ssh -i ~/.ssh/cbam_key.pem -q cbam@${DB01} <<\EOC
echo ">>> CBAM21 DB01 Check"

echo "Command : hostname"
hostname

echo ""&&echo ""

echo "# etcd status check"
systemctl status etcd | grep Active

echo ""&&echo ""

echo "# mariadb status check"
systemctl status mariadb | grep Active

echo ""&&echo ""

echo "# mongod status check"
systemctl status mongod | grep Active

echo ""&&echo ""

echo "# rabbitmq status check"
systemctl status rabbitmq-server | grep Active

echo ""&&echo ""

echo "# etcd-in-memory status check"
systemctl status etcd-in-memory | grep Active

echo ""&&echo ""

echo "Command : ntpq -pn"
ntpq -pn

echo ""&&echo ""

echo "# ectl cluster health check"
sudo ectl cluster-health | grep healthy

echo ""&&echo ""

echo "# ectlm cluster health check"
sudo ectlm cluster-health | grep healthy

echo ""&&echo ""

echo "# mongod health check"
sudo mongo --quiet --ipv6 --host cbamset/$(hostname -s)-dbfrontend --port "PortNumber" --ssl --sslCAFile /etc/ssl/certs/cbam-internal-ca.crt --sslPEMKeyFile /etc/ssl/certs/cbam-node-bundle-$(hostname -s).crt -u cbamroot -p $(sudo ectl get cbam_private/cluster/components/mongodb/admin_credentials/password) admin --eval "rs.status()" | grep -e name -e health -e stateStr

echo ""&&echo ""

echo "# mariadb health check"
sudo mysql -u cbamroot -p$(sudo ectl get cbam_private/cluster/components/mariadb/admin_credentials/password) -e "SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='wsrep_cluster_size'\\G;"

echo ""&&echo ""

echo "# rabbitmq server health check"
sudo rabbitmqctl cluster_status | grep "Running Nodes" -A5

echo ""&&echo ""

EOC
}

### DB02
function check_DB02(){
ssh -i ~/.ssh/cbam_key.pem -q cbam@${DB02} <<\EOC
echo ">>> CBAM21 DB02 Check"

echo "Command : hostname"
hostname

echo ""&&echo ""

echo "# etcd status check"
systemctl status etcd | grep Active

echo ""&&echo ""

echo "# mariadb status check"
systemctl status mariadb | grep Active

echo ""&&echo ""

echo "# mongod status check"
systemctl status mongod | grep Active

echo ""&&echo ""

echo "# rabbitmq server status check"
systemctl status rabbitmq-server | grep Active

echo ""&&echo ""

echo "# etcd-in-memory status check"
systemctl status etcd-in-memory | grep Active

echo ""&&echo ""

echo "Command : ntpq -pn"
ntpq -pn

echo ""&&echo ""

EOC
}

### DB03
function check_DB03(){
ssh -i ~/.ssh/cbam_key.pem -q cbam@${DB03} <<\EOC
echo ">>> CBAM21 DB03 Check"

echo "Command : hostname"
hostname

echo ""&&echo ""

echo "# etcd status check"
systemctl status etcd | grep Active

echo ""&&echo ""

echo "# mariadb status check"
systemctl status mariadb | grep Active

echo ""&&echo ""

echo "# mongod status check"
systemctl status mongod | grep Active

echo ""&&echo ""

echo "# rabbitmq server status check"
systemctl status rabbitmq-server | grep Active

echo ""&&echo ""

echo "# etcd-in-memory status check"
systemctl status etcd-in-memory | grep Active

echo ""&&echo ""

echo "Command : ntpq -pn"
ntpq -pn

EOC
}

echo ">>> CBAM21 Nodes ping check"

echo ""&&echo ""

ping_result=$(ping -w 3 ${AP01} | grep '100% packet loss')
if [[ -n $ping_result ]]; then
  echo "AP01 inactive."
else
  check_AP01
fi

ping_result=$(ping -w 3 ${AP02} | grep '100% packet loss')
if [[ -n $ping_result ]]; then
  echo "AP02 inactive."
else
  check_AP02
fi

ping_result=$(ping -w 3 ${DB01} | grep '100% packet loss')
if [[ -n $ping_result ]]; then
  echo "DB01 inactive."
else
  check_DB01
fi

ping_result=$(ping -w 3 ${DB02} | grep '100% packet loss')
if [[ -n $ping_result ]]; then
  echo "DB02 inactive."
else
  check_DB02
fi

ping_result=$(ping -w 3 ${DB03} | grep '100% packet loss')
if [[ -n $ping_result ]]; then
  echo "DB03 inactive."
else
  check_DB03
fi

echo ""&&echo ""

echo "CBAM21 Healthcheck finish"
date

echo ""