
作成理由
: すべてのStatusがOKなら内容確認せず、Healthcheckを終わらせるようにしたかった
問題点
実用性に問題があると言われて作成中断
#!/bin/bash
echo ">>> CBIS Healthcheck After Scale in/out"
echo ">>> HealthCheck date : `date`"
echo ">>> Source change to stackrc"
source /home/stack/stackrc
ok_status="OK"
ko_status="Not OK"
check_dirpath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles"
count_filepath="/home/stack/tmp/healthcheck_dir/count_ovscomp.txt"
check_filename="count_ovscomp.txt"
if test -d "${check_dirpath}"; then
echo ">>> Directory for CBIS healthcheck exists already"
else
echo ">>> Create for healthcheck_dir directory"
mkdir -p "${check_dirpath}"
fi
if test -e "${count_filepath}"; then
echo ">>> Since ${check_filename} file exists already, run a backup"
cp -p "${count_filepath}" "${count_filepath}.bak"
date >> "${count_filepath}"
echo ">>> Compute : `openstack server list | grep 'ovscompute' | wc -l`" >> "${count_filepath}"
else
echo ">>> Since Not found ${check_filename} file, create ${check_filename} file"
touch "${count_filepath}"
date >> "${count_filepath}"
echo ">>> Compute : `openstack server list | grep 'ovscompute' | wc -l`" >> "${count_filepath}"
fi
echo ""
if test -e "${count_filepath}".bak; then
tail -2 "${count_filepath}".bak
else
echo ">>> "${check_filename}".bak is not exist"
fi
tail -2 "${count_filepath}"
echo ""&&echo ""
echo "# Command : openstack stack list"
openstack stack list
openstack stack list | grep "overcloud" | cut -d '|' -f5 > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/Stack_Status_list.txt
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/Stack_Status_list.txt"
while read line; do
if [ "${line}" == "COMPLETE" ]; then
echo "# Check Stack Status="${ok_status}""
elif [ "${line}" == "UPDATE_COMPLETE" ]; then
echo "# Check Stack Status="${ok_status}""
else
echo "# Check Stack Status="${ko_status}""
break
fi
done < "${source_filepath}"
echo ""&&echo ""
echo "# Command : openstack cbis version"
openstack cbis version
openstack cbis version | grep "hotfixes" | wc -l > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/Patch_list.txt
line=`cat /home/stack/tmp/healthcheck_dir/cbis_checkfiles/Patch_list.txt`
if [ "${line}" -ge 1 ]; then
echo "# Check Patch Apply="${ok_status}""
else
echo "# Check Patch Apply="${ko_status}""
fi
echo ""&&echo ""
echo "# Command : openstack server list --sort Name"
openstack server list --sort Name
Controller_number=`openstack server list | grep Controller | cut -d '|' -f3 | sed 's/ overcloud-controller-//' | head -1`
echo ">>> Check: Status, Networks, Image, Flavor"
openstack server list --sort Name | grep 'ovscompute' | cut -d '|' -f4 | sed 's/^ //' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/Status_list.txt
openstack server list --sort Name | grep 'ovscompute' | cut -d '|' -f5 | sed 's/^ ctlplane=//' | sed 's/172.31.0.*/172.31.0/' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/Networks_list.txt
openstack server list --sort Name | grep 'ovscompute' | cut -d '|' -f6 | sed 's/^ //' | cut -d '_' -f1 > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/Image_list.txt
openstack server list --sort Name | grep 'ovscompute' | cut -d '|' -f7 | sed 's/^ //' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/Flavor_list.txt
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/Status_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "ACTIVE" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check Status="${ok_status}""
else
echo "# Check Status="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/Networks_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "172.31.0" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check Networks="${ok_status}""
else
echo "# Check Networks="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/Image_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "overcloud-full" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check Image="${ok_status}""
else
echo "# Check Image="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/Flavor_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "OvsCompute" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check Flavor="${ok_status}""
else
echo "# Check Flavor="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
echo ""&&echo ""
echo "# Command : openstack baremetal node list"
openstack baremetal node list
echo ">>> Check: Power State, Provisioning State, Maintenance"
openstack baremetal node list | grep 'server' | cut -d '|' -f5 | sed 's/^ //' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/Power_State_list.txt
openstack baremetal node list | grep 'server' | cut -d '|' -f6 | sed 's/^ //' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/Provisioning_State_list.txt
openstack baremetal node list | grep 'server' | cut -d '|' -f7 | sed 's/^ //' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/Maintenance_list.txt
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/Power_State_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "power on" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check Power State="${ok_status}""
else
echo "# Check Power State="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/Provisioning_State_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "active" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check Provisioning State="${ok_status}""
else
echo "# Check Provisioning State="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/Maintenance_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "False" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check Maintenance="${ok_status}""
else
echo "# Check Maintenance="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
echo ""&&echo ""
echo "# Command : openstack image list"
openstack image list
echo ">>> Source change to overcloudrc"
source /home/stack/overcloudrc
echo ""&&echo ""
echo "# Command : openstack hypervisor list --long --sort \"Hypervisor Hostname\""
openstack hypervisor list --long --sort "Hypervisor Hostname"
echo ">>> Check: State, vCPUs, Memory MB"
openstack hypervisor list --long --sort "Hypervisor Hostname" | grep 'ovscompute' | cut -d '|' -f6 | sed 's/^ //' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/State_list.txt
openstack hypervisor list --long --sort "Hypervisor Hostname" | grep 'ovscompute' | cut -d '|' -f8 | sed 's/^ //' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/vCPUs_list.txt
openstack hypervisor list --long --sort "Hypervisor Hostname" | grep 'ovscompute' | cut -d '|' -f10 | sed 's/^ 39288[0-9] /39288/' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/MemoryMB_list.txt
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/State_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "up" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check State="${ok_status}""
else
echo "# Check State="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/vCPUs_list.txt"
i=1
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "70" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check vCPUs="${ok_status}""
else
echo "# Check vCPUs="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/MemoryMB_list.txt"
i=1
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "39288" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check MemoryMBUsed="${ok_status}""
else
echo "# Check MemoryMBUsed="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
echo ""&&echo ""
echo "# Command : openstack availability zone list"
openstack availability zone list
echo ">>> Check: Zone Status"
openstack availability zone list | awk '{print $4}' | sed 's/|//' | sed '/^$/d' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/Zone_Status_list.txt
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/Zone_Status_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "available" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check Zone Status="${ok_status}""
else
echo "# Check Zone Status="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
echo ""&&echo ""
echo "# Command : openstack host list --sort Zone"
openstack host list --sort Zone
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/Zone_name_list.txt"
openstack host list --sort Zone | grep "compute" | cut -d "|" -f4 | sed 's/^ //' | tr -d ' ' | sort -u > "${source_filepath}"
echo ">>> All Compute : `openstack host list --sort Zone | grep -i "compute" | wc -l `"
echo ">>> Compute in each Zone"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ]; then
echo "${line} : `/usr/bin/openstack host list --sort Zone | grep "${line}" | wc -l`"
((i+=1))
elif [ $i -eq $end_number ]; then
echo "${line} : `/usr/bin/openstack host list --sort Zone | grep "${line}" | wc -l`"
break
else
break
fi
done < "${source_filepath}"
echo ""&&echo ""
echo "# Command : openstack compute service list"
openstack compute service list
echo ">>> Check: Status, State"
openstack compute service list | grep "ovscompute" | cut -d '|' -f6 | sed 's/^ //' | tr -d ' ' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/Status_list.txt
openstack compute service list | grep "ovscompute" | cut -d '|' -f7 | sed 's/^ //' | tr -d ' ' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/State_list.txt
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/Status_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "enabled" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check Status="${ok_status}""
else
echo "# Check Status="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/State_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "up" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check State="${ok_status}""
else
echo "# Check State="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
echo ""&&echo ""
echo "# Command : openstack volume service list"
openstack volume service list
echo ">>> Check: Status, State"
openstack volume service list | grep "tripleo" | cut -d '|' -f5 | sed 's/^ //' | tr -d ' ' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/Status_list.txt
openstack volume service list | grep "tripleo" | cut -d '|' -f6 | sed 's/^ //' | tr -d ' ' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/State_list.txt
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/Status_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "enabled" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check Status="${ok_status}""
else
echo "# Check Status="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/State_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "up" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check State="${ok_status}""
else
echo "# Check State="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
echo ""&&echo ""
echo "# Command : openstack volume type list"
openstack volume type list
echo ">>> Check: Is Public"
openstack volume type list | grep "tripleo" | cut -d '|' -f4 | sed 's/^ //' | tr -d ' ' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/IsPublic_list.txt
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/IsPublic_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "True" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check Is Public="${ok_status}""
else
echo "# Check Is Public="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
echo ""&&echo ""
echo "# Command : openstack volume type list --default"
openstack volume type list --default
echo ""&&echo ""
echo "# Command : openstack network agent list --sort Host"
openstack network agent list --sort Host
echo ">>> Check: Alive, State"
openstack network agent list --sort Host | grep "overcloud" | cut -d '|' -f6 | sed 's/^ //' | tr -d ' ' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/Alive_list.txt
openstack network agent list --sort Host | grep "overcloud" | cut -d '|' -f7 | sed 's/^ //' | tr -d ' ' > /home/stack/tmp/healthcheck_dir/cbis_checkfiles/State_list.txt
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/Alive_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == ":-)" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check Alive="${ok_status}""
else
echo "# Check Alive="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/State_list.txt"
i=1
end_number=`wc -l "${source_filepath}" 2>&1 | awk '{print $1}'`
while read line; do
if [ $i -lt $end_number ] && [ "${line}" == "UP" ]; then
((i+=1))
elif [ $i -eq $end_number ]; then
echo "# Check State="${ok_status}""
else
echo "# Check State="${ko_status}""
echo ">>> Line:$i:Error"
break
fi
done < "${source_filepath}"
echo ""&&echo ""
echo "# Command : o 0 -q sudo pcs status"
echo ""
source /usr/share/cbis/undercloud/tools/ssh-overcloud.sh Controller ${Controller_number} -q sudo pcs status
echo ""
echo ">>> Check Controllers 3 online"
source /usr/share/cbis/undercloud/tools/ssh-overcloud.sh Controller ${Controller_number} -q sudo pcs status | grep "^Online" | awk '{print $3}' | grep "[0-9]"
source /usr/share/cbis/undercloud/tools/ssh-overcloud.sh Controller ${Controller_number} -q sudo pcs status | grep "^Online" | awk '{print $4}' | grep "[0-9]"
source /usr/share/cbis/undercloud/tools/ssh-overcloud.sh Controller ${Controller_number} -q sudo pcs status | grep "^Online" | awk '{print $5}' | grep "[0-9]"
source_filepath="/home/stack/tmp/healthcheck_dir/cbis_checkfiles/PCS_Status_list.txt"
echo ""
echo ">>> PCS Status check"
source /usr/share/cbis/undercloud/tools/ssh-overcloud.sh Controller ${Controller_number} -q sudo pcs status | egrep "Started|Master|Slave|active|enabled" | wc -l > "${source_filepath}"
cat_source_file=`cat ${source_filepath}`
if [ "$cat_source_file" -ge 25 ]; then
echo "# Check PCS Status="${ok_status}""
else
echo "# Check PCS Status="${ko_status}""
fi
echo ""&&echo ""
function salt { source ~/venv/salt-ssh/bin/activate; salt-ssh -c /home/stack/salt/etc/salt/ --log-file /home/stack/salt/var/log/salt/ssh --no-host-keys "$@"; deactivate; }
echo ""&&echo ""
echo "# Command : salt \"*cont*\" cmd.run \"hostname -s | grep -o -E '[0-9]+' | tail -1 | xargs -i sudo docker exec galera-bundle-docker-{} clustercheck\""
salt "*cont*" cmd.run "hostname -s | grep -o -E '[0-9]+' | tail -1 | xargs -i sudo docker exec galera-bundle-docker-{} clustercheck"
echo ""&&echo
if [ "$1" = "--all" ]; then
echo "# Command : salt \"*\" cmd.run \"lscpu | grep -e 'CPU(s)' -e Core -e Socket -e NUMA -e Virtualization\""
salt "*" cmd.run "lscpu | grep -e 'CPU(s)' -e Core -e Socket -e NUMA -e Virtualization"
echo ""&&echo ""
echo "# Command : salt \"*\" cmd.run \"numactl -H\""
salt "*" cmd.run "numactl -H"
echo ""&&echo ""
echo "# Command : salt \"*cont*\" cmd.run \"numactl -s\""
salt "*cont*" cmd.run "numactl -s"
echo ""&&echo ""
echo "# Command : salt \"*comp*\" cmd.run \"numactl -s\""
salt "*comp*" cmd.run "numactl -s"
echo ""&&echo ""
echo "# Command : salt \"*\" cmd.run \"free -h\""
salt "*" cmd.run "free -h"
echo ""&&echo ""
echo "# Command : salt \"*comp*\" cmd.run \"grep -e HugePages_Total -e Hugepagesize /proc/meminfo\""
salt "*comp*" cmd.run "grep -e HugePages_Total -e Hugepagesize /proc/meminfo"
echo ""&&echo ""
echo "# Command : salt \"*\" cmd.run 'lspci | grep Ethernet'"
salt "*" cmd.run 'lspci | grep Ethernet'
echo ""&&echo ""
echo "# Command : salt \"*\" cmd.run \"ip l | grep -e em[12] -e p3p[12]\""
salt "*" cmd.run "ip l | grep -e em[12] -e p3p[12]"
echo ""&&echo ""
echo "# Command : salt \"*\" cmd.run \"head /sys/class/net/{em[12],p3p[12]}/device/numa_node\""
salt "*" cmd.run "head /sys/class/net/{em[12],p3p[12]}/device/numa_node"
echo ""&&echo ""
echo "# Command : salt \"*cont*\" cmd.run \"ip -4 a | grep -e mtu -e inet\""
salt "*cont*" cmd.run "ip -4 a | grep -e mtu -e inet"
echo ""&&echo ""
echo "# Command : salt \"*comp*\" cmd.run \"ip -4 a | grep -e mtu -e inet\""
salt "*comp*" cmd.run "ip -4 a | grep -e mtu -e inet"
echo ""&&echo ""
echo "# Command : salt \"*cont*\" cmd.run \"lsblk\""
salt "*cont*" cmd.run "lsblk"
echo ""&&echo ""
echo "# Command : salt \"*comp*\" cmd.run \"lsblk\""
salt "*comp*" cmd.run "lsblk"
echo ""&&echo ""
else
echo "skip"
echo ""&&echo ""
fi
echo "# Command : salt \"*cont*\" cmd.run \"ntpq -pn\""
salt "*cont*" cmd.run "ntpq -pn"
echo ""&&echo ""
echo "# Command : salt \"*comp*\" cmd.run \"ntpq -pn\""
salt "*comp*" cmd.run "ntpq -pn"
echo ""&&echo ""
echo ">>> CBIS Healthcheck Finish"
date
echo ""&&echo ""