Was switching script (Aliyun)

• 알리바바 클라우드 사용
  로드밸러서를 조절하여 WAS 서버 스위칭 및 재구동 SCRIPT

Environment Variables File (envar):

#: Title        : Switching WAS servers file
#: Description  : 스위칭할 WAS 서버들의 정보 기입

homeDir=$(realpath "$(dirname "$0")")

# variables
aliyunUser="***"

# ONLINE_WAS group
vServerGroupId="***"

# WAS instance id
declare -a instances=(
    "server WAS1|private IP|instance ID"
    "server WAS2|private IP|instance ID"
    "server WAS3|private IP|instance ID"
    "server WAS4|private IP|instance ID"
)

destPort="***"
shutdownPort="***"

Main Script:

#!/bin/bash
#: Title        : Switching WAS servers
#: Description  : 로드밸런서를 조절하여 WAS 서버 스위칭

# envar 폴더 임포트
. "${homeDir}/envar"

#================================ FUNCTIONS ================================

function timestamp() {
    echo "[$(date +"%Y-%m-%d %H:%M:%S")]"
}

# Delete known hosts
function delete_Hosts() {
    for instIP in "$@"; do
        ssh-keygen -R "${instIP}"
    done
}

# Add known hosts
function add_Hosts() {
    for instIP in "$@"; do
        ssh-keyscan -t ecdsa-sha2-nistp256 "${instIP}" >> ~/.ssh/known_hosts
    done
}

function ask_question {
    local switchNumber=$1

    # if there are no parameters set switchNumber to 1
    if [ -z "$switchNumber" ]; then
        switchNumber=1
    fi

    # if parameters not a number
    if [[ ! $switchNumber =~ ^[0-9]+$ ]]; then
        printf "\e[31m You can only use a number \e[0m\n"
        exit 1
    fi

    # if parameters more than servers
    if [ $switchNumber -gt ${#instances[@]} ]; then
        printf "\e[31m There are more than the current number of servers. \e[0m\n"
        exit 1
    fi

    # if parameters less than or equal to 0
    if [ $switchNumber -le 0 ]; then
        printf "\e[31m Please enter a number greater than 1 \e[0m\n"
        exit 1
    fi
}

function was_restart {
    local serverName=$1
    local instanceIp=$2
    local instanceId=$3

    printf "\e[32m$(timestamp) [${serverName}] stopping tomcat \e[0m\n"
    ssh "$instanceIp" -- /home/~/~/was stop
    sleep 10

    if [ $? -ne 0 ]; then
        printf "\e[31m$(timestamp) [${serverName}] failed to stop \e[0m\n"
        exit 1
    else
        printf "\e[32m$(timestamp) [${serverName}] successfully stopped tomcat \e[0m\n"
    fi

    printf "\e[32m$(timestamp) [${serverName}] starting tomcat \e[0m\n"
    ssh "$instanceIp" -- /home/~/~/was start

    if [ $? -ne 0 ]; then
        printf "\e[31m$(timestamp) [${serverName}] failed to start \e[0m\n"
        exit 1
    else
        printf "\e[32m$(timestamp) [${serverName}] successfully started tomcat \e[0m\n"
    fi
}

function was_weight_to_0 {
    local serverName=$1
    local instanceIp=$2
    local instanceId=$3

    aliyun -p "${aliyunUser}" slb SetVServerGroupAttribute --region '***' --RegionId '***' --endpoint '***'.aliyuncs.com --VServerGroupId "${vServerGroupId}" --BackendServers "[{ \"ServerId\": \"${instanceId}\", \"Weight\": \"0\", \"Port\": \"${destPort}\", \"Type\": \"ecs\" }]" > /dev/null 2>&1

    if [ $? -ne 0 ]; then
        printf "\e[31m$(timestamp) [${serverName}] failed to set weight to 0 \e[0m\n"
        exit 1
    else
        printf "\e[32m$(timestamp) [${serverName}] successfully set weight to 0 \e[0m\n"
    fi

    printf "\e[32m$(timestamp) [${serverName}] waiting for ${destPort}/tcp to be closed... \e[0m\n"

    while [[ $(ssh "$instanceIp" "netstat -ant | awk '/${destPort}/ && /ESTABLISHED/' | wc -l") -gt 0 ]]; do
        sleep 10
    done

    printf "\e[32m$(timestamp) [${serverName}] successfully closed ${destPort}/tcp \e[0m\n"
}

function was_weight_to_100 {
    local serverName=$1
    local instanceIp=$2
    local instanceId=$3

    printf "\e[32m$(timestamp) [${serverName}] waiting for ${shutdownPort}/tcp to be open... \e[0m\n"

    while [[ $(ssh "$instanceIp" "netstat -nlt | grep ${shutdownPort} | wc -l") -lt 1 ]]; do
        sleep 10
    done

    printf "\e[32m$(timestamp) [${serverName}] successfully opened ${shutdownPort}/tcp \e[0m\n"
    sleep 60 # Wait for CPU to stabilize before attaching to SLB

    aliyun -p "${aliyunUser}" slb SetVServerGroupAttribute --region '***' --RegionId '***' --endpoint '***'.aliyuncs.com --VServerGroupId "${vServerGroupId}" --BackendServers "[{ \"ServerId\": \"${instanceId}\", \"Weight\": \"100\", \"Port\": \"${destPort}\", \"Type\": \"ecs\" }]" > /dev/null 2>&1

    if [ $? -ne 0 ]; then
        printf "\e[31m$(timestamp) [${serverName}] failed to set weight to 100 \e[0m\n"
        exit 1
    else
        printf "\e[32m$(timestamp) [${serverName}] successfully set weight to 100 \e[0m\n"
    fi
}

function was_switching {
    local serverName=$1
    local instanceIp=$2
    local instanceId=$3

    # Database checking log
    checkDatabase "$serverName" "$instanceIp"

    # SLB weight to zero
    was_weight_to_0 "$serverName" "$instanceIp" "$instanceId"

    # WAS restart
    was_restart "$serverName" "$instanceIp" "$instanceId"

    # SLB weight to hundred
    was_weight_to_100 "$serverName" "$instanceIp" "$instanceId"
}

# Count server weight 0
function count_server_weight_0 {
    server_ids=$(aliyun -p "${aliyunUser}" slb DescribeVServerGroupAttribute --region '***' --RegionId '***' --endpoint '***'.aliyuncs.com --VServerGroupId "${vServerGroupId}" | jq -r '.BackendServers.BackendServer[] | select(.Weight == 0) | .ServerId')

    if [[ -z "$server_ids" ]]; then
        cnt=0
    else
        cnt=$(echo "$server_ids" | wc -w)
    fi
    echo "$cnt"
}

#================================== MAIN ==================================

switchNumber=$1 # 스위칭 서버 개수

ask_question "$switchNumber"

declare -a instanceNameArr
declare -a instanceIpArr
declare -a instanceIdArr

for instance in "${instances[@]}"; do
    IFS="|" read -r instanceName instanceIp instanceId <<< "$instance"
    instanceNameArr+=("$instanceName")
    instanceIpArr+=("$instanceIp")
    instanceIdArr+=("$instanceId")
done

delete_Hosts "${instanceIpArr[@]}"
add_Hosts "${instanceIpArr[@]}"

printf "\e[32m$(timestamp) WAS switching start... \e[0m\n"
start_time=$(date +%s)

# WAS switching
cnt=0

while true; do
    result=$(count_server_weight_0) # count was server weight 0
    if [ ${result} -lt ${switchNumber} ]; then
        i=${cnt}
        was_switching "${instanceNameArr[$i]}" "${instanceIpArr[$i]}" "${instanceIdArr[$i]}" &
        ((cnt++))
    fi
    sleep 10
    if [ ${cnt} -eq ${#instances[@]} ]; then
        break
    fi
done
wait

end_time=$(date +%s)
action_time=$((end_time - start_time))

printf "\e[32mSwitching WAS successfully finished \e[0m\n"
echo "Total action time: $action_time seconds"