#!/bin/bash

#
# Check
#

#set -x

#==============================================================================#
VERSION="v2015060900"
 
#------------------------------------------------------------------------------#
### damit auch alle Programme gefunden werden
PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
WGETOPTS="-4 -l 0 --no-check-certificate --no-proxy --no-dns-cache --no-cache --no-cookies --no-http-keep-alive --ignore-length --max-redirect=0 --unlink -T 2 -t 1 -nc"

#------------------------------------------------------------------------------#
#
# Die Verbindungsüberprüfung wird auf eine Anzahl von Prozessen verteilt,
# die in dieser Variablen gespeichert wird.
#
# Sollte diese Variable leer sein, wird die Anzahl der CPU-Kerne verwendet.
#PROZESSE=8

if [ -z "${PROZESSE}" ] ; then
        PROZESSE="$(grep -E '^processor' /proc/cpuinfo | wc -l)"
fi

#------------------------------------------------------------------------------#
### dieses Skript kann nicht einzeln gestartet werden

if [ -z "${LDIRECTORD_IFN_CF}" ] ; then
        ARBEITSVERZEICHNIS="$(dirname ${0})"
        echo "
        ${ARBEITSVERZEICHNIS}/LB_Modus.sh Start
        ${ARBEITSVERZEICHNIS}/LB_Modus.sh Stop
        ${ARBEITSVERZEICHNIS}/LB_Modus.sh Check
        "
        exit 1
fi

#==============================================================================#
### Funktionen

#------------------------------------------------------------------------------#
ip_out()
{
if [ -z "$(echo "${1}" | grep -E '[0-9][0-9]*.[0-9][0-9]*.[0-9][0-9]*.[0-9][0-9]*')" ] ; then
        host ${1} | awk '{print $NF}'
else
        echo ${1}
fi
}

#------------------------------------------------------------------------------#
check_now()
{
cat "${1}" | while read ZEILE
do
        #----------------------------------------------------------------------#
        #IFNAME="$(echo "${ZEILE}" | grep -Ev '^$' | awk -F'|' '{print $1}' | head -n1)"
        VIPADR="$(echo "${ZEILE}" | grep -Ev '^$' | awk -F'|' '{print $2}' | head -n1)"
        unset BLOCK
        BLOCK="$(echo "${ZEILE}" | tr -s '|' '\n')"
        VIRTUAL="$(echo "${BLOCK}" | grep -F 'virtual=' | sed 's/virtual=//;s/:[0-9]*//' | awk '{print $1}')"
 
        ### hier wird (wegen der Einfachheit) der Port von "virtual" auch fuer "real" verwendet
        TPORT="$(echo "${BLOCK}" | grep -F 'virtual=' | awk -F':' '{print $NF}' | awk '{print $1}')"
        ALLEREALIPS="$(echo "${BLOCK}" | grep -F 'real=' | sed 's/real=//;s/:[0-9]*//' | awk '{print $1}')"
        #FALLBACK="$(echo "${BLOCK}" | grep -F 'fallback=' | sed 's/fallback=//;s/:[0-9]*//' | awk '{print $1}')"
        FALLBACK="$(echo "${BLOCK}" | grep -F 'fallback=' | sed 's/fallback=//;' | awk '{print $1}')"
        #SCHEDULER="$(echo "${BLOCK}" | grep -F 'scheduler=' | sed 's/scheduler=//' | awk '{print $1}')"
        SERVICE="$(echo "${BLOCK}" | grep -F 'service=' | sed 's/service=//;s/["]//g' | awk '{print $1}')"
        REQUEST="$(echo "${BLOCK}" | grep -F 'request=' | sed 's/request=//;s/["]//g' | awk '{print $1}')"
        RECEIVE="$(echo "${BLOCK}" | grep -F 'receive=' | sed 's/receive=//;s/["]//g' | awk '{print $1}')"
        #PERSISTENT="$(echo "${BLOCK}" | grep -F 'persistent=' | sed 's/persistent=//;s/["]//g' | awk '{print $1}')"
        #PROTOCOL="$(echo "${BLOCK}" | grep -F 'protocol=' | sed 's/protocol=//;s/["]//g' | awk '{print $1}')"
 
        #----------------------------------------------------------------------#
        ### hier wird die FallBack-Verbindung wieder entfernt

        #echo "FALLBACK='${FALLBACK}'" >> /tmp/Check.log ; sync
        if [ -z "${FALLBACK}" ] ; then
                FB_AUS=""
        else
                FBEINTAG="$(echo "${FALLBACK}" | awk -F':' '{print $1}')"
                FBADR="$(ip_out ${FBEINTAG})"
                FBPORT="$(echo "${FALLBACK}" | awk -F':' '{print $2}')"

                FB_EIN="-a -t ${VIPADR}:${TPORT} -r ${FBADR}:${FBPORT} -m
                -e -t ${VIPADR}:${TPORT} -r ${FBADR}:${FBPORT} -w 1"

                FB_AUS="-e -t ${VIPADR}:${TPORT} -r ${FBADR}:${FBPORT} -w 0
                -d -t ${VIPADR}:${TPORT} -r ${FBADR}:${FBPORT}"
        fi
 
        #----------------------------------------------------------------------#

        (for REAL_IP in ${ALLEREALIPS}
        do
                TEMP="/tmp/ipvsadm_Check_$(head -c 1000 /dev/urandom | tr -cd '[:alnum:]' | cut -b-12)"

                #--------------------------------------------------------------#
                ### hier wird der FQDN der REAL-Hosts in eine IP umgewandelt bzw. die IP uebernommen
                RIPADR="$(echo "${IP_FQDN}" | fgrep "|${REAL_IP}" | awk -F'|' '{print $1}')"
                if [ -z "${RIPADR}" ] ; then
                        RIPADR="${REAL_IP}"
                fi
 
                #==============================================================#
                ### hier wird der IST-Zustand einer LB-Verbindung zum Real-Server ausgelesen
                #echo "3: ipvsadm -Ln | fgrep \"${RIPADR}:${TPORT}\""                                 >> /tmp/Check.log ; sync
                #echo "#-> 0: ipvsadm -Ln | fgrep '${RIPADR}:${TPORT}'" >> /tmp/Check.log ; sync
                IST_WEIGHT="$(ipvsadm -Lnt ${VIPADR}:${TPORT} | fgrep "${RIPADR}:${TPORT}" | awk '/Route/{sub(".*Route","");print $1}' | head -n 1)"
                if [ -z "${IST_WEIGHT}" ] ; then
                        IST_WEIGHT="$(ipvsadm -Ln | fgrep "${RIPADR}:${TPORT}" | awk '/Masq/{sub(".*Masq","");print $1}' | head -n 1)"
                        if [ "${IST_WEIGHT}" = "1" ] ; then
                                IST_WEIGHT="1"
                        else
                                IST_WEIGHT="0"
                        fi
                fi

                if [ -n "${IST_WEIGHT}" ] ; then
 
                        #echo "5: RIPADR:TPORT='${RIPADR}:${TPORT}'" >> /tmp/Check.log ; sync
                        #------------------------------------------------------#
                        if [ "smtp" = "${SERVICE}" ] ; then
                                #
                                # SMTP
                                #
                                #echo "----------------------------------------" >> /tmp/Check.log ; sync
                                #echo "6: wget ${WGETOPTS} -O ${TEMP} http://${RIPADR}:${TPORT}/${REQUEST}" >> /tmp/Check.log ; sync
                                wget -q ${WGETOPTS} -O ${TEMP} http://${RIPADR}:${TPORT}/${REQUEST}
                                BEKOMMEN="$(cat ${TEMP} | awk '/^220 /{print $1}' | head -n1 ; rm -f ${TEMP})"
                                #echo "BEKOMMEN='${BEKOMMEN}'" >> /tmp/Check.log ; sync
 
                                if [ "${BEKOMMEN}" = "220" ] ; then
                                        if [ "${IST_WEIGHT}" = "0" ] ; then
                                                echo "-e -t ${VIPADR}:${TPORT} -r ${RIPADR}:${TPORT} -w 1 (Weight 1)" | logger -t LoadBalancer
                                                echo "-e -t ${VIPADR}:${TPORT} -r ${RIPADR}:${TPORT} -w 1"
                                        fi
                                else
                                        if [ "${IST_WEIGHT}" = "1" ] ; then
                                                echo "-e -t ${VIPADR}:${TPORT} -r ${RIPADR}:${TPORT} -w 0 (Weight 0)" | logger -t LoadBalancer
                                                echo "-e -t ${VIPADR}:${TPORT} -r ${RIPADR}:${TPORT} -w 0"
                                        fi
                                fi
                        else
                                #
                                # HTTP / HTTPS
                                #
                                #echo "----------------------------------------" >> /tmp/Check.log ; sync
                                #echo "7: wget ${WGETOPTS} -O ${TEMP} ${SERVICE}://${RIPADR}:${TPORT}/${REQUEST}" >> /tmp/Check.log ; sync
                                VERBUNDEN="$(wget ${WGETOPTS} -O ${TEMP} ${SERVICE}://${RIPADR}:${TPORT}/${REQUEST} 2>&1 | awk '/connected|verbunden/{print $NF}')"
                                #echo "8: VERBUNDEN='${VERBUNDEN}'" >> /tmp/Check.log ; sync

                                BEKOMMEN="$(cat ${TEMP} | grep -Ev '^$' | sed 's/^[ \t]//g;s/[ \t]$//g' ; rm -f ${TEMP})"
                                #echo "9: BEKOMMEN='${BEKOMMEN}' | RECEIVE='${RECEIVE}'" >> /tmp/Check.log ; sync

                                if [ "${VERBUNDEN}" = "connected." -o "${VERBUNDEN}" = "verbunden." ] ; then
                                        #echo "- ipvsadm: 1" >> /tmp/Check.log ; sync
                                        if [ -z "${RECEIVE}" ] ; then
                                                #echo "- ipvsadm: 2" >> /tmp/Check.log ; sync
                                                if [ "${IST_WEIGHT}" = "0" ] ; then
                                                        echo "-e -t ${VIPADR}:${TPORT} -r ${RIPADR}:${TPORT} -w 1 (Weight 1)" | logger -t LoadBalancer
                                                        echo "-e -t ${VIPADR}:${TPORT} -r ${RIPADR}:${TPORT} -w 1"
                                                fi
                                        else
                                                #echo "- ipvsadm: 4" >> /tmp/Check.log ; sync
                                                if [ "${RECEIVE}" = "${BEKOMMEN}" ] ; then
                                                        #echo "- ipvsadm: 05" >> /tmp/Check.log ; sync
                                                        if [ "${IST_WEIGHT}" = "0" ] ; then
                                                                echo "-e -t ${VIPADR}:${TPORT} -r ${RIPADR}:${TPORT} -w 1 (Weight 1)" | logger -t LoadBalancer
                                                                echo "-e -t ${VIPADR}:${TPORT} -r ${RIPADR}:${TPORT} -w 1"
                                                        fi
                                                else
                                                        #echo "- ipvsadm: 07" >> /tmp/Check.log ; sync
                                                        if [ "${IST_WEIGHT}" = "1" ] ; then
                                                                echo "-e -t ${VIPADR}:${TPORT} -r ${RIPADR}:${TPORT} -w 0 (Weight 0)" | logger -t LoadBalancer
                                                                echo "-e -t ${VIPADR}:${TPORT} -r ${RIPADR}:${TPORT} -w 0"
                                                        fi
                                                fi
                                        fi
                                else
                                        if [ "${IST_WEIGHT}" = "1" ] ; then
                                                echo "-e -t ${VIPADR}:${TPORT} -r ${RIPADR}:${TPORT} -w 0 (Weight 0)" | logger -t LoadBalancer
                                                echo "-e -t ${VIPADR}:${TPORT} -r ${RIPADR}:${TPORT} -w 0"
                                        fi
                                fi
                        fi
                fi
                #==============================================================#
                ### FallBack
                if [ -n "${FALLBACK}" ] ; then
                        IPVSADM_LNT="$(ipvsadm -Lnt ${VIPADR}:${TPORT} | fgrep Route)"
                        ROUTE_SUM="$(echo "${IPVSADM_LNT}" | sed 's/.*Route//' | awk '{z=$1;s+=z}END{print s}')"
                        ROUTE_CL="$(echo "${IPVSADM_LNT}" | grep -Fv "> ${FBADR}:" | sed 's/.*Route//' | awk '{z=$1;s+=z}END{print s}')"
                        ROUTE_FB="$(echo "${IPVSADM_LNT}" | grep -F "> ${FBADR}:")"
                        if [ "${ROUTE_CL}" -eq "0" ] ; then
                                if [ -z "${ROUTE_FB}" ] ; then
                                        echo "Added fallback server: ${FBADR}:${FBPORT}" | logger -t LoadBalancer
                                        echo "${FB_EIN}"
                                fi
                        else
                                if [ -n "${ROUTE_FB}" ] ; then
                                        echo "Deleted fallback server: ${FBADR}:${FBPORT}" | logger -t LoadBalancer
                                        echo "${FB_AUS}"
                                fi
                        fi
                fi
 
        done) | ipvsadm -R #2>/dev/null
done

rm -f ${1}

### wenn Schluss ist, dann muss auch Schluss sein
if [ "${STARTZEIT}" != "$(cat ${RUN} 2>/dev/null)" ]
then
        echo "Ende von ${STARTZEIT}"
        exit 0
fi

}

#==============================================================================#
### Run

###
### Bearbeitung je Cluster
### mit mehreren Prozessen
### maximale Anzahl der Prozesse = ${PROZESSE}
###
echo "${LDIRECTORD_IFN_CF}" | nl | while read NR CLUSTER
do
        #----------------------------------------------------------------------#
        ### es duerfen nur max. soviele Prozesse laufen,
        ### wie CPU-Kerne im System stecken

        ANZAHL_LAUFEN="$(ps alx | fgrep -v grep | fgrep "/bin/bash /root/bin/LB_Check.sh" | wc -l)"
        #echo "ANZAHL_LAUFEN=${ANZAHL_LAUFEN}"

        while (( ${ANZAHL_LAUFEN} >= ${PROZESSE} ))
        do
                sleep 0.1
                ANZAHL_LAUFEN="$(ps alx | fgrep -v grep | fgrep "/bin/bash /root/bin/LB_Check.sh" | wc -l)"
                #echo "ANZAHL_LAUFEN=${ANZAHL_LAUFEN}"
        done

        #----------------------------------------------------------------------#
        ### wenn Schluss ist, dann muss auch Schluss sein

        while [ "${STARTZEIT}" != "$(cat ${RUN} 2>/dev/null)" ]
        do
                exit 0
        done

        #----------------------------------------------------------------------#
        ### Start

        DATENDATEI="/tmp/ipvsadm_Cluster_Check_${NR}"
        echo "${CLUSTER}" > ${DATENDATEI}
        ( check_now ${DATENDATEI} ) &
done