"Fossies" - the Fresh Open Source Software Archive

Member "mosshe/functions.mosshe" (26 Sep 2020, 31820 Bytes) of package /linux/privat/old/mosshe.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Bash source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file.

A hint: This file contains one or more very long lines, so maybe it is better readable using the pure text view mode that shows the contents as wrapped lines within the browser window.


    1 #!/bin/bash
    2 
    3 #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    4 #
    5 #   function library - please call with external script
    6 #
    7 #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    8 
    9 #############################################################################
   10 # MoSSHe: remote server monitoring environment
   11 #
   12 # Copyright (C) 2009- Volker Tanger
   13 #
   14 # This program is free software; you can redistribute it and/or
   15 # modify it under the terms of the GNU General Public License
   16 # as published by the Free Software Foundation; either version 2
   17 # of the License, or (at your option) any later version.
   18 #
   19 # For bug reports and suggestions or if you just want to talk to me please
   20 # contact me at volker.tanger@wyae.de
   21 #
   22 # Updates will be available at  http://www.wyae.de/software/mosshe/
   23 # please check there for updates prior to submitting patches!
   24 #
   25 # For list of changes please refer to the HISTORY file. Thanks.
   26 #############################################################################
   27 
   28 
   29 MOSSHEVERSION="20.9.25"
   30 
   31 #---------------------------------------------------------
   32 # init library and files
   33 #---------------------------------------------------------
   34 
   35 HTMLALERT="#ff0000"
   36 HTMLWARN="#ffff00"
   37 HTMLUNDEF="#5555ff"
   38 HTMLINFO="#ffffff"
   39 HTMLOK="#55ff55"
   40 
   41 DATUM=`date +%Y-%m-%d`
   42 ZEIT=`date +%H:%M:%S`
   43 DATIM="${DATUM};${ZEIT}"
   44 
   45 MOSSHELOG=`date +"${TEMPDIR}/mosshelog_%Y-%m-%d_%Hh%M"`
   46 
   47 
   48 # Lookup a Group short name via DNS TXT if no MYGROUP supplied
   49 # Currently only djbdns's dnstxt supported
   50 if [ -z "$MYGROUP" ]; then
   51     LEVELS=$(echo "$MYDOM" | grep -o '\.' | wc -l)
   52     LEVELS=$(( LEVELS - 1 ))
   53     for i in $(seq 1 $LEVELS); do
   54         MYGROUP=$([ `which dnstxt` ] && dnstxt mygroup.$(echo $MYDOM | cut -d. -f $i-))
   55         [ "$MYGROUP" ] && break
   56     done
   57 fi
   58 MYGROUP=${MYGROUP:-$MYDOM}
   59 
   60 #############################################################################
   61 ###     MoSShE internal functions
   62 #############################################################################
   63 
   64 MossheLog () {
   65    date +"%Y-%m-%d %H:%M:%S - $1 " >> $MOSSHELOG
   66 }
   67 
   68 
   69 #---------------------------------------------------------
   70 # Self-Check & Self-Locking - email to admin if in distress
   71 #---------------------------------------------------------
   72 
   73 MossheSelfCheck () {
   74     MossheLog "MossheSelfCheck"
   75     if [ -f $TEMPDIR/mosshe_lock.5 ]; then
   76         date | mail -s "$MYGROUP: $MYNAME - MoSShE in deadlock" $1
   77     rm -f $TEMPDIR/mosshe_lock.*
   78     killall mosshe
   79     fi
   80 
   81     if [ -f $TEMPDIR/mosshe_lock.4 ]; then date > $TEMPDIR/mosshe_lock.5; 
   82     elif [ -f $TEMPDIR/mosshe_lock.3 ]; then date > $TEMPDIR/mosshe_lock.4;
   83     elif [ -f $TEMPDIR/mosshe_lock.2 ]; then date > $TEMPDIR/mosshe_lock.3;
   84     elif [ -f $TEMPDIR/mosshe_lock.1 ]; then date > $TEMPDIR/mosshe_lock.2;
   85     elif [ -f $TEMPDIR/mosshe_lock.0 ]; then date > $TEMPDIR/mosshe_lock.1
   86     fi
   87 
   88     LOCKSEXIST=`ls $TEMPDIR/mosshe_lock.? 2>/dev/null`
   89     if [ -n "$LOCKSEXIST" ]; then
   90         echo "MossheSelfCheck failed: MoSShE locked.  "
   91     echo "Self-terminating.  Fix with rm -f $TEMPDIR/mosshe_lock.?" 
   92     echo " "
   93     echo " "
   94     cat ${TEMPDIR}/mosshelog_*
   95     date +"%Y-%m-%d %H:%M:%S - another MoSShE started and ran into lock"
   96         rm -f ${MOSSHELOG}
   97         kill "$$"
   98     fi
   99     rm -f ${TEMPDIR}/mosshelog_*
  100     MossheLog "MossheSelfCheck - successfully established lock"    
  101     date > $TEMPDIR/mosshe_lock.0
  102     
  103 #    rm "$WWWDIR/srv_$MYNAME.txt"
  104 }
  105 
  106 
  107 #---------------------------------------------------------
  108 # FinalizeLog - finalize, cleanup, HTMLize - any parameter to remove HTMLs first
  109 #---------------------------------------------------------
  110 FinalizeLog () {
  111     MossheLog "FinalizeLog $1"
  112 
  113     [ "$1" -a "$WWWDIR" ] && rm -f $WWWDIR/*.html
  114 
  115     if [ ! -f $WWWDIR/plotdata/dygraph-combined.js ]; then
  116         MossheLog "MossheFinalizeLog - graphing routines missing in $WWWDIR/plotdata/"
  117     fi
  118 
  119     typeset -i OKCOUNT FAILCOUNT
  120     OKCOUNT=0
  121     FAILCOUNT=0
  122     
  123     if [ -f $TEMPDIR/mosshe_collected.newstatus ]; then
  124         mv $TEMPDIR/mosshe_collected.newstatus $TEMPDIR/mosshe_collected.oldstatus
  125     else
  126         echo "MoSShE probably was restarted? Could not find $TEMPDIR/mosshe_collected.newstatus ..."
  127     fi
  128     
  129     cat $DATADIR/www/template_head.thtml \
  130         | sed -e "s/TIMESTAMP/$DATUM - $ZEIT/g" \
  131     | tee $TEMPDIR/mosshe_collected.html \
  132     > $TEMPDIR/mosshe_nonokcollected.html
  133     echo '<tr bgcolor="#dddddd" class="border datarowhead"><td><b>Timestamp</b></td><td><b>Group</b></td><td><b>System</b></td><td><b>Property</b></td><td><b>Status</b></td><td><b>Value</b></td><td><b>Detail</b></td></tr>' \
  134     | tee -a $TEMPDIR/mosshe_collected.html \
  135     >> $TEMPDIR/mosshe_nonokcollected.html
  136     EPOCH=`date +%s`
  137     rm -f $TEMPDIR/mosshe_nagiosresults.$$.tmp
  138     
  139     LASTLINEHEAD=;
  140     while read LINE; do
  141     DAT=`echo $LINE | cut -d ";" -f 1`
  142     TIM=`echo $LINE | cut -d ";" -f 2`
  143     GROUP=`echo $LINE | cut -d ";" -f 3`
  144     ME=`echo $LINE | cut -d ";" -f 4`
  145     SVC=`echo $LINE | cut -d ";" -f 5`
  146     STAT=`echo $LINE | cut -d ";" -f 6`
  147     VALUE=`echo $LINE | cut -d ";" -f 7`
  148     MSG=`echo $LINE | cut -d ";" -f 8`
  149     if [ "$STAT" = "ALERT" ]; then
  150         COLR=$HTMLALERT
  151         STATNUM=2
  152     elif [ "$STAT" = "WARN" ]; then
  153         COLR=$HTMLWARN
  154         STATNUM=1
  155     elif [ "$STAT" = "INFO" ]; then
  156         COLR=$HTMLINFO
  157         STATNUM=0
  158     elif [ "$STAT" = "OK" ]; then
  159         COLR=$HTMLOK
  160         STATNUM=0
  161     else
  162         COLR=$HTMLUNDEF
  163         STATNUM=3
  164     fi
  165 
  166     THISLINEHEAD="$COLR ${DAT} $GROUP $ME"
  167     if [ "$THISLINEHEAD" = "$LASTLINEHEAD" ]; then
  168         LINEHEADPROPOSAL="<tr class=\"datarow\"><td></td><td></td><td></td>"
  169     else    
  170         LINEHEADPROPOSAL="<tr height="2"><td colspan="7"></td></tr><tr class=\"datarow\"><td bgcolor=\"$COLR\" class=\"border\">${DAT} ${TIM}</td><td bgcolor=\"$COLR\" class=\"border\"><a href=\"$GROUP.html\">$GROUP</a></td><td bgcolor=\"$COLR\" class=\"border\"><a href=\"srv_$ME.html\">$ME</a></td>"
  171         LASTLINEHEAD=$THISLINEHEAD
  172     fi
  173     LINEHEAD="<!-- | GROUP=$GROUP | SERVER=$ME | -->$LINEHEADPROPOSAL"
  174     
  175     CHK=`echo $SVC | sed -e "s/\//\./g"`
  176     FN="$ME/$CHK"
  177     echo "$LINEHEAD<td bgcolor=\"$COLR\" class=\"border\"><a href=\"/mosshe/plotdata/?$FN\">$SVC</a> <a href=\"/mosshe/plotdata/?$FN.avg\">(l)</a></td><td bgcolor=\"$COLR\" class=\"border\">$STAT</td><td bgcolor=\"$COLR\" class=\"border\">$VALUE</td><td bgcolor=\"$COLR\" class=\"border\">$MSG</td></tr>" >> $TEMPDIR/mosshe_collected.html
  178     
  179     if [ "$STAT" = "OK" ]; then
  180         OKCOUNT=$(( OKCOUNT + 1 ))
  181     elif [ "$STAT" != "INFO" ]; then
  182          echo "$LINEHEAD<td bgcolor=\"$COLR\" class=\"border\"><a href=\"/mosshe/plotdata/?$FN\">$SVC</a> <a href=\"/mosshe/plotdata/?$FN.avg\">(l)</a></td><td bgcolor=\"$COLR\" class=\"border\">$STAT</td><td bgcolor=\"$COLR\" class=\"border\">$VALUE</td><td bgcolor=\"$COLR\" class=\"border\">$MSG</td></tr>" >> $TEMPDIR/mosshe_nonokcollected.html
  183         FAILCOUNT=$(( FAILCOUNT + 1 ))
  184     fi
  185 
  186     echo "$GROUP - $ME - $STAT - $SVC" >> $TEMPDIR/mosshe_collected.newstatus
  187     echo "${EPOCH};${GROUP};${ME};${SVC};${STATNUM};${MSG}" >> $TEMPDIR/mosshe_nagiosresults.$$.tmp
  188     done < "$TEMPDIR/tmp.$$.collected.tmp"
  189     
  190     echo '<tr height="2"><td colspan="7"></td></tr><tr bgcolor="#dddddd" class="border datarowhead"><td><b>Timestamp</b></td><td><b>Group</b></td><td><b>System</b></td><td><b>Property</b></td><td><b>Status</b></td><td><b>Value</b></td><td><b>Detail</b></td></tr>' \
  191     | tee -a $TEMPDIR/mosshe_collected.html \
  192     >> $TEMPDIR/mosshe_nonokcollected.html
  193     
  194     cat $DATADIR/www/template_foot.thtml \
  195         | sed -e "s/TIMESTAMP/$DATUM - $ZEIT/g" \
  196     | sed -e "s/OKCOUNT/$OKCOUNT/g" \
  197     | sed -e "s/FAILCOUNT/$FAILCOUNT/g" \
  198     | tee -a $TEMPDIR/mosshe_collected.html \
  199     >> $TEMPDIR/mosshe_nonokcollected.html
  200     
  201     mv $TEMPDIR/mosshe_collected.html $WWWDIR/index.html
  202     mv $TEMPDIR/mosshe_nonokcollected.html $WWWDIR/notok.html
  203     mv $TEMPDIR/tmp.$$.collected.tmp $WWWDIR/index.csv
  204     mv $TEMPDIR/mosshe_nagiosresults.$$.tmp $WWWDIR/nagios.csv
  205     if [ -f $TEMPDIR/mosshe_collected.oldstatus ]; then
  206         diff $TEMPDIR/mosshe_collected.oldstatus $TEMPDIR/mosshe_collected.newstatus | fgrep -v " - OK - " | fgrep " - " > $TEMPDIR/mosshe_collected.diff
  207     else
  208         echo "MoSShE probably was restarted? Could not find $TEMPDIR/mosshe_collected.oldstatus ..."
  209     fi
  210     
  211     rm -f $TEMPDIR/mosshe_lock.*
  212 
  213     TXTDIRLIST=`ls  $WWWDIR/srv_*.txt.new 2> /dev/null`
  214     if [ -n "$TXTDIRLIST" ]; then
  215       for F in $WWWDIR/srv_*.txt.new; do
  216     OLDF=`echo $F | sed -e "s/.txt.new/.txt/g"`
  217     rm "$OLDF"
  218     mv "$F" "$OLDF"
  219       done
  220     fi
  221 
  222     chmod -R ugo+rX ${WWWDIR}
  223     export OKCOUNT FAILCOUNT
  224 }
  225 
  226 #---------------------------------------------------------
  227 # SortGroups - create per-server pages
  228 #---------------------------------------------------------
  229 SortGroups () {
  230     MossheLog "SortGroups $1"
  231     cat $DATADIR/www/template_head.thtml \
  232         | sed -e "s/TIMESTAMP/$DATUM - $ZEIT/g" \
  233     > $TEMPDIR/groupcollected.html
  234     echo '<tr bgcolor="#dddddd" class="border datarowhead"><td><b>Group</b></td><td><b>Alert</b></td><td><b>Warn</b></td><td><b>Ok</b></td><td><b>Undef</b></td></tr>' \
  235     >> $TEMPDIR/groupcollected.html
  236     
  237     cat $DATADIR/www/template_head.thtml \
  238             | sed -e "s/TIMESTAMP/$DATUM - $ZEIT/g" \
  239         > $TEMPDIR/mosshe.$$.grouphead.tmp
  240     echo '<tr bgcolor="#dddddd" class="border datarowhead"><td><b>Timestamp</b></td><td><b>Group</b></td><td><b>System</b></td><td><b>Property</b></td><td><b>Status</b></td><td><b>Value</b></td><td><b>Detail</b></td></tr>' \
  241     >> $TEMPDIR/mosshe.$$.grouphead.tmp
  242 
  243     cat $DATADIR/www/template_foot.thtml \
  244         | fgrep -v "Totals: <div class=\"ok\">OKCOUNT checks are OK</div> - <div class=\"notok\">FAILCOUNT show problems</div>" \
  245     >> $TEMPDIR/mosshe_foot.$$.tmp
  246 
  247     cat $WWWDIR/index.csv  | cut -d ";" -f 3 | sort -u > $TEMPDIR/mosshe_grouplist.$$.tmp
  248     LASTGROUP=
  249     LASTSERVER=
  250     while read GROUPSERVER; do
  251         set `IFS=";"; echo $GROUPSERVER`
  252         GROUP=$1
  253         SERVER=$2
  254         if [ "$GROUP" = "$LASTGROUP" ]; then
  255         GROUP=
  256             [ "$SERVER" = "$LASTSERVER" ] && SERVER=
  257     fi
  258         if [ "$GROUP" -o "$SERVER" ]; then
  259         echo -n "<tr><td class=\"border\"><a href=\"${GROUP}.html\">${GROUP}</a></td></td>" >> $TEMPDIR/groupcollected.html
  260     else
  261         echo -n "<tr><td></td><td></td>" >> $TEMPDIR/groupcollected.html
  262     fi
  263     for STATUS in ALERT WARN OK UNDEF; do
  264         COUNT=`fgrep ";$STATUS;" $WWWDIR/index.csv | fgrep ";${GROUP};" | wc -l`
  265         if [ "$STATUS" = "ALERT" ]; then
  266         COLR=$HTMLALERT
  267         elif [ "$STATUS" = "WARN" ]; then
  268             COLR=$HTMLWARN
  269         elif [ "$STATUS" = "OK" ]; then
  270             COLR=$HTMLOK
  271         else
  272             COLR=$HTMLUNDEF
  273         fi
  274         if [ "$COUNT" = "0" ]; then
  275             COLR="#ffffff"
  276         fi
  277         echo -n "<td bgcolor=\"$COLR\" align=\"middle\" class=\"border\">$COUNT</td>" >> $TEMPDIR/groupcollected.html
  278     done
  279     echo '</tr>' >> $TEMPDIR/groupcollected.html
  280     
  281     cp $TEMPDIR/mosshe.$$.grouphead.tmp $TEMPDIR/mosshe.$$.group.tmp
  282     fgrep "| GROUP=$GROUP |" $WWWDIR/index.html >> $TEMPDIR/mosshe.$$.group.tmp 
  283     cat $TEMPDIR/mosshe_foot.$$.tmp >> $TEMPDIR/mosshe.$$.group.tmp
  284     mv $TEMPDIR/mosshe.$$.group.tmp "$WWWDIR/$GROUP.html"
  285     
  286         [ "$GROUP" ] && LASTGROUP=$GROUP   
  287         [ "$SERVER" ] && LASTSERVER=$SERVER
  288     done < $TEMPDIR/mosshe_grouplist.$$.tmp
  289     
  290     echo '<tr bgcolor="#dddddd" class="border datarowhead"><td><b>Group</b></td><td><b>Alert</b></td><td><b>Warn</b></td><td><b>Ok</b></td><td><b>Undef</b></td></tr>' \
  291     >> $TEMPDIR/groupcollected.html
  292     cat $TEMPDIR/mosshe_foot.$$.tmp >> $TEMPDIR/groupcollected.html
  293     mv $TEMPDIR/groupcollected.html $WWWDIR/groups.html
  294     chmod ugo+rX ${WWWDIR}/group*.html
  295     
  296     rm $TEMPDIR/mosshe_grouplist.$$.tmp $TEMPDIR/mosshe_foot.$$.tmp $TEMPDIR/mosshe.$$.grouphead.tmp
  297 
  298 
  299     #---------------------------------------------------------
  300     # SortGroupsExpanded - create per-server pages
  301     #---------------------------------------------------------
  302     cat $DATADIR/www/template_head.thtml \
  303         | sed -e "s/TIMESTAMP/$DATUM - $ZEIT/g" \
  304     > $TEMPDIR/servercollected.html
  305     echo '<tr bgcolor="#dddddd" class="border datarowhead"><td><b>Group</b></td><td><b>Server</b></td><td><b>Alert</b></td><td><b>Warn</b></td><td><b>Ok</b></td><td><b>Undef</b></td></tr>' \
  306     >> $TEMPDIR/servercollected.html
  307     
  308     cat $DATADIR/www/template_head.thtml \
  309             | sed -e "s/TIMESTAMP/$DATUM - $ZEIT/g" \
  310         > $TEMPDIR/mosshe.$$.serverhead.tmp
  311     echo '<tr bgcolor="#dddddd" class="border datarowhead"><td><b>Timestamp</b></td><td><b>Group</b></td><td><b>System</b></td><td><b>Property</b></td><td><b>Status</b></td><td><b>Value</b></td><td><b>Detail</b></td></tr>' \
  312     >> $TEMPDIR/mosshe.$$.serverhead.tmp
  313 
  314     cat $DATADIR/www/template_foot.thtml \
  315         | fgrep -v "Totals: <div class=\"ok\">OKCOUNT checks are OK</div> - <div class=\"notok\">FAILCOUNT show problems</div>" \
  316     >> $TEMPDIR/mosshe_foot.$$.tmp
  317 
  318     cat $WWWDIR/index.csv  | cut -d ";" -f 3,4 | sort -t ";" -k 1,2 -u > $TEMPDIR/mosshe_serverlist.$$.tmp
  319     LASTGROUP=
  320     LASTSERVER=
  321     while read GROUPSERVER; do
  322         set `IFS=";"; echo $GROUPSERVER`
  323         GROUP=$1
  324         SERVER=$2
  325         if [ "$GROUP" = "$LASTGROUP" ]; then
  326         GROUP=
  327             [ "$SERVER" = "$LASTSERVER" ] && SERVER=
  328     fi
  329         if [ "$GROUP" -o "$SERVER" ]; then
  330         echo -n "<tr><td class=\"border\"><a href=\"${GROUP}.html\">${GROUP}</a></td><td class=\"border\"><a href=\"srv_${SERVER}.html\">$SERVER</a></td>" >> $TEMPDIR/servercollected.html
  331     else
  332         echo -n "<tr><td></td><td></td>" >> $TEMPDIR/servercollected.html
  333     fi
  334     for STATUS in ALERT WARN OK UNDEF; do
  335         COUNT=`fgrep ";$STATUS;" $WWWDIR/index.csv | fgrep ";${GROUPSERVER};" | wc -l`
  336         if [ "$STATUS" = "ALERT" ]; then
  337         COLR=$HTMLALERT
  338         elif [ "$STATUS" = "WARN" ]; then
  339             COLR=$HTMLWARN
  340         elif [ "$STATUS" = "OK" ]; then
  341             COLR=$HTMLOK
  342         else
  343             COLR=$HTMLUNDEF
  344         fi
  345         if [ "$COUNT" = "0" ]; then
  346             COLR="#ffffff"
  347         fi
  348         echo -n "<td bgcolor=\"$COLR\" align=\"middle\" class=\"border\">$COUNT</td>" >> $TEMPDIR/servercollected.html
  349     done
  350     echo '</tr>' >> $TEMPDIR/servercollected.html
  351     
  352 #   cp $TEMPDIR/mosshe.$$.serverhead.tmp $TEMPDIR/mosshe.$$.server.tmp
  353 #   fgrep "| SERVER=$SERVER |" $WWWDIR/index.html >> $TEMPDIR/mosshe.$$.server.tmp  
  354 #   cat $TEMPDIR/mosshe_foot.$$.tmp >> $TEMPDIR/mosshe.$$.server.tmp
  355 #   mv $TEMPDIR/mosshe.$$.server.tmp "$WWWDIR/$1-$SERVER.html"
  356     
  357         [ "$GROUP" ] && LASTGROUP=$GROUP   
  358         [ "$SERVER" ] && LASTSERVER=$SERVER
  359     done < $TEMPDIR/mosshe_serverlist.$$.tmp
  360     
  361     echo '<tr bgcolor="#dddddd" class="border datarowhead"><td><b>Group</b></td><td><b>Server</b></td><td><b>Alert</b></td><td><b>Warn</b></td><td><b>Ok</b></td><td><b>Undef</b></td></tr>' \
  362     >> $TEMPDIR/servercollected.html
  363     cat $TEMPDIR/mosshe_foot.$$.tmp >> $TEMPDIR/servercollected.html
  364     mv $TEMPDIR/servercollected.html $WWWDIR/groupsexpanded.html
  365     chmod ugo+rX ${WWWDIR}/group*.html $WWWDIR/server*.html
  366     
  367     rm $TEMPDIR/mosshe_serverlist.$$.tmp $TEMPDIR/mosshe_foot.$$.tmp $TEMPDIR/mosshe.$$.serverhead.tmp
  368 }
  369 
  370 
  371 #---------------------------------------------------------
  372 # SortServers - create per-server pages
  373 #---------------------------------------------------------
  374 SortServers () {
  375     MossheLog "SortServers $1"
  376     cat $DATADIR/www/template_head.thtml \
  377         | sed -e "s/TIMESTAMP/$DATUM - $ZEIT/g" \
  378     > $TEMPDIR/servercollected.html
  379     echo '<tr bgcolor="#dddddd" class="border datarowhead"><td><b>Server</b></td><td><b>Alert</b></td><td><b>Warn</b></td><td><b>Ok</b></td><td><b>Undef</b></td></tr>' \
  380     >> $TEMPDIR/servercollected.html
  381     
  382     cat $DATADIR/www/template_head.thtml \
  383             | sed -e "s/TIMESTAMP/$DATUM - $ZEIT/g" \
  384         > $TEMPDIR/mosshe.$$.serverhead.tmp
  385     echo '<tr bgcolor="#dddddd"><td><b>Timestamp</b></td><td><b>Group</b></td><td><b>System</b></td><td><b>Property</b></td><td><b>Status</b></td><td><b>Value</b></td><td><b>Detail</b></td></tr>' \
  386     >> $TEMPDIR/mosshe.$$.serverhead.tmp
  387 
  388     cat $DATADIR/www/template_foot.thtml \
  389         | fgrep -v "Totals: <div class=\"ok\">OKCOUNT checks are OK</div> - <div class=\"notok\">FAILCOUNT show problems</div>" \
  390     >> $TEMPDIR/mosshe_foot.$$.tmp
  391 
  392     cat $WWWDIR/index.csv  | cut -d ";" -f 4 | sort -u > $TEMPDIR/mosshe_serverlist.$$.tmp
  393     while read SERVER; do
  394     echo -n "<tr><td class=\"border\"><a href=\"srv_${SERVER}.html\">$SERVER</a></td>" >> $TEMPDIR/servercollected.html
  395     for STATUS in ALERT WARN OK UNDEF; do
  396         COUNT=`fgrep ";$STATUS;" $WWWDIR/index.csv | fgrep ";${SERVER};" | wc -l`
  397         if [ "$STATUS" = "ALERT" ]; then
  398         COLR=$HTMLALERT
  399         elif [ "$STATUS" = "WARN" ]; then
  400             COLR=$HTMLWARN
  401         elif [ "$STATUS" = "OK" ]; then
  402             COLR=$HTMLOK
  403         else
  404             COLR=$HTMLUNDEF
  405         fi
  406         if [ "$COUNT" = "0" ]; then
  407             COLR="#ffffff"
  408         fi
  409         echo -n "<td bgcolor=\"$COLR\" align=\"middle\" class=\"border\">$COUNT</td>" >> $TEMPDIR/servercollected.html
  410     done
  411     echo '</tr>' >> $TEMPDIR/servercollected.html
  412     
  413     cp $TEMPDIR/mosshe.$$.serverhead.tmp $TEMPDIR/mosshe.$$.server.tmp
  414     fgrep "| SERVER=$SERVER |" $WWWDIR/index.html >> $TEMPDIR/mosshe.$$.server.tmp
  415     if [ -f "$WWWDIR/srv_$SERVER.txt" ]; then 
  416         echo "</table><p>" >> $TEMPDIR/mosshe.$$.server.tmp
  417         cat "$WWWDIR/srv_$SERVER.txt" >> $TEMPDIR/mosshe.$$.server.tmp
  418     fi
  419     cat $TEMPDIR/mosshe_foot.$$.tmp >> $TEMPDIR/mosshe.$$.server.tmp 
  420     mv $TEMPDIR/mosshe.$$.server.tmp "$WWWDIR/srv_$SERVER.html"
  421     chmod ugo+rX ${WWWDIR}/srv_$SERVER.html
  422     
  423     done < $TEMPDIR/mosshe_serverlist.$$.tmp
  424     
  425     echo '<tr bgcolor="#dddddd" class="border datarowhead"><td><b>Server</b></td><td><b>Alert</b></td><td><b>Warn</b></td><td><b>Ok</b></td><td><b>Undef</b></td></tr><p>'  >> $TEMPDIR/servercollected.html
  426     cat $TEMPDIR/mosshe_foot.$$.tmp >> $TEMPDIR/servercollected.html
  427     mv $TEMPDIR/servercollected.html $WWWDIR/servers.html
  428     chmod -R ugo+rX $WWWDIR/server*.html
  429     
  430     rm $TEMPDIR/mosshe_serverlist.$$.tmp $TEMPDIR/mosshe_foot.$$.tmp $TEMPDIR/mosshe.$$.serverhead.tmp
  431 }
  432 
  433 
  434 
  435 
  436 #############################################################################
  437 ###     Central Server Functions  /  ImportAgent + PassiveCheck
  438 #############################################################################
  439 
  440 
  441 
  442 #---------------------------------------------------------
  443 # ImportAgentCurl URL  -  inport and merge results from other agents
  444 #---------------------------------------------------------
  445 ImportAgentCurl () {
  446     MossheLog "ImportAgentCurl $1"
  447     URL=$1
  448     IMSV=`echo "$URL" | cut -d "/" -f 3 | cut -d ':' -f 1`
  449         if [ -z `which curl` ]; then
  450         echo "${DATIM};$MYGROUP;$MYNAME;ImportAgentCurl_$IMSV;UNDEF;-1;cURL not installed - cannot perform AgentImport"  >> $TEMPDIR/tmp.$$.collected.tmp
  451         else
  452         curl --connect-timeout $NETWAIT --silent $URL -o $TEMPDIR/tmp.$$.collected.tmp.import
  453         if [ $? -gt 0 ]; then
  454         echo "${DATIM};$MYGROUP;$MYNAME;ImportAgentCurl_$IMSV;ALERT;1;could not download $URL"  >> $TEMPDIR/tmp.$$.collected.tmp
  455         else
  456         echo "${DATIM};$MYGROUP;$MYNAME;ImportAgentCurl_$IMSV;OK;0;imported $URL"  >> $TEMPDIR/tmp.$$.collected.tmp
  457         cat $TEMPDIR/tmp.$$.collected.tmp.import | grep -E ';(OK|ALERT|WARN|UNDEF);' >> $TEMPDIR/tmp.$$.collected.tmp
  458         rm $TEMPDIR/tmp.$$.collected.tmp.import 
  459         fi
  460         fi
  461 }
  462 
  463 
  464 #---------------------------------------------------------
  465 # ImportAgentWget URL  -  inport and merge results from other agents
  466 #---------------------------------------------------------
  467 ImportAgentWget () {
  468     MossheLog "ImportAgentWget $1"
  469     URL=$1
  470     IMSV=`echo "$URL" | cut -d "/" -f 3 | cut -d ':' -f 1`
  471         if [ -z `which wget` ]; then
  472         echo "${DATIM};$MYGROUP;$MYNAME;ImportAgentWget_$IMSV;UNDEF;-1;cURL not installed - cannot perform AgentImport"  >> $TEMPDIR/tmp.$$.collected.tmp
  473         else
  474         wget --quiet --timeout=$NETWAIT --tries=2 $URL --output-document=$TEMPDIR/tmp.$$.collected.tmp.import
  475         if [ $? -gt 0 ]; then
  476         echo "${DATIM};$MYGROUP;$MYNAME;ImportAgentWget_$IMSV;ALERT;1;could not download $URL"  >> $TEMPDIR/tmp.$$.collected.tmp
  477         else
  478         echo "${DATIM};$MYGROUP;$MYNAME;ImportAgentWget_$IMSV;OK;0;imported $URL"  >> $TEMPDIR/tmp.$$.collected.tmp
  479         cat $TEMPDIR/tmp.$$.collected.tmp.import | grep -E ';(OK|ALERT|WARN|UNDEF);' >> $TEMPDIR/tmp.$$.collected.tmp
  480         rm $TEMPDIR/tmp.$$.collected.tmp.import 
  481         fi
  482         fi
  483 }
  484 
  485 
  486 #---------------------------------------------------------
  487 # ImportAgent URL  -  inport and merge results from other agents
  488 #---------------------------------------------------------
  489 ImportAgent () {
  490     MossheLog "ImportAgent $1"
  491     URL=$1
  492     IMSV=`echo "$URL" | cut -d "/" -f 3 | cut -d ':' -f 1`
  493     if [ -z `which lynx` ]; then
  494         echo "${DATIM};$MYGROUP;$MYNAME;ImportAgent_$IMSV;UNDEF;-1;LYNX not installed - cannot perform AgentImport"  >> $TEMPDIR/tmp.$$.collected.tmp
  495     else
  496         lynx -connect_timeout=$NETWAIT -width=1023 -dump $URL >> $TEMPDIR/tmp.$$.collected.tmp.import
  497         if [ $? -gt 0 ]; then
  498         echo "${DATIM};$MYGROUP;$MYNAME;ImportAgent_$IMSV;ALERT;1;could not download $URL"  >> $TEMPDIR/tmp.$$.collected.tmp
  499         else
  500         echo "${DATIM};$MYGROUP;$MYNAME;ImportAgent_$IMSV;OK;0;imported $URL"  >> $TEMPDIR/tmp.$$.collected.tmp
  501         cat $TEMPDIR/tmp.$$.collected.tmp.import | grep -E ';(OK|ALERT|WARN|UNDEF);' >> $TEMPDIR/tmp.$$.collected.tmp
  502         rm $TEMPDIR/tmp.$$.collected.tmp.import 
  503         fi
  504     fi
  505 }
  506 
  507 
  508 #---------------------------------------------------------
  509 # ReapPassiveChecks  servername  minutes.timeout  filename  -  inport and merge income results from other agents
  510 #---------------------------------------------------------
  511 ReapPassiveChecks () {
  512     MossheLog "ReapPassiveChecks $1"
  513     SRVNAME=$1
  514     MAXAGE=$2
  515     FILENAME=$3
  516 
  517     STATUS="UNDEF"
  518     MESSAGE="Something wrong with passive check for $SRVNAME file $FILENAME"
  519     VALUE=0
  520     
  521     if [ ! -s "$FILENAME" ]; then
  522         STATUS="ALERT"
  523     MESSAGE="Error: Check file $FILENAME for $SRVNAME not existing "
  524     VALUE=3
  525     else
  526         typeset -i FTIME=`ls -l --time-style=+%s $FILENAME | cut -d " " -f 6`
  527         typeset -i LTIME=`date +%s --date="$MAXAGE minutes ago"`
  528         if [ $FTIME -lt $LTIME ]; then 
  529             STATUS="WARN"
  530         MESSAGE="Problem: check file $FILENAME for $SRVNAME older than $MAXAGE minutes "
  531         VALUE=2
  532     else
  533             STATUS="OK"
  534         MESSAGE="Passive-check file $FILENAME for $SRVNAME seems to be current"
  535         VALUE=1
  536         cat "$FILENAME" | grep -E ';(OK|ALERT|WARN|UNDEF);'  >> $TEMPDIR/tmp.$$.collected.tmp
  537     fi
  538     fi
  539 
  540     echo "${DATIM};$MYGROUP;$SRVNAME;reappassive_$SRVNAME;$STATUS;$VALUE;$MESSAGE" >> $TEMPDIR/tmp.$$.collected.tmp
  541 }
  542 
  543 
  544 
  545 
  546 #############################################################################
  547 ###     Alerting
  548 #############################################################################
  549 
  550 
  551 #---------------------------------------------------------
  552 # AlertMailOnChange  mailaddress
  553 #---------------------------------------------------------
  554 AlertMailOnChange () {
  555 MossheLog "AlertMailOnChange $1"
  556 if [ -z `which mail` ]; then
  557     echo "MoSShE: 'mail' command not availabe, cannot send alerts!"
  558     MossheLog "AlertMailOnChange $1 - missing 'mail' command"
  559     logger -t MoSShE -p local0.alert "MoSShE: 'mail' command not availabe, cannot send alerts!"
  560     exit 255
  561 fi
  562 MAILTO=$1;
  563 if [ -s $TEMPDIR/mosshe_collected.diff ]; then
  564     echo "$WEBURL/notok.html " > $TEMPDIR/MAIL.$$
  565     echo " " >> $TEMPDIR/MAIL.$$
  566     echo " " >> $TEMPDIR/MAIL.$$
  567     echo "Status Change  <old  >new" >> $TEMPDIR/MAIL.$$
  568     echo "-------------------------" >> $TEMPDIR/MAIL.$$
  569     cat $TEMPDIR/mosshe_collected.diff >> $TEMPDIR/MAIL.$$
  570     echo " " >> $TEMPDIR/MAIL.$$
  571     echo " " >> $TEMPDIR/MAIL.$$
  572 
  573     echo "Non-OK Status (Overview)" >> $TEMPDIR/MAIL.$$
  574     echo "------------------------" >> $TEMPDIR/MAIL.$$
  575     fgrep -v " - OK - " $TEMPDIR/mosshe_collected.newstatus >> $TEMPDIR/MAIL.$$
  576     echo " " >> $TEMPDIR/MAIL.$$
  577     echo " " >> $TEMPDIR/MAIL.$$
  578 
  579     echo "Non-OK Status (Detail)" >> $TEMPDIR/MAIL.$$
  580     echo "----------------------" >> $TEMPDIR/MAIL.$$
  581     fgrep -v ";OK;" $WWWDIR/index.csv >> $TEMPDIR/MAIL.$$
  582     echo " " >> $TEMPDIR/MAIL.$$
  583     echo " " >> $TEMPDIR/MAIL.$$
  584 
  585     cat $TEMPDIR/MAIL.$$ | mail -s "MoSShE $MYGROUP: $MYNAME : Status change detected" $MAILTO
  586     rm $TEMPDIR/MAIL.$$
  587 fi
  588 }
  589 
  590 
  591 #---------------------------------------------------------
  592 # AlertMailOnChangeFor servername  mailaddress  subject
  593 #---------------------------------------------------------
  594 AlertMailOnChangeFor () {
  595 MossheLog "AlertMailOnChangeFor srv=$1 to=$2 msg=$3"
  596 if [ -z `which mail` ]; then
  597     echo "MoSShE: 'mail' command not availabe, cannot send alerts!"
  598     MossheLog "AlertMailOnChangeFor $1 - missing 'mail' command"
  599     logger -t MoSShE -p local0.alert "MoSShE: 'mail' command not availabe, cannot send alerts!"
  600     exit 255
  601 fi
  602 SERVER=$1
  603 MAILTO=$2
  604 SUBJECT=$3
  605 if [ -z "$SUBJECT" ]; then SUBJECT="Monitoring $SERVER : Status change on $MYGROUP: $MYNAME"; fi
  606 fgrep "$SERVER" $TEMPDIR/mosshe_collected.diff > $TEMPDIR/collected_filter.$$.diff
  607 if [ -s $TEMPDIR/collected_filter.$$.diff ]; then
  608     echo "http://$WEBURL/notok.html " > $TEMPDIR/MAIL.$$
  609     echo " " >> $TEMPDIR/MAIL.$$
  610     echo " " >> $TEMPDIR/MAIL.$$
  611     echo "Status Change  <old  >new" >> $TEMPDIR/MAIL.$$
  612     echo "-------------------------" >> $TEMPDIR/MAIL.$$
  613     cat $TEMPDIR/collected_filter.$$.diff >> $TEMPDIR/MAIL.$$
  614     echo " " >> $TEMPDIR/MAIL.$$
  615     echo " " >> $TEMPDIR/MAIL.$$
  616 
  617     echo "Non-OK Status (Overview)" >> $TEMPDIR/MAIL.$$
  618     echo "------------------------" >> $TEMPDIR/MAIL.$$
  619     fgrep -v " - OK - " $TEMPDIR/mosshe_collected.newstatus | fgrep "$SERVER" >> $TEMPDIR/MAIL.$$
  620     echo " " >> $TEMPDIR/MAIL.$$
  621     echo " " >> $TEMPDIR/MAIL.$$
  622 
  623     echo "Non-OK Status (Detail)" >> $TEMPDIR/MAIL.$$
  624     echo "----------------------" >> $TEMPDIR/MAIL.$$
  625     fgrep -v ";OK;" $WWWDIR/index.csv | fgrep "$SERVER" >> $TEMPDIR/MAIL.$$
  626     echo " " >> $TEMPDIR/MAIL.$$
  627     echo " " >> $TEMPDIR/MAIL.$$
  628 
  629     cat $TEMPDIR/MAIL.$$ | mail -s "$SUBJECT" $MAILTO
  630     rm $TEMPDIR/MAIL.$$
  631 fi
  632 rm $TEMPDIR/collected_filter.$$.diff
  633 }
  634 
  635 
  636 #---------------------------------------------------------
  637 # AlertMailAlways  mailaddress
  638 #---------------------------------------------------------
  639 AlertMailAlways () {
  640 MossheLog "AlertMailAlways $1"
  641 if [ -z `which mail` ]; then
  642     echo "MoSShE: 'mail' command not availabe, cannot send alerts!"
  643     MossheLog "AlertMailAlways $1 - missing 'mail' command"
  644     logger -t MoSShE -p local0.alert "MoSShE: 'mail' command not availabe, cannot send alerts!"
  645     exit 255
  646 fi
  647 MAILTO=$1;
  648 fgrep -v " - OK - " $TEMPDIR/mosshe_collected.newstatus > $TEMPDIR/tmp.$$.collected.newstatus.nonOK
  649 if [ -s $TEMPDIR/tmp.$$.collected.newstatus.nonOK ]; then
  650     echo "http://$WEBURL/notok.html " > $TEMPDIR/MAIL.$$
  651     echo " " >> $TEMPDIR/MAIL.$$
  652     echo " " >> $TEMPDIR/MAIL.$$
  653     echo "MoSSHe Alert/Warning on $MYGROUP: $MYNAME" >> $TEMPDIR/MAIL.$$
  654     echo "---------------------------------------" >> $TEMPDIR/MAIL.$$
  655 
  656     cat $TEMPDIR/tmp.$$.collected.newstatus.nonOK >> $TEMPDIR/MAIL.$$
  657     echo " " >> $TEMPDIR/MAIL.$$
  658     echo " " >> $TEMPDIR/MAIL.$$
  659 
  660     cat $TEMPDIR/MAIL.$$ | mail -s "MoSSHe monitoring: Alert/Warning for $MYGROUP: $MYNAME" $MAILTO
  661     rm $TEMPDIR/MAIL.$$
  662 fi
  663 rm $TEMPDIR/tmp.$$.collected.newstatus.nonOK
  664 }
  665 
  666 
  667 #---------------------------------------------------------
  668 # SyslogOnChange  facility
  669 #---------------------------------------------------------
  670 SyslogOnChange () {
  671 MossheLog "SyslogOnChange $1"
  672 FACILITY=${1-local0};
  673 
  674 if [ -s $TEMPDIR/mosshe_collected.diff ]; then
  675     while read LINE; do    
  676 #   echo "$LINE" | read UPDOWN SERVER DUMMY1 STATUS DUMMY2 CHECK
  677     UPDOWN=`echo $LINE | cut -d " " -f 1`
  678     GROUP=`echo $LINE | cut -d " " -f 2`
  679     SERVER=`echo $LINE | cut -d " " -f 4`
  680     STATUS=`echo $LINE | cut -d " " -f 6`
  681     CHECK=`echo $LINE | cut -d " " -f 8`
  682     FULLTEXT=`fgrep ";${SERVER};${CHECK};" $WWWDIR/index.csv | cut -d ";" -f 9-`
  683 
  684     if [ "$UPDOWN" = '<' ]; then
  685         logger -t MoSShE -p ${FACILITY}.info "OK - ${CHECK} @ ${GROUP}: ${SERVER} - ${FULLTEXT}"
  686     elif [ "$STATUS" = 'INFO' ]; then
  687         logger -t MoSShE -p ${FACILITY}.info "${STATUS} - ${CHECK} @ ${GROUP}: ${SERVER} info message"
  688     elif [ "$STATUS" = 'WARN' ]; then
  689         logger -t MoSShE -p ${FACILITY}.warn "${STATUS} - ${CHECK} @ ${GROUP}: ${SERVER} is not okay - ${FULLTEXT}"
  690     elif [ "$STATUS" = 'ALERT' ]; then
  691         logger -t MoSShE -p ${FACILITY}.alert "${STATUS} - ${CHECK} @ ${GROUP}: ${SERVER} is broken/critical - ${FULLTEXT}"
  692     else
  693         logger -t MoSShE -p ${FACILITY}.err "${STATUS} - ${CHECK} @ ${GROUP}: ${SERVER} check is not working - ${FULLTEXT}"
  694     fi
  695     done < $TEMPDIR/mosshe_collected.diff
  696 fi
  697 }
  698 
  699 
  700 #############################################################################
  701 ###     Logging
  702 #############################################################################
  703 
  704 
  705 #---------------------------------------------------------
  706 # LogTo filename
  707 #---------------------------------------------------------
  708 LogTo () {
  709     MossheLog "LogTo $1"
  710     LOGFILENAME=$1;
  711     mkdir -p `dirname $LOGFILENAME` 
  712     cat $WWWDIR/index.csv >> $LOGFILENAME
  713 }
  714 
  715 
  716 #---------------------------------------------------------
  717 # LogToDaily filename
  718 #---------------------------------------------------------
  719 LogToDaily () {
  720     MossheLog "LogToDaily $1"
  721     LOGFILENAME=$1;
  722     mkdir -p `dirname $LOGFILENAME` 
  723     cat $WWWDIR/index.csv >> $LOGFILENAME.$DATUM
  724 }
  725 
  726 
  727 #---------------------------------------------------------
  728 # LogToWeekly filename
  729 #---------------------------------------------------------
  730 LogToWeekly () {
  731     MossheLog "LogToWeekly $1"
  732     LOGFILENAME=$1;
  733     mkdir -p `dirname $LOGFILENAME` 
  734     cat $WWWDIR/index.csv >> $LOGFILENAME.`date +"%Ykw%W"`
  735 }
  736 
  737 
  738 #---------------------------------------------------------
  739 # LogToMonthly filename
  740 #---------------------------------------------------------
  741 LogToMonthly () {
  742     MossheLog "LogToMonthly $1"
  743     LOGFILENAME=$1;
  744     mkdir -p `dirname $LOGFILENAME` 
  745     cat $WWWDIR/index.csv >> $LOGFILENAME.`date +"%Y-%m"`
  746 }
  747 
  748 
  749 #---------------------------------------------------------
  750 # SLA_Eval servername checkname
  751 #---------------------------------------------------------
  752 SLA_Eval () {
  753     MossheLog "SLA_Eval srv=$1 chk=$2"
  754     SRV=$1;
  755     CHK=$2;
  756     cat $TEMPDIR/mosshe_collected.diff | fgrep $SRV | fgrep $CHK \
  757         | while read LINE; do
  758         UPDOWN=`echo $LINE | cut -d " " -f 1`
  759         SERVER=`echo $LINE | cut -d " " -f 2`
  760         CHECK=`echo $LINE | cut -d " " -f 6`
  761         if [ "$UPDOWN" = '>' ]; then
  762             STATUS=`echo $LINE | cut -d " " -f 4`
  763         echo -n "$SERVER;$CHECK;$STATUS;$DATIM;" >> $WWWDIR/SLA_${SRV}_${CHK}.txt
  764         else
  765             echo "OK;$DATIM" >> $WWWDIR/SLA_${SRV}_${CHK}.txt
  766         fi
  767     done
  768 }
  769 
  770 
  771 #---------------------------------------------------------
  772 # PlotDataFiles   MaxSize
  773 #---------------------------------------------------------
  774 PlotDataFiles () {
  775     MossheLog "PlotDataFiles $1"
  776     MAXSIZE=$1;
  777     cat $WWWDIR/index.csv \
  778         | cut -d ";" -f 1,2,4,5,7 \
  779     | sed -e "s/;/ /g" \
  780     | while read DAT TIM HOST CHCK VALU; do
  781         CHK=`echo $CHCK | sed -e "s/\//\./g"`
  782         FN="$WWWDIR/datalog/$HOST/$CHK"
  783         FNAVG="$WWWDIR/datalog/$HOST/$CHK.avg"
  784         mkdir -p "$WWWDIR/datalog/$HOST"
  785         echo "$DAT $TIM $VALU" >> "$FNAVG"
  786         echo "$DAT $TIM $VALU" >> "$FN"
  787         if [ -s "$WWWDIR/please_trim_the_logs" ]; then
  788         tail -n $MAXSIZE "$FNAVG" > "$FN"
  789         fi
  790     done
  791     rm -f "$WWWDIR/please_trim_the_logs"
  792 }
  793 
  794 
  795 
  796 
  797 #---------------------------------------------------------
  798 # PlotAvgDataFiles   AverageNum MaxSize
  799 #---------------------------------------------------------
  800 PlotAvgDataFiles () {
  801     echo 'PlotAvgDataFiles is deprecated (no longer working) - please remove the call from your MOSSHE file'
  802 }
  803 
  804 
  805 
  806 
  807 #############################################################################
  808 
  809 #############################################################################