#!/bin/sh

# bb-local.sh
#
# BIG BROTHER MONITORING SCRIPT
# Sean MacGuire - The MacLawran Group Inc.
# Version 1.4
# Mar 05th, 2000
#
# This program is Copyright (c) 1997-2000
# The MacLawran Group Inc.
# All Rights Reserved
#
# For each system we monitor we watch
#	disk	USES df 
#	cpu	USES uptime
#	procs	CHECKS A LIST
# 	msgs	LOOKS FOR WARNINGS
#
# For the network we care about
#	telnet	USES telnet and checks the results
#	http	USES lynx to check for valid output
#
# THEN SEND THE RESULTS TO THE DISPLAY SERVER USING bb

# echo "***** BBHOME IS SET TO $BBHOME"
# echo "***** BBTMP IS SET TO $BBTMP"

if test ! "$BBTMP"                      # GET DEFINITIONS IF NEEDED
then
	 # echo "*** LOADING BBDEF ***"
        . $BBHOME/etc/bbdef.sh          # INCLUDE STANDARD DEFINITIONS
fi

# Setup the name of the local host
# Save a version of the machine name with '.' instead of ',', if any are there
MACHINEDOTS=`echo $MACHINE | $SED 's/,/\./g'`
if [ "$FQDN" != "TRUE" ]
then
	OLDIFS=$IFS
	IFS='.'
	set $MACHINEDOTS >/dev/null
	IFS=$OLDIFS
	MACHINEMASK="${1}"
else
	MACHINEMASK="${MACHINEDOTS}"
fi

#
#=====================================================================
# DISK TEST
#=====================================================================
#
# echo "*** DISK TEST 1 ***"
COLOR="green"
REDLINE=""
YELLOWLINE=""

eval "$DFCMD | $EGREP \"$DFUSE\" | $EGREP -v \"$DFEXCLUDE\" | $SORT \"+${DFSORT}\" > $BBTMP/DFOUT"

cat $BBTMP/DFOUT | {
while read line
do
    set $line
    DISKNAME=$6
    DISKDEV=$1
    DISKPCT=`echo $5 | $SED "s/%//"`

    # echo "*** DISK TEST 2 ***"
    #
    # SUPPORT FOR NON-STANDARD df'S
    # Richard Dansereau <dansereau.richard@regis.uqam.ca>
    # Thanks to Thomas Alex <thomas.alex@commgensys.com>
    #
    case "$BBOSTYPE"
    in
        sco3)   DISKNAME=$1
                DISKPCT=`echo $6 | $SED "s/%//"`
		DISKDEV=$7	# Is this true ?
		;;
	irix)   # Thanks to:    David Brandon
		# and Jeff Younker - jeff@mdli.com
		DISKNAME=$7
		DISKPCT=`echo $6 | $SED "s/%//"`
		;;
    esac

    # echo "$DISKNAME $DISKPCT"

	DFWARNVAL=$DFWARN               # USE DEFAULT WARN VALUE
	DFPANICVAL=$DFPANIC             # USE DEFAULT PANIC VALUE

	#
	# NEW IN 1.08a
	# Robert Andre Croteau's custom disk file
	# IF WE HAVE A CUSTOM PARTITION FILE, USE IT.
	# DEFAULTS TO etc/bb-dftab
	# FORMAT:	partition:warn%:panic%
	# I.E.:		/usr:96:99
	#
	# multihost bb-dftab based on an idea of Jeff Younker<jeff@mdli.com>
	if test -f "${BBDFTAB}"
	then
		# File format:  [host:]partition:WARN:PANIC
		# Check for the partition without the leading host
		# echo "*** CHECKING FOR PARTITION $DISKNAME IN $BBDFTAB"
		DISKLINE=`$GREP "^[ 	]*${DISKNAME}[ 	]*:" $BBDFTAB`
		# echo "*** DISKLINE: $DISKLINE"
		if test "$DISKLINE" != ""
		then
			# echo "*** DISKLINE NOT NULL"
			set `echo $DISKLINE | $SED "s/:/ /g"`
        		DFWARNVAL=$2            # GET CURRENT FS WARNING LEVEL
        		DFPANICVAL=$3           # GET CURRENT FS PANIC LEVEL
			# echo "*** WARN AT $DFWARNVAL - PANIC AT $DFPANICVAL"
		else
			DISKLINE=`$GREP "^${MACHINEMASK}[   ]*:[ 	]*${DISKNAME}[ 	]*:" $BBDFTAB`
			# echo "*** DISKLINE: $DISKLINE"
			if test "$DISKLINE" = ""
			then
				DISKLINE=`$GREP "^${MACHINEMASK}\..**:[ 	]*${DISKNAME}[ 	]*:" $BBDFTAB`
			fi
			if test "$DISKLINE" = ""
			then
				DISKLINE=`$GREP "^localhost[ 	]*:[ 	]*${DISKNAME}[ 	]*:" $BBDFTAB`
			fi
			if test "$DISKLINE" = ""
			then
				DISKLINE=`$GREP "^[ 	]*:[ 	]*${DISKNAME}[ 	]*:" $BBDFTAB`
				if test "$DISKLINE" != ""
				then
					DISKLINE="localhost${DISKLINE}"
				fi
			fi
			if test "$DISKLINE" != ""
			then
				# echo "*** DISKLINE NOT NULL"
				set `echo $DISKLINE | $SED "s/:/ /g"`
        			DFWARNVAL=$3            # GET CURRENT FS WARNING LEVEL
        			DFPANICVAL=$4           # GET CURRENT FS PANIC LEVEL
				# echo "*** WARN AT $DFWARNVAL - PANIC AT $DFPANICVAL"
			fi
		fi
	fi

    	MARK=""
    	if test "$DISKPCT" -ge "$DFWARNVAL"         # ABOVE WARNING LEVEL
    	then
        	if test "$DISKPCT" -ge "$DFPANICVAL"    # ABOVE PANIC, RED ALERT !!!
        	then
            		COLOR="red"
            		MARK="PANIC"
	    		MARKLEVEL=$DFPANICVAL
	    		HIGHMARK="PANIC"
			GIFCOLOR="&red"
	        	REDLINE="${GIFCOLOR} ${DISKNAME} (${DISKPCT}%) has reached the defined disk space ${MARK} level (${MARKLEVEL}%)
${REDLINE}"
        	else
            		if test "$COLOR" = "green"          # ONLY IF NOT ALREADY SET
            		then
                		COLOR="yellow"
	        		HIGHMARK="WARNING"
            		fi
            		MARK="WARNING"
	    		MARKLEVEL=$DFWARNVAL
			GIFCOLOR="&yellow"
	        	YELLOWLINE="${GIFCOLOR} ${DISKNAME} (${DISKPCT}%) has reached the defined disk space ${MARK} level (${MARKLEVEL}%)
${YELLOWLINE}"
        	fi
    	fi
done

#
# NOW SEND THIS INFORMATION TO THE BIG BROTHER DISPLAY UNIT
#
if [ "$COLOR" = "green" ]
then
    LINE="status $MACHINE.disk $COLOR `date` - Disk partitions on $MACHINE OK

`cat $BBTMP/DFOUT`"
else
    LINE="status $MACHINE.disk $COLOR `date` - Disk on $MACHINE at $HIGHMARK level
${REDLINE}${YELLOWLINE}
`cat $BBTMP/DFOUT`"
fi

$BB $BBDISP "$LINE"

$RM -f $BBTMP/DFOUT

}
#
#=====================================================================
# CPU TEST AND UPTIME WARNINGS
# AUTOMATIC WARNING IF THE SYSTEM IS UP FOR LESS THAN 1 HOUR
#=====================================================================
#
# echo "*** CPU TEST ***"
set `$UPTIME`

echo $3 | $GREP "," > /dev/null 2>&1
if test "$?" = 0
then
	UP=`echo $3 | $SED "s/,//"`
else
	UP=`echo $3 $4 | $SED "s/,//"`
fi

shift;shift;shift;shift;

ARG=`$EXPR $# - 2`			# 5 MINUTE LOAD AVERAGE
COUNT=1
while test "$COUNT" -le "$ARG"
do
	shift;
	COUNT=`$EXPR $COUNT + 1`
done

AVG=`echo $1 | $SED "s/\,//g" | $SED "s/\.//" `
AVG=`$EXPR $AVG + 0`
# echo "LOAD AVERAGE: $AVG"

LINE="LOAD AVG on $MACHINE is ${AVG}"

if test "$AVG" -ge "$CPUPANIC"
then
	COLOR="red"
elif test "$AVG" -ge "$CPUWARN"
then
	COLOR="yellow"
else	
	COLOR="green"
fi

# echo "*** PS COMMAND $PS ***"

$PS > $BBTMP/bb.$$			# GET A PS LISTING
P=`$CAT $BBTMP/bb.$$ | $WC`		# NUMBER OF PROCESSES
P=`echo $P`
U=`$WHO | $WC`				# GET NUMBER OF USERS
U=`echo $U`

#
# AUTOMATIC WARNING IF A SYSTEM HAS BEEN UP LESS THAN AN HOUR
# THIS WILL HELP US NOTICE SPONTANEOUS REBOOTS!
#
echo $UP | $GREP "min" > /dev/null 2>&1
if test "$?" = "0"
then
	COLOR="yellow"
	LINE="Warning: Machine recently rebooted"
fi

#
# NOW SEND THIS INFORMATION TO THE BIG BROTHER DISPLAY UNIT
#
$BB $BBDISP "status $MACHINE.cpu $COLOR `date` up: $UP, $U users, $P procs, load=${AVG}
$LINE"

#
#=====================================================================
# PROCESSES THAT MUST EXIST
#=====================================================================
#
# echo "*** PROCESSES TEST ***"

GREENLINES=""
YELLOWLINES=""
REDLINES=""
STATLINE="No processes to check"
COLOR="green"

# if bb-proctab override defaults from bbdef.sh
# bb-proctab based on an idea of Jeff Younker<jeff@mdli.com>
# Thanks Jeff !

if [ -f "$BBHOME/etc/bb-proctab" ]
then
	# a line can start with
	# :....
	#  :....
	# localhost:...
	# www:....
	# www.bb4.com:...
	#
	# All mask should allow for spaces/tabs before :
	# Grep for each mask
	$RM -f $BBTMP/BBPROCS.$$
	for mask in "[ 	]*localhost[ 	]*:" "[ 	]*${MACHINEMASK}[ 	]*:" "[ 	]*${MACHINEMASK}\..*:"
	do
		# Save procs in FILE
		procsline=`$GREP "^${mask}" $BBHOME/etc/bb-proctab >>$BBTMP/BBPROCS.$$ 2>/dev/null`
	done

	# IF any line matched then process
	if [ -s $BBTMP/BBPROCS.$$ ]
	then
		# Get rid of defaults
		PROCS=""
		PAGEPROC=""
		$RM -f $BBTMP/PROCS.$$
		$RM -f $BBTMP/PAGEPROC.$$
		cat $BBTMP/BBPROCS.$$ | $SORT -u |
		while read line
		do
			# If anyt "::" are found, force a space in between to
			# make Solaris happy :(
			line=`echo "$line" | $SED 's/::/: :/g'`
			OLDIFS=$IFS
			IFS=':'
			set $line
			IFS=$OLDIFS
			# Results are sent to a file because some shells go into a subshell
			# during a while preceded by a pipe such that variables set within
			# that while will not be visible after exiting the while. The {} should
			# not send it within a subshell but I'm wary (especially SUN & DIGITAL)
			echo $2 >> $BBTMP/PROCS.$$
			echo $3 >> $BBTMP/PAGEPROC.$$
		done
		PROCS=`$CAT $BBTMP/PROCS.$$`
		PAGEPROC=`$CAT $BBTMP/PAGEPROC.$$`
		$RM -f $BBTMP/PROCS.$$
		$RM -f $BBTMP/PAGEPROC.$$
	fi
	$RM -f $BBTMP/BBPROCS.$$
fi

#
# All this crap is to be able to handle processes specified
# with "":  snmmpd "syslogd -s" xntpd
# The string "syslogd -s" will be matched against the PS output
#
# The ' ' in the "" construct is replaced with '&_' temporarely
# until the string is actually used which is reverted back to ' '
#
# That's because a constrcut like "syslogd -s" in BBPROCS would
# appear as "syslogd and -s" in the for loop if no substitution
# was done:
# for proc in $BBPROCS  -> would become for proc in "syslogd -s"
# but proc instead of being "syslogd -s" there would be 2 proc
# iteration: one for "syslogd and the other -s"   
#
# Any clearer ?
#
# Must inverse !"   " to "!    " also

BBPROCS=`echo $PROCS $PAGEPROC | $SED 's/\![ 	]*"/"\!/g'`
if [ -n "$BBPROCS" ]
then
	OLDIFS=$IFS
	IFS='"'
	set " "$BBPROCS >/dev/null 2>&1
	IFS=$OLDIFS
	BBPROCS=""
	while [ $# -gt 0 ]
	do
		BBPROCS="$BBPROCS $1"
		if [ $# -gt 1 ]
		then
			BBPROCS="$BBPROCS "`echo $2 | $SED 's/ /\&_/g'`
			shift
		fi
		shift
	done
fi
	
for proc in $BBPROCS
do
	proc=`echo "$proc" | $SED 's/\&_/ /g'`
	case $proc
	in
		*!* )
			proc=`echo "$proc" | $SED 's/!//g'`
			TESTREVERSED=TRUE
			OK=DOWN
			NOTOK=UP
			;;
		*)      
			TESTREVERSED=FALSE
			OK=UP
			NOTOK=DOWN
			;;
	esac

				# No need to exclude grep from
				# process list
				# thanks to Paul Gluhosky
				# <paul.gluhosky@yale.edu>
	$GREP "$proc" $BBTMP/bb.$$ > /dev/null 2>&1
	RC=$?
	# Check if service must NOT be running
	if [ "$TESTREVERSED" = "TRUE" ]
	then    
		if [ "$RC" -ne 0 ]
		then
			RC=0
		else
			RC=1
		fi
	fi   
	if test "$RC" != "0"
	then
		STATLINE="Some processes are in error"
		#
		# NOW SEE IF THIS IS A PROCESS WE SHOULD NOTIFY ON...
		#
		echo "$PAGEPROC" | $GREP "$proc" > /dev/null 2>&1
		if test "$?" = "0"
		then
			COLOR="red"
			REDLINES="&red $proc  $NOTOK
$REDLINES"
		else
			if test "$COLOR" != "red"	# WASN'T A PANIC
			then
				COLOR="yellow"		# SO JUST WARN
			fi
			YELLOWLINES="&yellow $proc  $NOTOK
$YELLOWLINES"
		fi	
	else
		if [ "$COLOR" = "green" ]
		then
			STATLINE="All processes are OK"
		fi
		GREENLINES="&green $proc  $OK
$GREENLINES"
	fi
done

#
# NOW SEND THIS INFORMATION TO THE BIG BROTHER DISPLAY UNIT
#
$BB $BBDISP "status $MACHINE.procs $COLOR `date` $STATLINE

${REDLINES}${YELLOWLINES}${GREENLINES}"

$RM -f $BBTMP/bb.$$

#
#=====================================================================
# NOTIFY ON BAD MESSAGES
#=====================================================================
#
# echo "*** MESSAGES TEST ***"

BADMSGFILE=""
for file in $MSGFILE
do
	if test ! -r "$file" 	# FILE IS NOT READABLE, ALWAYS AN ERROR
	then
		BADMSGFILE="$BADMSGFILE $file"
	elif test "$CHKMSGLEN" = "TRUE"
	then
		if [ ! -s "$file" ]	# FILE IS EMPTY!
		then
			BADMSGFILE="$BADMSGFILE $file"
		fi
	fi
done

# Are there any bad message files
BADLINE=""
if [ "$BADMSGFILE" = "" ]
then

	COLOR="green"
	MSGLINE="No bad messages"
	
	msgsfile=$BBTMP/BBMSG.$$
	for word in $MSGS
	do
        	$GREP "$word" $MSGFILE > $msgsfile
        	if [ "$?" -eq 0 ]                       # FOUND A BAD WORD
        	then
                	OLDIFS=$IFS
                	IFS=";"
                	for ignmsg in $IGNMSGS
                	do
                        	# Save in a file to keep new lines
                        	$GREP -v "$ignmsg" $msgsfile > ${msgsfile}.tmp
                        	mv ${msgsfile}.tmp ${msgsfile}
                	done
                	IFS=$OLDIFS
                	if [ ! -s "${msgsfile}" ]
                	then
                        	continue
                	fi
                	LINE=`cat ${msgsfile} | $TAIL -10`
			MSGLINE="$MACHINE - bad message(s) found:
		
"
			BADLINE="${BADLINE}${LINE}"
			echo "$PAGEMSG" | $GREP "$word" > /dev/null 2>&1
			if test "$?" = "0"
			then
				COLOR="red"
				break;			# ENOUGH ALREADY...
			else
				if test "$COLOR" = "green"
				then
					COLOR="yellow"
				fi
			fi
		fi
	done
else					# PROBLEM WITH THE MESSAGE FILE
	COLOR="red"
	MSGLINE=""
	for file in $BADMSGFILE
	do
		# DOES THE FILE EXIST ?
		if [ ! -f $file ] 
		then
			MSGLINE="$MSGLINE
$file has vanished"
		elif [ ! -r $file ] 	# READABLE
		then
			MSGLINE="$MSGLINE
$file is unreadable"
		else
			MSGLINE="$MSGLINE
$file is empty"
		fi
	done
fi
$RM -f ${msgsfile}

$BB $BBDISP "status $MACHINE.msgs $COLOR `date` ${MSGLINE}${BADLINE}"
