#!/bin/ksh
#
#ident	"@(#)staf.eng	1.51	04/12/08 SMI"
#
# Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
# Solaris Test Automation Facility (STAF) - Main Engine
#
# STAF Version Number:
#       This is the master version number.  Change this by hand.
#       to increment the version of the package.  The master
#       makefile in ../pkg uses this number.  Do not change the
#       format of the line, or the case, name, or spacing around
#       the "version" variable.
# 
version=2.4

wf_report=/export/home/staf_test.log # TEST CODE WF
# file/dir variables
PATH=$PATH:/usr/bin:.		# /usr/bin and current directory
export HOME=/			# root home directory
lvfile=/tmp/level.dat           # level file
stfile=/tmp/suite.dat           # suite file
nfile=/tmp/nethost.dat  	# network client file
rtfile=/tmp/runtime.dat         # file contains current run time
tcfile=/tmp/technology.dat      # technology file
opfile=/tmp/option.dat          # option file
startsec=0

wdir=/opt/SUNWstaf		# work directory
pdir=$wdir/prgs			# program directory
level=`cat $lvfile` 		# level of testing
suite=`cat $stfile` 		# system or controller cert
sysoption=`cat $opfile`		# system or motherboard
timepath=/opt/SUNWstaf/prgs 
	
hname=`hostname`		# test system
repdir=$wdir/reports/$hname	# report directory
cdir=$wdir/cmd-files/$hname	# command file directory
report=$repdir/staf.rpt		# current report file
repname=			# report filename
tlog=$repdir/staf.log		# current test log
logs=				# array of test log files
logname=			# log filename
fname=$1			# command file
ext=0				# file extension
rlog=			        # save log file option
dateh=				# date holder
level_num=			# level number holder
suite_level_log=    # log suite for result tar ball name
cksume_file=			# file used to save the cksum of the report

# clock variables
clock_time=0			# actual clock time
last_time=0			# last time check
time_factor=0			# calculate midnight
run_time=0			# running time

# test variables
i=0				# loop counter
x=0				# loop counter
group=0				# count for test groups
act_tests=0			# number of active test
cur_test=0			# current test number
test_inc=1			# test increment
num_tests=0			# number of running tests
inline=				# input value
args=				# test arguments
stop_time=			# array of test stop time
loop_time=			# array of loop times
max_loop_time=28800		# maximum allowable loop_time
num_inst=			# array of test number of instances
test_cmd=			# array of test commands
test_dir=			# array of test directory
tpid=				# array of test pid
test_basename=			# array of test case basename
test_name=			# array of test case name without .stf
time_sig=17			# elapsed time signal
loadmax=3			# max load average
latime=0			# check load time
laint="0:5"			# load time interval
st_time=0			# start time
test_rem=0			# test to remove
test_ran=0			# number of tests ran
kidpid=				# child pids
option=				# staf options
opval=				# staf options value
errstr=				# error string
count=0
count_wait=0


# functions

# remove temp files when test finished or stoped
clean()
{
#    if [ -f $lvfile ]
#    then
#	rm -f $lvfile
#    fi
#    if [ -f $stfile ]
#    then
#	rm -f $stfile
#    fi
#    if [ -f $nfile ]
#    then
#       rm -f $nfile
#    fi
    if [ -f $tcfile ]
    then
	rm -f $tcfile
    fi
    if [ -f $opfile ]
    then
	rm -f $opfile
    fi
    if [ -f $rtfile ]
    then
	rm -f $rtfile
    fi
}

# Convert h:m time to seconds for use by subtests
time_to_seconds()
{
	hours=`expr "$1" : '^\([^:]*\):'`
	minutes=`expr "$1" : '.*:\(.*\)$'`
	((seconds=hours*3600+minutes*60))
	echo $seconds
}

# kill test parent and children
kill_tests()
{
	i=0
	while [ $i -ne $act_tests ]
	do 
		kidpid=`$pdir/child ${tpid[$i]}`
		# kill test parent
		kill -$time_sig ${tpid[$i]} > /dev/null 2>&1
		# kill test children
		kill -9 $kidpid > /dev/null 2>&1
		dateh=`/bin/date -u +"\`echo \"\045r\"\`"`
		echo "	$dateh -- ${test_cmd[$i]} status = FAIL" >> $report
		((i=i+1))
	done
}

# move report/log files
mv_replog()
{
	
	count=0
	mv -f $report $repname
	for i in `ls $tlog.* | sort -t . -k 3,3n -k 4,4n 2>/dev/null`
	do
		grp=`echo "$i" | cut -f3 -d'.'`
		ext=`echo "$i" | cut -f4- -d'.'`

		if [  -z "${rlog[*]}" ]
		then
			if [ $1 = error ]; then
				mv -f $i $logname$grp.$ext
			else
				rm -f $i
			fi
		
		elif [ ${rlog[$count]} = true ]
		then	
			mv -f $i $logname$grp.$ext
		else
			
			if [ $1 = error ]; then
				mv -f $i $logname$grp.$ext
			else
				rm -f $i
			fi
		fi
		((count=count+1))
		
	done
	
}

# end report
end_report()
{
	# get minutes
	((x=(run_time%100)+(60-(st_time%100))))
	echo "Test: $run_time $st_time $x" >> $wf_report
	# add hour if minutes >= 60
	if [ $x -ge 60 ]
	then
		((run_time=run_time+100))
        	((x=x-60))
	fi
	(echo "\nSTAF Completion:\n----------------"
	dateh=`/bin/date -u +"\`echo \"\045D  \045r\"\`"`

	endsec=`$timepath/time_sec`
	runtimesec=`expr $endsec \- $startsec `
	hoursec=`expr $runtimesec \/ 3600`

	mintmpsec=`expr $runtimesec % 3600`
	minutesec=`expr $mintmpsec \/ 60`

	echo "Processing Stop Time:	$dateh"
#	echo "Elapsed Time:		Hours = $(((run_time/100)-((st_time/100)+1)))  Minutes = $x"
	echo "Elapsed Time:		Hours = $hoursec  Minutes = $minutesec"
	mv_replog $1
	echo "\n*** Solaris Test Automation Facility (STAF v$version) Processing Report - END ***\n") >> $report
	#mon_load end
	cksum $repname > $cksum_file
	clean
}

# start report
st_report()
{
	startsec=`$timepath/time_sec`
	(echo "*** Solaris Test Automation Facility (STAF v$version) Processing Report - BEGIN ***\n" 
	echo "Report Information:" 
	echo "===================" 
	echo "Note: all times reported in UTC\n"
	level_num=`echo $level | cut -f2 -dL`
	if [ $suite == "CC" ]
	then
		echo "Suite:			Controller Certification Level $level_num"
	else
		#add for motherboard certification 
		if [ "$sysoption" == "system" ]
		then
			echo "Suite:			System Certification Level $level_num"
		elif [ "$sysoption" == "motherboard" ]
		then
			echo "Suite:			Motherboard Certification Level $level_num"
		fi
		#end add
	fi
	echo "Processing Summary Report:	$report"
	dateh=`/bin/date -u +"\`echo \"\045D  \045r\"\`"`
	echo "Processing Start Time:		$dateh") >> $report
	
	# added by Matthew on Feb. 28, record suite and level used when save report to fix 4997040
	(echo "$suite"
	echo "$level") > $suite_level_log
}

# staf error
sys_err()
{
	echo "$*" >> $report
	end_report error
	exit 1
}

# get system info
get_si()
{
	(echo "Command File Name:		$fname\n"
	echo "`uname -n`:" 
	echo "----------") >> $report
	cd $pdir/si
	./system_info.sh >> $report
	(echo "Disk layout:"
	 echo "------------") >> $report
	df -k >> $report
	cd $wdir
	(echo "\nPatch Information:"
	echo "------------------"
	showrev -p) >> $report
	(echo "\nModule Information:"
	echo "------------------"
	modinfo) >> $report
	($pdir/si/usb_info.sh) >> $report
}

# get running time
get_time()
{
	dateh=`/bin/date -u +"\`echo \"\045H\045M\"\`"`
	clock_time=$((dateh))
	if [ $((last_time-clock_time)) -gt 0 ]
        then
                ((time_factor=time_factor+1))
        fi
        last_time=$clock_time
        if [ $time_factor -ne 0 ]
        then
                run_time=$(((2400*time_factor)+clock_time))
        else
                run_time=$(((24*time_factor)+clock_time))
        fi
}

# calculate stop time
calc_stop_time()
{
	if [ "$1" = "0:0" ]
	then
		echo "0"
	else	
		get_time
		# 1 minute is added because the time is only to the
		# nearest minute, which may mean tests getting 'timedout'
		# when they're actually correct
		((x=5+run_time+(`echo $1 | cut -f1 -d':'`*100)+`echo $1 | cut -f2 -d':'`))
		if [ $((x%100)) -ge 60 ]
		then
        		((x=x+40))
		fi
		echo $x
	fi
}

# remove test from queue
rem_test()
{
	test_rem=${num_inst[$1]}
	test_ran=$act_tests
	inline2=${test_cmd[$1]}
	x=0
	
	# loop through tests
	while [ $x -lt $num_tests ] && [ $num_tests -ne 0 ]
	do
		# remove test and instances
		if [ ${num_inst[$x]} -eq $test_rem ]
		then
			# if test running
			if [ $x -lt $act_tests ]
			then
				dateh=`/bin/date -u +"\`echo \"\045r\"\`"`
				# kill test
				if [ "$2" = "kill" ]
				then
					# notify parent for clean up
					kill -$time_sig ${tpid[$x]} > /dev/null 2>&1
					# kill children
					kidpid=`$pdir/child ${tpid[$x]}`
					kill -9 $kidpid > /dev/null 2>&1
					echo "  $dateh -- ${test_cmd[$x]} status = TIMEOUT" >> $report
					grp=`echo ${logs[$x]} | cut -f3 -d'.'`
					ext=`echo ${logs[$x]} | cut -f4- -d'.'`
					mv -f ${logs[$x]} $logname$grp.$ext.TIMEOUT_`/bin/date -u +"\`echo \"\045m\045d\045y_\045H\045M\045S\"\`"`
				fi
								
				# decrease active tests number
				((act_tests=act_tests-1))
			fi
			# decrease number of tests
			((num_tests=num_tests-1))
			i=$x
			while [ $i -lt $num_tests ]
			do
				# test pid
				tpid[$i]=${tpid[$i+1]}
				# test log
				logs[$i]=${logs[$i+1]}
				# test command
				test_cmd[$i]=${test_cmd[$i+1]}
				# test run time
				stop_time[$i]=${stop_time[$i+1]}
				# test directory
				test_dir[$i]=${test_dir[$i+1]}
				# test number of instances
				num_inst[$i]=${num_inst[$i+1]}
				((i=i+1))
			done
			# null array element
			tpid[$num_tests]=""; logs[$num_tests]=""; test_cmd[$num_tests]="";
			test_basename[$num_tests]=""; test_name[$num_tests]="";
			stop_time[$num_tests]=""; test_dir[$num_tests]=""; num_inst[$num_tests]=""
		else
			((x=x+1))
		fi
		
	done
#	echo "	$inline2 instances ran = $((test_ran-act_tests))" >> $report
}

# start test running
st_test()
{
	# change to test directory
	cd ${test_dir[$1]}
	# create test log
	if [ ! -e ${logs[$1]} ]
	then 
		touch ${logs[$1]}
		chmod 666 ${logs[$1]}
	fi
	# start test running
	${test_cmd[$1]} >> ${logs[$1]} 2>&1 &
	# store test pid
	tpid[$1]=$!
	# change back to work directory
	cd $wdir
}

# get load average
get_load()
{
	$pdir/loadavg | awk '{print $5}'
}

# add instances to test_cmd array
add_inst()
{
	i=0
	x=0
	while : 
	do
		if [ ${num_inst[$i]} -gt 1 ]
		then
			# test pid
			tpid[$num_tests]=0
			# test command
			test_cmd[$num_tests]=${test_cmd[$i]}
			# get test case name
			test_basename[$num_tests]=`echo ${test_cmd[$num_tests]} | awk '{print $1}'`
			test_name[$num_tests]=`basename ${test_basename[$num_tests]} ".stf"`
			# test log
			logs[$num_tests]=$tlog.$group.$act_tests.${test_name[$num_tests]}
			# test run time
			stop_time[$num_tests]=${stop_time[$i]}
			# test directory
			test_dir[$num_tests]=${test_dir[$i]}
			# test number of instances
			num_inst[$num_tests]=1
			((num_inst[$i]=${num_inst[$i]}-1))
			# extra time to add at end
			loop_time[$num_tests]=0
			tmp1=`expr $num_tests + $count_wait`
			tmp2=`expr $i + $count_wait`
			rlog[$tmp1]=${rlog[$tmp2]}
			((count=count+1))
			st_test $num_tests
			((act_tests=act_tests+1))
			((num_tests=num_tests+1))
			((x=x+1))
		fi
		((i=i+1))
		if [ $i -eq $act_tests ]
		then
			i=0
			if [ $x -eq 0 ] 
			then
				while [ $i -ne $act_tests ]
				do
					num_inst[$i]=$i
					((i=i+1))
				done
				break
			fi
			x=0
		fi
	done
}

# check for test errors
chk_err()
{
	errstr=`grep ERROR $1`
	if [ -n "$errstr" ]
	then
		echo "!ERROR detected in log: $1" >> $report
		echo "$errstr" >> $report
		# stop tests
		kill_tests
		# write report
		end_report error
		exit 1
	fi
}

# check for test pass (should only be executed after the end of a test)
chk_pass()
{
	passstr=`grep PASS $1`
	notestr=`grep NOTE $1`
	dateh=`/bin/date -u +"\`echo \"\045r\"\`"`

	if [ -n "$passstr" -o -n "$notestr" ]
	then
		echo "PASS detected in log: $1" >> $report
		echo "$passstr" >> $report
		echo "  $dateh -- $2 status = PASS" >> $report

	else
		echo "No PASS detected in log: $1" >> $report
		echo "  $dateh -- $2 status = FAIL" >> $report

		# Trigger a kill-all
		echo "!ERROR: Test terminated without a PASS. Checking for other ERRORs." >> $1
	fi
}

# Checks to see if a test has issued an LOOPTIME command in the logs
chk_looptime()
{
	looptime=`grep 'LOOPTIME:' ${logs[$cur_test]} | sort +1n | tail -1 | cut -f 2 -d : | sed -e 's/ //'`

	if [ -n "$looptime" -a "$looptime" -gt ${loop_time[$cur_test]} ]
	then
		# Force looptime and max_loop_time to be interperated as decimal rahter than octal
		looptime=$(( looptime ))
		max_loop_time=$(( max_loop_time ))


		# Calculate extra time, and save old

		# Check that it's not over the limit
		if [ "$looptime" -gt "$max_loop_time" ]
		then
			looptime="$max_loop_time"
		fi

		extratime=$(( looptime - loop_time[$cur_test] ))
		loop_time[$cur_test]=$looptime

		# Convert to minutes
		extratime=$(( looptime/60 ))

		mins=`expr ${stop_time[$cur_test]} : '.*\([0-9][0-9]\)'`
		hours=`expr ${stop_time[$cur_test]} : '\(.*\)[0-9][0-9]'`

		# Force interpartion in decimal rather than octal
		mins=$(( mins ))
		hours=$(( hours ))

		mins=$(( mins + extratime ))
		hours=$(( hours + mins/60 ))
		mins=$(( mins % 60 ))

		# Create a variable of width 2, for leading zero when necessary
		typeset -Z2 mins_2=$mins
		stop_time[$cur_test]=${hours}${mins_2}

	fi
}


# monitor running test(s)
runner()
{	
	# add instances
	num_tests=$act_tests
	add_inst
	# get load avg start time
	latime=`calc_stop_time $laint`
	# loop while monitoring tests
	while [ $act_tests -ne 0 ]
	do
	        # update run time
	        get_time
	        # get minutes
		((_minute=(run_time%100)+(60-(st_time%100))))
	        # add hour if minutes >= 60
		if [ $_minute -ge 60 ]
		then
		    ((_run_time=run_time+100))
		    ((_minute=_minute-60))
		fi
		echo "$(((_run_time/100)-((st_time/100)+1))):$_minute" > $rtfile

		cur_test=0
		# step through active tests
		while [ $cur_test -lt $act_tests ]
		do
		 	# check for errors
			chk_err ${logs[$cur_test]}
			chk_looptime ${logs[$cur_test]}
			# check test stop time
			get_time
			if [ ${stop_time[$cur_test]} -ne 0 ] && [ $run_time -ge ${stop_time[$cur_test]} ]
			then
				# time up - Signal ERROR and stop all tests
				echo "!ERROR: TIMEOUT!" >> ${logs[$cur_test]}
				chk_err ${logs[$cur_test]}
			# check if test running
			elif [ -z "`ps -e -o pid | grep -w ${tpid[$cur_test]}`" ]
			then
				# if test not running and run time = 0
				if [ ${stop_time[$cur_test]} -eq 0 ]
				then
					# remove test from queue
					chk_pass ${logs[$cur_test]} ${test_cmd[$cur_test]}
					chk_err ${logs[$cur_test]}
					rem_test $cur_test
					test_inc=0
				else
					# Test finished early
					chk_pass ${logs[$cur_test]} ${test_cmd[$cur_test]}
					chk_err ${logs[$cur_test]}
					rem_test $cur_test
					test_inc=0
				fi
			fi
			# inc test index
			((cur_test=cur_test+test_inc))
			test_inc=1		
		done
		# check load average and start new test
		if [ $act_tests -ne 0 ] && [ -n "${num_inst[$act_tests]}" ]
		then
			get_time
			if [ `get_load` -lt $loadmax ] && [ $run_time -ge $latime ]
			then
				dateh=`/bin/date -u +"\`echo \"\045r\"\`"`
				echo "	$dateh -- Starting instance of: ${test_cmd[$act_tests]}" >> $report
				st_test $act_tests
				((act_tests=act_tests+1))
				latime=`calc_stop_time $laint`
			fi
		fi
		# wait one
		sleep 1
	done
}

# process command file
proc_cmd_file()
{
	act_tests=0
	(echo "\nSTAF Command File:\n------------------"
	cat $fname
	echo "\nSTAF Commands Executed:\n-----------------------") >> $report
 	file_len=`wc -l $fname | awk '{print $1}'`
	st_time=$((`/bin/date -u +"\`echo \"\045H\045M\"\`"`))
	while [ $file_len -gt 0 ] 
	do
		inline=`tail -$file_len $fname | head -1`
		((file_len=file_len-1))

		case `echo "$inline" | cut -b1` in
		# test command
		[0-9])
			dateh=`/bin/date -u +"\`echo \"\045r\"\`"`
			echo "$dateh -- $inline" >> $report
			# get number of instance(s)
			num_inst[$act_tests]=`echo "$inline" | cut -f2 -d' '`
			# check for number
			expr ${num_inst[$act_tests]} + 0 > /dev/null 2>&1
			if [ $? -ne 0 ]
			then
				sys_err "\n!ERROR: Bad number of instances argument: ${num_inst[$act_tests]}\n"
			fi
			# if test instances 0, do not start test
			if [ ${num_inst[$act_tests]} -eq 0 ]
			then
				echo "Number of instances = 0"
			else
				# get start time
				if [ $act_tests -eq 0 ]
				then
					get_time
				fi

				# get test stop time
				stop_time[$act_tests]=`echo "$inline" | cut -f1 -d' '`
				# export runtime to test
				export RUNTIME
				RUNTIME=`time_to_seconds ${stop_time[$act_tests]}`
				stop_time[$act_tests]=`calc_stop_time ${stop_time[$act_tests]}`

				if [ $RUNTIME -eq 0 ]
				then
					echo "    Not running: Zero runtime."
				else
					# get test command and args
					test_cmd[$act_tests]=`echo "$inline" | cut -f3 -d' '`
					# get test directory
					test_dir[$act_tests]=`dirname ${test_cmd[$act_tests]}`
					# check for execute
					if [ ! -x ${test_cmd[$act_tests]} ]
					then
						sys_err "\n!ERROR: Cannot execute test: ${test_cmd[$act_tests]}\n"
					fi
					test_cmd[$act_tests]=`basename ${test_cmd[$act_tests]}`
					# get test case name, which used in log file name
					test_name[$act_tests]=`basename ${test_cmd[$act_tests]} ".stf"`
					args=`echo "$inline" | cut -f4- -d' '`
					if [ -n "$args" ]
					then
						test_cmd[$act_tests]=`echo "${test_cmd[$act_tests]} $args"`
					fi
					# assign log file to test
					logs[$act_tests]=$tlog.$group.$act_tests.${test_name[$act_tests]}
					# set extra time to zero to start with
					loop_time[$act_tests]=0
					# start test running
					st_test $act_tests
					# inc number of test
					((act_tests=act_tests+1))
				fi
			fi
		;;
		# unix command
		[a-z])
			# monitor test(s) & wait for completion
			if [ "$inline" = "wait" ]
			then 
				if [ $act_tests -gt 0 ]
				then
					
					runner
					((group=group+1))
				fi
				count_wait=$count
				wait
			else
				echo "$inline" >> $report
				# do unix command
				$inline >> $report 2>&1
				# if error
				if [ $? -ne 0 ]
				then
					echo "\n!ERROR: Bad command line: $inline\n" >> $report
					kill_tests
					end_report error
					exit 1
				fi
			fi
		;;
		# options
		[A-Z])
                                echo ">OPTION: \c" >> $report
                                option=`echo "$inline" | cut -f1 -d'='`
                                opval=`echo "$inline" | cut -f2 -d'='`
                                if [ "$option" = "LOG" ]
                                then
                                        rlog[$count]=$opval
                                        echo "LOG files saved = ${rlog[$count]}" >> $report
                                elif [ "$option" = "LOADMAX" ]
                                then
                                	loadmax=$opval
                                	echo "LOADMAX = $loadmax" >> $report
                                fi
                                ((count=count+1))
		;;
		esac
		
	done
	
	# no wait in command file, monitor test(s) & wait for completion
	if [ $act_tests -ne 0 ]
	then 
		runner
		wait
	fi
}

#Monitor system load average over course of test run
mon_load() {

	if [[ $1 == "end" ]]
	then
		pkill -9 -x load
		rm -f /tmp/ld.op
		rm -f /tmp/sh*
	fi
}

### MAIN ###

#Add by Martin Mei 02/09/04 ::=BEGIN
LANG=C; export LANG
#Add by Martin Mei 02/09/04 ::=END

# check running directory
if [ $PWD != $wdir ]
then
	cd $wdir
fi

# check report directory
if [ ! -d $repdir ]
then
	mkdir -m 777 -p $repdir
fi

# determine report filename
dateh=`/bin/date -u +"\`echo \"\045m\045d\045y_\045H\045M\045S\"\`"`
repname=$repdir/STAF_${suite}_${level}_$dateh.PROSUM

# added by Matthew Zhang on Feb. 28 for bug 4997040
suite_level_log=$repdir/.STAF_$dateh.SUITE_LEVEL.log

# determine cksum filename
cksum_file=$repdir/.$dateh.cks

# determine log filename
dateh=`/bin/date -u +"\`echo \"\045m\045d\045y_\045H\045M\045S\"\`"`
logname=$repdir/STAF_${suite}_${level}_$dateh.LOG_GROUP_

# check for report file
if [ -a $report ]
then
	ps -ef|egrep ".stf"| egrep -v egrep >/dev/null 2>&1 
	if [ $? -eq 0 ]
	then
		echo "\n!ERROR: STAF already running.\n"
		exit 1
	else
		echo "\n!ERROR: STAF status is not correct. Check STAF status.\n"
		exit 1
	fi
fi

# print start report
st_report

# check for root id
if [ `/usr/ucb/whoami` != root ]
then
	sys_err "\n!ERROR: This progam must be run as root.\n"
fi


# check for args
if [ $# -ne 1 ]
then
	tty -s
	if [ $? -ne 0 ]
	then
		read fname
		fname=`echo $fname | tr -d "\r"`
		if [ -z "$fname" ] 
		then
			sys_err "\nUSAGE: $0 [command file]\n"
		fi
	else
		sys_err "\nUSAGE: $0 [command file]\n"
	fi
fi

# check for command file existence
if [ ! -s $fname ]
then
	if [ -s $cdir/$fname ]
	then
		fname=$cdir/$fname
	else
		sys_err "\n!ERROR: Cannot find command file: $fname or $cdir/$fname\n"
	fi
fi

# print system info to report
get_si

# process command file
proc_cmd_file

# finish report
end_report pass
