/*----------------------------------------------------------------------------*/
/* Hobbit monitor network test tool.                                          */
/*                                                                            */
/* Copyright (C) 2003-2006 Henrik Storner <henrik@hswn.dk>                    */
/*                                                                            */
/* This program is released under the GNU General Public License (GPL),       */
/* version 2. See the file "COPYING" for details.                             */
/*                                                                            */
/*----------------------------------------------------------------------------*/

static char rcsid[] = "$Id: bbtest-net.c,v 1.243 2006/08/07 16:28:22 henrik Rel $";

#include <limits.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <ctype.h>
#include <netdb.h>
#include <sys/wait.h>
#include <rpc/rpc.h>
#include <fcntl.h>
#include <errno.h>

#include "libbbgen.h"

#ifdef HAVE_RPCENT_H
#include <rpc/rpcent.h>
#endif

#ifdef BROKEN_HPUX_NETDB
/* 
 * Some HP-UX include files fail to define RPC functions 
 * and structs that are purely standard. At the same time,
 * their own docs claim that the DO define them. Go figure ...
 */
struct rpcent {
	char    *r_name;        /* name of server for this rpc program */
	char    **r_aliases;    /* alias list */
	int     r_number;       /* rpc program number */
};
extern struct rpcent *getrpcbyname(char *);
#endif

#include "libbbgen.h"
#include "version.h"

#include "bbtest-net.h"
#include "dns.h"
#include "contest.h"
#include "httptest.h"
#include "httpresult.h"
#include "ldaptest.h"

char *reqenv[] = {
	"NONETPAGE",
	"BBHOSTS",
	"BBTMP",
	"BBHOME",
	"BB",
	"BBDISP",
	NULL
};

/* toolid values */
#define TOOL_CONTEST	0
#define TOOL_NSLOOKUP	1
#define TOOL_DIG	2
#define TOOL_NTP        3
#define TOOL_FPING      4
#define TOOL_HTTP       5
#define TOOL_MODEMBANK  6
#define TOOL_LDAP	7
#define TOOL_RPCINFO	8


RbtHandle	svctree;			/* All known services, has service_t records */
service_t	*pingtest = NULL;		/* Identifies the pingtest within svctree list */
int		pingcount = 0;
service_t	*dnstest = NULL;		/* Identifies the dnstest within svctree list */
service_t	*digtest = NULL;		/* Identifies the digtest within svctree list */
service_t	*httptest = NULL;		/* Identifies the httptest within svctree list */
service_t	*ldaptest = NULL;		/* Identifies the ldaptest within svctree list */
service_t	*rpctest = NULL;		/* Identifies the rpctest within svctree list */
service_t	*modembanktest = NULL;		/* Identifies the modembank test within svctree list */
RbtHandle       testhosttree;			/* All tested hosts, has testedhost_t records */
char		*nonetpage = NULL;		/* The "NONETPAGE" env. variable */
int		dnsmethod = DNS_THEN_IP;	/* How to do DNS lookups */
int 		timeout=10;			/* The timeout (seconds) for all TCP-tests */
char		*contenttestname = "content";   /* Name of the content checks column */
char		*ssltestname = "sslcert";       /* Name of the SSL certificate checks column */
char		*failtext = "not OK";
int             sslwarndays = 30;		/* If cert expires in fewer days, SSL cert column = yellow */
int             sslalarmdays = 10;		/* If cert expires in fewer days, SSL cert column = red */
char		*location = "";			/* BBLOCATION value */
int		hostcount = 0;
int		testcount = 0;
int		notesthostcount = 0;
char		**selectedhosts;
int		selectedcount = 0;
int		testuntagged = 0;
time_t		frequenttestlimit = 1800;	/* Interval (seconds) when failing hosts are retried frequently */
int		checktcpresponse = 0;
int		dotraceroute = 0;
int		fqdn = 1;
int		dosendflags = 1;
char		*pingcmd = NULL;
char		pinglog[PATH_MAX];
char		pingerrlog[PATH_MAX];
int		respcheck_color = COL_YELLOW;
int		extcmdtimeout = 30;
int		bigfailure = 0;

void dump_hostlist(void)
{
	RbtIterator handle;
	testedhost_t *walk;

	for (handle = rbtBegin(testhosttree); (handle != rbtEnd(testhosttree)); handle = rbtNext(testhosttree, handle)) {
		walk = (testedhost_t *)gettreeitem(testhosttree, handle);
		printf("Hostname: %s\n", textornull(walk->hostname));
		printf("\tIP           : %s\n", textornull(walk->ip));
		printf("\tHosttype     : %s\n", textornull(walk->hosttype));

		printf("\tFlags        :");
		if (walk->testip) printf(" testip");
		if (walk->dialup) printf(" dialup");
		if (walk->nosslcert) printf(" nosslcert");
		if (walk->dodns) printf(" dodns");
		if (walk->dnserror) printf(" dnserror");
		if (walk->repeattest) printf(" repeattest");
		if (walk->noconn) printf(" noconn");
		if (walk->noping) printf(" noping");
		if (walk->dotrace) printf(" dotrace");
		printf("\n");

		printf("\tbadconn      : %d:%d:%d\n", walk->badconn[0], walk->badconn[1], walk->badconn[2]);
		printf("\tdowncount    : %d started %s", walk->downcount, ctime(&walk->downstart));
		printf("\trouterdeps   : %s\n", textornull(walk->routerdeps));
		printf("\tdeprouterdown: %s\n", (walk->deprouterdown ? textornull(walk->deprouterdown->hostname) : ""));
		printf("\tldapauth     : '%s' '%s'\n", textornull(walk->ldapuser), textornull(walk->ldappasswd));
		printf("\tSSL alerts   : %d:%d\n", walk->sslwarndays, walk->sslalarmdays);
		printf("\n");
	}
}
void dump_testitems(void)
{
	RbtIterator handle;
	service_t *swalk;
	testitem_t *iwalk;

	for (handle = rbtBegin(svctree); handle != rbtEnd(svctree); handle = rbtNext(svctree, handle)) {
		swalk = (service_t *)gettreeitem(svctree, handle);

		printf("Service %s, port %d, toolid %d\n", swalk->testname, swalk->portnum, swalk->toolid);

		for (iwalk = swalk->items; (iwalk); iwalk = iwalk->next) {
			if (swalk == modembanktest) {
				modembank_t *mentry;
				int i;

				mentry = iwalk->privdata;
				printf("\tModembank   : %s\n", textornull(mentry->hostname));
				printf("\tStart-IP    : %s\n", u32toIP(mentry->startip));
				printf("\tBanksize    : %d\n", mentry->banksize);
				printf("\tOpen        :");
				for (i=0; i<mentry->banksize; i++) printf(" %d", mentry->responses[i]);
				printf("\n");
			}
			else {
				printf("\tHost        : %s\n", textornull(iwalk->host->hostname));
				printf("\ttestspec    : %s\n", textornull(iwalk->testspec));
				printf("\tFlags       :");
				if (iwalk->dialup) printf(" dialup");
				if (iwalk->reverse) printf(" reverse");
				if (iwalk->silenttest) printf(" silenttest");
				if (iwalk->alwaystrue) printf(" alwaystrue");
				printf("\n");
				printf("\tOpen        : %d\n", iwalk->open);
				printf("\tBanner      : %s\n", textornull(STRBUF(iwalk->banner)));
				printf("\tcertinfo    : %s\n", textornull(iwalk->certinfo));
				printf("\tDuration    : %ld.%06ld\n", (long int)iwalk->duration.tv_sec, (long int)iwalk->duration.tv_usec);
				printf("\tbadtest     : %d:%d:%d\n", iwalk->badtest[0], iwalk->badtest[1], iwalk->badtest[2]);
				printf("\tdowncount    : %d started %s", iwalk->downcount, ctime(&iwalk->downstart));
				printf("\n");
			}
		}

		printf("\n");
	}
}

testitem_t *find_test(char *hostname, char *testname)
{
	RbtIterator handle;
	testedhost_t *h;
	service_t *s;
	testitem_t *t;

	handle = rbtFind(svctree, testname);
	if (handle == rbtEnd(svctree)) return NULL;
	s = (service_t *)gettreeitem(svctree, handle);

	handle = rbtFind(testhosttree, hostname);
	if (handle == rbtEnd(testhosttree)) return NULL;
	h = (testedhost_t *)gettreeitem(testhosttree, handle);

	for (t=s->items; (t && (t->host != h)); t = t->next) ;

	return t;
}


char *deptest_failed(testedhost_t *host, char *testname)
{
	static char result[1024];

	char *depcopy;
	char depitem[MAX_LINE_LEN];
	char *p, *q;
	char *dephostname, *deptestname, *nextdep;
	testitem_t *t;

	if (host->deptests == NULL) return NULL;

	depcopy = strdup(host->deptests);
	sprintf(depitem, "(%s:", testname);
	p = strstr(depcopy, depitem);
	if (p == NULL) { xfree(depcopy); return NULL; }

	result[0] = '\0';
	dephostname = p+strlen(depitem);
	q = strchr(dephostname, ')');
	if (q) *q = '\0';

	/* dephostname now points to a list of "host1/test1,host2/test2" dependent tests. */
	while (dephostname) {
		p = strchr(dephostname, '/');
		if (p) {
			*p = '\0';
			deptestname = (p+1); 
		}
		else deptestname = "";

		p = strchr(deptestname, ',');
		if (p) {
			*p = '\0';
			nextdep = (p+1);
		}
		else nextdep = NULL;

		t = find_test(dephostname, deptestname);
		if (t && !t->open) {
			if (strlen(result) == 0) {
				strcpy(result, "\nThis test depends on the following test(s) that failed:\n\n");
			}

			if ((strlen(result) + strlen(dephostname) + strlen(deptestname) + 2) < sizeof(result)) {
				strcat(result, dephostname);
				strcat(result, "/");
				strcat(result, deptestname);
				strcat(result, "\n");
			}
		}

		dephostname = nextdep;
	}

	xfree(depcopy);
	if (strlen(result)) strcat(result, "\n\n");

	return (strlen(result) ? result : NULL);
}


service_t *add_service(char *name, int port, int namelen, int toolid)
{
	RbtIterator handle;
	service_t *svc;

	/* Avoid duplicates */
	handle = rbtFind(svctree, name);
	if (handle != rbtEnd(svctree)) {
		svc = (service_t *)gettreeitem(svctree, handle);
		return svc;
	}

	svc = (service_t *) malloc(sizeof(service_t));
	svc->portnum = port;
	svc->testname = strdup(name); 
	svc->toolid = toolid;
	svc->namelen = namelen;
	svc->items = NULL;
	rbtInsert(svctree, svc->testname, svc);

	return svc;
}

int getportnumber(char *svcname)
{
	struct servent *svcinfo;
	int result = 0;

	result = default_tcp_port(svcname);
	if (result == 0) {
		svcinfo = getservbyname(svcname, NULL);
		if (svcinfo) result = ntohs(svcinfo->s_port);
	}

	return result;
}

void load_services(void)
{
	char *netsvcs;
	char *p;

	netsvcs = strdup(init_tcp_services());

	p = strtok(netsvcs, " ");
	while (p) {
		add_service(p, getportnumber(p), 0, TOOL_CONTEST);
		p = strtok(NULL, " ");
	}
	xfree(netsvcs);

	/* Save NONETPAGE env. var in ",test1,test2," format for easy and safe grepping */
	nonetpage = (char *) malloc(strlen(xgetenv("NONETPAGE"))+3);
	sprintf(nonetpage, ",%s,", xgetenv("NONETPAGE"));
	for (p=nonetpage; (*p); p++) if (*p == ' ') *p = ',';
}


testedhost_t *init_testedhost(char *hostname)
{
	testedhost_t *newhost;

	hostcount++;
	newhost = (testedhost_t *) calloc(1, sizeof(testedhost_t));
	newhost->hostname = strdup(hostname);
	newhost->dotrace = dotraceroute;
	newhost->sslwarndays = sslwarndays;
	newhost->sslalarmdays = sslalarmdays;

	return newhost;
}

testitem_t *init_testitem(testedhost_t *host, service_t *service, char *testspec, 
                          int dialuptest, int reversetest, int alwaystruetest, int silenttest,
			  int sendasdata)
{
	testitem_t *newtest;

	testcount++;
	newtest = (testitem_t *) malloc(sizeof(testitem_t));
	newtest->host = host;
	newtest->service = service;
	newtest->dialup = dialuptest;
	newtest->reverse = reversetest;
	newtest->alwaystrue = alwaystruetest;
	newtest->silenttest = silenttest;
	newtest->senddata = sendasdata;
	newtest->testspec = (testspec ? strdup(testspec) : NULL);
	newtest->privdata = NULL;
	newtest->open = 0;
	newtest->banner = newstrbuffer(0);
	newtest->certinfo = NULL;
	newtest->certexpires = 0;
	newtest->duration.tv_sec = newtest->duration.tv_usec = -1;
	newtest->downcount = 0;
	newtest->badtest[0] = newtest->badtest[1] = newtest->badtest[2] = 0;
	newtest->next = NULL;

	return newtest;
}


int wanted_host(namelist_t *host, char *netstring)
{
	char *netlocation = bbh_item(host, BBH_NET);

	if (selectedcount == 0)
		return ((strlen(netstring) == 0) || 				   /* No BBLOCATION = do all */
			(netlocation && (strcmp(netlocation, netstring) == 0)) ||  /* BBLOCATION && matching NET: tag */
			(testuntagged && (netlocation == NULL)));		   /* No NET: tag for this host */
	else {
		/* User provided an explicit list of hosts to test */
		int i;

		for (i=0; (i < selectedcount); i++) {
			if (strcmp(selectedhosts[i], host->bbhostname) == 0) return 1;
		}
	}

	return 0;
}


void load_tests(void)
{
	char *p, *routestring = NULL;
	namelist_t *hosts, *hwalk;
	testedhost_t *h;

	hosts = load_hostnames(xgetenv("BBHOSTS"), "netinclude", get_fqdn());
	if (hosts == NULL) {
		errprintf("Cannot load bb-hosts\n");
		return;
	}

	/* Each network test tagged with NET:locationname */
	if (strlen(location) > 0) {
		routestring = (char *) malloc(strlen(location)+strlen("route_:")+1);
		sprintf(routestring, "route_%s:", location);
	}

	for (hwalk = hosts; (hwalk); hwalk = hwalk->next) {
		int anytests = 0;
		int ping_dialuptest = 0, ping_reversetest = 0;
		char *testspec;

		if (!wanted_host(hwalk, location)) continue;

		if (argnmatch(hwalk->bbhostname, "@dialup.")) {
			/* Modembank entry: "dialup displayname startIP count" */

			char *realname;
			testitem_t *newtest;
			modembank_t *newentry;
			int i, ip1, ip2, ip3, ip4, banksize;

			realname = hwalk->bbhostname + strlen("@dialup.");
			banksize = atoi(bbh_item(hwalk, BBH_BANKSIZE));
			sscanf(bbh_item(hwalk, BBH_IP), "%d.%d.%d.%d", &ip1, &ip2, &ip3, &ip4);

			newtest = init_testitem(NULL, modembanktest, NULL, 0, 0, 0, 0, 0);
			newtest->next = modembanktest->items;
			modembanktest->items = newtest;

			newtest->privdata = (void *)malloc(sizeof(modembank_t));
			newentry = (modembank_t *)newtest->privdata;
			newentry->hostname = realname;
			newentry->startip = IPtou32(ip1, ip2, ip3, ip4);
			newentry->banksize = banksize;
			newentry->responses = (int *) malloc(banksize * sizeof(int));
			for (i=0; i<banksize; i++) newentry->responses[i] = 0;

			/* No more to do for modembanks */
			continue;
		}


		h = init_testedhost(hwalk->bbhostname);

		p = bbh_custom_item(hwalk, "badconn:");
		if (p) sscanf(p+strlen("badconn:"), "%d:%d:%d", &h->badconn[0], &h->badconn[1], &h->badconn[2]);

		p = bbh_custom_item(hwalk, "route:");
		if (p) h->routerdeps = p + strlen("route:");
		if (routestring) {
			p = bbh_custom_item(hwalk, routestring);
			if (p) h->routerdeps = p + strlen(routestring);
		}

		if (bbh_item(hwalk, BBH_FLAG_NOCONN)) h->noconn = 1;
		if (bbh_item(hwalk, BBH_FLAG_NOPING)) h->noping = 1;
		if (bbh_item(hwalk, BBH_FLAG_TRACE)) h->dotrace = 1;
		if (bbh_item(hwalk, BBH_FLAG_NOTRACE)) h->dotrace = 0;
		if (bbh_item(hwalk, BBH_FLAG_TESTIP)) h->testip = 1;
		if (bbh_item(hwalk, BBH_FLAG_DIALUP)) h->dialup = 1;
		if (bbh_item(hwalk, BBH_FLAG_NOSSLCERT)) h->nosslcert = 1;
		if (bbh_item(hwalk, BBH_FLAG_LDAPFAILYELLOW)) h->ldapsearchfailyellow = 1;
		if (bbh_item(hwalk, BBH_FLAG_HIDEHTTP)) h->hidehttp = 1;

		p = bbh_item(hwalk, BBH_SSLDAYS);
		if (p) sscanf(p, "%d:%d", &h->sslwarndays, &h->sslalarmdays);

		p = bbh_item(hwalk, BBH_DEPENDS);
		if (p) h->deptests = p;

		p = bbh_item(hwalk, BBH_LDAPLOGIN);
		if (p) {
			h->ldapuser = strdup(p);
			h->ldappasswd = (strchr(h->ldapuser, ':'));
			if (h->ldappasswd) {
				*h->ldappasswd = '\0';
				h->ldappasswd++;
			}
		}

		p = bbh_item(hwalk, BBH_DESCRIPTION);
		if (p) {
			h->hosttype = strdup(p);
			p = strchr(h->hosttype, ':');
			if (p) *p = '\0';
		}

		testspec = bbh_item_walk(hwalk);
		while (testspec) {
			service_t *s = NULL;
			int dialuptest = 0, reversetest = 0, silenttest = 0, sendasdata = 0;
			int alwaystruetest = (bbh_item(hwalk, BBH_FLAG_NOCLEAR) != NULL);

			if (bbh_item_idx(testspec) == -1) {

				/* Test prefixes:
				 * - '?' denotes dialup test, i.e. report failures as clear.
				 * - '|' denotes reverse test, i.e. service should be DOWN.
				 * - '~' denotes test that ignores ping result (normally,
				 *       TCP tests are reported CLEAR if ping check fails;
				 *       with this flag report their true status)
				 */
				if (*testspec == '?') { dialuptest=1;     testspec++; }
				if (*testspec == '!') { reversetest=1;    testspec++; }
				if (*testspec == '~') { alwaystruetest=1; testspec++; }

				if (pingtest && argnmatch(testspec, pingtest->testname)) {
					char *p;

					/*
					 * Ping/conn test. Save any modifier flags for later use.
					 */
					ping_dialuptest = dialuptest;
					ping_reversetest = reversetest;
					p = strchr(testspec, '=');
					if (p) {
						char *ips;

						/* Extra ping tests - save them for later */
						h->extrapings = (extraping_t *)malloc(sizeof(extraping_t));
						h->extrapings->iplist = NULL;
						if (argnmatch(p, "=worst,")) {
							h->extrapings->matchtype = MULTIPING_WORST;
							ips = strdup(p+7);
						}
						else if (argnmatch(p, "=best,")) {
							h->extrapings->matchtype = MULTIPING_BEST;
							ips = strdup(p+6);
						}
						else {
							h->extrapings->matchtype = MULTIPING_BEST;
							ips = strdup(p+1);
						}

						do {
							ipping_t *newping = (ipping_t *)malloc(sizeof(ipping_t));

							newping->ip = ips;
							newping->open = 0;
							newping->banner = newstrbuffer(0);
							newping->next = h->extrapings->iplist;
							h->extrapings->iplist = newping;
							ips = strchr(ips, ',');
							if (ips) { *ips = '\0'; ips++; }
						} while (ips && (*ips));
					}
					s = NULL; /* Dont add the test now - ping is special (enabled by default) */
				}
				else if ((argnmatch(testspec, "ldap://")) || (argnmatch(testspec, "ldaps://"))) {
					/*
					 * LDAP test. This uses ':' a lot, so save it here.
					 */
#ifdef BBGEN_LDAP
					s = ldaptest;
					add_url_to_dns_queue(testspec);
#else
					errprintf("ldap test requested, but bbgen was built with no ldap support\n");
#endif
				}
				else if ((strcmp(testspec, "http") == 0) || (strcmp(testspec, "https") == 0)) {
					errprintf("http/https tests requires a full URL\n");
				}
				else if ( argnmatch(testspec, "http")         ||
					  argnmatch(testspec, "content=http") ||
					  argnmatch(testspec, "cont;http")    ||
					  argnmatch(testspec, "cont=")        ||
					  argnmatch(testspec, "nocont;http")  ||
					  argnmatch(testspec, "nocont=")      ||
					  argnmatch(testspec, "post;http")    ||
					  argnmatch(testspec, "post=")        ||
					  argnmatch(testspec, "nopost;http")  ||
					  argnmatch(testspec, "nopost=")      ||
					  argnmatch(testspec, "type;http")    ||
					  argnmatch(testspec, "type=")        )      {

					/* HTTP test. */
					bburl_t url;

					decode_url(testspec, &url);
					if (url.desturl->parseerror || (url.proxyurl && url.proxyurl->parseerror)) {
						s = NULL;
						errprintf("Invalid URL for http test - ignored: %s\n", testspec);
					}
					else {
						s = httptest;
						add_url_to_dns_queue(testspec);
					}
				}
				else if (argnmatch(testspec, "apache") || argnmatch(testspec, "apache=")) {
					char *userfmt = "cont=apache;%s;.";
					char *deffmt = "cont=apache;http://%s/server-status?auto;.";
					static char *statusurl = NULL;
					char *userurl;

					if (statusurl != NULL) xfree(statusurl);

					userurl = strchr(testspec, '='); 
					if (userurl) {
						bburl_t url;
						userurl++;

						decode_url(userurl, &url);
						if (url.desturl->parseerror || (url.proxyurl && url.proxyurl->parseerror)) {
							s = NULL;
							errprintf("Invalid URL for apache test - ignored: %s\n", testspec);
						}
						else {
							statusurl = (char *)malloc(strlen(userurl) + strlen(userfmt) + 1);
							sprintf(statusurl, userfmt, userurl);
							s = httptest;
						}
					}
					else {
						char *ip = bbh_item(hwalk, BBH_IP);
						statusurl = (char *)malloc(strlen(deffmt) + strlen(ip) + 1);
						sprintf(statusurl, deffmt, ip);
						s = httptest;
					}

					if (s) {
						testspec = statusurl;
						add_url_to_dns_queue(testspec);
						sendasdata = 1;
					}
				}
				else if (argnmatch(testspec, "rpc")) {
					/*
					 * rpc check via rpcinfo
					 */
					s = rpctest;
				}
				else if (argnmatch(testspec, "dns=")) {
					s = dnstest;
				}
				else if (argnmatch(testspec, "dig=")) {
					s = digtest;
				}
				else {
					/* 
					 * Simple TCP connect test. 
					 */
					char *option;
					RbtIterator handle;

					/* Remove any trailing ":s", ":q", ":Q", ":portnumber" */
					option = strchr(testspec, ':'); 
					if (option) { 
						*option = '\0'; 
						option++; 
					}
	
					/* Find the service */
					handle = rbtFind(svctree, testspec);
					s = ((handle == rbtEnd(svctree)) ? NULL : (service_t *)gettreeitem(svctree, handle));
					if (option && s) {
						/*
						 * Check if it is a service with an explicit portnumber.
						 * If it is, then create a new service record named
						 * "SERVICE_PORT" so we can merge tests for this service+port
						 * combination for multiple hosts.
						 *
						 * According to Hobbit docs, this type of services must be in
						 * BBNETSVCS - so it is known already.
						 */
						int specialport = 0;
						char *specialname;
						char *opt2 = strrchr(option, ':');

						if (opt2) {
							if (strcmp(opt2, ":s") == 0) {
								/* option = "portnumber:s" */
								silenttest = 1;
								*opt2 = '\0';
								specialport = atoi(option);
								*opt2 = ':';
							}
						}
						else if (strcmp(option, "s") == 0) {
							/* option = "s" */
							silenttest = 1;
							specialport = 0;
						}
						else {
							/* option = "portnumber" */
							specialport = atoi(option);
						}

						if (specialport) {
							specialname = (char *) malloc(strlen(s->testname)+10);
							sprintf(specialname, "%s_%d", s->testname, specialport);
							s = add_service(specialname, specialport, strlen(s->testname), TOOL_CONTEST);
							xfree(specialname);
						}
					}

					if (s) h->dodns = 1;
					if (option) *(option-1) = ':';
				}

				if (s) {
					testitem_t *newtest;

					anytests = 1;
					newtest = init_testitem(h, s, testspec, dialuptest, reversetest, alwaystruetest, silenttest, sendasdata);
					newtest->next = s->items;
					s->items = newtest;

					if (s == httptest) h->firsthttp = newtest;
					else if (s == ldaptest) h->firstldap = newtest;
				}
			}

			testspec = bbh_item_walk(NULL);
		}

		if (pingtest && !h->noconn) {
			/* Add the ping check */
			testitem_t *newtest;

			anytests = 1;
			newtest = init_testitem(h, pingtest, NULL, ping_dialuptest, ping_reversetest, 1, 0, 0);
			newtest->next = pingtest->items;
			pingtest->items = newtest;
			h->dodns = 1;
		}


		/* 
		 * Setup badXXX values.
		 *
		 * We need to do this last, because the testitem_t records do
		 * not exist until the test has been created.
		 *
		 * So after parsing the badFOO tag, we must find the testitem_t
		 * record created earlier for this test (it may not exist).
		 */
		testspec = bbh_item_walk(hwalk);
		while (testspec) {
			char *testname, *timespec, *badcounts;
			int badclear, badyellow, badred;
			int inscope;
			testitem_t *twalk;
			service_t *swalk;

			if (strncmp(testspec, "bad", 3) != 0) {
				/* Not a bad* tag - skip it */
				testspec = bbh_item_walk(NULL);
				continue;
			}


			badclear = badyellow = badred = 0;
			inscope = 1;

			testname = testspec+strlen("bad");
			badcounts = strchr(testspec, ':');
			if (badcounts) {
				if (sscanf(badcounts, ":%d:%d:%d", &badclear, &badyellow, &badred) != 3) {
					errprintf("Incorrect 'bad' counts: '%s'\n", badcounts);
					badcounts = NULL;
				}
			}
			timespec = strchr(testspec, '-');
			if (timespec) inscope = periodcoversnow(timespec);

			if (strlen(testname) && badcounts && inscope) {
				char *p;
				RbtIterator handle;
				twalk = NULL;

				p = strchr(testname, ':'); if (p) *p = '\0';
				handle = rbtFind(svctree, testname);
				swalk = ((handle == rbtEnd(svctree)) ? NULL : (service_t *)gettreeitem(svctree, handle));
				if (p) *p = ':';
				if (swalk) {
					if (swalk == httptest) twalk = h->firsthttp;
					else if (swalk == ldaptest) twalk = h->firstldap;
					else for (twalk = swalk->items; (twalk && (twalk->host != h)); twalk = twalk->next) ;
				}

				if (twalk) {
					twalk->badtest[0] = badclear;
					twalk->badtest[1] = badyellow;
					twalk->badtest[2] = badred;
				}
				else {
					dbgprintf("No test for badtest spec host=%s, test=%s\n",
						h->hostname, testname);
				}
			}

			testspec = bbh_item_walk(NULL);
		}


		if (anytests) {
			RbtStatus res;

			/* 
			 * Check for a duplicate host def. Causes all sorts of funny problems.
			 * However, dont drop the second definition - to do this, we will have
			 * to clean up the testitem lists as well, or we get crashes when 
			 * tests belong to a non-existing host.
			 */

			res = rbtInsert(testhosttree, h->hostname, h);
			if (res == RBT_STATUS_DUPLICATE_KEY) {
				errprintf("Host %s appears twice in bb-hosts! This may cause strange results\n", h->hostname);
			}
	
			strcpy(h->ip, bbh_item(hwalk, BBH_IP));
			if (!h->testip && (dnsmethod != IP_ONLY)) add_host_to_dns_queue(h->hostname);
		}
		else {
			/* No network tests for this host, so ignore it */
			dbgprintf("Did not find any network tests for host %s\n", h->hostname);
			xfree(h);
			notesthostcount++;
		}

	}

	return;
}

char *ip_to_test(testedhost_t *h)
{
	char *dnsresult;
	int nullip = (strcmp(h->ip, "0.0.0.0") == 0);

	if (!nullip && (h->testip || (dnsmethod == IP_ONLY))) {
		/* Already have the IP setup */
	}
	else if (h->dodns) {
		dnsresult = dnsresolve(h->hostname);

		if (dnsresult) {
			strcpy(h->ip, dnsresult);
		}
		else if ((dnsmethod == DNS_THEN_IP) && !nullip) {
			/* Already have the IP setup */
		}
		else {
			/* Cannot resolve hostname */
			h->dnserror = 1;
			errprintf("bbtest-net: Cannot resolve IP for host %s\n", h->hostname);
		}
	}

	return h->ip;
}


void load_ping_status(void)
{
	FILE *statusfd;
	char statusfn[PATH_MAX];
	char l[MAX_LINE_LEN];
	char host[MAX_LINE_LEN];
	int  downcount;
	time_t downstart;
	RbtIterator handle;
	testedhost_t *h;

	sprintf(statusfn, "%s/ping.%s.status", xgetenv("BBTMP"), location);
	statusfd = fopen(statusfn, "r");
	if (statusfd == NULL) return;

	while (fgets(l, sizeof(l), statusfd)) {
		unsigned int uidownstart;
		if (sscanf(l, "%s %d %u", host, &downcount, &uidownstart) == 3) {
			downstart = uidownstart;
			handle = rbtFind(testhosttree, host);
			if (handle != rbtEnd(testhosttree)) {
				h = (testedhost_t *)gettreeitem(testhosttree, handle);
				if (!h->noping && !h->noconn) {
					h->downcount = downcount;
					h->downstart = downstart;
				}
			}
		}
	}

	fclose(statusfd);
}

void save_ping_status(void)
{
	FILE *statusfd;
	char statusfn[PATH_MAX];
	testitem_t *t;
	int didany = 0;

	sprintf(statusfn, "%s/ping.%s.status", xgetenv("BBTMP"), location);
	statusfd = fopen(statusfn, "w");
	if (statusfd == NULL) return;

	for (t=pingtest->items; (t); t = t->next) {
		if (t->host->downcount) {
			fprintf(statusfd, "%s %d %u\n", t->host->hostname, t->host->downcount, (unsigned int)t->host->downstart);
			didany = 1;
			t->host->repeattest = ((time(NULL) - t->host->downstart) < frequenttestlimit);
		}
	}

	fclose(statusfd);
	if (!didany) unlink(statusfn);
}

void load_test_status(service_t *test)
{
	FILE *statusfd;
	char statusfn[PATH_MAX];
	char l[MAX_LINE_LEN];
	char host[MAX_LINE_LEN];
	int  downcount;
	time_t downstart;
	RbtIterator handle;
	testedhost_t *h;
	testitem_t *walk;

	sprintf(statusfn, "%s/%s.%s.status", xgetenv("BBTMP"), test->testname, location);
	statusfd = fopen(statusfn, "r");
	if (statusfd == NULL) return;

	while (fgets(l, sizeof(l), statusfd)) {
		unsigned int uidownstart;
		if (sscanf(l, "%s %d %u", host, &downcount, &uidownstart) == 3) {
			downstart = uidownstart;
			handle = rbtFind(testhosttree, host);
			if (handle != rbtEnd(testhosttree)) {
				h = (testedhost_t *)gettreeitem(testhosttree, handle);
				if (test == httptest) walk = h->firsthttp;
				else if (test == ldaptest) walk = h->firstldap;
				else for (walk = test->items; (walk && (walk->host != h)); walk = walk->next) ;

				if (walk) {
					walk->downcount = downcount;
					walk->downstart = downstart;
				}
			}
		}
	}

	fclose(statusfd);
}

void save_test_status(service_t *test)
{
	FILE *statusfd;
	char statusfn[PATH_MAX];
	testitem_t *t;
	int didany = 0;

	sprintf(statusfn, "%s/%s.%s.status", xgetenv("BBTMP"), test->testname, location);
	statusfd = fopen(statusfn, "w");
	if (statusfd == NULL) return;

	for (t=test->items; (t); t = t->next) {
		if (t->downcount) {
			fprintf(statusfd, "%s %d %u\n", t->host->hostname, t->downcount, (unsigned int)t->downstart);
			didany = 1;
			t->host->repeattest = ((time(NULL) - t->downstart) < frequenttestlimit);
		}
	}

	fclose(statusfd);
	if (!didany) unlink(statusfn);
}


void save_frequenttestlist(int argc, char *argv[])
{
	FILE *fd;
	char fn[PATH_MAX];
	RbtIterator handle;
	testedhost_t *h;
	int didany = 0;
	int i;

	sprintf(fn, "%s/frequenttests.%s", xgetenv("BBTMP"), location);
	fd = fopen(fn, "w");
	if (fd == NULL) return;

	for (i=1; (i<argc); i++) {
		if (!argnmatch(argv[i], "--report")) fprintf(fd, "%s ", argv[i]);
	}
	for (handle = rbtBegin(testhosttree); (handle != rbtEnd(testhosttree)); handle = rbtNext(testhosttree, handle)) {
		h = (testedhost_t *)gettreeitem(testhosttree, handle);
		if (h->repeattest) {
			fprintf(fd, "%s ", h->hostname);
			didany = 1;
		}
	}

	fclose(fd);
	if (!didany) unlink(fn);
}


void run_nslookup_service(service_t *service)
{
	testitem_t	*t;
	char		*lookup;

	for (t=service->items; (t); t = t->next) {
		if (!t->host->dnserror) {
			if (t->testspec && (lookup = strchr(t->testspec, '='))) {
				lookup++; 
			}
			else {
				lookup = t->host->hostname;
			}

			t->open = (dns_test_server(ip_to_test(t->host), lookup, t->banner) == 0);
		}
	}
}

void run_ntp_service(service_t *service)
{
	testitem_t	*t;
	char		cmd[1024];
	char		*p;
	char		cmdpath[PATH_MAX];

	p = xgetenv("NTPDATE");
	strcpy(cmdpath, (p ? p : "ntpdate"));
	for (t=service->items; (t); t = t->next) {
		if (!t->host->dnserror) {
			sprintf(cmd, "%s -u -q -p 2 %s 2>&1", cmdpath, ip_to_test(t->host));
			t->open = (run_command(cmd, "no server suitable for synchronization", t->banner, 1, extcmdtimeout) == 0);
		}
	}
}


void run_rpcinfo_service(service_t *service)
{
	testitem_t	*t;
	char		cmd[1024];
	char		*p;
	char		cmdpath[PATH_MAX];

	p = xgetenv("RPCINFO");
	strcpy(cmdpath, (p ? p : "rpcinfo"));
	for (t=service->items; (t); t = t->next) {
		if (!t->host->dnserror && (t->host->downcount == 0)) {
			sprintf(cmd, "%s -p %s 2>&1", cmdpath, ip_to_test(t->host));
			t->open = (run_command(cmd, NULL, t->banner, 1, extcmdtimeout) == 0);
		}
	}
}


int start_ping_service(service_t *service)
{
	testitem_t *t;
	char *cmd;
	char **cmdargs;
	int pfd[2];
	int status;

	/*
	 * The idea here is to run ping in a separate process, in parallel
	 * with some other time-consuming task (the TCP network tests).
	 * We cannot use the simple "popen()/pclose()" interface, because
	 *   a) ping doesn't start the tests until EOF is reached on stdin
	 *   b) EOF on stdin happens with pclose(), but it will also wait
	 *      for the process to finish.
	 *
	 * Therefore this slightly more complex solution, which in essence
	 * forks a new process running "hobbitping 2>&1 1>$BBTMP/ping.$$"
	 * The output is then picked up by the finish_ping_service().
	 */

	pingcmd = strdup(getenv_default("FPING", "hobbitping", NULL));
	pingcmd = realloc(pingcmd, strlen(pingcmd)+5);
	strcat(pingcmd, " -Ae");

	sprintf(pinglog, "%s/ping-stdout.%lu", xgetenv("BBTMP"), (unsigned long)getpid());
	sprintf(pingerrlog, "%s/ping-stderr.%lu", xgetenv("BBTMP"), (unsigned long)getpid());

	/* Setup command line and arguments */
	cmdargs = setup_commandargs(pingcmd, &cmd);

	/* Get a pipe FD */
	status = pipe(pfd);
	if (status == -1) {
		errprintf("Could not create pipe for hobbitping\n");
		return -1;
	}

	/* Now fork off the ping child-process */
	status = fork();
	if (status < 0) {
		errprintf("Could not fork() the ping child\n");
		return -1;
	}
	else if (status == 0) {
		/*
		 * child must have
		 *  - stdin fed from the parent
		 *  - stdout going to a file
		 *  - stderr going to another file. This is important, as
		 *    putting it together with stdout will wreak havoc when 
		 *    we start parsing the output later on. We could just 
		 *    dump it to /dev/null, but it might be useful to see
		 *    what went wrong.
		 */
		int outfile, errfile;

		outfile = open(pinglog, O_CREAT|O_WRONLY|O_TRUNC, S_IRUSR|S_IWUSR);
		if (outfile == -1) errprintf("Cannot create file %s : %s\n", pinglog, strerror(errno));
		errfile = open(pingerrlog, O_CREAT|O_WRONLY|O_TRUNC, S_IRUSR|S_IWUSR);
		if (errfile == -1) errprintf("Cannot create file %s : %s\n", pingerrlog, strerror(errno));

		if ((outfile == -1) || (errfile == -1)) {
			/* Ouch - cannot create our output files. Abort. */
			exit(98);
		}

		status = dup2(pfd[0], STDIN_FILENO);
		status = dup2(outfile, STDOUT_FILENO);
		status = dup2(errfile, STDERR_FILENO);
		close(pfd[0]); close(pfd[1]); close(outfile); close(errfile);

		execvp(cmd, cmdargs);

		/* Should never go here ... just kill the child */
		fprintf(stderr, "hobbitping invocation failed: %s\n", strerror(errno));
		exit(99);
	}
	else {
		/* parent */
		char ip[IP_ADDR_STRLEN+1];	/* Must have room for the \n at the end also */

		close(pfd[0]);
		pingcount = 0;

		/* Feed the IP's to test to the child */
		for (t=service->items; (t); t = t->next) {
			if (!t->host->dnserror && !t->host->noping) {
				sprintf(ip, "%s\n", ip_to_test(t->host));
				status = write(pfd[1], ip, strlen(ip));
				pingcount++;
				if (t->host->extrapings) {
					ipping_t *walk;

					for (walk = t->host->extrapings->iplist; (walk); walk = walk->next) {
						sprintf(ip, "%s\n", walk->ip);
						status = write(pfd[1], ip, strlen(ip));
						pingcount++;
					}
				}
			}
		}

		close(pfd[1]);	/* This is when ping starts doing tests */
	}

	return 0;
}


int finish_ping_service(service_t *service)
{
	testitem_t	*t;
	FILE		*logfd;
	char 		*p;
	char		l[MAX_LINE_LEN];
	char		pingip[MAX_LINE_LEN];
	int		ip1, ip2, ip3, ip4;
	int		pingstatus, failed = 0;

	/* 
	 * Wait for the ping child to finish.
	 * If we're lucky, it will be done already since it has run
	 * while we were doing tcp tests.
	 */
	wait(&pingstatus);
	switch (WEXITSTATUS(pingstatus)) {
	  case 0: /* All hosts reachable */
	  case 1: /* Some hosts unreachable */
	  case 2: /* Some IP's not found (should not happen) */
		break;

	  case 3: /* Bad command-line args, or not suid-root */
		failed = 1;
		errprintf("Execution of '%s' failed - program not suid root?\n", pingcmd);
		break;

	  case 98:
		failed = 1;
		errprintf("hobbitping child could not create outputfiles in %s\n", xgetenv("$BBTMP"));
		break;

	  case 99:
		failed = 1;
		errprintf("Could not run the command '%s' (exec failed)\n", pingcmd);
		break;

	  default:
		failed = 1;
		errprintf("Execution of '%s' failed with error-code %d\n", 
			pingcmd, WEXITSTATUS(pingstatus));
	}

	/* Load status of previously failed tests */
	load_ping_status();

	/* Open the new ping result file */
	logfd = fopen(pinglog, "r");
	if (logfd == NULL) { 
		failed = 1;
		errprintf("Cannot open ping output file %s\n", pinglog);
	}

	/* Copy error messages to the Hobbit logfile */
	if (failed) {
		FILE *errfd;
		char buf[1024];
			
		errfd = fopen(pingerrlog, "r");
		if (errfd && fgets(buf, sizeof(buf), errfd)) {
			errprintf("%s", buf);
		}
		if (errfd) fclose(errfd);
	}
	if (!debug) unlink(pingerrlog);

	if (failed) {
		/* Flag all ping tests as "undecided" */
		bigfailure = 1;
		for (t=service->items; (t); t = t->next) t->open = -1;
		goto cleanup;
	}

	/* The test did run, and we have a result-file. Look at it. */
	while (fgets(l, sizeof(l), logfd)) {
		p = strchr(l, '\n'); if (p) *p = '\0';
		if (sscanf(l, "%d.%d.%d.%d ", &ip1, &ip2, &ip3, &ip4) == 4) {

			sprintf(pingip, "%d.%d.%d.%d", ip1, ip2, ip3, ip4);

			/*
			 * Need to loop through all testitems - there may be multiple entries for
			 * the same IP-address.
			 */
			for (t=service->items; (t); t = t->next) {
				if (strcmp(t->host->ip, pingip) == 0) {
					if (t->open) dbgprintf("More than one ping result for %s\n", pingip);
					t->open = (strstr(l, "is alive") != NULL);
					t->banner = dupstrbuffer(l);
				}

				if (t->host->extrapings) {
					ipping_t *walk;
					for (walk = t->host->extrapings->iplist; (walk); walk = walk->next) {
						if (strcmp(walk->ip, pingip) == 0) {
							if (t->open) dbgprintf("More than one ping result for %s\n", pingip);
							walk->open = (strstr(l, "is alive") != NULL);
							walk->banner = dupstrbuffer(l);
						}
					}
				}
			}
		}
	}

	/* 
	 * Handle the router dependency stuff. I.e. for all hosts
	 * where the ping test failed, go through the list of router
	 * dependencies and if one of the dependent hosts also has 
	 * a failed ping test, point the dependency there.
	 */
	for (t=service->items; (t); t = t->next) {
		if (!t->open && t->host->routerdeps) {
			testitem_t *router;

			strcpy(l, t->host->routerdeps);
			p = strtok(l, ",");
			while (p && (t->host->deprouterdown == NULL)) {
				for (router=service->items; 
					(router && (strcmp(p, router->host->hostname) != 0)); 
					router = router->next) ;

				if (router && !router->open) t->host->deprouterdown = router->host;

				p = strtok(NULL, ",");
			}
		}
	}

cleanup:
	if (logfd) fclose(logfd);
	if (!debug) unlink(pinglog);

	return 0;
}

void run_modembank_service(service_t *service)
{
	testitem_t	*t;
	char		cmd[1024];
	char		startip[IP_ADDR_STRLEN], endip[IP_ADDR_STRLEN];
	char		*p;
	char		cmdpath[PATH_MAX];
	FILE		*cmdpipe;
	char		l[MAX_LINE_LEN];
	int		ip1, ip2, ip3, ip4;

	for (t=service->items; (t); t = t->next) {
		modembank_t *req = (modembank_t *)t->privdata;

		p = xgetenv("FPING");
		strcpy(cmdpath, (p ? p : "hobbitping"));
		strcpy(startip, u32toIP(req->startip));
		strcpy(endip, u32toIP(req->startip + req->banksize - 1));
		sprintf(cmd, "%s -g -Ae %s %s 2>/dev/null", cmdpath, startip, endip);

		dbgprintf("Running command: '%s'\n", cmd);
		cmdpipe = popen(cmd, "r");
		if (cmdpipe == NULL) {
			errprintf("Could not run the hobbitping command %s\n", cmd);
			return;
		}

		while (fgets(l, sizeof(l), cmdpipe)) {
			dbgprintf("modembank response: %s", l);

			if (sscanf(l, "%d.%d.%d.%d ", &ip1, &ip2, &ip3, &ip4) == 4) {
				unsigned int idx = IPtou32(ip1, ip2, ip3, ip4) - req->startip;

				if (idx >= req->banksize) {
					errprintf("Unexpected response for IP not in bank - %d.%d.%d.%d", 
						  ip1, ip2, ip3, ip4);
				}
				else {
					req->responses[idx] = (strstr(l, "is alive") != NULL);
				}
			}
		}
		pclose(cmdpipe);

		if (debug) {
			int i;

			dbgprintf("Results for modembank start=%s, length %d\n", u32toIP(req->startip), req->banksize);
			for (i=0; (i<req->banksize); i++)
				dbgprintf("\t%s is %d\n", u32toIP(req->startip+i), req->responses[i]);
		}
	}
}


int decide_color(service_t *service, char *svcname, testitem_t *test, int failgoesclear, char *cause)
{
	int color = COL_GREEN;
	int countasdown = 0;
	char *deptest = NULL;

	*cause = '\0';
	if (service == pingtest) {
		/*
		 * "noconn" is handled elsewhere.
		 * "noping" always sends back a status "clear".
		 * If DNS error, return red and count as down.
		 */
		if (test->open == -1) {
			/* Failed to run the ping utility. */
			strcpy(cause, "Hobbit system failure");
			return COL_CLEAR;
		}
		else if (test->host->noping) { 
			/* Ping test disabled - go "clear". End of story. */
			strcpy(cause, "Ping test disabled (noping)");
			return COL_CLEAR; 
		}
		else if (test->host->dnserror) { 
			strcpy(cause, "DNS lookup failure");
			color = COL_RED; countasdown = 1; 
		}
		else {
			if (test->host->extrapings == NULL) {
				/* Red if (open=0, reverse=0) or (open=1, reverse=1) */
				if ((test->open + test->reverse) != 1) { 
					sprintf(cause, "Host %s respond to ping", (test->open ? "does" : "does not"));
					color = COL_RED; countasdown = 1; 
				}
			}
			else {
				/* Host with many pings */
				int totalcount = 1;
				int okcount = test->open;
				ipping_t *walk;

				for (walk = test->host->extrapings->iplist; (walk); walk = walk->next) {
					if (walk->open) okcount++;
					totalcount++;
				}

				switch (test->host->extrapings->matchtype) {
				  case MULTIPING_BEST:
					  if (okcount == 0) {
						  color = COL_RED;
						  countasdown = 1;
						  sprintf(cause, "Host does not respond to ping on any of %d IP's", 
							  totalcount);
					  }
					  break;
				  case MULTIPING_WORST:
					  if (okcount < totalcount) {
						  color = COL_RED;
						  countasdown = 1;
						  sprintf(cause, "Host responds to ping on %d of %d IP's",
							  okcount, totalcount);
					  }
					  break;
				}
			}
		}

		/* Handle the "route" tag dependencies. */
		if ((color == COL_RED) && test->host->deprouterdown) { 
			char *routertext;

			routertext = test->host->deprouterdown->hosttype;
			if (routertext == NULL) routertext = xgetenv("BBROUTERTEXT");
			if (routertext == NULL) routertext = "router";

			strcat(cause, "\nIntermediate ");
			strcat(cause, routertext);
			strcat(cause, " down ");
			color = COL_YELLOW; 
		}

		/* Handle "badconn" */
		if ((color == COL_RED) && (test->host->downcount < test->host->badconn[2])) {
			if      (test->host->downcount >= test->host->badconn[1]) color = COL_YELLOW;
			else if (test->host->downcount >= test->host->badconn[0]) color = COL_CLEAR;
			else                                                      color = COL_GREEN;
		}

		/* Run traceroute , but not on dialup or reverse-test hosts */
		if ((color == COL_RED) && test->host->dotrace && !test->host->dialup && !test->reverse && !test->dialup) {
			char cmd[PATH_MAX];

			if (xgetenv("TRACEROUTE")) {
				sprintf(cmd, "%s %s 2>&1", xgetenv("TRACEROUTE"), test->host->ip);
			}
			else {
				sprintf(cmd, "traceroute -n -q 2 -w 2 -m 15 %s 2>&1", test->host->ip);
			}
			test->host->traceroute = newstrbuffer(0);
			run_command(cmd, NULL, test->host->traceroute, 0, extcmdtimeout);
		}
	}
	else {
		/* TCP test */
		if (test->host->dnserror) { 
			strcpy(cause, "DNS lookup failure");
			color = COL_RED; countasdown = 1; 
		}
		else {
			if (test->reverse) {
				/*
				 * Reverse tests go RED when open.
				 * If not open, they may go CLEAR if the ping test failed
				 */

				if (test->open) { 
					strcpy(cause, "Service responds when it should not");
					color = COL_RED; countasdown = 1; 
				}
				else if (failgoesclear && (test->host->downcount != 0) && !test->alwaystrue) {
					strcpy(cause, "Host appears to be down");
					color = COL_CLEAR; countasdown = 0;
				}
			}
			else {
				if (!test->open) {
					if (failgoesclear && (test->host->downcount != 0) && !test->alwaystrue) {
						strcpy(cause, "Host appears to be down");
						color = COL_CLEAR; countasdown = 0;
					}
					else {
						tcptest_t *tcptest = (tcptest_t *)test->privdata;

						strcpy(cause, "Service unavailable");
						if (tcptest) {
							switch (tcptest->errcode) {
							  case CONTEST_ETIMEOUT: 
								strcat(cause, " (connect timeout)"); 
								break;
							  case CONTEST_ENOCONN : 
								strcat(cause, " (");
								strcat(cause, strerror(tcptest->connres));
								strcat(cause, ")");
								break;
							  case CONTEST_EDNS    : 
								strcat(cause, " (DNS error)"); 
								break;
							  case CONTEST_EIO     : 
								strcat(cause, " (I/O error)"); 
								break;
							  case CONTEST_ESSL    : 
								strcat(cause, " (SSL error)"); 
								break;
							}
						}
						color = COL_RED; countasdown = 1;
					}
				}
				else {
					/* Check if we got the expected data */
					if (checktcpresponse && (service->toolid == TOOL_CONTEST) && !tcp_got_expected((tcptest_t *)test->privdata)) {
						strcpy(cause, "Unexpected service response");
						color = respcheck_color; countasdown = 1;
					}
				}
			}
		}

		/* Handle test dependencies */
		if ( failgoesclear && (color == COL_RED) && !test->alwaystrue && (deptest = deptest_failed(test->host, test->service->testname)) ) {
			strcpy(cause, deptest);
			color = COL_CLEAR;
		}

		/* Handle the "badtest" stuff for other tests */
		if ((color == COL_RED) && (test->downcount < test->badtest[2])) {
			if      (test->downcount >= test->badtest[1]) color = COL_YELLOW;
			else if (test->downcount >= test->badtest[0]) color = COL_CLEAR;
			else                                          color = COL_GREEN;
		}
	}


	/* Dialup hosts and dialup tests report red as clear */
	if ( ((color == COL_RED) || (color == COL_YELLOW)) && (test->host->dialup || test->dialup) && !test->reverse) { 
		strcat(cause, "\nDialup host or service");
		color = COL_CLEAR; countasdown = 0; 
	}

	/* If a NOPAGENET service, downgrade RED to YELLOW */
	if (color == COL_RED) {
		char *nopagename;

		/* Check if this service is a NOPAGENET service. */
		nopagename = (char *) malloc(strlen(svcname)+3);
		sprintf(nopagename, ",%s,", svcname);
		if (strstr(nonetpage, svcname) != NULL) color = COL_YELLOW;
		xfree(nopagename);
	}

	if (service == pingtest) {
		if (countasdown) {
			test->host->downcount++; 
			if (test->host->downcount == 1) test->host->downstart = time(NULL);
		}
		else test->host->downcount = 0;
	}
	else {
		if (countasdown) {
			test->downcount++; 
			if (test->downcount == 1) test->downstart = time(NULL);
		}
		else test->downcount = 0;
	}
	return color;
}


void send_results(service_t *service, int failgoesclear)
{
	testitem_t	*t;
	int		color;
	char		msgline[4096];
	char		msgtext[4096];
	char		causetext[1024];
	char		*svcname;

	svcname = strdup(service->testname);
	if (service->namelen) svcname[service->namelen] = '\0';

	dbgprintf("Sending results for service %s\n", svcname);

	for (t=service->items; (t); t = t->next) {
		char flags[10];
		int i;

		i = 0;
		flags[i++] = (t->open ? 'O' : 'o');
		flags[i++] = (t->reverse ? 'R' : 'r');
		flags[i++] = ((t->dialup || t->host->dialup) ? 'D' : 'd');
		flags[i++] = (t->alwaystrue ? 'A' : 'a');
		flags[i++] = (t->silenttest ? 'S' : 's');
		flags[i++] = (t->host->testip ? 'T' : 't');
		flags[i++] = (t->host->dodns ? 'L' : 'l');
		flags[i++] = (t->host->dnserror ? 'E' : 'e');
		flags[i++] = '\0';

		color = decide_color(service, svcname, t, failgoesclear, causetext);

		init_status(color);
		if (dosendflags) 
			sprintf(msgline, "status %s.%s %s <!-- [flags:%s] --> %s %s %s ", 
				commafy(t->host->hostname), svcname, colorname(color), 
				flags, timestamp, 
				svcname, ( ((color == COL_RED) || (color == COL_YELLOW)) ? "NOT ok" : "ok"));
		else
			sprintf(msgline, "status %s.%s %s %s %s %s ", 
				commafy(t->host->hostname), svcname, colorname(color), 
				timestamp, 
				svcname, ( ((color == COL_RED) || (color == COL_YELLOW)) ? "NOT ok" : "ok"));

		if (t->host->dnserror) {
			strcat(msgline, ": DNS lookup failed");
			sprintf(msgtext, "\nUnable to resolve hostname %s\n\n", t->host->hostname);
		}
		else {
			sprintf(msgtext, "\nService %s on %s is ", svcname, t->host->hostname);
			switch (color) {
			  case COL_GREEN: 
				  strcat(msgtext, "OK ");
				  strcat(msgtext, (t->reverse ? "(down)" : "(up)"));
				  strcat(msgtext, "\n");
				  break;

			  case COL_RED:
			  case COL_YELLOW:
				  if ((service == pingtest) && t->host->deprouterdown) {
					char *routertext;

					routertext = t->host->deprouterdown->hosttype;
					if (routertext == NULL) routertext = xgetenv("BBROUTERTEXT");
					if (routertext == NULL) routertext = "router";

					strcat(msgline, ": Intermediate ");
					strcat(msgline, routertext);
					strcat(msgline, " down");

					sprintf(msgtext+strlen(msgtext), 
						"%s.\nThe %s %s (IP:%s) is not reachable, causing this host to be unreachable.\n",
						failtext, routertext, 
						((testedhost_t *)t->host->deprouterdown)->hostname,
						((testedhost_t *)t->host->deprouterdown)->ip);
				  }
				  else {
					sprintf(msgtext+strlen(msgtext), "%s : %s\n", failtext, causetext);
				  }
				  break;

			  case COL_CLEAR:
				  strcat(msgtext, "OK\n");
				  if (service == pingtest) {
					  if (t->host->deprouterdown) {
						char *routertext;

						routertext = t->host->deprouterdown->hosttype;
						if (routertext == NULL) routertext = xgetenv("BBROUTERTEXT");
						if (routertext == NULL) routertext = "router";

						strcat(msgline, ": Intermediate ");
						strcat(msgline, routertext);
						strcat(msgline, " down");

						strcat(msgtext, "\nThe ");
						strcat(msgtext, routertext); strcat(msgtext, " ");
						strcat(msgtext, ((testedhost_t *)t->host->deprouterdown)->hostname);
						strcat(msgtext, " (IP:");
						strcat(msgtext, ((testedhost_t *)t->host->deprouterdown)->ip);
						strcat(msgtext, ") is not reachable, causing this host to be unreachable.\n");
					  }
					  else if (t->host->noping) {
						  strcat(msgline, ": Disabled");
						  strcat(msgtext, "Ping check disabled (noping)\n");
					  }
					  else if (t->host->dialup) {
						  strcat(msgline, ": Disabled (dialup host)");
						  strcat(msgtext, "Dialup host\n");
					  }
					  else if (t->open == -1) {
						  strcat(msgline, ": System failure of the ping test");
						  strcat(msgtext, "Hobbit system error\n");
					  }
					  /* "clear" due to badconn: no extra text */
				  }
				  else {
					  /* Non-ping test clear: Dialup test or failed ping */
					  strcat(msgline, ": Ping failed, or dialup host/service");
					  strcat(msgtext, "Dialup host/service, or test depends on another failed test\n");
					  strcat(msgtext, causetext);
				  }
				  break;
			}
			strcat(msgtext, "\n");
		}
		strcat(msgline, "\n");
		addtostatus(msgline);
		addtostatus(msgtext);

		if ((service == pingtest) && t->host->downcount) {
			sprintf(msgtext, "\nSystem unreachable for %d poll periods (%u seconds)\n",
				t->host->downcount, (unsigned int)(time(NULL) - t->host->downstart));
			addtostatus(msgtext);
		}

		if (STRBUFLEN(t->banner)) {
			if (service == pingtest) {
				sprintf(msgtext, "\n&%s %s\n", colorname(t->open ? COL_GREEN : COL_RED), STRBUF(t->banner));
				addtostatus(msgtext);
				if (t->host->extrapings) {
					ipping_t *walk;
					for (walk = t->host->extrapings->iplist; (walk); walk = walk->next) {
						if (STRBUFLEN(walk->banner)) {
							sprintf(msgtext, "&%s %s\n", 
								colorname(walk->open ? COL_GREEN : COL_RED), STRBUF(walk->banner));
							addtostatus(msgtext);
						}
					}
				}
			}
			else {
				addtostatus("\n"); addtostrstatus(t->banner); addtostatus("\n");
			}
		}

		if ((service == pingtest) && t->host->traceroute && (STRBUFLEN(t->host->traceroute) > 0)) {
			addtostatus("Traceroute results:\n");
			addtostrstatus(t->host->traceroute);
			addtostatus("\n");
		}

		if (t->duration.tv_sec != -1) {
			sprintf(msgtext, "\nSeconds: %u.%02u\n", 
				(unsigned int)t->duration.tv_sec, (unsigned int)t->duration.tv_usec / 10000);
			addtostatus(msgtext);
		}
		addtostatus("\n\n");
		finish_status();
	}
}


void send_modembank_results(service_t *service)
{
	testitem_t	*t;
	char		msgline[1024];
	int		i, color, inuse;
	char		startip[IP_ADDR_STRLEN], endip[IP_ADDR_STRLEN];

	for (t=service->items; (t); t = t->next) {
		modembank_t *req = (modembank_t *)t->privdata;

		inuse = 0;
		strcpy(startip, u32toIP(req->startip));
		strcpy(endip, u32toIP(req->startip + req->banksize - 1));

		init_status(COL_GREEN);		/* Modembanks are always green */
		sprintf(msgline, "status dialup.%s %s %s FROM %s TO %s DATA ", 
			req->hostname, colorname(COL_GREEN), timestamp, startip, endip);
		addtostatus(msgline);
		for (i=0; i<req->banksize; i++) {
			if (req->responses[i]) {
				color = COL_GREEN;
				inuse++;
			}
			else {
				color = COL_CLEAR;
			}

			sprintf(msgline, "%s ", colorname(color));
			addtostatus(msgline);
		}

		sprintf(msgline, "\n\nUsage: %d of %d (%d%%)\n", inuse, req->banksize, ((inuse * 100) / req->banksize));
		addtostatus(msgline);
		finish_status();
	}
}


void send_rpcinfo_results(service_t *service, int failgoesclear)
{
	testitem_t	*t;
	int		color;
	char		msgline[1024];
	char		*msgbuf;
	char		causetext[1024];

	msgbuf = (char *)malloc(4096);

	for (t=service->items; (t); t = t->next) {
		char *wantedrpcsvcs = NULL;
		char *p;

		/* First see if the rpcinfo command succeeded */
		*msgbuf = '\0';

		color = decide_color(service, service->testname, t, failgoesclear, causetext);
		p = strchr(t->testspec, '=');
		if (p) wantedrpcsvcs = strdup(p+1);

		if ((color == COL_GREEN) && STRBUFLEN(t->banner) && wantedrpcsvcs) {
			char *rpcsvc, *aline;

			rpcsvc = strtok(wantedrpcsvcs, ",");
			while (rpcsvc) {
				struct rpcent *rpcinfo;
				int  svcfound = 0;
				int  aprogram;
				int  aversion;
				char aprotocol[10];
				int  aport;

				rpcinfo = getrpcbyname(rpcsvc);
				aline = STRBUF(t->banner); 
				while ((!svcfound) && rpcinfo && aline && (*aline != '\0')) {
					p = strchr(aline, '\n');
					if (p) *p = '\0';

					if (sscanf(aline, "%d %d %s %d", &aprogram, &aversion, aprotocol, &aport) == 4) {
						svcfound = (aprogram == rpcinfo->r_number);
					}

					aline = p;
					if (p) {
						*p = '\n';
						aline++;
					}
				}

				if (svcfound) {
					sprintf(msgline, "&%s Service %s (ID: %d) found on port %d\n", 
						colorname(COL_GREEN), rpcsvc, rpcinfo->r_number, aport);
				}
				else if (rpcinfo) {
					color = COL_RED;
					sprintf(msgline, "&%s Service %s (ID: %d) NOT found\n", 
						colorname(COL_RED), rpcsvc, rpcinfo->r_number);
				}
				else {
					color = COL_RED;
					sprintf(msgline, "&%s Unknown RPC service %s\n",
						colorname(COL_RED), rpcsvc);
				}
				strcat(msgbuf, msgline);

				rpcsvc = strtok(NULL, ",");
			}
		}

		if (wantedrpcsvcs) xfree(wantedrpcsvcs);

		init_status(color);
		sprintf(msgline, "status %s.%s %s %s %s %s, %s\n\n", 
			commafy(t->host->hostname), service->testname, colorname(color), timestamp, 
			service->testname, 
			( ((color == COL_RED) || (color == COL_YELLOW)) ? "NOT ok" : "ok"),
			causetext);
		addtostatus(msgline);

		/* The summary of wanted RPC services */
		addtostatus(msgbuf);

		/* rpcinfo output */
		if (t->open) {
			if (STRBUFLEN(t->banner)) {
				addtostatus("\n\n");
				addtostrstatus(t->banner);
			}
			else {
				sprintf(msgline, "\n\nNo output from rpcinfo -p %s\n", t->host->ip);
				addtostatus(msgline);
			}
		}
		else {
			addtostatus("\n\nCould not connect to the portmapper service\n");
			if (STRBUFLEN(t->banner)) addtostrstatus(t->banner);
		}
		finish_status();
	}

	xfree(msgbuf);
}


void send_sslcert_status(testedhost_t *host)
{
	int color = -1;
	RbtIterator handle;
	service_t *s;
	testitem_t *t;
	char msgline[1024];
	char *sslmsg;
	int sslmsgsize;
	time_t now = time(NULL);
	char *certowner;

	sslmsgsize = 4096;
	sslmsg = (char *)malloc(sslmsgsize);
	*sslmsg = '\0';

	for (handle = rbtBegin(svctree); handle != rbtEnd(svctree); handle = rbtNext(svctree, handle)) {
		s = (service_t *)gettreeitem(svctree, handle);
		certowner = s->testname;

		for (t=s->items; (t); t=t->next) {
			if ((t->host == host) && t->certinfo && (t->certexpires > 0)) {
				int sslcolor = COL_GREEN;

				if (s == httptest) certowner = ((http_data_t *)t->privdata)->url;
				else if (s == ldaptest) certowner = t->testspec;

				if (t->certexpires < (now+host->sslwarndays*86400)) sslcolor = COL_YELLOW;
				if (t->certexpires < (now+host->sslalarmdays*86400)) sslcolor = COL_RED;
				if (sslcolor > color) color = sslcolor;

				if (t->certexpires > now) {
					sprintf(msgline, "\n&%s SSL certificate for %s expires in %u days\n\n", 
						colorname(sslcolor), certowner,
						(unsigned int)((t->certexpires - now) / 86400));
				}
				else {
					sprintf(msgline, "\n&%s SSL certificate for %s expired %u days ago\n\n", 
						colorname(sslcolor), certowner,
						(unsigned int)((now - t->certexpires) / 86400));
				}

				if ((strlen(msgline)+strlen(sslmsg) + strlen(t->certinfo)) > sslmsgsize) {
					sslmsgsize += (4096 + strlen(t->certinfo) + strlen(msgline));
					sslmsg = (char *)realloc(sslmsg, sslmsgsize);
				}
				strcat(sslmsg, msgline);
				strcat(sslmsg, t->certinfo);
			}
		}
	}

	if (color != -1) {
		/* Send off the sslcert status report */
		init_status(color);
		sprintf(msgline, "status %s.%s %s %s\n", 
			commafy(host->hostname), ssltestname, colorname(color), timestamp);
		addtostatus(msgline);
		addtostatus(sslmsg);
		addtostatus("\n\n");
		finish_status();
	}

	xfree(sslmsg);
}

int main(int argc, char *argv[])
{
	RbtIterator handle;
	service_t *s;
	testedhost_t *h;
	testitem_t *t;
	int argi;
	int concurrency = 0;
	char *pingcolumn = "";
	char *egocolumn = NULL;
	int failgoesclear = 0;		/* IPTEST_2_CLEAR_ON_FAILED_CONN */
	int dumpdata = 0;
	int runtimewarn;		/* 300 = default BBSLEEP setting */
	int servicedumponly = 0;
	int pingrunning = 0;

	if (init_ldap_library() != 0) {
		errprintf("Failed to initialize ldap library\n");
		return 1;
	}

	if (xgetenv("CONNTEST") && (strcmp(xgetenv("CONNTEST"), "FALSE") == 0)) pingcolumn = NULL;
	runtimewarn = (xgetenv("BBSLEEP") ? atol(xgetenv("BBSLEEP")) : 300);

	for (argi=1; (argi < argc); argi++) {
		if      (argnmatch(argv[argi], "--timeout=")) {
			char *p = strchr(argv[argi], '=');
			p++; timeout = atoi(p);
		}
		else if (argnmatch(argv[argi], "--conntimeout=")) {
			int newtimeout;
			char *p = strchr(argv[argi], '=');
			p++; newtimeout = atoi(p);
			if (newtimeout > timeout) timeout = newtimeout;
			errprintf("Deprecated option '--conntimeout' should not be used\n");
		}
		else if (argnmatch(argv[argi], "--cmdtimeout=")) {
			char *p = strchr(argv[argi], '=');
			p++; extcmdtimeout = atoi(p);
		}
		else if (argnmatch(argv[argi], "--concurrency=")) {
			char *p = strchr(argv[argi], '=');
			p++; concurrency = atoi(p);
		}
		else if (argnmatch(argv[argi], "--dns-timeout=") || argnmatch(argv[argi], "--dns-max-all=")) {
			char *p = strchr(argv[argi], '=');
			p++; dnstimeout = atoi(p);
		}
		else if (argnmatch(argv[argi], "--dns=")) {
			char *p = strchr(argv[argi], '=');
			p++;
			if (strcmp(p, "only") == 0)      dnsmethod = DNS_ONLY;
			else if (strcmp(p, "ip") == 0)   dnsmethod = IP_ONLY;
			else                             dnsmethod = DNS_THEN_IP;
		}
		else if (strcmp(argv[argi], "--no-ares") == 0) {
			use_ares_lookup = 0;
		}
		else if (argnmatch(argv[argi], "--maxdnsqueue=")) {
			char *p = strchr(argv[argi], '=');
			max_dns_per_run = atoi(p+1);
		}
		else if (argnmatch(argv[argi], "--dnslog=")) {
			char *fn = strchr(argv[argi], '=');
			dnsfaillog = fopen(fn+1, "w");
		}
		else if (argnmatch(argv[argi], "--report=") || (strcmp(argv[argi], "--report") == 0)) {
			char *p = strchr(argv[argi], '=');
			if (p) {
				egocolumn = strdup(p+1);
			}
			else egocolumn = "bbtest";
			timing = 1;
		}
		else if (strcmp(argv[argi], "--test-untagged") == 0) {
			testuntagged = 1;
		}
		else if (argnmatch(argv[argi], "--frequenttestlimit=")) {
			char *p = strchr(argv[argi], '=');
			p++; frequenttestlimit = atoi(p);
		}
		else if (strcmp(argv[argi], "--timelimit=") == 0) {
			char *p = strchr(argv[argi], '=');
			p++; runtimewarn = atol(p);
		}
		else if (strcmp(argv[argi], "--huge=") == 0) {
			char *p = strchr(argv[argi], '=');
			p++; warnbytesread = atoi(p);
		}

		/* Options for TCP tests */
		else if (strcmp(argv[argi], "--checkresponse") == 0) {
			checktcpresponse = 1;
		}
		else if (argnmatch(argv[argi], "--checkresponse=")) {
			char *p = strchr(argv[argi], '=');
			checktcpresponse = 1;
			respcheck_color = parse_color(p+1);
			if (respcheck_color == -1) {
				errprintf("Invalid colorname in '%s' - using yellow\n", argv[argi]);
				respcheck_color = COL_YELLOW;
			}
		}
		else if (strcmp(argv[argi], "--no-flags") == 0) {
			dosendflags = 0;
		}

		/* Options for PING tests */
		else if (argnmatch(argv[argi], "--ping")) {
			char *p = strchr(argv[argi], '=');
			if (p) {
				p++; pingcolumn = p;
			}
			else pingcolumn = "";
		}
		else if (strcmp(argv[argi], "--noping") == 0) {
			pingcolumn = NULL;
		}
		else if (strcmp(argv[argi], "--trace") == 0) {
			dotraceroute = 1;
		}
		else if (strcmp(argv[argi], "--notrace") == 0) {
			dotraceroute = 0;
		}

		/* Options for HTTP tests */
		else if (argnmatch(argv[argi], "--content=")) {
			char *p = strchr(argv[argi], '=');
			contenttestname = strdup(p+1);
		}

		/* Options for SSL certificates */
		else if (argnmatch(argv[argi], "--ssl=")) {
			char *p = strchr(argv[argi], '=');
			ssltestname = strdup(p+1);
		}
		else if (strcmp(argv[argi], "--no-ssl") == 0) {
			ssltestname = NULL;
		}
		else if (argnmatch(argv[argi], "--sslwarn=")) {
			char *p = strchr(argv[argi], '=');
			p++; sslwarndays = atoi(p);
		}
		else if (argnmatch(argv[argi], "--sslalarm=")) {
			char *p = strchr(argv[argi], '=');
			p++; sslalarmdays = atoi(p);
		}

		/* Debugging options */
		else if (strcmp(argv[argi], "--debug") == 0) {
			debug = 1;
		}
		else if (argnmatch(argv[argi], "--dump")) {
			char *p = strchr(argv[argi], '=');

			if (p) {
				if (strcmp(p, "=before") == 0) dumpdata = 1;
				else if (strcmp(p, "=after") == 0) dumpdata = 2;
				else dumpdata = 3;
			}
			else dumpdata = 2;

			debug = 1;
		}
		else if (strcmp(argv[argi], "--no-update") == 0) {
			dontsendmessages = 1;
		}
		else if (strcmp(argv[argi], "--timing") == 0) {
			timing = 1;
		}

		/* Informational options */
		else if (strcmp(argv[argi], "--services") == 0) {
			servicedumponly = 1;
		}
		else if (strcmp(argv[argi], "--version") == 0) {
			printf("bbtest-net version %s\n", VERSION);
			if (ssl_library_version) printf("SSL library : %s\n", ssl_library_version);
			if (ldap_library_version) printf("LDAP library: %s\n", ldap_library_version);
			printf("\n");
			return 0;
		}
		else if ((strcmp(argv[argi], "--help") == 0) || (strcmp(argv[argi], "-?") == 0)) {
			printf("bbtest-net version %s\n\n", VERSION);
			printf("Usage: %s [options] [host1 host2 host3 ...]\n", argv[0]);
			printf("General options:\n");
			printf("    --timeout=N                 : Timeout (in seconds) for service tests\n");
			printf("    --concurrency=N             : Number of tests run in parallel\n");
			printf("    --dns-timeout=N             : DNS lookups timeout and fail after N seconds [30]\n");
			printf("    --dns=[only|ip|standard]    : How IP's are decided\n");
			printf("    --no-ares                   : Use the system resolver library for hostname lookups\n");
			printf("    --dnslog=FILENAME           : Log failed hostname lookups to file FILENAME\n");
			printf("    --report[=COLUMNNAME]       : Send a status report about the running of bbtest-net\n");
			printf("    --test-untagged             : Include hosts without a NET: tag in the test\n");
			printf("    --frequenttestlimit=N       : Seconds after detecting failures in which we poll frequently\n");
			printf("    --timelimit=N               : Warns if the complete test run takes longer than N seconds [BBSLEEP]\n");
			printf("\nOptions for simple TCP service tests:\n");
			printf("    --checkresponse             : Check response from known services\n");
			printf("    --no-flags                  : Dont send extra bbgen test flags\n");
			printf("\nOptions for PING (connectivity) tests:\n");
			printf("    --ping[=COLUMNNAME]         : Enable ping checking, default columname is \"conn\"\n");
			printf("    --noping                    : Disable ping checking\n");
			printf("    --trace                     : Run traceroute on all hosts where ping fails\n");
			printf("    --notrace                   : Disable traceroute when ping fails (default)\n");
			printf("\nOptions for HTTP/HTTPS (Web) tests:\n");
			printf("    --content=COLUMNNAME        : Define default columnname for CONTENT checks (content)\n");
			printf("\nOptions for SSL certificate tests:\n");
			printf("    --ssl=COLUMNNAME            : Define columnname for SSL certificate checks (sslcert)\n");
			printf("    --no-ssl                    : Disable SSL certificate check\n");
			printf("    --sslwarn=N                 : Go yellow if certificate expires in less than N days (default:30)\n");
			printf("    --sslalarm=N                : Go red if certificate expires in less than N days (default:10)\n");
			printf("\nDebugging options:\n");
			printf("    --no-update                 : Send status messages to stdout instead of to bbd\n");
			printf("    --timing                    : Trace the amount of time spent on each series of tests\n");
			printf("    --debug                     : Output debugging information\n");
			printf("    --dump[=before|=after|=all] : Dump internal memory structures before/after tests run\n");
			printf("    --maxdnsqueue=N             : Only queue N DNS lookups at a time\n");
			printf("\nInformational options:\n");
			printf("    --services                  : Dump list of known services and exit\n");
			printf("    --version                   : Show program version and exit\n");
			printf("    --help                      : Show help text and exit\n");

			return 0;
		}
		else if (strncmp(argv[argi], "-", 1) == 0) {
			errprintf("Unknown option %s - try --help\n", argv[argi]);
		}
		else {
			/* Must be a hostname */
			if (selectedcount == 0) selectedhosts = (char **) malloc(argc*sizeof(char *));
			selectedhosts[selectedcount++] = strdup(argv[argi]);
		}
	}

	svctree = rbtNew(name_compare);
	testhosttree = rbtNew(name_compare);
	init_timestamp();
	envcheck(reqenv);
	fqdn = get_fqdn();

	/* Setup SEGV handler */
	setup_signalhandler(egocolumn ? egocolumn : "bbtest");

	if (xgetenv("BBLOCATION")) location = strdup(xgetenv("BBLOCATION"));
	if (pingcolumn && (strlen(pingcolumn) == 0)) pingcolumn = xgetenv("PINGCOLUMN");
	if (pingcolumn && xgetenv("IPTEST_2_CLEAR_ON_FAILED_CONN")) {
		failgoesclear = (strcmp(xgetenv("IPTEST_2_CLEAR_ON_FAILED_CONN"), "TRUE") == 0);
	}
	if (xgetenv("NETFAILTEXT")) failtext = strdup(xgetenv("NETFAILTEXT"));

	if (debug) {
		int i;
		printf("Command: bbtest-net");
		for (i=1; (i<argc); i++) printf(" '%s'", argv[i]);
		printf("\n");
		printf("Environment BBLOCATION='%s'\n", textornull(xgetenv("BBLOCATION")));
		printf("Environment CONNTEST='%s'\n", textornull(xgetenv("CONNTEST")));
		printf("Environment IPTEST_2_CLEAR_ON_FAILED_CONN='%s'\n", textornull(xgetenv("IPTEST_2_CLEAR_ON_FAILED_CONN")));
		printf("\n");
	}

	add_timestamp("bbtest-net startup");

	load_services();
	if (servicedumponly) {
		dump_tcp_services();
		return 0;
	}

	dnstest = add_service("dns", getportnumber("domain"), 0, TOOL_NSLOOKUP);
	digtest = add_service("dig", getportnumber("domain"), 0, TOOL_DIG);
	add_service("ntp", getportnumber("ntp"),    0, TOOL_NTP);
	rpctest  = add_service("rpc", getportnumber("sunrpc"), 0, TOOL_RPCINFO);
	httptest = add_service("http", getportnumber("http"),  0, TOOL_HTTP);
	ldaptest = add_service("ldapurl", getportnumber("ldap"), strlen("ldap"), TOOL_LDAP);
	if (pingcolumn) pingtest = add_service(pingcolumn, 0, 0, TOOL_FPING);
	modembanktest = add_service("dialup", 0, 0, TOOL_MODEMBANK);
	add_timestamp("Service definitions loaded");

	load_tests();
	add_timestamp(use_ares_lookup ? "Tests loaded" : "Tests loaded, hostname lookups done");

	flush_dnsqueue();
	if (use_ares_lookup) add_timestamp("DNS lookups completed");

	if (dumpdata & 1) { dump_hostlist(); dump_testitems(); }

	/* Ping checks first */
	if (pingtest && pingtest->items) pingrunning = (start_ping_service(pingtest) == 0);

	/* Load current status files */
	for (handle = rbtBegin(svctree); handle != rbtEnd(svctree); handle = rbtNext(svctree, handle)) {
		s = (service_t *)gettreeitem(svctree, handle);
		if (s != pingtest) load_test_status(s);
	}

	/* First run the TCP/IP and HTTP tests */
	for (handle = rbtBegin(svctree); handle != rbtEnd(svctree); handle = rbtNext(svctree, handle)) {
		s = (service_t *)gettreeitem(svctree, handle);
		if ((s->items) && (s->toolid == TOOL_CONTEST)) {
			char tname[128];

			for (t = s->items; (t); t = t->next) {
				if (!t->host->dnserror) {
					strcpy(tname, s->testname);
					if (s->namelen) tname[s->namelen] = '\0';
					t->privdata = (void *)add_tcp_test(ip_to_test(t->host), s->portnum, tname, NULL, 
									   NULL, t->silenttest, NULL, 
									   NULL, NULL, NULL);
				}
			}
		}
	}
	for (t = httptest->items; (t); t = t->next) add_http_test(t);
	add_timestamp("Test engine setup completed");

	do_tcp_tests(timeout, concurrency);
	add_timestamp("TCP tests completed");

	if (pingrunning) {
		char msg[512];

		finish_ping_service(pingtest); 
		sprintf(msg, "PING test completed (%d hosts)", pingcount);
		add_timestamp(msg);
		combo_start();
		send_results(pingtest, failgoesclear);
		if (selectedhosts == 0) save_ping_status();
		combo_end();
		add_timestamp("PING test results sent");
	}

	if (debug) {
		show_tcp_test_results();
		show_http_test_results(httptest);
	}

	for (handle = rbtBegin(svctree); handle != rbtEnd(svctree); handle = rbtNext(svctree, handle)) {
		s = (service_t *)gettreeitem(svctree, handle);
		if ((s->items) && (s->toolid == TOOL_CONTEST)) {
			for (t = s->items; (t); t = t->next) { 
				/*
				 * If the test fails due to DNS error, t->privdata is NULL
				 */
				if (t->privdata) {
					char *p;
					int i;
					tcptest_t *testresult = (tcptest_t *)t->privdata;

					t->open = testresult->open;
					t->banner = dupstrbuffer(testresult->banner);
					t->certinfo = testresult->certinfo;
					t->certexpires = testresult->certexpires;
					t->duration.tv_sec = testresult->duration.tv_sec;
					t->duration.tv_usec = testresult->duration.tv_usec;

					/* Binary data in banner ... */
					for (i=0, p=STRBUF(t->banner); (i < STRBUFLEN(t->banner)); i++, p++) {
						if (!isprint((int)*p) && !isspace((int)*p)) *p = '.';
					}
				}
			}
		}
	}
	for (t = httptest->items; (t); t = t->next) {
		if (t->privdata) {
			http_data_t *testresult = (http_data_t *)t->privdata;

			t->certinfo = testresult->tcptest->certinfo;
			t->certexpires = testresult->tcptest->certexpires;
		}
	}

	add_timestamp("Test result collection completed");


	/* Run the ldap tests */
	for (t = ldaptest->items; (t); t = t->next) add_ldap_test(t);
	add_timestamp("LDAP test engine setup completed");

	run_ldap_tests(ldaptest, (ssltestname != NULL), timeout);
	add_timestamp("LDAP tests executed");

	if (debug) show_ldap_test_results(ldaptest);
	for (t = ldaptest->items; (t); t = t->next) {
		if (t->privdata) {
			ldap_data_t *testresult = (ldap_data_t *)t->privdata;

			t->certinfo = testresult->certinfo;
			t->certexpires = testresult->certexpires;
		}
	}
	add_timestamp("LDAP tests result collection completed");


	/* dns, dig, ntp tests */
	for (handle = rbtBegin(svctree); handle != rbtEnd(svctree); handle = rbtNext(svctree, handle)) {
		s = (service_t *)gettreeitem(svctree, handle);
		if (s->items) {
			switch(s->toolid) {
				case TOOL_NSLOOKUP:
					run_nslookup_service(s); 
					add_timestamp("NSLOOKUP tests executed");
					break;
				case TOOL_DIG:
					run_nslookup_service(s); 
					add_timestamp("DIG tests executed");
					break;
				case TOOL_NTP:
					run_ntp_service(s); 
					add_timestamp("NTP tests executed");
					break;
				case TOOL_RPCINFO:
					run_rpcinfo_service(s); 
					add_timestamp("RPC tests executed");
					break;
				case TOOL_MODEMBANK:
					run_modembank_service(s); 
					add_timestamp("Modembank tests executed");
					break;
			}
		}
	}

	combo_start();
	for (handle = rbtBegin(svctree); handle != rbtEnd(svctree); handle = rbtNext(svctree, handle)) {
		s = (service_t *)gettreeitem(svctree, handle);
		switch (s->toolid) {
			case TOOL_CONTEST:
			case TOOL_NSLOOKUP:
			case TOOL_DIG:
			case TOOL_NTP:
				send_results(s, failgoesclear);
				break;

			case TOOL_FPING:
			case TOOL_HTTP:
			case TOOL_LDAP:
				/* These handle result-transmission internally */
				break;

			case TOOL_MODEMBANK:
				send_modembank_results(s);
				break;

			case TOOL_RPCINFO:
				send_rpcinfo_results(s, failgoesclear);
				break;
		}
	}
	for (handle = rbtBegin(testhosttree); (handle != rbtEnd(testhosttree)); handle = rbtNext(testhosttree, handle)) {
		h = (testedhost_t *)gettreeitem(testhosttree, handle);
		send_http_results(httptest, h, h->firsthttp, nonetpage, failgoesclear);
		send_content_results(httptest, h, nonetpage, contenttestname, failgoesclear);
		send_ldap_results(ldaptest, h, nonetpage, failgoesclear);
		if (ssltestname && !h->nosslcert) send_sslcert_status(h);
	}

	combo_end();
	add_timestamp("Test results transmitted");

	/*
	 * The list of hosts to test frequently because of a failure must
	 * be saved - it is then picked up by the frequent-test ext script
	 * that runs bbtest-net again with the frequent-test hosts as
	 * parameter.
	 *
	 * Should the retest itself update the frequent-test file ? It
	 * would allow us to kick hosts from the frequent-test file sooner.
	 * However, it is simpler (no races) if we just let the normal
	 * test-engine be alone in updating the file. 
	 * At the worst, we'll re-test a host going up a couple of times
	 * too much.
	 *
	 * So for now update the list only if we ran with no host-parameters.
	 */
	if (selectedcount == 0) {
		/* Save current status files */
		for (handle = rbtBegin(svctree); handle != rbtEnd(svctree); handle = rbtNext(svctree, handle)) {
			s = (service_t *)gettreeitem(svctree, handle);
			if (s != pingtest) save_test_status(s);
		}
		/* Save frequent-test list */
		save_frequenttestlist(argc, argv);
	}

	shutdown_ldap_library();
	add_timestamp("bbtest-net completed");

	if (dumpdata & 2) { dump_hostlist(); dump_testitems(); }

	/* Tell about us */
	if (egocolumn) {
		char msgline[4096];
		char *timestamps;
		int color;

		/* Go yellow if it runs for too long */
		if (total_runtime() > runtimewarn) {
			errprintf("WARNING: Runtime %ld longer than time limit (%ld)\n", total_runtime(), runtimewarn);
		}
		color = (errbuf ? COL_YELLOW : COL_GREEN);
		if (bigfailure) color = COL_RED;

		combo_start();
		init_status(color);
		sprintf(msgline, "status %s.%s %s %s\n\n", xgetenv("MACHINE"), egocolumn, colorname(color), timestamp);
		addtostatus(msgline);

		sprintf(msgline, "bbtest-net version %s\n", VERSION);
		addtostatus(msgline);
		if (ssl_library_version) {
			sprintf(msgline, "SSL library : %s\n", ssl_library_version);
			addtostatus(msgline);
		}
		if (ldap_library_version) {
			sprintf(msgline, "LDAP library: %s\n", ldap_library_version);
			addtostatus(msgline);
		}

		sprintf(msgline, "\nStatistics:\n Hosts total           : %8d\n Hosts with no tests   : %8d\n Total test count      : %8d\n Status messages       : %8d\n Alert status msgs     : %8d\n Transmissions         : %8d\n", 
			hostcount, notesthostcount, testcount, bbstatuscount, bbnocombocount, bbmsgcount);
		addtostatus(msgline);
		sprintf(msgline, "\nDNS statistics:\n # hostnames resolved  : %8d\n # succesful           : %8d\n # failed              : %8d\n # calls to dnsresolve : %8d\n",
			dns_stats_total, dns_stats_success, dns_stats_failed, dns_stats_lookups);
		addtostatus(msgline);
		sprintf(msgline, "\nTCP test statistics:\n # TCP tests total     : %8d\n # HTTP tests          : %8d\n # Simple TCP tests    : %8d\n # Connection attempts : %8d\n # bytes written       : %8ld\n # bytes read          : %8ld\n",
			tcp_stats_total, tcp_stats_http, tcp_stats_plain, tcp_stats_connects, 
			tcp_stats_written, tcp_stats_read);
		addtostatus(msgline);

		if (errbuf) {
			addtostatus("\n\nError output:\n");
			addtostatus(errbuf);
		}

		show_timestamps(&timestamps);
		addtostatus(timestamps);

		finish_status();
		combo_end();
	}
	else show_timestamps(NULL);

	if (dnsfaillog) fclose(dnsfaillog);

	return 0;
}

