#!/usr/bin/perl -w -I../../lib/
#
# $Id: db-wrapnet.pl,v 1.13 2002/03/05 23:43:22 fpmip Exp $
#
# Script to get XML data from a dr-server and sort it in the right directories
# for the db-backend-daemon to find and process.
#
# Author : Tycho Fruru <tycho@fruru.com>
#
# Copyright (C) 2001 Tycho Fruru <tycho@fruru.com>
# Copyright (C) 2001 Conostix S.A. Luxembourg - Tycho Fruru <tycho@conostix.com>
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.  See <http://www.fsf.org/copyleft/gpl.txt>.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
# Goal : fetcher la vache

use DBI;
use XML::Simple;
use Data::Dumper;
use LWP;

use IPFC::Log;

require "../../lib/ipfcsign-xmlconfig.pm";
require "../../lib/ipfcsign.pm";

if (defined($ARGV[0])) {
	$DB_WRAPNET_CONFIG_FILE = $ARGV[0];
} else {
	$DB_WRAPNET_CONFIG_FILE = "./db-wrapnet-config.xml";
}

if (defined($ARGV[1])) {
	$DB_WRAPNET_KEYS_FILE = $ARGV[1];
} else {
	$DB_WRAPNET_KEYS_FILE = "./db-wrapnet-keys.xml";
}

while (1) {
# read our configuration file
	$config_file = XMLin($DB_WRAPNET_CONFIG_FILE,
			     forcearray=>1,keyattr=>[],contentkey=>[]);
#	print Dumper($config_file);

	IPFC::Log::set_verbosity($config_file->{'verbosity'}->[0]);
		
	IPFC::Sign::set_config(IPFC::Sign::Xmlconfig::read_config($DB_WRAPNET_KEYS_FILE));	
	
# %url_mapping will enable us to easily find the right entry in the config file for
# a certain freeform url identification (see agent_url_id in table agent_url_ids)

	my $i = 0;
	
	while (defined($config_file->{'url-set'}->[$i])) {
		$url_mapping{$config_file->{'url-set'}->[$i]->{'id'}->[0]} = $i;
		$i++;
	}
	
	$dbh = DBI->connect($config_file->{'db-connect-string'}->[0],
			    $config_file->{'db-username'}->[0],
			    $config_file->{'db-password'}->[0]) or
		die "Database connection failed";
	$sth = $dbh->prepare(
			"select distinct(agent_url_id) from agent_url_ids") or
		die "Database query failed";
	$sth->execute() or die "Database query execute failed";
	while (($agent_url_id) = $sth->fetchrow_array) {
		push @agent_url_ids,$agent_url_id;
	}
	$sth->finish;

	$sth = $dbh->prepare("select agent_id from agent_url_ids where agent_url_id=?") or
		die "Database query failed";
	while ($agent_url_id = pop @agent_url_ids) {
		$sth->execute($agent_url_id) or die "Database query execute failed";
		while (($agent_id) = $sth->fetchrow_array) {
			$agent_config_location{$agent_id}=$url_mapping{$agent_url_id};
		}
	}
# $agent_config_location{$agent_id} gives the index in url-set where definitions for this
# agent are to be found

	$sth->finish;
	$dbh->disconnect;
	
#	print $agent_config_location{2};
	
	my $url_set_index = -1;
	my $url_seen = 0;
        my $ua = new LWP::UserAgent( keep_alive => 1 );
	$ua->agent("db-wrapnet");

	while (defined($config_file->{'url-set'}->[++$url_set_index])) {
		my $url_set = $config_file->{'url-set'}->[$url_set_index];
		IPFC::Log::logger(2,"Starting on $url_set->{'id'}->[0]");

		my $authentication = $url_set->{'authentication'}->[0];
		my $user;
		my $pass;
		
		if (defined($authentication)) {
			if ($authentication eq "basic") {
				$user = $url_set->{'username'}->[0];
				$pass = $url_set->{'password'}->[0];
			} else {
				die "Unknown authentication method $authentication !";
			}
		} else {
			die "No authentication is not very secure ...";
		}

		my $ca_certificate_file = $url_set->{'ca-certificate-file'}->[0];
		if (defined($ca_certificate_file)) {
			$ENV{'HTTPS_CA_FILE'}=$ca_certificate_file;
		}
		
		my $client_certificate_file = $url_set->{'client-certificate-file'}->[0];
		if (defined($client_certificate_file)) {
			$ENV{'HTTPS_CERT_FILE'}=$client_certificate_file;
		}
		
		my $client_certificate_key = $url_set->{'client-certificate-key'}->[0];
		if (defined($client_certificate_key)) {
			$ENV{'HTTPS_CERT_KEY'}=$client_certificate_key;
		}
		
		my $base_url = $url_set->{'url'}->[0];
		
		my %url_list;
		my @url_list;
		my @base_url_list;		

		my @url_extensions = @{$url_set->{'subdir'}};
		
		@url_extensions=($url_set->{'processeddir'}->[0],@url_extensions);

		my $directory_status = 1;
		
		foreach $url_extension (@url_extensions) {
			my $request=new HTTP::Request GET => "$base_url/$url_extension/";
			if ($authentication eq "basic") {
				$request->authorization_basic($user,$pass);
			}
			my $result=$ua->request($request);
		
			if (!$result->is_success) {
				IPFC::Log::logger(1,"Problem getting directory listing $base_url/$url_extension/");
				$directory_status = 0;
			}
			my $directory_listing=$result->content;
			my @directory_listing=split('\n',$directory_listing);
			#print @directory_listing;
			
			my $directory_index = 0;
			while (defined($directory_listing[$directory_index])) {
# ugly hack to get the right files ... but it seems to work for now
				if ($directory_listing[$directory_index] =~ /^<IMG SRC=\"\/icons\/unknown.gif\" ALT=\"\[   \]\"> <A HREF=\"(.*)\">/) {
					$url_list{"$url_extension/$1"}="noted";
				}
				$directory_index++;
			}
		}
		
		if ($directory_status == 0) {
			next;			# there was a problem getting the directory listing and we'll try again
		}
		
		foreach $url (keys %url_list) {
			if (($url !~ /\.ok$/) && ($url !~ /\.processed$/)) {	# not an OK or processed file
				$url =~ /\/([^\/]*)$/ ; # get the filename part
				my $base_url = $1;
				if (defined($url_list{$url.".ok"})) { 	# which has corresponding OK file...
				  if (!defined($url_list{"$url_set->{'processeddir'}->[0]/$base_url.processed"})) { # but no corresponding processed file...
					push @url_list,$url;
					#push @url_list,$url.".ok";
				  }
				}
			}
		}
		
		if ($#url_list >= 0) {
			$url_seen = 1;
		}
		
		foreach $url (@url_list) {
			my $request = new HTTP::Request GET => "$base_url/$url";
			if ($authentication eq "basic") {
				$request->authorization_basic($user,$pass);
			}
			my $result=$ua->request($request);
			if (!$result->is_success) {
				IPFC::Log::logger(2,"Problem getting file $base_url/$url");
				next;
			}
			
			my $xml_file=$result->content;
			
			my $xml_message = IPFC::Sign::check_signature($xml_file);
			
			if (!defined($xml_message)) {
				IPFC::Log::logger(1,"Wrong signature in XML-file $base_url/$url");
				$url_seen = 0; # wait after this set of files (to avoid tight looping)
				next;
			}
			
			my $xml_parsed;
			
			eval { 
				$xml_parsed = XMLin($xml_message,forcearray=>1,keyattr=>[],contentkey=>[]);
			} ;
			
			if ($@) {
				IPFC::Log::logger(1,"Problem parsing XML-file $base_url/$url");
				$url_seen = 0; # wait after this set of files (to avoid tight looping)
				next;
			}
			
			my $xml_agent_id = $xml_parsed->{'agent'}->[0]->{'id'};
			
			if (defined($xml_agent_id) && (!ref($xml_agent_id)) && ($agent_config_location{$xml_agent_id} == $url_set_index)) {
				my $filename = $config_file->{'download-directory'}->[0]."/".$xml_agent_id."/".$url;

#				if (-e $filename) {
#					die "The file $filename exists already";
#				}
				
				open XMLFILE,">",$filename or die "Cannot open file $filename for writing";
				print XMLFILE $xml_file or die "Cannot write on $filename";
				close XMLFILE;
				
				# make a nice .ok file so the db-backend-daemon knows all of it is there and gets to see it ...
				open OKFILE,">",$filename.".ok" or die "Cannot open file $filename.ok for writing";
				close OKFILE;
				
				# we have the file, we have the ok file ... processing is finished, push the .processed file
				
#				print "base_url = $base_url\n";
#				print "url = $url\n";
				
				$url =~ /\/([^\/]*)$/;
				my $processed_url = "$base_url/processed/$1.processed";
				
#				print "processed_url = $processed_url\n";
				
				my $processed_request = new HTTP::Request PUT => $processed_url;
				if ($authentication eq "basic") {
					$processed_request->authorization_basic($user,$pass);
				}
				$processed_request->content("");
				my $result=$ua->request($processed_request);
		
				if (!$result->is_success) {
					IPFC::Log::logger(1,"There was a problem putting the processed file $processed_url on the dr-server !");
				} else {
					IPFC::Log::logger(2,"File $base_url/$url OK...");
				}

				
			} else { # this agentid is not supposed to write to this part of the webserver ... what is happening ?
				IPFC::Log::logger(1,"Agent has written in the wrong place on the dr-server ... please investigate !");
				IPFC::Log::logger(1,"url = $base_url$url");
				if (defined($xml_agent_id)) { 
					IPFC::Log::logger(1,"agentid in xml file = $xml_agent_id");
					IPFC::Log::logger(1,"expected url-set = $config_file->{'url-set'}->[$agent_config_location{$xml_agent_id}]->{'id'}->[0]");
				} else {
					IPFC::Log::logger(1,"no agentid was found in xml file !");
				}
				IPFC::Log::logger(1,"actual url-set = $url_set->{'id'}->[0]");
			}
		}				
	}
	
	if (not $url_seen) {
			IPFC::Log::logger(3,"No files to treat, sleeping $config_file->{'poll-interval'}->[0] seconds...");
			sleep $config_file->{'poll-interval'}->[0];
	}
	
}
