/* dictfmt.c -- 
 * Created: Sun Jul 20 20:17:11 1997 by faith@acm.org
 * Revised: Sun Jul  5 19:25:18 1998 by faith@acm.org
 * Copyright 1997, 1998 Rickard E. Faith (faith@acm.org)
 * 
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 1, or (at your option) any
 * later version.
 * 
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 675 Mass Ave, Cambridge, MA 02139, USA.
 * 
 * $Id: dictfmt.c,v 1.6 1998/07/06 01:33:40 faith Exp $
 *
 * Sun Jul 5 18:48:33 1998: added patches for Gutenberg's '1995 CIA World
 * Factbook' from David Frey <david@eos.lugs.ch>.
 *
 */

#include "config.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <ctype.h>

#if HAVE_GETOPT_H
#include <getopt.h>
#endif

#define FMT_MAXPOS 65
#define FMT_INDENT  0

#define JARGON    1
#define FOLDOC    2
#define EASTON    3
#define PERIODIC  4
#define HITCHCOCK 5
#define CIA1995   6

#define BSIZE 10240

static int  Debug;
static FILE *str;

static unsigned char b64_list[] =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";

/* |b64_encode| encodes |val| in a printable base 64 format.  A MSB-first
   encoding is generated. */

static const char *b64_encode( unsigned long val )
{
   static char   result[7];
   int    i;

   result[0] = b64_list[ (val & 0xc0000000) >> 30 ];
   result[1] = b64_list[ (val & 0x3f000000) >> 24 ];
   result[2] = b64_list[ (val & 0x00fc0000) >> 18 ];
   result[3] = b64_list[ (val & 0x0003f000) >> 12 ];
   result[4] = b64_list[ (val & 0x00000fc0) >>  6 ];
   result[5] = b64_list[ (val & 0x0000003f)       ];
   result[6] = 0;

   for (i = 0; i < 5; i++) if (result[i] != b64_list[0]) return result + i;
   return result + 5;
}

static FILE *fmt_str;
static int  fmt_indent;
static int  fmt_pos;
static int  fmt_pending;
static int  fmt_hwcount;
static int  fmt_maxpos = FMT_MAXPOS;

static void fmt_openindex( const char *filename )
{
   char buffer[1024];

   if (!filename) return;

   sprintf( buffer, "sort -df > %s\n", filename );
   
   if (!(fmt_str = popen( buffer, "w" ))) {
      fprintf( stderr, "Cannot open %s for write\n", buffer );
      exit(1);
   }
}

static void fmt_newline( void )
{
   int i;
   
   fputc('\n', str);
   for (i = 0; i < fmt_indent; i++) fputc(' ', str);
   fmt_pos = fmt_indent;
   fmt_pending = 0;
}

static void fmt_string( const char *s )
{
   char *sdup = malloc( strlen(s) + 1 );
   char *pt = sdup;
   char *p = sdup;
#if 0
   char *t;
#endif
   int  len;

#if 1
   strcpy( sdup, s );
#else
   for (t = sdup; *s; s++) {
      if (*s == '_') *t++ = ' ';
      else *t++ = *s;
   }
   *t = '\0';
#endif

   while ((pt = strchr(pt, ' '))) {
      *pt++ = '\0';
      len = strlen(p);
      if (fmt_pending && fmt_pos + len > fmt_maxpos) {
	 fmt_newline();
      }
      if (fmt_pending) {
	 fputc(' ', str);
	 ++fmt_pos;
	 fmt_pending = 0;
      }
      fprintf( str, "%s", p );
      fmt_pos += len;
      p = pt;
      fmt_pending = 1;
   }
   
   len = strlen(p);
   if (fmt_pending && fmt_pos + len > fmt_maxpos) {
      fmt_newline();
   }
   if (len && fmt_pending) {
      fputc(' ', str);
      ++fmt_pos;
      fmt_pending = 0;
   }
   if (!len) {
      fmt_pending = 1;
   } else {
      fprintf( str, "%s", p );
      fmt_pos += len;
   }

   free(sdup);
}

static void fmt_newheadword( const char *word, int flag )
{
   static char prev[1024] = "";
   static int  start = 0;
   static int  end;

   fmt_indent = 0;
   if (*prev) fmt_newline();
   fflush(stdout);
   end = ftell(str);
   
   if (fmt_str && *prev) {
      fprintf( fmt_str, "%s\t%s\t", prev, b64_encode(start) );
      fprintf( fmt_str, "%s\n", b64_encode(end-start) );
   }
   if (word) {
      strcpy(prev,word);
      start = end;
      if (flag) {
	 fmt_string(word);
	 fmt_indent += FMT_INDENT;
	 fmt_newline();
      }
   }

   if (fmt_hwcount && !(fmt_hwcount % 100)) {
      fprintf( stderr, "%10d headwords\r", fmt_hwcount );
   }
   ++fmt_hwcount;
}

static void fmt_closeindex( void )
{
   fmt_newheadword(NULL,0);
   if (fmt_str) pclose( fmt_str );
   fprintf( stderr, "%12d headwords\n", fmt_hwcount );
}

int main( int argc, char **argv )
{
   int        c;
   int        type = 0;
   char       buffer[1024];
   char       buffer2[1024];
   char       indexname[1024];
   char       dataname[1024];
   const char *url = "unknown";
   const char *sname = "unknown";
   int        header = 0;
   time_t     t;
   char       *pt;
   char       *s, *d;
   char       *buf;

   while ((c = getopt(argc, argv, "jfephDu:s:c:")) != EOF)
      switch (c) {
      case 'j': type = JARGON;    break;
      case 'f': type = FOLDOC;    break;
      case 'e': type = EASTON;    break;
      case 'p': type = PERIODIC;  break;
      case 'h': type = HITCHCOCK; break;
      case 'D': ++Debug;          break;
      case 'u': url = optarg;     break;
      case 's': sname = optarg;   break;
      case 'c':
	 switch (*optarg) {
	 case '5': type = CIA1995; break;
	 default:  fprintf( stderr,
			    "Only CIA 1995 (-c5) currently supported\n" );
	 exit(1);
	 }
	 break;
      default:
	 fprintf( stderr,
		  "usage: dictfmt [-jfephD] [-c5] -u url -s short basename\n");
	 exit(1);
      }

   if (optind + 1 != argc) {
      fprintf( stderr,
	       "usage: dictfmt [-jfephD] [-c5] -u url -s short basename\n" );
      exit(1);
   }

   sprintf( indexname, "%s.index", argv[optind] );
   sprintf( dataname,  "%s.dict", argv[optind] );
   
   fmt_openindex( indexname );
   if (Debug) {
      str = stdout;
   } else {
      if (!(str = fopen(dataname, "w"))) {
	 fprintf(stderr, "Cannot open %s for write\n", dataname);
	 exit(1);
      }
   }

   fmt_newheadword("00-database-url",1);
   fmt_string( "     " );
   fmt_string( url );

   fmt_newheadword("00-database-short",1);
   fmt_string( "     " );
   fmt_string( sname );

   fmt_newheadword("00-database-info",1);
   fmt_string("This file was converted from the original database on:" );
   fmt_newline();
   time(&t);
   sprintf( buffer, "          %25.25s", ctime(&t) );
   fmt_string( buffer );
   fmt_newline();
   fmt_newline();
   fmt_string( "The original data is available from:" );
   fmt_newline();
   fmt_string( "     " );
   fmt_string( url );
   fmt_newline();
   fmt_newline();
   fmt_string(
      "The original data was distributed with the notice shown below."
      "  No additional restrictions are claimed.  Please redistribute"
      " this changed version under the same conditions and restriction"
      " that apply to the original version." );
   fmt_newline();
   fmt_indent += 3;
   fmt_newline();
   fmt_maxpos = 200;		/* Don't wrap */

   while (fgets(buf = buffer,BSIZE-1,stdin)) {
      if (strlen(buffer))
	 buffer[strlen(buffer)-1] = '\0'; /* remove newline */
      
      switch (type) {
      case HITCHCOCK:
	 if (strlen(buffer) == 1) {
	    header = 1;
	    continue;;
	 }
	 if (header) {
	    strcpy( buffer2, buffer );
	    if ((pt = strchr( buffer2, ','))) {
	       *pt = '\0';
	       fmt_newheadword(buffer2, 0);
	    }
	 }
	 break;
      case EASTON:
	 strcpy( buffer2, buffer );
	 for (s = buffer2, d = buffer; *s; ++s) {
	    if (*s == '<') {
	       header = 1;
	       switch (s[1]) {
	       case 'I': *d++ = '_'; break;
	       case 'A':
		  if (s[3] == 'N') goto skip;
		  *d++ = '{';
		  break;
	       case 'P': goto skip;
	       case 'B': goto copy;
	       case '/':
		  switch(s[2]) {
		  case 'I': *d++ = '_'; break;
		  case 'A': *d++ = '}'; break;
		  case 'B': goto copy;
		  default:
		     fprintf( stderr,
			      "Unknown tag: %s (%c%c)\n",
			      buffer2, s[1], s[2] );
		     exit(1);
		  }
		  break;
	       default:
		  fprintf( stderr, "Unknown tag: %s (%c)\n", buffer2, s[1] );
		  exit(1);
	       }
	       while (*s && *s != '>') s++;
	       continue;
	    }
      copy:
	    *d++ = *s;
	 }
	 *d = '\0';
#if 0
	 printf( "BEFORE: %s\n", buffer2 );
	 printf( "AFTER: %s\n", buffer );
#endif

	 if (*buffer == '<') {
	    switch (buffer[1]) {
	    case 'B':
	       if ((pt = strstr( buffer+3, " - </B>" ))) {
		  *pt = '\0';
		  fmt_newheadword(buffer+3, 0);
		  fmt_indent += 3;
		  memmove( buf, buffer+3, strlen(buffer+3)+1 );
	       } else {
		  fprintf( stderr, "No end: %s\n", buffer );
		  exit(1);
	       }
	       break;
	    default:
	       fprintf( stderr, "Unknown: %s\n", buffer );
	       exit(1);
	    }
	 } else {
	    if (buffer[0] == ' ' && buffer[1] == ' ') fmt_newline();
	 }
	 break;
      case JARGON:
	 switch (*buffer) {
	 case ':':
	    header = 1;
	    if ((pt = strchr( buffer+1, ':' ))) {
	       s = pt + 1;
	       if (*s == ':') ++s;
	       
	       *pt = '\0';
	       fmt_newheadword(buffer+1, 0);

	       memmove( buf, buffer+1, strlen(buffer+1));
	       memmove( pt-1, s, strlen(s)+1 ); /* move \0 also */
	    }
	    break;
	 case '*':
	 case '=':
	 case '-':
	    if (buffer[0] == buffer[1]
		&& buffer[0] == buffer[2]
		&& buffer[0] == buffer[3])
	       continue;		/* Skip lines with *'s and ='s */
	 }
	 break;
      case PERIODIC:
	 switch (*buffer) {
	 case '%':
	    if (buffer[1] == 'h') {
	       if (!header) {
		  header = 1;
		  continue;
	       } else {
		  fmt_newheadword(buffer+3,1);
		  continue;
	       }
	    } else if (buffer[1] == 'd') {
	       continue;
	    }	    
	    break;
	 }
	 break;
      case FOLDOC:
	 if (*buffer && *buffer != ' ' && *buffer != '\t') {
	    if (header < 2) {
	       ++header;
	    } else {
	       fmt_newheadword(buffer,1);
	       continue;
	    }
	 }
	 if (*buf == '\t') {
	    memmove( buf+2, buf, strlen(buf)+1 ); /* move \0 */
	    buf[0] = buf[1] = buf[2] = ' ';
	 }
	 break;
      case CIA1995:
	 if (*buffer == '@') {
	    buf++;
	 } else if (strncmp(buffer, "_____",5) == 0) {
	    fgets(buf = buffer,BSIZE-1,stdin); /* empty line */
	    
	    fgets(buf = buffer,BSIZE-1,stdin);
	    if (strlen(buffer))
	       buffer[strlen(buffer)-1] = '\0'; /* remove newline */
	    
	    while (isspace(*buf)) buf++;
	    if (*buf != '\0') {
	       char *p;
	       int l; /* flag: downcase the letter */
	       
	       header=1;
	       
	       /* Downcase `buf' sensibly */
	       
	       /* Leave first character in upper case, 
		  but downcase the rest */
	       p=buf; p++; l=1; 
	       while (*p != '\0') {
		  if (isspace(*p)) l=0;
		  else {
		     if (l) *p=tolower(*p); 
		     l=1;
		  }
		  p++;
	       }
	       fmt_newheadword(buf,0);
	    }
 	 }
 	 break;
      default:
	 fprintf(stderr, "Unknown input format type %d\n", type );
	 exit(2);
      }
      fmt_string(buf);
      fmt_newline();
 skip:
   }
   
   fmt_newheadword(NULL,0);

   fmt_closeindex();
   fclose(str);
   return 0;
}
