/* *************************************************************
   RWhois Software

   Copyright (c) 1994 Scott Williamson and Mark Kosters
   Copyright (c) 1996-2000 Network Solutions, Inc.

   See the file LICENSE for conditions of use and distribution.
**************************************************************** */

#include "fileinfo.h"

#include "auth_area.h"
#include "schema.h"
#include "defines.h"
#include "fileutils.h"
#include "log.h"
#include "misc.h"
#include "strutil.h"
#include "main_config.h"

#define MASTER_FILE_LIST    "local.db"
#define MASTER_FILE_LIST_W  "local.db.write"
#define MASTER_FILE_LIST_B  "local.db.bak"

#define LOCK_BLOCKING_TIME  5 /* in USLEEP_WAIT_PERIODs */


typedef enum
{
  MFL_READ,
  MFL_WRITE,
  MFL_BACKUP
} master_inst_type;

/* ------------------- Local Functions ---------------- */

static mkdb_file_type select_type PROTO((char *ftype));
static char *select_str_from_type PROTO((mkdb_file_type type));
static file_struct *read_file_struct PROTO((FILE *fp));
static int write_file_struct PROTO((FILE *fp, file_struct *fi, int not_last));
static int read_file_list PROTO((char *index_file, dl_list_type *file_list));
static void write_file_list PROTO((char *filename, dl_list_type *file_list));
static int next_file_no PROTO((dl_list_type *file_list));
static char *generate_file_name PROTO((char *tmp_filename, char *base_filename,
                                       dl_list_type *full_file_list));
static int get_master_index_file PROTO((class_struct     *class,
                                        auth_area_struct *auth_area,
                                        master_inst_type instance_type,
                                        char             *index_file));
static int add_file_struct PROTO((file_struct    *file,
                                  dl_list_type   *file_list,
                                  mkdb_lock_type lock_mode));
static int delete_file_no PROTO((int file_no, dl_list_type *file_list));
static int lock_unlock_file_no PROTO((int            file_no,
                                      mkdb_lock_type lock,
                                      dl_list_type   *file_list));
static int update_file_no PROTO((file_struct  *update_file,
                                 dl_list_type *file_list));
static int add_file_list PROTO((dl_list_type   *file_list,
                                dl_list_type   *full_file_list,
                                mkdb_lock_type lock_mode));
static int delete_file_list PROTO((dl_list_type *file_list,
                                  dl_list_type *full_file_list));
static int mod_file_list PROTO((dl_list_type *file_list,
                                dl_list_type *full_file_list));
static int lock_unlock_file_list PROTO((dl_list_type   *file_list,
                                        dl_list_type   *full_file_list,
                                        mkdb_lock_type lock));
static int install_write_file_list PROTO((class_struct     *class,
                                          auth_area_struct *auth_area));

/* ---- file list reading and writing primitives --- */

/* select_mkdb_file_type: returns the database file type (data or
   index) based on the tag. */
static mkdb_file_type
select_type(ftype)
  char  *ftype;
{
  if (STR_EQ(ftype, MKDB_EXACT_INDEX_STR) ||
      STR_EQ(ftype, MKDB_OLD_INDEX_STR) ||
      STR_EQ(ftype, MKDB_OLD_INDEX_FIRST_STR) ||
      STR_EQ(ftype, MKDB_OLD_INDEX_WORD_STR))
  {
    return MKDB_EXACT_INDEX_FILE;
  }
  if (STR_EQ(ftype, MKDB_SOUNDEX_INDEX_STR))
  {
    return MKDB_SOUNDEX_INDEX_FILE;
  }
  if (STR_EQ(ftype, MKDB_CIDR_INDEX_STR))
  {
    return MKDB_CIDR_INDEX_FILE;
  }

  if (STR_EQ(ftype, MKDB_DATA_FILE_STR))
  {
    return MKDB_DATA_FILE;
  }

  return MKDB_NO_FILE;
}

static char *
select_str_from_type(type)
  mkdb_file_type    type;
{
  switch (type)
  {
  case MKDB_EXACT_INDEX_FILE:
    return MKDB_EXACT_INDEX_STR;
  case MKDB_SOUNDEX_INDEX_FILE:
    return MKDB_SOUNDEX_INDEX_STR;
  case MKDB_CIDR_INDEX_FILE:
    return MKDB_CIDR_INDEX_STR;
  case MKDB_DATA_FILE:
    return MKDB_DATA_FILE_STR;
  default:
    return NULL;
  }
}

static file_struct *
read_file_struct(fp)
  FILE  *fp;
{
  char        line[MAX_LINE + 1];
  char        tag[MAX_LINE];
  char        datum[MAX_LINE];
  file_struct *fi;

  bzero(line, sizeof(line));

  fi = xcalloc(1, sizeof(*fi));

  while ((readline(fp, line, MAX_LINE)) != NULL)
  {
    if (new_record(line))
    {
      break;
    }

    if (parse_line(line, tag, datum))
    {
      if (STR_EQ(tag, MKDB_TYPE_TAG))
      {
        fi->type = select_type(datum);
      }
      else if (STR_EQ(tag, MKDB_FILE_TAG))
      {
        fi->filename = xstrdup(datum);
      }
      else if (STR_EQ(tag, MKDB_NUMRECS_TAG))
      {
        fi->num_recs = atol(datum);
      }
      else if (STR_EQ(tag, MKDB_FILE_NO_TAG))
      {
        fi->file_no = atoi(datum);
      }
      else if (STR_EQ(tag, MKDB_SIZE_TAG))
      {
#if defined(OFF_T64) && defined(HAVE_ATOLL)
        fi->size = atoll(datum);
#else
        fi->size = atol(datum);
#endif
      }
      else if (STR_EQ(tag, MKDB_LOCK_TAG))
      {
        fi->lock = true_false(datum);
      }
      else
      {
        log(L_LOG_WARNING, MKDB, "unknown file list tag: %s",
            tag);
      }
    }
    else
    {
      log(L_LOG_WARNING, MKDB, "malformed file list line: %s",
          line);
      return NULL;
    }
  }

  /* check to see if we have a valid file_struct; for now, this means
     a real type, a real name, and a positive file number */
  if (fi->filename && *fi->filename &&
      (fi->type > MKDB_NO_FILE) &&
      (fi->file_no >= 0))
  {
    return(fi);
  }

  /* otherwise, we trash the record */
  destroy_file_struct_data(fi);

  return NULL;
}

static int
write_file_struct(fp, fi, not_last)
  FILE        *fp;
  file_struct *fi;
  int         not_last;
{

  if (!fp || !fi)
  {
    return FALSE;
  }

  fprintf(fp, "%s:%s\n", MKDB_TYPE_TAG, select_str_from_type(fi->type));
  fprintf(fp, "%s:%s\n", MKDB_FILE_TAG, fi->filename);
  fprintf(fp, "%s:%d\n", MKDB_FILE_NO_TAG, fi->file_no);
#ifdef OFF_T64
  fprintf(fp, "%s:%lld\n", MKDB_SIZE_TAG, fi->size);
#else
  fprintf(fp, "%s:%ld\n", MKDB_SIZE_TAG, fi->size);
#endif
  fprintf(fp, "%s:%ld\n", MKDB_NUMRECS_TAG, fi->num_recs);
  fprintf(fp, "%s:%s\n", MKDB_LOCK_TAG, on_off(fi->lock));

  if (not_last)
  {
    fprintf(fp, "---\n");
  }

  return TRUE;
}

/* read_file_list: reads a file_list data file into the file_list
   structure, by type.  Returns TRUE on success. */
static int
read_file_list(index_file, file_list)
  char            *index_file;
  dl_list_type    *file_list;
{
  FILE              *fp = NULL;
  file_struct       *fi;
  int               tries = 5; /* in USLEEP_WAIT_PERIODs */

  /* can't work with null ptrs */
  if (!index_file || !file_list)
  {
    log(L_LOG_ERR, MKDB, "read_file_list: null data detected");
    return FALSE;
  }

  /* adjust for the lack of usleep */
#ifndef HAVE_USLEEP
  tries = tries / USLEEP_SEC_CONV; if (tries == 0) tries++;
#endif

  fp = fopen(index_file, "r");
  while (fp == NULL && errno == ENOENT && tries > 0)
  {
    tries--;
#ifdef HAVE_USLEEP
    usleep(USLEEP_WAIT_PERIOD);
#else
    sleep(1);
#endif
    fp = fopen(index_file, "r");
  }

  if (!fp)
  {
    if (errno == ENOENT)
    {
/*       log(L_LOG_DEBUG, MKDB, */
/*           "master file list '%s' not present; area assumed to be unindexed", */
/*           index_file); */
      return TRUE;
    }
    else
    {
      log(L_LOG_ERR, MKDB,
          "could not open master index file '%s': %s", index_file,
          strerror(errno));

      return FALSE;
    }
  }

  while ( (fi = read_file_struct(fp)) )
  {
    dl_list_append(file_list, fi);
  }

  fclose(fp);

  return TRUE;
}

/* write_file_list: writes a file list to the master index file. The
   entire file_list must be given to this routine. */
static void
write_file_list(filename, file_list)
  char         *filename;
  dl_list_type *file_list;
{
  FILE         *fp;
  file_struct  *fi;
  int          not_done;

  /* note: locking should be taken care of in the enclosing routine */
  fp = fopen(filename, "w");
  if (!fp)
  {
    log(L_LOG_ERR, MKDB, "could not open '%s' for write: %s", filename,
        strerror(errno));
    return;
  }

  not_done = dl_list_first(file_list);
  while (not_done)
  {
    fi = dl_list_value(file_list);

    not_done = dl_list_next(file_list);

    write_file_struct(fp, fi, not_done);
  }

  fclose(fp);
}

/* next_file_no: returns the next file number.  Assumes a) the
   file_list is the full file list, and b) that they are sequentially
   increasing (that is, the last element has the highest number */
static int
next_file_no(file_list)
  dl_list_type  *file_list;
{
  file_struct   *file;

  if (dl_list_empty(file_list))
  {
    return(0);
  }

  dl_list_last(file_list);

  file = dl_list_value(file_list);
  if (!file)
  {
    /* FIXME: maybe should log error here */
    return(0);
  }

  return(file->file_no + 1);
}

static char *
generate_file_name(tmp_filename, base_filename, full_file_list)
  char         *tmp_filename;
  char         *base_filename;
  dl_list_type *full_file_list;
{
  file_struct  *f;
  char         template[MAX_FILE + 1];
  char         real_fname[MAX_FILE + 1];
  char         dir[MAX_FILE];
  char         file[MAX_FILE];
  char         base_file[MAX_FILE];
  char         base_dir[MAX_FILE];
  int          not_done;
  int          index_no = 0;

  if (NOT_STR_EXISTS(base_filename))
  {
    log(L_LOG_DEBUG, MKDB,
        "generate_file_name: falling back to temporary file name");
    return xstrdup(tmp_filename);
  }

  split_path(base_filename, base_dir, base_file);

  index_no = 0;
  not_done = dl_list_last(full_file_list);

  /* find the last instance of a file matching the base_filename template */
  while (not_done)
  {
    f = dl_list_value(full_file_list);
    split_path(f->filename, dir, file);

    log(L_LOG_DEBUG, MKDB, "comparing '%s' with '%s'", file, base_file);

    if (sscanf(file, base_file, &index_no) > 0)
    {
      log(L_LOG_DEBUG, MKDB, "comparison succeeed, index no = %d", index_no);
      index_no++;
      break;
    }

    not_done = dl_list_prev(full_file_list);
  }
  sprintf(template, "%s/%s", dir, base_file);
  sprintf(real_fname, template, index_no);

  log(L_LOG_DEBUG, MKDB, "generate_file_name: generated '%s'",
      real_fname);

  return xstrdup(real_fname);
}

/* fills the 'index_file' varible with the path and name of the master
   index file, given the instance type (read, write, backup, etc) */
static int
get_master_index_file(class, auth_area, instance_type, index_file)
  class_struct     *class;
  auth_area_struct *auth_area;
  master_inst_type instance_type;
  char             *index_file;
{
  if (!class || !class->name || !index_file)
  {
    return FALSE;
  }

  switch (instance_type)
  {
  case MFL_READ:
    sprintf(index_file, "%s/%s", class->db_dir, MASTER_FILE_LIST);
    break;
  case MFL_WRITE:
    sprintf(index_file, "%s/%s", class->db_dir, MASTER_FILE_LIST_W);
    break;
  case MFL_BACKUP:
    sprintf(index_file, "%s/%s", class->db_dir, MASTER_FILE_LIST_B);
    break;
  }

  return TRUE;
}

/* adds a file structure to a list, checking for duplication and
   assigning the index number.  */
static int
add_file_struct(file, file_list, lock_mode)
  file_struct    *file;
  dl_list_type   *file_list;
  mkdb_lock_type lock_mode;
{
  file_struct *tmp_file;
  char        file_path[MAX_FILE + 1];

  if (NOT_STR_EXISTS(file->filename) && NOT_STR_EXISTS(file->tmp_filename))
  {
    log(L_LOG_ERR, MKDB, "generate_file_name: null data detected");
    return FALSE;
  }

  /* check to see if we must generate a "real" filename first */
  if (NOT_STR_EXISTS(file->filename))
  {
    file->filename = generate_file_name(file->tmp_filename,
                                        file->base_filename,
                                        file_list);

    if (STR_EXISTS(file->filename))
    {
      if (link(file->tmp_filename, file->filename) >= 0)
      {
        unlink(file->tmp_filename);
      }
      else
      {
        log(L_LOG_WARNING, MKDB,
            "attempt to move temporary file '%s' to '%s' failed: %s",
            file->tmp_filename, file->filename, strerror(errno));
        free(file->filename);
        file->filename = xstrdup(file->tmp_filename);
      }
    }
  }

  /* standardize the file name */
  if (!canonicalize_path(file_path, MAX_FILE, file->filename,
                         get_root_dir(), FALSE, FALSE))
  {
    return(-1);
  }

  tmp_file = find_file_by_name(file_list, file_path, MKDB_ALL_FILES);

  /* file was already in there */
  if (tmp_file)
  {

    /* either way, we would like the file_no to be filled out */
    file->file_no = tmp_file->file_no;

    /* if it is equivalent, then we are done */
    if (tmp_file->type     == file->type &&
        tmp_file->size     == file->size &&
        tmp_file->num_recs == file->num_recs)
    {
      return TRUE;
    }

    /* update the sucker */
    tmp_file->type     = file->type;
    tmp_file->size     = file->size;
    tmp_file->num_recs = file->num_recs;
  }
  else
  {
    /* otherwise, we add it */
    file->lock         = lock_mode;
    file->file_no      = next_file_no(file_list);

    tmp_file           = copy_file_struct(file);

    dl_list_append(file_list, tmp_file);
  }

  return TRUE;
}

/* deletes a given file from a file list */
static int
delete_file_no(file_no, file_list)
  int          file_no;
  dl_list_type *file_list;
{
  file_struct *file;
  int         not_done;

  not_done = dl_list_first(file_list);
  while (not_done)
  {
    file = dl_list_value(file_list);

    if (file->file_no == file_no)
    {
      dl_list_delete(file_list);
      return TRUE;
    }

    not_done = dl_list_next(file_list);
  }

  return FALSE;
}

/* locks or unlocks a given file number with a file list */
static int
lock_unlock_file_no(file_no, lock, file_list)
  int            file_no;
  mkdb_lock_type lock;
  dl_list_type   *file_list;
{
  file_struct *file;
  int         not_done;

  not_done = dl_list_first(file_list);
  while (not_done)
  {
    file = dl_list_value(file_list);

    if (file->file_no == file_no)
    {
      file->lock = lock;
      return TRUE;
    }

    not_done = dl_list_next(file_list);
  }

  return FALSE;
}

/* updates the statistics of the corresponding (by file no) file
   structure in list.  */
static int
update_file_no(update_file, file_list)
  file_struct  *update_file;
  dl_list_type *file_list;
{
  file_struct *file;
  int         not_done;

  not_done = dl_list_first(file_list);
  while (not_done)
  {
    file = dl_list_value(file_list);

    if (file->file_no == update_file->file_no)
    {
      file->size     = update_file->size;
      file->num_recs = update_file->num_recs;
      return TRUE;
    }

    not_done = dl_list_next(file_list);
  }

  return FALSE;
}

/* add the files in the list to the full file list */
static int
add_file_list(file_list, full_file_list, lock_mode)
  dl_list_type     *file_list;
  dl_list_type     *full_file_list;
  mkdb_lock_type   lock_mode;
{
  file_struct   *file;
  int           not_done;

  not_done = dl_list_first(file_list);
  while (not_done)
  {
    file = dl_list_value(file_list);

    if (!add_file_struct(file, full_file_list, lock_mode))
    {
      return FALSE;
    }

    not_done = dl_list_next(file_list);
  }

  return TRUE;
}

/* delete the files in the list from the full file list */
static int
delete_file_list(file_list, full_file_list)
  dl_list_type *file_list;
  dl_list_type *full_file_list;
{
  file_struct *file;
  int         not_done;

  not_done = dl_list_first(file_list);

  while (not_done)
  {
    file = dl_list_value(file_list);
    if (!file)
    {
      log(L_LOG_WARNING, MKDB,
          "attempted to delete non-existent file %s from master file list",
          file->filename);
      not_done = dl_list_next(file_list);
      continue;
    }

    delete_file_no(file->file_no, full_file_list);
    not_done = dl_list_next(file_list);
  }

  return TRUE;
}

/* modifies the statistics of the files in the list in the full file
   list */
static int
mod_file_list(file_list, full_file_list)
  dl_list_type *file_list;
  dl_list_type *full_file_list;
{
  file_struct   *file;
  int           not_done;

  not_done = dl_list_first(file_list);
  while (not_done)
  {
    file = dl_list_value(file_list);

    if (!update_file_no(file, full_file_list))
    {
      log(L_LOG_WARNING, MKDB,
          "failed to update file number %d in master file list",
          file->file_no);
      continue;
    }

    not_done = dl_list_next(file_list);
  }

  return TRUE;
}

static int
lock_unlock_file_list(file_list, full_file_list, lock)
  dl_list_type   *file_list;
  dl_list_type   *full_file_list;
  mkdb_lock_type lock;
{
  file_struct   *file;
  int           not_done;

  not_done = dl_list_first(file_list);
  while (not_done)
  {
    file = dl_list_value(file_list);

    if (!lock_unlock_file_no(file->file_no, lock, full_file_list))
    {
      log(L_LOG_WARNING, MKDB,
          "failed to lock/unlock file number %d in master file list",
          file->file_no);
      not_done = dl_list_next(file_list);
      continue;
    }

    not_done = dl_list_next(file_list);
  }

  return TRUE;
}

static int
install_write_file_list(class, auth_area)
  class_struct     *class;
  auth_area_struct *auth_area;
{
  char w_index_file_name[MAX_FILE + 1];
  char r_index_file_name[MAX_FILE + 1];
  char b_index_file_name[MAX_FILE + 1];

  if (!get_master_index_file(class, auth_area, MFL_READ, r_index_file_name))
  {
    log(L_LOG_ERR, MKDB, "");
    return FALSE;
  }

  get_master_index_file(class, auth_area, MFL_WRITE, w_index_file_name);
  get_master_index_file(class, auth_area, MFL_BACKUP, b_index_file_name);

  /* first remove the backup file, if it exists */
  if (file_exists(b_index_file_name))
  {
    if (unlink(b_index_file_name) < 0)
    {
      log(L_LOG_WARNING, MKDB,
          "could not remove master file list backup '%s': %s",
          b_index_file_name, strerror(errno));
    }
  }

  /* now move the current read file into the backup position */
  if (file_exists(r_index_file_name))
  {
    if (link(r_index_file_name, b_index_file_name) < 0 ||
        unlink(r_index_file_name) < 0)
    {
      log(L_LOG_WARNING, MKDB,
          "could not move read master file list '%s' to backup: %s",
          r_index_file_name, strerror(errno));
    }
  }

  /* finally, move the current write file into the read file position */
  if (link(w_index_file_name, r_index_file_name) < 0 ||
      unlink(w_index_file_name) < 0)
  {
    log(L_LOG_ERR, MKDB,
        "could not move write master file list '%s' to read: %s",
        w_index_file_name, strerror(errno));
    return FALSE;
  }

  return TRUE;
}

/* check to see if an area is indexed at all.  This is so we don't
   have to spin looking for a possibly transient master index file. (3
   stat() calls are much cheaper than any amount of waiting... */
static int
is_area_indexed(class, auth_area)
  class_struct *class;
  auth_area_struct *auth_area;
{
  char w_index_file_name[MAX_FILE + 1];
  char r_index_file_name[MAX_FILE + 1];
  char b_index_file_name[MAX_FILE + 1];

  /* first we attempt to calculate all the names of the master index
     files. */
  if (!get_master_index_file(class, auth_area, MFL_READ, r_index_file_name))
  {
    log(L_LOG_ERR, MKDB, "");
    return FALSE;
  }

  get_master_index_file(class, auth_area, MFL_WRITE, w_index_file_name);
  get_master_index_file(class, auth_area, MFL_BACKUP, b_index_file_name);

  /* now we see if any of them exist.  It is theortically possible to
     miss all three due to some wacky process scheduling, but is is
     probably very unlikely */

  if (file_exists(r_index_file_name))
  {
    return TRUE;
  }

  if (file_exists(b_index_file_name))
  {
    return TRUE;
  }

  if (file_exists(w_index_file_name))
  {
    return TRUE;
  }

  return FALSE;
}

/* ------------------- Public Functions --------------- */


int
mkdb_file_type_equals(type1, type2)
  mkdb_file_type type1;
  mkdb_file_type type2;
{
  if (type1 == type2) return TRUE;

  if (type1 == MKDB_NO_FILE || type2 == MKDB_NO_FILE) return FALSE;

  if (type1 == MKDB_ALL_FILES || type2 == MKDB_ALL_FILES) return TRUE;

  if (type1 == MKDB_ALL_INDEX_FILES)
  {
    if (type2 >= MKDB_EXACT_INDEX_FILE && type2 < MKDB_MAX_FILE_TYPE)
    {
      return TRUE;
    }
    return FALSE;
  }

  if (type2 == MKDB_ALL_INDEX_FILES)
  {
    if (type1 >= MKDB_EXACT_INDEX_FILE && type1 < MKDB_MAX_FILE_TYPE)
    {
      return TRUE;
    }
    return FALSE;
  }

  return FALSE;
}

file_struct *
copy_file_struct(fi)
  file_struct *fi;
{
  file_struct *copy;

  copy = xcalloc(1, sizeof(*fi));

  /* copy the main result */
  bcopy(fi, copy, sizeof(*copy));

  /* realloc sub allocations */
  if (fi->filename)
  {
    copy->filename = xstrdup(fi->filename);
  }

  if (fi->tmp_filename)
  {
    copy->tmp_filename = xstrdup(fi->tmp_filename);
  }

  if (fi->base_filename)
  {
    copy->base_filename = xstrdup(fi->base_filename);
  }

  return(copy);
}

int
copy_file_list(target_file_list, source_file_list)
  dl_list_type *target_file_list;
  dl_list_type *source_file_list;
{
  int         not_done;
  file_struct *f;
  file_struct *f_copy;

  if (!target_file_list || !source_file_list)
  {
    log(L_LOG_ERR, MKDB, "copy_file_list: null data detected");
    return FALSE;
  }

  not_done = dl_list_first(source_file_list);

  while (not_done)
  {
    f = dl_list_value(source_file_list);

    f_copy = copy_file_struct(f);

    dl_list_append(target_file_list, f_copy);

    not_done = dl_list_next(source_file_list);
  }

  return TRUE;
}
/* get_dir: given class and auth_area (or just auth_area) return the
     data directory in 'dir'.  Return TRUE on success. */
int
get_dir(class_name, auth_area_name, dir)
  char  *class_name;
  char  *auth_area_name;
  char  *dir;
{
  auth_area_struct *auth_area;
  class_struct     *class;

  if (!auth_area_name ||
      !*auth_area_name)
  {
    return(FALSE);
  }

  auth_area = find_auth_area_by_name(auth_area_name);
  if (auth_area == NULL)
  {
    log(L_LOG_ERR, MKDB, "no auth-area '%s'", auth_area_name);
    return(FALSE);
  }
  if (!class_name ||
      !*class_name)
  {
    if (!(auth_area->data_dir))
    {
      return(FALSE);
    }

    strcpy(dir, auth_area->data_dir);

    return(TRUE);
  }

  class = find_class_by_name(auth_area->schema, class_name);
  if (class == NULL)
  {
   log(L_LOG_ERR, MKDB, "no class '%s' in auth-area '%s'",
             class_name, auth_area_name);
    return(FALSE);
  }
  if (!(class->db_dir))
  {
    return(FALSE);
  }

  strcpy(dir, class->db_dir);

  return(TRUE);
}

/* get_file_list: reads in records from the master file list
     pointed to by class & auth_area, and appends them to file_list,
     which should already be initialized. */
int
get_file_list(class, auth_area, file_list)
  class_struct     *class;
  auth_area_struct *auth_area;
  dl_list_type     *file_list;
{
  char  index_file[MAX_FILE + 1];

  /* calculate the master file list name */
  bzero((char *)index_file, sizeof(index_file));
  if (!get_master_index_file(class, auth_area, MFL_READ, index_file))
  {
    return FALSE;
  }

  /* first check to see if the area appears to be indexed in order to
     avoid a possibly lengthy read_file_list() call */
  if (! is_area_indexed(class, auth_area))
  {
    return TRUE;
  }

  if (!read_file_list(index_file, file_list))
  {
    dl_list_destroy(file_list);
    return FALSE;
  }

  return TRUE;
}

/* get_file: fills 'file_list' with entries of type 'type' given class
     and auth_area (by name), or just auth_area.  Returns TRUE on
     success. */
int
get_file(class_name, auth_area_name, file_list)
  char            *class_name;
  char            *auth_area_name;
  dl_list_type    *file_list;
{
  auth_area_struct *auth_area;
  schema_struct    *schema;
  class_struct     *class;
  dl_list_type     *class_list;
  int              not_done;

  if (!auth_area_name ||
      !*auth_area_name)
  {
    return(FALSE);
  }

  if (!file_list)
  {
    return(FALSE);
  }

  /* initialize the list */
  dl_list_default(file_list, FALSE, destroy_file_struct_data);

  auth_area = find_auth_area_by_name(auth_area_name);
  if (auth_area == NULL)
  {
    log(L_LOG_ERR, MKDB, "no auth-area '%s'", auth_area_name);
    return(FALSE);
  }

  if (class_name && *class_name)
  {
    class = find_class_by_name(auth_area->schema, class_name);
    if (class == NULL)
    {
      log(L_LOG_ERR, MKDB, "no class '%s' in auth-area '%s'",
                class_name, auth_area_name);
      return(FALSE);
    }

    if (!get_file_list(class, auth_area, file_list))
    {
      return FALSE;
    }
  }
  else
  {
    schema = auth_area->schema;
    if (!schema)
    {
      return(FALSE);
    }

    class_list = &(schema->class_list);

    not_done = dl_list_first(class_list);
    while (not_done)
    {
      class = dl_list_value(class_list);

      if (!get_file_list(class, auth_area, file_list))
      {
        return FALSE;
      }

      not_done = dl_list_next(class_list);
    }
  }

  return TRUE;
}

int
filter_file_list(result_list, type, master_list)
  dl_list_type   *result_list;
  mkdb_file_type type;
  dl_list_type   *master_list;
{
  int         not_done;
  file_struct *fi;
  file_struct *copy;

  /* reject null data */
  if (!result_list || !master_list)
  {
    log(L_LOG_WARNING, MKDB, "filter_file_list: null data detected");
    return FALSE;
  }

  not_done = dl_list_first(master_list);
  while (not_done)
  {
    fi = dl_list_value(master_list);
    /* IF a) they're looking for all files or
          b) its the right type and it isn't locked or
          c) they looking for all index files and it isn't locked
       THEN add it to the result list */
    if ( (type == MKDB_ALL_FILES) ||
         (mkdb_file_type_equals(type, fi->type) && fi->lock == MKDB_LOCK_OFF) )
    {
      copy = copy_file_struct(fi);
      dl_list_append(result_list, copy);
    }

    not_done = dl_list_next(master_list);
  } /* while (not_done) */

  return TRUE;
}


int
unlink_master_file_list(class, auth_area)
  class_struct     *class;
  auth_area_struct *auth_area;
{
  char  index_file[MAX_FILE];

  if (!get_master_index_file(class, auth_area, MFL_READ, index_file))
  {
    return FALSE;
  }

  if (unlink(index_file) < 0)
  {
    log(L_LOG_WARNING, MKDB, "could not delete master file list '%s': %s",
        index_file, strerror(errno));

    return FALSE;
  }

  /* delete the write and backup files, too, if possible */
  if (get_master_index_file(class, auth_area, MFL_WRITE, index_file))
  {
    unlink(index_file);
  }

  if (get_master_index_file(class, auth_area, MFL_BACKUP, index_file))
  {
    unlink(index_file);
  }

  return TRUE;
}


file_struct *
add_single_file(class, auth_area, file_name, type, num_recs)
  class_struct     *class;
  auth_area_struct *auth_area;
  char             *file_name;
  mkdb_file_type   type;
  long             num_recs;
{
  file_struct  *file;
  file_struct  *list_file;
  dl_list_type file_list;

  dl_list_default(&file_list, FALSE, destroy_file_struct_data);

  file = build_base_file_struct(file_name, type, num_recs);
  if (!file)
  {
    return NULL;
  }

  dl_list_append(&file_list, file);

  if (!modify_file_list(class, auth_area, &file_list, NULL, NULL, NULL,
                        NULL))
  {
    dl_list_destroy(&file_list);
    return NULL;
  }

  dl_list_first(&file_list);
  list_file = dl_list_value(&file_list);

  /* copy the list copy of the file_struct, because the list copy will
     be free()d when the list is destroyed */
  file           = xmemdup(list_file, sizeof(*file));
  file->filename = xstrdup(list_file->filename);

  dl_list_destroy(&file_list);

  return(file);
}



int
modify_file_list(class, auth_area, add_list, delete_list,
                 mod_list, unlock_list, lock_list)
  class_struct     *class;
  auth_area_struct *auth_area;
  dl_list_type     *add_list;
  dl_list_type     *delete_list;
  dl_list_type     *mod_list;
  dl_list_type     *unlock_list;
  dl_list_type     *lock_list;
{
  dl_list_type   full_file_list;
  char           write_index_file[MAX_FILE + 1];
  int            lock_fd   = -1;
  mkdb_lock_type lock_mode = MKDB_LOCK_ON;

  if (!class || !auth_area)
  {
    return FALSE;
  }

  dl_list_default(&full_file_list, FALSE, destroy_file_struct_data);

  /* first, we establish the write lock; This prevents staleness
     problems. */
  if (!get_master_index_file(class, auth_area, MFL_WRITE, write_index_file))
  {
    return FALSE;
  }

  if (!get_placeholder_lock(write_index_file, LOCK_BLOCKING_TIME, &lock_fd))
  {
    log(L_LOG_ERR, MKDB,
        "could not obtain lock for master index file '%s': %s",
        write_index_file, strerror(errno));
    return FALSE;
  }

  log(L_LOG_DEBUG, MKDB, "master file write start: %d", (int) getpid());

  /* read the current master file list */
  if (!get_file_list(class, auth_area, &full_file_list))
  {
    release_placeholder_lock(write_index_file, lock_fd);
    return FALSE;
  }

  /* guess if we want to add files locked (inactive) or not.  If we
     passed in an unlock list, then we probably want to unlock added
     files too.

     FIXME: this should be done via a parameter, or some other logic */
  if (unlock_list && !dl_list_empty(unlock_list))
  {
    lock_mode = MKDB_LOCK_OFF;
  }

  /* add loop */
  if (add_list)
  {
    add_file_list(add_list, &full_file_list, lock_mode);
  }

  /* delete loop */
  if (delete_list)
  {
    delete_file_list(delete_list, &full_file_list);
  }

  /* modify loop */
  if (mod_list)
  {
    mod_file_list(mod_list, &full_file_list);
  }

  /* unlock loop */
  if (unlock_list)
  {
    lock_unlock_file_list(unlock_list, &full_file_list, MKDB_LOCK_OFF);
  }

  /* lock loop */
  if (lock_list)
  {
    lock_unlock_file_list(lock_list, &full_file_list, MKDB_LOCK_ON);
  }

  /* write the file list back out */
  write_file_list(write_index_file, &full_file_list);
  install_write_file_list(class, auth_area);

  /* now, we can release the lock */
  log(L_LOG_DEBUG, MKDB, "master file write end: %d", (int) getpid());

  release_placeholder_lock(write_index_file, lock_fd);

  dl_list_destroy(&full_file_list);

  return TRUE;
}


void
unlink_file_list(file_list)
  dl_list_type *file_list;
{
  file_struct  *file;
  int          not_done;

  if (!file_list)
  {
    return;
  }

  if (!dl_list_empty(file_list))
  {
    not_done = dl_list_first(file_list);
    while (not_done)
    {
      file = dl_list_value(file_list);

      if (file && file->filename &&
          file_exists(file->filename))
      {
        unlink(file->filename);
      }

      not_done = dl_list_next(file_list);
    }
  }
}


/* find_file_by_id: searches a file list for a file_struct with
     'id'.  Returns NULL if not found. */
file_struct *
find_file_by_id(list, id, type)
  dl_list_type   *list;
  int            id;
  mkdb_file_type type;
{
  int           not_done;
  file_struct   *fi;

  not_done = dl_list_first(list);

  while (not_done)
  {
    fi = dl_list_value(list);
    if (fi && fi->file_no == id && mkdb_file_type_equals(type, fi->type))
    {
      return(fi);
    }

    not_done = dl_list_next(list);
  }

  return NULL;
}


/* find_file_by_name: searches a file list for a file_struct with
   filename 'name'.  Returns NULL if not found. */
file_struct *
find_file_by_name(list, name, type)
  dl_list_type   *list;
  char           *name;
  mkdb_file_type type;
{
  int           not_done;
  file_struct   *fi;

  not_done = dl_list_first(list);

  while (not_done)
  {
    fi = dl_list_value(list);
    if (fi && STR_EQ(fi->filename, name) &&
        mkdb_file_type_equals(type, fi->type))
    {
      return(fi);
    }

    not_done = dl_list_next(list);
  }

  return NULL;
}

long
records_in_auth_area(auth_area)
  auth_area_struct *auth_area;
{
  dl_list_type master_file_list;
  dl_list_type data_file_list;
  file_struct  *fi;
  long         count    = 0;
  int          not_done;

  dl_list_default(&master_file_list, FALSE, destroy_file_struct_data);
  dl_list_default(&data_file_list, FALSE, destroy_file_struct_data);

  if (!get_file(NULL, auth_area->name, &master_file_list))
  {
    return 0;
  }

  filter_file_list(&data_file_list, MKDB_DATA_FILE, &master_file_list);

  not_done = dl_list_first(&data_file_list);
  while (not_done)
  {
    fi = dl_list_value(&data_file_list);
    count += fi->num_recs;

    not_done = dl_list_next(&data_file_list);
  }

  dl_list_destroy(&master_file_list);
  dl_list_destroy(&data_file_list);

  return(count);
}

/* --------------- File List Constructors ------------- */

/* build_base_file_struct: given a path, a type and the number of
     records, allocate and return a resultant file structure. */
file_struct *
build_base_file_struct(file_name, type, num_recs)
  char           *file_name;
  mkdb_file_type type;
  long           num_recs;
{
  struct stat    sb;
  int            status;
  file_struct    *file;

  status = stat(file_name, &sb);

  if (status < 0) {
    /* stat itself failed (file doesn't exist) */
    return NULL;
  }

  file = xcalloc(1, sizeof(*file));

  file->filename  = xstrdup(file_name);
  file->type      = type;
  file->size      = sb.st_size;
  file->num_recs  = num_recs;

  return(file);
}

file_struct *
build_tmp_base_file_struct(tmp_filename, base_template, type, num_recs)
  char           *tmp_filename;
  char           *base_template;
  mkdb_file_type type;
  long           num_recs;
{
  struct stat    sb;
  int            status;
  file_struct    *file;

  status = stat(tmp_filename, &sb);

  if (status < 0) {
    /* stat itself failed (file doesn't exist) */
    return NULL;
  }

  file = xcalloc(1, sizeof(*file));

  file->filename      = NULL;
  file->type          = type;
  file->size          = sb.st_size;
  file->num_recs      = num_recs;
  file->tmp_filename  = xstrdup(tmp_filename);
  if (base_template)
  {
    file->base_filename = xstrdup(base_template);
  }

  return(file);
}


int
build_file_list_by_names(file_list, type, base_dir, num_names, names)
  dl_list_type      *file_list;
  mkdb_file_type    type;
  char              *base_dir;
  int               num_names;
  char              **names;
{
  file_struct   *df;
  char          path[MAX_FILE + 1];
  int           status = FALSE;
  int           i;

  bzero(path, sizeof(path));

  for (i = 0; i < num_names; i++)
  {
    /* since add_file_struct will normalize the path, we just take the
       easy way out and convert it to a full path */
    if (names[i] && *names[i])
    {
      path_rel_to_full(path, MAX_FILE, names[i], base_dir);
      df = build_base_file_struct(path, type, 0);

      if (df)
      {
        dl_list_append(file_list, df);
        status = TRUE;
      }
    }
  }

  return(status);
}

int
build_file_list_by_suffix(file_list, type, base_dir, suffix)
  dl_list_type   *file_list;
  mkdb_file_type type;
  char           *base_dir;
  char           *suffix;
{
  file_struct   *fi;
  DIR           *dir_fp;
  struct dirent *entry;
  char          path[MAX_FILE + 1];
  char          rev_suffix[MAX_FILE];
  char          *d_name;
  int           suffix_len;

  if (!file_list || !base_dir || !*base_dir || !suffix || !*suffix)
  {
    return FALSE;
  }

  bzero(path, sizeof(path));
  suffix_len = strlen(suffix);
  strcpy(rev_suffix, suffix);
  strrev(rev_suffix);

  dir_fp = opendir(base_dir);
  if (!dir_fp)
  {
    log(L_LOG_ERR, MKDB, "could not open directory '%s': %s", base_dir,
              strerror(errno));
    return FALSE;
  }

  while ( (entry = readdir(dir_fp)) != NULL )
  {
    d_name = (char *)(entry->d_name);
    if (!strncmp(rev_suffix, strrev(d_name), suffix_len))
    {
      path_rel_to_full(path, MAX_FILE, strrev(d_name), base_dir);
      fi = build_base_file_struct(path, type, 0);

      if (fi)
      {
        dl_list_append(file_list, fi);
      }
    }
  }

  closedir(dir_fp);
  return TRUE;
}

/* --------------- Destructor Components  ------------- */

int
destroy_file_struct_data(data)
  file_struct   *data;
{
  if (!data)
  {
    return TRUE;
  }

  if (data->filename)
  {
    free(data->filename);
  }

  if (data->tmp_filename)
  {
    free(data->tmp_filename);
  }

  if (data->base_filename)
  {
    free(data->base_filename);
  }

  if (data->fp)
  {
    fclose(data->fp);
  }

  free(data);

  return TRUE;
}


