@@ -17,6 +17,7 @@

 #include "ssi.h"

 #ifdef CLUSTALO

 #include <limits.h>

+#include <inttypes.h>

 #endif

 static sqd_uint32 v20magic = 0xf3f3e9b1; /* SSI 1.0: "ssi1" + 0x80808080 */

@@ -747,9 +748,9 @@ SSIAddPrimaryKeyToIndex(SSIINDEX *g, char *key, int fh,

 	      (unsigned long) L);

     } else {

 #ifdef CLUSTALO

-        fprintf(g->ptmp, "%s\t%d\t%llu\t%llu\t%lu\n", 

+        fprintf(g->ptmp, "%s\t%d\t%" PRIu64 "\t%" PRIu64 "\t%lu\n",

                 key, fh, (unsigned long long)r_off->off.i64, 

-                d_off == NULL? 0 : (unsigned long long) d_off->off.i64, 

+                d_off == NULL ? 0 : (unsigned long long) d_off->off.i64,

 	      (unsigned long) L);

 #else

         fprintf(g->ptmp, "%s\t%d\t%llu\t%llu\t%lu\n", 

@@ -1403,7 +1404,7 @@ activate_external_sort(SSIINDEX *g)

 	      (unsigned long) g->pkeys[i].d_off.off.i32, 

 	      (unsigned long) g->pkeys[i].len);

     } else {

-      fprintf(g->ptmp, "%s\t%u\t%llu\t%llu\t%lu\n", 

+      fprintf(g->ptmp, "%s\t%u\t%" PRIu64 "\t%" PRIu64 "\t%lu\n",

 	      g->pkeys[i].key, g->pkeys[i].fnum,

 	      (unsigned long long) g->pkeys[i].r_off.off.i64, 

 	      (unsigned long long) g->pkeys[i].d_off.off.i64, 

new file mode 100644

@@ -0,0 +1,1537 @@

+/*****************************************************************

+ * SQUID - a library of functions for biological sequence analysis

+ * Copyright (C) 1992-2002 Washington University School of Medicine

+ * 

+ *     This source code is freely distributed under the terms of the

+ *     GNU General Public License. See the files COPYRIGHT and LICENSE

+ *     for details.

+ *****************************************************************/

+

+#include <stdio.h>

+#include <stdlib.h>

+#include <string.h>

+#include <sys/stat.h>

+#include <sys/types.h>

+#include <unistd.h>

+#include "squid.h"

+#include "ssi.h"

+#ifdef CLUSTALO

+#include <limits.h>

+#endif

+

+static sqd_uint32 v20magic = 0xf3f3e9b1; /* SSI 1.0: "ssi1" + 0x80808080 */

+static sqd_uint32 v20swap  = 0xb1e9f3f3; /* byteswapped */

+

+static int read_i16(FILE *fp, sqd_uint16 *ret_result);

+static int read_i32(FILE *fp, sqd_uint32 *ret_result);

+static int read_i64(FILE *fp, sqd_uint64 *ret_result);

+static int read_offset(FILE *fp, char mode, SSIOFFSET *ret_offset);

+static int write_i16(FILE *fp, sqd_uint16 n);

+static int write_i32(FILE *fp, sqd_uint32 n);

+static int write_i64(FILE *fp, sqd_uint64 n);

+static int write_offset(FILE *fp, SSIOFFSET *offset);

+static int binary_search(SSIFILE *sfp, char *key, int klen, SSIOFFSET *base, 

+			 sqd_uint32 recsize, sqd_uint32 maxidx);

+static int indexfile_position(SSIFILE *sfp, SSIOFFSET *base, sqd_uint32 len,

+			      sqd_uint32 n);

+static void clear_ssifile(SSIFILE *sfp);

+static sqd_uint64 current_index_size(SSIINDEX *g);

+static int        activate_external_sort(SSIINDEX *g);

+static int        load_indexfile(SSIFILE *sfp);

+static int        parse_pkey_info(char *buf, char mode, struct ssipkey_s *pkey);

+static int        parse_skey_info(char *buf, struct ssiskey_s *skey);

+

+/* Function: SSIOpen()

+ * Date:     SRE, Sun Dec 31 12:40:03 2000 [St. Louis]

+ *

+ * Purpose:  Opens the SSI index file {filename} and returns

+ *           a SSIFILE * stream thru {ret_sfp}.

+ *           The caller must eventually close this stream using

+ *           SSIClose(). More than one index file can be open

+ *           at once.

+ *

+ * Args:     filename - full path to a SSI index file

+ *

+ * Returns:  Returns 0 on success, nonzero on failure.

+ */

+int

+SSIOpen(char *filename, SSIFILE **ret_sfp)

+{

+  SSIFILE  *sfp = NULL;

+  int       status;

+  if ((sfp = malloc(sizeof(SSIFILE))) == NULL)   return SSI_ERR_MALLOC;

+  if ((sfp->fp = fopen(filename, "rb")) == NULL) {

+    free(sfp);

+    return SSI_ERR_NOFILE;    

+  }

+  status = load_indexfile(sfp);

+  *ret_sfp = sfp;

+  return status;

+}

+/* load_indexfile(): given a SSIFILE structure with an open and positioned 

+ *    stream (fp) -- but no other data loaded -- read the next SSIFILE

+ *    in from disk. We use this routine without its SSIOpen() wrapper

+ *    as part of the external mergesort when creating large indices.

+ */

+static int

+load_indexfile(SSIFILE *sfp)

+{

+  sqd_uint32   magic;

+  sqd_uint16   i;		/* counter over files */

+  int          status;		/* overall return status if an error is thrown */

+

+  status = SSI_ERR_BADFORMAT; /* default: almost every kind of error is a bad format error */

+

+  sfp->filename   = NULL;

+  sfp->fileformat = NULL;

+  sfp->fileflags  = NULL;

+  sfp->bpl        = NULL;

+  sfp->rpl        = NULL;

+  sfp->nfiles     = 0;          

+  if (! read_i32(sfp->fp, &magic))               {status = SSI_ERR_BADMAGIC;  goto FAILURE; }

+  if (magic != v20magic && magic != v20swap)     {status = SSI_ERR_BADMAGIC;  goto FAILURE; }

+  if (! read_i32(sfp->fp, &(sfp->flags))) goto FAILURE; 

+

+  /* If we have 64-bit offsets, make sure we can deal with them.

+   */

+#ifndef HAS_64BIT_FILE_OFFSETS  

+  if ((sfp->flags & SSI_USE64_INDEX) ||

+      (sfp->flags & SSI_USE64))

+    { status = SSI_ERR_NO64BIT; goto FAILURE; }

+#endif

+

+  sfp->imode = (sfp->flags & SSI_USE64_INDEX) ? SSI_OFFSET_I64 : SSI_OFFSET_I32;

+  sfp->smode = (sfp->flags & SSI_USE64) ?       SSI_OFFSET_I64 : SSI_OFFSET_I32;

+

+  if (! read_i16(sfp->fp, &(sfp->nfiles)))     goto FAILURE;

+  if (! read_i32(sfp->fp, &(sfp->nprimary)))   goto FAILURE;

+  if (! read_i32(sfp->fp, &(sfp->nsecondary))) goto FAILURE;

+  if (! read_i32(sfp->fp, &(sfp->flen)))       goto FAILURE;

+  if (! read_i32(sfp->fp, &(sfp->plen)))       goto FAILURE;

+  if (! read_i32(sfp->fp, &(sfp->slen)))       goto FAILURE;

+  if (! read_i32(sfp->fp, &(sfp->frecsize)))   goto FAILURE;

+  if (! read_i32(sfp->fp, &(sfp->precsize)))   goto FAILURE;

+  if (! read_i32(sfp->fp, &(sfp->srecsize)))   goto FAILURE;

+  

+  if (! read_offset(sfp->fp, sfp->imode, &(sfp->foffset))) goto FAILURE;

+  if (! read_offset(sfp->fp, sfp->imode, &(sfp->poffset))) goto FAILURE;

+  if (! read_offset(sfp->fp, sfp->imode, &(sfp->soffset))) goto FAILURE;

+

+  /* Read the file information and keep it.

+   * We expect the number of files to be small, so reading it

+   * once should be advantageous overall. If SSI ever had to

+   * deal with large numbers of files, you'd probably want to

+   * read file information on demand.

+   */

+  if (sfp->nfiles == 0)                                                   goto FAILURE;

+  if ((sfp->filename=malloc(sizeof(char *)    *sfp->nfiles)) == NULL)   {status = SSI_ERR_MALLOC; goto FAILURE; }

+  for (i = 0; i < sfp->nfiles; i++) sfp->filename[i] = NULL; 

+  if ((sfp->fileformat=malloc(sizeof(sqd_uint32)*sfp->nfiles)) == NULL) {status = SSI_ERR_MALLOC; goto FAILURE; }

+  if ((sfp->fileflags =malloc(sizeof(sqd_uint32)*sfp->nfiles)) == NULL) {status = SSI_ERR_MALLOC; goto FAILURE; }

+  if ((sfp->bpl     =malloc(sizeof(sqd_uint32)*sfp->nfiles)) == NULL)   {status = SSI_ERR_MALLOC; goto FAILURE; }

+  if ((sfp->rpl     =malloc(sizeof(sqd_uint32)*sfp->nfiles)) == NULL)   {status = SSI_ERR_MALLOC; goto FAILURE; }

+

+  for (i = 0; i < sfp->nfiles; i++) 

+    {

+      /* We have to explicitly position, because header and file 

+       * records may expand in the future; frecsize and foffset 

+       * give us forwards compatibility. 

+       */ 

+      if (indexfile_position(sfp, &(sfp->foffset), sfp->frecsize, i) !=0)  goto FAILURE;

+      if ((sfp->filename[i] =malloc(sizeof(char)*sfp->flen)) == NULL)        {status = SSI_ERR_MALLOC; goto FAILURE; }

+      if (fread(sfp->filename[i],sizeof(char),sfp->flen, sfp->fp)!=sfp->flen) goto FAILURE;

+      if (! read_i32(sfp->fp, &(sfp->fileformat[i])))                             goto FAILURE;

+      if (! read_i32(sfp->fp, &(sfp->fileflags[i])))                              goto FAILURE;

+      if (! read_i32(sfp->fp, &(sfp->bpl[i])))                                    goto FAILURE;

+      if (! read_i32(sfp->fp, &(sfp->rpl[i])))                                    goto FAILURE;

+    }

+  

+  /* Success. Return 0.

+   */

+  return 0;			

+

+ FAILURE:

+  /* Failure: free the damaged structure, return status code.

+   */

+  SSIClose(sfp);

+  return status;

+}

+

+

+

+/* Function: SSIGetOffsetByName()

+ * Date:     SRE, Sun Dec 31 13:55:31 2000 [St. Louis]

+ *

+ * Purpose:  Looks up the string {key} in the open index {sfp}.

+ *           {key} can be either a primary or secondary key. If {key}

+ *           is found, {*ret_fh} contains a unique handle on

+ *           the file that contains {key} (suitable for an SSIFileInfo()

+ *           call, or for comparison to the handle of the last file

+ *           that was opened for retrieval), and {offset} is filled 

+ *           in with the offset in that file.

+ *           

+ * Args:     sfp         - open index file

+ *           key         - string to search for

+ *           ret_fh      - RETURN: handle on file that key is in

+ *           ret_offset  - RETURN: offset of the start of that key's record

+ *

+ * Returns:  0 on success.

+ *           non-zero on error.

+ */

+int

+SSIGetOffsetByName(SSIFILE *sfp, char *key, int *ret_fh,

+		   SSIOFFSET *ret_offset)

+{

+  int         status;

+  sqd_uint16  fnum;

+

+  /* Look in the primary keys.

+   */

+  status = binary_search(sfp, key, sfp->plen, &(sfp->poffset), sfp->precsize,

+			 sfp->nprimary);

+  if (status == 0) {		

+    /* We found it as a primary key; get our data & return.

+     */

+    if (! read_i16(sfp->fp, &fnum)) return SSI_ERR_NODATA;

+    *ret_fh = (int) fnum;

+    if (! read_offset(sfp->fp, sfp->smode, ret_offset))  return SSI_ERR_NODATA;

+

+    return 0;	/* success! (we don't need the other key data) */

+  } else if (status == SSI_ERR_NO_SUCH_KEY) {

+    /* Not in the primary keys? OK, try the secondary keys.

+     */

+    if (sfp->nsecondary > 0) {

+      char *pkey;

+      status = binary_search(sfp, key, sfp->slen, &(sfp->soffset), sfp->srecsize,

+			     sfp->nsecondary);

+      if (status != 0) return status;

+      if ((pkey = malloc(sizeof(char) * sfp->plen)) == NULL) return SSI_ERR_MALLOC;

+      if (fread(pkey, sizeof(char), sfp->plen, sfp->fp) != sfp->plen) return SSI_ERR_NODATA;

+

+      status = SSIGetOffsetByName(sfp, pkey, ret_fh, ret_offset);

+      free(pkey);

+    }

+    return status;

+

+  } else return status;		

+  /*NOTREACHED*/

+}

+

+/* Function: SSIGetOffsetByNumber()

+ * Date:     SRE, Mon Jan  1 19:42:42 2001 [St. Louis]

+ *

+ * Purpose:  Looks up primary key #{n} in the open index {sfp}.

+ *           {n} ranges from 0..nprimary-1. When key #{n} 

+ *           is found, {*ret_fh} contains a unique 

+ *           handle on the file that contains {key} (suitable

+ *           for an SSIFileInfo() call, or for comparison to 

+ *           the handle of the last file that was opened for retrieval),

+ *           and {offset} is filled in with the offset in that file.

+ *           

+ * Args:     sfp        - open index file

+ *           n          - primary key number to retrieve.

+ *           ret_fh     - RETURN: handle on file that key is in

+ *           ret_offset - RETURN: offset of the start of that key's record

+ *

+ * Returns:  0 on success.

+ *           non-zero on error.

+ */

+int

+SSIGetOffsetByNumber(SSIFILE *sfp, int n, int *ret_fh, SSIOFFSET *ret_offset)

+{

+  sqd_uint16 fnum;

+  char      *pkey;

+

+  if (n >= sfp->nprimary) return SSI_ERR_NO_SUCH_KEY;

+  if (indexfile_position(sfp, &(sfp->poffset), sfp->precsize, n) != 0) 

+    return SSI_ERR_SEEK_FAILED;

+

+  if ((pkey = malloc(sizeof(char) * sfp->plen)) == NULL) return SSI_ERR_MALLOC;

+  if (fread(pkey, sizeof(char), sfp->plen, sfp->fp) != sfp->plen) return SSI_ERR_NODATA;

+  if (! read_i16(sfp->fp, &fnum))                      return SSI_ERR_NODATA;

+  if (! read_offset(sfp->fp, sfp->smode, ret_offset))  return SSI_ERR_NODATA;  

+  *ret_fh = fnum;

+  free(pkey);

+  return 0;

+}

+

+/* Function: SSIGetSubseqOffset()

+ * Date:     SRE, Mon Jan  1 19:49:31 2001 [St. Louis]

+ *

+ * Purpose:  Implements SSI_FAST_SUBSEQ.

+ * 

+ *           Looks up a primary or secondary {key} in the open

+ *           index {sfp}. Asks for the nearest offset to a

+ *           subsequence starting at position {requested_start}

+ *           in the sequence (numbering the sequence 1..L). 

+ *           If {key} is found, on return, {ret_fh}

+ *           contains a unique handle on the file that contains 

+ *           {key} (suitable for an SSIFileInfo() call, or for 

+ *           comparison to the handle of the last file that was 

+ *           opened for retrieval); {record_offset} contains the

+ *           disk offset to the start of the record; {data_offset}

+ *           contains the disk offset either exactly at the requested

+ *           residue, or at the start of the line containing the

+ *           requested residue; {ret_actual_start} contains the 

+ *           coordinate (1..L) of the first valid residue at or

+ *           after {data_offset}. {ret_actual_start} is <= 

+ *           {requested_start}. 

+ *

+ * Args:     sfp             - open index file

+ *           key             - primary or secondary key to find

+ *           requested_start - residue we'd like to start at (1..L)

+ *           ret_fh          - RETURN: handle for file the key is in

+ *           record_offset   - RETURN: offset of entire record

+ *           data_offset     - RETURN: offset of subseq (see above)

+ *           ret_actual_start- RETURN: coord (1..L) of residue at data_offset

+ *

+ * Returns:  0 on success, non-zero on failure.

+ */

+int

+SSIGetSubseqOffset(SSIFILE *sfp, char *key, int requested_start,

+		    int *ret_fh, SSIOFFSET *record_offset,

+		    SSIOFFSET *data_offset, int *ret_actual_start)

+{

+  int        status;

+  sqd_uint32 len;

+  int        r, b, i, l;	/* tmp variables for "clarity", to match docs */

+  

+  /* Look up the key. Rely on the fact that SSIGetOffsetByName()

+   * leaves the index file positioned at the rest of the data for this key.

+   */

+  status = SSIGetOffsetByName(sfp, key, ret_fh, record_offset);

+  if (status != 0) return status;

+

+  /* Check that we're allowed to do subseq lookup on that file.

+   */

+  if (! (sfp->fileflags[*ret_fh] & SSI_FAST_SUBSEQ))

+    return SSI_ERR_NO_SUBSEQS;

+

+  /* Read the data we need for subseq lookup

+   */

+  if (! read_offset(sfp->fp, sfp->smode, data_offset)) return SSI_ERR_NODATA;

+  if (! read_i32(sfp->fp, &len))                         return SSI_ERR_NODATA;

+

+  /* Set up tmp variables for clarity of equations below,

+   * and to make them match documentation (ssi-format.tex).

+   */

+  r = sfp->rpl[*ret_fh];    /* residues per line */

+  b = sfp->bpl[*ret_fh];    /* bytes per line    */

+  i = requested_start;	    /* start position 1..L */

+  l = (i-1)/r;		    /* data line # (0..) that the residue is on */

+  if (r == 0 || b == 0) return SSI_ERR_NO_SUBSEQS;

+  if (i < 0 || i > len) return SSI_ERR_RANGE;

+  

+  /* When b = r+1, there's nothing but sequence on each data line (and the \0),

+   * and we can find each residue precisely.

+   */

+  if (b == r+1) {

+    if (sfp->smode == SSI_OFFSET_I32) {

+      data_offset->mode    = SSI_OFFSET_I32;

+      data_offset->off.i32 = data_offset->off.i32 + l*b + (i-1)%r;

+    } else if (sfp->smode == SSI_OFFSET_I64) {

+      data_offset->mode    = SSI_OFFSET_I64;

+      data_offset->off.i64 = data_offset->off.i64 + l*b + (i-1)%r;

+    } 

+    *ret_actual_start = requested_start;

+  } else { 

+    /* else, there's other stuff on seq lines, so the best

+     * we can do easily is to position at start of relevant line.

+     */

+    if (sfp->smode == SSI_OFFSET_I32) {

+      data_offset->mode    = SSI_OFFSET_I32;

+      data_offset->off.i32 = data_offset->off.i32 + l*b;

+    } else if (sfp->smode == SSI_OFFSET_I64) {

+      data_offset->mode    = SSI_OFFSET_I64;

+      data_offset->off.i64 = data_offset->off.i64 + l*b;

+    } 

+    /* yes, the eq below is = 1 + (i-1)/r*r but it's not = i. that's an integer /. */

+    *ret_actual_start = 1 + l*r;

+  }

+  return 0;

+}

+

+/* Function: SSISetFilePosition()

+ * Date:     SRE, Tue Jan  2 09:13:46 2001 [St. Louis]

+ *

+ * Purpose:  Uses {offset} to sets the file position for {fp}, usually an

+ *           open sequence file, relative to the start of the file.

+ *           Hides the details of system-dependent shenanigans necessary for

+ *           file positioning in large (>2 GB) files. 

+ *           

+ *           Behaves just like fseek(fp, offset, SEEK_SET) for 32 bit

+ *           offsets and <2 GB files.

+ *           

+ *           Warning: if all else fails, in desperation, it will try to

+ *           use fsetpos(). This requires making assumptions about fpos_t

+ *           that may be unwarranted... assumptions that ANSI C prohibits

+ *           me from making... though I believe the ./configure

+ *           script robustly tests whether I can play with fpos_t like this.

+ *

+ * Args:     fp      - file to position.

+ *           offset  - SSI offset relative to file start.

+ *                 

+ * Returns:  0 on success, nonzero on error.

+ */

+int

+SSISetFilePosition(FILE *fp, SSIOFFSET *offset)

+{

+  if (offset->mode == SSI_OFFSET_I32) {

+    if (fseek(fp, offset->off.i32, SEEK_SET) != 0)       return SSI_ERR_SEEK_FAILED;

+  }

+#ifndef HAS_64BIT_FILE_OFFSETS

+  else return SSI_ERR_NO64BIT;

+#elif defined HAVE_FSEEKO && SIZEOF_OFF_T == 8

+  else if (fseeko(fp, offset->off.i64, SEEK_SET) != 0)   return SSI_ERR_SEEK_FAILED;

+#elif defined HAVE_FSEEKO64 && SIZEOF_OFF64_T == 8

+  else if (fseeko64(fp, offset->off.i64, SEEK_SET) != 0) return SSI_ERR_SEEK_FAILED;

+#elif defined HAVE_FSEEK64

+  else if (fseek64(fp, offset->off.i64, SEEK_SET) != 0)  return SSI_ERR_SEEK_FAILED;

+#elif defined ARITHMETIC_FPOS_T && SIZEOF_FPOS_T == 8

+  else if (fsetpos(fp, &(offset->off.i64)) != 0)         return SSI_ERR_SEEK_FAILED;

+#endif

+  return 0;

+}

+

+

+/* Function: SSIFileInfo()

+ * Date:     SRE, Tue Jan  2 10:31:01 2001 [St. Louis]

+ *

+ * Purpose:  Given a file number {fh} in an open index file

+ *           {sfp}, retrieve file name {ret_filename} and

+ *           the file format {ret_format}. 

+ *           

+ *           {ret_filename} is a pointer to a string maintained

+ *           internally by {sfp}. It should not be free'd; 

+ *           SSIClose(sfp) takes care of it.

+ *

+ * Args:     sfp          - open index file

+ *           fh           - handle on file to look up

+ *           ret_filename - RETURN: name of file n

+ *           ret_format   - RETURN: format of file n

+ *

+ * Returns:  0 on success, nonzero on failure.

+ */

+int

+SSIFileInfo(SSIFILE *sfp, int fh, char **ret_filename, int *ret_format)

+{

+  if (fh < 0 || fh >= sfp->nfiles) return SSI_ERR_BADARG;

+  *ret_filename = sfp->filename[fh];

+  *ret_format   = sfp->fileformat[fh];

+  return 0;

+}

+

+/* Function: SSIClose()

+ * Date:     SRE, Sun Dec 31 14:56:37 2000 [St. Louis]

+ *

+ * Purpose:  Close an open {SSIFILE *}.

+ *

+ * Args:     sfp - index file to close.

+ *

+ * Returns:  (void)

+ */

+void

+SSIClose(SSIFILE *sfp) 

+{

+  if (sfp != NULL) {

+    clear_ssifile(sfp);

+    if (sfp->fp       != NULL) fclose(sfp->fp);

+    free(sfp);

+  }

+}  

+/* clear_ssifile(): free the innards of SSIFILE, without 

+ * destroying the structure or closing the stream.

+ */

+static void

+clear_ssifile(SSIFILE *sfp)

+{

+  int i;

+

+  if (sfp->filename != NULL) {

+    for (i = 0; i < sfp->nfiles; i++) 

+      if (sfp->filename[i] != NULL) free(sfp->filename[i]);

+    free(sfp->filename);

+  }

+  if (sfp->fileformat   != NULL) free(sfp->fileformat);

+  if (sfp->fileflags    != NULL) free(sfp->fileflags);

+  if (sfp->bpl          != NULL) free(sfp->bpl);

+  if (sfp->rpl          != NULL) free(sfp->rpl);

+}

+  

+

+/* Function: SSIRecommendMode()

+ * Date:     SRE, Fri Feb 16 08:23:47 2001 [St. Louis]

+ *

+ * Purpose:  Examines the file and determines whether it should be

+ *           indexed with large file support or not; returns 

+ *           SSI_OFFSET_I32 for most files, SSI_OFFSET_I64 for large

+ *           files, or -1 on failure.

+ *

+ * Args:     file - name of file to check for size

+ *

+ * Returns:  -1 on failure (including case where file is too big)

+ *           SSI_OFFSET_I32 for most files (<= 2^31-1 bytes)

+ *           SSI_OFFSET_I64 for large files (> 2^31-1 bytes)

+ */

+int

+SSIRecommendMode(char *file)

+{

+#if HAVE_STAT64

+  struct stat64 s1;

+  if (stat64(file, &s1) == 0) {

+    if (s1.st_size <= 2146483647L) return SSI_OFFSET_I32;

+    else                           return SSI_OFFSET_I64;

+  }

+#else 

+  struct stat s2;

+  if (stat(file, &s2) == 0) {

+    if (s2.st_size <= 2146483647L) return SSI_OFFSET_I32;

+    else                           return SSI_OFFSET_I64;

+  }

+#endif

+  return -1;

+}

+ 

+

+/* Function: SSICreateIndex()

+ * Date:     SRE, Tue Jan  2 11:23:25 2001 [St. Louis]

+ *

+ * Purpose:  Creates and initializes a SSI index structure. 

+ *           Sequence file offset type is specified by {mode}.

+ *

+ * Args:     mode    - SSI_OFFSET_I32 or SSI_OFFSET_I64, sequence file index mode.

+ *

+ * Returns:  ptr to new index structure, or NULL on failure.

+ *           Caller is responsible for free'ing the returned

+ *           structure with SSIFreeIndex().

+ */

+SSIINDEX *

+SSICreateIndex(int mode)

+{

+  SSIINDEX *g;

+

+  g = NULL;

+  if ((g = malloc(sizeof(SSIINDEX))) == NULL)                               goto FAILURE;

+  g->smode    = mode;

+  g->imode    = SSI_OFFSET_I32;	/* index always starts as 32-bit; may get upgraded later */

+  g->external = FALSE;

+  g->max_ram  = SSI_MAXRAM;

+

+#ifndef HAS_64BIT_FILE_OFFSETS

+  if (mode == SSI_OFFSET_I64) 

+    Die("\

+Can't create a 64-bit SSI index on this system, sorry;\n\

+I don't have 64-bit file offset functions available.\n");

+#endif

+

+  g->filenames  = NULL;

+  g->fileformat = NULL;

+  g->bpl        = NULL;

+  g->rpl        = NULL;

+  g->flen       = 0;

+  g->nfiles     = 0;

+

+  g->pkeys         = NULL;

+  g->plen          = 0;

+  g->nprimary      = 0;

+  g->ptmpfile      = "tmp.ssi.1"; /* hardcoded, for now. */

+  g->ptmp          = NULL;

+  

+  g->skeys         = NULL;

+  g->slen          = 0;

+  g->nsecondary    = 0;

+  g->stmpfile      = "tmp.ssi.2"; /* hardcoded, for now. */

+  g->stmp          = NULL;

+

+  /* All mallocs must go after NULL initializations, because of the cleanup strategy;

+   * we'll try to free anything non-NULL if a malloc fails.

+   */

+  if ((g->filenames = malloc(sizeof(char *)     * SSI_FILE_BLOCK)) == NULL) goto FAILURE;

+  if ((g->fileformat= malloc(sizeof(sqd_uint32) * SSI_FILE_BLOCK)) == NULL) goto FAILURE; 

+  if ((g->bpl       = malloc(sizeof(sqd_uint32) * SSI_FILE_BLOCK)) == NULL) goto FAILURE; 

+  if ((g->rpl       = malloc(sizeof(sqd_uint32) * SSI_FILE_BLOCK)) == NULL) goto FAILURE; 

+  

+  if ((g->pkeys = malloc(sizeof(struct ssipkey_s)* SSI_KEY_BLOCK))== NULL)  goto FAILURE;

+  if ((g->skeys = malloc(sizeof(struct ssipkey_s)* SSI_KEY_BLOCK))== NULL)  goto FAILURE;

+

+  return g;

+

+ FAILURE:

+  SSIFreeIndex(g);		/* free the damaged structure */

+  return NULL;

+}

+

+/* Function: SSIGetFilePosition()

+ * Date:     SRE, Tue Jan  2 09:59:26 2001 [St. Louis]

+ *

+ * Purpose:  Fills {ret_offset} with the current disk

+ *           offset of {fp}, relative to the start of the file. 

+ *           {mode} is set to either SSI_OFFSET_I32 or 

+ *           SSI_OFFSET_I64. If {mode} is _I32 (32 bit), just wraps

+ *           a call to ftell(); otherwise deals with system-dependent

+ *           details of 64-bit file offsets.

+ *

+ * Args:     fp         - open stream

+ *           mode       - SSI_OFFSET_I32 or SSI_OFFSET_I64

+ *           ret_offset - RETURN: file position       

+ *

+ * Returns:  0 on success. nonzero on error.

+ */

+int 

+SSIGetFilePosition(FILE *fp, int mode, SSIOFFSET *ret_offset)

+{

+  if (mode == SSI_OFFSET_I32) 

+    {

+      ret_offset->mode    = SSI_OFFSET_I32;

+      ret_offset->off.i32 = ftell(fp);

+      if (ret_offset->off.i32 == -1) return SSI_ERR_TELL_FAILED;

+    }

+  else if (mode != SSI_OFFSET_I64) abort(); /* only happens on a coding error */

+  else {

+    ret_offset->mode    = SSI_OFFSET_I64;

+#ifndef HAS_64BIT_FILE_OFFSETS

+    return SSI_ERR_NO64BIT;

+#elif defined HAVE_FTELLO && SIZEOF_OFF_T == 8

+    if ((ret_offset->off.i64 = ftello(fp)) == -1)   return SSI_ERR_TELL_FAILED;

+#elif defined HAVE_FTELLO64 && SIZEOF_OFF64_T == 8

+    if ((ret_offset->off.i64 = ftello64(fp)) == -1) return SSI_ERR_TELL_FAILED;

+#elif defined HAVE_FTELL64

+    if ((ret_offset->off.i64 = ftell64(fp)) == -1)  return SSI_ERR_TELL_FAILED;

+#elif defined ARITHMETIC_FPOS_T && SIZEOF_FPOS_T == 8

+    if (fgetpos(fp, &(ret_offset->off.i64)) != 0)   return SSI_ERR_TELL_FAILED;

+#endif

+  }

+  return 0;

+}

+

+/* Function: SSIAddFileToIndex()

+ * Date:     SRE, Tue Jan  2 12:54:36 2001 [St. Louis]

+ *

+ * Purpose:  Adds the sequence file {filename}, which is known to 

+ *           be in format {fmt}, to the index {g}. Creates and returns

+ *           a unique filehandle {fh} for then associating primary keys

+ *           with this file using SSIAddPrimaryKeyToIndex().

+ *

+ * Args:     g         - active index

+ *           filename  - file to add 

+ *           fmt       - format code for this file (e.g. SQFILE_FASTA)

+ *           ret_fh    - RETURN: unique handle for this file

+ *

+ * Returns:  0 on success; nonzero on error.

+ */

+int

+SSIAddFileToIndex(SSIINDEX *g, char *filename, int fmt, int *ret_fh)

+{

+  int n;

+  

+  if (g->nfiles >= SSI_MAXFILES) return SSI_ERR_TOOMANY_FILES;

+

+  n = strlen(filename);

+  if ((n+1) > g->flen) g->flen = n+1;

+

+  g->filenames[g->nfiles]  = FileTail(filename, FALSE);

+  g->fileformat[g->nfiles] = fmt;

+  g->bpl[g->nfiles]        = 0;

+  g->rpl[g->nfiles]        = 0;

+  *ret_fh                  = g->nfiles;   /* handle is simply = file number */

+  g->nfiles++;

+

+  if (g->nfiles % SSI_FILE_BLOCK == 0) {

+    g->filenames = realloc(g->filenames,  sizeof(char *) * (g->nfiles+SSI_FILE_BLOCK));

+    if (g->filenames == NULL) return SSI_ERR_MALLOC;

+    g->fileformat= realloc(g->fileformat, sizeof(sqd_uint32) * (g->nfiles+SSI_FILE_BLOCK));

+    if (g->fileformat == NULL) return SSI_ERR_MALLOC;

+    g->bpl       = realloc(g->bpl,        sizeof(sqd_uint32) * (g->nfiles+SSI_FILE_BLOCK));

+    if (g->bpl == NULL) return SSI_ERR_MALLOC;

+    g->rpl       = realloc(g->rpl,        sizeof(sqd_uint32) * (g->nfiles+SSI_FILE_BLOCK));

+    if (g->rpl == NULL) return SSI_ERR_MALLOC;

+  }

+  return 0;

+}

+

+

+/* Function: SSISetFileForSubseq()

+ * Date:     SRE, Tue Jan  9 10:02:05 2001 [St. Louis]

+ *

+ * Purpose:  Set SSI_FAST_SUBSEQ for the file indicated by

+ *           filehandle {fh} in the index {g}, setting

+ *           parameters {bpl} and {rpl} to the values given.

+ *           {bpl} is the number of bytes per sequence data line.

+ *           {rpl} is the number of residues per sequence data line. 

+ *           Caller must be sure that {bpl} and {rpl} do not change

+ *           on any line of any sequence record in the file

+ *           (except for the last data line of each record). If

+ *           this is not the case in this file, SSI_FAST_SUBSEQ

+ *           will not work, and this routine should not be

+ *           called.

+ *

+ * Args:     g    - the active index

+ *           fh   - handle for file to set SSI_FAST_SUBSEQ on

+ *           bpl  - bytes per data line

+ *           rpl  - residues per data line

+ *

+ * Returns:  0 on success; 1 on error.

+ */

+int

+SSISetFileForSubseq(SSIINDEX *g, int fh, int bpl, int rpl)

+{

+  if (fh < 0 || fh >= g->nfiles) return SSI_ERR_BADARG;

+  if (bpl <= 0 || rpl <= 0)      return SSI_ERR_BADARG;

+  g->bpl[fh] = bpl;

+  g->rpl[fh] = rpl;

+  return 0;

+}

+

+

+/* Function: SSIAddPrimaryKeyToIndex()

+ * Date:     SRE, Tue Jan  2 11:50:54 2001 [St. Louis]

+ *

+ * Purpose:  Put primary key {key} in the index {g}, while telling

+ *           the index this primary key is in the file associated

+ *           with filehandle {fh} (returned by a previous call

+ *           to SSIAddFileToIndex()), and its record starts at 

+ *           position {r_off} in the file.

+ *           

+ *           {d_off} and {L} are optional; they may be left unset

+ *           by passing NULL and 0, respectively. (If one is

+ *           provided, both must be provided.) If they are provided,

+ *           {d_off} gives the position of the first line of sequence

+ *           data in the record, and {L} gives the length of

+ *           the sequence in residues. They are used when 

+ *           SSI_FAST_SUBSEQ is set for this file. If SSI_FAST_SUBSEQ

+ *           is not set for the file, {d_off} and {L} will be

+ *           ignored by the index reading API even if they are stored

+ *           by the index writing API, so it doesn't hurt for the 

+ *           indexing program to provide them; typically they

+ *           won't know whether it's safe to set SSI_FAST_SUBSEQ

+ *           for the whole file until the whole file has been

+ *           read and every key has already been added to the index.

+ *           

+ * Args:     g      - active index

+ *           key    - primary key to add

+ *           fh     - handle on file that this key's in 

+ *           r_off  - offset to start of record

+ *           d_off  - offset to start of sequence data

+ *           L      - length of sequence, or 0

+ *

+ * Returns:  0 on success, nonzero on error.

+ */

+int

+SSIAddPrimaryKeyToIndex(SSIINDEX *g, char *key, int fh,

+			SSIOFFSET *r_off, SSIOFFSET *d_off, int L)

+{

+  int n;			/* a string length */

+  

+  if (fh >= SSI_MAXFILES)         return SSI_ERR_TOOMANY_FILES;

+  if (g->nprimary >= SSI_MAXKEYS) return SSI_ERR_TOOMANY_KEYS;

+  if (L > 0 && d_off == NULL) abort(); /* need both. */

+

+  /* Before adding the key: check how big our index is.

+   * If it's getting too large, switch to external mode.

+   */

+  if (!g->external && current_index_size(g) >= g->max_ram) 

+    if (activate_external_sort(g) != 0)  return SSI_ERR_NOFILE;

+

+  /* Update maximum pkey length, if needed.

+   */

+  n = strlen(key);

+  if ((n+1) > g->plen) g->plen = n+1;

+

+  /* External mode? Simply append to disk...

+   */

+  if (g->external) {

+    if (g->smode == SSI_OFFSET_I32) {

+      fprintf(g->ptmp, "%s\t%d\t%lu\t%lu\t%lu\n", 

+	      key, fh, (unsigned long) r_off->off.i32, 

+	      (unsigned long) (d_off == NULL? 0 : d_off->off.i32),

+	      (unsigned long) L);

+    } else {

+#ifdef CLUSTALO

+        fprintf(g->ptmp, "%s\t%d\t%llu\t%llu\t%lu\n", 

+                key, fh, (unsigned long long)r_off->off.i64, 

+                d_off == NULL? 0 : (unsigned long long) d_off->off.i64, 

+	      (unsigned long) L);

+#else

+        fprintf(g->ptmp, "%s\t%d\t%llu\t%llu\t%lu\n", 

+	      key, fh, r_off->off.i64, 

+	      d_off == NULL? 0 : d_off->off.i64, 

+	      (unsigned long) L);

+#endif

+    }

+    g->nprimary++;

+    return 0;

+  }

+

+  /* Else: internal mode, keep keys in memory...

+   */

+  if ((g->pkeys[g->nprimary].key = sre_strdup(key, n)) == NULL) return SSI_ERR_MALLOC;

+  g->pkeys[g->nprimary].fnum  = (sqd_uint16) fh;

+  g->pkeys[g->nprimary].r_off = *r_off;

+  if (d_off != NULL && L > 0) {

+    g->pkeys[g->nprimary].d_off = *d_off;

+    g->pkeys[g->nprimary].len   = L;

+  } else {

+	/* yeah, this looks stupid, but look: we have to give a valid

+           looking, non-NULL d_off of some sort, or writes will fail. 

+           It's going to be unused anyway. */

+    g->pkeys[g->nprimary].d_off = *r_off;

+    g->pkeys[g->nprimary].len   = 0;

+  }

+  g->nprimary++;

+

+  if (g->nprimary % SSI_KEY_BLOCK == 0) {

+    g->pkeys = realloc(g->pkeys, sizeof(struct ssipkey_s) * (g->nprimary+SSI_KEY_BLOCK));

+    if (g->pkeys == NULL) return SSI_ERR_MALLOC;

+  }

+  return 0;

+}

+

+

+/* Function: SSIAddSecondaryKeyToIndex()

+ * Date:     SRE, Tue Jan  2 12:44:40 2001 [St. Louis]

+ *

+ * Purpose:  Puts secondary key {key} in the index {g}, associating

+ *           it with primary key {pkey} that was previously

+ *           registered by SSIAddPrimaryKeyToIndex().

+ *

+ * Args:     g    - active index 

+ *           key  - secondary key to add             

+ *           pkey - primary key to associate this key with

+ *

+ * Returns:  0 on success, nonzero on failure.

+ */

+int

+SSIAddSecondaryKeyToIndex(SSIINDEX *g, char *key, char *pkey)

+{

+  int n;			/* a string length */

+  

+  if (g->nsecondary >= SSI_MAXKEYS) return SSI_ERR_TOOMANY_KEYS;

+

+  /* Before adding the key: check how big our index is.

+   * If it's getting too large, switch to external mode.

+   */

+  if (!g->external && current_index_size(g) >= g->max_ram) 

+    if (activate_external_sort(g) != 0)  return SSI_ERR_NOFILE;

+

+  /* Update maximum secondary key length, if necessary.

+   */

+  n = strlen(key);

+  if ((n+1) > g->slen) g->slen = n+1;

+

+  /* if external mode: write info to disk.

+   */

+  if (g->external) {

+    fprintf(g->stmp, "%s\t%s\n", key, pkey);

+    g->nsecondary++;

+    return 0;

+  }

+

+  /* else, internal mode... store info in memory.

+   */

+  if ((g->skeys[g->nsecondary].key  = sre_strdup(key, n))   == NULL) return SSI_ERR_MALLOC;

+  if ((g->skeys[g->nsecondary].pkey = sre_strdup(pkey, -1)) == NULL) return SSI_ERR_MALLOC;

+  g->nsecondary++;

+

+  if (g->nsecondary % SSI_KEY_BLOCK == 0) {

+    g->skeys = realloc(g->skeys, sizeof(struct ssiskey_s) * (g->nsecondary+SSI_KEY_BLOCK));

+    if (g->skeys == NULL) return SSI_ERR_MALLOC;

+  }

+  return 0;

+}

+

+

+

+

+/* Function: SSIWriteIndex()

+ * Date:     SRE, Tue Jan  2 13:55:56 2001 [St. Louis]

+ *

+ * Purpose:  Writes complete index {g} in SSI format to a 

+ *           binary file {file}. Does all           

+ *           the overhead of sorting the primary and secondary keys,