DatabaseDocument.java

/* DatabaseDocument.java
 *
 * created: 2005
 *
 * This file is part of Artemis
 * 
 * Copyright (C) 2005  Genome Research Limited
 * 
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 *
 */

package uk.ac.sanger.artemis.util;

import uk.ac.sanger.artemis.Options;
import uk.ac.sanger.artemis.io.ChadoCanonicalGene;
import uk.ac.sanger.artemis.io.DocumentEntry;
import uk.ac.sanger.artemis.io.GFF3Encoder;
import uk.ac.sanger.artemis.io.GFFStreamFeature;
import uk.ac.sanger.artemis.io.PartialSequence;
import uk.ac.sanger.artemis.io.Range;
import uk.ac.sanger.artemis.io.ReadFormatException;
import uk.ac.sanger.artemis.chado.ArtemisUtils;
import uk.ac.sanger.artemis.chado.ChadoCvTermView;
import uk.ac.sanger.artemis.chado.ChadoTransactionManager;
import uk.ac.sanger.artemis.chado.FeatureForUpdatingResidues;
import uk.ac.sanger.artemis.chado.IBatisDAO;
import uk.ac.sanger.artemis.chado.JdbcDAO;
import uk.ac.sanger.artemis.chado.GmodDAO;
import uk.ac.sanger.artemis.chado.ChadoTransaction;
import uk.ac.sanger.artemis.components.database.DatabaseEntrySource;
import uk.ac.sanger.artemis.components.genebuilder.GeneUtils;
import uk.ac.sanger.artemis.components.Splash;
import uk.ac.sanger.artemis.util.DatabaseLocationParser;

import org.gmod.schema.sequence.Feature;
import org.gmod.schema.sequence.FeatureProp;
import org.gmod.schema.sequence.FeatureLoc;
import org.gmod.schema.sequence.FeaturePub;
import org.gmod.schema.sequence.FeatureRelationship;
import org.gmod.schema.sequence.FeatureSynonym;
import org.gmod.schema.sequence.FeatureCvTerm;
import org.gmod.schema.sequence.FeatureCvTermProp;
import org.gmod.schema.sequence.FeatureCvTermDbXRef;
import org.gmod.schema.sequence.FeatureCvTermPub;
import org.gmod.schema.cv.Cv;
import org.gmod.schema.cv.CvTerm;
import org.gmod.schema.general.Db;
import org.gmod.schema.general.DbXRef;
import org.gmod.schema.organism.Organism;
import org.gmod.schema.pub.PubDbXRef;
import org.gmod.schema.pub.Pub;
import org.postgresql.largeobject.LargeObjectManager;

import com.ibatis.common.jdbc.SimpleDataSource;

import java.sql.*;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.io.*;
import java.net.ConnectException;
import java.net.InetAddress;
import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Set;
import java.util.Vector;
import java.util.Enumeration;
import java.util.List;
import java.util.Iterator;
import java.util.Collection;

import javax.swing.JOptionPane;
import javax.swing.JPasswordField;

/**
 * Objects of this class are Documents created from a relational database.
 */
public class DatabaseDocument extends Document
{
  private String name = null;

  /** source feature_id */
  private String srcFeatureId = "1";

  /** database schema */
  private String schema = "public";

  private static Hashtable<Integer, CvTerm> cvterms;
  
  private InputStreamProgressListener progress_listener;

  /** JDBC DAO */
  private JdbcDAO jdbcDAO = null;

  /** iBatis DAO */
  private static IBatisDAO connIB = null;

  private ByteBuffer[] gff_buffer;

  private ByteBuffer gff_buff;

  /** entries to split into - each is given a name and the features within the entry */
  private static String[][][] TYPES = 
  { 
      { {"repeats"}   , {"repeat_region", "direct_repeat"} }, 
      { {"EST"}       , {"EST_match", "match_part"} },
      { {"contig+gap"}, {"contig", "gap"}}
  };

  /** true if splitting the GFF into entries */
  private boolean splitGFFEntry;

  private boolean iBatis = false;

  private JPasswordField pfield;
  
  private boolean singleSchema = true;

  //private List schema_list;
  
  private static List<String> organismNames;
  
  private boolean gene_builder;
  
  // include children in reading from the database
  private boolean readChildren = true;
  
  // range to retrieve features for
  private Range range;
  
  private Feature geneFeature;
  
  private Hashtable<String, Feature> idFeatureStore;
  
  private boolean lazyFeatureLoad = true;
  
  public static String EXONMODEL  = "exon-model";
  public static String TRANSCRIPT = "mRNA";
  public static boolean CHADO_INFER_CDS = false;
  
  /** list of controlled_curation CV names */
  private static Vector<String> cvControledCuratioNames;
  
  private static CvTermThread cvThread;
  
  // controlled vocabulary
  /** controlled_curation controlled vocabulary */
  public static String CONTROLLED_CURATION_TAG_CVNAME = 
                                 "CC_";
  /**  controlled vocabulary */
  public static String PRODUCTS_TAG_CVNAME = "genedb_products";
  public static String RILEY_TAG_CVNAME = "RILEY";
  
  private static org.apache.log4j.Logger logger4j = 
    org.apache.log4j.Logger.getLogger(DatabaseDocument.class);

  
  /**
   * 
   * Create a new Document from a database.
   * 
   * @param location
   *          This should be a URL string giving:
   *          jdbc:postgresql://host:port/database_name?user=username
   * 
   */
  public DatabaseDocument(String location, JPasswordField pfield)
  {
    super(location);
    this.pfield = pfield;

    if(location.indexOf('=') > -1)
      this.schema = location.substring( location.indexOf('=')+ 1);
    
    if(System.getProperty("ibatis") != null ||
       System.getProperty("jdbc") == null)
    {
      iBatis = true;
      System.setProperty("chado", location);
    }
    initMDC(this);
  }

  
  /**
   * 
   * Create a new Document from a database.
   * 
   * @param location
   *          This should be a URL string giving:
   *          jdbc:postgresql://host:port/database_name?user=username
   * @param feature_id
   *          ID of a feature to be extracted.
   * 
   */
  public DatabaseDocument(String location, JPasswordField pfield,
                          String srcFeatureId, String schema)
  {
    super(location);
    this.pfield = pfield;

    this.srcFeatureId = srcFeatureId;
    this.schema = schema;

    if(System.getProperty("ibatis") != null ||
       System.getProperty("jdbc") == null)
    {
      iBatis = true;
      System.setProperty("chado", location);
    }
    initMDC(this);
  }

  /**
   * 
   * Create a new Document from a database.
   * 
   * @param location
   *          This should be a URL string giving:
   *          jdbc:postgresql://host:port/database_name?user=username
   * @param srcFeatureId
   *          ID of a feature to be extracted.
   * @param splitGFFEntry
   *          split into separate entries based on feature types.
   * @param progress_listener
   *          input stream progress listener
   * 
   */
  public DatabaseDocument(String location, JPasswordField pfield,
                          String srcFeatureId, String schema, boolean splitGFFEntry,
                          InputStreamProgressListener progress_listener)
  {
    super(location);
    this.pfield = pfield;
    this.srcFeatureId = srcFeatureId;
    this.schema = schema;
    this.splitGFFEntry = splitGFFEntry;
    this.progress_listener = progress_listener;
    if(System.getProperty("ibatis") != null ||
       System.getProperty("jdbc") == null)
    {
      iBatis = true;
      System.setProperty("chado", location);
    }
    
    reset(location, schema);
    initMDC(this);
  }

  /**
   * Used by the gene builder to read a database entry
   * for a single gene.
   * @param location
   * @param pfield
   * @param srcFeatureId
   * @param schema
   * @param gene_builder
   */
  public DatabaseDocument(String location, JPasswordField pfield,
          String srcFeatureId, String schema, boolean gene_builder)
  {
    super(location);
    this.pfield = pfield;
    this.srcFeatureId = srcFeatureId;
    this.schema = schema;
    this.gene_builder = gene_builder;

    if(System.getProperty("ibatis") != null ||
       System.getProperty("jdbc") == null)
    {
      iBatis = true;
      System.setProperty("chado", location);
    }
    initMDC(this);
  }
  
  public DatabaseDocument(String location, JPasswordField pfield,
                          String srcFeatureId, String schema,
                          ByteBuffer gff_buff, String name)
  {
    super(location);
    this.pfield = pfield;
    this.srcFeatureId = srcFeatureId;
    this.schema = schema;
    this.gff_buff = gff_buff;
    this.name = name;
    if(System.getProperty("ibatis") != null ||
       System.getProperty("jdbc") == null)
    {
      iBatis = true;
      System.setProperty("chado", location);
    }
    initMDC(this);
  }
  
  /**
   * Use another DatabaseDocument to make a new document.
   * @param originalDocument
   * @param srcFeatureId
   * @param schema
   * @param gene_builder
   * @param region_grab
   * @param progress_listener
   */
  public DatabaseDocument (final DatabaseDocument originalDocument,
             final String schema, final Feature geneFeature,
             final Range range,
             final InputStreamProgressListener progress_listener)
  {
    this((String)originalDocument.getLocation(), 
         originalDocument.getPfield(),
         "-1", schema, false);
    this.progress_listener = progress_listener;
    this.range = range;
    this.geneFeature = geneFeature;
  }
  
  /**
   * Use another DatabaseDocument to make a new document.
   * @param originalDocument
   * @param srcFeatureId
   * @param schema
   * @param gene_builder
   * @param region_grab
   * @param progress_listener
   */
  public DatabaseDocument (final DatabaseDocument originalDocument,
             final String srcFeatureId, 
             final String schema, 
             final boolean gene_builder,
             final InputStreamProgressListener progress_listener)
  {
    this((String)originalDocument.getLocation(), 
         originalDocument.getPfield(),
         srcFeatureId, schema, gene_builder);
    this.progress_listener = progress_listener;
  }
  
  
  public static void initMDC(final DatabaseDocument doc)
  {
    // add username & host to MDC data for logging
    try
  	{
      org.apache.log4j.MDC.put("username",doc.getUserName());
  	}
  	catch(NullPointerException npe)
	  {
	    org.apache.log4j.MDC.put("username",System.getProperty("user.name"));
	  }
	
	  try 
	  {
	    org.apache.log4j.MDC.put("host",
    		  InetAddress.getLocalHost().getHostAddress());
	  } 
	  catch(Exception e) {}
  }
  
  public void setReadChildren(final boolean readChildren)
  {
    this.readChildren = readChildren; 
  }
  
  /**
   * Reset the schema.
   * @param location
   * @param schema
   */
  private void reset(String location, String schema)
  {
    this.schema = schema;

    if(!location.endsWith("="+schema))
    {
      int index = location.lastIndexOf('=');
      setLocation(location.substring(0,index+1) + schema);
      if(iBatis && connIB != null)
      {
        try
        {
          connIB.close();
        }
        catch(SQLException e)
        {
          logger4j.warn(e.getMessage());
        }
        connIB  = null;
      }
      
      jdbcDAO = null;
      System.setProperty("chado", (String)getLocation());
      logger4j.debug((String)getLocation());
    }
  }
  
  /**
   * Reset connection.
   */
  public void reset()
  {
    if(iBatis && connIB != null)
    {
      try
      {
        connIB.close();
      }
      catch(SQLException e)
      {
        logger4j.warn(e.getMessage());
      }
      connIB  = null;
    }
    
    jdbcDAO = null;
  }

  /**
   * Append a String to the Document location.
   * @param name  the name to append.
   */
  public Document append(String name) throws IOException
  {
    return new DatabaseDocument( ((String)getLocation()) + name, pfield);
  }

  /**
   * Return the name of this Document (the last element of the Document
   * location).
   */
  public String getName()
  {
    if(name == null)
    {
      int ind     = ((String) getLocation()).indexOf("?");
      String name = ((String) getLocation()).substring(0, ind);
      ind = name.lastIndexOf("/");
      return name.substring(ind + 1);
    }
    return name;
  }


  /**
  *  Set the name of this document.
  */
  public void setName(String name)
  {
    this.name = name;
  }


  public DatabaseDocument createDatabaseDocument()
  {
    return new DatabaseDocument( (String)getLocation(), pfield,
                                  srcFeatureId, schema );
  }
  
  /**
   * Return true if and only if the Document refered to by this object exists
   * and is readable. Always returns true.
   */
  public boolean readable()
  {
    return true;
  }

  /**
   * Return true if and only if the Document refered to by this object exists
   * and can be written to. Always returns false.
   */
  public boolean writable()
  {
    return true;
  }

  /**
   * Create a new InputStream object from this Document. The contents of the
   * Document can be read from the InputStream.
   * 
   * @exception IOException
   *              Thrown if the Document can't be read from (for example if it
   *              doesn't exist).
   */
  public InputStream getInputStream() throws IOException
  {
    ByteArrayInputStream instream;

    if(gff_buff != null)
    {
      instream = new ByteArrayInputStream(gff_buff.getBytes());
      return instream;
    }

    try
    {
      GmodDAO dao = getDAO();

      if(gene_builder)
      {
        // creating a gene builder
        List<String> schemaList = new Vector<String>();
        schemaList.add(schema);
          
        ByteBuffer bb = getGeneFeature(srcFeatureId,
                                       schemaList, dao, readChildren);          
        return new ByteArrayInputStream(bb.getBytes());
      }
      else if(range != null)
      {
        //
        // Retrieve all features within a range
       // List schemaList = new Vector();
       // schemaList.add(schema);
        final Feature srcFeature;
        if(geneFeature != null)
        {
          Collection<FeatureLoc> featureLocs = geneFeature.getFeatureLocsForFeatureId();
          Iterator<FeatureLoc> it = featureLocs.iterator();
          final FeatureLoc featureLoc = it.next();

          int srcfeatureid = featureLoc.getFeatureBySrcFeatureId().getFeatureId();
          srcFeature = dao.getFeatureById(srcfeatureid);
          setName(srcFeature.getUniqueName());
          this.srcFeatureId = Integer.toString(srcfeatureid);     
        }
        else
        {
          srcFeature = dao.getFeatureById(Integer.parseInt(srcFeatureId));
        }
        
        final ByteBuffer entryBuffer = getFeaturesInRange(srcFeature, range, dao);
        getChadoSequence(srcFeature, entryBuffer);

        return new ByteArrayInputStream(entryBuffer.getBytes());
      }
      
      ByteBuffer entryBuffer = new ByteBuffer();
      try
      {
        DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
        Calendar cal = Calendar.getInstance();

        entryBuffer.append("##gff-version 3\n");
        entryBuffer.append("#created " + dateFormat.format(cal.getTime()) + "\n");
        
        ByteBuffer sequenceBuffer = new ByteBuffer();
        if(dao instanceof IBatisDAO)
          ((IBatisDAO) dao).startTransaction();

        logger4j.debug("RETRIEVE SOURCE FEATURE FROM: "+getLocation());
        Feature srcFeature = getChadoSequence(dao, sequenceBuffer);
        
        entryBuffer.append("##sequence-region " + srcFeature.getUniqueName() +
            " 1 " + srcFeature.getResidues().length + "\n");
        gff_buffer = getGff(dao, srcFeature);
        
        if(splitGFFEntry)
        {
          if(gff_buffer[0].size() > 0)
            entryBuffer.append(gff_buffer[0]);
        }
        else
        {
          for(int i = 0; i < gff_buffer.length; i++)
          {
            if(gff_buffer[i].size() > 0)
              entryBuffer.append(gff_buffer[i]);
          }
        }

        if(dao instanceof IBatisDAO)
          ((IBatisDAO) dao).commitTransaction();
        entryBuffer.append(sequenceBuffer);
      }
      finally
      {
        if(dao instanceof IBatisDAO)
          ((IBatisDAO) dao).endTransaction();
      }

      instream = new ByteArrayInputStream(entryBuffer.getBytes());
      return instream;
    }
    catch(RuntimeException re)
    {
      JOptionPane.showMessageDialog(null, "Problems Reading...\n" +
          re.getMessage(),
          "Problems Reading From the Database ",
          JOptionPane.ERROR_MESSAGE);
      
      re.printStackTrace();
    }
    catch(java.sql.SQLException sqlExp)
    {
      JOptionPane.showMessageDialog(null, "Problems Reading...\n" +
          sqlExp.getMessage(),
          "Problems Reading From the Database ",
          JOptionPane.ERROR_MESSAGE);
      
      sqlExp.printStackTrace();
    }

    return null;
  }

  /**
   * 
   * Called (by DatabaseEntrySource) to retrieve all the documents for each
   * entry created.
   * 
   */
  public DatabaseDocument[] getGffDocuments(String location, String id,
                                            String schema)
  {
    int nentries = 0;
    for(int i = 1; i < gff_buffer.length; i++)
    {
      if(gff_buffer[i].size() > 0)
        nentries++;
    }

    DatabaseDocument[] new_docs = new DatabaseDocument[nentries];
    nentries = 0;
    for(int i = 1; i < gff_buffer.length; i++)
    {
      if(gff_buffer[i].size() == 0)
        continue;

      String name = TYPES[i-1][0][0];

      new_docs[nentries] = new DatabaseDocument(location, pfield, id, schema,
                                                gff_buffer[i], name);
      nentries++;
    }

    return new_docs;
  }

  /**
   * Create an array of GFF lines.
   * @param dao                 the data access object 
   * @param parentFeatureID     the parent identifier for the features to 
   *                            extract
   * @return   the <code>ByteBuffer</code> array of GFF lines
   */
  private ByteBuffer[] getGff(final GmodDAO dao, 
                              final Feature srcFeature)
  {
    //final int srcfeature_id = Integer.parseInt(srcFeatureId);
    
    logger4j.debug("BUILD GFF FEATURES");
    
    // build srcfeature object
    FeatureLoc featureloc = new FeatureLoc();
    featureloc.setFeatureBySrcFeatureId(srcFeature);
    Feature child = new Feature();
    
    // ignore match_part (BLAST HSPs)
    CvTerm cvTerm;
    try
    {
      cvTerm = getCvTermByCvAndCvTerm("match_part", "sequence");
    }
    catch(NullPointerException ne)
    {
      cvTerm = dao.getCvTermByNameAndCvName("match_part", "sequence");
    }
    
    child.setFeatureLoc(featureloc);
    child.setAnalysis(false);
    child.setCvTerm(cvTerm);
    
    final List<Feature> featList = dao.getFeaturesByLocatedOnFeature(child);
    final ByteBuffer[] buffers = new ByteBuffer[TYPES.length + 1];
    for(int i = 0; i < buffers.length; i++)
      buffers[i] = new ByteBuffer();
    
    ByteBuffer this_buff;
    int feature_size = featList.size();
    final Hashtable<String, Feature> id_store = new Hashtable<String, Feature>(feature_size);

    // build feature store
    for(int i = 0; i < feature_size; i++)
    {
      Feature feat = featList.get(i);
      id_store.put(Integer.toString(feat.getFeatureId()), feat);
    }

    if(lazyFeatureLoad)
      idFeatureStore = id_store;

    // get all dbrefs & synonyms etc
    final Hashtable<Integer, List<String>> dbxrefs;
    final Hashtable<Integer, List<FeatureSynonym>> synonym;
    final Hashtable<Integer, List<FeatureCvTerm>> featureCvTerms;
    final Hashtable<Integer, List<FeatureCvTermDbXRef>> featureCvTermDbXRefs;
    Hashtable<Integer, List<FeatureCvTermPub>> featureCvTermPubs = null;
    final Hashtable<Integer, List<FeaturePub>> featurePubs;
    final List<PubDbXRef> pubDbXRefs;
    
    if(lazyFeatureLoad)
    {
      dbxrefs = null;
      synonym = null;
      featureCvTerms = null;
      featureCvTermDbXRefs = null;
      featureCvTermPubs = null;
      featurePubs = null;
      pubDbXRefs = null;
    }
    else
    {
      dbxrefs= IBatisDAO.mergeDbXRef(
        dao.getFeatureDbXRefsBySrcFeature(srcFeature));
      synonym = getAllFeatureSynonyms(
        dao.getFeatureSynonymsBySrcFeature(srcFeature));
      featureCvTerms = getFeatureCvTermsByFeature(dao, 
        dao.getFeatureCvTermsBySrcFeature(srcFeature));
      featureCvTermDbXRefs = getFeatureCvTermDbXRef(dao, 
        dao.getFeatureCvTermDbXRefBySrcFeature(srcFeature));
      
      try
      {
        featureCvTermPubs = getFeatureCvTermPub(dao, 
          dao.getFeatureCvTermPubBySrcFeature(srcFeature));
      } 
      catch(Exception e) 
      { 
        e.printStackTrace();
        if(dao instanceof IBatisDAO)
        {
          try
          {
            ((IBatisDAO) dao).endTransaction();
            ((IBatisDAO) dao).startTransaction();
          }
          catch(SQLException sqle){}  
        }
      }
      featurePubs = getFeaturePubs(dao,
          dao.getFeaturePubsBySrcFeature(srcFeature));
      pubDbXRefs= dao.getPubDbXRef();
    }
    
    // create gff byte stream
    for(int i = 0; i < feature_size; i++)
    { 
      // select buffer based on feature type
      Feature feat = featList.get(i);
      int type_id = feat.getCvTerm().getCvTermId();
      String typeName = getCvtermName(type_id, dao, gene_builder);
      this_buff = buffers[0];

      for(int j = 0; j < TYPES.length; j++)
      {
        for(int k=0; k<TYPES[j][1].length; k++)
          if(TYPES[j][1][k].equals(typeName))
            this_buff = buffers[j+1];
      }

      chadoToGFF(feat, srcFeature.getUniqueName(),
                 dbxrefs, synonym, featureCvTerms,
                 pubDbXRefs, featureCvTermDbXRefs, featureCvTermPubs,
                 featurePubs,
                 id_store, dao, 
                 feat.getFeatureLoc(), this_buff, gene_builder);
       
      if( i%10 == 0 || i == feature_size-1)
        progress_listener.progressMade("Read from database: " + 
                                       feat.getUniqueName());
    }

    return buffers;
  }

  /**
   * Get a <code>Hashtable</code> of feature_id keys and their corresponding 
   * feature_synonym
   * 
   */
  private Hashtable<Integer, List<FeatureSynonym>> getAllFeatureSynonyms(final List<FeatureSynonym> list) 
  {   
    Hashtable<Integer, List<FeatureSynonym>> synonym = new Hashtable<Integer, List<FeatureSynonym>>();
    Integer featureId;
    List<FeatureSynonym> value;
    FeatureSynonym alias;
    
    for(int i=0; i<list.size(); i++)
    {
      alias = list.get(i);
      featureId = new Integer(alias.getFeature().getFeatureId());
      if(synonym.containsKey(featureId))
        value = synonym.get(featureId);
      else
        value = new Vector<FeatureSynonym>();
      
      value.add(alias);
      synonym.put(featureId, value);
    }
    
    return synonym;
  }
  
  /**
   * Get FeaturePub's (i.e. /literature qualifiers).
   * @param dao
   * @param list
   * @return
   */
  private Hashtable<Integer, List<FeaturePub>> getFeaturePubs(final GmodDAO dao,
                                                              final List<FeaturePub> list)
  {
    final Hashtable<Integer, List<FeaturePub>> featurePubs = new Hashtable<Integer, List<FeaturePub>>();
    Integer featureId;
    List<FeaturePub> value;
    FeaturePub featurePub;
    
    for(int i=0; i<list.size(); i++)
    {
      featurePub = list.get(i);
      featureId = new Integer(featurePub.getFeature().getFeatureId());
      if(featurePubs.containsKey(featureId))
        value = featurePubs.get(featureId);
      else
        value = new Vector<FeaturePub>();
      
      value.add(featurePub);
      featurePubs.put(featureId, value);
    }
    
    return featurePubs;
  }
  
  /**
   * @param dao
   * @param chadoFeature null if we want them all
   * @return
   */
  private Hashtable<Integer, List<FeatureCvTerm>> getFeatureCvTermsByFeature(
                                final GmodDAO dao, 
                                final List<FeatureCvTerm> list)
  {
    Hashtable<Integer, List<FeatureCvTerm>> featureCvTerms = new Hashtable<Integer, List<FeatureCvTerm>>();
    Integer featureId;
    List<FeatureCvTerm> value;
    FeatureCvTerm feature_cvterm;
    
    for(int i=0; i<list.size(); i++)
    {
      feature_cvterm = list.get(i);
      featureId = new Integer(feature_cvterm.getFeature().getFeatureId());
      if(featureCvTerms.containsKey(featureId))
        value = featureCvTerms.get(featureId);
      else
        value = new Vector<FeatureCvTerm>();
      
      value.add(feature_cvterm);
      featureCvTerms.put(featureId, value);
    }
    return featureCvTerms;
  }
  
  /**
   * 
   * @param dao
   * @param chadoFeature null if we want all
   * @return
   */
  private Hashtable<Integer, List<FeatureCvTermDbXRef>> getFeatureCvTermDbXRef(
      final GmodDAO dao, final List<FeatureCvTermDbXRef> list)
  {
    if(list == null || list.size() == 0)
      return null;
    
    Integer featureCvTermDbXRefId;
    List<FeatureCvTermDbXRef> value;
    
    Hashtable<Integer, List<FeatureCvTermDbXRef>> featureCvTermDbXRefs = 
      new Hashtable<Integer, List<FeatureCvTermDbXRef>>(list.size());
    for(int i=0; i<list.size(); i++)
    {
      FeatureCvTermDbXRef featureCvTermDbXRef =
        list.get(i);
      
      featureCvTermDbXRefId = new Integer(
          featureCvTermDbXRef.getFeatureCvTerm().getFeatureCvTermId());
      
      if(featureCvTermDbXRefs.containsKey(featureCvTermDbXRefId))
        value = featureCvTermDbXRefs.get(featureCvTermDbXRefId);
      else
        value = new Vector<FeatureCvTermDbXRef>();
      
      value.add(featureCvTermDbXRef);
      featureCvTermDbXRefs.put(featureCvTermDbXRefId, value);
    }
     
    return featureCvTermDbXRefs;
  }
  
  private Hashtable<Integer, List<FeatureCvTermPub>> getFeatureCvTermPub(
                                        final GmodDAO dao,
                                        final List<FeatureCvTermPub> list)
  {
    if(list == null || list.size() == 0)
      return null;

    Integer featureCvTermId;
    List<FeatureCvTermPub> value;

    Hashtable<Integer, List<FeatureCvTermPub>> featureCvTermPubs = 
      new Hashtable<Integer, List<FeatureCvTermPub>>(list.size());
    for(int i=0; i<list.size(); i++)
    {
      FeatureCvTermPub featureCvTermPub =
        list.get(i);
      
      featureCvTermId = new Integer(
          featureCvTermPub.getFeatureCvTerm().getFeatureCvTermId());
      
      if(featureCvTermPubs.containsKey(featureCvTermId))
        value = featureCvTermPubs.get(featureCvTermId);
      else
        value = new Vector<FeatureCvTermPub>();
      
      value.add(featureCvTermPub);
      featureCvTermPubs.put(featureCvTermId, value);
    }
    return featureCvTermPubs;
  }
  
  /**
   * Retrieve the features in a given range
   * @param srcFeature
   * @param range
   * @param dao
   * @return
   */
  private ByteBuffer getFeaturesInRange(final Feature srcFeature,
                                        final Range range, 
                                        final GmodDAO dao)
  { 
    ByteBuffer buff = new ByteBuffer();

    logger4j.debug("GET FEATURES IN RANGE:: "+range.toString());
    List featuresInRange = dao.getFeaturesByRange(range.getStart()-1, 
                                range.getEnd(), 0, srcFeature, null);
    
    List<Integer> featureIds = new Vector<Integer>(featuresInRange.size());
    for(int i=0; i<featuresInRange.size(); i++)
    {
      Feature thisFeature = (Feature)featuresInRange.get(i);
      featureIds.add(new Integer(thisFeature.getFeatureId()));
    }
    
    FeatureLoc featureLoc = new FeatureLoc();
    featureLoc.setFmin(new Integer(range.getStart()));
    featureLoc.setFmax(new Integer(range.getEnd()));
    srcFeature.setFeatureLoc(featureLoc);
    
    Hashtable<Integer, List<String>> dbxrefs = IBatisDAO.mergeDbXRef(
        dao.getFeatureDbXRefsBySrcFeature(srcFeature));
    Hashtable<Integer, List<FeatureSynonym>> synonym = getAllFeatureSynonyms(
        dao.getFeatureSynonymsBySrcFeature(srcFeature));
    Hashtable<Integer, List<FeatureCvTerm>> featureCvTerms = getFeatureCvTermsByFeature(dao, 
        dao.getFeatureCvTermsBySrcFeature(srcFeature));
    Hashtable<Integer, List<FeatureCvTermDbXRef>> featureCvTermDbXRefs = getFeatureCvTermDbXRef(dao, 
        dao.getFeatureCvTermDbXRefBySrcFeature(srcFeature));
    Hashtable<Integer, List<FeatureCvTermPub>> featureCvTermPubs = getFeatureCvTermPub(dao, 
        dao.getFeatureCvTermPubBySrcFeature(srcFeature));
    Hashtable<Integer, List<FeaturePub>> featurePubs = getFeaturePubs(dao,
        dao.getFeaturePubsBySrcFeature(srcFeature));

    List<PubDbXRef> pubDbXRefs = dao.getPubDbXRef();
    
    Hashtable<String, Feature> id_store = new Hashtable<String, Feature>(featuresInRange.size());

    // build feature name store
    for(int i = 0; i < featuresInRange.size(); i++)
    {
      Feature chadoFeature = (Feature)featuresInRange.get(i);
      String featureId = Integer.toString(chadoFeature.getFeatureId());
      id_store.put(featureId, chadoFeature);
    }