/** File: DataRowStatisticsIndexMap.java */
 

/**
 * Class DataRowStatisticsIndexMap is used to add statistics on the data to 
 * the IndexMap which could be saved as a .sidx file. 
 * The ftIM IndexMap is read from the
 * [TODO] add discussion on which data can be used.
 * <PRE>
 * <B>List of Methods</B><BR>
 * ===================
 * DataRowStatisticsIndexMap() - Constructor
 * setIndexMapTable() - set the IndexMap FileTable ftIM to use
 * setDataTable() - set the data FileTable ft to use if any
 * createStatisticsIndexMapFile() - create & write Statistics Index Map file
 * readIndexMapFile() - read the IndexMap FileTable to ftIM. 
 * readDataFileHeader() - read the data file FileTable to ft.
 * computeRowStatistics() - compute row statistics of data (max,min,mean,stddev)
 * computeGlobalStatistics() - compute global stats of (max,min,mean,stddev) data
 * addRowStatisticsToIndexMap() - add the row stats to the IndexMap ftIM table.
 * addGlobalStatisticsToIndexMapHdr() - add 2 header rows hdr[0:1] for global stats.
 * writeStatisticsIndexMap() - write extended IndexMap w/statistics as .sidx file.
 * 
 *</PRE>
 *<P> 
 * This code is available at the HTMLtools project on SourceForge at
 * <A HREF="http://htmltools.sourceforge.net/">http://htmltools.sourceforge.net/</A>
 * under the "Common Public License Version 1.0" 
 * <A HREF="http://www.opensource.org/licenses/cpl1.0.php">
 * http://www.opensource.org/licenses/cpl1.0.php</A>.<P></P>
 * <P>
 * It was derived and refactored from the open source
 * MAExplorer (http://maexplorer.sourceforge.net/), and
 * Open2Dprot (http://Open2Dprot.sourceforge.net/) Table modules.
 * <P>
 * $Date: 2009/09/15 11:45:56 $ $Revision: 1.34 $
 * <BR> 
 * Copyright 2008, 2009 by Peter Lemkin 
 * E-Mail: lemkin@users.sourceforge.net 
 * <A HREF="http://lemkingroup.com/">http://lemkingroup.com/</A>
 * <BR>
 */

public class DataRowStatisticsIndexMap extends Thread 
{ /* DataRowStatisticsIndexMap */
 
  
  /** converter link */
  private HTMLtools
    cvt= null;

  /** FileTable of the IndexMap */
  private FileTable 
    ftIM= null;

  /** FileTable of the Data table */
  private FileTable 
    ft= null;
  
  /* ----------- File name paths ----- */
  
  /** The full path of the Table data file (.idx extension).
   * We do not read the entire data file into Table ft, but
   * rather random access (all) rows based on the indexMap
   * Table ftIM.
   */
  private String
    dataFilePath= null;
  
  /** The full path of the Table Index Map file (.idx extension).
   * The table is read into ftIM. We do not write the table, but 
   * just read it.
   */
  private String
    indexMapFilePath= null;
  
  /** The full path of the Statistics Index Map file (.sidx extension).
   * The table the expanded ftIM table after we add the row and global 
   * statistics. We do not read the table, but just write it.
   */
  private String
    statIndexMapFilePath= null;
  
  /** Number of of digits in output statistics for .sidx Table */
  private int
    precision= 0;
  
  
  /* ----------- local -dropColumn list for speed in testing ----- */
    
  /** List of columns in FileTable ft that are in the -dropColumn list.
   * This is of size nDropListCols.
   */
  private int
    dropListCols[]= null;
  
  /** Size of dropListCols[] list of columns in FileTable ft that are in 
   * the -dropColumn list.
   */
  private int
    nDropListCols= 0;
  
  
  /* ----------- ftIM Table column indexes ----- */
  
  /* Setup index-map keyword index column for "StartByte" */
  private int
    idxStartByte= -1;

  /* Setup index-map keyword index column for "EndByte" */
  private int
    idxEndByte= -1;

  /* Setup extended ftIM keyword index column for row "Min" data */
  private int
    idxMinRow= -1;
  
  /* Setup extended ftIM keyword index column for row "Max" data */
  private int
    idxMaxRow= -1;
  
  /* Setup extended ftIM keyword index column for row "Mean" data */
  private int
    idxMeanRow= -1;
  
  /* Setup extended ftIM keyword index column for row "StdDev" data */
  private int
    idxStdDevRow= -1;

  
  /* ----------- ftIM Table column indexes ----- */
  /** Number of data columns in the data Table including the drop columns. */
  private int 
    nDcols= 0;
  
  /** Number of data columns in the data Table NOT including the drop columns.
   * - computed */
  private int 
    nD2cols= 0;
  
  /** Number of data rows in the index map Table. */
  private int 
    nIMrows= 0;

  /** Computed row data min values from the data Tables. */
  private float
    minRowVal[]= null;

  /** Computed row data max values from the data Tables. */
  private float
    maxRowVal[]= null;

  /** Computed row data mean values from the data Tables. */
  private float
    meanRowVal[]= null;

  /** Computed row data stdDev values from the data Tables. */
  private float
    stdDevRowVal[]= null;
  
  /** Global min value from data tables */
  private float
    glbMinRowVal= 0.0F;
  
  /** Global max value from data tables */
  private float
    glbMaxRowVal= 0.0F;
  
  /** Global mean value from data tables */
  private float
    glbMeanRowVal= 0.0F;
  
  /** Global stdDev value from data tables */
  private float
    glbStdDevRowVal= 0.0F;

  /** Global min value from data tables */
  private double
    glbMinRowValSum= 0.0;
  
  /** Global min value from data tables */
  private double
    glbMaxRowValSum= 0.0;
  
  /** Global min value from data tables */
  private double
    glbMeanRowValSum= 0.0;
    
  /** Global min value from data tables */
  private double
    glbStdDevRowValSum= 0.0;
  

  /**
   * DataRowStatisticsIndexMap() - Constructor
   * @param cvt is an instance of converter
   * @param dataFilePath is the data .txt file that has an associated
   *        .idx IndexMap file with the same base name. The .sidx 
   *        file to be created has the same base name
   */
  public DataRowStatisticsIndexMap(HTMLtools cvt, 
                                   String dataFilePath)
  { /* DataRowStatisticsIndexMap */
    this.cvt= cvt; 
    this.dataFilePath= dataFilePath;
    
    /* compute the IndexMap and Statistics Index Map file path names. */
    String dfPath= dataFilePath;
    this.indexMapFilePath= dfPath.substring(0,dfPath.length()-4)+".idx";
    this.statIndexMapFilePath= dfPath.substring(0,dfPath.length()-4)+".sidx";
    
    /* Setup the local -dropColumns list */
    nDropListCols= -1;
    dropListCols= null;    
  } /* DataRowStatisticsIndexMap */

  
  /** 
   * setIndexMapTable() - set the IndexMap FileTable ftIM to use
   * if it is already in memory.
  */
  public void setIndexMapTable(FileTable ftIM)
  { this.ftIM= ftIM; }

  
  /** 
   * setDataTable() - set the data FileTable ft to use if any
   * if it is already in memory.
   */
  public void setDataTable(FileTable ft)
  { this.ft= ft; }

  
  /** 
   * setTablePrecision() - set number of digits in output statistics.
   * default is 2.
   */
  public void setTablePrecision(int precision)
  { 
    if(precision<=0)
      precision= 2;
    this.precision= precision; 
  }
  
  

  /** 
   * createStatisticsIndexMapFile() - create & write Statistics Index Map file
   * @return true if succeed
   */
  public boolean createStatisticsIndexMapFile()
  { /* createStatisticsIndexMapFile */

    /* [1] Read the IndexMap FileTable to ftIM. */
    if(!readIndexMapFile())
      return(false);
    
    /* [2] Read the data file FileTable to ft. */
    if(!readDataFileHeader())
      return(false);
    
    /* [3] compute row statistics of data (max,min,mean,stddev) */
    if(!computeRowStatistics())
      return(false);
    
    /* [4] compute global stats of (max,min,mean,stddev) data */
    if(!computeGlobalStatistics())
      return(false);
      
    /* [5]  add the row stats to the IndexMap ftIM table. */
    if(!addRowStatisticsToIndexMap())
      return(false);
    
    /* [6] Add 2nd header row hdr[0] for global stats. */
    if(!addGlobalStatisticsToIndexMapHdr())
      return(false);

    /*  [7] write extended IndexMap w/statistics as .sidx file. 
     * Note that ftIM has been extended in step [6].
     */
    if(!writeStatisticsIndexMap(ftIM))
      return(false);
      
    return(true);
  } /* createStatisticsIndexMapFile */
  

  /** 
   * readIndexMapFile() - read the IndexMap FileTable to ftIM.
   * This can be used if the IndexMap file has not been read yet.
   * @return true if succeed with (ftIM, idxStartByte, idxEndByte)
   *         set up.
   */
  public boolean readIndexMapFile()
  { /* readIndexMapFile */
    ftIM= new FileTable("Index-Map-Table");
    if(!ftIM.readAndParseTable(indexMapFilePath))
    {
      UtilCM.logMsg("Problem reading Index Map file '"+
                    indexMapFilePath+"' \n  "+ftIM.errMsgLog);
      return(false);
    }     
    /* Clean (remove) header  enclosing whitespace */
    ftIM.trimTableEnclWhitespace(true,false);
    
    /* Setup index-map keyword index column values */
    idxStartByte= ftIM.lookupFieldIdx("StartByte");
    idxEndByte= ftIM.lookupFieldIdx("EndByte"); 
    
    return(true);
  } /* readIndexMapFile */
  
  
  /** 
   * readDataFileHeader() - read the data file FileTable to ft.
   * @return true if succeed with ft set up.
   */
  public boolean readDataFileHeader()
  { /* readDataFileHeader */
    ft= new FileTable("Index-Map-Table");
    if(!ft.readAndParseTableFields(dataFilePath))
    {
      UtilCM.logMsg("Problem reading data file Table '"+
                    dataFilePath+"' \n  "+ft.errMsgLog);
      return(false);
    }     
    /* Clean (remove) header  enclosing whitespace */
    ft.trimTableEnclWhitespace(true,false);

    /* Setup the local -dropColumns list */
    nDropListCols=0;
    dropListCols= null;
    if(cvt.nDropColNames>0)
    { /* make local drop list */
      String tFields[]= ft.tFields;
      dropListCols= new int[cvt.nDropColNames];  /* worst case */
      for(int i=0;i<cvt.nDropColNames;i++)
      { /* test i'th drop col against the ft.tFields[k] list */
        String dropName= cvt.dropColNames[i];
        for(int k=0;k<ft.tCols;k++)
          if(dropName.equals(tFields[k]))
          {
            dropListCols[nDropListCols++]= k;
            break;
          }
      } /* test i'th drop col against the ft.tFields[k] list */
    } /* make local drop list */
        
    return(true);
  } /* readDataFileHeader */
  
  
  /** 
    * isDropColumn() - test if column cTest is a -drop list column.
    * If not in list of no list, return false.
    * @param cTest is the column index to test
    * @return true if it is a drop list column.
    */
   public boolean isDropColumn(int cTest)
   { /* isDropColumn */
     if(nDropListCols==0)
       return(false);
     
     for(int c=0;c<nDropListCols;c++)
       if(cTest==dropListCols[c])
         return(true);
     
     return(false);
   } /* isDropColumn */

  
 /** 
   * computeRowStatistics() - compute row statistics of data 
   * (max,min,mean,stddev).
   * @return true if succeed
   */
  public boolean computeRowStatistics()
  { /* computeRowStatistics */
    /* [1] Allocate storage for the computations */
    nIMrows= ftIM.tRows;
    nDcols= ft.tCols;      /* Includes the drop columns at this point */
    nD2cols= 0;            /* Does NOT includes the drop columns - computed */
    minRowVal= new float[nIMrows];
    maxRowVal= new float[nIMrows];
    meanRowVal= new float[nIMrows];
    stdDevRowVal= new float[nIMrows];

    /* reset global sums */
    glbMinRowValSum= 0.0;
    glbMaxRowValSum= 0.0;
    glbMeanRowValSum= 0.0;
    glbStdDevRowValSum= 0.0;
    
    /* [1.1] Get the Seek list */
    long startSeekByte[]= new long[nIMrows];
    String sSeek= null;
    String tData[][]= ftIM.tData;
    for(int r=0;r<nIMrows;r++)
    { /* build the seek list */
      sSeek= tData[r][idxStartByte]; 
      startSeekByte[r]= UtilCM.cvs2l(sSeek,-1L);
    }
    
    /* [2] Read the data file row data using the random access seek
     * pointers from the ftIM Table and compute row statistics
     * into the above arrays.
     */
    String
      sRow,
      sRowData[]= null;     /* it should be [nDcols] size */
    int nD2cols= nDcols - nDropListCols; /* ignore -drop columns */
    double 
      rowData[]= new double[nD2cols];
    FileTable 
      fio= cvt.fio;
    fio.initFileIO();            /* Reset random access file defaults */
    for(int r=0;r<nIMrows;r++)
    { /* process row r */
      /*Check if abort processing by GUI Cancel */
      if(cvt.isAbortProcessingFlag())
        return(false);
      
      /* Update the status line if using the GUI */
      if((r & 0xEF)==0)
        cvt.incrUpdateStatusLine("Extracting Flip Row: "+r);
      
      /* [2.1] Get the row r as a string */
      long startLineSeek= startSeekByte[r];
      sRow= fio.readRandomAccessLine(dataFilePath,startLineSeek);
      if(sRow==null)
      {
        UtilCM.logMsg("Problem seeking row ["+r+ 
                      "] start seek byte pointers from '"+
                       cvt.flipDataFile + 
                       "'\n with startSeekByte="+startSeekByte[r]+
                       "\n"+fio.errMsgLog);
        return (false);        
      }
      
      /* [2.2] Convert the numeric data from the row cells in the row
       * to a String array. Then convert it to a float array for
       * use in computing the statistics.
       */
       sRowData= UtilCM.cvs2ArrayNullFill(sRow, "\t", "");
      
      if(sRowData.length!=nDcols)
      {
        UtilCM.logMsg("Problem data row length("+sRowData.length+
                      ") .NEQ. tCols("+nDcols+") for row ["+r+
                      "] start seek byte pointers from '"+
                      dataFilePath + "'.\n");
        return (false);        
      }
      //UtilCM.trimArrayEnclWhitespace(sRowData);
      
      /* Convert to float array. At end c2 should be nD2cols. */
      int c2= 0;
      for(int c=0;c<nDcols;c++)
        if(!isDropColumn(c))
          rowData[c2++]= UtilCM.cvs2d(sRowData[c]);
      
      /* [2.3] Compute the (min,max,mean,stdDev) for the row */
      double
        data,
        rMin=  1000000000.0,
        rMax= -1000000000.0,
        rSum= 0.0,
        rMean= 0.0,
        rDiff= 0.0,
        rDiffSq= 0.0,
        rSumDiffSq= 0.0,
        rStdDev= 0.0;
      for(int c=0;c<nD2cols;c++)
      { /* compute min, max and rSum */
        data= rowData[c];
        rMin= Math.min(rMin, data);
        rMax= Math.max(rMax, data);
        rSum += data;
      }  
      rMean= rSum/nD2cols;

      for(int c=0;c<nD2cols;c++)
      { /* compute rDiff, rDiffSQ and rSum */
        if(isDropColumn(c))
          continue;
        data= rowData[c];
        rDiff= (data - rMean);
        rDiffSq= (rDiff * rDiff);
        rSumDiffSq += rDiffSq;
      }
      rStdDev= Math.sqrt((1.0/(nD2cols-1)) * rSumDiffSq);
      
      /* [2.4] Save the statistics in row arrays */
      minRowVal[r]= (float)rMin;
      maxRowVal[r]= (float)rMax;
      meanRowVal[r]= (float)rMean;
      stdDevRowVal[r]= (float)rStdDev;   
      
      /* [2.5] Compute the running global sums */
      glbMinRowValSum += rMin;
      glbMaxRowValSum += rMax;
      glbMeanRowValSum += rMean;
      glbStdDevRowValSum += rStdDev;
    } /* process row r */
    
    return(true);
  } /* computeRowStatistics */

  
  /** 
   * computeGlobalStatistics() - compute global statistics of 
   * (max,min,mean,stddev) data from global sums computed in 
   * computeRowStatistics.
   * @return true if succeed
   */
  public boolean computeGlobalStatistics()
  { /* computeGlobalStatistics */    
    /* Compute and same global min,max,mean,stdDev values from data tables */
    glbMinRowVal= (float)(glbMinRowValSum/nIMrows);
    glbMaxRowVal= (float)(glbMaxRowValSum/nIMrows);
    glbMeanRowVal= (float)(glbMeanRowValSum/nIMrows);
    glbStdDevRowVal= (float)(glbStdDevRowValSum/nIMrows);
    
    return(true);
  } /* computeGlobalStatistics */
  

  /** 
   * addRowStatisticsToIndexMap() - add row statistics(Min,Max,Mean,StdDev)
   * column data to the ftIM index map table after the ftIM table
   * has been read into memory.
   * @return true if succeed
   */
  public boolean addRowStatisticsToIndexMap()
  { /* addRowStatisticsToIndexMap */
    if(ftIM==null || ftIM.tRows==0)
      return(false);
    if(precision<=0)
      precision= 2;
    
    /* Create (min,max,mean,stdDev) String data columns. */
    String
      minColData[]= new String[nIMrows],
      maxColData[]= new String[nIMrows],
      meanColData[]= new String[nIMrows],
      stdDevColData[]= new String[nIMrows];
    for(int r=0;r<nIMrows;r++)
    {
      minColData[r]= UtilCM. cvf2s(minRowVal[r], precision);
      maxColData[r]= UtilCM. cvf2s(maxRowVal[r], precision);
      meanColData[r]= UtilCM. cvf2s(meanRowVal[r], precision);
      stdDevColData[r]= UtilCM. cvf2s(stdDevRowVal[r], precision);
    }

    /* Add new (min,max,mean,stdDev) columns to IndexMap Table */
    ftIM.addColumnToTable("Min", minColData);
    ftIM.addColumnToTable("Max", maxColData);
    ftIM.addColumnToTable("Mean", meanColData);
    ftIM.addColumnToTable("StdDev", stdDevColData);
      
    /* Setup index-map keyword index column values */
    idxMinRow= ftIM.lookupFieldIdx("Min");
    idxMaxRow= ftIM.lookupFieldIdx("Max");
    idxMeanRow= ftIM.lookupFieldIdx("Mean");
    idxStdDevRow= ftIM.lookupFieldIdx("StdDev"); 
          
     return(true);
  } /* addRowStatisticsToIndexMap */
  

  /** 
   * addGlobalStatisticsToIndexMapHdr() - add 2 header rows hdr[0:1] for global 
   * statistics where
   * <PRE>
   *  Hdr[0] has names
   *   ("Global Min", "Global Max", "Global Mean","Global StdDev")
   *  Hdr[0] has global values
   *   ( glbMinRowVal, glbMaxRowVal, glbMeanRowVal, glbStdDevRowVal)
   *  </PRE>
   * @return true if succeed
   */
  public boolean addGlobalStatisticsToIndexMapHdr()
  { /* addGlobalStatisticsToIndexMapHdr */
    /* [1] Compute the new header arrays to use */
    int nIMcols= ftIM.tCols;
    String hdrNames[][]= new String[3][];   
    for(int k=0;k<3;k++)
      hdrNames[k]= new String[nIMcols];
    for(int c=0;c<nIMcols;c++)
    {
      hdrNames[0][c]= "";
      hdrNames[1][c]= "";
      hdrNames[2][c]= ftIM.tFields[c];
    }

    /* Add the Global stats column names */
    hdrNames[0][idxMinRow]= "Global Min";
    hdrNames[0][idxMaxRow]= "Global Max";
    hdrNames[0][idxMeanRow]= "Global Mean";
    hdrNames[0][idxStdDevRow]= "Global StdDev";
    
    /* Add the Global stats data */
    hdrNames[1][idxMinRow]= (""+glbMinRowVal);
    hdrNames[1][idxMaxRow]= (""+glbMaxRowVal);
    hdrNames[1][idxMeanRow]= (""+glbMeanRowVal);
    hdrNames[1][idxStdDevRow]= (""+glbStdDevRowVal);
    
    /* [2] Update the Image Map header to three rows of header */
    if(!ftIM.setHeadersToTable(hdrNames, 3, ftIM.tCols))
      return(false);
    
    return(true);
  } /* addGlobalStatisticsToIndexMapHdr */

  
  /** 
   * writeStatisticsIndexMap() - write extended IndexMap with statistics 
   * as a .sidx file.
   * @return true if succeed
   */
  public boolean writeStatisticsIndexMap(FileTable ftSIM)
  { /* writeStatisticsIndexMap */ 
    if(!ftSIM.writeTableToTabDelimFile(statIndexMapFilePath,true))
    {
      UtilCM.logMsg("Problem writing 'Statistics Index Map' Table file'" + 
                    statIndexMapFilePath + "'\n");
      return (false);
    }
    UtilCM.logMsg("Saved 'Statistics Index Map' Table .sidx file '"+
                  statIndexMapFilePath+"'.\n"); 

    return(true);
  } /* writeStatisticsIndexMap */
  
  
} /* end of class DataRowStatisticsIndexMap */