* [REFACTOR] to separate JTVconverter program from HTMLtools. *
*
* List of Methods*
* =================== * JTVconvert() - Constructor * convertAllJTVfiles() - convert a list of sub directories of JTV * getUnzippedJTVzipFileData() - get array of name & data for zip file. * getJTVsubDirData() - get array of name & data for the JTV directory. * cvtJTVfileSet() - convert a sub directory JTV file set * createTableFromStr() - create Table by parsing tab-delim. string * createTableFromFile() - create Table from file tab-delim. string. * createTableAtKeywordLine() - create Table starting at hdr keyword. * makeMapTable() - create map Table from ftL2S and ftMAS. * applyMapTableToCDT() - map 'mAdb ID's to CDT array names. * reorderNAMEsubfieldsInCDT() - reorder 'NAME' subfields in CDT all data rows * copyJTVjarFilesToJTVOutputDir() - copy JTV jar files to JTVOutputDir. * reZipConvertedFiles() - ReZip converted files to output JTV directory. *
* This code is available at the HTMLtools project on SourceForge at * http://htmltools.sourceforge.net/ * under the "Common Public License Version 1.0" * * http://www.opensource.org/licenses/cpl1.0.php.
** It was derived and refactored from the open source * MAExplorer (http://maexplorer.sourceforge.net/), and * Open2Dprot (http://Open2Dprot.sourceforge.net/) Table modules. *
* $Date: 2009/07/13 11:45:56 $ $Revision: 1.28 $
*
* Copyright 2008, 2009 by Peter Lemkin
* E-Mail: lemkin@users.sourceforge.net
* http://lemkingroup.com/
*
*/
public class JTVconvert
{
/* Note all global variables are in Globals.java except the help
* messages which are in HelpMsgs.java.
*/
public HTMLtools
cvt;
/** Global fileTable instance */
public FileTable
fio;
/**
* JTVconvert() - Constructor
*/
public JTVconvert(HTMLtools cvt)
{ /* JTVconvert */
this.cvt= cvt;
this.fio= cvt.fio;
} /* JTVconvert */
/**
* convertAllJTVfiles() - convert a list of sub directories of JTV
* file sets by reading the three files from the each of the sub
* directories in the jtvInputDir directory. It maps the .cdt file
* in each sub directory to use the toHdrName column of the
* equivalent mapHdrFile map Table instead of the
* "EID:'mAdb ID'" as generated by mAdb. The mapping between "mAdb ID"
* and short array names is done using the jtv_mAdbArraySummaryFile
* Table map. It then writes out the JTV subset to a created
* sub directory in jtvOutputDir which has the same base name as
* the input JTV sub directory being processed. The arguments are
* in the Global variables
*
*
* The full switch is four arguments:
* '-jtvNamesMap:{mAdbArraySummaryFile,mapHdrFile,fromHdrName,toHdrName}'.
* The optional switches are: '-jtvInputDir:{jtvInputDir}' and
* '-jtvOutputDir:{jtvOutputSubDir}' to set the directories to
* other than the defaults ("JTVinput" and "JTVoutput").
* The JTV Global variable definitions are:
* jtvInputDir - where JTV input sub directories are found
* jtvOutputDir - where JTV output sub directories are written
* jtvUseMapColName - column name used when remap the 'mAdb ID' data
* jtv_mAdbArraySummaryFile - mAdb Array Summary file Table of "mAdb ID"s
* jtv_mapHdrNamesFile - long to short Affy name mapping Table.
*
* @return true if write out all of the files, false if there is an
* error or abortProcessingFlag was set..
*/
public boolean convertAllJTVfiles()
{ /* convertAllJTVfiles */
/* [1] Get and initialize the lists of JTV sub directories */
String
jtvInputDir= cvt.jtvInputDir,
jtvOutputDir= cvt.jtvOutputDir,
jtv_mAdbArraySummaryFile= cvt.jtv_mAdbArraySummaryFile,
jtv_mapHdrNamesFile= cvt.jtv_mapHdrNamesFile;
String
inputSubDirList[]= null, /* List of JTV subdirectories */
jtvInputFilePath= null, /* Single JTV directory or zip file path */
jtvFileBaseName= null; /* Single JTV directory or zip file base name */
int
fromL2SmapColIdx= cvt.jtvUseMapFromColNbr-1,
toL2SmapColIdx= cvt.jtvUseMapToColNbr-1;
/* Force it to use the current file separators. */
jtvInputDir= fio.mapPathFileSeparators(jtvInputDir);
jtvOutputDir= fio.mapPathFileSeparators(jtvOutputDir);
/* Get list of data to convert. This will either be a list
* of sub directories or .zip files to convert.
* Try looking for .zip files before looking for sub directories.
*/
if(cvt.jtvSpecificInputFileList!=null)
{ /* get files from the "-files:f1,f2,...,fn" list */
inputSubDirList= UtilCM.cvs2Array(cvt.jtvSpecificInputFileList,",");
}
else
{ /* get zip files from the inputDataDir */
inputSubDirList= fio.getFilesInDir(jtvInputDir,".zip");
}
/* It not zip directory then look for regular directory */
if(inputSubDirList==null)
inputSubDirList= fio.getSubDirsInDir(jtvInputDir); /* try sub dirs */
if(inputSubDirList==null)
{ /* no data */
UtilCM.logMsg("Bad JTV conversion, no subdirectories or zip files in" +
" jtvInputDir '"+ jtvInputDir+"'\n");
return(false);
}
int nDirs= inputSubDirList.length;
UtilCM.logMsg("JTV Input directory= '"+jtvInputDir+"'\n"+
"JTV Output directory= '"+jtvOutputDir+"'\n"+
"# of JTV sub directories= '"+nDirs+"'\n");
for(int i=0;i* Unzip the JTV data into dataList[0:nEntries-1][0:1]. * where: * dataList[*][0] = file names * dataList[*][1] = unzipped data for each file ** @param zipFileName is a JTV zip file to read * @return true if succeed */ private String[][] getUnzippedJTVzipFileData(String jtvZipFileNamePath) { /* getUnzippedJTVzipFileData */ String dataList[][]= fio.readAllZipEntries(jtvZipFileNamePath); /* Validate the data list */ if(dataList==null || dataList.length!=3) return(null); /* Check for the three file extensions are in alphabetical order * [0]".atr", [1].cdt", [2]".gtr" */ int atrIdx= -1, cdtIdx= -1, gtrIdx= -1; for(int i=0;i<3;i++) if(dataList[i][0].endsWith(".atr")) atrIdx= i; else if(dataList[i][0].endsWith(".cdt")) cdtIdx= i; else if(dataList[i][0].endsWith(".gtr")) gtrIdx= i; /* Make sorted copy */ if(atrIdx==-1 || cdtIdx==-1 || gtrIdx==-1) return(null); /* File contents should exist */ String sData[]= new String[3]; sData[0]= dataList[0][1]; sData[1]= dataList[1][1]; sData[2]= dataList[2][1]; if(sData[0].length()==0 || sData[1].length()==0 || sData[2].length()==0) return(null); /* This sorts it alphabetically */ dataList[0][1] = new String(sData[atrIdx]); dataList[1][1] = new String(sData[cdtIdx]); dataList[2][1] = new String(sData[gtrIdx]); return(dataList); } /* getUnzippedJTVzipFileData */ /** * getJTVsubDirData() - get array of name & data for the JTV directory. * There should be 3 files in the unzipped directory ending in * ".atr", .cdt", ".gtr" *
* Return the JTV data in dataList[0:nEntries-1][0:1]. * where: * dataList[*][0] = file names * dataList[*][1] = data for each file ** @param inputSubDir is a JTV subdiredtory to read * @return true if succeed */ private String[][] getJTVsubDirData(String inputSubDir) { /* getJTVsubDirData */ String dataList[][]= new String[3][], jtvInputSubDir= cvt.jtvInputDir+cvt.fileSep+inputSubDir; /* Get the list of file names */ String jtvATRsrc[]= fio.getFilesInDir(jtvInputSubDir,".atr"), jtvCDTsrc[]= fio.getFilesInDir(jtvInputSubDir,".cdt"), jtvGTRsrc[]= fio.getFilesInDir(jtvInputSubDir,".gtr"); /* Check for the three file extensions */ if(jtvATRsrc==null || jtvCDTsrc==null || jtvGTRsrc==null || jtvATRsrc.length!=1 || jtvCDTsrc.length!=1 || jtvGTRsrc.length!=1) return(null); /* Setup the file paths */ String jtvATRinfile= jtvATRsrc[0], jtvCDTinfile= jtvCDTsrc[0], jtvGTRinfile= jtvGTRsrc[0], jtvATRinPath= jtvInputSubDir+cvt.fileSep+jtvATRinfile, jtvCDTinPath= jtvInputSubDir+cvt.fileSep+jtvCDTinfile, jtvGTRinPath= jtvInputSubDir+cvt.fileSep+jtvGTRinfile; /* Read the three input files */ String jtvATRstr= fio.readFileAsString(jtvATRinPath), jtvCDTstr= fio.readFileAsString(jtvCDTinPath), jtvGTRstr= fio.readFileAsString(jtvGTRinPath); /* File contents should exist */ if(jtvATRstr==null || jtvCDTstr==null || jtvGTRstr==null) return(null); /* make 2nd dim */ for(int i=0;i<3;i++) dataList[i]= new String[2]; dataList[0][0]= jtvATRinfile; /* Save the names */ dataList[1][0]= jtvCDTinfile; dataList[2][0]= jtvGTRinfile; dataList[0][1]= jtvATRstr; /* Save the data */ dataList[1][1]= jtvCDTstr; dataList[2][1]= jtvGTRstr; return(dataList); } /* getJTVsubDirData */ /** * cvtJTVfileSet() - convert a sub directory JTV file set * by reading the three files from the (jtvInputDir+inputSubDir) * sub directory with (.atr,.cdt,.gtr) file extensions. It then maps * the .cdt file to use the jtvUseMapColNbr column of the equivalent * jtv_mapHdrNamesFile instead of the "EID:'mAdb ID'" * as generated for the JTV by mAdb. The mapping between "mAdb ID" * and short array names is done using the jtv_mAdbArraySummaryFile * Table map. It then writes out the JTV subset to a created * sub directory (jtvOutputDir+inputSubDir) which has the same * base name as the input directory. *
* [REFACTOR] to separate JTVconverter program from HTMLtools.
* @param jtvInputFilePath - is the input sub directory to process (not
* the full path - need to preface with jtvInputDir
* to get the full path). If the inputSubDir is
* a file ending in .zip, then read the dataList
* by unzipping that file. *
* @param jtvFileBaseName is the single JTV directory or zip file base name
* @param jtvInputDir - directory to find the set of input JTV
* sub directories, each with (.atr,.cdt,.gtr)
* extension files.
* @param jtvOutputDir - directory to write the converted JTV output
* sub directory.
* @param fromL2SmapColIdx - column index used when remap the
* 'mAdb ID' data FROM a mapping data name with this
* column in the jtv_mapHdrNamesFile map.
* @param toL2SmapColIdx - column index used when remap the
* 'mAdb ID' data TO a new data name with this column in
* the jtv_mapHdrNamesFile map.
* @param jtv_mAdbArraySummaryFile - name of the mAdb Array Summary file
* that contains a (MAS) Table with "mAdb ID" and
* the corresponding "Array Name".
* @param jtv_mapHdrNamesFile - is the name of the Long to
* Short array name mapping file that contains a
* (L2S)Table with the array names as well as the
* alternate array names that are used in the mapping.
* @return true if succeed,
*/
private boolean cvtJTVfileSet(String jtvInputFilePath,
String jtvFileBaseName,
String jtvInputDir,
String jtvOutputDir,
int fromL2SmapColIdx,
int toL2SmapColIdx,
String jtv_mAdbArraySummaryFile,
String jtv_mapHdrNamesFile)
{ /* cvtJTVfileSet */
/* [1] Lookup the names and data of the 3 input files in the JTV
* sub directory. Get array of name & data for the JTV directory.
* There should be 3 files in the unzipped directory ending in
* ".atr", .cdt", ".gtr"
* It creates a data structure of the JTV data in
* dataList[0:nEntries-1][0:1]
* where:
* dataList[*][0] = file names
* dataList[*][1] = data for each file.
* Note that it will unzip and get the data if the inputSubDir
* is actually a .zip file, otherwise it will read the contents
* of the sub directory.
*/
String
dataList[][]= null;
if(jtvInputFilePath.endsWith(".zip"))
{
dataList= getUnzippedJTVzipFileData(jtvInputFilePath);
if(dataList==null)
{ /* Bad data */
UtilCM.logMsg("Bad JTV data, no files in zip file '" +
jtvInputFilePath+"'\n");
return(false);
}
}
else
{ /* Get list of files in the directory */
dataList= getJTVsubDirData(jtvInputFilePath);
if(dataList==null)
{ /* Bad data */
UtilCM.logMsg("Bad JTV data, no files in directory '" +
jtvInputFilePath+"'\n");
return(false);
}
}
/* [1.1] Get the names and paths of the three input files */
String
jtvATRinfile= dataList[0][0],
jtvCDTinfile= dataList[1][0],
jtvGTRinfile= dataList[2][0],
baseInputFile= jtvATRinfile.substring(0,jtvATRinfile.length()-4),
jtvOutputSubDir= jtvOutputDir +cvt.fileSep + jtvFileBaseName;
/* Remove the ".zip" if it is there in the output directory */
if(jtvOutputSubDir.endsWith(".zip"))
jtvOutputSubDir=
jtvOutputSubDir.substring(0,jtvOutputSubDir.length()-4);
/* [1.2] Get the data from the three input files. */
String
jtvATRstr= dataList[0][1],
jtvCDTstr= dataList[1][1],
jtvGTRstr= dataList[2][1];
UtilCM.logMsg(" Read JTV files from \n '"+
jtvInputFilePath+cvt.fileSep+"'\n"+
" "+jtvATRinfile+"\n"+
" "+jtvCDTinfile+"\n"+
" "+jtvGTRinfile+"\n");
/* [2] Try to parse the .cdt file string as a Table. */
FileTable ftCDT= createTableFromStr(jtvCDTstr, "CDT-Table");
if(ftCDT==null)
{ /* Bad CDT data */
UtilCM.logMsg("Bad JTV conversion, bad .cdt Table file data for '" +
jtvInputFilePath+"' CDF file '"+
jtvCDTinfile+"'\n"+FileTable.lastErrMsgLog+"\n");
/* Clean (remove) header and data enclosing whitespace */
ftCDT.trimTableEnclWhitespace(true,true);
return(false);
}
/* [3] Use the jtv_mapHdrNamesFile Table ftL2S (setup by Switches
* parser) that contains is the list of the target names to remap the
* .cdt column headers.
*/
FileTable ftL2S= cvt.ftL2S;
if(ftL2S==null)
{ /* Bad Long2Short data */
UtilCM.logMsg("Bad JTV conversion, bad header names map Table file, '" +
jtv_mapHdrNamesFile+"'\n"+FileTable.lastErrMsgLog+"\n");
return(false);
}
/* [4] Read the ftMAS Table from the jtv_mapHdrNamesFile
* that contains is the list of the target names to remap the
* .cdt column headers.
*/
FileTable ftMAS= createTableAtKeywordLine(jtv_mAdbArraySummaryFile,
"mAdbArraySummary-Table",
"mAdb ID");
if(ftMAS==null)
{ /* Bad mAdbArraySummary data */
UtilCM.logMsg("Bad JTV conversion, bad mAdbArraySummary map Table file, '" +
jtv_mAdbArraySummaryFile+"'\n"+
FileTable.lastErrMsgLog+"\n");
return(false);
}
/* [5] Create a map Table from ftL2S and ftMAS. Do this by
* creating Map Table ftM from the ftL2S Table by adding the
* "mAdb ID" data in ftL2S to the matching array name row
* in ftL2S using the ftMAS "Array Name".
*/
FileTable ftM= makeMapTable(ftL2S, ftMAS, fromL2SmapColIdx,
toL2SmapColIdx, "Map-Table");
if(ftM==null)
{ /* problem creating Map-Table data */
UtilCM.logMsg("Problem creating JTV Map-Table.\n"+
FileTable.lastErrMsgLog+"\n");
return(false);
}
/* [6] Map 'mAdb ID's to CDT array names. Do this by mapping
* Table ftM "mAdbID" data to map the header names doing
* partial match of the ftCDT.tFields[] data of the form
* "EID:#" where the # is the matching ftM "mAdbID" data for
* ftM row rM. For each match, replace the header with ftM
* jtvToMapColNbr data for row rM.
*/
String toFieldName= ftL2S.tFields[toL2SmapColIdx];
int
idx_toNameMap_M= ftM.lookupFieldIdx(toFieldName),
idx_mAdbID_M= ftM.lookupFieldIdx("mAdb ID");
if(!applyMapTableToCDT(ftM, ftCDT, idx_toNameMap_M, idx_mAdbID_M))
{ /* problem making map table */
UtilCM.logMsg("Problem mapping JTV CDT file 'mAdb ID' to" +
" array ID names.\n");
return(false);
}
else
UtilCM.logMsg(" Finished mapping CDT file 'mAdb ID' to " +
"array ID names.\n");
/* [6.1] Reorder 'NAME' subfields in CDT all rows of data to improve
* usability in JTV browser.
* Reorder:
* "WID:... || xxxxxx_at || MAP:... || gene -- geneDescr. || RID:..."
* to
* "xxxxxx_at || gene -- geneDescr. || WID:... || MAP:... || RID:..."
*/
if(!reorderNAMEsubfieldsInCDT(ftCDT))
{ /* problem making map table */
UtilCM.logMsg("Problem reordering 'NAME' subfields in all CDT data.\n");
return(false);
}
else
UtilCM.logMsg(" Finished reordering 'NAME' subfields in CDT all data.\n");
/* [7] Convert the modified ftCDT Table to a tab-delimited
* string to write out.
*/
String
jtvCDTstrMapped= ftCDT.cvtTableToTabDelimStr(true);
/* [8] Create output sub directory with the same name if it does
* not already exist. Since it will reside in jtvOutputDir
* it is ok to have the same name as the input sub directory.
*/
if(jtvOutputSubDir.endsWith(".zip"))
jtvOutputSubDir= jtvOutputSubDir.substring(0,jtvOutputSubDir.length()-4);
if(!fio.makeDirectory(jtvOutputSubDir,true))
{ /* Ouput problem */
UtilCM.logMsg("Bad JTV conversion, can't create output directory, '" +
jtvOutputSubDir+"'\n");
return(false);
}
/* [9] Create output file name paths. Note for the outBaseName
* we can use either the input base name or a hard wired name
* "JTVcvt". Since the JTV applet arg list gives the
* sub directory path, it is not a problem if the actual files
* have the same name.
* */
String
outBaseName= baseInputFile; /* same as input files */
outBaseName= "JTVcvt"; /* or use alternate base file name */
/* Ensure right file separator */
jtvOutputSubDir= fio.mapPathFileSeparators(jtvOutputSubDir);
fio.makeDirectory(jtvOutputSubDir,true);
String
jtvATRoutPath= jtvOutputSubDir+cvt.fileSep+outBaseName+".atr",
jtvCDToutPath= jtvOutputSubDir+cvt.fileSep+outBaseName+".cdt",
jtvGTRoutPath= jtvOutputSubDir+cvt.fileSep+outBaseName+".gtr";
fio.makePathSubDirs(jtvOutputSubDir);
/* [10] Copy & rename the .atr & .gtr files to the output directory */
boolean
atrOK= fio.writeStringToFile(jtvATRoutPath, jtvATRstr),
cdtOK= fio.writeStringToFile(jtvCDToutPath, jtvCDTstrMapped),
gtrOK= fio.writeStringToFile(jtvGTRoutPath, jtvGTRstr);
if(!atrOK || !cdtOK || !gtrOK)
{ /* Output problem */
UtilCM.logMsg("Problem writing JTV files:"+
" ATR("+atrOK+")"+" CDT("+cdtOK+")"+
" GTR("+gtrOK+")"+"'\n");
return(false);
}
UtilCM.logMsg(" Created JTV output files in directory\n '"+
jtvOutputSubDir+cvt.fileSep+"'\n"+
" "+outBaseName+".atr"+"\n"+
" "+outBaseName+".cdt"+"\n"+
" "+outBaseName+".gtr"+"\n");
return(true);
} /* cvtJTVfileSet */
/**
* createTableFromStr() - create Table by parsing tab-delim. string
* @param tblStr - a tab-delimited string
* @param tblName - of the table
* @return the FileTable created, else null
*/
private FileTable createTableFromStr(String tblStr, String tblName)
{ /* createTableFromStr */
FileTable ft= new FileTable(tblName);
ft.setHasTableHeaderFlag(true);
ft.setDuplicateFieldsFlag(false);
ft.setNbrTableHdrLines(1);
ft.setRmvTrailingBlankLinesFlag(false);
ft.setHasEmptyLineBeforeTableFlag(false);
/* Create the ft Table from the string. */
if(!ft.parseTableFromString(tblStr))
return(null);
/* Clean (remove) header and data enclosing whitespace */
ft.trimTableEnclWhitespace(true,true);
return(ft);
} /* createTableFromStr */
/**
* createTableFromFile() - create Table from file tab-delim. string.
* @param filePath - full path of the table file to read
* @param tblName - of the table
* @return the FileTable created, else null
*/
private FileTable createTableFromFile(String filePath,
String tblName)
{ /* createTableFromFile */
FileTable ft= new FileTable(tblName);
ft.setHasTableHeaderFlag(true);
ft.setDuplicateFieldsFlag(false);
ft.setNbrTableHdrLines(1);
ft.setRmvTrailingBlankLinesFlag(false);
ft.setHasEmptyLineBeforeTableFlag(false);
/* Read the table file as a ft Table. */
if(!ft.readAndParseTable(filePath))
return(null);
/* Clean (remove) header and data enclosing whitespace */
ft.trimTableEnclWhitespace(true,true);
return(ft);
} /* createTableFromFile */
/**
* createTableAtKeywordLine() - create Table starting at hdr keyword.
* It starts parsing the tab-delimited string read from the file
* at the header row that contains the keyword.
* @param filePath - full path of the table file to read
* @param tblName - of the table
* @param keyword - contained in the table header row.
* @return the FileTable created, else null
*/
private FileTable createTableAtKeywordLine(String filePath,
String tblName,
String keyword)
{ /* createTableAtKeywordLine */
FileTable ft= new FileTable(tblName);
ft.setHasTableHeaderFlag(true);
ft.setDuplicateFieldsFlag(false);
ft.setNbrTableHdrLines(1);
ft.setRmvTrailingBlankLinesFlag(false);
ft.setHasEmptyLineBeforeTableFlag(false);
ft.setStartTableAtColStr(keyword); /* skip front of file */
/* Read the table file as a ft Table. */
if(!ft.readAndParseTable(filePath))
return(null);
/* Clean (remove) header and data enclosing whitespace */
ft.trimTableEnclWhitespace(true,true);
return(ft);
} /* createTableAtKeywordLine */
/**
* makeMapTable() - create map Table from ftL2S and ftMAS.
* Create Map Table ftM from the ftL2S Table by adding the
* "mAdb ID" data in ftL2S to the matching array name row
* in ftL2S using the ftMAS "Array Name".
* Note: This mapping depends on formats of:
* 1) mAdb "Array Summary Table" .CEL file name in field "Array Name"
* and the 2) GSP "LongToShortMap" generated from the "Affy .CEL file"
* field in the individual "EGxxx" Tables.
* Because of the special dependence on the mAdb "Array Summary Table"
* this uses special [HACK]s.
* @param ftL2S - LongToShortMap Table
* @param ftMAS - mAdb Array Summary Table
* @param fromL2SmapColIdx - column index used when remap the
* 'mAdb ID' data FROM a mapping data name with this
* column in the jtv_mapHdrNamesFile map.
* @param toL2SmapColIdx - column index used when remap the
* 'mAdb ID' data TO a new data name with this column in
* the jtv_mapHdrNamesFile map.
* @param tblName - of the table
* @return the FileTable map (ftM) created, else null
*/
private FileTable makeMapTable(FileTable ftL2S, FileTable ftMAS,
int fromL2SmapColIdx, int toL2SmapColIdx,
String tblName)
{ /* makeMapTable */
/* [1] Create Map Table ftM of the ftMAS "mAdb ID" data to the
* GSP IDs in ftL2S using the .CEL "Array Name" in the
* mAdb Array Summary Table.
*/
FileTable ftM= new FileTable("Map-Table");
int
idx_fromMapName_M= -1,
idx_toMapName_M= -1,
idx_mAdb_ID_M= -1;
/* [2] Copy the map columns from Table ftL2S to ftM Table. */
String
colName,
colData[]= null;
if(ftL2S==null || fromL2SmapColIdx<0 || toL2SmapColIdx<0)
return(null); /* bad indexes */
for(int c=0;c