/*
* Store the start index and count of probe sets for the given chromosome.
*/
void CopyNumberResultWriter::SetChromosomeProbeSetIndexInformation(u_int8_t chr, int startIndex, int count)
{
ostringstream str;
str << (int) chr;
wstring schr = StringUtils::ConvertMBSToWCS(str.str());
ParameterNameValueType param;
param.SetName(schr + L":start");
param.SetValueInt32(startIndex);
chrStartStop.push_back(param);
param.SetName(schr + L":count");
param.SetValueInt32(count);
chrStartStop.push_back(param);
param.SetName(schr + L":display");
param.SetValueAscii(ChromosomeToString(chr));
chrStartStop.push_back(param);
}
void CHPData::SetInt32ToGenericHdr(const std::wstring& name, int32_t value)
{
ParameterNameValueType paramType;
paramType.SetName(name);
paramType.SetValueInt32(value);
GenericDataHeader* hdr = genericData.Header().GetGenericDataHdr();
hdr->AddNameValParam(paramType);
}
void TestFileGenerator::WriteSmallDatFileWithGridAndSubgridsAndParameters()
{
std::string filename = "small_DAT_file_with_subgrids_and_parameters";
WriteDatFile(filename, L"Hg-small", 4, 5, true);
DATData data(filename);
AddGridAndSubgrids(data, 2.5, 10);
ParameterNameValueType nvt;
nvt.SetName(L"Zip Code");
nvt.SetValueInt32(95051);
data.AddGridAlignmentAlgorithmParameter(nvt);
nvt.SetName(L"County");
nvt.SetValueText(L"Santa Clara");
data.AddGridAlignmentAlgorithmParameter(nvt);
DATFileUpdater update(data);
update.Update();
}
/**
* No more probesets will be processed, this is a chance to finish outputting
* results and clean up.
* @param qMethod - Quantification method that was used.
* @return true if success, false otherwise.
*/
bool QuantMethodExprCCCHPReport::finish(QuantMethod &qMethod)
{
// Sanity to check we saw all the probe sets we were expecting.
if (m_CurrentProbeSetCount != m_Info.m_NumProbeSets) {
Err::errAbort("QuantMethodExprCCCHPReport::finish() - Expecting: " + ToStr(m_Info.m_NumProbeSets) +
" but got: " + ToStr(m_CurrentProbeSetCount) + ". Command Console CHP file will be corrupt.");
}
// Flush remaining signal entries in the buffer.
m_ExpressionQuantificationBufferWriter.FlushBuffer();
// Rewrite CHP files to get chip summary entires
Verbose::out(1,"Creating final files for CHP output");
Verbose::progressBegin(1, ToStr("Finalizing Expression CHP Files"),
m_CHPFileNames.size(), 1, m_CHPFileNames.size());
try {
for (unsigned int chip = 0; chip < m_CHPFileNames.size(); chip++) {
// open up tmp chp file to pull results from
GenericData data;
GenericFileReader reader;
std::string filename = m_CHPFileNames[chip]+".tmp";
reader.SetFilename(filename);
reader.ReadHeader(data);
GenericDataHeader* hdr = data.Header().GetGenericDataHdr();
GenericDataHeader updateHdr;
for (int source = 0; source < m_ChipSummaries.size(); source++) {
ChipSummary::metricDefVec_t metricDefs = m_ChipSummaries[source]->getMetricDefs();
for (int i = 0; i < metricDefs.size(); i++) {
ChipSummary::Metric metric;
if (!m_ChipSummaries[source]->getMetric(chip, metricDefs[i].m_name, metric)) {
Err::errAbort("QuantMethodExprCCCHPReport: metric '" + metricDefs[i].m_name +
"' was not found");
}
std::wstring mName(CHIP_SUMMARY_PARAMETER_NAME_PREFIX);
mName += StringUtils::ConvertMBSToWCS(metric.m_Name);
ParameterNameValueType param;
if (hdr->FindNameValParam(mName, param) == false) {
Err::errAbort("QuantMethodExprCCCHPReport: metric name '" + StringUtils::ConvertWCSToMBS(mName) +
"' could not be found in the header of " + filename);
}
switch (param.GetParameterType()) {
case ParameterNameValueType::Int8Type:
param.SetValueInt8((int8_t)metric.m_Integer);
break;
case ParameterNameValueType::UInt8Type:
param.SetValueUInt8((u_int8_t)metric.m_Integer);
break;
case ParameterNameValueType::Int16Type:
param.SetValueInt16((int16_t)metric.m_Integer);
break;
case ParameterNameValueType::UInt16Type:
param.SetValueUInt16((u_int16_t)metric.m_Integer);
break;
case ParameterNameValueType::Int32Type:
param.SetValueInt32((int32_t)metric.m_Integer);
break;
case ParameterNameValueType::UInt32Type:
param.SetValueUInt32((u_int32_t)metric.m_Integer);
break;
case ParameterNameValueType::FloatType:
param.SetValueFloat((float)metric.m_Double);
break;
case ParameterNameValueType::TextType:
param.SetValueText(StringUtils::ConvertMBSToWCS(metric.m_String), (int) metric.m_String.length());
break;
case ParameterNameValueType::AsciiType:
if (metric.m_String.size() > 256) {
Err::errAbort("QuantMethodExprCCCHPReport: string header parameter too long, name = '" +
metric.m_Name + "', value = '" + metric.m_String + "'");
}
param.SetValueAscii(metric.m_String, (int) metric.m_String.length());
break;
default:
Err::errAbort("QuantMethodExprCCCHPReport: unknown header parameter type found in file " +
filename);
}
updateHdr.AddNameValParam(param);
}
}
std::ofstream os;
Fs::aptOpen(os, filename, std::ios::out|std::ios::binary|std::ios::in);
if (!os) {
Err::errAbort("QuantMethodExprCCCHPReport: file " + filename +
" could not be opened for writing");
}
GenericDataHeaderUpdater updater;
updater.Update(os, updateHdr, *hdr);
os.close();
Verbose::progressStep(1);
}
} catch (...) {
removeAllChps();
Err::errAbort("Error in creating final CHP output.");
}
Verbose::progressEnd(1, ToStr("Done."));
// Remove .tmp extension
for (unsigned int i = 0; i < m_CHPFileNames.size(); i++) {
std::string from = m_CHPFileNames[i] + ".tmp";
std::string to = m_CHPFileNames[i];
if (!Fs::fileRename(from.c_str(),to.c_str())) {
removeAllChps();
Err::errAbort("Unable to rename '" + from + "' to '" + to + "'");
}
}
removeTmpChps();
return true;
}
/**
* Get set up for a run of reporting probesets. Often used to open file
* streams and print headers to files etc.
*
* @param qMethod - Quantification method to be used.
* @param layout - Where the probesets, probes, etc are on the chip.
*
* @return true if success, false otherwise.
*/
bool QuantMethodExprCCCHPReport::prepare(QuantMethod &qMethod, const IntensityMart &iMart)
{
QuantExprMethod *eMethod = dynamic_cast<QuantExprMethod *>(&qMethod);
if (eMethod == NULL) { Err::errAbort("Can only use a QuantMethodExprReport with a QuantExprMethod."); }
setupFileNames(iMart);
int nfiles = m_CHPFileNames.size();
// Make sure our output directory exists.
if (!Fs::isWriteableDir(m_Prefix.c_str()) &&
(Fs::mkdirPath(m_Prefix, false) != APT_OK)) {
APT_ERR_ABORT("Can't make or write to directory: " + m_Prefix);
}
removeAllChps();
// Get CEL file GUIDs
///@todo This be computed by the engine and passed in via AnalysisInfo
m_celGuids.resize(nfiles);
std::string tmp_unc_name;
for (int chip=0; chip<nfiles; chip++) {
FusionCELData cel;
try {
tmp_unc_name=Fs::convertToUncPath(m_CELFileNames[chip]);
cel.SetFileName(tmp_unc_name.c_str());
if (!cel.ReadHeader()) {
Err::errAbort("Unable to read CEL file: "+FS_QUOTE_PATH(tmp_unc_name));
}
GenericData *gdata = cel.GetGenericData();
if (gdata != NULL) {
m_celGuids[chip] = gdata->Header().GetGenericDataHdr()->GetFileId();
}
cel.Close();
}
catch (...) {
Err::errAbort("Unable to read CEL file " + tmp_unc_name);
}
}
int maxProbeSetNameLength = 0;
for (int i=0; i<m_Info.m_ProbesetNames.size(); i++) {
int len = (int)strlen(m_Info.m_ProbesetNames.at(i));
if (m_Info.m_ProbesetDisplayNames.size() > 0 && m_Info.m_ProbesetDisplayNames.at(i) != NULL)
len = (int)strlen(m_Info.m_ProbesetDisplayNames.at(i));
maxProbeSetNameLength = Max(maxProbeSetNameLength, len);
}
// Prepare headers for all CHP files.
wstring algName = StringUtils::ConvertMBSToWCS(m_Info.m_AlgName);
wstring algVersion = StringUtils::ConvertMBSToWCS(m_Info.m_AlgVersion);
wstring chipType = StringUtils::ConvertMBSToWCS(m_Info.m_ChipType);
// For each chip, precreate all probeset signal entries (default to 0.0).
Verbose::out(1,"QuantMethodExprCCCHPReport: Creating temporary files for CHP output");
for (int chip=0; chip<nfiles; chip++) {
try {
ParameterNameValueType param;
// Create tmp chp file
std::string tmp_chp_name=m_CHPFileNames[chip] + ".tmp";
CHPQuantificationData *data = new CHPQuantificationData(tmp_chp_name);
m_TmpChpFiles.push_back(tmp_chp_name);
// set parent header
FusionCELData cel;
try {
tmp_unc_name=Fs::convertToUncPath(m_CELFileNames[chip]);
cel.SetFileName(tmp_unc_name.c_str());
if (!cel.ReadHeader()) {
Err::errAbort("Unable to read CEL file: "+FS_QUOTE_PATH(tmp_unc_name));
}
GenericData *gdata = cel.GetGenericData();
if (gdata != NULL) {
data->GetFileHeader()->GetGenericDataHdr()->AddParent(*gdata->Header().GetGenericDataHdr());
}
cel.Close();
}
catch (...) {
Err::errAbort("Unable to read CEL file: "+FS_QUOTE_PATH(tmp_unc_name));
}
data->SetEntryCount(m_Info.m_NumProbeSets, maxProbeSetNameLength);
data->SetAlgName(algName);
data->SetAlgVersion(algVersion);
data->SetArrayType(chipType);
param.SetName(L"program-name");
param.SetValueText(StringUtils::ConvertMBSToWCS(m_Info.m_ProgramName));
data->GetGenericData().Header().GetGenericDataHdr()->AddNameValParam(param);
param.SetName(L"program-version");
param.SetValueText(StringUtils::ConvertMBSToWCS(m_Info.m_ProgramVersion));
data->GetGenericData().Header().GetGenericDataHdr()->AddNameValParam(param);
param.SetName(L"program-company");
param.SetValueText(StringUtils::ConvertMBSToWCS(m_Info.m_ProgramCompany));
data->GetGenericData().Header().GetGenericDataHdr()->AddNameValParam(param);
// Add algorithm parameters to list.
ParameterNameValueTypeList paramList;
assert(m_Info.m_ParamNames.size() == m_Info.m_ParamValues.size());
for (int i=0; i<m_Info.m_ParamNames.size(); i++) {
if (m_Info.m_ParamValues[i].length() > 0) {
param.SetName(StringUtils::ConvertMBSToWCS(m_Info.m_ParamNames[i]));
param.SetValueText(StringUtils::ConvertMBSToWCS(m_Info.m_ParamValues[i]));
paramList.push_back(param);
}
}
// Add list of all CEL GUIDs in batch
///@todo should this be computed by the engine and passed in via AnalysisInfo?
string prefix = "apt-opt-";
for (int chip=0; chip<m_CHPFileNames.size(); chip++) {
if (m_celGuids[chip].empty() == false) {
string paramName = prefix + "cel-guid-" + ToStr(chip+1);
param.SetName(StringUtils::ConvertMBSToWCS(paramName));
param.SetValueText(StringUtils::ConvertMBSToWCS(m_celGuids[chip]));
paramList.push_back(param);
}
}
data->AddAlgParams(paramList);
// Add the run report parameters to the list
ParameterNameValueTypeList summaryParamList;
std::string blankStr(256, ' ');
for (int source=0; source<m_ChipSummaries.size(); source++) {
ChipSummary::metricDefVec_t metricDefs = m_ChipSummaries[source]->getMetricDefs();
for (int i = 0; i < metricDefs.size(); i++) {
param.SetName(StringUtils::ConvertMBSToWCS(metricDefs[i].m_name));
if (metricDefs[i].m_type == ChipSummary::Metric::Double) {
param.SetValueFloat(-1.0);
}
else if (metricDefs[i].m_type == ChipSummary::Metric::Integer) {
param.SetValueInt32(-1);
}
else if (metricDefs[i].m_type == ChipSummary::Metric::String) {
param.SetValueAscii(blankStr);
}
else {
Err::errAbort("QuantMethodExprCCCHPReport: Unable to handle unknown type: " +
ToStr(metricDefs[i].m_type) );
}
summaryParamList.push_back(param);
}
}
data->AddSummaryParams(summaryParamList);
ProbeSetQuantificationData entry;
CHPQuantificationFileWriter writer(*data);
writer.SeekToDataSet(); // seek to data table location
for (int index=0; index<m_Info.m_ProbesetNames.size(); index++) {
if (m_Info.m_ProbesetDisplayNames.size() > 0 && m_Info.m_ProbesetDisplayNames[index] != NULL)
entry.name = m_Info.m_ProbesetDisplayNames[index];
else
entry.name = m_Info.m_ProbesetNames[index];
entry.quantification = 0.0f;
writer.WriteEntry(entry);
}
delete data;
}
catch (...) {
Err::errAbort("QuantMethodExprCHPReport::prepare() - Unable to write header and/or precreate signal entries to file: " + m_CHPFileNames[chip] + ".tmp");
}
}
// initialize expression signal buffer writer
m_ExpressionQuantificationBufferWriter.Initialize(&m_TmpChpFiles);
return true;
}
void CalvinCHPMultiDataFileUpdaterTest::CreateReferenceFile2()
{
CHPMultiDataData data(TEST2_FILE);
vector<ColumnInfo> cols;
ParameterNameValueType nv;
ByteColumn bcol(L"byte");
cols.push_back(bcol);
UByteColumn ubcol(L"ubyte");
cols.push_back(ubcol);
ShortColumn scol(L"short");
cols.push_back(scol);
UShortColumn uscol(L"ushort");
cols.push_back(uscol);
IntColumn icol(L"int");
cols.push_back(icol);
UIntColumn uicol(L"uint");
cols.push_back(uicol);
FloatColumn fcol(L"float");
cols.push_back(fcol);
ASCIIColumn acol(L"ascii", 7);
cols.push_back(acol);
UnicodeColumn tcol(L"text", 10);
cols.push_back(tcol);
ProbeSetMultiDataGenotypeData e;
data.SetEntryCount(GenotypeMultiDataType, 4, 10, cols);
CHPMultiDataFileWriter *writer = new CHPMultiDataFileWriter(data);
nv.SetName(L"byte");
nv.SetValueInt8(8);
e.metrics.push_back(nv);
nv.SetName(L"ubyte");
nv.SetValueUInt8(8);
e.metrics.push_back(nv);
nv.SetName(L"short");
nv.SetValueInt16(16);
e.metrics.push_back(nv);
nv.SetName(L"ushort");
nv.SetValueUInt16(16);
e.metrics.push_back(nv);
nv.SetName(L"int");
nv.SetValueInt32(32);
e.metrics.push_back(nv);
nv.SetName(L"uint");
nv.SetValueUInt32(32);
e.metrics.push_back(nv);
nv.SetName(L"float");
nv.SetValueFloat(44.0f);
e.metrics.push_back(nv);
nv.SetName(L"ascii");
nv.SetValueAscii("ascii");
e.metrics.push_back(nv);
nv.SetName(L"text");
nv.SetValueText(L"text");
e.metrics.push_back(nv);
writer->SeekToDataSet(GenotypeMultiDataType);
e.name = "1";
e.call = 1;
e.confidence = 10.0f;
writer->WriteEntry(e);
e.name = "2";
e.call = 2;
e.confidence = 20.0f;
writer->WriteEntry(e);
e.name = "3";
e.call = 3;
e.confidence = 30.0f;
writer->WriteEntry(e);
e.name = "4";
e.call = 4;
e.confidence = 40.0f;
writer->WriteEntry(e);
delete writer;
}
void CalvinCHPMultiDataFileUpdaterTest::CreateReferenceFile3()
{
CHPMultiDataData data(TEST3_FILE);
vector<ColumnInfo> cols;
ParameterNameValueType nv;
ByteColumn bcol(L"byte");
cols.push_back(bcol);
UByteColumn ubcol(L"ubyte");
cols.push_back(ubcol);
ShortColumn scol(L"short");
cols.push_back(scol);
UShortColumn uscol(L"ushort");
cols.push_back(uscol);
IntColumn icol(L"int");
cols.push_back(icol);
UIntColumn uicol(L"uint");
cols.push_back(uicol);
FloatColumn fcol(L"float");
cols.push_back(fcol);
ASCIIColumn acol(L"ascii", 7);
cols.push_back(acol);
UnicodeColumn tcol(L"text", 10);
cols.push_back(tcol);
ProbeSetMultiDataCopyNumberData e;
ProbeSetMultiDataCytoRegionData c;
data.SetEntryCount(CopyNumberMultiDataType, 4, 10, cols);
data.SetEntryCount(CytoMultiDataType, 2, 10);
CHPMultiDataFileWriter *writer = new CHPMultiDataFileWriter(data);
nv.SetName(L"byte");
nv.SetValueInt8(8);
e.metrics.push_back(nv);
nv.SetName(L"ubyte");
nv.SetValueUInt8(8);
e.metrics.push_back(nv);
nv.SetName(L"short");
nv.SetValueInt16(16);
e.metrics.push_back(nv);
nv.SetName(L"ushort");
nv.SetValueUInt16(16);
e.metrics.push_back(nv);
nv.SetName(L"int");
nv.SetValueInt32(32);
e.metrics.push_back(nv);
nv.SetName(L"uint");
nv.SetValueUInt32(32);
e.metrics.push_back(nv);
nv.SetName(L"float");
nv.SetValueFloat(44.0f);
e.metrics.push_back(nv);
nv.SetName(L"ascii");
nv.SetValueAscii("ascii");
e.metrics.push_back(nv);
nv.SetName(L"text");
nv.SetValueText(L"text");
e.metrics.push_back(nv);
writer->SeekToDataSet(CopyNumberMultiDataType);
e.name = "1";
e.chr = 1;
e.position = 10;
writer->WriteEntry(e);
e.name = "2";
e.chr = 2;
e.position = 20;
writer->WriteEntry(e);
e.name = "3";
e.chr = 3;
e.position = 30;
writer->WriteEntry(e);
e.name = "4";
e.chr = 4;
e.position = 40;
writer->WriteEntry(e);
writer->SeekToDataSet(CytoMultiDataType);
c.name = "1";
c.chr= 1;
c.startPosition = 1;
c.stopPosition = 2;
c.call = 1;
c.confidenceScore = 10.0f;
writer->WriteEntry(c);
c.name = "2";
c.chr= 2;
c.startPosition = 2;
c.stopPosition = 3;
c.call = 2;
c.confidenceScore = 20.0f;
writer->WriteEntry(c);
delete writer;
}
void TestFileGenerator::WriteRemaingSmallCelFileWithGridParameters(CelFileData& data)
{
data.SetIntensityCount(25);
data.SetStdDevCount(25);
data.SetPixelCount(25);
data.SetOutlierCount(2);
data.SetMaskCount(3);
data.SetRows(5);
data.SetCols(5);
data.SetArrayType(L"Hg-small");
data.SetLibraryPackageName(L"Hg-small-lib-package");
data.SetMasterFileName(L"Hg-small-master-file");
data.SetAlgorithmName(L"Feature Extraction");
ParameterNameValueType nvt;
nvt.SetName(L"percentile");
nvt.SetValueFloat(0.75f);
data.AddAlgorithmParameter(nvt);
nvt.SetName(L"outlierlow");
nvt.SetValueFloat(1.004f);
data.AddAlgorithmParameter(nvt);
nvt.SetName(L"CellMargin");
nvt.SetValueInt32(2);
data.AddAlgorithmParameter(nvt);
// Add grid
wchar_t* gridParams[] = {L"GridULX", L"GridULY", L"GridURX", L"GridURY", L"GridLRX", L"GridLRY", L"GridLLX", L"GridLLY" };
for (int32_t i = 0; i < 8; ++i)
{
nvt.SetName(gridParams[i]);
nvt.SetValueFloat(2.0f + (float)i);
data.AddAlgorithmParameter(nvt);
}
CelFileWriter* writer = new CelFileWriter(data);
FloatVector vInten;
FloatVector vStdev;
Int16Vector vPixels;
for (int i=0; i<25; ++i)
{
vInten.push_back(100.0f*i);
vStdev.push_back(.5*i);
vPixels.push_back(25);
}
// Do some writing
writer->WriteIntensities(vInten);
writer->WriteStdDevs(vStdev);
writer->WritePixels(vPixels);
// XYCoordVector
XYCoordVector outlier;
XYCoord xy(0,0);
outlier.push_back(xy);
xy.xCoord = 1;
xy.yCoord = 2;
outlier.push_back(xy);
writer->WriteOutlierCoords(outlier);
XYCoordVector masked;
xy.xCoord = 1;
xy.yCoord = 0;
masked.push_back(xy);
xy.xCoord = 2;
xy.yCoord = 1;
masked.push_back(xy);
xy.xCoord = 3;
xy.yCoord = 2;
masked.push_back(xy);
writer->WriteMaskCoords(masked);
delete writer;
}
