RECT CBitmapFont::CalculateRect(int id)
{
RECT rCell;
rCell.left = (id % GetNumCols()) * GetCharWidth();
rCell.top = (id / GetNumCols()) * GetCharHeight();
rCell.right = rCell.left + GetCharWidth();
rCell.bottom = rCell.top + GetCharHeight();
return rCell;
}
void DiagonalBlockSparseMatrix::Solve(const Eigen::MatrixXd& rhs, Eigen::MatrixXd& sol)
{
int numRows = GetNumRows();
int numCols = GetNumCols();
LOG_IF(FATAL, numRows != numCols) << "DiagonalBlockSparseMatrix is not a square matrix and noninvertible.";
int rhsCols = rhs.cols();
int rhsRows = rhs.rows();
LOG_IF(FATAL, numCols != rhsRows) << "DiagonalBlockSparseMatrix and right hand side matrix are of different dimensions.";
sol.resize(rhsRows, rhsCols);
int numUntilLast = 0;
int numDiagElements = static_cast<int>(mDiagElements.size());
for (int i = 0; i < numDiagElements; ++i)
{
int numColsElement = mDiagElements[i].GetNumCols();
Eigen::MatrixXd partSol(numColsElement, rhsCols);
Eigen::MatrixXd partRhs = rhs.block(numUntilLast, 0, numColsElement, rhsCols);
#if LLT_SOLVE
mDiagElements[i].LltSolve(partRhs, partSol);
#else
mDiagElements[i].ConjugateGradientSolve(partRhs, partSol);
#endif
sol.block(numUntilLast, 0, numColsElement, rhsCols) = partSol;
numUntilLast += numColsElement;
}
}
void NDMask::MarkSectionAs(const std::vector<size_t>& sectionOffset, const NDShape& sectionShape, MaskKind maskKind)
{
// TODO: Implement batching of masking operation for masks residing on GPUs to avoid making
// GPU invocations for each MaskSection call.
if (sectionOffset.size() > m_maskShape.Rank())
LogicError("NDMask::MaskSection: The sectionOffset cannot have dimensionality higher than the rank of 'this' mask");
if (sectionShape.Rank() > m_maskShape.Rank())
LogicError("NDMask::MaskSection: The section shape cannot have an axes count higher than the rank of 'this' mask");
std::vector<size_t> offset(m_maskShape.Rank(), 0);
for (size_t i = 0; i < sectionOffset.size(); ++i)
offset[i] = sectionOffset[i];
NDShape shape = sectionShape.AppendShape(NDShape(m_maskShape.Rank() - sectionShape.Rank(), NDShape::InferredDimension));
auto maskMatrix = GetMatrix();
size_t rowOffset = offset[0];
size_t colOffset = offset[1];
size_t sliceRowLength = (shape[0] != NDShape::InferredDimension) ? shape[0] : (maskMatrix->GetNumRows() - rowOffset);
size_t sliceColLength = (shape[1] != NDShape::InferredDimension) ? shape[1] : (maskMatrix->GetNumCols() - colOffset);
if ((rowOffset == 0) && (sliceRowLength == maskMatrix->GetNumRows()))
maskMatrix->ColumnSlice(colOffset, sliceColLength).SetValue((char)maskKind);
else
{
// Since Matrix does not support strides in the row dimension, we will need to create separate slices for each column
for (size_t i = colOffset; i < (colOffset + sliceColLength); ++i)
{
auto column = maskMatrix->ColumnSlice(i, 1);
column.Reshape(1, maskMatrix->GetNumRows());
column.ColumnSlice(rowOffset, sliceRowLength).SetValue((char)maskKind);
}
}
}
NDArrayView::NDArrayView(const NDShape& viewShape, const SparseIndexType* colStarts, const SparseIndexType* rowIndices, const ElementType* nonZeroValues, size_t numNonZeroValues, const DeviceDescriptor& device, bool readOnly/* = false*/)
: NDArrayView(AsDataType<ElementType>(), device, StorageFormat::SparseCSC, viewShape, false, AllocateTensorView<ElementType>(viewShape, StorageFormat::SparseCSC, device))
{
if ((colStarts == nullptr) || (rowIndices == nullptr) || (nonZeroValues == nullptr) || (numNonZeroValues == 0) || (numNonZeroValues > viewShape.TotalSize()))
InvalidArgument("Invalid sparse CSC format initial data specified for NDArrayView construction");
auto sparseMatrix = GetWritableMatrix<ElementType>(1);
sparseMatrix->SetMatrixFromCSCFormat(colStarts, rowIndices, nonZeroValues, numNonZeroValues, sparseMatrix->GetNumRows(), sparseMatrix->GetNumCols());
m_isReadOnly = readOnly;
}
bool MatrixMarketReader<FloatType>::MMReadHeader( const std::string &filename )
{
FILE *mm_file = ::fopen( filename.c_str( ), "r" );
if( mm_file == NULL )
{
printf( "Cannot Open Matrix-Market File !\n" );
return 1;
}
if ( MMReadHeader( mm_file ) )
{
printf ("Matrix not supported !\n");
return 2;
}
// If symmetric MM stored file, double the reported size
if( mm_is_symmetric( Typecode ) )
nNZ <<= 1;
::fclose( mm_file );
std::clog << "Matrix: " << filename << " [nRow: " << GetNumRows( ) << "] [nCol: " << GetNumCols( ) << "] [nNZ: " << GetNumNonZeroes( ) << "]" << std::endl;
return 0;
}
MBLayoutPtr SequencePacker::PackSparseStream(const StreamBatch& batch, size_t streamIndex)
{
assert(m_outputStreamDescriptions[streamIndex]->m_storageType == StorageType::sparse_csc);
// compute the aggregate nnz count of all the sequence in the batch.
size_t nnzCount = 0;
for (const auto& sequence : batch)
{
SparseSequenceDataPtr sparseSequence = static_pointer_cast<SparseSequenceData>(sequence);
nnzCount += sparseSequence->m_totalNnzCount;
}
if (nnzCount > numeric_limits<IndexType>::max())
{
RuntimeError("Minibatch NNZ count (%" PRIu64 ") exceeds the maximum allowed "
"value (%" PRIu64 ")\n", nnzCount, (size_t)numeric_limits<IndexType>::max());
}
const auto& stream = m_inputStreamDescriptions[streamIndex];
assert(stream->m_storageType == StorageType::sparse_csc);
auto elementSize = GetSizeByType(stream->m_elementType);
auto indexSize = sizeof(IndexType);
auto pMBLayout = CreateMBLayout(batch);
// Compute the required buffer size:
// size of nnz type + nnz * (size of the element type) + nnz * (size of the row index type) +
// (number of columns + 1) * (size of the column index type).
size_t requiredSize =
sizeof(nnzCount) +
nnzCount * (elementSize + indexSize) +
indexSize * (pMBLayout->GetNumCols() + 1);
auto& buffer = m_streamBuffers[m_currentBufferIndex][streamIndex];
if (buffer.m_size < requiredSize)
{
buffer.Resize(requiredSize);
}
auto* destination = buffer.m_data.get();
// insert the nnzCount as the first element in the buffer.
memcpy(destination, &nnzCount, sizeof(nnzCount));
// create two pointers to the memory blocks inside the buffer,
// one for data portion and anther -- for indices.
auto* dataDst = destination + sizeof(nnzCount);
auto* indicesDst = dataDst + elementSize* nnzCount;
// column index for the current sample (= number of nnz value packed so far).
IndexType columnOffset = 0;
// a vector to store column index for each sample in the resulting (packed) matrix.
vector<IndexType> sparseColumnIndices;
// a vector to keep track of the offsets into each input sequence,
// there an offset is the number of nnz values packed so far. Current sample
// values/indices start of the offset position in the sequence data/index array
vector<IndexType> sequenceOffsets(batch.size(), 0);
vector<MBLayout::SequenceInfo> sequenceInfos(pMBLayout->GetAllSequences());
// sort the vector in ascending order of the parallel sequence index.
sort(sequenceInfos.begin(), sequenceInfos.end(),
[](const MBLayout::SequenceInfo& a, const MBLayout::SequenceInfo& b){ return a.s < b.s; });
// Iterate over the all time steps in the layout (total number of samples/columns
// in a parallel sequence), traversing the layout in horizontal direction.
for (auto timeStep = 0; timeStep < pMBLayout->GetNumTimeSteps(); ++timeStep)
{
// For each time step, iterate over all sequences in the minibatch,
// traversing the layout in vertical direction.
for (const auto& sequenceInfo : sequenceInfos)
{
// skip the sequence if it does not intersect with the time step
if (timeStep < sequenceInfo.tBegin || timeStep >= sequenceInfo.tEnd)
{
continue;
}
// store the offset of the current column )...
sparseColumnIndices.push_back(columnOffset);
auto seqId = sequenceInfo.seqId;
if (seqId == GAP_SEQUENCE_ID)
{
continue;
}
// compute the index of the sample inside the sequence.
size_t sampleIndex = timeStep - sequenceInfo.tBegin;
const auto& sequence = batch[seqId];
// make sure the index less than the sequence length in samples.
assert(sampleIndex < sequence->m_numberOfSamples);
auto& sequenceOffset = sequenceOffsets[seqId];
SparseSequenceDataPtr sparseSequence = static_pointer_cast<SparseSequenceData>(sequence);
IndexType nnz = sparseSequence->m_nnzCounts[sampleIndex];
// compute the sample offset in bytes.
size_t sampleOffset = sequenceOffset * elementSize;
// copy all nzz values from source sequence into the buffer.
const auto* dataSrc = reinterpret_cast<const char*>(sequence->GetDataBuffer()) + sampleOffset;
memcpy(dataDst, dataSrc, nnz * elementSize);
dataDst += nnz * elementSize; // advance the destination pointer
// copy all nzz value indices from source sequence into the buffer.
const auto* indicesSrc = sparseSequence->m_indices + sequenceOffset;
memcpy(indicesDst, indicesSrc, nnz * indexSize);
indicesDst += nnz * indexSize; // advance the destination pointer
sequenceOffset += nnz;
columnOffset += nnz;
}
}
// at this point each element in sequenceOffsets should be equal to the total
// nnz count of the respective sequence and the sum of all elements - to the
// overall nnz count.
assert(accumulate(sequenceOffsets.begin(), sequenceOffsets.end(), 0) == nnzCount);
// check the distance between data and index destination pointers.
assert(indicesDst == dataDst + nnzCount * indexSize);
// after we packed all samples, the column offset must be equal to the total nnz count.
assert(columnOffset == nnzCount);
sparseColumnIndices.push_back(columnOffset);
// check that the number of column indices == N + 1 (where N is the number of
// column in the packed matrix)
assert((pMBLayout->GetNumCols() + 1) == sparseColumnIndices.size());
// verify that there's enough space in the buffer for the array of column indices.
assert(indicesDst + sparseColumnIndices.size()*indexSize <= destination + requiredSize);
// copy column indices into the buffer.
memcpy(indicesDst, sparseColumnIndices.data(), sparseColumnIndices.size() * indexSize);
return pMBLayout;
}
MBLayoutPtr SequencePacker::PackDenseStream(const StreamBatch& batch, size_t streamIndex)
{
assert(m_outputStreamDescriptions[streamIndex]->m_storageType == StorageType::dense);
const auto& stream = m_inputStreamDescriptions[streamIndex];
auto& buffer = m_streamBuffers[m_currentBufferIndex][streamIndex];
size_t sampleSize = GetSampleSize(m_outputStreamDescriptions[streamIndex]);
auto pMBLayout = CreateMBLayout(batch);
size_t requiredSize = pMBLayout->GetNumCols() * sampleSize;
if (buffer.m_size < requiredSize)
{
buffer.Resize(requiredSize);
}
auto elementSize = GetSizeByType(stream->m_elementType);
const auto& sequenceInfos = pMBLayout->GetAllSequences();
// Iterate over sequences in the layout, copy samples from the
// source sequences into the buffer (at appropriate offsets).
for (int i = 0; i < sequenceInfos.size(); ++i)
{
const auto& sequenceInfo = sequenceInfos[i];
// skip gaps
if (sequenceInfo.seqId == GAP_SEQUENCE_ID)
{
continue;
}
const auto& sequence = batch[sequenceInfo.seqId];
size_t numSamples = sequence->m_numberOfSamples;
assert(numSamples == sequenceInfo.GetNumTimeSteps());
char* bufferPtr = buffer.m_data.get();
// Iterate over all samples in the sequence, keep track of the sample offset (which is especially
// important for sparse input, where offset == number of preceding nnz elements).
for (size_t sampleIndex = 0, sampleOffset = 0; sampleIndex < numSamples; ++sampleIndex)
{
// Compute the offset into the destination buffer, using the layout information
// to get the column index corresponding to the given sample.
auto destinationOffset = pMBLayout->GetColumnIndex(sequenceInfo, sampleIndex) * sampleSize;
// verify that there's enough space left in the buffer to fit a full sample.
assert(destinationOffset <= buffer.m_size - sampleSize);
auto* destination = bufferPtr + destinationOffset;
if (stream->m_storageType == StorageType::dense)
{
// verify that the offset (an invariant for dense).
assert(sampleOffset == sampleIndex * sampleSize);
PackDenseSample(destination, sequence, sampleOffset, sampleSize);
sampleOffset += sampleSize;
}
else if (stream->m_storageType == StorageType::sparse_csc)
{
// TODO: make type casts members of the SparseSequenceData
SparseSequenceDataPtr sparseSequence = static_pointer_cast<SparseSequenceData>(sequence);
// make sure that the sequence meta-data is correct.
assert(numSamples == sparseSequence->m_nnzCounts.size());
PackSparseSampleAsDense(destination, sparseSequence, sampleIndex, sampleOffset, sampleSize, elementSize);
// move the offset by nnz count of the sample.
sampleOffset += sparseSequence->m_nnzCounts[sampleIndex];
// verify that the offset is within the bounds (less or equal
// to the total nnz count of the sequence).
assert(sampleOffset <= sparseSequence->m_totalNnzCount);
}
else
{
RuntimeError("Storage type %d is not supported.", (int)stream->m_storageType);
}
}
}
return pMBLayout;
}
int main(int argc, char *argv[]) {
char *inFileName = NULL;
char *outBaseName = NULL;
char *flowOrder = NULL;
int nFrame = 0;
// Parse command line arguments
processArgs(argc,argv,&inFileName,&outBaseName,&flowOrder,&nFrame);
// Determine number of flows, make sure it is a multiple of nFrame
int nLines = GetNumLines(inFileName);
if(0 != (nLines % nFrame)) {
fprintf(stderr,"Number of lines in input file %s is %d, should be a multiple of nFrames %d\n",inFileName,nLines,nFrame);
exit(EXIT_FAILURE);
}
int nFlow = nLines/nFrame;
// Determine number of wells, make sure we have at least one to process
int nCols;
nCols = GetNumCols(inFileName);
if(nCols < 2) {
fprintf(stderr,"Number of columns in input file %s is %d, should be at least 2 (1 background + multiple wells)\n",inFileName,nCols);
exit(EXIT_FAILURE);
}
int nWell = nCols-1;
// Read the input data
double *dat = NULL;
double *bkg = NULL;
readData(inFileName,nLines,nWell,&dat,&bkg);
//
// Fit background model
//
int my_nuc_block[NUMFB];
// TODO: Objects should be isolated!!!!
GlobalDefaultsForBkgModel::SetFlowOrder(flowOrder);
GlobalDefaultsForBkgModel::GetFlowOrderBlock(my_nuc_block,0,NUMFB);
InitializeLevMarSparseMatrices(my_nuc_block);
float sigma_guess=1.2;
float t0_guess=23;
float dntp_uM=50.0;
BkgModel *bkgmodel = new BkgModel(nWell,nFrame,sigma_guess,t0_guess,dntp_uM);
// Iterate over flows
struct bead_params p;
struct reg_params rp;
short *imgBuffer = new short[nWell * nFrame];
short *bkgBuffer = new short[nFrame];
float *out_sig = new float[nWell*nFlow];
float *out_sim_fg = new float[nWell*nFlow*nFrame];
float *out_sim_bg = new float[nFlow*nFrame];
memset(out_sim_fg,0,nWell*nFlow*nFrame);
memset(out_sim_bg,0,nFlow*nFrame);
int fitFramesPerFlow = 0;
for(int iFlow=0, iFlowBatch=0; iFlow < nFlow; iFlow++) {
//copy well data into imgBuffer and bkg data into bkgBufer
int flowOffset = iFlow * nFrame;
for(int iWell=0, iImg=0; iWell<nWell; iWell++)
for(int iFrame=0; iFrame<nFrame; iFrame++, iImg++)
imgBuffer[iImg] = (short) dat[iWell*nLines+flowOffset+iFrame];
for(int iFrame=0; iFrame<nFrame; iFrame++)
bkgBuffer[iFrame] = (short) bkg[iFlow*nFrame+iFrame];
// Pass data to background model
bool last_flow = (iFlow == (nFlow-1));
bkgmodel->ProcessImage(imgBuffer,bkgBuffer,iFlow,last_flow,false);
// If the model has been fit, store results
if ((((1+iFlow) % NUMFB) == 0) || last_flow) {
bkgmodel->GetRegParams(&rp);
for (int iWell=0; iWell < nWell; iWell++) {
bkgmodel->GetParams(iWell,&p);
// Store estimated signal per well
for (int iFlowParam=0, iFlowOut=iFlow-NUMFB+1; iFlowParam < NUMFB; iFlowParam++, iFlowOut++)
out_sig[iWell*nFlow+iFlowOut] = p.Ampl[iFlowParam]*p.Copies;
// Simulate data from the fitted model
float *fg,*bg,*feval,*isig,*pf;
int tot_frms = bkgmodel->GetModelEvaluation(iWell,&p,&rp,&fg,&bg,&feval,&isig,&pf);
fitFramesPerFlow = floor(tot_frms/(double)NUMFB);
for (int iFlowParam=0, iFlowOut=iFlow-NUMFB+1; iFlowParam < NUMFB; iFlowParam++, iFlowOut++) {
for (int iFrame=0; iFrame < fitFramesPerFlow; iFrame++) {
out_sim_fg[iWell*nLines+iFlowOut*nFrame+iFrame] = fg[iFlowParam*fitFramesPerFlow+iFrame];
if(iWell==0)
out_sim_bg[iFlowOut*nFrame+iFrame] = bg[iFlowParam*fitFramesPerFlow+iFrame];
}
}
}
iFlowBatch++;
}
}
// Cleanup
delete [] imgBuffer;
delete [] bkgBuffer;
delete bkgmodel;
CleanupLevMarSparseMatrices();
GlobalDefaultsForBkgModel::StaticCleanup();
free(dat);
free(bkg);
// Write out results
char *outFileSig = new char[strlen(outBaseName)+50];
sprintf(outFileSig,"%s.intensityEstimate.txt",outBaseName);
writeMatrix(outFileSig,out_sig,nFlow,nWell);
delete [] out_sig;
char *outFileSimFg = new char[strlen(outBaseName)+50];
sprintf(outFileSimFg,"%s.fittedSignal.txt",outBaseName);
writeMatrix(outFileSimFg,out_sim_fg,nFlow*nFrame,nWell);
delete [] out_sim_fg;
char *outFileSimBg = new char[strlen(outBaseName)+50];
sprintf(outFileSimBg,"%s.fittedBkg.txt",outBaseName);
writeMatrix(outFileSimBg,out_sim_bg,nFlow*nFrame,1);
delete [] out_sim_bg;
printf("\n");
printf("Simulated data has %d frames per flow\n",fitFramesPerFlow);
exit(EXIT_SUCCESS);
}
