void app::init_sound()
{
//Do sound setup
numOutputChannels = 8;
numInputChannels = 0;
global_sampleRate = 44100;
global_buffsize = 2048;
numWindows = 1;
//Setup the sound stream
//pa_soundstream.listDevices();
int outputIndex = findNOutputChannelIndex(numOutputChannels);
if(outputIndex == -1)
{
cout << "Could not find " << numOutputChannels << " channel output device." << endl;
soft_exit();
}
pa_soundstream.setDeviceID(outputIndex);
pa_soundstream.setup(this,numOutputChannels,numInputChannels,global_sampleRate,global_buffsize,numWindows);
//Volume init
global_volume = 0.0;
//Volumes
fr_vol = 1; //Front right vol
fl_vol = 1;
sr_vol = 1;
sl_vol = 1;
//Init audio effects
init_audio_effects();
}
void *
BigAllocator::allocate(size_t amountToAllocate)
{
//
// Round up to the allocation granularity.
//
if ((size_t)allocPointer % allocationGranularity != 0) {
allocPointer = (char *)((size_t)allocPointer + allocationGranularity - (size_t)allocPointer % allocationGranularity);
_ASSERT((size_t)allocPointer % allocationGranularity == 0);
}
if (allocPointer + amountToAllocate > basePointer + maxMemory) {
fprintf(stderr, "BigAllocator: allocating too much memory, %lld > %lld\n", allocPointer + amountToAllocate - basePointer , maxMemory);
soft_exit(1);
}
void *retVal = allocPointer;
allocPointer += amountToAllocate;
#if _DEBUG
if (nCanaries < maxCanaries) {
_ASSERT(allocPointer + sizeof(unsigned) <= basePointer + maxMemory);
canaries[nCanaries] = (unsigned *)allocPointer;
*canaries[nCanaries] = canaryValue;
nCanaries++;
allocPointer += sizeof(unsigned);
}
#endif // DEBUG
return retVal;
}
void
Genome::startContig(const char *contigName)
{
if (nContigs == maxContigs) {
//
// Reallocate (maybe we're sequencing a tree that's got lots of chromosomes).
//
int newMaxContigs = maxContigs * 2;
Contig *newContigs = new Contig[newMaxContigs];
if (NULL == newContigs) {
WriteErrorMessage("Genome: unable to reallocate contig array to size %d\n",newMaxContigs);
soft_exit(1);
}
for (int i = 0; i < nContigs; i++) {
newContigs[i] = contigs[i];
}
delete [] contigs;
contigs = newContigs;
maxContigs = newMaxContigs;
}
contigs[nContigs].beginningOffset = nBases;
size_t len = strlen(contigName) + 1;
contigs[nContigs].name = new char[len];
contigs[nContigs].nameLength = (unsigned)len-1;
strncpy(contigs[nContigs].name,contigName,len);
contigs[nContigs].name[len-1] = '\0';
nContigs++;
}
void BigDealloc(void *memory)
{
if (NULL == memory) return;
// Figure out the size we had allocated
char *startAddress = ((char *) memory) - sizeof(size_t);
size_t sizeAllocated = *((size_t *) startAddress);
if (munmap(startAddress, sizeAllocated) != 0) {
perror("munmap");
soft_exit(1);
}
}
void
Genome::addData(const char *data, GenomeDistance len)
{
if (nBases + len > GenomeLocationAsInt64(maxBases)) {
WriteErrorMessage("Tried to write beyond allocated genome size (or tried to write into a genome that was loaded from a file).\n"
"Size = %lld\n", GenomeLocationAsInt64(maxBases));
soft_exit(1);
}
memcpy(bases + nBases,data,len);
nBases += (unsigned)len;
}
void
Genome::addData(const char *data, size_t len)
{
if ((size_t)nBases + len > maxBases) {
WriteErrorMessage("Tried to write beyond allocated genome size (or tried to write into a genome that was loaded from a file).\n"
"Size = %lld\n",(_int64)maxBases);
soft_exit(1);
}
memcpy(bases + nBases,data,len);
nBases += (unsigned)len;
}
SimpleReadWriter(const FileFormat* i_format, DataWriter* i_writer, const Genome* i_genome, bool i_killIfTooSlow, bool i_emitInternalScore, char *i_internalScoreTag, bool i_ignoreAlignmentAdjustmentsForOm)
: format(i_format), writer(i_writer), genome(i_genome), killIfTooSlow(i_killIfTooSlow), lastTooSlowCheck(0), emitInternalScore(i_emitInternalScore), ignoreAlignmentAdjustmentsForOm(i_ignoreAlignmentAdjustmentsForOm)
{
if (emitInternalScore) {
if (strlen(i_internalScoreTag) != 2) {
WriteErrorMessage("SimpleReadWriter: bogus internal score tag\n");
soft_exit(1);
}
strcpy(internalScoreTag, i_internalScoreTag);
} else {
internalScoreTag[0] = '\0';
}
}
Genome::Genome(GenomeDistance i_maxBases, GenomeDistance nBasesStored, unsigned i_chromosomePadding, unsigned i_maxContigs)
: maxBases(i_maxBases), minLocation(0), maxLocation(i_maxBases), chromosomePadding(i_chromosomePadding), maxContigs(i_maxContigs),
mappedFile(NULL)
{
bases = ((char *) BigAlloc(nBasesStored + 2 * N_PADDING)) + N_PADDING;
if (NULL == bases) {
WriteErrorMessage("Genome: unable to allocate memory for %llu bases\n", GenomeLocationAsInt64(maxBases));
soft_exit(1);
}
// Add N's for the N_PADDING bases before and after the genome itself
memset(bases - N_PADDING, 'n', N_PADDING);
memset(bases + nBasesStored, 'n', N_PADDING);
nBases = 0;
nContigs = 0;
contigs = new Contig[maxContigs];
contigsByName = NULL;
}
void
SimpleReadWriter::checkIfTooSlow()
{
const _int64 tooSlowCheckPeriod = 5 * 60 * 1000; // 5 min in ms
const _int64 tooSlowCheckMinReadsPerCheckPeriod = 5 * 60 * 1000; // One read/ms (or 1000 reads/s, but just on this thread).
if (killIfTooSlow) {
_int64 now = timeInMillis();
if (lastTooSlowCheck + tooSlowCheckPeriod <= now) {
if (lastTooSlowCheck != 0 && writesSinceLastTooSlowCheck < tooSlowCheckMinReadsPerCheckPeriod) {
WriteErrorMessage("Only wrote %lld writes during a %lld minute check period; we're probably out of memory and are giving up because of -kts\n", writesSinceLastTooSlowCheck, tooSlowCheckPeriod / (60 * 1000));
soft_exit(1);
}
lastTooSlowCheck = now;
writesSinceLastTooSlowCheck = 0;
}
writesSinceLastTooSlowCheck++;
} // if (killIfTooSlow)
}
bool app::generalSettings(string name, string value)
{
//do_close variable
if(name == app::Setting_Do_Close)
{
if(value == "1")
{
soft_exit();
}
}
//Global volume
else if(name == app::Setting_GlobalVolume)
{
global_volume = atof(value.c_str());
}
else
{
return false;
}
return true;
}
void *BigAlloc(
#endif
size_t sizeToAllocate,
size_t *sizeAllocated)
{
// Make space to include the allocated size at the start of our region; this is necessary
// so that we can BigDealloc the memory later.
sizeToAllocate += sizeof(size_t);
const size_t ALIGN_SIZE = 4096;
if (sizeToAllocate % ALIGN_SIZE != 0) {
sizeToAllocate += ALIGN_SIZE - (sizeToAllocate % ALIGN_SIZE);
}
if (sizeAllocated != NULL) {
*sizeAllocated = sizeToAllocate - sizeof(size_t);
}
int flags = MAP_PRIVATE|MAP_ANONYMOUS;
#ifdef USE_HUGETLB
flags |= MAP_HUGETLB;
#endif
char *mem = (char *) mmap(NULL, sizeToAllocate, PROT_READ|PROT_WRITE, flags, -1, 0);
if (mem == MAP_FAILED) {
perror("mmap");
soft_exit(1);
}
#if (defined(MADV_HUGEPAGE) && !defined(USE_HUGETLB))
// Tell Linux to use huge pages for this range
if (BigAllocUseHugePages) {
if (madvise(mem, sizeToAllocate, MADV_HUGEPAGE) == -1) {
fprintf(stderr, "WARNING: failed to enable huge pages -- your kernel may not support it\n");
}
}
#endif
// Remember the size allocated in the first sizeof(size_t) bytes
*((size_t *) mem) = sizeToAllocate;
return (void *) (mem + sizeof(size_t));
}
Genome::Genome(unsigned i_maxBases, unsigned nBasesStored, unsigned i_chromosomePadding)
: maxBases(i_maxBases), minOffset(0), maxOffset(i_maxBases), chromosomePadding(i_chromosomePadding)
{
bases = ((char *) BigAlloc(nBasesStored + 2 * N_PADDING)) + N_PADDING;
if (NULL == bases) {
WriteErrorMessage("Genome: unable to allocate memory for %llu bases\n",(_int64)maxBases);
soft_exit(1);
}
// Add N's for the N_PADDING bases before and after the genome itself
memset(bases - N_PADDING, 'n', N_PADDING);
memset(bases + nBasesStored, 'n', N_PADDING);
nBases = 0;
maxContigs = 32; // A power of two that's bigger than the usual number of chromosomes, so we don't have to
// reallocate in practice.
nContigs = 0;
contigs = new Contig[maxContigs];
contigsByName = NULL;
}
void RunDaemonMode(int argc, const char **argv)
{
if (argc < 2 || argc > 3) {
daemonUsage();
}
printf("SNAP in daemon mode, waiting for commands to execute\n");
const char *pipeName = argc == 3 ? argv[2] : DEFAULT_NAMED_PIPE_NAME;
CommandPipe = OpenNamedPipe(pipeName, true);
if (NULL == CommandPipe) {
WriteErrorMessage("Unable to open named pipe for command IO.\n");
soft_exit(1);
}
const size_t commandBufferSize = 10000; // Yes, this is fixed size, no it's not a buffer overflow. The named pipe reader just quits if it's too long.
char commandBuffer[commandBufferSize];
//
// Format of commands is argc (in ascii) followed by argc arguments, each in one line.
//
for (;;) {
if (!ReadFromNamedPipe(CommandPipe, commandBuffer, commandBufferSize)) {
CloseNamedPipe(CommandPipe);
CommandPipe = NULL;
WriteStatusMessage("Named pipe closed. Exiting\n");
soft_exit_no_print(0);
}
int argc = atoi(commandBuffer);
if (0 == argc) {
WriteErrorMessage("Expected argument count on named pipe, got '%s'; ignoring.\n", commandBuffer);
} else {
char **argv = new char*[argc];
for (int i = 0; i < argc; i++) {
argv[i] = new char[commandBufferSize];
if (!ReadFromNamedPipe(CommandPipe, argv[i], commandBufferSize)) {
CloseNamedPipe(CommandPipe);
CommandPipe = NULL;
WriteStatusMessage("Error reading argument #%d from named pipe.\n", i);
soft_exit(1);
}
} // for each arg
if (argc > 1 && strcmp(argv[1], "exit") == 0) {
WriteStatusMessage("SNAP server exiting by request\n");
WriteToNamedPipe(CommandPipe, CommandExecutedString);
soft_exit_no_print(1);
}
printf("Executing command: ");
for (int i = 1; i < argc; i++) {
printf("%s ", argv[i]);
}
printf("\n");
ProcessNonDaemonCommands(argc, (const char **) argv);
printf("\n");
for (int i = 0; i < argc; i++) {
delete[] argv[i];
argv[i] = NULL;
}
delete[] argv;
argv = NULL;
}
WriteToNamedPipe(CommandPipe, CommandExecutedString);
}
}
SNAPHashTable *SNAPHashTable::loadFromBlob(GenericFile_Blob *loadFile)
{
SNAPHashTable *table = new SNAPHashTable();
unsigned fileMagic;
if (sizeof(magic) != loadFile->read(&fileMagic, sizeof(magic))) {
WriteErrorMessage("Magic number mismatch on hash table load. %d != %d\n", fileMagic, magic);
soft_exit(1);
}
if (fileMagic != magic) {
WriteErrorMessage("SNAPHashTable: magic number mismatch. Perhaps you have a corruped index. %d != %d\n", fileMagic, magic);
soft_exit(1);
}
if (sizeof(table->tableSize) != loadFile->read(&table->tableSize, sizeof(table->tableSize))) {
WriteErrorMessage("SNAPHashTable::SNAPHashTable fread table size failed\n");
soft_exit(1);
}
if (sizeof(table->usedElementCount) != loadFile->read(&table->usedElementCount, sizeof(table->usedElementCount))) {
WriteErrorMessage("SNAPHashTable::SNAPHashTable fread used element count failed\n");
soft_exit(1);
}
if (sizeof(table->keySizeInBytes) != loadFile->read(&table->keySizeInBytes, sizeof(table->keySizeInBytes))) {
WriteErrorMessage("SNAPHashTable::SNAPHashTable fread keySizeInBytes size failed. Perhaps this is an old format hash table and needs to be rebuilt.\n");
soft_exit(1);
}
if (table->keySizeInBytes < 4 || table->keySizeInBytes > 8) {
WriteErrorMessage("SNAPHashTable::SNAPHashTable Key size must be between 4 and 8 inclusive. Perhaps this is an old format hash table and needs to be rebuilt.\n");
soft_exit(1);
}
if (sizeof(table->valueSizeInBytes) != loadFile->read(&table->valueSizeInBytes, sizeof(table->valueSizeInBytes))) {
WriteErrorMessage("SNAPHashTable::SNAPHashTable fread dataSizeInBytes size failed. Perhaps this is an old format hash table and needs to be rebuilt.\n");
soft_exit(1);
}
if (table->valueSizeInBytes == 0 || table->valueSizeInBytes > sizeof(_uint64)) {
//
// It must be at least one byte, because we need that much for the unused value value. The code stuffs
// values into _uint64, so it can't be bigger than that.
//
WriteErrorMessage(
"SNAPHashTable::SNAPHashTable value size in bytes (%d) must be between 1 and 8. Perhaps you have a hash table from a future version of SNAP? Or else it's corrupt.\n", table->valueSizeInBytes);
soft_exit(1);
}
if (sizeof(table->valueCount) != loadFile->read(&table->valueCount, sizeof(table->valueCount))) {
WriteErrorMessage("SNAPHashTable::SNAPHashTable: value count failed to read.\n");
soft_exit(1);
}
if (table->valueCount == 0 || table->valueCount > 2) {
// Technically, > 2 would work fine with the code, but SNAP doesn't use it, so the check is here to detect corruption.
WriteErrorMessage("SNAPHashTable::SNAPHashTable: invalid value count (%d), possible corruption or bad file format.\n", table->valueCount);
soft_exit(1);
}
table->invalidValueValue = 0; // Need this in case valueSizeInBytes < sizeof(ValueType)
if (table->valueSizeInBytes != loadFile->read(&table->invalidValueValue, table->valueSizeInBytes)) {
WriteErrorMessage("SNAPHashTable::SNAPHashTable: unable to read invalid value value\n");
soft_exit(1);
}
if (table->tableSize <= 0) {
WriteErrorMessage("SNAPHashTable::SNAPHashTable Zero or negative hash table size\n");
soft_exit(1);
}
table->elementSize = table->keySizeInBytes + table->valueSizeInBytes * table->valueCount;
size_t bytesMapped;
table->Table = loadFile->mapAndAdvance(table->tableSize * table->elementSize, &bytesMapped);
if (bytesMapped != table->tableSize * table->elementSize) {
WriteErrorMessage("SNAPHashTable: unable to map table\n");
soft_exit(1);
}
table->ownsMemoryForTable = false;
return table;
}
void *BigAllocInternal(
size_t sizeToAllocate,
size_t *sizeAllocated,
bool reserveOnly,
size_t *pageSize)
/*++
Routine Description:
Allocate memory, using large pages if both appropriate and possible, and always using
VirtualAlloc (meaning that this will always use at least one VM page, so you shouldn't
use it for small stuff, only gigantic data structures for which you want to reduce TLB
misses and cache misses on the page table). Use malloc or new for ordinary allocations.
Arguments:
sizeToAllocate - The amount of memory that is needed
sizeAllocated - Optional parameter that if provided returns the amount of memory actually allocated, which
will always be >= sizeToAllocate (unless the allocation fails).
reserveOnly - If TRUE, will only reserve address space, must call BigCommit to commit memory
pageSize - Optional parameter that if provided returns the page size (not large page size)
Return Value:
pointer to the memory allocated, or NULL if the allocation failed.
--*/
{
if (sizeToAllocate == 0) {
sizeToAllocate = 1;
}
static bool warningPrinted = false;
void *allocatedMemory;
SYSTEM_INFO systemInfo[1];
GetSystemInfo(systemInfo);
size_t virtualAllocSize = ((sizeToAllocate + systemInfo->dwPageSize - 1) / systemInfo->dwPageSize) * systemInfo->dwPageSize;
if (pageSize != NULL) {
*pageSize = systemInfo->dwPageSize;
}
//
// Try to do the VirtualAlloc using large pages if the size we're getting is at last one large page.
// Callers should have asserted the SeLockMemoryPrivilege if they want large pages.
//
size_t largePageSize = GetLargePageMinimum();
DWORD commitFlag = reserveOnly ? 0 : MEM_COMMIT;
if (0 != largePageSize && virtualAllocSize >= largePageSize) {
//
// Start by asserting the SeLockMemoryPrivilege, which is necessary for large page allocations. It's overkill to
// do this every time, it only has to happen once/thread. However, a BigAllocation is a big deal and shouldn't be
// happening very much, so we just don't worry about the extra cost.
//
BOOL assertPrivilegeWorked = AssertPrivilege("SeLockMemoryPrivilege");
DWORD assertPrivilegeError = GetLastError();
size_t largePageSizeToAllocate = ((virtualAllocSize + largePageSize - 1) / largePageSize) * largePageSize;
#if _DEBUG
largePageSizeToAllocate += largePageSize; // For the guard page.
#endif // DEBUG
allocatedMemory = (BYTE *)VirtualAlloc(0,largePageSizeToAllocate,commitFlag|MEM_RESERVE|((BigAllocUseHugePages && !reserveOnly) ? MEM_LARGE_PAGES : 0),PAGE_READWRITE);
if (NULL != allocatedMemory) {
#if _DEBUG
DWORD oldProtect;
if (!VirtualProtect((char *)allocatedMemory + virtualAllocSize, systemInfo->dwPageSize, PAGE_NOACCESS, &oldProtect)) {
static bool printedVirtualProtectedWarning = false;
if (! printedVirtualProtectedWarning) {
fprintf(stderr,"VirtualProtect for guard page failed, %d\n", GetLastError());
printedVirtualProtectedWarning = true;
}
}
largePageSizeToAllocate -= largePageSize; // Back out the guard page
#endif // DEBUG
if (NULL != sizeAllocated) {
*sizeAllocated = largePageSizeToAllocate;
}
return allocatedMemory;
} else if (!warningPrinted) {
//
// The first time we fail, print out a warning and then fall back to VirtualAlloc. We want be able to use
// the fallback because the caller might not be able to assert the appropriate privilege and we'd still like
// to run. The check for printing only once isn't thread safe, so you might get more than one printed
// if multiple threads fail at the same time.
//
warningPrinted = true;
fprintf(stderr,"BigAlloc: WARNING: Unable to allocate large page memory, %d. Falling back to VirtualAlloc. Performance may be adversely affected. Size = %lld\n", GetLastError(), largePageSizeToAllocate);
if (!assertPrivilegeWorked || GetLastError() == 1314) { // TODO: Look up the error code name for 1314.
fprintf(stderr,"BigAlloc: Unable to assert the SeLockMemoryPrivilege (%d), which is probably why it failed.\n",assertPrivilegeError);
fprintf(stderr,"Try secpol.msc, then SecuritySettings, Local Policies, User Rights Assignment.\n");
fprintf(stderr,"Then double click 'Lock Pages in Memory,' add the current user directly or by being\n");
fprintf(stderr,"In a group and then reboot (you MUST reboot) for it to work.\n");
}
}
}
allocatedMemory = (BYTE *)VirtualAlloc(0,virtualAllocSize,commitFlag|MEM_RESERVE,PAGE_READWRITE);
if (NULL != allocatedMemory && NULL != sizeAllocated) {
*sizeAllocated = virtualAllocSize;
}
if (NULL == allocatedMemory) {
fprintf(stderr,"BigAlloc of size %lld failed.\n", sizeToAllocate);
soft_exit(1);
}
return allocatedMemory;
}
bool
SimpleReadWriter::writeReads(
const ReaderContext& context,
Read *read,
SingleAlignmentResult *results,
int nResults,
bool firstIsPrimary)
{
char* buffer;
size_t size;
size_t used;
bool result = false;
for (int i = 0; i < nResults; i++) {
if (results[i].status == NotFound) {
results[i].location = InvalidGenomeLocation;
}
}
//
// We need to keep track of the offsets of all of the alignments in the output buffer so we can commit them. However,
// we want to avoid dynamic memory allocation as much as possible. So, we have a static buffer on the stack that's big enough
// for the great majority of cases, and then allocate dynamically if that's too small. Makes for annoying, but efficient
// code.
//
const int staticUsedBufferSize = 2000;
size_t staticUsedBuffer[staticUsedBufferSize];
GenomeLocation staticFinalLocationsBuffer[staticUsedBufferSize];
size_t *usedBuffer;
GenomeLocation *finalLocations;
if (nResults <= staticUsedBufferSize) {
usedBuffer = staticUsedBuffer;
finalLocations = staticFinalLocationsBuffer;
} else {
usedBuffer = new size_t[nResults];
finalLocations = new GenomeLocation[nResults];
}
for (int pass = 0; pass < 2; pass++) { // Make two passes, one with whatever buffer space is left and one with a clean buffer.
bool blewBuffer = false;
if (!writer->getBuffer(&buffer, &size)) {
goto done;
}
used = 0;
for (int whichResult = 0; whichResult < nResults; whichResult++) {
int addFrontClipping = 0;
read->setAdditionalFrontClipping(0);
int cumulativeAddFrontClipping = 0;
finalLocations[whichResult] = results[whichResult].location;
while (!format->writeRead(context, &lvc, buffer + used, size - used, &usedBuffer[whichResult], read->getIdLength(), read, results[whichResult].status,
results[whichResult].mapq, finalLocations[whichResult], results[whichResult].direction, (whichResult > 0) || !firstIsPrimary, &addFrontClipping)) {
if (0 == addFrontClipping) {
blewBuffer = true;
break;
}
// redo if read modified (e.g. to add soft clipping, or move alignment for a leading I.
const Genome::Contig *originalContig = results[whichResult].status == NotFound ? NULL
: genome->getContigAtLocation(results[whichResult].location);
const Genome::Contig *newContig = results[whichResult].status == NotFound ? NULL
: genome->getContigAtLocation(results[whichResult].location + addFrontClipping);
if (newContig == NULL || newContig != originalContig || finalLocations[whichResult] + addFrontClipping > originalContig->beginningLocation + originalContig->length - genome->getChromosomePadding()) {
//
// Altering this would push us over a contig boundary. Just give up on the read.
//
results[whichResult].status = NotFound;
results[whichResult].location = InvalidGenomeLocation;
finalLocations[whichResult] = InvalidGenomeLocation;
} else {
cumulativeAddFrontClipping += addFrontClipping;
if (addFrontClipping > 0) {
read->setAdditionalFrontClipping(cumulativeAddFrontClipping);
}
finalLocations[whichResult] = results[whichResult].location + cumulativeAddFrontClipping;
}
} // while formatting doesn't work
if (blewBuffer) {
break;
}
used += usedBuffer[whichResult];
_ASSERT(used <= size);
if (used > 0xffffffff) {
WriteErrorMessage("SimpleReadWriter:writeReads: used too big\n");
soft_exit(1);
}
} // for each result.
if (!blewBuffer) {
//
// Everything worked OK.
//
for (int whichResult = 0; whichResult < nResults; whichResult++) {
writer->advance((unsigned)usedBuffer[whichResult], finalLocations[whichResult]);
}
result = true;
goto done;
}
if (pass == 1) {
WriteErrorMessage("Failed to write into fresh buffer; trying providing the -wbs switch with a larger value\n");
soft_exit(1);
}
if (!writer->nextBatch()) {
goto done;
}
} // for each pass (i.e., not empty, empty buffer)
done:
if (usedBuffer != staticUsedBuffer) {
delete[] usedBuffer;
usedBuffer = NULL;
delete[] finalLocations;
finalLocations = NULL;
}
read->setAdditionalFrontClipping(0);
return result;
}
const Genome *
Genome::loadFromFile(const char *fileName, unsigned chromosomePadding, unsigned i_minOffset, unsigned length)
{
GenericFile *loadFile;
unsigned nBases,nContigs;
if (!openFileAndGetSizes(fileName,&loadFile,&nBases,&nContigs)) {
//
// It already printed an error. Just fail.
//
return NULL;
}
if (0 == length) {
length = nBases - i_minOffset;
} else {
//
// Don't let length go beyond nBases.
//
length = __min(length,nBases - i_minOffset);
}
Genome *genome = new Genome(nBases,length, chromosomePadding);
genome->nBases = nBases;
genome->nContigs = genome->maxContigs = nContigs;
genome->contigs = new Contig[nContigs];
genome->minOffset = i_minOffset;
if (i_minOffset >= nBases) {
WriteErrorMessage("Genome::loadFromFile: specified minOffset %u >= nBases %u\n",i_minOffset,nBases);
}
genome->maxOffset = i_minOffset + length;
static const unsigned contigNameBufferSize = 512;
char contigNameBuffer[contigNameBufferSize];
unsigned n;
size_t contigSize;
char *curName;
for (unsigned i = 0; i < nContigs; i++) {
if (NULL == loadFile->gets(contigNameBuffer, contigNameBufferSize)){
WriteErrorMessage("Unable to read contig description\n");
delete genome;
return NULL;
}
for (n = 0; n < contigNameBufferSize; n++){
if (contigNameBuffer[n] == ' ') {
contigNameBuffer[n] = '\0';
break;
}
}
genome->contigs[i].beginningOffset = atoi(contigNameBuffer);
contigNameBuffer[n] = ' ';
n++; // increment n so we start copying at the position after the space
contigSize = strlen(contigNameBuffer + n) - 1; //don't include the final \n
genome->contigs[i].name = new char[contigSize + 1];
genome->contigs[i].nameLength = (unsigned)contigSize;
curName = genome->contigs[i].name;
for (unsigned pos = 0; pos < contigSize; pos++) {
curName[pos] = contigNameBuffer[pos + n];
}
curName[contigSize] = '\0';
}
//
// Skip over the miserable \n that gets left in the file.
//
/* char newline;
if (1 != fread(&newline,1,1,loadFile)) {
WriteErrorMessage("Genome::loadFromFile: Unable to read expected newline\n");
delete genome;
return NULL;
}
if (newline != 10) {
WriteErrorMessage("Genome::loadFromFile: Expected newline to be 0x0a, got 0x%02x\n",newline);
delete genome;
return NULL;
}
*/
if (0 != loadFile->advance(i_minOffset)) {
WriteErrorMessage("Genome::loadFromFile: _fseek64bit failed\n");
soft_exit(1);
}
size_t retval;
if (length != (retval = loadFile->read(genome->bases,length))) {
WriteErrorMessage("Genome::loadFromFile: fread of bases failed; wanted %u, got %d\n", length, retval);
loadFile->close();
delete loadFile;
delete genome;
return NULL;
}
loadFile->close();
delete loadFile;
genome->fillInContigLengths();
genome->sortContigsByName();
return genome;
}
bool
SimpleReadWriter::writePairs(
const ReaderContext& context,
Read **reads /* array of size NUM_READS_PER_PAIR */,
PairedAlignmentResult *result,
int nResults,
SingleAlignmentResult **singleResults /* array of size NUM_READS_PER_PAIR*/,
int *nSingleResults /* array of size NUM_READS_PER_PAIR*/,
bool firstIsPrimary)
{
bool retVal = false;
//
// We need to write all alignments for the pair into the same buffer, so that a write from
// some other thread doesn't separate them. We make two passes, trying to write into the
// existing buffer, and then into a clean one. If that doesn't work, abort the alignment
// run and ask for a bigger write buffer.
//
const int staticUsedBufferSize = 2000;
size_t staticUsedBuffer[NUM_READS_PER_PAIR][staticUsedBufferSize];
GenomeLocation staticLocationBuffer[NUM_READS_PER_PAIR][staticUsedBufferSize];
GenomeLocation *finalLocations[NUM_READS_PER_PAIR];
size_t *usedBuffer[NUM_READS_PER_PAIR];
if (nResults + nSingleResults[0] <= staticUsedBufferSize && nResults + nSingleResults[1] <= staticUsedBufferSize) {
usedBuffer[0] = staticUsedBuffer[0];
usedBuffer[1] = staticUsedBuffer[1];
finalLocations[0] = staticLocationBuffer[0];
finalLocations[1] = staticLocationBuffer[1];
} else {
usedBuffer[0] = new size_t[nResults * NUM_READS_PER_PAIR + nSingleResults[0] + nSingleResults[1]];
usedBuffer[1] = usedBuffer[0] + nResults + nSingleResults[0];
finalLocations[0] = new GenomeLocation[nResults * NUM_READS_PER_PAIR + nSingleResults[0] + nSingleResults[1]];
finalLocations[1] = finalLocations[0] + nResults + nSingleResults[0];
}
//
// For paired reads, we need to have the same QNAME for both of them, and it needs to be unique among all other
// reads in the dataset. For now, all we do is see if the read names end in /1 and /2, and if so truncate them.
//
size_t idLengths[NUM_READS_PER_PAIR];
idLengths[0] = reads[0]->getIdLength();
idLengths[1] = reads[1]->getIdLength();
if (idLengths[0] == idLengths[1] && idLengths[0] > 2 && reads[0]->getId()[idLengths[0]-2] == '/' && reads[1]->getId()[idLengths[0]-2] == '/') {
char lastChar0, lastChar1;
lastChar0 = reads[0]->getId()[idLengths[0] - 1];
lastChar1 = reads[1]->getId()[idLengths[1] - 1];
if ((lastChar0 == '1' || lastChar0 == '2') && (lastChar1 == '1' || lastChar1 == '2') &&
lastChar0 != lastChar1) {
idLengths[0] -= 2;
idLengths[1] -= 2;
}
}
for (int pass = 0; pass < 2; pass++) {
char* buffer;
size_t size;
size_t used = 0;
bool fitInBuffer = true;
if (!writer->getBuffer(&buffer, &size)) {
goto done;
}
//
// Write all of the pair alignments into the buffer.
//
for (int whichAlignmentPair = 0; whichAlignmentPair < nResults; whichAlignmentPair++) {
reads[0]->setAdditionalFrontClipping(0);
reads[1]->setAdditionalFrontClipping(0);
GenomeLocation locations[2];
locations[0] = result[whichAlignmentPair].status[0] != NotFound ? result[whichAlignmentPair].location[0] : InvalidGenomeLocation;
locations[1] = result[whichAlignmentPair].status[1] != NotFound ? result[whichAlignmentPair].location[1] : InvalidGenomeLocation;
int writeOrder[2]; // The order in which we write the reads, which is just numerical by genome location. SO writeOrder[0] gets written first, and writeOrder[1] second.
if (locations[0] <= locations[1]) {
writeOrder[0] = 0;
writeOrder[1] = 1;
} else {
writeOrder[0] = 1;
writeOrder[1] = 0;
}
bool secondReadLocationChanged;
int cumulativePositiveAddFrontClipping[NUM_READS_PER_PAIR] = { 0, 0 };
do {
size_t tentativeUsed = 0;
secondReadLocationChanged = false;
for (int firstOrSecond = 0; firstOrSecond < NUM_READS_PER_PAIR; firstOrSecond++) { // looping over the order in which the reads are written, not the order in which they arrived
int whichRead = writeOrder[firstOrSecond];
//
// Loop until we get a write with no additional front clipping.
//
int addFrontClipping = 0;
while (!format->writeRead(context, &lvc, buffer + used + tentativeUsed, size - used - tentativeUsed, &usedBuffer[firstOrSecond][whichAlignmentPair],
idLengths[whichRead], reads[whichRead], result[whichAlignmentPair].status[whichRead], result[whichAlignmentPair].mapq[whichRead], locations[whichRead], result[whichAlignmentPair].direction[whichRead],
whichAlignmentPair != 0 || !firstIsPrimary, &addFrontClipping, true, writeOrder[firstOrSecond] == 0,
reads[1 - whichRead], result[whichAlignmentPair].status[1 - whichRead], locations[1 - whichRead], result[whichAlignmentPair].direction[1 - whichRead],
result[whichAlignmentPair].alignedAsPair)) {
if (0 == addFrontClipping || locations[whichRead] == InvalidGenomeLocation) {
//
// We failed because we ran out of buffer.
//
goto blownBuffer;
}
if (1 == firstOrSecond) {
//
// If the location of the second read changed, we need to redo the first one as well, because it includes an offset to the second read
//
secondReadLocationChanged = true;
}
const Genome::Contig *originalContig = genome->getContigAtLocation(locations[whichRead]);
const Genome::Contig *newContig = genome->getContigAtLocation(locations[whichRead] + addFrontClipping);
if (newContig != originalContig || NULL == newContig || locations[whichRead] + addFrontClipping > originalContig->beginningLocation + originalContig->length - genome->getChromosomePadding()) {
//
// Altering this would push us over a contig boundary. Just give up on the read.
//
result[whichAlignmentPair].status[whichRead] = NotFound;
result[whichAlignmentPair].location[whichRead] = InvalidGenomeLocation;
locations[whichRead] = InvalidGenomeLocation;
} else {
if (addFrontClipping > 0) {
cumulativePositiveAddFrontClipping[firstOrSecond] += addFrontClipping;
reads[whichRead]->setAdditionalFrontClipping(cumulativePositiveAddFrontClipping[firstOrSecond]);
}
locations[whichRead] += addFrontClipping;
}
} // While formatting didn't work
tentativeUsed += usedBuffer[firstOrSecond][whichAlignmentPair];
} // for first or second read
} while (secondReadLocationChanged);
used += usedBuffer[0][whichAlignmentPair] + usedBuffer[1][whichAlignmentPair];
//
// Both reads are written into the buffer. Save the final locations we used for when we commit.
//
for (int whichRead = 0; whichRead < NUM_READS_PER_PAIR; whichRead++) {
finalLocations[whichRead][whichAlignmentPair] = locations[whichRead];
}
} // for each pair.
//
// Now write the single alignments.
//
for (int whichRead = 0; whichRead < NUM_READS_PER_PAIR; whichRead++) {
for (int whichAlignment = 0; whichAlignment < nSingleResults[whichRead]; whichAlignment++) {
int addFrontClipping;
reads[whichRead]->setAdditionalFrontClipping(0);
GenomeLocation location = singleResults[whichRead][whichAlignment].status != NotFound ? singleResults[whichRead][whichAlignment].location : InvalidGenomeLocation;
int cumulativePositiveAddFrontClipping = 0;
while (!format->writeRead(context, &lvc, buffer + used, size - used, &usedBuffer[whichRead][nResults + whichAlignment], reads[whichRead]->getIdLength(),
reads[whichRead], singleResults[whichRead][whichAlignment].status, singleResults[whichRead][whichAlignment].mapq, location, singleResults[whichRead][whichAlignment].direction,
true, &addFrontClipping)) {
if (0 == addFrontClipping) {
goto blownBuffer;
}
const Genome::Contig *originalContig = genome->getContigAtLocation(location);
const Genome::Contig *newContig = genome->getContigAtLocation(location + addFrontClipping);
if (newContig != originalContig || NULL == newContig || location + addFrontClipping > originalContig->beginningLocation + originalContig->length - genome->getChromosomePadding()) {
//
// Altering this would push us over a contig boundary. Just give up on the read.
//
singleResults[whichRead][whichAlignment].status = NotFound;
location = InvalidGenomeLocation;
} else {
if (addFrontClipping > 0) {
cumulativePositiveAddFrontClipping += addFrontClipping;
reads[whichRead]->setAdditionalFrontClipping(cumulativePositiveAddFrontClipping);
}
location += addFrontClipping;
}
}
finalLocations[whichRead][nResults + whichAlignment] = location;
used += usedBuffer[whichRead][nResults + whichAlignment];
} // For each single alignment of a read
} // For each read
//
// They all fit into the buffer.
//
//
// Commit the updates for the pairs.
//
for (int whichReadPair = 0; whichReadPair < nResults; whichReadPair++) {
for (int firstOrSecond = 0; firstOrSecond < NUM_READS_PER_PAIR; firstOrSecond++) {
// adjust for write order
int writeFirstOrSecond = (!!firstOrSecond) ^ (finalLocations[0][whichReadPair] > finalLocations[1][whichReadPair]); // goofy looking !! converts int to bool
writer->advance((unsigned)usedBuffer[firstOrSecond][whichReadPair],
finalLocations[writeFirstOrSecond][whichReadPair] == InvalidGenomeLocation ? finalLocations[1 - writeFirstOrSecond][whichReadPair] : finalLocations[writeFirstOrSecond][whichReadPair]);
}
}
//
// Now commit the updates for the single reads.
//
for (int whichRead = 0; whichRead < NUM_READS_PER_PAIR; whichRead++) {
for (int whichAlignment = 0; whichAlignment < nSingleResults[whichRead]; whichAlignment++) {
writer->advance((unsigned)usedBuffer[whichRead][nResults + whichAlignment], finalLocations[whichRead][nResults + whichAlignment]);
}
}
retVal = true;
break;
blownBuffer:
if (pass > 0) {
WriteErrorMessage("Unable to fit all alignments for one read pair into a single write buffer. Increase the size of the write buffer with -wbs, or reduce the number of alignments with -om or -omax\n");
WriteErrorMessage("Read id: '%.*s'\n", reads[0]->getIdLength(), reads[0]->getId());
soft_exit(1);
}
if (!writer->nextBatch()) {
goto done;
}
} // For each buffer full pass
done:
if (usedBuffer[0] != staticUsedBuffer[0]) {
delete[] usedBuffer[0];
usedBuffer[0] = usedBuffer[1] = NULL;
delete[] finalLocations[0];
finalLocations[0] = finalLocations[1] = NULL;
}
reads[0]->setAdditionalFrontClipping(0);
reads[1]->setAdditionalFrontClipping(0);
return retVal;
}
const Genome *
Genome::loadFromFile(const char *fileName, unsigned chromosomePadding, GenomeLocation minLocation, GenomeDistance length, bool map)
{
GenericFile *loadFile;
GenomeDistance nBases;
unsigned nContigs;
if (!openFileAndGetSizes(fileName, &loadFile, &nBases, &nContigs, map)) {
//
// It already printed an error. Just fail.
//
return NULL;
}
GenomeLocation maxLocation(nBases);
if (0 == length) {
length = maxLocation - minLocation;
} else {
//
// Don't let length go beyond nBases.
//
length = __min(length, maxLocation - minLocation);
maxLocation = minLocation + length;
}
Genome *genome = new Genome(nBases, length, chromosomePadding);
genome->nBases = nBases;
genome->nContigs = genome->maxContigs = nContigs;
genome->contigs = new Contig[nContigs];
genome->minLocation = minLocation;
if (GenomeLocationAsInt64(minLocation) >= nBases) {
WriteErrorMessage("Genome::loadFromFile: specified minOffset %u >= nBases %u\n", GenomeLocationAsInt64(minLocation), nBases);
soft_exit(-1);
}
genome->maxLocation = maxLocation;
static const unsigned contigNameBufferSize = 512;
char contigNameBuffer[contigNameBufferSize];
unsigned n;
size_t contigSize;
char *curName;
for (unsigned i = 0; i < nContigs; i++) {
if (NULL == loadFile->gets(contigNameBuffer, contigNameBufferSize)){
WriteErrorMessage("Unable to read contig description\n");
delete genome;
return NULL;
}
for (n = 0; n < contigNameBufferSize; n++){
if (contigNameBuffer[n] == ' ') {
contigNameBuffer[n] = '\0';
break;
}
}
_int64 contigStart;
if (1 != sscanf(contigNameBuffer, "%lld", &contigStart)) {
WriteErrorMessage("Unable to parse contig start in genome file '%s', '%s%'\n", fileName, contigNameBuffer);
soft_exit(1);
}
genome->contigs[i].beginningLocation = GenomeLocation(contigStart);
contigNameBuffer[n] = ' ';
n++; // increment n so we start copying at the position after the space
contigSize = strlen(contigNameBuffer + n) - 1; //don't include the final \n
genome->contigs[i].name = new char[contigSize + 1];
genome->contigs[i].nameLength = (unsigned)contigSize;
curName = genome->contigs[i].name;
for (unsigned pos = 0; pos < contigSize; pos++) {
curName[pos] = contigNameBuffer[pos + n];
}
curName[contigSize] = '\0';
}
if (0 != loadFile->advance(GenomeLocationAsInt64(minLocation))) {
WriteErrorMessage("Genome::loadFromFile: _fseek64bit failed\n");
soft_exit(1);
}
size_t readSize;
if (map) {
GenericFile_map *mappedFile = (GenericFile_map *)loadFile;
genome->bases = (char *)mappedFile->mapAndAdvance(length, &readSize);
genome->mappedFile = mappedFile;
mappedFile->prefetch();
} else {
readSize = loadFile->read(genome->bases, length);
loadFile->close();
delete loadFile;
loadFile = NULL;
}
if (length != readSize) {
WriteErrorMessage("Genome::loadFromFile: fread of bases failed; wanted %u, got %d\n", length, readSize);
delete loadFile;
delete genome;
return NULL;
}
genome->fillInContigLengths();
genome->sortContigsByName();
return genome;
}
