PICO_FUNC picoext_getEngineMemUsage(
pico_Engine engine,
pico_Int16 resetIncremental,
pico_Int32 *outUsedBytes,
pico_Int32 *outIncrUsedBytes,
pico_Int32 *outMaxUsedBytes
)
{
pico_Status status = PICO_OK;
if (!picoctrl_isValidEngineHandle((picoctrl_Engine) engine)) {
status = PICO_ERR_INVALID_HANDLE;
} else if ((outUsedBytes == NULL) || (outIncrUsedBytes == NULL) || (outMaxUsedBytes == NULL)) {
status = PICO_ERR_NULLPTR_ACCESS;
} else {
picoos_Common common = picoctrl_engGetCommon((picoctrl_Engine) engine);
status = getMemUsage(common, resetIncremental != 0, outUsedBytes, outIncrUsedBytes, outMaxUsedBytes);
}
return status;
}
PICO_FUNC picoext_getSystemMemUsage(
pico_System system,
pico_Int16 resetIncremental,
pico_Int32 *outUsedBytes,
pico_Int32 *outIncrUsedBytes,
pico_Int32 *outMaxUsedBytes
)
{
pico_Status status = PICO_OK;
if (!is_valid_system_handle(system)) {
status = PICO_ERR_INVALID_HANDLE;
} else if ((outUsedBytes == NULL) || (outIncrUsedBytes == NULL) || (outMaxUsedBytes == NULL)) {
status = PICO_ERR_NULLPTR_ACCESS;
} else {
picoos_Common common = pico_sysGetCommon(system);
status = getMemUsage(common, resetIncremental != 0, outUsedBytes, outIncrUsedBytes, outMaxUsedBytes);
}
return status;
}
/*
double get_cpu_use_rate(const struct cpu_usage_struct *cur,
const struct cpu_usage_struct *old)
{
double user,sys,nice,idle,total;
double cpu_rate;
user = (double)(cur->cpu_user - old->cpu_user);
sys = (double)(cur->cpu_sys - old->cpu_sys);
nice = (double)(cur->cpu_nice - old->cpu_nice);
idle = (double)(cur->cpu_idle - old->cpu_idle);
total = user + sys + nice + idle;
cpu_rate = (1-idle/total)*100;
return total;
}
double get_cpu_us_rate(const struct cpu_usage_struct *cur,
const struct cpu_usage_struct *old)
{
double user,sys,nice,idle,total;
double cpu_rate;
user = (double)(cur->cpu_user - old->cpu_user);
sys = (double)(cur->cpu_sys - old->cpu_sys);
nice = (double)(cur->cpu_nice - old->cpu_nice);
idle = (double)(cur->cpu_idle - old->cpu_idle);
total = user + sys + nice + idle;
cpu_rate = (user/total)*100;
return cpu_rate;
}
double get_cpu_free_rate(const struct cpu_usage_struct *cur,
const struct cpu_usage_struct *old)
{
double user,sys,nice,idle,total;
double free_rate;
user = (double)(cur->cpu_user - old->cpu_user);
sys = (double)(cur->cpu_sys - old->cpu_sys);
nice = (double)(cur->cpu_nice - old->cpu_nice);
idle = (double)(cur->cpu_idle - old->cpu_idle);
total = user + sys + nice + idle;
free_rate = (idle /total)*100;
return free_rate;
}
int get_cpuinfo_from_proc_stat(struct cpu_usage_struct *usage)
{
FILE *fp = NULL;
char tmp[10];
fp = fopen(CPU_FILE_PROC_STAT,"r");
if(fp == NULL)
{
perror("fopen");
return -1;
}
printf("%s,%d\n",__FILE__,__LINE__);
fscanf(fp,"%s %lu %lu %lu %lu",tmp,&(usage->cpu_user),&(usage->cpu_sys),
&(usage->cpu_nice),&(usage->cpu_idle));
printf("%s %d\n",__FILE__,__LINE__);
fclose(fp);
printf("%s %d\n",__FILE__,__LINE__);
return 1;
}
*/
int main()
{
//struct cpu_usage_struct *cur,*old,cpu_usage;
struct cpu_usage_struct cpu_usage;
struct mem_usage_struct mem;
double use_rate,free_rate,rate;
unsigned long total;
/*old = (struct cpu_usage_struct*)malloc(sizeof(struct cpu_usage_struct));
if(old == NULL)
{
perror("malloc error");
return -1;
}
cur = (struct cpu_usage_struct*)malloc(sizeof(struct cpu_usage_struct));
if(cur == NULL)
{
perror("malloc error");
return -1;
}
get_cpuinfo_from_proc_stat(old);
sleep(1);
get_cpuinfo_from_proc_stat(cur);
rate=cur->cpu_user+cur->cpu_sys+cur->cpu_nice+cur->cpu_idle;
use_rate = get_cpu_use_rate(cur,old);
free_rate = get_cpu_free_rate(cur,old);
//use_rate=getCpuUsage(&cpu_usage);
printf("us:%.1f%%\n use_rate:%.1lf%%\nfree_rate:%.1lf%%\n",100*cur->cpu_user/rate,use_rate,free_rate);
*/
total=getCpuUsage(&cpu_usage);
if (total>0)
printf("<h4>用户使用率\t%.1f%%\t系统使用率\t%.1f%%</h4>",cpu_usage.cpu_user*100.0/total,cpu_usage.cpu_sys*100.0/total);
getMemUsage(&mem);
if (mem.total>0)
printf("总数=%lu 已用=%lu 可用=%lu 缓冲区=%lu 已缓存=%lu 使用率:%.1f%%\n",mem.total,mem.used,mem.free,mem.buffers,mem.cached,(mem.used-mem.buffers-mem.cached)*100.0/mem.total);
return 1;
}
int readAllReads(char *fileName1, char *fileName2, int compressed,
unsigned char *fastq, unsigned char pairedEnd, Read **seqList,
unsigned int *seqListSize, unsigned int ListSize, unsigned int AccListSize) {
double startTime=getTime();
char * seq;
char * qual;
char seq1[SEQ_MAX_LENGTH];
char rseq1[SEQ_MAX_LENGTH];
char qual1[SEQ_MAX_LENGTH];
char seq2[SEQ_MAX_LENGTH];
char rseq2[SEQ_MAX_LENGTH];
char qual2[SEQ_MAX_LENGTH];
char dummy[SEQ_MAX_LENGTH];
int discarded = 0;
int seqCnt = 0;
Read *list = NULL;
int nCnt1;
int nCnt2;
list = getMem(sizeof(Read)*ListSize, "list @readAllReads()");
while(ListSize > seqCnt && readFirstSeq(dummy)) {
int i = 0;
int _mtmp = 36;
if(dummy[0] == '#' || dummy[0] == '>' || dummy[0] == ' ' ||
dummy[0] == '\r' || dummy[0] == '\n')
continue;
strtok(dummy, "\t ");
seq = strtok(NULL, "\t ");
qual = strtok(NULL, "\t ");
for(i = 0; i < _mtmp - 1; i++) {
seq1[i] = toupper(seq[i]);
qual1[i] = qual[i];
}
for(i = 0; i < _mtmp - 1; i++) {
seq2[i] = toupper(seq[i + _mtmp - 1]);
qual2[i] = qual[i + _mtmp - 1];
}
seq1[_mtmp - 1] = seq2[_mtmp - 1] = qual1[_mtmp - 1] = qual2[_mtmp - 1] = '\0';
nCnt1 = 0; nCnt2 = 0;
for (i=0; i<_mtmp; i++)
{
if (seq1[i] == 'N')
nCnt1++;
if (seq2[i] == 'N')
nCnt2++;
}
if (nCnt1 > errThreshold || nCnt2 > errThreshold)
{
discarded += 2;
continue;
}
if (errThreshold == 255) {
if (cropSize > 0) {
errThreshold = (int) ceil(cropSize * 0.04);
fprintf(stdout, "Sequence length: %d bp. Error threshold is set to %d bp.\n",
cropSize, errThreshold);
}
else {
errThreshold = (int) ceil((strlen(seq1)) * 0.04);
fprintf(stdout, "Sequence length: %d bp. Error threshold is set to %d bp.\n",
((int)strlen(seq1)), errThreshold);
}
fprintf(stdout, "You can override this value using the -e parameter.\n");
}
list[seqCnt].hits = getMem (1 + 3 * _mtmp + 3 + _mtmp, "list.hits @readAllReads()");
list[seqCnt].seq = list[seqCnt].hits + 1;
list[seqCnt].rseq = list[seqCnt].seq + _mtmp + 1;
list[seqCnt].qual = list[seqCnt].rseq + _mtmp + 1;
list[seqCnt].name = list[seqCnt].qual + _mtmp + 1;
list[seqCnt].hashValue = getMem(sizeof(short) * _mtmp, "list.hashValue @readAllReads()");
list[seqCnt].rhashValue = getMem(sizeof(short) * _mtmp, "list.rhashValue @readAllReads()");
list[seqCnt].readNumber = seqCnt;
list[seqCnt].hits[0] = 0;
reverseComplement(seq1, rseq1, _mtmp - 1); // DHL Modify
rseq1[_mtmp - 1] = '\0';
for (i=0; i<_mtmp-1; i++) {
list[seqCnt].seq[i] = seq1[i];
list[seqCnt].rseq[i] = rseq1[i];
list[seqCnt].qual[i] = qual1[i];
}
if (!pairedEndMode)
sprintf(list[seqCnt].name, "%s_%d/1", mappingOutput, (seqCnt + AccListSize) / 2);
else
sprintf(list[seqCnt].name, "%s_%d", mappingOutput, (seqCnt + AccListSize) / 2);
list[seqCnt].seq[_mtmp - 1] = list[seqCnt].rseq[_mtmp - 1] = list[seqCnt].qual[_mtmp - 1]='\0';
seqCnt++;
list[seqCnt].hits = getMem (1 + 3 * _mtmp + 3 + _mtmp, "list.hits @readAllReads()");
list[seqCnt].seq = list[seqCnt].hits + 1;
list[seqCnt].rseq = list[seqCnt].seq + _mtmp+1;
list[seqCnt].qual = list[seqCnt].rseq + _mtmp+1;
list[seqCnt].name = list[seqCnt].qual + _mtmp+1;
list[seqCnt].hashValue = getMem(sizeof(short) * _mtmp, "list.hashValue @readAllReads()");
list[seqCnt].rhashValue = getMem(sizeof(short) * _mtmp, "list.rhashValue @readAllReads()");
list[seqCnt].readNumber = seqCnt;
list[seqCnt].hits[0] = 0;
reverseComplement(seq2, rseq2, _mtmp - 1); // DHL Modify
rseq2[_mtmp - 1] = '\0';
for (i=0; i<_mtmp; i++) {
list[seqCnt].seq[i] = seq2[i];
list[seqCnt].rseq[i] = rseq2[i];
list[seqCnt].qual[i] = qual2[i];
}
if (!pairedEndMode)
sprintf(list[seqCnt].name, "%s_%d/2", mappingOutput, (seqCnt + AccListSize) / 2);
else
sprintf(list[seqCnt].name, "%s_%d", mappingOutput, (seqCnt + AccListSize) / 2);
list[seqCnt].seq[_mtmp - 1] = list[seqCnt].rseq[_mtmp - 1] = list[seqCnt].qual[_mtmp - 1]='\0';
seqCnt++;
}
if (seqCnt <= 0) {
//fprintf(stdout, "ERROR: No reads can be found for mapping\n");
fprintf(stdout, "==== End of Input Reads ====\n"); // DHL: read slice
return 0;
}
//qsort(list, seqCnt, sizeof(Read), toCompareRead);
adjustQual(list, seqCnt);
*seqList = list;
*seqListSize = seqCnt;
_r_seq = list;
_r_seqCnt = seqCnt;
if (pairedEnd)
discarded *= 2;
if (seqCnt > 1) {
fprintf(stdout, "==== %d sequences are read in %0.2f. (%d discarded) [Mem:%0.2f M] ====\n",
seqCnt, (getTime()-startTime), discarded, getMemUsage());
}
else {
fprintf(stdout, "==== %d sequence is read in %0.2f. (%d discarded) [Mem:%0.2f M] ====\n",
seqCnt, (getTime()-startTime), discarded, getMemUsage());
}
return seqCnt;
}
void printMemoryUsage()
{
printf("[Memory usage:% d KB]\n", getMemUsage());
}
int readChunk(Read **seqList, unsigned int *seqListSize)
{
double startTime=getTime();
char seq1[SEQ_MAX_LENGTH];
char name1[SEQ_MAX_LENGTH];
char qual1[SEQ_MAX_LENGTH];
char seq2[SEQ_MAX_LENGTH];
char name2[SEQ_MAX_LENGTH];
char qual2[SEQ_MAX_LENGTH];
char dummy[SEQ_MAX_LENGTH];
int size;
int maxCnt = 0;
_r_seqCnt = 0;
_r_readMemUsage = 0;
int i;//, len;
int namelen;
while( (namelen = readFirstSeq(name1,1)) )
{
if (pairedEndMode)
{
if (name1[namelen-2]=='/' && name1[namelen-1]=='1')
{
namelen -= 2;
name1[namelen]='\0';
}
}
size = sizeof(uint16_t) + (SEQ_LENGTH * 2) + QUAL_LENGTH + 3 + (CMP_SEQ_LENGTH << 4) + namelen +/* 1 +*/ 4;
_r_seq[_r_seqCnt].hits = getMem(size);
_r_readMemUsage += size;
_r_seq[_r_seqCnt].seq = (char *)(_r_seq[_r_seqCnt].hits + 1);
_r_seq[_r_seqCnt].rseq = (char *)(_r_seq[_r_seqCnt].seq + SEQ_LENGTH + 1);
_r_seq[_r_seqCnt].qual = (char *)(_r_seq[_r_seqCnt].rseq + SEQ_LENGTH + 1);
_r_seq[_r_seqCnt].cseq = (CompressedSeq *)(_r_seq[_r_seqCnt].qual + QUAL_LENGTH + 1);
_r_seq[_r_seqCnt].crseq = (CompressedSeq *)(_r_seq[_r_seqCnt].cseq + CMP_SEQ_LENGTH);
_r_seq[_r_seqCnt].name = (char *)(_r_seq[_r_seqCnt].crseq + CMP_SEQ_LENGTH);
_r_seq[_r_seqCnt].alphCnt = (unsigned char *)(_r_seq[_r_seqCnt].name + namelen);// + 1);
_r_seq[_r_seqCnt].hits[0] = 0;
for (i=1; i<namelen+1; i++)
_r_seq[_r_seqCnt].name[i-1] = name1[i];
if ( readFirstSeq(_r_seq[_r_seqCnt].seq,2) != SEQ_LENGTH)
{
fprintf(stdout, "ERR: Inconsistent read length for %s\n", name1);
exit(EXIT_FAILURE);
}
if ( _r_fastq )
{
readFirstSeq(dummy,3);
readFirstSeq(_r_seq[_r_seqCnt].qual,4);
}
else
{
_r_seq[_r_seqCnt].qual = "*";
}
_r_seqCnt++;
if (pairedEndMode)
{
_r_seq[_r_seqCnt].hits = getMem(size);
_r_readMemUsage += size;
_r_seq[_r_seqCnt].seq = (char *) (_r_seq[_r_seqCnt].hits + 1);
_r_seq[_r_seqCnt].rseq = (char *)(_r_seq[_r_seqCnt].seq + SEQ_LENGTH + 1);
_r_seq[_r_seqCnt].qual = (char *)(_r_seq[_r_seqCnt].rseq + SEQ_LENGTH + 1);
_r_seq[_r_seqCnt].cseq = (CompressedSeq *)(_r_seq[_r_seqCnt].qual + QUAL_LENGTH + 1);
_r_seq[_r_seqCnt].crseq = (CompressedSeq *)(_r_seq[_r_seqCnt].cseq + CMP_SEQ_LENGTH);
_r_seq[_r_seqCnt].name = (char *)(_r_seq[_r_seqCnt].crseq + CMP_SEQ_LENGTH);
_r_seq[_r_seqCnt].alphCnt = (unsigned char *)(_r_seq[_r_seqCnt].name + namelen);// + 1);
_r_seq[_r_seqCnt].hits[0] = 0;
readSecondSeq(name2, 1);
for (i=1; i<namelen+1; i++)
_r_seq[_r_seqCnt].name[i-1] = name1[i];
if ( readSecondSeq(_r_seq[_r_seqCnt].seq,2) != SEQ_LENGTH)
{
fprintf(stdout, "ERR: Inconsistent read length for %s\n", name1);
exit(EXIT_FAILURE);
}
if ( _r_fastq )
{
readSecondSeq(dummy,3);
readSecondSeq(_r_seq[_r_seqCnt].qual,4);
}
else
{
_r_seq[_r_seqCnt].qual = "*";
}
_r_seqCnt++;
}
if (_r_seqCnt >= _r_maxSeqCnt)
break;
}
*seqList = _r_seq;
*seqListSize = _r_seqCnt;
if (_r_seqCnt > 0)
{
preProcessReadsMT();
fprintf(stdout, "| *Reading Input* | %15.2f | XXXXXXXXXXXXXXX | %15.2f | XXXXXXXXXXXXXXX %15d |\n", (getTime()-startTime), getMemUsage(), _r_seqCnt );
_r_firstIteration = 0;
}
else if (_r_firstIteration)
{
fprintf(stdout, "ERR: No reads for mapping\n");
exit(EXIT_FAILURE);
}
if (_r_seqCnt < _r_maxSeqCnt) // reached end of file
return 0;
else
return 1;
}
int main(int argc, char *argv[])
{
if (!parseCommandLine(argc, argv))
return 1;
configHashTable();
/****************************************************
* INDEXING
***************************************************/
if (indexingMode)
{
int i;
/********************************
* Regulard Mode
********************************/
if (!bisulfiteMode)
{
configHashTable();
for (i = 0; i < fileCnt; i++)
{
generateHashTable(fileName[i][0], fileName[i][1]);
}
}
else
/********************************
* Bisulfite Mode
********************************/
{ // TODO
}
}
/****************************************************
* SEARCHING
***************************************************/
else
{
Read *seqList;
unsigned int seqListSize;
int fc;
int samplingLocsSize;
int *samplingLocs;
double totalLoadingTime = 0;
double totalMappingTime = 0;
double startTime;
double loadingTime;
double mappingTime;
double lstartTime;
double ppTime;
double tmpTime;;
char *prevGen = getMem(CONTIG_NAME_SIZE);
prevGen[0]='\0';
char *curGen;
int flag;
double maxMem=0;
char fname1[FILE_NAME_LENGTH];
char fname2[FILE_NAME_LENGTH];
char fname3[FILE_NAME_LENGTH];
char fname4[FILE_NAME_LENGTH];
char fname5[FILE_NAME_LENGTH];
// Loading Sequences & Sampling Locations
startTime = getTime();
if (bisulfiteMode && !pairedEndMode && seqFile1 == NULL)
{
//TODO
}
else
{
if (!readAllReads(seqFile1, seqFile2, seqCompressed, &seqFastq, pairedEndMode, &seqList, &seqListSize))
{
return 1;
}
}
//loadSamplingLocations(&samplingLocs, &samplingLocsSize);
totalLoadingTime += getTime()-startTime;
if (pairedEndMode)
{
//Switching to Inferred Size
minPairEndedDistance = minPairEndedDistance - SEQ_LENGTH + 2;
maxPairEndedDistance = maxPairEndedDistance - SEQ_LENGTH + 2;
if (pairedEndDiscordantMode)
{
maxPairEndedDiscordantDistance = maxPairEndedDiscordantDistance - SEQ_LENGTH + 2;
minPairEndedDiscordantDistance = minPairEndedDiscordantDistance - SEQ_LENGTH + 2;
}
/* The size between the ends;
minPairEndedDistance = minPairEndedDistance + SEQ_LENGTH + 1;
maxPairEndedDistance = maxPairEndedDistance + SEQ_LENGTH + 1;
if (pairedEndDiscordantMode)
{
maxPairEndedDiscordantDistance = maxPairEndedDiscordantDistance + SEQ_LENGTH + 1;
minPairEndedDiscordantDistance = minPairEndedDiscordantDistance + SEQ_LENGTH + 1;
}*/
sprintf(fname1, "%s__%s__1", mappingOutputPath, mappingOutput);
sprintf(fname2, "%s__%s__2", mappingOutputPath, mappingOutput);
sprintf(fname3, "%s__%s__disc", mappingOutputPath, mappingOutput);
sprintf(fname4, "%s__%s__oea1", mappingOutputPath, mappingOutput);
sprintf(fname5, "%s__%s__oea2", mappingOutputPath, mappingOutput);
unlink(fname1);
unlink(fname2);
unlink(fname3);
unlink(fname4);
unlink(fname5);
}
// Preparing output
initOutput(mappingOutput, outCompressed);
fprintf(stdout, "-----------------------------------------------------------------------------------------------------------\n");
fprintf(stdout, "| %15s | %15s | %15s | %15s | %15s %15s |\n","Genome Name","Loading Time", "Mapping Time", "Memory Usage(M)","Total Mappings","Mapped reads");
fprintf(stdout, "-----------------------------------------------------------------------------------------------------------\n");
/********************************
* Regular Mode
********************************/
if (!bisulfiteMode)
{
if (!pairedEndMode)
{
for (fc = 0; fc <fileCnt; fc++)
{
if (!initLoadingHashTable(fileName[fc][1]))
{
return 1;
}
loadSamplingLocations(&samplingLocs, &samplingLocsSize);
mappingTime = 0;
loadingTime = 0;
prevGen[0] = '\0';
flag = 1;
do
{
flag = loadHashTable ( &tmpTime ); // Reading a fragment
curGen = getRefGenomeName();
// First Time
if (flag && prevGen[0]== '\0')
{
sprintf(prevGen, "%s", curGen);
}
if ( !flag || strcmp(prevGen, curGen)!=0)
{
fprintf(stdout, "| %15s | %15.2f | %15.2f | %15.2f | %15lld %15lld |\n",
prevGen,loadingTime, mappingTime, maxMem, mappingCnt , mappedSeqCnt);
fflush(stdout);
totalMappingTime += mappingTime;
totalLoadingTime += loadingTime;
loadingTime = 0;
mappingTime = 0;
maxMem = 0;
if (!flag)
{
break;
}
}
else if (progressRep && mappingTime != 0)
{
fprintf(stdout, "| %15s | %15.2f | %15.2f | %15.2f | %15lld %15lld |\n",
prevGen,loadingTime, mappingTime, maxMem, mappingCnt , mappedSeqCnt);
fflush(stdout);
}
sprintf(prevGen, "%s", curGen);
loadingTime += tmpTime;
lstartTime = getTime();
initFAST(seqList, seqListSize, samplingLocs, samplingLocsSize, fileName[fc][0]);
mapSingleEndSeq();
mappingTime += getTime() - lstartTime;
if (maxMem < getMemUsage())
{
maxMem = getMemUsage();
}
} while (flag);
} // end for;
finalizeFAST();
finalizeLoadingHashTable();
}
// Pairedend Mapping Mode
else
{
for (fc = 0; fc <fileCnt; fc++)
{
if (!initLoadingHashTable(fileName[fc][1]))
{
return 1;
}
loadSamplingLocations(&samplingLocs, &samplingLocsSize);
mappingTime = 0;
loadingTime = 0;
prevGen[0] = '\0';
flag = 1;
do
{
flag = loadHashTable ( &tmpTime ); // Reading a fragment
curGen = getRefGenomeName();
// First Time
if (flag && prevGen[0]== '\0')
{
sprintf(prevGen, "%s", curGen);
}
if ( !flag || strcmp(prevGen, curGen)!=0)
{
// DISCORDANT
lstartTime = getTime();
outputPairedEnd();
mappingTime += getTime() - lstartTime;
//DISCORDANT
fprintf(stdout, "| %15s | %15.2f | %15.2f | %15.2f | %15lld %15lld |\n",
prevGen,loadingTime, mappingTime, maxMem, mappingCnt , mappedSeqCnt);
fflush(stdout);
totalMappingTime += mappingTime;
totalLoadingTime += loadingTime;
loadingTime = 0;
mappingTime = 0;
maxMem = 0;
if (!flag)
{
break;
}
}
else if (progressRep && mappingTime != 0)
{
fprintf(stdout, "| %15s | %15.2f | %15.2f | %15.2f | %15lld %15lld |\n",
prevGen,loadingTime, mappingTime, maxMem, mappingCnt , mappedSeqCnt);
fflush(stdout);
}
sprintf(prevGen, "%s", curGen);
loadingTime += tmpTime;
lstartTime = getTime();
initFAST(seqList, seqListSize, samplingLocs, samplingLocsSize, fileName[fc][0]);
mapPairedEndSeq();
mappingTime += getTime() - lstartTime;
if (maxMem < getMemUsage())
{
maxMem = getMemUsage();
}
} while (flag);
} // end for;
finalizeLoadingHashTable();
if (pairedEndDiscordantMode)
{
lstartTime = getTime();
outputPairedEndDiscPP();
ppTime = getTime() - lstartTime;
}
finalizeFAST();
}
}
/********************************
* Bisulfite Mode
********************************/
{
//TODO
}
finalizeOutput();
fprintf(stdout, "-----------------------------------------------------------------------------------------------------------\n");
fprintf(stdout, "%19s%16.2f%18.2f\n\n", "Total:",totalLoadingTime, totalMappingTime);
if (pairedEndDiscordantMode)
fprintf(stdout, "Post Processing Time: %18.2f \n", ppTime);
fprintf(stdout, "%-30s%10.2f\n","Total Time:", totalMappingTime+totalLoadingTime);
fprintf(stdout, "%-30s%10d\n","Total No. of Reads:", seqListSize);
fprintf(stdout, "%-30s%10lld\n","Total No. of Mappings:", mappingCnt);
fprintf(stdout, "%-30s%10.0f\n\n","Avg No. of locations verified:", ceil((float)verificationCnt/seqListSize));
int cof = (pairedEndMode)?2:1;
if (progressRep && maxHits != 0)
{
int frequency[maxHits+1];
int i;
for ( i=0 ; i <= maxHits; i++)
{
frequency[i] = 0;
}
for (fc = 0; fc < seqListSize; fc++)
{
frequency[*(seqList[fc*cof].hits)]++;
}
frequency[maxHits] = completedSeqCnt;
for ( i=0 ; i <= maxHits; i++)
{
fprintf(stdout, "%-30s%10d%10d%10.2f%%\n","Reads Mapped to ", i, frequency[i], 100*(float)frequency[i]/(float)seqListSize);
}
}
if (strcmp(unmappedOutput, "") == 0)
{
char fn[strlen(mappingOutputPath) + strlen(mappingOutput) + 6 ];
sprintf(fn, "%s%s.nohit", mappingOutputPath, mappingOutput );
finalizeReads(fn);
}
else
{
finalizeReads(unmappedOutput);
}
freeMem(prevGen, CONTIG_NAME_SIZE);
}
return 0;
}
void reMem(void *ptr, size_t oldsize, size_t newsize)
{
ptr = realloc(ptr, newsize);
if (ptr == NULL){
fprintf(stderr, "Cannot reallocate memory. Currently addressed memory = %0.2f MB, requested memory = %0.2f MB.\nCheck the available main memory, and if you have user limits (ulimit -v).\n", getMemUsage(), (double)newsize);
exit(0);
}
memUsage+=newsize-oldsize;
}
void * getMem(size_t size)
{
void *ret;
ret = malloc(size);
if (ret == NULL){
fprintf(stderr, "Cannot allocate memory. Currently addressed memory = %0.2f MB, requested memory = %0.2f MB.\nCheck the available main memory, and if you have user limits (ulimit -v).\n", getMemUsage(), (double)size);
exit(0);
}
memUsage+=size;
return ret;
}
int readAllReads(char *fileName1,
char *fileName2,
int compressed,
unsigned char *fastq,
unsigned char pairedEnd,
Read **seqList,
unsigned int *seqListSize)
{
double startTime=getTime();
char seq1[SEQ_MAX_LENGTH];
char rseq1[SEQ_MAX_LENGTH];
char name1[SEQ_MAX_LENGTH];
char qual1[SEQ_MAX_LENGTH];
char seq2[SEQ_MAX_LENGTH];
char rseq2[SEQ_MAX_LENGTH];
char name2[SEQ_MAX_LENGTH];
char qual2[SEQ_MAX_LENGTH];
char dummy[SEQ_MAX_LENGTH];
char ch;
int err1, err2;
int nCnt;
int discarded = 0;
int seqCnt = 0;
int maxCnt = 0;
int i;
Read *list = NULL;
// connect pair socket
printf("attempting to connect to ventilator @ %s\n", ventNode);
void *readreceiver = zmq_socket (context, ZMQ_PULL);
zmq_connect (readreceiver, ventNode);
char *message = s_recv (readreceiver);
maxCnt = atoi(message);
printf("... succesfull!\n");
printf("ready to receive %d reads\n", maxCnt);
free (message);
*fastq = 0;
list = getMem(sizeof(Read)*maxCnt);
// start receiving
int recvdCnt = 0;
while (1){
char *message = s_recv (readreceiver);
char *p = NULL;
p = strtok (message," ");
sprintf(name1, p);
p = strtok (NULL," ");
sprintf(seq1, p);
//printf(n)
free(message);
if (strcmp(name1,"DONE")==0){ break;} // get out of loop if the last read is received
sprintf(qual1, "*"); // ignore all quality for now
recvdCnt++;
for (i=0; i<strlen(name1); i++){if (name1[i] == '\n') {name1[i]='\0';}}
for (i=0; i<strlen(seq1); i++){if (seq1[i] == '\n') {seq1[i]='\0';}}
//printf("%d: %s %s\n", recvdCnt, name1, seq1);
//count all the Ns in the sequence and set err1 appropriately
nCnt = 0;
err1 = 0;
for (i=0; i<strlen(seq1); i++)
{
if (seq1[i] == 'N') {nCnt++;}
}
if (nCnt > errThreshold){
err1 = 1;
}
if (!err1){
int _mtmp = strlen(seq1);
list[seqCnt].hits = getMem (1+3*_mtmp+3+strlen(name1)+1);
list[seqCnt].seq = list[seqCnt].hits + 1;
list[seqCnt].rseq = list[seqCnt].seq + _mtmp+1;
list[seqCnt].qual = list[seqCnt].rseq + _mtmp+1;
list[seqCnt].name = list[seqCnt].qual + _mtmp+1;
reverseComplete(seq1, rseq1, _mtmp);
rseq1[_mtmp] = '\0';
int i;
list[seqCnt].hits[0] = 0;
for (i=0; i<=_mtmp; i++)
{
list[seqCnt].seq[i] = seq1[i];
list[seqCnt].rseq[i] = rseq1[i] ;
list[seqCnt].qual[i] = qual1[i];
}
sprintf(list[seqCnt].name,"%s%c", ((char*)name1)+1,'\0');
seqCnt++;
}
else { // sequence had too many Ns in it!
discarded++;
}
}
// CLOSE PAIR SOCKET
zmq_close (readreceiver);
if (seqCnt > 0)
{
QUAL_LENGTH = SEQ_LENGTH = strlen(list[0].seq);
if (! *fastq)
{
QUAL_LENGTH = 1;
}
//fprintf(stderr, "%d %d\n", SEQ_LENGTH, QUAL_LENGTH);
}
else
{
fprintf(stdout, "ERR: No reads can be found for mapping\n");
fprintf(stdout, "recvdCnt: %d\n", recvdCnt);
fprintf(stdout, "seqCnt: %d\n", seqCnt);
fprintf(stdout, "discarded: %d\n", discarded);
char out_msg [255];
// tell controller that no reads were received!
sprintf (out_msg, "MAPPER INPUT %s %d %d",mapperID, seqCnt, discarded);
s_send (requester, out_msg);
s_recv (requester);
return 0;
}
*seqList = list;
*seqListSize = seqCnt;
_r_seq = list;
_r_seqCnt = seqCnt;
fprintf(stdout, "%d sequences are read in %0.2f. (%d discarded) [Mem:%0.2f M]\n", seqCnt, (getTime()-startTime), discarded, getMemUsage());
//totalLoadingTime+=getTime()-startTime;
char out_msg [255];
sprintf (out_msg, "MAPPER INPUT %s %d %d",mapperID, seqCnt, discarded);
s_send (requester, out_msg);
char *msg = s_recv (requester);
if (strcmp(msg,"RESTART")==0){
printf("RESTART received from controller. Terminating!\n");
return 0; // end the while loop and terminate the mapper
}
else if (strcmp(msg,"WAIT")==0){
// wait for appropriate signal on subsocket. not using request-reply socket so that system does not hang!
printf("WAIT received from controller via subsocket.\n");
char *msg2 = s_recv (subsocket);
char killsig[255];
strcpy(killsig,mapperID);
strcat(killsig," RESTART");
if (strcmp(msg2,killsig)==0){
free(msg2);
free(msg);
printf("RESTART received from controller via subsocket. Terminating!\n");
return 0; // terminate the mapper
}
free(msg2);
}
free(msg);
printf("GO received from controller. STARTING MAPPING!\n");
return 1;
}
void* getMem( size_t size)
{
void* ret;
ret = malloc( size);
if( ret == NULL)
{
fprintf( stderr, "Cannot allocate memory. Currently addressed memory = %0.2f MB, requested memory = %0.2f MB.\nCheck the available main memory.\n", getMemUsage(), ( float) ( size / 1048576.0));
exit( 0);
}
memUsage = memUsage + size;
return ret;
}
int readAllReads(char *fileName1,
char *fileName2,
int compressed,
unsigned char *fastq,
unsigned char pairedEnd,
Read **seqList,
unsigned int *seqListSize)
{
double startTime=getTime();
char seq1[SEQ_MAX_LENGTH];
char rseq1[SEQ_MAX_LENGTH];
char name1[SEQ_MAX_LENGTH];
char qual1[SEQ_MAX_LENGTH];
char seq2[SEQ_MAX_LENGTH];
char rseq2[SEQ_MAX_LENGTH];
char name2[SEQ_MAX_LENGTH];
char qual2[SEQ_MAX_LENGTH];
char dummy[SEQ_MAX_LENGTH];
char ch;
int err1, err2;
int nCnt;
int discarded = 0;
int seqCnt = 0;
int maxCnt = 0;
int i;
Read *list = NULL;
// new vars
char * BUFF_1 = NULL;
char * BUFF_2 = NULL;
unsigned long int BUFF_1_pos = 0;
unsigned long int BUFF_2_pos = 0;
char READ_BUFFER[SEQ_MAX_LENGTH];
size_t read_len;
size_t curr_pos;
void * _r_fp1;
void * _r_fp2;
if (!compressed)
{
_r_fp1 = fileOpen( fileName1, "r");
if (_r_fp1 == NULL) return 0;
if ( pairedEnd && fileName2 != NULL ){
_r_fp2 = fileOpen ( fileName2, "r" );
if (_r_fp2 == NULL) return 0;
}
else{
_r_fp2 = _r_fp1;
}
readSeq = &readline_TXT;
}
else{
_r_fp1 = fileOpenGZ (fileName1, "r");
if (_r_fp1 == NULL){
return 0;
}
if ( pairedEnd && fileName2 != NULL ){
_r_fp2 = fileOpenGZ ( fileName2, "r" );
if (_r_fp2 == NULL) return 0;
}
else{
_r_fp2 = _r_fp1;
}
readSeq = &readline_GZ;
}
//READ INTO 1 or 2 buffers
///READ IN read 1 into the buffer
curr_pos=0;
while (readSeq(_r_fp1,READ_BUFFER)){
read_len = strlen(READ_BUFFER);
BUFF_1=(char*)(realloc(BUFF_1,curr_pos+read_len));
//strcpy(&BUFF_1[curr_pos],READ_BUFFER);
memcpy(&BUFF_1[curr_pos],READ_BUFFER,read_len);
curr_pos+=read_len;
maxCnt++;
//printf("%s",READ_BUFFER);
}
if (pairedEnd){
curr_pos=0;
while (readSeq(_r_fp2,READ_BUFFER)){
read_len = strlen(READ_BUFFER);
BUFF_2=(char*)(realloc(BUFF_2,curr_pos+read_len));
//strcpy(&BUFF_2[curr_pos],READ_BUFFER);
memcpy(&BUFF_1[curr_pos],READ_BUFFER,read_len);
curr_pos+=read_len;
}
}
//printf("%s",BUFF_1);
//exit(1);
printf("read in complete\n");
printf("%d lines\n",maxCnt);
//printf("%s",BUFF_1);
if (BUFF_1[0] == '>')
*fastq = 0;
else
*fastq = 1;
// Counting the number of lines in the file
//while (readSeq(dummy)) maxCnt++;
if (!compressed){
fclose(_r_fp1);
if (pairedEnd) fclose(_r_fp2);
}else{
gzclose(_r_fp1);
if (pairedEnd) gzclose(_r_fp2);
}
///AFTER HERE, no changes except reads in from stream
// Calculating the Maximum # of sequences
if (*fastq)
{
maxCnt /= 4;
}
else
{
maxCnt /= 2;
}
if (pairedEnd && fileName2 != NULL )
maxCnt *= 2;
list = getMem(sizeof(Read)*maxCnt);
//while( readSeq(name1) )
while(scanBUF(BUFF_1,name1,&BUFF_1_pos)==1)
{
err1 = 0;
err2 = 0;
//readSeq(seq1);
scanBUF(BUFF_1,seq1,&BUFF_1_pos);
name1[strlen(name1)-1] = '\0';
for (i=0; i<strlen(name1);i++)
{
if (name1[i] == ' ')
{
name1[i] = '\0';
break;
}
}
if ( *fastq )
{
scanBUF(BUFF_1,dummy,&BUFF_1_pos);
scanBUF(BUFF_1,qual1,&BUFF_1_pos);
//readSeq(dummy);
//readSeq(qual1);
qual1[strlen(qual1)-1] = '\0';
}
else
{
sprintf(qual1, "*");
}
// Cropping
if (cropSize > 0)
{
seq1[cropSize] = '\0';
if ( *fastq )
qual1[cropSize] = '\0';
}
nCnt = 0;
for (i=0; i<strlen(seq1); i++)
{
seq1[i] = toupper (seq1[i]);
if (seq1[i] == 'N')
{
nCnt++;
}
else if (isspace(seq1[i]))
{
seq1[i] = '\0';
break;
}
}
if (nCnt > errThreshold)
{
err1 = 1;
}
// Reading the second seq of pair-ends
if (pairedEnd)
{
scanBUF(BUFF_2,name2,&BUFF_2_pos);
scanBUF(BUFF_2,seq2,&BUFF_2_pos);
//readSeq(name2);
//readSeq(seq2);
name2[strlen(name2)-1] = '\0';
for (i=0; i<strlen(name2);i++)
{
if (name2[i] == ' ')
{
name2[i] = '\0';
break;
}
}
if ( *fastq )
{
//readSeq(dummy);
//readSeq(qual2);
scanBUF(BUFF_2,dummy,&BUFF_2_pos);
scanBUF(BUFF_2,qual2,&BUFF_2_pos);
qual2[strlen(qual2)-1] = '\0';
}
else
{
sprintf(qual2, "*");
}
// Cropping
if (cropSize > 0)
{
seq2[cropSize] = '\0';
if ( *fastq )
qual2[cropSize] = '\0';
}
nCnt = 0;
for (i=0; i<strlen(seq2); i++)
{
seq2[i] = toupper (seq2[i]);
if (seq2[i] == 'N')
{
nCnt++;
}
else if (isspace(seq2[i]))
{
seq2[i] = '\0';
}
}
if (nCnt > errThreshold)
{
err2 = 1;
}
}
if (!pairedEnd && !err1)
{
int _mtmp = strlen(seq1);
list[seqCnt].hits = getMem (1+3*_mtmp+3+strlen(name1)+1);
list[seqCnt].seq = list[seqCnt].hits + 1;
list[seqCnt].rseq = list[seqCnt].seq + _mtmp+1;
list[seqCnt].qual = list[seqCnt].rseq + _mtmp+1;
list[seqCnt].name = list[seqCnt].qual + _mtmp+1;
reverseComplete(seq1, rseq1, _mtmp);
int i;
list[seqCnt].hits[0] = 0;
for (i=0; i<=_mtmp; i++)
{
list[seqCnt].seq[i] = seq1[i];
list[seqCnt].rseq[i] = rseq1[i] ;
list[seqCnt].qual[i] = qual1[i];
}
list[seqCnt].rseq[_mtmp]=list[seqCnt].qual[_mtmp]='\0';
sprintf(list[seqCnt].name,"%s%c", ((char*)name1)+1,'\0');
seqCnt++;
}
else if (pairedEnd && !err1 && !err2)
{
// Naming Conventions X/1, X/2 OR X
int tmplen = strlen(name1);
if (strcmp(name1, name2) != 0)
{
tmplen = strlen(name1)-2;
}
//first seq
int _mtmp = strlen(seq1);
list[seqCnt].hits = getMem (1+3*_mtmp+3+tmplen+1);
list[seqCnt].seq = list[seqCnt].hits + 1;
list[seqCnt].rseq = list[seqCnt].seq + _mtmp+1;
list[seqCnt].qual = list[seqCnt].rseq + _mtmp+1;
list[seqCnt].name = list[seqCnt].qual + _mtmp+1;
reverseComplete(seq1, rseq1, _mtmp);
int i;
list[seqCnt].hits[0] = 0;
for (i=0; i<=_mtmp; i++)
{
list[seqCnt].seq[i] = seq1[i];
list[seqCnt].rseq[i] = rseq1[i] ;
list[seqCnt].qual[i] = qual1[i];
}
name1[tmplen]='\0';
list[seqCnt].rseq[_mtmp]=list[seqCnt].qual[_mtmp]='\0';
sprintf(list[seqCnt].name,"%s%c", ((char*)name1)+1,'\0');
seqCnt++;
//second seq
list[seqCnt].hits = getMem (1+3*_mtmp+3+tmplen+1);
list[seqCnt].seq = list[seqCnt].hits + 1;
list[seqCnt].rseq = list[seqCnt].seq + _mtmp+1;
list[seqCnt].qual = list[seqCnt].rseq + _mtmp+1;
list[seqCnt].name = list[seqCnt].qual + _mtmp+1;
reverseComplete(seq2, rseq2, _mtmp);
list[seqCnt].hits[0] = 0;
for (i=0; i<=_mtmp; i++)
{
list[seqCnt].seq[i] = seq2[i];
list[seqCnt].rseq[i] = rseq2[i];
list[seqCnt].qual[i] = qual2[i];
}
name2[tmplen]='\0';
list[seqCnt].rseq[_mtmp]=list[seqCnt].qual[_mtmp]='\0';
sprintf(list[seqCnt].name,"%s%c", ((char*)name2)+1,'\0');
seqCnt++;
}
else
{
discarded++;
}
}
if (seqCnt > 0)
{
QUAL_LENGTH = SEQ_LENGTH = strlen(list[0].seq);
if (! *fastq)
{
QUAL_LENGTH = 1;
}
//fprintf(stderr, "%d %d\n", SEQ_LENGTH, QUAL_LENGTH);
}
else
{
fprintf(stdout, "ERR: No reads can be found for mapping, %d discarded\n", discarded);
return 1;
}
if (pairedEnd)
{
// seqCnt /= 2;
}
*seqList = list;
*seqListSize = seqCnt;
_r_seq = list;
_r_seqCnt = seqCnt;
free(BUFF_1);
if (pairedEnd) free(BUFF_2);
fprintf(stdout, "%d sequences are read in %0.2f. (%d discarded) [Mem:%0.2f M]\n", seqCnt, (getTime()-startTime), discarded, getMemUsage());
//totalLoadingTime+=getTime()-startTime;
/*
fprintf(stdout,"HERE\n");
int j;
for (j=0;j<maxCnt;j++){
fprintf(stdout,"%s\n",list[j].seq);
}
exit(1);
*/
return 1;
}
