Exemplo n.º 1
0
// main k-mer counting function, shared between minia and dsk
// verbose == 0 : stderr progress bar
// verbose >= 1 : print basic status
// verbose >= 2 : print extra partition information
// write_count == True: include kmer count in results file, in that form:
//           - save kmer count for each kmer in the resulting binary file
//           - the very first four bytes of the result file are the kmer length
void sorting_count(Bank *Sequences, char *prefix, int max_memory, int max_disk_space, bool write_count, int verbose)
{

    // create a temp dir from the prefix
    char temp_dir[1024];
    sprintf(temp_dir,"%s_temp",prefix);

    // clear the temp folder (needs to be done before estimating disk space)
    DIR*            dp;
    struct dirent*  ep;
    char            p_buf[512] = {0};
    dp = opendir(temp_dir);
    while ( (dp != NULL) && ((ep = readdir(dp)) != NULL)) {
        sprintf(p_buf, "%s/%s", temp_dir, ep->d_name);
        remove(p_buf);
    }
    if(dp != NULL)
        closedir(dp);

    if (max_disk_space == 0)
    {
        // default max disk space
        struct statvfs buffer ;
        char current_path[1000];
        getcwd(current_path,sizeof(current_path));
        // int ret =
        statvfs(current_path, &buffer);
        int available = (int)(((double)buffer.f_bavail * (double)buffer.f_bsize) / 1024 / 1024);
	uint32_t tt_new_temp = (uint32_t) (((double)Sequences->filesizes)/(1024*1024));
        printf("Available disk space in %s: %d  %u %llu MB\n",current_path,available,tt_new_temp,Sequences->filesizes); // not working in osx (is that a TODO then?)
        max_disk_space = min((uint32_t)available/2, tt_new_temp);
    } 
    if (max_disk_space <= 0) // still 0?
        max_disk_space = 10000; // = default for osx

    // estimate number of iterations TODO Check if multiplication with totalKmers is actually required or not. It may be just increasing number of partitions for no reason
    //uint64_t volume = totalKmers*Sequences->estimate_kmers_volume(smallestKmer);  //Since there are totalKmers no of kmers and an upper bound can be estimated by using the smallest size of kmer. Added by Raunaq
    uint64_t volume = Sequences->estimate_kmers_volume(smallestKmer);  //Since there are totalKmers no of kmers and an upper bound can be estimated by using the smallest size of kmer. Added by Raunaq
    uint32_t nb_passes = ( volume / max_disk_space ) + 1;
    int passes_hash ;
    
    int nb_threads=1;
    
#if OMP
    use_compressed_reads =true;
    nb_threads = 8;
    max_memory /= nb_threads;
    max_memory = max (max_memory,1);
#endif
    
    // temp bugfix: don't use compressed reads for long reads
    if (Sequences->estimate_max_readlen() > 1000000)
        use_compressed_reads = false;
    
    
    uint64_t volume_per_pass,volume_per_partition;
    uint32_t nb_partitions;
    int partitions_hash;

    // loop to lower the number of partitions below the maximum number of simulatenously open files
    do
    {
        volume_per_pass = volume / nb_passes;
        nb_partitions = ( volume_per_pass * totalKmers / max_memory ) + 1; 
	//printf("volume per pass and total volume %llu %llu \n",volume_per_pass,(unsigned long long)volume);
        // if partitions are hashed instead of sorted, adjust for load factor
        // (as in the worst case, all kmers in the partition are distinct and partition may be slightly bigger due to hash-repartition)
        if (use_hashing)
        {
            nb_partitions = (uint32_t) ceil((float) nb_partitions / load_factor);
            nb_partitions = ((nb_partitions * OAHash::size_entry() ) + sizeof(key_type)-1) / sizeof(key_type); // also adjust for hash overhead
        }

        struct rlimit lim;
        int max_open_files = 1000;
        int err = getrlimit(RLIMIT_NOFILE, &lim);
        if (err == 0)
            max_open_files = lim.rlim_cur / 2;
        if (nb_partitions >= max_open_files)
            nb_passes++;
        else
            break;
    }
    while (1);
    volume_per_partition= volume_per_pass/nb_partitions;
    passes_hash = ceil(log(nb_passes)/log(4));
    partitions_hash = ceil(log(nb_partitions)/log(4));
    int size_for_reestimation = ceil((passes_hash + partitions_hash)*1.8);
    double * lmer_counts = (double * ) malloc(sizeof(long)*pow(4,size_for_reestimation));
    long * lmers_for_hash = (long * ) malloc(sizeof(long)*pow(4,size_for_reestimation));
    int * partitions_for_lmers =(int * ) malloc(sizeof(int)*pow(4,size_for_reestimation));
    Sequences->count_kmers_for_small_value(size_for_reestimation,lmer_counts);
    int temp_partition=reestimate_partitions(size_for_reestimation,volume_per_partition,lmer_counts,lmers_for_hash,partitions_for_lmers);
    unordered_map<long,int> part_hash;
    int total_lmers=pow(4,size_for_reestimation);
    for(int it=0;it<total_lmers;it++)
    {
	pair<long,int> temp_pair(lmers_for_hash[it],partitions_for_lmers[it]);
        part_hash.insert (temp_pair); // Add element to the hash 
    }
    //uint64_t up_passes_size = volume_per_pass;
      	do
	{
		//recompute the number of partitions based on updated partitions estimate
		nb_partitions = ceil(temp_partition*1.0/nb_passes);
		struct rlimit lim;
	        int max_open_files = 1000;
	        int err = getrlimit(RLIMIT_NOFILE, &lim);
	        if (err == 0)
        	    max_open_files = lim.rlim_cur / 2;
	        if (nb_partitions >= max_open_files)
        	    nb_passes++;
	        else
        	    break;
	}while(1);
    	printf("no of partitions before %lu and after %d passes %lu \n",nb_partitions*nb_passes,temp_partition,nb_passes);
    uint64_t total_IO =   volume * 2LL * 1024LL*1024LL   ;// in bytes  +   nb_passes * ( volume / (sizeof(kmer_type)*4) )    ; // in bytes
    uint64_t temp_IO = 0;
    BinaryBankConcurrent * redundant_partitions_file[nb_partitions]; 
    char redundant_filename[nb_partitions][256];
    kmer_type kmer;
    int max_read_length = KMERSBUFFER_MAX_READLEN;
    kmer_type * kmer_table_seq = (kmer_type * ) malloc(sizeof(kmer_type)*max_read_length); ;
    kmer_type * kmer_length_table_seq = (kmer_type * ) malloc(sizeof(kmer_type)*max_read_length);

    BinaryReads *  binread = NULL;
    if(use_compressed_reads)
        binread = new BinaryReads(return_file_name(binary_read_file),true);

    fprintf(stderr,"Sequentially counting ~%llu MB of kmers with %d partition(s) and %d passes using %d thread(s), ~%d MB of memory and ~%d MB of disk space\n", (unsigned long long)volume, nb_partitions,nb_passes, nb_threads, max_memory * nb_threads, max_disk_space);

    STARTWALL(count);

    mkdir(temp_dir, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
    
    // Open totalKmers files to store counts of totalKmers different k's
    BinaryBankConcurrent * SolidKmers[totalKmers];
    for (int s=0;s<totalKmers;s++) 
    {	
		char temp[1024];
		sprintf(temp,"%s.%d",return_file_name(solid_kmers_file),Kmerlist[s]);
		uint64_t exp = (((uint64_t)1)<<(Kmerlist[s]*2))-1;
		SolidKmers[s] = new BinaryBankConcurrent(temp,sizeof(kmer),true,nb_threads);
		//printf("kmer is %d exp is %llu \n",Kmerlist[s],exp);
		//BinaryBankConcurrent * SolidKmers = new BinaryBankConcurrent(return_file_name(solid_kmers_file),sizeof(kmer),true,nb_threads);

	    if (write_count)
	    {
        	// write k-mer nbits as the first 4 bytes; and actual k-mer size as the next 4 bits
      		  uint32_t kmer_nbits = sizeof(kmer) * 8;
   	     	SolidKmers[s]->write_buffered(&kmer_nbits, 4,0);
        	SolidKmers[s]->write_buffered(&Kmerlist[s], 4,0);
        	SolidKmers[s]->flush(0);
        }
   }

    int64_t estimated_NbReads = Sequences->estimate_nb_reads();
    char * rseq;
    int readlen;
    int64_t NbSolid = 0;
    int64_t * NbSolid_omp = (int64_t  *) calloc(nb_threads,sizeof(int64_t));
    //long total_kmers_per_partition[nb_partitions]; //guillaume probably commented it because updating this variable would require synchronization
    long distinct_kmers_per_partition[nb_partitions];
    uint64_t  * histo_count = (uint64_t  *) calloc(10001,sizeof(uint64_t));


#if OMP
    uint64_t  **  histo_count_omp = (uint64_t  **) calloc(nb_threads,sizeof(uint64_t *));
    for(int ii=0;ii<nb_threads;ii++)
    {
        histo_count_omp[ii]= (uint64_t  *) calloc(10001,sizeof(uint64_t));
    }
#endif
    

    
   
    //start by the conversion of the file to binary format

    if(use_compressed_reads)
    {
        char * pt_begin;
        int idx =0 ;
        int64_t NbRead = 0;
        Progress progress_conversion;
       // progress_conversion.timer_mode=1; // to switch to timer mode (show elapsed and estimated remaining time)
        progress_conversion.init(estimated_NbReads,"First step: Converting input file into Binary format");
        
        Sequences->rewind_all();
        while(1)
        {
            if(! Sequences->get_next_seq(&rseq,&readlen)) break; // read  original fasta file
            if(readlen > max_read_length) // realloc kmer_table_seq if needed
            {
                max_read_length = 2*readlen;
                kmer_table_seq = (kmer_type * ) realloc(kmer_table_seq,sizeof(kmer_type)*max_read_length);
            	kmer_length_table_seq = (kmer_type * ) realloc(kmer_length_table_seq,sizeof(kmer_type)*max_read_length);
	    }
            
            pt_begin = rseq;
            //should be ok
            while (pt_begin < (rseq+ readlen))
            {
                idx=0; // start a new read

                //skips NN
                while (*pt_begin =='N' && pt_begin < (rseq+ readlen))
                {
                    pt_begin ++;
                }
                // goes to next N or end of seq
                while ( (pt_begin[idx] !='N') &&  ((pt_begin +idx) < (rseq+ readlen))  )
                {
                    idx++;
                }
                
                //we have a seq beginning at  pt_begin of size idx  ,without any N, will be treated as a read:
                binread->write_read(pt_begin,idx);
		revcomp_sequence(pt_begin,idx); // reverse complement the string 
		binread->write_read(pt_begin,idx); // write reverse complement string 
		revcomp_sequence(pt_begin,idx); // restore the string 

		pt_begin += idx;
            }
            
            // binread->write_read(rseq,readlen);
            
            
            NbRead++;
            if ((NbRead%10000)==0)
            {
                progress_conversion.inc(10000);
            }
        }
	//printf("Number of reads converted to binary %d \n",NbRead);
        progress_conversion.finish();
        binread->close();

    }
    ///fin conversion
    if (clear_cache)
    {
#ifdef OSX
        system("purge");
#else
        system("echo 3 > /proc/sys/vm/drop_caches");
#endif
    }
    
    
    
#if SINGLE_BAR
    Progress progress;
    char message[1000];
    sprintf(message,"Counting kmers");
    progress.timer_mode=1;
    if (verbose == 0 )
        progress.init(total_IO,message);
#endif
    
    //use_compressed_reads=false; // for testing compute_kmer_from_one_seq 
    // how many times we will traverse the whole reads file (has an influence on temp disk space)

   uint64_t iter_partition=0;
    for (uint32_t current_pass = 0; current_pass < nb_passes; current_pass ++)
    {
	// stop computing if all partitions are done Added by Raunaq
        if (iter_partition==temp_partition)
		break;
	if(use_compressed_reads ) //open binary reads for reading
            binread->open(false);
        
        STARTWALL(debpass);
        STARTWALL(debw);
	int initial_value = current_pass*nb_partitions;
        for (uint32_t p=0;p<nb_partitions;p++)
        {
            sprintf(redundant_filename[p],"%s/partition%d.redundant_kmers",temp_dir,p);
            redundant_partitions_file[p] =  new BinaryBankConcurrent (redundant_filename[p],sizeof(kmer_type),true, nb_threads);
            distinct_kmers_per_partition[p]=0;
       	}
	int final_value = ((current_pass+1)*nb_partitions)-1;
	printf("Storing k-mers in partition files between %d and %d \n",initial_value,final_value);
        Sequences->rewind_all();
#if !SINGLE_BAR
        Progress progress;
        progress.timer_mode=1; // to switch to timer mode (show elapsed and estimated remaining time)
        char message[1000];
        sprintf(message,"Pass %d/%d, Step 1: partitioning",current_pass+1,nb_passes);
        if (verbose == 0 )
            progress.init(estimated_NbReads,message);
#endif
     

        
        //current_pass> 0 &&
#if OMP
#pragma omp parallel if(use_compressed_reads)  num_threads(nb_threads)
#endif
        {
            int64_t  nbkmers_written =0;
            int tid =0;
            int64_t NbRead = 0;
            int64_t nread =0;
            int64_t tempread =0;
	    long it_zero_wrt =0;
#if OMP

            tid = omp_get_thread_num();
#endif
            int nreads_in_buffer= 1000;
            KmersBuffer * kbuff =NULL;
            if(use_compressed_reads)
            {
                kbuff = new KmersBuffer (binread, 1000000,  nreads_in_buffer); //buffer size (in nb of kmers), seq per task // the buffer is per thread
                kbuff->binary_read_file = binread->binary_read_file;
            }

            kmer_type * kmer_table ;
            kmer_type * kmer_length_info ; // Added by Raunaq, to store the length of read into the partitions file
	    while(1)
            {

                //read the fasta file
                if(use_compressed_reads) // && current_pass>0
                {
                    nread = kbuff->readkmers();
                    if( nread < 0) break;
                    NbRead+= nread;
                    tempread+= nread;
                }
                else
                {
                    if(! Sequences->get_next_seq(&rseq,&readlen)) break; // read  original fasta file
                    if(readlen > max_read_length) // realloc kmer_table_seq if needed
                    {
                        max_read_length = 2*readlen;
                        kmer_table_seq = (kmer_type * ) realloc(kmer_table_seq,sizeof(kmer_type)*max_read_length);
            		kmer_length_table_seq = (kmer_type * ) realloc(kmer_length_table_seq,sizeof(kmer_type)*max_read_length);
                    }

                }

//                if(use_compressed_reads ) //write compressed read file at first pass //&& current_pass==0
//                    binread->write_read(rseq,readlen);

                int i;
                int nbkmers =readlen-sizeKmer+1;  

                if( use_compressed_reads) //current_pass >0 &&
                {
                    nbkmers = kbuff->nkmers;
                    kmer_table = kbuff->kmers_buffer;
		    kmer_length_info = kbuff->kmer_length;
                } 
                else //old fashion   
                {
                    compute_kmer_table_from_one_seq(readlen,rseq,kmer_table_seq,kmer_length_table_seq,Kmerlist[totalKmers-1]); // Added by Raunaq for computing kmers for all values of k 
                    nbkmers =readlen-Kmerlist[totalKmers-1]+1;  
                    kmer_table = kmer_table_seq;
		    kmer_length_info = kmer_length_table_seq;
                    NbRead++;
                    //printf("Number of kmers read from seq %d \n",nbkmers);
		}
		
                nbkmers_written= 0;
		char  temp_kmer[256];
		int zero;
                //compute the kmers stored in the buffer kmer_table
                for (i=0; i<nbkmers; i++)
                {
                    kmer_type lkmer;
					kmer_type lkmer_length;
                    // kmer = extractKmerFromRead(rseq,i,&graine,&graine_revcomp);

                    lkmer = kmer_table[i];
					lkmer_length = kmer_length_info[i];
		   // zero = code2seq(lkmer,temp_kmer);
					long pass_lkmer = code2first_n_nucleotide(lkmer,size_for_reestimation);
					unordered_map<long,int>::const_iterator got = part_hash.find(pass_lkmer);
					int p;// compute in which partition this kmer falls into
					if(got==part_hash.end())
						continue;
					else
						p = got->second; 
                    // check if this kmer should be included in the current pass
                    if(!(p >= initial_value && p<= final_value))
						continue;


/*		
#ifdef _ttmath
                    (reduced_kmer % nb_partitions).ToInt(p);
#else
                    p = reduced_kmer % nb_partitions;
#endif
*/
					p = p - current_pass*nb_partitions;  
                    nbkmers_written++;

                    redundant_partitions_file[p]->write_element_buffered(&lkmer,tid); // save this kmer to the right partition file
					redundant_partitions_file[p]->write_buffered(&lkmer_length,sizeof(lkmer_length),tid,false); // save the kmer length next to the kmer in the same partition file
		    // total_kmers_per_partition[p]++; // guillaume probably commented it because updating this variable would require synchronization

                }
                //NbRead++;
#if SINGLE_BAR
                if(verbose==0)
                {
                if (nb_threads == 1)
                    progress.inc(nbkmers_written * sizeof(kmer_type));
                else
                    progress.inc(nbkmers_written * sizeof(kmer_type),tid);
                }
#endif
             //   if ((NbRead%10000)==0)
                if(tempread> 10000)
                {
                    tempread -= 10000;
                    if (verbose)
                        fprintf (stderr,"%cPass %d/%d, loop through reads to separate (redundant) kmers into partitions, processed %lluM reads out of %lluM",13,current_pass+1,nb_passes,(unsigned long long)(NbRead/1000/1000),(unsigned long long)(estimated_NbReads/1000/1000));
#if !SINGLE_BAR
                    else
                        if (nb_threads == 1)
                            progress.set(NbRead);
                        else
                            progress.inc(10000,tid);
#endif
                }
            } //end while
           // printf("Count of zero in write is %lu \n",it_zero_wrt);
            if(use_compressed_reads)
                delete kbuff;
        } // end OMP 


        
#if !SINGLE_BAR
        if (verbose == 0)
        {
            if (nb_threads == 1)
             progress.finish();
            else
              progress.finish_threaded();  // here only one thread
            
            sprintf(message,"Pass %d/%d, Step 2: computing kmer count per partition",current_pass+1,nb_passes);
            progress.init(nb_partitions+1,message);
        }
#endif
        
        if (verbose)fprintf(stderr,"\n");

        if (verbose >= 2)
        {
            STOPWALL(debw,"Writing redundant kmers");
        }
        STARTWALL(debtri);
	


            for (uint32_t p=0;p<nb_partitions;p++)
            {	
				redundant_partitions_file[p]->close();
                redundant_partitions_file[p]->open(false);
            }



        // for better timing: clear the file cache, since the partitions may still be in memory, that's unfair to low mem machines
        if (clear_cache)
        {
#ifdef OSX
            system("purge");
#else
            system("echo 3 > /proc/sys/vm/drop_caches");
#endif
        }

        //quick and dirty parall with omp, testing
        //todo if we want omp and histo : separate histo_count tab per thread that needs to be merged at the end
        // TODO to guillaume: remove that todo above, because it is done, right?
        kmer_type lkmer,lkmer_length,lkmer_temp,exp;
	long it_zero=0;
	OAHash * hash;
	int p,s;
#if OMP 
        //omp_set_numthreads(2);  //num_threads(2) //if(!output_histo) num_threads(nb_threads)
#pragma omp parallel for private (p,s,lkmer,lkmer_length,hash,lkmer_temp,exp)  num_threads(nb_threads)
#endif        
        // load, sort each partition to output solid kmers
        for ( p=0;p<nb_partitions;p++)
        {
			char temp_kmer[256];  // bug check code 
			int zero;
			kmer_type lkmer_revcomp; // to store revcomps
				
           	bool use_hashing_for_this_partition = use_hashing;
			if(hybrid_mode)
			{
				if(   (redundant_partitions_file[p]->nb_elements()*sizeof(kmer_type)) <  (max_memory*1024LL*1024LL) )  // Maintain totalKmers hash for each partition file
				{	
					use_hashing_for_this_partition = false;
				}
				else
				{
					use_hashing_for_this_partition = true;
				}
			}
            int tid =0;
			//int s;
			//Computing if hashing should be used or not for this partition
#if OMP
            tid = omp_get_thread_num();
#endif
            //use_hashing_for_this_partition = false;  //to check the vector part of the code
           	if (use_hashing_for_this_partition)
            {
                // hash partition and save to solid file
				
				hash = new OAHash(max_memory*1024LL*1024LL/2); // One hash to store all types of k-mer lengths

				uint64_t nkmers_read=0;
				redundant_partitions_file[p]->read_element_buffered(&lkmer_length);

				while (redundant_partitions_file[p]->read_element_buffered(&lkmer))
				{
			
					if(lkmer_length == Kmerlist[0])  //only add the largest k-mer 
						hash->increment(lkmer,convert_to_int(lkmer_length));
					else
					{
						unordered_map<int,int>::const_iterator got = kmerlength_map.find(convert_to_int(lkmer_length));
						exp = (((kmer_type)1)<<(got->second*2))-1;
						lkmer_temp = lkmer & exp;
						hash->increment(lkmer_temp,got->second);

					}	
					if(!redundant_partitions_file[p]->read_element_buffered(&lkmer_length)) 
					{
						break;
					}
					nkmers_read++;
#if SINGLE_BAR
					if(verbose==0 && nkmers_read==10000)
					{
						if (nb_threads == 1)
							progress.inc(nkmers_read*sizeof(kmer_type));
						else
							progress.inc(nkmers_read*sizeof(kmer_type),tid);
						nkmers_read=0;
					}
#endif
                }
                
                
				if (verbose >= 2)
					 printf("Pass %d/%d partition %d/%d hash load factor: %0.3f\n",current_pass+1,nb_passes,p+1,nb_partitions,hash->load_factor());
                	for( s=0;s<totalKmers;s++) 
					{
						OAHash * temp_ = new OAHash(max_memory*1024LL*1024LL/2);
						hash->start_iterator();
						while (hash->next_iterator())
                				{
							uint_abundance_t abundance = hash->iterator->value;
        	       		 			uint_abundance_t abund_tid = (current_pass+1)*100+p;
							if(output_histo)
							{
							 uint_abundance_t saturated_abundance;
							 saturated_abundance = (abundance >= 10000) ? 10000 : abundance;
#if OMP
							 histo_count_omp[tid][saturated_abundance]++;
#else
					
							 histo_count[saturated_abundance]++;
#endif
							}
							int length_kmer = hash->iterator->length;
							lkmer = hash->iterator->key;
	                    				if (abundance >= nks && abundance <= max_couv && length_kmer == Kmerlist[s])
							{
								//write if lkmer is the smaller of it and its reverse complement
								lkmer_revcomp = revcomp(lkmer,length_kmer);
								if(lkmer < lkmer_revcomp)
								{
								SolidKmers[s]->write_element_buffered(&(hash->iterator->key),tid);
							
								 NbSolid_omp[tid]++;
								if (write_count)
										SolidKmers[s]->write_buffered(&abundance, sizeof(abundance),tid, false);
								}
							}
		                    			distinct_kmers_per_partition[p]++;
							if(s!=totalKmers-1)
							{
								if(length_kmer == Kmerlist[s])
								{
									exp = (((kmer_type)1)<<(Kmerlist[s+1]*2))-1;
									lkmer_temp = lkmer & exp;
									temp_->increment_by_value(lkmer_temp,abundance,Kmerlist[s+1]);
								}else {
									temp_->increment_by_value(lkmer,abundance,length_kmer);
								}
							}
						}
						hash->~OAHash();
						hash = temp_;
					}
				hash->~OAHash();
			//printf("All hashes closed and destroyed \n");
			}
            
			else
			{
				// This part does it in slower fashion
				// sort partition and save to solid file 
        	    //vector < kmer_type > kmers;
				vector < kmer_type > kmers[totalKmers];
                uint64_t nkmers_read=0;
               	//int s=0; 
				
				redundant_partitions_file[p]->read_element_buffered(&lkmer_length);
				while (redundant_partitions_file[p]->read_element_buffered (&lkmer))
				{
    		        for(s=0;s<totalKmers;s++)
					{
						//kmer_type lkmer_temp;
						//kmer_type exp;
						if(lkmer_length<Kmerlist[s])
							continue;
						if(s==0)
							kmers[s].push_back (lkmer);
						else
						{
							exp = (((kmer_type)1)<<(Kmerlist[s]*2))-1;
							lkmer_temp = lkmer & exp; // Converting the kmer to its smaller equivalent in binary 
							kmers[s].push_back (lkmer_temp);
						}
                    }
					nkmers_read++;
					if(!redundant_partitions_file[p]->read_element_buffered(&lkmer_length)) break;  //Added to get the next length of kmer
#if SINGLE_BAR
					if(verbose==0 && nkmers_read==10000)
					{
						if (nb_threads == 1)
							progress.inc(nkmers_read*sizeof(kmer_type));
						else
							progress.inc(nkmers_read*sizeof(kmer_type),tid);
						nkmers_read=0;
					}
#endif
                }
                
                for(s=0;s<totalKmers;s++)
               	{
					sort (kmers[s].begin (), kmers[s].end ());
                
					kmer_type previous_kmer = *(kmers[s].begin ());
					uint_abundance_t abundance = 0;
					for (vector < kmer_type >::iterator it = kmers[s].begin (); it != kmers[s].end ();it++)
					{
						kmer_type current_kmer = *it;
					
						if (current_kmer == previous_kmer)
							abundance++;
						else
						{
							if(output_histo)
							{
									uint_abundance_t saturated_abundance;
									saturated_abundance = (abundance >= 10000) ? 10000 : abundance;
#if OMP
									histo_count_omp[tid][saturated_abundance]++;
#else
									histo_count[saturated_abundance]++;
#endif
					
							}
							if (abundance >= nks  && abundance <= max_couv)
							{
								 NbSolid_omp[tid]++;
								 SolidKmers[s]->write_element_buffered(&previous_kmer,tid);
						
								 if (write_count)
									SolidKmers[s]->write_buffered(&abundance, sizeof(abundance),tid, false);
							}		
								abundance = 1;
							distinct_kmers_per_partition[p]++;
						}
						previous_kmer = current_kmer;
					}
                
                //last kmer
					distinct_kmers_per_partition[p]++;
					if(output_histo)
					{
							uint_abundance_t saturated_abundance;
							saturated_abundance = (abundance >= 10000) ? 10000 : abundance;
#if OMP
							histo_count_omp[tid][saturated_abundance]++;
#else
							histo_count[saturated_abundance]++;
#endif
				
					}
					if (abundance >= nks && abundance <= max_couv)
					{
							NbSolid_omp[tid]++;
							SolidKmers[s]->write_element_buffered(&previous_kmer,tid);
				
							if (write_count)
							   SolidKmers[s]->write_buffered(&abundance, sizeof(abundance),tid, false);
				
					}	
				}
			}
            
            
		//printf("Done writing kmers for all K \n");
            
            	if (verbose >= 1)
                	fprintf(stderr,"%cPass %d/%d, loaded and sorted partition %d/%d, found %lld solid kmers so far",13,current_pass+1,nb_passes,p+1,nb_partitions,(long long)(NbSolid_omp[tid]));
            
		//printf("Done writing kmers for all K %d check 1 \n",p);
            	if (verbose >= 2)
                	printf("\nPass %d/%d partition %d/%d %ld distinct kmers\n",current_pass+1,nb_passes,p+1,nb_partitions,/*total_kmers_per_partition[p],*/distinct_kmers_per_partition[p]);
            
#if !SINGLE_BAR
            	if (verbose == 0 && nb_threads==1)
                	progress.inc(1);
            	else if (verbose == 0 && nb_threads>1)
                	progress.inc(1,tid);
#endif
            
            	//if(redundant_partitions_file[p]->find_error()) {
		//	printf("Error in the binary file \n");
		//}
            	redundant_partitions_file[p]->close();
		
            	remove(redundant_filename[p]);
 
        } // end for partitions

#if OMP
        //merge histo
        if(output_histo)

        {
            for (int cc=1; cc<10001; cc++) {
                uint64_t sum_omp = 0;
                for(int ii=0;ii<nb_threads;ii++)
                {
                    sum_omp += histo_count_omp[ii][cc];
                }
                histo_count[cc] = sum_omp;
            }
        }
#endif
        
#if !SINGLE_BAR
        if (verbose == 0 && nb_threads == 1)
            progress.finish();
        else if (verbose == 0 && nb_threads > 1 )
            progress.finish_threaded();
#endif

        if (verbose) fprintf(stderr,"\n");

        if (verbose >= 2)
        {
            STOPWALL(debtri,"Reading and sorting partitions");
            STOPWALL(debpass,"Pass total");

        }
       
	//printf("Done writing kmers for all K check 4 \n");
        if(use_compressed_reads)
            binread->close();
        
        //delete
            for (uint32_t p=0;p<nb_partitions;p++)
            {
                delete redundant_partitions_file[p] ;
            }
        
    }
	//printf("Done writing kmers for all K check 5 \n");

    //single bar
#if SINGLE_BAR
    if (verbose == 0 && nb_threads == 1)
        progress.finish();
    else if (verbose == 0 && nb_threads > 1 )
        progress.finish_threaded();
#endif
    
    if(output_histo)
    {
        FILE * histo_file = fopen(return_file_name(histo_file_name),"w");
        for (int cc=1; cc<10001; cc++) {
            fprintf(histo_file,"%i\t%llu\n",cc,(unsigned long long)(histo_count[cc]));
        }
        fclose(histo_file);
    }
    free(histo_count);

    NbSolid = NbSolid_omp[0];
#if OMP
    NbSolid=0;
    for(int ii=0;ii<nb_threads;ii++)
    {
        NbSolid += NbSolid_omp[ii];
    }
#endif
   for ( int s=0;s<totalKmers;s++) 
    	SolidKmers[s]->close();
    printf("\nSaved %lld solid kmers\n",(long long)NbSolid);
    rmdir(temp_dir);

    STOPWALL(count,"Counted kmers");
    fprintf(stderr,"\n------------------ Counted kmers and kept those with abundance >=%i,     \n",nks);
} 
Exemplo n.º 2
0
int FormatColumnMatrix::read(NameAssignment* nameMap){
	try {		

		string firstName, secondName;
		float distance;
		int nseqs = nameMap->size();

		list = new ListVector(nameMap->getListVector());
	
		Progress* reading = new Progress("Formatting matrix:     ", nseqs * nseqs);

		int lt = 1;
		int refRow = 0;	//we'll keep track of one cell - Cell(refRow,refCol) - and see if it's transpose
		int refCol = 0; //shows up later - Cell(refCol,refRow).  If it does, then its a square matrix

		//need to see if this is a square or a triangular matrix...
		
		ofstream out;
		string tempOutFile = filename + ".temp";
		m->openOutputFile(tempOutFile, out);
	
		while(fileHandle && lt == 1){  //let's assume it's a triangular matrix...
		
			if (m->control_pressed) { out.close();  m->mothurRemove(tempOutFile); fileHandle.close();  delete reading; return 0; }
		
			fileHandle >> firstName >> secondName >> distance;	// get the row and column names and distance
	
			map<string,int>::iterator itA = nameMap->find(firstName);
			map<string,int>::iterator itB = nameMap->find(secondName);
			if(itA == nameMap->end()){  m->mothurOut("AAError: Sequence '" + firstName + "' was not found in the names file, please correct\n"); exit(1);  }
			if(itB == nameMap->end()){  m->mothurOut("ABError: Sequence '" + secondName + "' was not found in the names file, please correct\n"); exit(1);  }

			if (distance == -1) { distance = 1000000; }
		
			if((distance < cutoff) && (itA != itB)){
				if(refRow == refCol){		// in other words, if we haven't loaded refRow and refCol...
					refRow = itA->second;
					refCol = itB->second;
					
					//making it square
					out << itA->second << '\t' << itB->second << '\t' << distance << endl;
					out << itB->second << '\t' << itA->second << '\t' << distance << endl;
				}
				else if(refRow == itA->second && refCol == itB->second){	lt = 0;		} //you are square
				else if(refRow == itB->second && refCol == itA->second){	lt = 0;		}  //you are square
				else{	//making it square
					out << itA->second << '\t' << itB->second << '\t' << distance << endl;
					out << itB->second << '\t' << itA->second << '\t' << distance << endl;
				}
				
				reading->update(itA->second * nseqs / 2);
			}
			m->gobble(fileHandle);
		}
		out.close();
		fileHandle.close();
	
		string squareFile;
		if(lt == 0){  // oops, it was square
			squareFile = filename;
		}else{ squareFile = tempOutFile; }
		
		//sort file by first column so the distances for each row are together
		string outfile = m->getRootName(squareFile) + "sorted.dist.temp";
		
		//use the unix sort 
		#if defined (__APPLE__) || (__MACH__) || (linux) || (__linux)
			string command = "sort -n " + squareFile + " -o " + outfile;
			system(command.c_str());
		#else //sort using windows sort
			string command = "sort " + squareFile + " /O " + outfile;
			system(command.c_str());
		#endif
		
		if (m->control_pressed) { m->mothurRemove(tempOutFile); m->mothurRemove(outfile); delete reading; return 0; }

		//output to new file distance for each row and save positions in file where new row begins
		ifstream in;
		m->openInputFile(outfile, in);
		
		distFile = outfile + ".rowFormatted";
		m->openOutputFile(distFile, out);
		
		rowPos.resize(nseqs, -1);
		int currentRow;
		int first, second;
		float dist;
		map<int, float> rowMap;
		map<int, float>::iterator itRow;
		
		//get first currentRow
		in >> first;
		currentRow = first;
		
		string firstString = toString(first);
		for(int k = 0; k < firstString.length(); k++)  {   in.putback(firstString[k]);  }
		
		while(!in.eof()) {
			
			if (m->control_pressed) { in.close(); out.close(); m->mothurRemove(distFile); m->mothurRemove(tempOutFile); m->mothurRemove(outfile); delete reading; return 0; }
			
			in >> first >> second >> dist; m->gobble(in);
			
			if (first != currentRow) {
				//save position in file of each new row
				rowPos[currentRow] = out.tellp();
				
				out << currentRow << '\t' << rowMap.size() << '\t';
				
				for (itRow = rowMap.begin(); itRow != rowMap.end(); itRow++) {
					out << itRow->first << '\t' << itRow->second << '\t';
				}
				out << endl;
				
				currentRow = first;
				rowMap.clear();
				
				//save row you just read
				if (dist < cutoff) {
					rowMap[second] = dist;
				}
			}else{
				if (dist < cutoff) {
					rowMap[second] = dist;
				}
			}
		}
		
		//print last Row
		//save position in file of each new row
		rowPos[currentRow] = out.tellp();
		
		out << currentRow << '\t' << rowMap.size() << '\t';
		
		for (itRow = rowMap.begin(); itRow != rowMap.end(); itRow++) {
			out << itRow->first << '\t' << itRow->second << '\t';
		}
		out << endl;
		
		
		in.close();
		out.close();
		
		if (m->control_pressed) {  m->mothurRemove(distFile); m->mothurRemove(tempOutFile); m->mothurRemove(outfile);  delete reading; return 0; }
		
		m->mothurRemove(tempOutFile);
		m->mothurRemove(outfile);
		
		reading->finish();
		
		delete reading;
		list->setLabel("0");
		
		if (m->control_pressed) {  m->mothurRemove(distFile);  return 0; }

		return 1;

	}
	catch(exception& e) {
		m->errorOut(e, "FormatColumnMatrix", "read");
		exit(1);
	}
}
Exemplo n.º 3
0
int ParsimonyCommand::execute() {
	try {
	
		if (abort == true) { if (calledHelp) { return 0; }  return 2;	}
		
		
		//randomtree will tell us if user had their own treefile or if they just want the random distribution
		//user has entered their own tree
		if (randomtree == "") { 
			
			m->setTreeFile(treefile);
			
            TreeReader* reader;
            if (countfile == "") { reader = new TreeReader(treefile, groupfile, namefile); }
            else { reader = new TreeReader(treefile, countfile); }
            T = reader->getTrees();
            ct = T[0]->getCountTable();
            delete reader;
	
			if(outputDir == "") { outputDir += m->hasPath(treefile); }
            map<string, string> variables; 
            variables["[filename]"] = outputDir + m->getSimpleName(treefile) +  ".";
            
			output = new ColumnFile(getOutputFileName("parsimony",variables), itersString);
			outputNames.push_back(getOutputFileName("parsimony",variables));
			outputTypes["parsimony"].push_back(getOutputFileName("parsimony",variables));
				
			sumFile = getOutputFileName("psummary",variables);
			m->openOutputFile(sumFile, outSum);
			outputNames.push_back(sumFile);
			outputTypes["psummary"].push_back(sumFile);
		}else { //user wants random distribution
			getUserInput();
				
			if(outputDir == "") { outputDir += m->hasPath(randomtree); }
			output = new ColumnFile(outputDir+ m->getSimpleName(randomtree), itersString);
			outputNames.push_back(outputDir+ m->getSimpleName(randomtree));
			outputTypes["parsimony"].push_back(outputDir+ m->getSimpleName(randomtree));
		}
			
		//set users groups to analyze
		SharedUtil util;
		vector<string> mGroups = m->getGroups();
		vector<string> tGroups = ct->getNamesOfGroups();
		util.setGroups(mGroups, tGroups, allGroups, numGroups, "parsimony");	//sets the groups the user wants to analyze
		util.getCombos(groupComb, mGroups, numComp);
		m->setGroups(mGroups);
			
		if (numGroups == 1) { numComp++; groupComb.push_back(allGroups); }
			
		Parsimony pars;
		counter = 0;
	
		Progress* reading;
		reading = new Progress("Comparing to random:", iters);
		
		if (m->control_pressed) { 
			delete reading; delete output;
			delete ct; for (int i = 0; i < T.size(); i++) { delete T[i]; }
			if (randomtree == "") {  outSum.close();  }
			for (int i = 0; i < outputNames.size(); i++) {	m->mothurRemove(outputNames[i]); } outputTypes.clear();
			m->clearGroups();
			return 0;
		}
			
		
		//get pscore for users tree
		userData.resize(numComp,0);  //data = AB, AC, BC, ABC.
		randomData.resize(numComp,0);  //data = AB, AC, BC, ABC.
		rscoreFreq.resize(numComp);  
		uscoreFreq.resize(numComp);  
		rCumul.resize(numComp);  
		uCumul.resize(numComp);  
		userTreeScores.resize(numComp);  
		UScoreSig.resize(numComp); 
				
		if (randomtree == "") {
			//get pscores for users trees
			for (int i = 0; i < T.size(); i++) {
				userData = pars.getValues(T[i], processors, outputDir);  //data = AB, AC, BC, ABC.
				
				if (m->control_pressed) { 
					delete reading; delete output;
					delete ct; for (int i = 0; i < T.size(); i++) { delete T[i]; }
					if (randomtree == "") {  outSum.close();  }
					for (int i = 0; i < outputNames.size(); i++) {	m->mothurRemove(outputNames[i]); } outputTypes.clear();
					m->clearGroups();
					return 0;
				}


				//output scores for each combination
				for(int k = 0; k < numComp; k++) {

					//update uscoreFreq
					map<int,double>::iterator it = uscoreFreq[k].find(userData[k]);
					if (it == uscoreFreq[k].end()) {//new score
						uscoreFreq[k][userData[k]] = 1;
					}else{ uscoreFreq[k][userData[k]]++; }
					
					//add users score to valid scores
					validScores[userData[k]] = userData[k];
					
					//save score for summary file
					userTreeScores[k].push_back(userData[k]);
				}
			}
			
			//get pscores for random trees
			for (int j = 0; j < iters; j++) {
								
				//create new tree with same num nodes and leaves as users
				randT = new Tree(ct);

				//create random relationships between nodes
				randT->assembleRandomTree();

				//get pscore of random tree
				randomData = pars.getValues(randT, processors, outputDir);
				
				if (m->control_pressed) { 
					delete reading;  delete output; delete randT;
					if (randomtree == "") {  outSum.close();  }
					for (int i = 0; i < outputNames.size(); i++) {	m->mothurRemove(outputNames[i]); } outputTypes.clear();
					delete ct; for (int i = 0; i < T.size(); i++) { delete T[i]; }
					m->clearGroups();
					return 0;
				}
					
				for(int r = 0; r < numComp; r++) {
					//add trees pscore to map of scores
					map<int,double>::iterator it = rscoreFreq[r].find(randomData[r]);
					if (it != rscoreFreq[r].end()) {//already have that score
						rscoreFreq[r][randomData[r]]++;
					}else{//first time we have seen this score
						rscoreFreq[r][randomData[r]] = 1;
					}
			
					//add randoms score to validscores
					validScores[randomData[r]] = randomData[r];
				}
				
				//update progress bar
				reading->update(j);
				
				delete randT;
			}

		}else {
			//get pscores for random trees
			for (int j = 0; j < iters; j++) {
								
				//create new tree with same num nodes and leaves as users
				randT = new Tree(ct);
				//create random relationships between nodes

				randT->assembleRandomTree();
				
				if (m->control_pressed) { 
					delete reading; delete output; delete randT; delete ct; 
					for (int i = 0; i < outputNames.size(); i++) {	m->mothurRemove(outputNames[i]); } outputTypes.clear(); return 0;
				}


				//get pscore of random tree
				randomData = pars.getValues(randT, processors, outputDir);
				
				if (m->control_pressed) { 
					delete reading; delete output; delete randT; delete ct; 
					for (int i = 0; i < outputNames.size(); i++) {	m->mothurRemove(outputNames[i]); } outputTypes.clear(); return 0;
				}
			
				for(int r = 0; r < numComp; r++) {
					//add trees pscore to map of scores
					map<int,double>::iterator it = rscoreFreq[r].find(randomData[r]);
					if (it != rscoreFreq[r].end()) {//already have that score
						rscoreFreq[r][randomData[r]]++;
					}else{//first time we have seen this score
						rscoreFreq[r][randomData[r]] = 1;
					}
			
					//add randoms score to validscores
					validScores[randomData[r]] = randomData[r];
				}
				
				//update progress bar
				reading->update(j);
				
				delete randT;
			}
		}

		for(int a = 0; a < numComp; a++) {
			float rcumul = 0.0000;
			float ucumul = 0.0000;
			//this loop fills the cumulative maps and put 0.0000 in the score freq map to make it easier to print.
			for (map<int,double>::iterator it = validScores.begin(); it != validScores.end(); it++) { 
				if (randomtree == "") {
					map<int,double>::iterator it2 = uscoreFreq[a].find(it->first);
					//user data has that score 
					if (it2 != uscoreFreq[a].end()) { uscoreFreq[a][it->first] /= T.size(); ucumul+= it2->second;  }
					else { uscoreFreq[a][it->first] = 0.0000; } //no user trees with that score
					//make uCumul map
					uCumul[a][it->first] = ucumul;
				}
			
				//make rscoreFreq map and rCumul
				map<int,double>::iterator it2 = rscoreFreq[a].find(it->first);
				//get percentage of random trees with that info
				if (it2 != rscoreFreq[a].end()) {  rscoreFreq[a][it->first] /= iters; rcumul+= it2->second;  }
				else { rscoreFreq[a][it->first] = 0.0000; } //no random trees with that score
				rCumul[a][it->first] = rcumul;
			}
			
			//find the signifigance of each user trees score when compared to the random trees and save for printing the summary file
			for (int h = 0; h < userTreeScores[a].size(); h++) {
				UScoreSig[a].push_back(rCumul[a][userTreeScores[a][h]]);
			}
		}
		
		if (m->control_pressed) { 
				delete reading; delete output;
				delete ct; for (int i = 0; i < T.size(); i++) { delete T[i]; }
				if (randomtree == "") {  outSum.close();  }
				for (int i = 0; i < outputNames.size(); i++) {	m->mothurRemove(outputNames[i]); } outputTypes.clear();
				return 0;
		}
		
		//finish progress bar
		reading->finish();
		delete reading;
		
		printParsimonyFile();
		if (randomtree == "") { printUSummaryFile(); }
				
        delete output; delete ct; for (int i = 0; i < T.size(); i++) { delete T[i]; }
		
		if (m->control_pressed) { for (int i = 0; i < outputNames.size(); i++) {	m->mothurRemove(outputNames[i]); } outputTypes.clear(); return 0;}
		
		m->mothurOutEndLine();
		m->mothurOut("Output File Names: "); m->mothurOutEndLine();
		for (int i = 0; i < outputNames.size(); i++) {	m->mothurOut(outputNames[i]); m->mothurOutEndLine();	}
		m->mothurOutEndLine();

		
		return 0;
		
	}
	catch(exception& e) {
		m->errorOut(e, "ParsimonyCommand", "execute");
		exit(1);
	}
}