Ejemplo n.º 1
0
void TestPlugin::call_MY_TEST_MPI_TAG_TIME_BOMB(Message*message){
	uint64_t*buffer=message->getBuffer();

	if(buffer[0]==0){
		cout<<"The bomb exploded on rank "<<m_core->getRank()<<" !"<<endl;

		// kill everyone

		m_core->sendEmptyMessageToAll(MY_TEST_MPI_TAG_STOP_AND_DIE);
	}else{
		uint64_t*bufferOut=(uint64_t*)m_core->getOutboxAllocator()->allocate(1*sizeof(uint64_t));
		bufferOut[0]=buffer[0]-1;

		cout<<"Remaining time before the explosion is "<<bufferOut[0]<<" according to rank "<<m_core->getRank()<<endl;

		// compute the next destination
		Rank destination=m_core->getRank()+1;
		if(destination==m_core->getSize())
			destination=0;

		Message aMessage(bufferOut,1,destination,MY_TEST_MPI_TAG_TIME_BOMB,m_core->getRank());

		// send the bomb to another rank
		m_core->getOutbox()->push_back(&aMessage);
	}
}
Ejemplo n.º 2
0
void GenomeNeighbourhood::call_RAY_MPI_TAG_NEIGHBOURHOOD_DATA(Message*message){
	
	MessageUnit*incoming=(MessageUnit*)message->getBuffer();
	int position=0;

	/* progressions are on the 'F' strand.
 * if a contig is on the 'R' strand, then the actual position is
 * really length(contig) - position
 *
 * it is sent this way because only master has the length on contigs
 *
 */
	PathHandle leftContig=incoming[position++];
	Strand leftVertexStrand=incoming[position++];
	int leftProgressionInContig=incoming[position++];

	PathHandle rightContig=incoming[position++];
	char rightVertexStrand=incoming[position++];
	int rightProgressionInContig=incoming[position++];

	int gapSizeInKmers=incoming[position++];

	#ifdef ASSERT
	assert(gapSizeInKmers >= 1);
	assert(m_rank==0x00);
	assert(m_contigLengths->count(leftContig)>0);
	assert(m_contigLengths->count(rightContig)>0);
	assert(leftProgressionInContig < m_contigLengths->operator[](leftContig));
	assert(rightProgressionInContig< m_contigLengths->operator[](rightContig));
	assert(leftProgressionInContig>=0);
	assert(rightProgressionInContig>=0);
	assert(leftVertexStrand=='F' || leftVertexStrand=='R');
	assert(rightVertexStrand=='F' || rightVertexStrand == 'R');
	#endif

/*
	if(rightVertexStrand=='R'){
		rightProgressionInContig=m_contigLengths->operator[](rightContig)-rightProgressionInContig;
	}

	// get the position on the actual strand
	if(leftVertexStrand=='R'){
		leftProgressionInContig=m_contigLengths->operator[](leftContig)-leftProgressionInContig;
	}
*/

	NeighbourPair pair(leftContig,leftVertexStrand,leftProgressionInContig,
				rightContig,rightVertexStrand,rightProgressionInContig,
				gapSizeInKmers);

	m_finalList.push_back(pair);

	int period=m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_NEIGHBOURHOOD_DATA);

	MessageUnit*buffer=(MessageUnit*)m_outboxAllocator->allocate(period*sizeof(MessageUnit));
	Message aMessage(buffer,period,message->getSource(),RAY_MPI_TAG_NEIGHBOURHOOD_DATA_REPLY,
		m_parameters->getRank());

	m_core->getOutbox()->push_back(&aMessage);
}
Ejemplo n.º 3
0
void GenomeNeighbourhood::sendRightNeighbours(){

	if(m_neighbourIndex < (int)m_rightNeighbours.size()){

		if(!m_sentEntry){
			m_sentEntry=true;

			// send a message to request the links of the current vertex
			MessageUnit*buffer=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(Kmer));

			Rank destination=0x0;
			int period=m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_NEIGHBOURHOOD_DATA);

			#ifdef ASSERT
			assert(period > 0);
			#endif

			int outputPosition=0;

			buffer[outputPosition++]=m_contigNames->at(m_contigIndex);
			buffer[outputPosition++]='F';
			buffer[outputPosition++]=m_contigs->at(m_contigIndex).size()-1;

			buffer[outputPosition++]=m_rightNeighbours[m_neighbourIndex].getContig();
			buffer[outputPosition++]=m_rightNeighbours[m_neighbourIndex].getStrand();
			buffer[outputPosition++]=m_rightNeighbours[m_neighbourIndex].getProgression();

			buffer[outputPosition++]=m_rightNeighbours[m_neighbourIndex].getDepth();

			Message aMessage(buffer,period,
				destination,RAY_MPI_TAG_NEIGHBOURHOOD_DATA,m_rank);

			m_virtualCommunicator->pushMessage(m_workerId,&aMessage);

			m_receivedReply=false;

		}else if(!m_receivedReply && m_virtualCommunicator->isMessageProcessed(m_workerId)){

			vector<MessageUnit> elements;
			m_virtualCommunicator->getMessageResponseElements(m_workerId,&elements);


			m_receivedReply=true;
			m_sentEntry=false;

			m_neighbourIndex++;
		}


	}else{

		#ifdef DEBUG_LEFT_PATHS
		cout<<"[DEBUG_LEFT_PATHS] processed left paths: "<<m_leftNeighbours.size()<<endl;
		#endif


		m_sentRightNeighbours=true;

	}
}
Ejemplo n.º 4
0
void SeedingData::call_RAY_SLAVE_MODE_SEND_SEED_LENGTHS(){
	if(!m_initialized){
		for(int i=0;i<(int)m_SEEDING_seeds.size();i++){
			int length=getNumberOfNucleotides(m_SEEDING_seeds[i].size(),
				m_parameters->getWordSize());
			m_slaveSeedLengths[length]++;
		}
		m_iterator=m_slaveSeedLengths.begin();
		m_initialized=true;
		m_communicatorWasTriggered=false;


		m_virtualCommunicator->resetCounters();
	}

	if(m_inbox->size()==1&&(*m_inbox)[0]->getTag()==RAY_MPI_TAG_SEND_SEED_LENGTHS_REPLY)
		m_communicatorWasTriggered=false;
	
	if(m_communicatorWasTriggered)
		return;

	if(m_iterator==m_slaveSeedLengths.end()){
		Message aMessage(NULL,0,MASTER_RANK,
			RAY_MPI_TAG_IS_DONE_SENDING_SEED_LENGTHS,getRank());
		m_outbox->push_back(aMessage);
		(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
		return;
	}
	
	MessageUnit*messageBuffer=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
	int maximumPairs=MAXIMUM_MESSAGE_SIZE_IN_BYTES/sizeof(MessageUnit)/2;
	int i=0;
	while(i<maximumPairs && m_iterator!=m_slaveSeedLengths.end()){
		int length=m_iterator->first;
		int count=m_iterator->second;
		messageBuffer[2*i]=length;
		messageBuffer[2*i+1]=count;
		i++;
		m_iterator++;
	}

	Message aMessage(messageBuffer,2*i,MASTER_RANK,
		RAY_MPI_TAG_SEND_SEED_LENGTHS,getRank());
	m_outbox->push_back(aMessage);
}
Ejemplo n.º 5
0
//--------------------------------------------------------------
void Resizer::RefsReceived(BMessage *message) 
{ 
    entry_ref ref;
    
	// get the ref from the message
    if(message->FindRef("refs", &ref) == B_OK)
    {
        BMessage aMessage(B_SIMPLE_DATA); // Make a new message
        aMessage.AddRef( "refs", &ref ); // Copy the ref into it
       	Fenetre->PostMessage(&aMessage, Fenetre->Main); // Post the message via the window
    }
}
Ejemplo n.º 6
0
void SeedingData::call_RAY_SLAVE_MODE_SEND_SEED_LENGTHS(){
	if(!m_initialized){

		m_virtualCommunicator->resetCounters();

		m_initialized = true;
	}

	Message aMessage(NULL,0,MASTER_RANK,
		RAY_MPI_TAG_IS_DONE_SENDING_SEED_LENGTHS,getRank());
	m_outbox->push_back(&aMessage);
	(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
}
Ejemplo n.º 7
0
void ProcessTimer::execute()
{
    ORWELL_LOG_TRACE("ProcessTimer::execute : broadcast Gamestate");

    GameState aGameState;
    aGameState.set_playing(m_game->getIsRunning());

    if (m_game->getWinner())
    {
        aGameState.set_winner(*m_game->getWinner());
    }

    RawMessage aMessage("all_clients", "GameState", aGameState.SerializeAsString());
    m_publisher->send( aMessage );
}
Ejemplo n.º 8
0
BMessenger GetMessenger(void)
{
	BMessenger aResult;
	status_t aErr = B_OK;
	BMessenger aDeskbar(DBAR_SIGNATURE, -1, &aErr);
	if (aErr != B_OK)return aResult;

	BMessage aMessage(B_GET_PROPERTY);
	
	aMessage.AddSpecifier("Messenger");
	aMessage.AddSpecifier("Shelf");
	aMessage.AddSpecifier("View", "Status");
	aMessage.AddSpecifier("Window", "Deskbar");
	
	BMessage aReply;

	if (aDeskbar.SendMessage(&aMessage, &aReply, 1000000, 1000000) == B_OK)
		aReply.FindMessenger("result", &aResult);
	return aResult;
}
Ejemplo n.º 9
0
void TestPlugin::call_MY_TEST_SLAVE_MODE_STEP_C(){

	cout<<"I am "<<m_core->getRank()<<" doing call_MY_TEST_SLAVE_MODE_STEP_C, now I die"<<endl;
	cout<<"This is over "<<endl;

	if(m_core->getRank()==0){
		uint64_t*buffer=(uint64_t*)m_core->getOutboxAllocator()->allocate(1*sizeof(uint64_t));
		buffer[0]=100;

		// compute the next destination
		Rank destination=m_core->getRank()+1;
		if(destination==m_core->getSize())
			destination=0;

		Message aMessage(buffer,1,destination,MY_TEST_MPI_TAG_TIME_BOMB,m_core->getRank());

		// send the bomb to another rank
		m_core->getOutbox()->push_back(&aMessage);
	}

	// do nothing now
	m_core->getSwitchMan()->setSlaveMode(RAY_SLAVE_MODE_DO_NOTHING);
}
Ejemplo n.º 10
0
void VerticesExtractor::call_RAY_SLAVE_MODE_ADD_EDGES(){

	MACRO_COLLECT_PROFILING_INFORMATION();

	if(this->m_outbox==NULL){
		m_rank=m_parameters->getRank();
		this->m_mode=m_mode;
		this->m_outbox=m_outbox;
		this->m_outboxAllocator=m_outboxAllocator;
	}

	if(m_finished){
		return;
	}

	if(!m_checkedCheckpoint){
		m_checkedCheckpoint=true;
		if(m_parameters->hasCheckpoint("GenomeGraph")){
			cout<<"Rank "<<m_parameters->getRank()<<": checkpoint GenomeGraph exists, not extracting vertices."<<endl;
			Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_VERTICES_DISTRIBUTED,m_parameters->getRank());
			m_outbox->push_back(aMessage);
			m_finished=true;
			return;
		}
	}

	#ifdef ASSERT
	assert(m_pendingMessages>=0);
	#endif
	if(m_pendingMessages!=0){
		return;
	}

	if(m_mode_send_vertices_sequence_id%10000==0 &&m_mode_send_vertices_sequence_id_position==0
		&&m_mode_send_vertices_sequence_id<(int)m_myReads->size()){

		string reverse="";
		if(m_reverseComplementVertex==true){
			reverse="(reverse complement) ";
		}
		printf("Rank %i is adding edges %s[%i/%i]\n",m_parameters->getRank(),reverse.c_str(),(int)m_mode_send_vertices_sequence_id+1,(int)m_myReads->size());
		fflush(stdout);

		m_derivative.addX(m_mode_send_vertices_sequence_id);
		m_derivative.printStatus(SLAVE_MODES[RAY_SLAVE_MODE_ADD_EDGES],RAY_SLAVE_MODE_ADD_EDGES);
		m_derivative.printEstimatedTime(m_myReads->size());
	}

	if(m_mode_send_vertices_sequence_id==(int)m_myReads->size()){

		MACRO_COLLECT_PROFILING_INFORMATION();

		// flush data
		flushAll(m_outboxAllocator,m_outbox,m_parameters->getRank());

		if(m_pendingMessages==0){
			#ifdef ASSERT
			assert(m_bufferedDataForIngoingEdges.isEmpty());
			assert(m_bufferedDataForOutgoingEdges.isEmpty());
			#endif

			Message aMessage(NULL,0, MASTER_RANK, RAY_MPI_TAG_VERTICES_DISTRIBUTED,m_parameters->getRank());
			m_outbox->push_back(aMessage);
			m_finished=true;
			printf("Rank %i is adding edges [%i/%i] (completed)\n",m_parameters->getRank(),(int)m_mode_send_vertices_sequence_id,(int)m_myReads->size());
			fflush(stdout);
			m_bufferedDataForIngoingEdges.showStatistics(m_parameters->getRank());
			m_bufferedDataForOutgoingEdges.showStatistics(m_parameters->getRank());

			m_derivative.writeFile(&cout);
		}
	}else{

		MACRO_COLLECT_PROFILING_INFORMATION();

/*
 * Decode the DNA sequence 
 * and store it in a local buffer.
 */
		if(m_mode_send_vertices_sequence_id_position==0){
			(*m_myReads)[(m_mode_send_vertices_sequence_id)]->getSeq(m_readSequence,m_parameters->getColorSpaceMode(),false);
		
			//cout<<"DEBUG Read="<<*m_mode_send_vertices_sequence_id<<" color="<<m_parameters->getColorSpaceMode()<<" Seq= "<<m_readSequence<<endl;
		}

		int len=strlen(m_readSequence);

		if(len<m_parameters->getWordSize()){
			m_hasPreviousVertex=false;
			(m_mode_send_vertices_sequence_id)++;
			(m_mode_send_vertices_sequence_id_position)=0;
			return;
		}

		MACRO_COLLECT_PROFILING_INFORMATION();

		char memory[1000];

		int maximumPosition=len-m_parameters->getWordSize()+1;
		
		#ifdef ASSERT
		assert(m_readSequence!=NULL);
		#endif

		int p=(m_mode_send_vertices_sequence_id_position);
		memcpy(memory,m_readSequence+p,m_parameters->getWordSize());
		memory[m_parameters->getWordSize()]='\0';

		MACRO_COLLECT_PROFILING_INFORMATION();

		if(isValidDNA(memory)){

			MACRO_COLLECT_PROFILING_INFORMATION();

			Kmer currentForwardKmer=wordId(memory);

			/* TODO: possibly don't flush k-mer that are not lower. not sure it that would work though. -Seb */

/*
 *                   previousForwardKmer   ->   currentForwardKmer
 *                   previousReverseKmer   <-   currentReverseKmer
 */


/*
 * Push the kmer
 */


			MACRO_COLLECT_PROFILING_INFORMATION();

			if(m_hasPreviousVertex){

				MACRO_COLLECT_PROFILING_INFORMATION();

				// outgoing edge
				// PreviousVertex(*) -> CurrentVertex
				Rank outgoingRank=m_parameters->_vertexRank(&m_previousVertex);
				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForOutgoingEdges.addAt(outgoingRank,m_previousVertex.getU64(i));
				}
				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForOutgoingEdges.addAt(outgoingRank,currentForwardKmer.getU64(i));
				}


				if(m_bufferedDataForOutgoingEdges.flush(outgoingRank,2*KMER_U64_ARRAY_SIZE,RAY_MPI_TAG_OUT_EDGES_DATA,m_outboxAllocator,m_outbox,m_parameters->getRank(),false)){
					m_pendingMessages++;
				}

				// ingoing edge
				// PreviousVertex -> CurrentVertex(*)
				Rank ingoingRank=m_parameters->_vertexRank(&currentForwardKmer);
				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForIngoingEdges.addAt(ingoingRank,m_previousVertex.getU64(i));
				}
				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForIngoingEdges.addAt(ingoingRank,currentForwardKmer.getU64(i));
				}


				if(m_bufferedDataForIngoingEdges.flush(ingoingRank,2*KMER_U64_ARRAY_SIZE,RAY_MPI_TAG_IN_EDGES_DATA,m_outboxAllocator,m_outbox,m_parameters->getRank(),false)){
					m_pendingMessages++;
				}

				MACRO_COLLECT_PROFILING_INFORMATION();
			}

			// reverse complement
			//
			Kmer currentReverseKmer=currentForwardKmer.
				complementVertex(m_parameters->getWordSize(),m_parameters->getColorSpaceMode());


			if(m_hasPreviousVertex){
				MACRO_COLLECT_PROFILING_INFORMATION();

				// outgoing edge
				// 
				Rank outgoingRank=m_parameters->_vertexRank(&currentReverseKmer);

				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForOutgoingEdges.addAt(outgoingRank,currentReverseKmer.getU64(i));
				}
				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForOutgoingEdges.addAt(outgoingRank,m_previousVertexRC.getU64(i));
				}

				MACRO_COLLECT_PROFILING_INFORMATION();


				if(m_bufferedDataForOutgoingEdges.flush(outgoingRank,2*KMER_U64_ARRAY_SIZE,RAY_MPI_TAG_OUT_EDGES_DATA,m_outboxAllocator,m_outbox,m_parameters->getRank(),false)){

					m_pendingMessages++;
				}

				MACRO_COLLECT_PROFILING_INFORMATION();

				// ingoing edge
				Rank ingoingRank=m_parameters->_vertexRank(&m_previousVertexRC);

				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForIngoingEdges.addAt(ingoingRank,currentReverseKmer.getU64(i));
				}
				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForIngoingEdges.addAt(ingoingRank,m_previousVertexRC.getU64(i));
				}

				MACRO_COLLECT_PROFILING_INFORMATION();


				if(m_bufferedDataForIngoingEdges.flush(ingoingRank,2*KMER_U64_ARRAY_SIZE,RAY_MPI_TAG_IN_EDGES_DATA,m_outboxAllocator,m_outbox,m_parameters->getRank(),false)){
					m_pendingMessages++;
				}
				MACRO_COLLECT_PROFILING_INFORMATION();
			}

			// there is a previous vertex.
			m_hasPreviousVertex=true;
			m_previousVertex=currentForwardKmer;
			m_previousVertexRC=currentReverseKmer;
		}else{
			m_hasPreviousVertex=false;
		}

		MACRO_COLLECT_PROFILING_INFORMATION();

		(m_mode_send_vertices_sequence_id_position++);

		if((m_mode_send_vertices_sequence_id_position)==maximumPosition){
			m_hasPreviousVertex=false;
			(m_mode_send_vertices_sequence_id)++;
			(m_mode_send_vertices_sequence_id_position)=0;
		}
	}
	MACRO_COLLECT_PROFILING_INFORMATION();
}
Ejemplo n.º 11
0
void Partitioner::call_RAY_MASTER_MODE_COUNT_FILE_ENTRIES(){
	/** tell every peer to count entries in files in parallel */
	if(!m_initiatedMaster){
		m_initiatedMaster=true;
		m_ranksDoneCounting=0;
		m_ranksDoneSending=0;
		for(int destination=0;destination<m_parameters->getSize();destination++){
			Message aMessage(NULL,0,destination,RAY_MPI_TAG_COUNT_FILE_ENTRIES,m_parameters->getRank());
			m_outbox->push_back(aMessage);
		}
	/** a peer rank finished counting the entries in its files */
	}else if(m_inbox->size()>0 && m_inbox->at(0)->getTag()== RAY_MPI_TAG_COUNT_FILE_ENTRIES_REPLY){
		m_ranksDoneCounting++;
		/** all peers have finished */
		if(m_ranksDoneCounting==m_parameters->getSize()){
			for(int destination=0;destination<m_parameters->getSize();destination++){
				Message aMessage(NULL,0,destination,RAY_MPI_TAG_REQUEST_FILE_ENTRY_COUNTS,m_parameters->getRank());
				m_outbox->push_back(aMessage);
			}
		}
	/** a peer send the count for one file */
	}else if(m_inbox->size()>0 && m_inbox->at(0)->getTag()== RAY_MPI_TAG_FILE_ENTRY_COUNT){
		MessageUnit*buffer=m_inbox->at(0)->getBuffer();
		int file=buffer[0];
		LargeCount count=buffer[1];
		m_masterCounts[file]=count;

		if(m_parameters->hasOption("-debug-partitioner"))
			cout<<"Rank "<<m_parameters->getRank()<<" received from "<<m_inbox->at(0)->getSource()<<" File "<<file<<" Entries "<<count<<endl;
		/** reply to the peer */
		Message aMessage(NULL,0,m_inbox->at(0)->getSource(),RAY_MPI_TAG_FILE_ENTRY_COUNT_REPLY,m_parameters->getRank());
		m_outbox->push_back(aMessage);
	/** a peer finished sending file counts */
	}else if(m_inbox->size()>0 && m_inbox->at(0)->getTag()== RAY_MPI_TAG_REQUEST_FILE_ENTRY_COUNTS_REPLY){
		m_ranksDoneSending++;
		/** all peers have finished */
		if(m_ranksDoneSending==m_parameters->getSize()){

			for(int i=0;i<(int)m_masterCounts.size();i++){
				if(m_parameters->hasOption("-debug-partitioner"))
					cout<<"Rank "<<m_parameters->getRank()<< " File "<<i<<" Count "<<m_masterCounts[i]<<endl;
				m_parameters->setNumberOfSequences(i,m_masterCounts[i]);
			}
			m_masterCounts.clear();

			/* write the number of sequences */
			ostringstream fileName;
			fileName<<m_parameters->getPrefix();
			fileName<<"NumberOfSequences.txt";
			ofstream f2(fileName.str().c_str());

/* Write a sequence partition too,
 * it contains the number of entries in each file.
 */
			ostringstream fileNameForFiles;
			fileNameForFiles<<m_parameters->getPrefix();
			fileNameForFiles<<"FilePartition.txt";
			ofstream partitionStream(fileNameForFiles.str().c_str());

			f2<<"Files: "<<m_parameters->getNumberOfFiles()<<endl;
			f2<<endl;

			partitionStream<<"#File	Name	FirstSequence	LastSequence	NumberOfSequences"<<endl;

			LargeCount totalSequences=0;
			for(int i=0;i<(int)m_parameters->getNumberOfFiles();i++){
				f2<<"FileNumber: "<<i<<endl;
				f2<<"	FilePath: "<<m_parameters->getFile(i)<<endl;

				LargeCount entries=m_parameters->getNumberOfSequences(i);
				f2<<" 	NumberOfSequences: "<<entries<<endl;

				if(entries>0){
					f2<<"	FirstSequence: "<<totalSequences<<endl;
					f2<<"	LastSequence: "<<totalSequences+entries-1<<endl;
				}

				f2<<endl;

				if(entries>0){
					partitionStream<<i<<"	"<<m_parameters->getFile(i);
					partitionStream<<"	"<<totalSequences;
					partitionStream<<"	"<<totalSequences+entries-1;
					partitionStream<<"	"<<entries<<endl;
				}

				totalSequences+=entries;
			}
			
			partitionStream.close();

			f2<<endl;
			f2<<"Summary"<<endl;
			f2<<"	NumberOfSequences: "<<totalSequences<<endl;
			f2<<"	FirstSequence: 0"<<endl;
			f2<<"	LastSequence: "<<totalSequences-1<<endl;
			f2.close();
			cout<<"Rank "<<m_parameters->getRank()<<" wrote "<<fileName.str()<<endl;


			/* write the partition */
			ostringstream fileName2;
			fileName2<<m_parameters->getPrefix();
			fileName2<<"SequencePartition.txt";
			ofstream f3(fileName2.str().c_str());

			LargeCount perRank=totalSequences/m_parameters->getSize();
			f3<<"#Rank	FirstSequence	LastSequence	NumberOfSequences"<<endl;
			for(int i=0;i<m_parameters->getSize();i++){
				LargeIndex first=i*perRank;
				LargeIndex last=first+perRank-1;

				if(i==m_parameters->getSize()-1){
					last=totalSequences-1;
				}
				
				LargeCount count=last-first+1;

				f3<<i<<"\t"<<first<<"\t"<<last<<"\t"<<count<<endl;
			}
			f3.close();
			cout<<"Rank "<<m_parameters->getRank()<<" wrote "<<fileName2.str()<<endl;

			m_switchMan->closeMasterMode();
		}
	}
}
Ejemplo n.º 12
0
void SeedingData::computeSeeds(){
	if(!m_initiatedIterator){
		m_last=time(NULL);

		m_SEEDING_i=0;

		m_activeWorkerIterator=m_activeWorkers.begin();
		m_splayTreeIterator.constructor(m_subgraph,m_wordSize,m_parameters);
		m_initiatedIterator=true;
		m_maximumAliveWorkers=30000;
		#ifdef ASSERT
		m_splayTreeIterator.hasNext();
		#endif
	}

	m_virtualCommunicator->processInbox(&m_activeWorkersToRestore);

	if(!m_virtualCommunicator->isReady()){
		return;
	}

	// flush all mode is necessary to empty buffers and 
	// restart things from scratch..

	// 1. iterate on active workers
	if(m_activeWorkerIterator!=m_activeWorkers.end()){
		uint64_t workerId=*m_activeWorkerIterator;
		#ifdef ASSERT
		assert(m_aliveWorkers.count(workerId)>0);
		assert(!m_aliveWorkers[workerId].isDone());
		#endif
		m_virtualCommunicator->resetLocalPushedMessageStatus();

		//force the worker to work until he finishes or pushes something on the stack
		while(!m_aliveWorkers[workerId].isDone()&&!m_virtualCommunicator->getLocalPushedMessageStatus()){
			m_aliveWorkers[workerId].work();
		}

		if(m_virtualCommunicator->getLocalPushedMessageStatus()){
			m_waitingWorkers.push_back(workerId);
		}
		if(m_aliveWorkers[workerId].isDone()){
			m_workersDone.push_back(workerId);
			vector<Kmer> seed=*(m_aliveWorkers[workerId].getSeed());

			int nucleotides=seed.size()+(m_wordSize)-1;

			// only consider the long ones.
			if(nucleotides>=m_parameters->getMinimumContigLength()){
				
				Kmer firstVertex=seed[0];
				Kmer lastVertex=seed[seed.size()-1];
				Kmer firstReverse=m_parameters->_complementVertex(&lastVertex);

				if(firstVertex<firstReverse){
					printf("Rank %i discovered a seed with %i vertices\n",m_rank,(int)seed.size());
					fflush(stdout);

					if(m_parameters->showMemoryUsage()){
						showMemoryUsage(m_rank);
					}
					m_SEEDING_seeds.push_back(seed);
				}
			}
		}
		m_activeWorkerIterator++;
	}else{
		updateStates();

		//  add one worker to active workers
		//  reason is that those already in the pool don't communicate anymore -- 
		//  as for they need responses.
		if(!m_virtualCommunicator->getGlobalPushedMessageStatus()&&m_activeWorkers.empty()){
			// there is at least one worker to start
			// AND
			// the number of alive workers is below the maximum
			if(m_SEEDING_i<(uint64_t)m_subgraph->size()&&(int)m_aliveWorkers.size()<m_maximumAliveWorkers){
				if(m_SEEDING_i % 100000 ==0){
					printf("Rank %i is creating seeds [%i/%i]\n",getRank(),(int)m_SEEDING_i+1,(int)m_subgraph->size());
					fflush(stdout);

					if(m_parameters->showMemoryUsage()){
						showMemoryUsage(m_rank);
					}
				}
				#ifdef ASSERT
				if(m_SEEDING_i==0){
					assert(m_completedJobs==0&&m_activeWorkers.size()==0&&m_aliveWorkers.size()==0);
				}
				#endif
				Vertex*node=m_splayTreeIterator.next();
				Kmer vertexKey=*(m_splayTreeIterator.getKey());

				int coverage=node->getCoverage(&vertexKey);
				int minimum=5;
				if(coverage<minimum){
					m_completedJobs++;
				}else{
					m_aliveWorkers[m_SEEDING_i].constructor(&vertexKey,m_parameters,m_outboxAllocator,m_virtualCommunicator,m_SEEDING_i);
					m_activeWorkers.insert(m_SEEDING_i);
				}

				int population=m_aliveWorkers.size();
				if(population>m_maximumWorkers){
					m_maximumWorkers=population;
				}

				m_SEEDING_i++;

				// skip the reverse complement as we don't really need it anyway.
			}else{
				m_virtualCommunicator->forceFlush();
			}
		}

		// brace yourself for the next round
		m_activeWorkerIterator=m_activeWorkers.begin();
	}

	#ifdef ASSERT
	assert((int)m_aliveWorkers.size()<=m_maximumAliveWorkers);
	#endif

	if((int)m_subgraph->size()==m_completedJobs){
		(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
		printf("Rank %i has %i seeds\n",m_rank,(int)m_SEEDING_seeds.size());
		fflush(stdout);
		printf("Rank %i is creating seeds [%i/%i] (completed)\n",getRank(),(int)m_SEEDING_i,(int)m_subgraph->size());
		fflush(stdout);
		printf("Rank %i: peak number of workers: %i, maximum: %i\n",m_rank,m_maximumWorkers,m_maximumAliveWorkers);
		fflush(stdout);
		m_virtualCommunicator->printStatistics();
		Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_SEEDING_IS_OVER,getRank());
		m_outbox->push_back(aMessage);

		if(m_parameters->showMemoryUsage()){
			showMemoryUsage(m_rank);
		}

		#ifdef ASSERT
		assert(m_aliveWorkers.size()==0);
		assert(m_activeWorkers.size()==0);
		#endif

		// sort the seeds by length
		std::sort(m_SEEDING_seeds.begin(),m_SEEDING_seeds.end(),myComparator_sort);
	}
}
Ejemplo n.º 13
0
void SeedingData::call_RAY_SLAVE_MODE_START_SEEDING(){
	if(!m_initiatedIterator){
		m_last=time(NULL);

		m_SEEDING_i=0;

		m_activeWorkerIterator=m_activeWorkers.begin();
		m_splayTreeIterator.constructor(m_subgraph,m_wordSize,m_parameters);
		m_initiatedIterator=true;
		m_maximumAliveWorkers=32768;

		#ifdef ASSERT
		m_splayTreeIterator.hasNext();
		#endif


		m_virtualCommunicator->resetCounters();
	}

	if(!m_checkedCheckpoint){
		if(m_parameters->hasCheckpoint("Seeds")){
			cout<<"Rank "<<m_parameters->getRank()<<": checkpoint Seeds exists, not computing seeds."<<endl;
			(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
			Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_SEEDING_IS_OVER,getRank());
			m_outbox->push_back(&aMessage);

			loadCheckpoint();

			return;
		}
		m_checkedCheckpoint=true;
	}

	m_virtualCommunicator->processInbox(&m_activeWorkersToRestore);

	if(!m_virtualCommunicator->isReady()){
		return;
	}

	// flush all mode is necessary to empty buffers and 
	// restart things from scratch..

	// 1. iterate on active workers
	if(m_activeWorkerIterator!=m_activeWorkers.end()){
		WorkerHandle workerId=*m_activeWorkerIterator;
		#ifdef ASSERT
		assert(m_aliveWorkers.count(workerId)>0);
		assert(!m_aliveWorkers[workerId].isDone());
		#endif
		m_virtualCommunicator->resetLocalPushedMessageStatus();

		//force the worker to work until he finishes or pushes something on the stack
		while(!m_aliveWorkers[workerId].isDone()&&!m_virtualCommunicator->getLocalPushedMessageStatus()){
			m_aliveWorkers[workerId].work();
		}

		if(m_virtualCommunicator->getLocalPushedMessageStatus()){
			m_waitingWorkers.push_back(workerId);
		}
		if(m_aliveWorkers[workerId].isDone()){
			m_workersDone.push_back(workerId);
			GraphPath*seed=m_aliveWorkers[workerId].getSeed();

			int nucleotides=getNumberOfNucleotides(seed->size(),m_wordSize);

			if(seed->size() > 0 && m_debugSeeds){
				cout<<"Raw seed length: "<<nucleotides<<" nucleotides"<<endl;
			}

			#ifdef ASSERT
			assert(nucleotides==0 || nucleotides>=m_wordSize);
			#endif

			SeedWorker*worker=&(m_aliveWorkers[workerId]);

			if(worker->isHeadADeadEnd() && worker->isTailADeadEnd()){
			
				m_skippedObjectsWithTwoDeadEnds++;

			}else if(worker->isHeadADeadEnd()){

				m_skippedObjectsWithDeadEndForHead++;

			}else if(worker->isTailADeadEnd()){

				m_skippedObjectsWithDeadEndForTail++;

			}else if(worker->isBubbleWeakComponent()){

				m_skippedObjectsWithBubbleWeakComponent++;

			// only consider the long ones.
			}else if(nucleotides>=m_parameters->getMinimumContigLength()){
				#ifdef SHOW_DISCOVERIES
				printf("Rank %i discovered a seed with %i vertices\n",m_rank,(int)seed.size());
				#endif
				
				#ifdef ASSERT
				assert(seed->size()>0);
				#endif

				Kmer firstVertex;
				seed->at(0,&firstVertex);
				Kmer lastVertex;
				seed->at(seed->size()-1,&lastVertex);
				Kmer firstReverse=m_parameters->_complementVertex(&lastVertex);

				int minimumNucleotidesForVerbosity=1024;

				bool verbose=nucleotides>=minimumNucleotidesForVerbosity;

				if(m_debugSeeds){
					verbose=true;
				}

				if(firstVertex<firstReverse){

					if(verbose){
						printf("Rank %i stored a seed with %i vertices\n",m_rank,(int)seed->size());
					}

					if(m_parameters->showMemoryUsage() && verbose){
						showMemoryUsage(m_rank);
					}

					GraphPath*theSeed=seed;

					theSeed->computePeakCoverage();
		
					CoverageDepth peakCoverage=theSeed->getPeakCoverage();

					if(verbose)
						cout<<"Got a seed, peak coverage: "<<peakCoverage;
	
					/* ignore the seed if it has too much coverage. */
					if(peakCoverage >= m_minimumSeedCoverageDepth
						&& peakCoverage <= m_parameters->getMaximumSeedCoverage()){

						if(verbose)
							cout<<", adding seed."<<endl;

						m_SEEDING_seeds.push_back(*theSeed);
		
						m_eligiblePaths++;
					}else{

						if(verbose)
							cout<<", ignoring seed."<<endl;
			
						m_skippedNotEnoughCoverage++;
					}
				}else{
					m_skippedNotMine++;
				}
			}else{
				m_skippedTooShort++;
			}
		}
		m_activeWorkerIterator++;
	}else{
		updateStates();

		//  add one worker to active workers
		//  reason is that those already in the pool don't communicate anymore -- 
		//  as for they need responses.
		if(!m_virtualCommunicator->getGlobalPushedMessageStatus()&&m_activeWorkers.empty()){
			// there is at least one worker to start
			// AND
			// the number of alive workers is below the maximum
			if(m_SEEDING_i<m_subgraph->size()&&(int)m_aliveWorkers.size()<m_maximumAliveWorkers){
				if(m_SEEDING_i % 100000 ==0){
					printf("Rank %i is creating seeds [%i/%i]\n",getRank(),(int)m_SEEDING_i+1,(int)m_subgraph->size());

					if(m_parameters->showMemoryUsage()){
						showMemoryUsage(m_rank);
					}
				}

				#ifdef ASSERT
				if(m_SEEDING_i==0){
					assert(m_completedJobs==0&&m_activeWorkers.size()==0&&m_aliveWorkers.size()==0);
				}
				#endif

				m_splayTreeIterator.next();
				Kmer vertexKey=*(m_splayTreeIterator.getKey());

				m_aliveWorkers[m_SEEDING_i].constructor(&vertexKey,m_parameters,m_outboxAllocator,m_virtualCommunicator,m_SEEDING_i,
RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,
RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE
);
				if(m_debugSeeds)
					m_aliveWorkers[m_SEEDING_i].enableDebugMode();

				m_activeWorkers.insert(m_SEEDING_i);

				int population=m_aliveWorkers.size();
				if(population>m_maximumWorkers){
					m_maximumWorkers=population;
				}

				m_SEEDING_i++;

				// skip the reverse complement as we don't really need it anyway.
			}else{
				m_virtualCommunicator->forceFlush();
			}
		}

		// brace yourself for the next round
		m_activeWorkerIterator=m_activeWorkers.begin();
	}

	#ifdef ASSERT
	assert((int)m_aliveWorkers.size()<=m_maximumAliveWorkers);
	#endif

	if((int)m_subgraph->size()==m_completedJobs){

		printf("Rank %i has %i seeds\n",m_rank,(int)m_SEEDING_seeds.size());
		printf("Rank %i is creating seeds [%i/%i] (completed)\n",getRank(),(int)m_SEEDING_i,(int)m_subgraph->size());
		printf("Rank %i: peak number of workers: %i, maximum: %i\n",m_rank,m_maximumWorkers,m_maximumAliveWorkers);
		m_virtualCommunicator->printStatistics();

		cout<<"Rank "<<m_rank<<" runtime statistics for seeding algorithm: "<<endl;
		cout<<"Rank "<<m_rank<<" Skipped paths because of dead end for head: "<<m_skippedObjectsWithDeadEndForHead<<endl;
		cout<<"Rank "<<m_rank<<" Skipped paths because of dead end for tail: "<<m_skippedObjectsWithDeadEndForTail<<endl;
		cout<<"Rank "<<m_rank<<" Skipped paths because of two dead ends: "<<m_skippedObjectsWithTwoDeadEnds<<endl;
		cout<<"Rank "<<m_rank<<" Skipped paths because of bubble weak component: "<<m_skippedObjectsWithBubbleWeakComponent<<endl;
		cout<<"Rank "<<m_rank<<" Skipped paths because of short length: "<<m_skippedTooShort<<endl;
		cout<<"Rank "<<m_rank<<" Skipped paths because of bad ownership: "<<m_skippedNotMine<<endl;
		cout<<"Rank "<<m_rank<<" Skipped paths because of low coverage: "<<m_skippedNotEnoughCoverage<<endl;
		cout<<"Rank "<<m_rank<<" Eligible paths: "<<m_eligiblePaths<<endl;

		#ifdef ASSERT
		assert(m_eligiblePaths==(int)m_SEEDING_seeds.size());
		#endif

		(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
		Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_SEEDING_IS_OVER,getRank());
		m_outbox->push_back(&aMessage);

		if(m_parameters->showMemoryUsage()){
			showMemoryUsage(m_rank);
		}

		#ifdef ASSERT
		assert(m_aliveWorkers.size()==0);
		assert(m_activeWorkers.size()==0);
		#endif

		// sort the seeds by length
		std::sort(m_SEEDING_seeds.begin(),
			m_SEEDING_seeds.end(),myComparator_sort);
	}
}
Ejemplo n.º 14
0
void Partitioner::call_RAY_SLAVE_MODE_COUNT_FILE_ENTRIES(){
	/** initialize the slave */
	if(!m_initiatedSlave){
		m_initiatedSlave=true;
		m_currentFileToCount=0;
		m_currentlySendingCounts=false;
		m_sentCount=false;
		
		/* possibly read the checkpoint */
		if(m_parameters->hasCheckpoint("Partition")){
			ifstream f(m_parameters->getCheckpointFile("Partition").c_str());
			cout<<"Rank "<<m_parameters->getRank()<<" is reading checkpoint Partition"<<endl;
			int count=0;
			f.read((char*)&count,sizeof(int));
			for(int i=0;i<count;i++){
				int file=-1;
				LargeCount sequences=0;
				f.read((char*)&file,sizeof(int));
				f.read((char*)&sequences,sizeof(LargeCount));

				#ifdef ASSERT
				assert(file>=0);
				assert(m_slaveCounts.count(file)==0);
				#endif
			
				m_slaveCounts[file]=sequences;

				#ifdef ASSERT
				assert(m_slaveCounts.count(file)>0);
				#endif
				cout<<"Rank "<<m_parameters->getRank()<<": from checkpoint Partition, file "<<file<<" has "<<sequences<<" sequences."<<endl;
			}
			f.close();
			m_currentFileToCount=m_parameters->getNumberOfFiles();
		}
	/* all files were processed, tell control peer that we are done */
	}else if(m_currentFileToCount==m_parameters->getNumberOfFiles()){
		Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_COUNT_FILE_ENTRIES_REPLY,m_parameters->getRank());
		m_outbox->push_back(aMessage);
		/* increment it so we don't go here again. */
		m_currentFileToCount++;

		/* Here we write the checkpoint Partition */
		if(m_parameters->writeCheckpoints() && !m_parameters->hasCheckpoint("Partition")){
			ofstream f(m_parameters->getCheckpointFile("Partition").c_str());
			cout<<"Rank "<<m_parameters->getRank()<<" is writing checkpoint Partition"<<endl;
			int count=m_slaveCounts.size();
			f.write((char*)&count,sizeof(int));

			for(map<int,LargeCount>::iterator i=m_slaveCounts.begin();
				i!=m_slaveCounts.end();i++){
				int file=i->first;
				LargeCount sequences=i->second;
				f.write((char*)&file,sizeof(int));
				f.write((char*)&sequences,sizeof(LargeCount));
			}

			f.close();
		}
	/** count sequences in a file */
	}else if(m_currentFileToCount<m_parameters->getNumberOfFiles()){
		int rankInCharge=m_currentFileToCount%m_parameters->getSize();
		if(rankInCharge==m_parameters->getRank()){
			/** count the entries in the file */
			string file=m_parameters->getFile(m_currentFileToCount);
			//cout<<"Rank "<<m_parameters->getRank()<<" Reading "<<file<<endl;
			int res=m_loader.load(file,false);
			if(res==EXIT_FAILURE){
				cout<<"Rank "<<m_parameters->getRank()<<" Error: "<<file<<" failed to load properly..."<<endl;
			}
			m_slaveCounts[m_currentFileToCount]=m_loader.size();

			m_loader.clear();

			cout<<"Rank "<<m_parameters->getRank()<<": File "<<file<<" (Number "<<m_currentFileToCount<<") has "<<m_slaveCounts[m_currentFileToCount]<<" sequences"<<endl;
		}
		m_currentFileToCount++;

	/** control peer asks the slave to send counts */
	}else if(m_inbox->size()>0 && m_inbox->at(0)->getTag() == RAY_MPI_TAG_REQUEST_FILE_ENTRY_COUNTS){
		m_currentFileToSend=0;
		m_currentlySendingCounts=true;
	/** sending counts */
	}else if(m_currentlySendingCounts){
		if(m_currentFileToSend<m_parameters->getNumberOfFiles()){
			int rankInCharge=m_currentFileToSend%m_parameters->getSize();
			/** skip the file, we are not in charge */
			if(rankInCharge!=m_parameters->getRank()){
				m_currentFileToSend++;
			/** send the count and wait for a reply to continue */
			}else if(!m_sentCount){
				MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(MAXIMUM_MESSAGE_SIZE_IN_BYTES);
				message[0]=m_currentFileToSend;
				message[1]=m_slaveCounts[m_currentFileToSend];
				Message aMessage(message,2,MASTER_RANK,RAY_MPI_TAG_FILE_ENTRY_COUNT,m_parameters->getRank());
				m_outbox->push_back(aMessage);
				m_sentCount=true;
			/** we got a reply, let's continue */
			}else if(m_inbox->size()>0 && m_inbox->at(0)->getTag() == RAY_MPI_TAG_FILE_ENTRY_COUNT_REPLY){
				m_sentCount=false;
				m_currentFileToSend++;
			}
		/** all counts were processed, report this to the control peer */
		}else{
			Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_REQUEST_FILE_ENTRY_COUNTS_REPLY,m_parameters->getRank());
			m_outbox->push_back(aMessage);
			m_slaveCounts.clear();

			m_switchMan->setSlaveMode(RAY_SLAVE_MODE_DO_NOTHING);
		}
	}
}
Ejemplo n.º 15
0
void SequencesIndexer::call_RAY_SLAVE_MODE_INDEX_SEQUENCES(){
	if(!m_initiatedIterator){
		m_theSequenceId=0;

		m_activeWorkerIterator.constructor(&m_activeWorkers);
		m_initiatedIterator=true;
		m_maximumAliveWorkers=32768;


		m_virtualCommunicator->resetCounters();
	}

	if(!m_checkedCheckpoint){
		if(m_parameters->hasCheckpoint("OptimalMarkers") && m_parameters->hasCheckpoint("ReadOffsets")){
			cout<<"Rank "<<m_parameters->getRank()<<": checkpoint OptimalMarkers exists, not selecting markers."<<endl;
			(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
			Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS_REPLY,m_rank);
			m_outbox->push_back(&aMessage);
			return;
		}
		m_checkedCheckpoint=true;
	}

	m_virtualCommunicator->processInbox(&m_activeWorkersToRestore);

	if(!m_virtualCommunicator->isReady()){
		return;
	}

	if(m_activeWorkerIterator.hasNext()){
		WorkerHandle workerId=m_activeWorkerIterator.next()->getKey();

		#ifdef ASSERT
		assert(m_aliveWorkers.find(workerId,false)!=NULL);
		assert(!m_aliveWorkers.find(workerId,false)->getValue()->isDone());
		#endif
		m_virtualCommunicator->resetLocalPushedMessageStatus();

		//force the worker to work until he finishes or pushes something on the stack
		while(!m_aliveWorkers.find(workerId,false)->getValue()->isDone()&&!m_virtualCommunicator->getLocalPushedMessageStatus()){
			m_aliveWorkers.find(workerId,false)->getValue()->work();
		}

		if(m_virtualCommunicator->getLocalPushedMessageStatus()){
			m_waitingWorkers.push_back(workerId);
		}
		if(m_aliveWorkers.find(workerId,false)->getValue()->isDone()){
			m_workersDone.push_back(workerId);
		}
	}else{
		updateStates();

		//  add one worker to active workers
		//  reason is that those already in the pool don't communicate anymore -- 
		//  as for they need responses.
		if(!m_virtualCommunicator->getGlobalPushedMessageStatus()&&m_activeWorkers.size()==0){
			// there is at least one worker to start
			// AND
			// the number of alive workers is below the maximum
			if(m_theSequenceId<(int)m_myReads->size()&&(int)m_aliveWorkers.size()<m_maximumAliveWorkers){
				if(m_theSequenceId%100000==0){
					printf("Rank %i is selecting optimal read markers [%i/%i]\n",m_rank,m_theSequenceId+1,(int)m_myReads->size());

					m_derivative.addX(m_theSequenceId);
					m_derivative.printStatus(SLAVE_MODES[RAY_SLAVE_MODE_INDEX_SEQUENCES],RAY_SLAVE_MODE_INDEX_SEQUENCES);
					m_derivative.printEstimatedTime(m_myReads->size());

					if(m_parameters->showMemoryUsage())
						showMemoryUsage(m_rank);
				}

				#ifdef ASSERT
				if(m_theSequenceId==0){
					assert(m_completedJobs==0&&m_activeWorkers.size()==0&&m_aliveWorkers.size()==0);
				}
				assert(m_theSequenceId<(int)m_myReads->size());
				#endif


				bool flag;
				m_aliveWorkers.insert(m_theSequenceId,&m_workAllocator,&flag)->getValue()->constructor(m_theSequenceId,m_parameters,m_outboxAllocator,m_virtualCommunicator,
					m_theSequenceId,m_myReads,&m_workAllocator,&m_readMarkerFile,&m_forwardStatistics,
					&m_reverseStatistics,
	RAY_MPI_TAG_ATTACH_SEQUENCE,
	RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE
);

				m_activeWorkers.insert(m_theSequenceId,&m_workAllocator,&flag);
				int population=m_aliveWorkers.size();
				if(population>m_maximumWorkers){
					m_maximumWorkers=population;
				}

				m_theSequenceId++;
			}else{
				m_virtualCommunicator->forceFlush();
			}
		}

		m_activeWorkerIterator.constructor(&m_activeWorkers);
	}

	#ifdef ASSERT
	assert((int)m_aliveWorkers.size()<=m_maximumAliveWorkers);
	#endif

	if((int)m_myReads->size()==m_completedJobs){
		printf("Rank %i is selecting optimal read markers [%i/%i] (completed)\n",m_rank,(int)m_myReads->size(),(int)m_myReads->size());
		printf("Rank %i: peak number of workers: %i, maximum: %i\n",m_rank,m_maximumWorkers,m_maximumAliveWorkers);
		(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
		Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS_REPLY,m_rank);
		m_outbox->push_back(&aMessage);

		m_derivative.writeFile(&cout);

		m_virtualCommunicator->printStatistics();

		if(m_parameters->showMemoryUsage()){
			showMemoryUsage(m_rank);
		}

		#ifdef ASSERT
		assert(m_aliveWorkers.size()==0);
		assert(m_activeWorkers.size()==0);
		#endif

		int freed=m_workAllocator.getNumberOfChunks()*m_workAllocator.getChunkSize();
		m_workAllocator.clear();

		if(m_parameters->showMemoryUsage()){
			cout<<"Rank "<<m_parameters->getRank()<<": Freeing unused assembler memory: "<<freed/1024<<" KiB freed"<<endl;
			showMemoryUsage(m_rank);
		}

		if(m_parameters->hasOption("-write-read-markers")){
			m_readMarkerFile.close();
		}

		if(m_parameters->hasOption("-write-marker-summary")){

			ostringstream file1;
			file1<<m_parameters->getPrefix()<<"Rank"<<m_parameters->getRank()<<".ForwardMarkerSummary.txt";
			string fileName1=file1.str();
			ofstream f1(fileName1.c_str());

			for(map<int,map<int,int> >::iterator i=m_forwardStatistics.begin();i!=m_forwardStatistics.end();i++){
				int offset=i->first;
				for(map<int,int>::iterator j=i->second.begin();j!=i->second.end();j++){
					int coverage=j->first;
					int count=j->second;
					f1<<offset<<"	"<<coverage<<"	"<<count<<endl;
				}
			}
			f1.close();

			ostringstream file2;
			file2<<m_parameters->getPrefix()<<"Rank"<<m_parameters->getRank()<<".ReverseMarkerSummary.txt";
			string fileName2=file2.str();
			ofstream f2(fileName2.c_str());

			for(map<int,map<int,int> >::iterator i=m_reverseStatistics.begin();i!=m_reverseStatistics.end();i++){
				int offset=i->first;
				for(map<int,int>::iterator j=i->second.begin();j!=i->second.end();j++){
					int coverage=j->first;
					int count=j->second;
					f2<<offset<<"	"<<coverage<<"	"<<count<<endl;
				}
			}
			f2.close();

		}

		m_forwardStatistics.clear();
		m_reverseStatistics.clear();
	}
}
Ejemplo n.º 16
0
void ClientAgent::MessageReceived(BMessage* msg)
{
	switch (msg->what) {
	// 22/8/99: this will now look for "text" to add to the
	// fInput view. -jamie
	case M_INPUT_FOCUS: {
		if (msg->HasString("text")) {
			BString newtext;
			newtext = fInput->Text();
			newtext.Append(msg->FindString("text"));
			fInput->SetText(newtext.String());
		}
		fInput->MakeFocus(true);
		// We don't like your silly selecting-on-focus.
		fInput->TextView()->Select(fInput->TextView()->TextLength(),
								   fInput->TextView()->TextLength());
	} break;

	case M_CLIENT_QUIT: {
		if (fIsLogging && !(msg->HasBool("vision:shutdown") && msg->FindBool("vision:shutdown"))) {
			BMessage logMessage(M_UNREGISTER_LOGGER);
			logMessage.AddString("name", fId.String());
			fSMsgr.SendMessage(&logMessage);
		}

		BMessage deathchant(M_CLIENT_SHUTDOWN);
		deathchant.AddPointer("agent", this);
		fSMsgr.SendMessage(&deathchant);

	} break;

	case M_THEME_FOREGROUND_CHANGE: {
		int16 which(msg->FindInt16("which"));
		if (which == C_INPUT || which == C_INPUT_BACKGROUND) {
			fActiveTheme->ReadLock();
			rgb_color fInputColor(fActiveTheme->ForegroundAt(C_INPUT));
			fInput->TextView()->SetFontAndColor(&fActiveTheme->FontAt(F_INPUT), B_FONT_ALL,
												&fInputColor);
			fInput->TextView()->SetViewColor(fActiveTheme->ForegroundAt(C_INPUT_BACKGROUND));
			fActiveTheme->ReadUnlock();
			fInput->TextView()->Invalidate();
		}
	} break;

	case M_THEME_FONT_CHANGE: {
		int16 which(msg->FindInt16("which"));
		if (which == F_INPUT) {
			fActiveTheme->ReadLock();
			rgb_color fInputColor(fActiveTheme->ForegroundAt(C_INPUT));
			fInput->TextView()->SetFontAndColor(&fActiveTheme->FontAt(F_INPUT), B_FONT_ALL,
												&fInputColor);
			fActiveTheme->ReadUnlock();
			Invalidate();
		}
	} break;

	case M_STATE_CHANGE: {
		if (msg->HasBool("bool")) {
			bool shouldStamp(vision_app->GetBool("timestamp"));
			if (fTimeStampState != shouldStamp) {
				if ((fTimeStampState = shouldStamp))
					fText->SetTimeStampFormat(vision_app->GetString("timestamp_format"));
				else
					fText->SetTimeStampFormat(NULL);
			}

			bool shouldLog = vision_app->GetBool("log_enabled");

			if (fIsLogging != shouldLog) {
				if ((fIsLogging = shouldLog)) {
					BMessage logMessage(M_REGISTER_LOGGER);
					logMessage.AddString("name", fId.String());
					fSMsgr.SendMessage(&logMessage);
				} else {
					BMessage logMessage(M_UNREGISTER_LOGGER);
					logMessage.AddString("name", fId.String());
					fSMsgr.SendMessage(&logMessage);
				}
			}
		} else if (msg->HasBool("string")) {
			BString which(msg->FindString("which"));
			if (which == "timestamp_format")
				fText->SetTimeStampFormat(vision_app->GetString("timestamp_format"));
		}
	} break;

	case M_SUBMIT_INPUT: {
		fCancelMLPaste = false;
		int32 which(0);

		msg->FindInt32("which", &which);

		if (msg->HasPointer("invoker")) {
			BInvoker* invoker(NULL);
			msg->FindPointer("invoker", reinterpret_cast<void**>(&invoker));
			delete invoker;
		}

		switch (which) {
		case PASTE_CANCEL:
			break;

		case PASTE_MULTI:
		case PASTE_MULTI_NODELAY: {
			BMessage* buffer(new BMessage(*msg));
			thread_id tid;

			// if there is some text in the input control already, submit it before
			// starting the timed paste
			if (fInput->TextView()->TextLength() != 0) {
				BString inputData(fInput->TextView()->Text());
				Submit(inputData.String(), true, true);
			}

			buffer->AddPointer("agent", this);
			buffer->AddPointer("window", Window());
			if (which == PASTE_MULTI_NODELAY) buffer->AddBool("delay", false);
			tid = spawn_thread(TimedSubmit, "Timed Submit", B_LOW_PRIORITY, buffer);
			resume_thread(tid);
		} break;

		case PASTE_SINGLE: {
			BString buffer;
			for (int32 i = 0; msg->HasString("data", i); ++i) {
				const char* data;
				msg->FindString("data", i, &data);
				buffer += (i ? " " : "");
				buffer += data;
			}

			int32 start, finish;

			if (msg->FindInt32("selstart", &start) == B_OK) {
				msg->FindInt32("selend", &finish);
				if (start != finish) fInput->TextView()->Delete(start, finish);

				if ((start == 0) && (finish == 0)) {
					fInput->TextView()->Insert(fInput->TextView()->TextLength(), buffer.String(),
											   buffer.Length());
					fInput->TextView()->Select(fInput->TextView()->TextLength(),
											   fInput->TextView()->TextLength());
				} else {
					fInput->TextView()->Insert(start, buffer.String(), buffer.Length());
					fInput->TextView()->Select(start + buffer.Length(), start + buffer.Length());
				}
			} else {
				fInput->TextView()->Insert(buffer.String());
				fInput->TextView()->Select(fInput->TextView()->TextLength(),
										   fInput->TextView()->TextLength());
			}
			fInput->TextView()->ScrollToSelection();
		} break;

		default:
			break;
		}
	} break;

	case M_PREVIOUS_INPUT: {
		fHistory->PreviousBuffer(fInput);
	} break;

	case M_NEXT_INPUT: {
		fHistory->NextBuffer(fInput);
	} break;

	case M_SUBMIT: {
		const char* buffer(NULL);
		bool clear(true), add2history(true);

		msg->FindString("input", &buffer);

		if (msg->HasBool("clear")) msg->FindBool("clear", &clear);

		if (msg->HasBool("history")) msg->FindBool("history", &add2history);

		Submit(buffer, clear, add2history);
	} break;

	case M_LAG_CHANGED: {
		msg->FindString("lag", &fMyLag);

		if (!IsHidden())
			vision_app->pClientWin()->pStatusView()->SetItemValue(STATUS_LAG, fMyLag.String());
	} break;

	case M_DISPLAY: {
		const char* buffer;

		for (int32 i = 0; msg->HasMessage("packed", i); ++i) {
			BMessage packed;

			msg->FindMessage("packed", i, &packed);
			packed.FindString("msgz", &buffer);
			Display(buffer, packed.FindInt32("fore"), packed.FindInt32("back"),
					packed.FindInt32("font"));
		}
	} break;

	case M_CHANNEL_MSG: {
		BString theNick;
		const char* theMessage(NULL);
		bool hasNick(false);
		bool isAction(false);
		BString knownAs;

		msg->FindString("nick", &theNick);
		msg->FindString("msgz", &theMessage);

		BString tempString;
		BString nickString;

		if (theMessage[0] == '\1') {
			BString aMessage(theMessage);
			aMessage.RemoveFirst("\1ACTION ");
			aMessage.RemoveLast("\1");

			tempString = " ";
			tempString += aMessage;
			tempString += "\n";

			nickString = "* ";
			nickString += theNick;
			isAction = true;
		} else {
			Display("<", theNick == fMyNick ? C_MYNICK : C_NICK);
			Display(theNick.String(), C_NICKDISPLAY);
			Display(">", theNick == fMyNick ? C_MYNICK : C_NICK);
			tempString += " ";
			tempString += theMessage;
			tempString += '\n';
		}

		// scan for presence of nickname, highlight if present
		if (theNick != fMyNick) FirstKnownAs(tempString, knownAs, &hasNick);

		tempString.Prepend(nickString);

		int32 dispColor = C_TEXT;
		if (hasNick) {
			BWindow* window(NULL);
			dispColor = C_MYNICK;
			if ((window = Window()) != NULL && !window->IsActive())
				system_beep(kSoundEventNames[(uint32)seNickMentioned]);
		} else if (isAction)
			dispColor = C_ACTION;

		Display(tempString.String(), dispColor);
	} break;

	case M_CHANGE_NICK: {
		const char* oldNick(NULL);

		msg->FindString("oldnick", &oldNick);

		if (fMyNick.ICompare(oldNick) == 0) fMyNick = msg->FindString("newnick");

		BMessage display;
		if (msg->FindMessage("display", &display) == B_NO_ERROR)
			ClientAgent::MessageReceived(&display);
	} break;

	case M_LOOKUP_WEBSTER: {
		BString lookup;
		msg->FindString("string", &lookup);
		lookup = StringToURI(lookup.String());
		lookup.Prepend("http://www.m-w.com/cgi-bin/dictionary?va=");
		vision_app->LoadURL(lookup.String());
	} break;

	case M_LOOKUP_GOOGLE: {
		BString lookup;
		msg->FindString("string", &lookup);
		lookup = StringToURI(lookup.String());
		lookup.Prepend("http://www.google.com/search?q=");
		vision_app->LoadURL(lookup.String());
	} break;

	case M_LOOKUP_ACRONYM: {
		BString lookup;
		msg->FindString("string", &lookup);
		lookup = StringToURI(lookup.String());
		lookup.Prepend("http://www.acronymfinder.com/af-query.asp?String=exact&Acronym=");
		lookup.Append("&Find=Find");
		vision_app->LoadURL(lookup.String());
	} break;

	case B_ESCAPE:
		fCancelMLPaste = true;
		break;

	case M_DCC_COMPLETE: {
		/// set up ///
		BString nick, file, size, type, completionMsg("[@] "), fAck;
		int32 rate, xfersize;
		bool completed(true);

		msg->FindString("nick", &nick);
		msg->FindString("file", &file);
		msg->FindString("size", &size);
		msg->FindString("type", &type);
		msg->FindInt32("transferred", &xfersize);
		msg->FindInt32("transferRate", &rate);

		BPath pFile(file.String());

		fAck << xfersize;

		if (size.ICompare(fAck)) completed = false;

		/// send mesage ///
		if (completed)
			completionMsg << S_CLIENT_DCC_SUCCESS;
		else
			completionMsg << S_CLIENT_DCC_FAILED;

		if (type == "SEND")
			completionMsg << S_CLIENT_DCC_SENDTYPE << pFile.Leaf() << S_CLIENT_DCC_TO;
		else
			completionMsg << S_CLIENT_DCC_RECVTYPE << pFile.Leaf() << S_CLIENT_DCC_FROM;

		completionMsg << nick << " (";

		if (!completed) completionMsg << fAck << "/";

		completionMsg << size << S_CLIENT_DCC_SIZE_UNITS "), ";
		completionMsg << rate << S_CLIENT_DCC_SPEED_UNITS "\n";

		Display(completionMsg.String(), C_CTCP_RPY);
	} break;

	default:
		BView::MessageReceived(msg);
	}
}
Ejemplo n.º 17
0
void GenomeNeighbourhood::processLinks(int mode){

	#ifdef ASSERT
	assert(!m_stackOfVertices.empty());
	assert(m_stackOfVertices.size() == m_stackOfDepths.size());
	#endif

	Kmer currentKmer=m_stackOfVertices.top();
	int depth=m_stackOfDepths.top();

	if(!m_directDone){
		if(!m_fetchedPaths){

			fetchPaths(mode);
		}else{
		
			#ifdef DEBUG_SIDE
			if(mode==FETCH_PARENTS){
				cout<<"[DEBUG_SIDE] Fetched direct paths for mode FETCH_PARENTS, n="<<m_paths<<endl;
			}else if(mode==FETCH_CHILDREN){
				cout<<"[DEBUG_SIDE] Fetched direct paths for mode FETCH_CHILDREN, n="<<m_paths<<endl;
			}
			#endif


			#ifdef DEBUG_NEIGHBOURHOOD_COMMUNICATION
			cout<<"m_directDone= True"<<endl;
			#endif

			m_directDone=true;
			m_fetchedPaths=false;
			m_numberOfPathsRequested=false;
			m_numberOfPathsReceived=false;

			m_reverseDone=false;

			m_reverseStrand=true;

		}
/**/
	}else if(!m_reverseDone){
		if(!m_fetchedPaths){

			fetchPaths(mode);

		}else{
			
			#ifdef DEBUG_SIDE
			if(mode==FETCH_PARENTS){
				cout<<"[DEBUG_SIDE] Fetched reverse paths for mode FETCH_PARENTS, n="<<m_paths<<endl;
			}else if(mode==FETCH_CHILDREN){
				cout<<"[DEBUG_SIDE] Fetched reverse paths for mode FETCH_CHILDREN, n="<<m_paths<<endl;
			}
			#endif

		
			#ifdef DEBUG_NEIGHBOURHOOD_COMMUNICATION
			cout<<"m_reverseDone= True"<<endl;
			#endif
	
			m_reverseDone=true;
		}
/**/

	}else if(m_fetchedPaths && !m_linksRequested){

		#ifdef DEBUG_NEIGHBOURHOOD_COMMUNICATION
		cout<<"Sending message RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT to "<<endl;
		#endif

		// send a message to request the links of the current vertex
		MessageUnit*buffer=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(Kmer));
		int bufferPosition=0;
		currentKmer.pack(buffer,&bufferPosition);
	
		Rank destination=m_parameters->_vertexRank(&currentKmer);

		Message aMessage(buffer,m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT),
			destination,RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT,m_rank);

		m_virtualCommunicator->pushMessage(m_workerId,&aMessage);

		m_linksRequested=true;
		m_linksReceived=false;

		#ifdef DEBUG_NEIGHBOURHOOD_COMMUNICATION
		cout<<"Message sent, RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT, will wait for a reply "<<endl;
		#endif

	}else if(m_fetchedPaths &&
		!m_linksReceived && m_virtualCommunicator->isMessageProcessed(m_workerId)){

		#ifdef DEBUG_NEIGHBOURHOOD_COMMUNICATION
		cout<<"Message received, RAY_MPI_TAG_GET_VERTEX_EDGES_COMPACT"<<endl;
		#endif

		vector<MessageUnit> elements;
		m_virtualCommunicator->getMessageResponseElements(m_workerId,&elements);

		#ifdef ASSERT
		assert((int)elements.size()>=2);
		#endif

		uint8_t edges=elements[0];

		#ifdef ASSERT
		int coverage=elements[1];
		assert(coverage>0);
		#endif

		vector<Kmer> parents=currentKmer._getIngoingEdges(edges,m_parameters->getWordSize());
		vector<Kmer> children=currentKmer._getOutgoingEdges(edges,m_parameters->getWordSize());

		#ifdef DEBUG_NEIGHBOURHOOD_COMMUNICATION
		cout<<"information: "<<parents.size()<<" parents, "<<children.size()<<" children"<<endl;
		#endif

		int nextDepth=depth+1;

		// remove the current vertex from the stack
		m_stackOfVertices.pop();
		m_stackOfDepths.pop();

		m_visited.insert(currentKmer);

		// add new links
		
		vector<Kmer>*links=&parents;

		#ifdef ASSERT
		assert(mode==FETCH_CHILDREN || mode==FETCH_PARENTS);
		#endif

		if(mode==FETCH_CHILDREN){
			links=&children;

		}else if(mode==FETCH_PARENTS){
			links=&parents;
		}

		#ifdef ASSERT
		assert(m_stackOfDepths.size()==m_stackOfVertices.size());
		#endif

		/** we don't continue if we found something interesting... **/

		for(int i=0;i<(int)links->size();i++){

			#ifdef ASSERT
			assert(i<(int) links->size());
			#endif

			Kmer newKmer=links->at(i);

			if(nextDepth<= m_maximumDepth && m_visited.count(newKmer)==0

				&& !m_foundPathsForThisVertex){ /* avoid exploring too much when something is already on the table **/
		
				m_stackOfVertices.push(newKmer);
				m_stackOfDepths.push(nextDepth);
			}
		}

		m_linksReceived=true;

	}else if(m_fetchedPaths && m_linksRequested && m_linksReceived){
		
		// restart the adventure

		resetKmerStates();

	}
}
Ejemplo n.º 18
0
void SequencesIndexer::attachReads(ArrayOfReads*m_myReads,
				RingAllocator*m_outboxAllocator,
				StaticVector*m_outbox,
				int*m_mode,
				int m_wordSize,
				int m_size,
				int m_rank
			){
	if(!m_initiatedIterator){
		m_theSequenceId=0;

		m_activeWorkerIterator.constructor(&m_activeWorkers);
		m_initiatedIterator=true;
		m_maximumAliveWorkers=30000;
	}

	m_virtualCommunicator->processInbox(&m_activeWorkersToRestore);


	if(!m_virtualCommunicator->isReady()){
		return;
	}

	if(m_activeWorkerIterator.hasNext()){
		uint64_t workerId=m_activeWorkerIterator.next()->getKey();
		#ifdef ASSERT
		assert(m_aliveWorkers.find(workerId,false)!=NULL);
		assert(!m_aliveWorkers.find(workerId,false)->getValue()->isDone());
		#endif
		m_virtualCommunicator->resetLocalPushedMessageStatus();

		//force the worker to work until he finishes or pushes something on the stack
		while(!m_aliveWorkers.find(workerId,false)->getValue()->isDone()&&!m_virtualCommunicator->getLocalPushedMessageStatus()){
			m_aliveWorkers.find(workerId,false)->getValue()->work();
		}

		if(m_virtualCommunicator->getLocalPushedMessageStatus()){
			m_waitingWorkers.push_back(workerId);
		}
		if(m_aliveWorkers.find(workerId,false)->getValue()->isDone()){
			m_workersDone.push_back(workerId);
		}
	}else{
		updateStates();

		//  add one worker to active workers
		//  reason is that those already in the pool don't communicate anymore -- 
		//  as for they need responses.
		if(!m_virtualCommunicator->getGlobalPushedMessageStatus()&&m_activeWorkers.size()==0){
			// there is at least one worker to start
			// AND
			// the number of alive workers is below the maximum
			if(m_theSequenceId<(int)m_myReads->size()&&(int)m_aliveWorkers.size()<m_maximumAliveWorkers){
				if(m_theSequenceId%10000==0){
					printf("Rank %i is selecting optimal read markers [%i/%i]\n",m_rank,m_theSequenceId+1,(int)m_myReads->size());
					fflush(stdout);
					if(m_parameters->showMemoryUsage())
						showMemoryUsage(m_rank);
				}

				#ifdef ASSERT
				if(m_theSequenceId==0){
					assert(m_completedJobs==0&&m_activeWorkers.size()==0&&m_aliveWorkers.size()==0);
				}
				#endif
				char sequence[4000];
				#ifdef ASSERT
				assert(m_theSequenceId<(int)m_myReads->size());
				#endif

				m_myReads->at(m_theSequenceId)->getSeq(sequence,m_parameters->getColorSpaceMode(),false);

				bool flag;
				m_aliveWorkers.insert(m_theSequenceId,&m_workAllocator,&flag)->getValue()->constructor(m_theSequenceId,sequence,m_parameters,m_outboxAllocator,m_virtualCommunicator,
					m_theSequenceId,m_myReads,&m_workAllocator);
				m_activeWorkers.insert(m_theSequenceId,&m_workAllocator,&flag);
				int population=m_aliveWorkers.size();
				if(population>m_maximumWorkers){
					m_maximumWorkers=population;
				}

				m_theSequenceId++;
			}else{
				m_virtualCommunicator->forceFlush();
			}
		}

		m_activeWorkerIterator.constructor(&m_activeWorkers);
	}

	#ifdef ASSERT
	assert((int)m_aliveWorkers.size()<=m_maximumAliveWorkers);
	#endif

	if((int)m_myReads->size()==m_completedJobs){
		printf("Rank %i is selecting optimal read markers [%i/%i] (completed)\n",m_rank,(int)m_myReads->size(),(int)m_myReads->size());
		fflush(stdout);
		printf("Rank %i: peak number of workers: %i, maximum: %i\n",m_rank,m_maximumWorkers,m_maximumAliveWorkers);
		fflush(stdout);
		(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
		Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_MASTER_IS_DONE_ATTACHING_READS_REPLY,m_rank);
		m_outbox->push_back(aMessage);

		m_virtualCommunicator->printStatistics();

		if(m_parameters->showMemoryUsage()){
			showMemoryUsage(m_rank);
		}

		#ifdef ASSERT
		assert(m_aliveWorkers.size()==0);
		assert(m_activeWorkers.size()==0);
		#endif

		int freed=m_workAllocator.getNumberOfChunks()*m_workAllocator.getChunkSize();
		m_workAllocator.clear();

		if(m_parameters->showMemoryUsage()){
			cout<<"Rank "<<m_parameters->getRank()<<": Freeing unused assembler memory: "<<freed/1024<<" KiB freed"<<endl;
			showMemoryUsage(m_rank);
		}
	}
}
Ejemplo n.º 19
0
void GenomeNeighbourhood::fetchPaths(int mode){

/** stop the search when something is found **/
/** this will speed things up, but will report less hits because of repeated k-mers **/

	bool stopWhenSomethingIsFound=true;

	Kmer currentKmer=m_stackOfVertices.top();
	int depth=m_stackOfDepths.top();

	Kmer kmer=currentKmer;

	if(m_reverseStrand){
		kmer=kmer.complementVertex(m_parameters->getWordSize(),m_parameters->getColorSpaceMode());
	}

	if(!m_numberOfPathsRequested){

		#ifdef DEBUG_SIDE
		if(mode==FETCH_PARENTS){
			cout<<"[DEBUG_SIDE] FETCH_PARENTS RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE"<<endl;
		}else if(mode==FETCH_CHILDREN){
			cout<<"[DEBUG_SIDE] FETCH_CHILDREN RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE"<<endl;
		}
		#endif


		#ifdef DEBUG_NEIGHBOURHOOD_COMMUNICATION
		cout<<"Sending RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE"<<endl;
		#endif

		// send a message to request the links of the current vertex
		MessageUnit*buffer=(MessageUnit*)m_outboxAllocator->allocate(1*sizeof(Kmer));
		int bufferPosition=0;
		kmer.pack(buffer,&bufferPosition);
	
		Rank destination=m_parameters->_vertexRank(&kmer);

		Message aMessage(buffer,m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE),
			destination,RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE,m_rank);

		m_virtualCommunicator->pushMessage(m_workerId,&aMessage);

		m_numberOfPathsReceived=false;

		// keep up the good work for now
		m_linksRequested=false;
		m_linksReceived=false;

		m_numberOfPathsRequested=true;

	}else if(!m_numberOfPathsReceived && m_virtualCommunicator->isMessageProcessed(m_workerId)){

		vector<MessageUnit> elements;
		m_virtualCommunicator->getMessageResponseElements(m_workerId,&elements);

		m_paths=elements[0];

		#ifdef DEBUG_NEIGHBOURHOOD_COMMUNICATION
		cout<<"Received reply for RAY_MPI_TAG_ASK_VERTEX_PATHS_SIZE"<<endl;
		#endif

		#ifdef DEBUG_NEIGHBOURHOOD_PATHS
		cout<<"Number of paths: "<<m_paths<<" depth: "<<depth<<" useReverse= "<<m_reverseStrand<<endl;
		#endif

		m_numberOfPathsReceived=true;
		m_pathIndex=0;

		m_requestedPath=false;
		m_receivedPath=false;

	}else if(m_numberOfPathsReceived && m_pathIndex == m_paths){

		#ifdef DEBUG_NEIGHBOURHOOD_PATHS
		cout<<"Index completed."<<endl;
		#endif

		m_fetchedPaths=true;

	
		#ifdef DEBUG_SIDE
		if(mode==FETCH_PARENTS){
			cout<<"[DEBUG_SIDE] FETCH_PARENTS finished fetching n="<<m_paths<<endl;
		}else if(mode==FETCH_CHILDREN){
			cout<<"[DEBUG_SIDE] FETCH_CHILDREN finished fetching n="<<m_paths<<endl;
		}
		#endif


	}else if(m_numberOfPathsReceived && !m_requestedPath){

		Rank destination=kmer.vertexRank(m_parameters->getSize(),
			m_parameters->getWordSize(),m_parameters->getColorSpaceMode());

		int elementsPerQuery=m_virtualCommunicator->getElementsPerQuery(RAY_MPI_TAG_ASK_VERTEX_PATH);

		MessageUnit*message=(MessageUnit*)m_outboxAllocator->allocate(elementsPerQuery);

		int outputPosition=0;
		kmer.pack(message,&outputPosition);
		message[outputPosition++]=m_pathIndex;

		#ifdef DEBUG_NEIGHBOURHOOD_COMMUNICATION
		cout<<"Sending RAY_MPI_TAG_ASK_VERTEX_PATH"<<endl;
		#endif

		Message aMessage(message,elementsPerQuery,destination,
			RAY_MPI_TAG_ASK_VERTEX_PATH,m_parameters->getRank());

		m_virtualCommunicator->pushMessage(m_workerId,&aMessage);

		m_requestedPath=true;
		m_receivedPath=false;

	}else if(m_numberOfPathsReceived && !m_receivedPath && m_virtualCommunicator->isMessageProcessed(m_workerId)){
		vector<MessageUnit> response;
		m_virtualCommunicator->getMessageResponseElements(m_workerId,&response);

		int bufferPosition=0;

		/* skip the k-mer because we don't need it */
		bufferPosition+=KMER_U64_ARRAY_SIZE;
		PathHandle pathIdentifier=response[bufferPosition++];
		int progression=response[bufferPosition++];

		
		int minimumDepth=1;

		#ifdef DEBUG_NEIGHBOURHOOD_PATHS
		cout<<"Received path mode=";
		if(mode==FETCH_PARENTS){
			cout<<"FETCH_PARENTS";
		}else{
			cout<<"FETCH_CHILDREN";
		}
		cout<<" path is "<<pathIdentifier<<endl;
		#endif

		/** add the path **/

		if(pathIdentifier != (*m_contigNames)[m_contigIndex]
			//&& m_foundContigs.count(pathIdentifier)==0
			&& depth >= minimumDepth){

			char strand='F';
			if(m_reverseStrand){
				strand='R';
			}

			Neighbour friendlyNeighbour(strand,depth,pathIdentifier,progression);
		
			if(stopWhenSomethingIsFound){
				m_foundPathsForThisVertex=true;
			}

			if(mode==FETCH_PARENTS){
				m_leftNeighbours.push_back(friendlyNeighbour);

			}else if(mode == FETCH_CHILDREN){

				m_rightNeighbours.push_back(friendlyNeighbour);
			}

			m_foundContigs.insert(pathIdentifier);
		}

		m_pathIndex++;

		m_receivedPath=true;

		#ifdef DEBUG_NEIGHBOURHOOD_COMMUNICATION
		cout<<"Received reply for RAY_MPI_TAG_ASK_VERTEX_PATH"<<endl;
		#endif

		m_requestedPath=false;
		m_receivedPath=false;
	}
}
Ejemplo n.º 20
0
void VerticesExtractor::process(int*m_mode_send_vertices_sequence_id,
				ArrayOfReads*m_myReads,
				bool*m_reverseComplementVertex,
				int rank,
				StaticVector*m_outbox,
				bool*m_mode_send_vertices,
				int wordSize,
				int size,
				RingAllocator*m_outboxAllocator,
				int*m_mode
				){
	if(this->m_outbox==NULL){
		m_rank=rank;
		this->m_mode=m_mode;
		this->m_outbox=m_outbox;
		this->m_outboxAllocator=m_outboxAllocator;
	}
	#ifdef ASSERT
	assert(m_pendingMessages>=0);
	#endif
	if(m_pendingMessages!=0){
		return;
	}

	if(m_finished){
		return;
	}

	if(*m_mode_send_vertices_sequence_id%100000==0 &&m_mode_send_vertices_sequence_id_position==0
	&&*m_mode_send_vertices_sequence_id<(int)m_myReads->size()){
		string reverse="";
		if(*m_reverseComplementVertex==true){
			reverse="(reverse complement) ";
		}
		printf("Rank %i is computing vertices & edges %s[%i/%i]\n",rank,reverse.c_str(),(int)*m_mode_send_vertices_sequence_id+1,(int)m_myReads->size());
		fflush(stdout);
	}

	if(*m_mode_send_vertices_sequence_id>(int)m_myReads->size()-1){
		// flush data
		flushAll(m_outboxAllocator,m_outbox,rank);
		if(m_pendingMessages==0){
			#ifdef ASSERT
			assert(m_bufferedData.isEmpty());
			assert(m_bufferedDataForIngoingEdges.isEmpty());
			assert(m_bufferedDataForOutgoingEdges.isEmpty());
			#endif

			Message aMessage(NULL,0, MASTER_RANK, RAY_MPI_TAG_VERTICES_DISTRIBUTED,rank);
			m_outbox->push_back(aMessage);
			m_finished=true;
			printf("Rank %i is computing vertices & edges [%i/%i] (completed)\n",rank,(int)*m_mode_send_vertices_sequence_id,(int)m_myReads->size());
			fflush(stdout);
			m_bufferedData.showStatistics(m_parameters->getRank());
			m_bufferedDataForIngoingEdges.showStatistics(m_parameters->getRank());
			m_bufferedDataForOutgoingEdges.showStatistics(m_parameters->getRank());
		}
	}else{
		if(m_mode_send_vertices_sequence_id_position==0){
			(*m_myReads)[(*m_mode_send_vertices_sequence_id)]->getSeq(m_readSequence,m_parameters->getColorSpaceMode(),false);
		
			//cout<<"DEBUG Read="<<*m_mode_send_vertices_sequence_id<<" color="<<m_parameters->getColorSpaceMode()<<" Seq= "<<m_readSequence<<endl;
		}
		int len=strlen(m_readSequence);

		if(len<wordSize){
			m_hasPreviousVertex=false;
			(*m_mode_send_vertices_sequence_id)++;
			(m_mode_send_vertices_sequence_id_position)=0;
			return;
		}

		char memory[1000];
		int lll=len-wordSize+1;
		
		#ifdef ASSERT
		assert(m_readSequence!=NULL);
		#endif

		int p=(m_mode_send_vertices_sequence_id_position);
		memcpy(memory,m_readSequence+p,wordSize);
		memory[wordSize]='\0';
		if(isValidDNA(memory)){
			Kmer a=wordId(memory);

			int rankToFlush=0;

			rankToFlush=m_parameters->_vertexRank(&a);
			for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
				m_bufferedData.addAt(rankToFlush,a.getU64(i));
			}

			if(m_bufferedData.flush(rankToFlush,KMER_U64_ARRAY_SIZE,RAY_MPI_TAG_VERTICES_DATA,m_outboxAllocator,m_outbox,rank,false)){
				m_pendingMessages++;
			}

			if(m_hasPreviousVertex){
				// outgoing edge
				int outgoingRank=m_parameters->_vertexRank(&m_previousVertex);
				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForOutgoingEdges.addAt(outgoingRank,m_previousVertex.getU64(i));
				}
				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForOutgoingEdges.addAt(outgoingRank,a.getU64(i));
				}

				if(m_bufferedDataForOutgoingEdges.needsFlushing(outgoingRank,2*KMER_U64_ARRAY_SIZE)){
					if(m_bufferedData.flush(outgoingRank,KMER_U64_ARRAY_SIZE,RAY_MPI_TAG_VERTICES_DATA,m_outboxAllocator,m_outbox,rank,true)){
						m_pendingMessages++;
					}
				}

				if(m_bufferedDataForOutgoingEdges.flush(outgoingRank,2*KMER_U64_ARRAY_SIZE,RAY_MPI_TAG_OUT_EDGES_DATA,m_outboxAllocator,m_outbox,rank,false)){
					m_pendingMessages++;
				}

				// ingoing edge
				int ingoingRank=m_parameters->_vertexRank(&a);
				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForIngoingEdges.addAt(ingoingRank,m_previousVertex.getU64(i));
				}
				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForIngoingEdges.addAt(ingoingRank,a.getU64(i));
				}

				if(m_bufferedDataForIngoingEdges.needsFlushing(ingoingRank,2*KMER_U64_ARRAY_SIZE)){
					if(m_bufferedData.flush(ingoingRank,KMER_U64_ARRAY_SIZE,RAY_MPI_TAG_VERTICES_DATA,m_outboxAllocator,m_outbox,rank,true)){
						m_pendingMessages++;
					}
				}

				if(m_bufferedDataForIngoingEdges.flush(ingoingRank,2*KMER_U64_ARRAY_SIZE,RAY_MPI_TAG_IN_EDGES_DATA,m_outboxAllocator,m_outbox,rank,false)){
					m_pendingMessages++;
				}
			}

			// reverse complement
			Kmer b=complementVertex(&a,wordSize,m_parameters->getColorSpaceMode());

			rankToFlush=m_parameters->_vertexRank(&b);
			for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
				m_bufferedData.addAt(rankToFlush,b.getU64(i));
			}

			if(m_bufferedData.flush(rankToFlush,KMER_U64_ARRAY_SIZE,RAY_MPI_TAG_VERTICES_DATA,m_outboxAllocator,m_outbox,rank,false)){
				m_pendingMessages++;
			}

			if(m_hasPreviousVertex){
				// outgoing edge
				int outgoingRank=m_parameters->_vertexRank(&b);
				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForOutgoingEdges.addAt(outgoingRank,b.getU64(i));
				}
				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForOutgoingEdges.addAt(outgoingRank,m_previousVertexRC.getU64(i));
				}

				if(m_bufferedDataForOutgoingEdges.needsFlushing(outgoingRank,2*KMER_U64_ARRAY_SIZE)){
					if(m_bufferedData.flush(outgoingRank,1*KMER_U64_ARRAY_SIZE,RAY_MPI_TAG_VERTICES_DATA,m_outboxAllocator,m_outbox,rank,true)){
						m_pendingMessages++;
					}
				}

				if(m_bufferedDataForOutgoingEdges.flush(outgoingRank,2*KMER_U64_ARRAY_SIZE,RAY_MPI_TAG_OUT_EDGES_DATA,m_outboxAllocator,m_outbox,rank,false)){
					m_pendingMessages++;
				}

				// ingoing edge
				int ingoingRank=m_parameters->_vertexRank(&m_previousVertexRC);
				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForIngoingEdges.addAt(ingoingRank,b.getU64(i));
				}
				for(int i=0;i<KMER_U64_ARRAY_SIZE;i++){
					m_bufferedDataForIngoingEdges.addAt(ingoingRank,m_previousVertexRC.getU64(i));
				}

				if(m_bufferedDataForIngoingEdges.needsFlushing(ingoingRank,2*KMER_U64_ARRAY_SIZE)){
					if(m_bufferedData.flush(ingoingRank,1*KMER_U64_ARRAY_SIZE,RAY_MPI_TAG_VERTICES_DATA,m_outboxAllocator,m_outbox,rank,true)){
						m_pendingMessages++;
					}
				}

				if(m_bufferedDataForIngoingEdges.flush(ingoingRank,2*KMER_U64_ARRAY_SIZE,RAY_MPI_TAG_IN_EDGES_DATA,m_outboxAllocator,m_outbox,rank,false)){
					m_pendingMessages++;
				}
			}

			// there is a previous vertex.
			m_hasPreviousVertex=true;
			m_previousVertex=a;
			m_previousVertexRC=b;
		}else{
			m_hasPreviousVertex=false;
		}

		(m_mode_send_vertices_sequence_id_position++);
		if((m_mode_send_vertices_sequence_id_position)==lll){
			m_hasPreviousVertex=false;
			(*m_mode_send_vertices_sequence_id)++;
			(m_mode_send_vertices_sequence_id_position)=0;
		}
	}
}
Ejemplo n.º 21
0
void DepthFirstSearchData::depthFirstSearchBidirectional(Kmer a,int maxDepth,
        bool*edgesRequested,bool*vertexCoverageRequested,bool*vertexCoverageReceived,
        RingAllocator*outboxAllocator,int size,int theRank,StaticVector*outbox,
        int*receivedVertexCoverage,SeedingData*seedingData,
        int minimumCoverage,bool*edgesReceived,Parameters*parameters) {

#ifdef ASSERT
    int wordSize=parameters->getWordSize();
#endif

    if(!m_doChoice_tips_dfs_initiated) {
        m_outgoingEdges.clear();
        m_ingoingEdges.clear();

        m_depthFirstSearchVisitedVertices.clear();
        m_depthFirstSearchVisitedVertices_vector.clear();
        m_depthFirstSearchVisitedVertices_depths.clear();
        while(m_depthFirstSearchVerticesToVisit.size()>0) {
            m_depthFirstSearchVerticesToVisit.pop();
        }
        while(m_depthFirstSearchDepths.size()>0) {
            m_depthFirstSearchDepths.pop();
        }
        m_maxDepthReached=false;
        m_depthFirstSearchVerticesToVisit.push(a);
        m_depthFirstSearchDepths.push(0);
        m_depthFirstSearch_maxDepth=0;
        m_doChoice_tips_dfs_initiated=true;
        m_doChoice_tips_dfs_done=false;
        m_coverages.clear();
        (*edgesRequested)=false;
        (*vertexCoverageRequested)=false;
    }
    if(m_depthFirstSearchVerticesToVisit.size()>0) {
        Kmer vertexToVisit=m_depthFirstSearchVerticesToVisit.top();

        if(!(*vertexCoverageRequested)) {

            if(m_depthFirstSearchVisitedVertices.count(vertexToVisit)>0) {
                m_depthFirstSearchVerticesToVisit.pop();
                m_depthFirstSearchDepths.pop();
                return;
            }

            (*vertexCoverageRequested)=true;
            (*vertexCoverageReceived)=false;

            MessageUnit*message=(MessageUnit*)(*outboxAllocator).allocate(KMER_U64_ARRAY_SIZE*sizeof(MessageUnit));
            int bufferPosition=0;
            vertexToVisit.pack(message,&bufferPosition);
            int dest=parameters->vertexRank(&vertexToVisit);
            Message aMessage(message,bufferPosition,dest,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE,theRank);
            (*outbox).push_back(&aMessage);
        } else if((*vertexCoverageReceived)) {
            if(!(*edgesRequested)) {
                m_coverages[vertexToVisit]=(*receivedVertexCoverage);

#ifdef ASSERT
                if(m_depthFirstSearchVisitedVertices.count(vertexToVisit)>0) {
                    cout<<"Already visited: "<<vertexToVisit.idToWord(wordSize,parameters->getColorSpaceMode())<<" root is "<<a.idToWord(wordSize,parameters->getColorSpaceMode())<<endl;
                }
                assert(m_depthFirstSearchVisitedVertices.count(vertexToVisit)==0);
                assert(*receivedVertexCoverage>0);
#endif

                if((*receivedVertexCoverage)>0) {
                    m_depthFirstSearchVisitedVertices.insert(vertexToVisit);
                } else {
#ifdef ASSERT
                    assert(false);
#endif
                    // don't visit it.
                    m_depthFirstSearchVerticesToVisit.pop();
                    m_depthFirstSearchDepths.pop();
                    (*edgesRequested)=false;
                    (*vertexCoverageRequested)=false;
                    return;
                }
                int theDepth=m_depthFirstSearchDepths.top();

                if(theDepth> m_depthFirstSearch_maxDepth) {
                    m_depthFirstSearch_maxDepth=theDepth;
                }

                // visit the vertex, and ask next edges.
                MessageUnit*message=(MessageUnit*)(*outboxAllocator).allocate(1*sizeof(MessageUnit));
                int bufferPosition=0;
                vertexToVisit.pack(message,&bufferPosition);
                int destination=parameters->vertexRank(&vertexToVisit);
                Message aMessage(message,bufferPosition,destination,RAY_MPI_TAG_REQUEST_VERTEX_EDGES,theRank);

                (*outbox).push_back(&aMessage);
                (*edgesRequested)=true;
                (*edgesReceived)=false;
            } else if((*edgesReceived)) {
                Kmer vertexToVisit=m_depthFirstSearchVerticesToVisit.top();
                int theDepth=m_depthFirstSearchDepths.top();

#ifdef ASSERT

                assert(theDepth>=0);
                assert(theDepth<=maxDepth);
#endif

                int newDepth=theDepth+1;

                m_depthFirstSearchVerticesToVisit.pop();
                m_depthFirstSearchDepths.pop();

                // the first 4 elements are padding
                // the 5th is the number of outgoing edges
                // following are the outgoing edges
                // following is the number of ingoing edges
                // following are the ingoing edges.

                vector<Kmer > outgoingEdges=seedingData->m_SEEDING_receivedOutgoingEdges;

                for(int i=0; i<(int)outgoingEdges.size(); i++) {
                    Kmer nextVertex=outgoingEdges[i];

                    if(m_depthFirstSearchVisitedVertices.size()>=MAX_VERTICES_TO_VISIT) {
                        continue;
                    }
                    if(m_depthFirstSearchVisitedVertices.count(nextVertex)>0) {
                        continue;
                    }
                    if(newDepth>maxDepth) {
                        m_maxDepthReached=true;
                        continue;
                    }
                    m_depthFirstSearchVerticesToVisit.push(nextVertex);
                    m_depthFirstSearchDepths.push(newDepth);

                    m_depthFirstSearchVisitedVertices_vector.push_back(vertexToVisit);
                    m_depthFirstSearchVisitedVertices_vector.push_back(nextVertex);
                    m_depthFirstSearchVisitedVertices_depths.push_back(newDepth);
                }

#ifdef ASSERT
                if(m_outgoingEdges.count(vertexToVisit)>0) {
                    cout<<vertexToVisit.idToWord(wordSize,parameters->getColorSpaceMode())<<" is already in the data structure "<<m_outgoingEdges[vertexToVisit].size()<<" v. "<<outgoingEdges.size()<<endl;
                }
                assert(m_outgoingEdges.count(vertexToVisit)==0);
#endif

                m_outgoingEdges[vertexToVisit]=outgoingEdges;

                vector<Kmer> ingoingEdges=seedingData->m_SEEDING_receivedIngoingEdges;

                for(int i=0; i<(int)ingoingEdges.size(); i++) {
                    Kmer nextVertex=ingoingEdges[i];

                    if(m_depthFirstSearchVisitedVertices.size()>=MAX_VERTICES_TO_VISIT) {
                        continue;
                    }
                    if(m_depthFirstSearchVisitedVertices.count(nextVertex)>0) {
                        continue;
                    }
                    if(newDepth>maxDepth) {
                        m_maxDepthReached=true;
                        continue;
                    }
                    m_depthFirstSearchVerticesToVisit.push(nextVertex);
                    m_depthFirstSearchDepths.push(newDepth);

                    // reverse the order.
                    m_depthFirstSearchVisitedVertices_vector.push_back(nextVertex);
                    m_depthFirstSearchVisitedVertices_vector.push_back(vertexToVisit);
                    m_depthFirstSearchVisitedVertices_depths.push_back(newDepth);
                }

                (*edgesRequested)=false;
                (*vertexCoverageRequested)=false;

#ifdef ASSERT
                assert(m_ingoingEdges.count(vertexToVisit)==0);

#endif

                m_ingoingEdges[vertexToVisit]=ingoingEdges;
            }
        }
    } else {
        m_doChoice_tips_dfs_done=true;
#ifdef SHOW_MINI_GRAPH
        cout<<"</MiniGraph>"<<endl;
#endif
    }
}
Ejemplo n.º 22
0
void SeedingData::computeSeeds(){
	if(!m_initiatedIterator){
		m_last=time(NULL);

		m_SEEDING_i=0;

		m_activeWorkerIterator=m_activeWorkers.begin();
		m_splayTreeIterator.constructor(m_subgraph,m_wordSize,m_parameters);
		m_initiatedIterator=true;
		m_maximumAliveWorkers=32768;

		#ifdef ASSERT
		m_splayTreeIterator.hasNext();
		#endif
	}

	if(!m_checkedCheckpoint){
		if(m_parameters->hasCheckpoint("Seeds")){
			cout<<"Rank "<<m_parameters->getRank()<<": checkpoint Seeds exists, not computing seeds."<<endl;
			(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
			Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_SEEDING_IS_OVER,getRank());
			m_outbox->push_back(aMessage);

			loadCheckpoint();

			return;
		}
		m_checkedCheckpoint=true;
	}

	m_virtualCommunicator->processInbox(&m_activeWorkersToRestore);

	if(!m_virtualCommunicator->isReady()){
		return;
	}

	// flush all mode is necessary to empty buffers and 
	// restart things from scratch..

	// 1. iterate on active workers
	if(m_activeWorkerIterator!=m_activeWorkers.end()){
		uint64_t workerId=*m_activeWorkerIterator;
		#ifdef ASSERT
		assert(m_aliveWorkers.count(workerId)>0);
		assert(!m_aliveWorkers[workerId].isDone());
		#endif
		m_virtualCommunicator->resetLocalPushedMessageStatus();

		//force the worker to work until he finishes or pushes something on the stack
		while(!m_aliveWorkers[workerId].isDone()&&!m_virtualCommunicator->getLocalPushedMessageStatus()){
			m_aliveWorkers[workerId].work();
		}

		if(m_virtualCommunicator->getLocalPushedMessageStatus()){
			m_waitingWorkers.push_back(workerId);
		}
		if(m_aliveWorkers[workerId].isDone()){
			m_workersDone.push_back(workerId);
			vector<Kmer> seed=*(m_aliveWorkers[workerId].getSeed());

			int nucleotides=seed.size()+(m_wordSize)-1;

			if(seed.size() > 0 && m_parameters->debugSeeds()){
				cout<<"Raw seed length: "<<nucleotides<<" nucleotides"<<endl;
			}

			// only consider the long ones.
			if(nucleotides>=m_parameters->getMinimumContigLength()){
				#ifdef SHOW_DISCOVERIES
				printf("Rank %i discovered a seed with %i vertices\n",m_rank,(int)seed.size());
				#endif
				
				Kmer firstVertex=seed[0];
				Kmer lastVertex=seed[seed.size()-1];
				Kmer firstReverse=m_parameters->_complementVertex(&lastVertex);

				if(firstVertex<firstReverse){
					printf("Rank %i stored a seed with %i vertices\n",m_rank,(int)seed.size());
					fflush(stdout);

					if(m_parameters->showMemoryUsage()){
						showMemoryUsage(m_rank);
					}
					m_SEEDING_seeds.push_back(seed);
				}
			}
		}
		m_activeWorkerIterator++;
	}else{
		updateStates();

		//  add one worker to active workers
		//  reason is that those already in the pool don't communicate anymore -- 
		//  as for they need responses.
		if(!m_virtualCommunicator->getGlobalPushedMessageStatus()&&m_activeWorkers.empty()){
			// there is at least one worker to start
			// AND
			// the number of alive workers is below the maximum
			if(m_SEEDING_i<(uint64_t)m_subgraph->size()&&(int)m_aliveWorkers.size()<m_maximumAliveWorkers){
				if(m_SEEDING_i % 100000 ==0){
					printf("Rank %i is creating seeds [%i/%i]\n",getRank(),(int)m_SEEDING_i+1,(int)m_subgraph->size());
					fflush(stdout);

					if(m_parameters->showMemoryUsage()){
						showMemoryUsage(m_rank);
					}
				}
				#ifdef ASSERT
				if(m_SEEDING_i==0){
					assert(m_completedJobs==0&&m_activeWorkers.size()==0&&m_aliveWorkers.size()==0);
				}
				#endif

				m_splayTreeIterator.next();
				Kmer vertexKey=*(m_splayTreeIterator.getKey());

				m_aliveWorkers[m_SEEDING_i].constructor(&vertexKey,m_parameters,m_outboxAllocator,m_virtualCommunicator,m_SEEDING_i);
				m_activeWorkers.insert(m_SEEDING_i);

				int population=m_aliveWorkers.size();
				if(population>m_maximumWorkers){
					m_maximumWorkers=population;
				}

				m_SEEDING_i++;

				// skip the reverse complement as we don't really need it anyway.
			}else{
				m_virtualCommunicator->forceFlush();
			}
		}

		// brace yourself for the next round
		m_activeWorkerIterator=m_activeWorkers.begin();
	}

	#ifdef ASSERT
	assert((int)m_aliveWorkers.size()<=m_maximumAliveWorkers);
	#endif

	if((int)m_subgraph->size()==m_completedJobs){
		printf("Rank %i has %i seeds\n",m_rank,(int)m_SEEDING_seeds.size());
		fflush(stdout);
		printf("Rank %i is creating seeds [%i/%i] (completed)\n",getRank(),(int)m_SEEDING_i,(int)m_subgraph->size());
		fflush(stdout);
		printf("Rank %i: peak number of workers: %i, maximum: %i\n",m_rank,m_maximumWorkers,m_maximumAliveWorkers);
		fflush(stdout);
		m_virtualCommunicator->printStatistics();

		(*m_mode)=RAY_SLAVE_MODE_DO_NOTHING;
		Message aMessage(NULL,0,MASTER_RANK,RAY_MPI_TAG_SEEDING_IS_OVER,getRank());
		m_outbox->push_back(aMessage);

		if(m_parameters->showMemoryUsage()){
			showMemoryUsage(m_rank);
		}

		#ifdef ASSERT
		assert(m_aliveWorkers.size()==0);
		assert(m_activeWorkers.size()==0);
		#endif

		// sort the seeds by length
		std::sort(m_SEEDING_seeds.begin(),m_SEEDING_seeds.end(),myComparator_sort);

		/** write seeds for debugging purposes */
		if(m_parameters->hasOption("-write-seeds")){
			ostringstream fileName;
			fileName<<m_parameters->getPrefix()<<"Rank"<<m_parameters->getRank()<<".RaySeeds.fasta";
			ofstream f(fileName.str().c_str());
			for(int i=0;i<(int)m_SEEDING_seeds.size();i++){
				uint64_t id=getPathUniqueId(m_parameters->getRank(),i);
				f<<">RaySeed-"<<id<<endl;

				f<<addLineBreaks(convertToString(&(m_SEEDING_seeds[i]),
					m_parameters->getWordSize(),m_parameters->getColorSpaceMode()),
					m_parameters->getColumns());
			}
			f.close();
		}

	}
}
Ejemplo n.º 23
0
/*
 * do a depth first search with max depth of maxDepth;
 */
void DepthFirstSearchData::depthFirstSearch(Kmer root,Kmer a,int maxDepth,
        bool*edgesRequested,bool*vertexCoverageRequested,bool*vertexCoverageReceived,
        RingAllocator*outboxAllocator,int size,int theRank,StaticVector*outbox,
        int*receivedVertexCoverage,vector<Kmer>*receivedOutgoingEdges,
        int minimumCoverage,bool*edgesReceived,int wordSize,Parameters*parameters) {
    if(!m_doChoice_tips_dfs_initiated) {
        m_depthFirstSearchVisitedVertices.clear();
        m_depthFirstSearchVisitedVertices_vector.clear();

        // add an arc
        m_depthFirstSearchVisitedVertices_vector.push_back(root);
        m_depthFirstSearchVisitedVertices_vector.push_back(a);

        m_depthFirstSearchVisitedVertices_depths.clear();
        while(m_depthFirstSearchVerticesToVisit.size()>0) {
            m_depthFirstSearchVerticesToVisit.pop();
        }
        while(m_depthFirstSearchDepths.size()>0) {
            m_depthFirstSearchDepths.pop();
        }
        m_maxDepthReached=false;
        m_depthFirstSearchVerticesToVisit.push(a);
        m_depthFirstSearchVisitedVertices.insert(a);
        m_depthFirstSearchDepths.push(0);
        m_depthFirstSearch_maxDepth=0;
        m_doChoice_tips_dfs_initiated=true;
        m_doChoice_tips_dfs_done=false;
        m_coverages.clear();
        (*edgesRequested)=false;
        (*vertexCoverageRequested)=false;
#ifdef SHOW_MINI_GRAPH
        cout<<"<MiniGraph>"<<endl;
        cout<<root->idToWord(wordSize,parameters->getColorSpaceMode())<<" -> "<<a->idToWord(wordSize,parameters->getColorSpaceMode())<<endl;
#endif
    }
    if(m_depthFirstSearchVerticesToVisit.size()>0) {
        Kmer vertexToVisit=m_depthFirstSearchVerticesToVisit.top();
        if(!(*vertexCoverageRequested)) {
            (*vertexCoverageRequested)=true;
            (*vertexCoverageReceived)=false;

            MessageUnit*message=(MessageUnit*)(*outboxAllocator).allocate(KMER_U64_ARRAY_SIZE*sizeof(MessageUnit));
            int j=0;
            vertexToVisit.pack(message,&j);
            int dest=parameters->vertexRank(&vertexToVisit);

            Message aMessage(message,j,dest,RAY_MPI_TAG_REQUEST_VERTEX_COVERAGE,theRank);
            (*outbox).push_back(&aMessage);
        } else if((*vertexCoverageReceived)) {
            if(!(*edgesRequested)) {
                m_coverages[vertexToVisit]=(*receivedVertexCoverage);
                m_depthFirstSearchVisitedVertices.insert(vertexToVisit);
                int theDepth=m_depthFirstSearchDepths.top();

                if(theDepth> m_depthFirstSearch_maxDepth) {
                    m_depthFirstSearch_maxDepth=theDepth;
                }

                // visit the vertex, and ask next edges.
                MessageUnit*message=(MessageUnit*)(*outboxAllocator).allocate(1*sizeof(MessageUnit));
                int bufferPosition=0;
                vertexToVisit.pack(message,&bufferPosition);
                int destination=parameters->vertexRank(&vertexToVisit);
                Message aMessage(message,bufferPosition,destination,RAY_MPI_TAG_REQUEST_VERTEX_OUTGOING_EDGES,theRank);
                (*outbox).push_back(&aMessage);
                (*edgesRequested)=true;
                (*edgesReceived)=false;
            } else if((*edgesReceived)) {
                Kmer vertexToVisit=m_depthFirstSearchVerticesToVisit.top();
                int theDepth=m_depthFirstSearchDepths.top();
#ifdef ASSERT
                assert(theDepth>=0);
                assert(theDepth<=maxDepth);
#endif
                int newDepth=theDepth+1;

                m_depthFirstSearchVerticesToVisit.pop();
                m_depthFirstSearchDepths.pop();

                for(int i=0; i<(int)(*receivedOutgoingEdges).size(); i++) {
                    Kmer nextVertex=(*receivedOutgoingEdges)[i];
                    if(m_depthFirstSearchVisitedVertices.count(nextVertex)>0) {
                        continue;
                    }
                    if(newDepth>maxDepth) {
                        m_maxDepthReached=true;
                        continue;
                    }

                    if(m_depthFirstSearchVisitedVertices.size()<MAX_VERTICES_TO_VISIT) {
                        // add an arc
                        m_depthFirstSearchVisitedVertices_vector.push_back(vertexToVisit);
                        m_depthFirstSearchVisitedVertices_vector.push_back(nextVertex);

                        // add the depth for the vertex
                        m_depthFirstSearchVisitedVertices_depths.push_back(newDepth);

                        // stacks
                        m_depthFirstSearchVerticesToVisit.push(nextVertex);
                        m_depthFirstSearchDepths.push(newDepth);
                    }


#ifdef SHOW_MINI_GRAPH
                    cout<<vertexToVisit->idToWord(wordSize,parameters->getColorSpaceMode())<<" -> "<<nextVertex->idToWord(wordSize,parameters->getColorSpaceMode())<<endl;
#endif
                }
                (*edgesRequested)=false;
                (*vertexCoverageRequested)=false;
            }
        }
    } else {
        m_doChoice_tips_dfs_done=true;
#ifdef SHOW_MINI_GRAPH
        cout<<"</MiniGraph>"<<endl;
#endif
    }
}