示例#1
0
	class ImageSetIndex *ForEachBaselineAction::GetNextIndex()
	{
		boost::mutex::scoped_lock lock(_mutex);
		while(_loopIndex->IsValid())
		{
			if(IsBaselineSelected(*_loopIndex))
			{
				ImageSetIndex *newIndex = _loopIndex->Copy();
				_loopIndex->Next();

				return newIndex;
			}
			_loopIndex->Next();
		}
		return 0;
	}
	std::unique_ptr<ImageSetIndex> ForEachBaselineAction::GetNextIndex()
	{
		std::lock_guard<std::mutex> lock(_mutex);
		while(_loopIndex->IsValid())
		{
			if(IsBaselineSelected(*_loopIndex))
			{
				std::unique_ptr<ImageSetIndex> newIndex = _loopIndex->Clone();
				_loopIndex->Next();

				return newIndex;
			}
			_loopIndex->Next();
		}
		return nullptr;
	}
示例#3
0
	void ForEachBaselineAction::Perform(ArtifactSet &artifacts, ProgressListener &progress)
	{
		if(!artifacts.HasImageSet())
		{
			progress.OnStartTask(*this, 0, 1, "For each baseline (no image set)");
			progress.OnEndTask(*this);
			AOLogger::Warn <<
				"I executed a ForEachBaselineAction without an active imageset: something is\n"
				"likely wrong. Check your strategy and the input files.\n";
		} else if(_selection == Current)
		{
			ActionBlock::Perform(artifacts, progress);
		} else
		{
			ImageSet *imageSet = artifacts.ImageSet();
			MSImageSet *msImageSet = dynamic_cast<MSImageSet*>(imageSet);
			if(msImageSet != 0)
			{
				// Check memory usage
				ImageSetIndex *tempIndex = msImageSet->StartIndex();
				size_t timeStepCount = msImageSet->ObservationTimesVector(*tempIndex).size();
				delete tempIndex;
				size_t channelCount = msImageSet->GetBandInfo(0).channels.size();
				size_t estMemorySizePerThread = 8/*bp complex*/ * 4 /*polarizations*/ * timeStepCount * channelCount * 3 /* approx copies of the data that will be made in memory*/;
				AOLogger::Debug << "Estimate of memory each thread will use: " << estMemorySizePerThread/(1024*1024) << " MB.\n";
				size_t compThreadCount = _threadCount;
				if(compThreadCount > 0) --compThreadCount;
				if(estMemorySizePerThread * compThreadCount > 12ul*1024ul*1024ul*1024ul)
				{
					size_t maxThreads = (12ul * 1024ul * 1024ul * 1024ul) / estMemorySizePerThread;
					if(maxThreads < 1) maxThreads = 1;
					AOLogger::Warn <<
						"WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING!\n"
						"This measurement set is TOO LARGE to be processed with " << _threadCount << " threads!\n" <<
						_threadCount << " threads would require " << ((estMemorySizePerThread*compThreadCount)/(1024*1024)) << " MB of memory approximately.\n"
						"Number of threads that will actually be used: " << maxThreads << "\n"
						"This might hurt performance a lot!\n\n";
					_threadCount = maxThreads;
				}
			}
			if(!_antennaeToSkip.empty())
			{
				AOLogger::Debug << "The following antenna's will be skipped: ";
				for(std::set<size_t>::const_iterator i=_antennaeToSkip.begin();i!=_antennaeToSkip.end(); ++i)
					AOLogger::Debug << (*i) << ' ';
				AOLogger::Debug <<'\n';
			}
			if(!_antennaeToInclude.empty())
			{
				AOLogger::Debug << "Only the following antenna's will be included: ";
				for(std::set<size_t>::const_iterator i=_antennaeToInclude.begin();i!=_antennaeToInclude.end(); ++i)
					AOLogger::Debug << (*i) << ' ';
				AOLogger::Debug <<'\n';
			}

			if(artifacts.MetaData() != 0)
			{
				_hasInitAntennae = true;
				if(artifacts.MetaData()->HasAntenna1())
					_initAntenna1 = artifacts.MetaData()->Antenna1();
				else
					_hasInitAntennae = false;
				if(artifacts.MetaData()->HasAntenna2())
					_initAntenna2 = artifacts.MetaData()->Antenna2();
				else
					_hasInitAntennae = false;
			}
			_artifacts = &artifacts;
			
			_initPartIndex = 0;
			_finishedBaselines = false;
			_baselineCount = 0;
			_baselineProgress = 0;
			_nextIndex = 0;
			
			// Count the baselines that are to be processed
			ImageSetIndex *iteratorIndex = imageSet->StartIndex();
			while(iteratorIndex->IsValid())
			{
				if(IsBaselineSelected(*iteratorIndex))
					++_baselineCount;
				iteratorIndex->Next();
			}
			delete iteratorIndex;
			AOLogger::Debug << "Will process " << _baselineCount << " baselines.\n";
			
			// Initialize thread data and threads
			_loopIndex = imageSet->StartIndex();
			_progressTaskNo = new int[_threadCount];
			_progressTaskCount = new int[_threadCount];
			progress.OnStartTask(*this, 0, 1, "Initializing");

			boost::thread_group threadGroup;
			ReaderFunction reader(*this);
			threadGroup.create_thread(reader);
			
			size_t mathThreads = mathThreadCount();
			for(unsigned i=0;i<mathThreads;++i)
			{
				PerformFunction function(*this, progress, i);
				threadGroup.create_thread(function);
			}
			
			threadGroup.join_all();
			progress.OnEndTask(*this);

			if(_resultSet != 0)
			{
				artifacts = *_resultSet;
				delete _resultSet;
			}

			delete[] _progressTaskCount;
			delete[] _progressTaskNo;

			delete _loopIndex;

			if(_exceptionOccured)
				throw std::runtime_error("An exception occured in one of the sub tasks of the (multi-threaded) \"For-each baseline\"-action: the RFI strategy will not continue.");
		}
	}
	void ForEachBaselineAction::Perform(ArtifactSet &artifacts, ProgressListener &progress)
	{
		if(!artifacts.HasImageSet())
		{
			progress.OnStartTask(*this, 0, 1, "For each baseline (no image set)");
			progress.OnEndTask(*this);
			Logger::Warn <<
				"I executed a ForEachBaselineAction without an active imageset: something is\n"
				"likely wrong. Check your strategy and the input files.\n";
		} else if(_selection == Current)
		{
			ActionBlock::Perform(artifacts, progress);
		} else
		{
			ImageSet& imageSet = artifacts.ImageSet();
			MSImageSet* msImageSet = dynamic_cast<MSImageSet*>(&imageSet);
			if(msImageSet != 0)
			{
				// Check memory usage
				std::unique_ptr<ImageSetIndex> tempIndex = msImageSet->StartIndex();
				size_t timeStepCount = msImageSet->ObservationTimesVector(*tempIndex).size();
				tempIndex.reset();
				size_t channelCount = msImageSet->GetBandInfo(0).channels.size();
				double estMemorySizePerThread =
					8.0/*bp complex*/ * 4.0 /*polarizations*/ *
					double(timeStepCount) * double(channelCount) *
					3.0 /* approx copies of the data that will be made in memory*/;
				Logger::Debug << "Estimate of memory each thread will use: " << memToStr(estMemorySizePerThread) << ".\n";
				size_t compThreadCount = _threadCount;
				if(compThreadCount > 0) --compThreadCount;
				
				int64_t memSize = System::TotalMemory();
				Logger::Debug << "Detected " << memToStr(memSize) << " of system memory.\n";
				
				if(estMemorySizePerThread * double(compThreadCount) > memSize)
				{
					size_t maxThreads = size_t(memSize / estMemorySizePerThread);
					if(maxThreads < 1) maxThreads = 1;
					Logger::Warn <<
						"This measurement set is TOO LARGE to be processed with " << _threadCount << " threads!\n" <<
						_threadCount << " threads would require " << memToStr(estMemorySizePerThread*compThreadCount) << " of memory approximately.\n"
						"Number of threads that will actually be used: " << maxThreads << "\n"
						"This might hurt performance a lot!\n\n";
					_threadCount = maxThreads;
				}
			}
			if(dynamic_cast<FilterBankSet*>(&imageSet) != nullptr && _threadCount != 1)
			{
				Logger::Info << "This is a Filterbank set -- disabling multi-threading\n";
				_threadCount = 1;
			}
			if(!_antennaeToSkip.empty())
			{
				Logger::Debug << "The following antennas will be skipped: ";
				for(std::set<size_t>::const_iterator i=_antennaeToSkip.begin();i!=_antennaeToSkip.end(); ++i)
					Logger::Debug << (*i) << ' ';
				Logger::Debug <<'\n';
			}
			if(!_antennaeToInclude.empty())
			{
				Logger::Debug << "Only the following antennas will be included: ";
				for(std::set<size_t>::const_iterator i=_antennaeToInclude.begin();i!=_antennaeToInclude.end(); ++i)
					Logger::Debug << (*i) << ' ';
				Logger::Debug <<'\n';
			}

			if(artifacts.MetaData() != 0)
			{
				_hasInitAntennae = true;
				if(artifacts.MetaData()->HasAntenna1())
					_initAntenna1 = artifacts.MetaData()->Antenna1();
				else
					_hasInitAntennae = false;
				if(artifacts.MetaData()->HasAntenna2())
					_initAntenna2 = artifacts.MetaData()->Antenna2();
				else
					_hasInitAntennae = false;
			}
			_artifacts = &artifacts;
			
			_initPartIndex = 0;
			_finishedBaselines = false;
			_baselineCount = 0;
			_baselineProgress = 0;
			_nextIndex = 0;
			
			// Count the baselines that are to be processed
			std::unique_ptr<ImageSetIndex> iteratorIndex = imageSet.StartIndex();
			while(iteratorIndex->IsValid())
			{
				if(IsBaselineSelected(*iteratorIndex))
					++_baselineCount;
				iteratorIndex->Next();
			}
			iteratorIndex.reset();
			Logger::Debug << "Will process " << _baselineCount << " baselines.\n";
			
			// Initialize thread data and threads
			_loopIndex = imageSet.StartIndex();
			_progressTaskNo = new int[_threadCount];
			_progressTaskCount = new int[_threadCount];
			progress.OnStartTask(*this, 0, 1, "Initializing");

			std::vector<std::thread> threadGroup;
			ReaderFunction reader(*this);
			threadGroup.emplace_back(reader);
			
			size_t mathThreads = mathThreadCount();
			for(unsigned i=0;i<mathThreads;++i)
			{
				PerformFunction function(*this, progress, i);
				threadGroup.emplace_back(function);
			}
			for(std::thread& t : threadGroup)
				t.join();
			progress.OnEndTask(*this);

			if(_resultSet != 0)
			{
				artifacts = *_resultSet;
				delete _resultSet;
			}

			delete[] _progressTaskCount;
			delete[] _progressTaskNo;

			_loopIndex.reset();

			if(_exceptionOccured)
				throw std::runtime_error("An exception occured in one of the sub tasks of the (multi-threaded) \"For-each baseline\"-action: the RFI strategy will not continue.");
		}
	}