void ForEachBaselineAction::Perform(ArtifactSet &artifacts, ProgressListener &progress) { if(!artifacts.HasImageSet()) { progress.OnStartTask(*this, 0, 1, "For each baseline (no image set)"); progress.OnEndTask(*this); Logger::Warn << "I executed a ForEachBaselineAction without an active imageset: something is\n" "likely wrong. Check your strategy and the input files.\n"; } else if(_selection == Current) { ActionBlock::Perform(artifacts, progress); } else { ImageSet& imageSet = artifacts.ImageSet(); MSImageSet* msImageSet = dynamic_cast<MSImageSet*>(&imageSet); if(msImageSet != 0) { // Check memory usage std::unique_ptr<ImageSetIndex> tempIndex = msImageSet->StartIndex(); size_t timeStepCount = msImageSet->ObservationTimesVector(*tempIndex).size(); tempIndex.reset(); size_t channelCount = msImageSet->GetBandInfo(0).channels.size(); double estMemorySizePerThread = 8.0/*bp complex*/ * 4.0 /*polarizations*/ * double(timeStepCount) * double(channelCount) * 3.0 /* approx copies of the data that will be made in memory*/; Logger::Debug << "Estimate of memory each thread will use: " << memToStr(estMemorySizePerThread) << ".\n"; size_t compThreadCount = _threadCount; if(compThreadCount > 0) --compThreadCount; int64_t memSize = System::TotalMemory(); Logger::Debug << "Detected " << memToStr(memSize) << " of system memory.\n"; if(estMemorySizePerThread * double(compThreadCount) > memSize) { size_t maxThreads = size_t(memSize / estMemorySizePerThread); if(maxThreads < 1) maxThreads = 1; Logger::Warn << "This measurement set is TOO LARGE to be processed with " << _threadCount << " threads!\n" << _threadCount << " threads would require " << memToStr(estMemorySizePerThread*compThreadCount) << " of memory approximately.\n" "Number of threads that will actually be used: " << maxThreads << "\n" "This might hurt performance a lot!\n\n"; _threadCount = maxThreads; } } if(dynamic_cast<FilterBankSet*>(&imageSet) != nullptr && _threadCount != 1) { Logger::Info << "This is a Filterbank set -- disabling multi-threading\n"; _threadCount = 1; } if(!_antennaeToSkip.empty()) { Logger::Debug << "The following antennas will be skipped: "; for(std::set<size_t>::const_iterator i=_antennaeToSkip.begin();i!=_antennaeToSkip.end(); ++i) Logger::Debug << (*i) << ' '; Logger::Debug <<'\n'; } if(!_antennaeToInclude.empty()) { Logger::Debug << "Only the following antennas will be included: "; for(std::set<size_t>::const_iterator i=_antennaeToInclude.begin();i!=_antennaeToInclude.end(); ++i) Logger::Debug << (*i) << ' '; Logger::Debug <<'\n'; } if(artifacts.MetaData() != 0) { _hasInitAntennae = true; if(artifacts.MetaData()->HasAntenna1()) _initAntenna1 = artifacts.MetaData()->Antenna1(); else _hasInitAntennae = false; if(artifacts.MetaData()->HasAntenna2()) _initAntenna2 = artifacts.MetaData()->Antenna2(); else _hasInitAntennae = false; } _artifacts = &artifacts; _initPartIndex = 0; _finishedBaselines = false; _baselineCount = 0; _baselineProgress = 0; _nextIndex = 0; // Count the baselines that are to be processed std::unique_ptr<ImageSetIndex> iteratorIndex = imageSet.StartIndex(); while(iteratorIndex->IsValid()) { if(IsBaselineSelected(*iteratorIndex)) ++_baselineCount; iteratorIndex->Next(); } iteratorIndex.reset(); Logger::Debug << "Will process " << _baselineCount << " baselines.\n"; // Initialize thread data and threads _loopIndex = imageSet.StartIndex(); _progressTaskNo = new int[_threadCount]; _progressTaskCount = new int[_threadCount]; progress.OnStartTask(*this, 0, 1, "Initializing"); std::vector<std::thread> threadGroup; ReaderFunction reader(*this); threadGroup.emplace_back(reader); size_t mathThreads = mathThreadCount(); for(unsigned i=0;i<mathThreads;++i) { PerformFunction function(*this, progress, i); threadGroup.emplace_back(function); } for(std::thread& t : threadGroup) t.join(); progress.OnEndTask(*this); if(_resultSet != 0) { artifacts = *_resultSet; delete _resultSet; } delete[] _progressTaskCount; delete[] _progressTaskNo; _loopIndex.reset(); if(_exceptionOccured) throw std::runtime_error("An exception occured in one of the sub tasks of the (multi-threaded) \"For-each baseline\"-action: the RFI strategy will not continue."); } }
void ForEachBaselineAction::Perform(ArtifactSet &artifacts, ProgressListener &progress) { if(!artifacts.HasImageSet()) { progress.OnStartTask(*this, 0, 1, "For each baseline (no image set)"); progress.OnEndTask(*this); AOLogger::Warn << "I executed a ForEachBaselineAction without an active imageset: something is\n" "likely wrong. Check your strategy and the input files.\n"; } else if(_selection == Current) { ActionBlock::Perform(artifacts, progress); } else { ImageSet *imageSet = artifacts.ImageSet(); MSImageSet *msImageSet = dynamic_cast<MSImageSet*>(imageSet); if(msImageSet != 0) { // Check memory usage ImageSetIndex *tempIndex = msImageSet->StartIndex(); size_t timeStepCount = msImageSet->ObservationTimesVector(*tempIndex).size(); delete tempIndex; size_t channelCount = msImageSet->GetBandInfo(0).channels.size(); size_t estMemorySizePerThread = 8/*bp complex*/ * 4 /*polarizations*/ * timeStepCount * channelCount * 3 /* approx copies of the data that will be made in memory*/; AOLogger::Debug << "Estimate of memory each thread will use: " << estMemorySizePerThread/(1024*1024) << " MB.\n"; size_t compThreadCount = _threadCount; if(compThreadCount > 0) --compThreadCount; if(estMemorySizePerThread * compThreadCount > 12ul*1024ul*1024ul*1024ul) { size_t maxThreads = (12ul * 1024ul * 1024ul * 1024ul) / estMemorySizePerThread; if(maxThreads < 1) maxThreads = 1; AOLogger::Warn << "WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING!\n" "This measurement set is TOO LARGE to be processed with " << _threadCount << " threads!\n" << _threadCount << " threads would require " << ((estMemorySizePerThread*compThreadCount)/(1024*1024)) << " MB of memory approximately.\n" "Number of threads that will actually be used: " << maxThreads << "\n" "This might hurt performance a lot!\n\n"; _threadCount = maxThreads; } } if(!_antennaeToSkip.empty()) { AOLogger::Debug << "The following antenna's will be skipped: "; for(std::set<size_t>::const_iterator i=_antennaeToSkip.begin();i!=_antennaeToSkip.end(); ++i) AOLogger::Debug << (*i) << ' '; AOLogger::Debug <<'\n'; } if(!_antennaeToInclude.empty()) { AOLogger::Debug << "Only the following antenna's will be included: "; for(std::set<size_t>::const_iterator i=_antennaeToInclude.begin();i!=_antennaeToInclude.end(); ++i) AOLogger::Debug << (*i) << ' '; AOLogger::Debug <<'\n'; } if(artifacts.MetaData() != 0) { _hasInitAntennae = true; if(artifacts.MetaData()->HasAntenna1()) _initAntenna1 = artifacts.MetaData()->Antenna1(); else _hasInitAntennae = false; if(artifacts.MetaData()->HasAntenna2()) _initAntenna2 = artifacts.MetaData()->Antenna2(); else _hasInitAntennae = false; } _artifacts = &artifacts; _initPartIndex = 0; _finishedBaselines = false; _baselineCount = 0; _baselineProgress = 0; _nextIndex = 0; // Count the baselines that are to be processed ImageSetIndex *iteratorIndex = imageSet->StartIndex(); while(iteratorIndex->IsValid()) { if(IsBaselineSelected(*iteratorIndex)) ++_baselineCount; iteratorIndex->Next(); } delete iteratorIndex; AOLogger::Debug << "Will process " << _baselineCount << " baselines.\n"; // Initialize thread data and threads _loopIndex = imageSet->StartIndex(); _progressTaskNo = new int[_threadCount]; _progressTaskCount = new int[_threadCount]; progress.OnStartTask(*this, 0, 1, "Initializing"); boost::thread_group threadGroup; ReaderFunction reader(*this); threadGroup.create_thread(reader); size_t mathThreads = mathThreadCount(); for(unsigned i=0;i<mathThreads;++i) { PerformFunction function(*this, progress, i); threadGroup.create_thread(function); } threadGroup.join_all(); progress.OnEndTask(*this); if(_resultSet != 0) { artifacts = *_resultSet; delete _resultSet; } delete[] _progressTaskCount; delete[] _progressTaskNo; delete _loopIndex; if(_exceptionOccured) throw std::runtime_error("An exception occured in one of the sub tasks of the (multi-threaded) \"For-each baseline\"-action: the RFI strategy will not continue."); } }
void SpatialCompositionAction::Perform(ArtifactSet &artifacts, ProgressListener &progress) { size_t imageCount = artifacts.ContaminatedData().ImageCount(); std::vector<Image2DPtr> images(imageCount); for(size_t p=0;p<imageCount;++p) images[p] = Image2D::CreateZeroImagePtr(artifacts.ContaminatedData().ImageWidth(), artifacts.ContaminatedData().ImageHeight()); std::string filename = artifacts.ImageSet()->File(); SpatialMSImageSet set(filename); ImageSetIndex *index = set.StartIndex(); size_t progressStep = 0, totalProgress = artifacts.ContaminatedData().ImageWidth() * artifacts.ContaminatedData().ImageHeight()/256; while(index->IsValid()) { TimeFrequencyData *data = set.LoadData(*index); SpatialMatrixMetaData metaData(set.SpatialMetaData(*index)); for(size_t p=0;p!=imageCount;++p) { switch(_operation) { case SumCrossCorrelationsOperation: images[p]->SetValue(metaData.TimeIndex(), metaData.ChannelIndex(), sumCrossCorrelations(data->GetImage(p))); break; case SumAutoCorrelationsOperation: images[p]->SetValue(metaData.TimeIndex(), metaData.ChannelIndex(), sumAutoCorrelations(data->GetImage(p))); break; case EigenvalueDecompositionOperation: { num_t value = eigenvalue(data->GetImage(p), data->GetImage(p+1)); images[p]->SetValue(metaData.TimeIndex(), metaData.ChannelIndex(), value); images[p+1]->SetValue(metaData.TimeIndex(), metaData.ChannelIndex(), 0.0); ++p; } break; case EigenvalueRemovalOperation: { std::pair<num_t, num_t> value = removeEigenvalue(data->GetImage(p), data->GetImage(p+1)); images[p]->SetValue(metaData.TimeIndex(), metaData.ChannelIndex(), value.first); images[p+1]->SetValue(metaData.TimeIndex(), metaData.ChannelIndex(), value.second); ++p; } break; } } delete data; index->Next(); ++progressStep; progress.OnProgress(*this, progressStep/256, totalProgress); } delete index; TimeFrequencyData newRevisedData = artifacts.RevisedData(); for(size_t p=0;p<imageCount;++p) newRevisedData.SetImage(p, images[p]); newRevisedData.SetMask(artifacts.RevisedData()); TimeFrequencyData *contaminatedData = TimeFrequencyData::CreateTFDataFromDiff(artifacts.ContaminatedData(), newRevisedData); contaminatedData->SetMask(artifacts.ContaminatedData()); artifacts.SetRevisedData(newRevisedData); artifacts.SetContaminatedData(*contaminatedData); delete contaminatedData; }