void ImagerAction::Perform(ArtifactSet &artifacts, ProgressListener &progress) { boost::mutex::scoped_lock lock(_imagerMutex); UVImager *imager = artifacts.Imager(); if(imager == 0) throw BadUsageException("No imager available to create image."); TimeFrequencyData &data = artifacts.ContaminatedData(); TimeFrequencyMetaDataCPtr metaData = artifacts.MetaData(); if(data.PolarisationCount() > 1) { TimeFrequencyData *tmp = data.CreateTFData(StokesIPolarisation); data = *tmp; delete tmp; } bool btPlaneImager = true; if(btPlaneImager) { typedef double ImagerNumeric; BaselineTimePlaneImager<ImagerNumeric> btImager; BandInfo band = metaData->Band(); Image2DCPtr inputReal = data.GetRealPart(), inputImag = data.GetImaginaryPart(); Mask2DCPtr mask = data.GetSingleMask(); size_t width = inputReal->Width(); for(size_t t=0;t!=width;++t) { UVW uvw = metaData->UVW()[t]; size_t channelCount = inputReal->Height(); std::vector<std::complex<ImagerNumeric> > data(channelCount); for(size_t ch=0;ch!=channelCount;++ch) { if(mask->Value(t, ch)) data[ch] = std::complex<ImagerNumeric>(0.0, 0.0); else data[ch] = std::complex<ImagerNumeric>(inputReal->Value(t, ch), inputImag->Value(t, ch)); } btImager.Image(uvw.u, uvw.v, uvw.w, band.channels[0].frequencyHz, band.channels[1].frequencyHz-band.channels[0].frequencyHz, channelCount, &(data[0]), imager->FTReal()); } } else { progress.OnStartTask(*this, 0, 1, "Imaging baseline"); for(size_t y=0;y<data.ImageHeight();++y) { imager->Image(data, metaData, y); progress.OnProgress(*this, y, data.ImageHeight()); } progress.OnEndTask(*this); } }
void ActionBlock::Perform(ArtifactSet &artifacts, ProgressListener &listener) { size_t nr = 0; unsigned totalWeight = Weight(); for(const_iterator i=begin();i!=end();++i) { Action *action = *i; unsigned weight = action->Weight(); listener.OnStartTask(*this, nr, totalWeight, action->Description(), weight); action->Perform(artifacts, listener); listener.OnEndTask(*this); nr += weight; } }
void ForEachBaselineAction::SetProgress(ProgressListener &progress, int no, int count, std::string taskName, int threadId) { boost::mutex::scoped_lock lock(_mutex); _progressTaskNo[threadId] = no; _progressTaskCount[threadId] = count; size_t totalCount = 0, totalNo = 0; for(size_t i=0;i<_threadCount;++i) { totalCount += _progressTaskCount[threadId]; totalNo += _progressTaskNo[threadId]; } progress.OnEndTask(*this); std::stringstream str; str << "T" << threadId << ": " << taskName; progress.OnStartTask(*this, totalNo, totalCount, str.str()); }
void ForEachBaselineAction::Perform(ArtifactSet &artifacts, ProgressListener &progress) { if(!artifacts.HasImageSet()) { progress.OnStartTask(*this, 0, 1, "For each baseline (no image set)"); progress.OnEndTask(*this); AOLogger::Warn << "I executed a ForEachBaselineAction without an active imageset: something is\n" "likely wrong. Check your strategy and the input files.\n"; } else if(_selection == Current) { ActionBlock::Perform(artifacts, progress); } else { ImageSet *imageSet = artifacts.ImageSet(); MSImageSet *msImageSet = dynamic_cast<MSImageSet*>(imageSet); if(msImageSet != 0) { // Check memory usage ImageSetIndex *tempIndex = msImageSet->StartIndex(); size_t timeStepCount = msImageSet->ObservationTimesVector(*tempIndex).size(); delete tempIndex; size_t channelCount = msImageSet->GetBandInfo(0).channels.size(); size_t estMemorySizePerThread = 8/*bp complex*/ * 4 /*polarizations*/ * timeStepCount * channelCount * 3 /* approx copies of the data that will be made in memory*/; AOLogger::Debug << "Estimate of memory each thread will use: " << estMemorySizePerThread/(1024*1024) << " MB.\n"; size_t compThreadCount = _threadCount; if(compThreadCount > 0) --compThreadCount; if(estMemorySizePerThread * compThreadCount > 12ul*1024ul*1024ul*1024ul) { size_t maxThreads = (12ul * 1024ul * 1024ul * 1024ul) / estMemorySizePerThread; if(maxThreads < 1) maxThreads = 1; AOLogger::Warn << "WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING!\n" "This measurement set is TOO LARGE to be processed with " << _threadCount << " threads!\n" << _threadCount << " threads would require " << ((estMemorySizePerThread*compThreadCount)/(1024*1024)) << " MB of memory approximately.\n" "Number of threads that will actually be used: " << maxThreads << "\n" "This might hurt performance a lot!\n\n"; _threadCount = maxThreads; } } if(!_antennaeToSkip.empty()) { AOLogger::Debug << "The following antenna's will be skipped: "; for(std::set<size_t>::const_iterator i=_antennaeToSkip.begin();i!=_antennaeToSkip.end(); ++i) AOLogger::Debug << (*i) << ' '; AOLogger::Debug <<'\n'; } if(!_antennaeToInclude.empty()) { AOLogger::Debug << "Only the following antenna's will be included: "; for(std::set<size_t>::const_iterator i=_antennaeToInclude.begin();i!=_antennaeToInclude.end(); ++i) AOLogger::Debug << (*i) << ' '; AOLogger::Debug <<'\n'; } if(artifacts.MetaData() != 0) { _hasInitAntennae = true; if(artifacts.MetaData()->HasAntenna1()) _initAntenna1 = artifacts.MetaData()->Antenna1(); else _hasInitAntennae = false; if(artifacts.MetaData()->HasAntenna2()) _initAntenna2 = artifacts.MetaData()->Antenna2(); else _hasInitAntennae = false; } _artifacts = &artifacts; _initPartIndex = 0; _finishedBaselines = false; _baselineCount = 0; _baselineProgress = 0; _nextIndex = 0; // Count the baselines that are to be processed ImageSetIndex *iteratorIndex = imageSet->StartIndex(); while(iteratorIndex->IsValid()) { if(IsBaselineSelected(*iteratorIndex)) ++_baselineCount; iteratorIndex->Next(); } delete iteratorIndex; AOLogger::Debug << "Will process " << _baselineCount << " baselines.\n"; // Initialize thread data and threads _loopIndex = imageSet->StartIndex(); _progressTaskNo = new int[_threadCount]; _progressTaskCount = new int[_threadCount]; progress.OnStartTask(*this, 0, 1, "Initializing"); boost::thread_group threadGroup; ReaderFunction reader(*this); threadGroup.create_thread(reader); size_t mathThreads = mathThreadCount(); for(unsigned i=0;i<mathThreads;++i) { PerformFunction function(*this, progress, i); threadGroup.create_thread(function); } threadGroup.join_all(); progress.OnEndTask(*this); if(_resultSet != 0) { artifacts = *_resultSet; delete _resultSet; } delete[] _progressTaskCount; delete[] _progressTaskNo; delete _loopIndex; if(_exceptionOccured) throw std::runtime_error("An exception occured in one of the sub tasks of the (multi-threaded) \"For-each baseline\"-action: the RFI strategy will not continue."); } }
void ForEachMSAction::Perform(ArtifactSet &artifacts, ProgressListener &progress) { unsigned taskIndex = 0; FinishAll(); for(std::vector<std::string>::const_iterator i=_filenames.begin();i!=_filenames.end();++i) { std::string filename = *i; progress.OnStartTask(*this, taskIndex, _filenames.size(), std::string("Processing measurement set ") + filename); bool skip = false; if(_skipIfAlreadyProcessed) { MeasurementSet set(filename); if(set.HasRFIConsoleHistory()) { skip = true; AOLogger::Info << "Skipping " << filename << ",\n" "because the set contains AOFlagger history and -skip-flagged was given.\n"; } } if(!skip) { std::auto_ptr<ImageSet> imageSet(ImageSet::Create(filename, _baselineIOMode, _readUVW)); bool isMS = dynamic_cast<MSImageSet*>(&*imageSet) != 0; if(isMS) { MSImageSet *msImageSet = static_cast<MSImageSet*>(&*imageSet); msImageSet->SetDataColumnName(_dataColumnName); msImageSet->SetSubtractModel(_subtractModel); } imageSet->Initialize(); if(_loadOptimizedStrategy) { rfiStrategy::DefaultStrategy::TelescopeId telescopeId; unsigned flags; double frequency, timeResolution, frequencyResolution; rfiStrategy::DefaultStrategy::DetermineSettings(*imageSet, telescopeId, flags, frequency, timeResolution, frequencyResolution); RemoveAll(); rfiStrategy::DefaultStrategy::LoadFullStrategy( *this, telescopeId, flags, frequency, timeResolution, frequencyResolution ); if(_threadCount != 0) rfiStrategy::Strategy::SetThreadCount(*this, _threadCount); } std::auto_ptr<ImageSetIndex> index(imageSet->StartIndex()); artifacts.SetImageSet(&*imageSet); artifacts.SetImageSetIndex(&*index); InitializeAll(); ActionBlock::Perform(artifacts, progress); FinishAll(); artifacts.SetNoImageSet(); index.reset(); imageSet.reset(); if(isMS) writeHistory(*i); } progress.OnEndTask(*this); ++taskIndex; } InitializeAll(); }
void ForEachBaselineAction::Perform(ArtifactSet &artifacts, ProgressListener &progress) { if(!artifacts.HasImageSet()) { progress.OnStartTask(*this, 0, 1, "For each baseline (no image set)"); progress.OnEndTask(*this); Logger::Warn << "I executed a ForEachBaselineAction without an active imageset: something is\n" "likely wrong. Check your strategy and the input files.\n"; } else if(_selection == Current) { ActionBlock::Perform(artifacts, progress); } else { ImageSet& imageSet = artifacts.ImageSet(); MSImageSet* msImageSet = dynamic_cast<MSImageSet*>(&imageSet); if(msImageSet != 0) { // Check memory usage std::unique_ptr<ImageSetIndex> tempIndex = msImageSet->StartIndex(); size_t timeStepCount = msImageSet->ObservationTimesVector(*tempIndex).size(); tempIndex.reset(); size_t channelCount = msImageSet->GetBandInfo(0).channels.size(); double estMemorySizePerThread = 8.0/*bp complex*/ * 4.0 /*polarizations*/ * double(timeStepCount) * double(channelCount) * 3.0 /* approx copies of the data that will be made in memory*/; Logger::Debug << "Estimate of memory each thread will use: " << memToStr(estMemorySizePerThread) << ".\n"; size_t compThreadCount = _threadCount; if(compThreadCount > 0) --compThreadCount; int64_t memSize = System::TotalMemory(); Logger::Debug << "Detected " << memToStr(memSize) << " of system memory.\n"; if(estMemorySizePerThread * double(compThreadCount) > memSize) { size_t maxThreads = size_t(memSize / estMemorySizePerThread); if(maxThreads < 1) maxThreads = 1; Logger::Warn << "This measurement set is TOO LARGE to be processed with " << _threadCount << " threads!\n" << _threadCount << " threads would require " << memToStr(estMemorySizePerThread*compThreadCount) << " of memory approximately.\n" "Number of threads that will actually be used: " << maxThreads << "\n" "This might hurt performance a lot!\n\n"; _threadCount = maxThreads; } } if(dynamic_cast<FilterBankSet*>(&imageSet) != nullptr && _threadCount != 1) { Logger::Info << "This is a Filterbank set -- disabling multi-threading\n"; _threadCount = 1; } if(!_antennaeToSkip.empty()) { Logger::Debug << "The following antennas will be skipped: "; for(std::set<size_t>::const_iterator i=_antennaeToSkip.begin();i!=_antennaeToSkip.end(); ++i) Logger::Debug << (*i) << ' '; Logger::Debug <<'\n'; } if(!_antennaeToInclude.empty()) { Logger::Debug << "Only the following antennas will be included: "; for(std::set<size_t>::const_iterator i=_antennaeToInclude.begin();i!=_antennaeToInclude.end(); ++i) Logger::Debug << (*i) << ' '; Logger::Debug <<'\n'; } if(artifacts.MetaData() != 0) { _hasInitAntennae = true; if(artifacts.MetaData()->HasAntenna1()) _initAntenna1 = artifacts.MetaData()->Antenna1(); else _hasInitAntennae = false; if(artifacts.MetaData()->HasAntenna2()) _initAntenna2 = artifacts.MetaData()->Antenna2(); else _hasInitAntennae = false; } _artifacts = &artifacts; _initPartIndex = 0; _finishedBaselines = false; _baselineCount = 0; _baselineProgress = 0; _nextIndex = 0; // Count the baselines that are to be processed std::unique_ptr<ImageSetIndex> iteratorIndex = imageSet.StartIndex(); while(iteratorIndex->IsValid()) { if(IsBaselineSelected(*iteratorIndex)) ++_baselineCount; iteratorIndex->Next(); } iteratorIndex.reset(); Logger::Debug << "Will process " << _baselineCount << " baselines.\n"; // Initialize thread data and threads _loopIndex = imageSet.StartIndex(); _progressTaskNo = new int[_threadCount]; _progressTaskCount = new int[_threadCount]; progress.OnStartTask(*this, 0, 1, "Initializing"); std::vector<std::thread> threadGroup; ReaderFunction reader(*this); threadGroup.emplace_back(reader); size_t mathThreads = mathThreadCount(); for(unsigned i=0;i<mathThreads;++i) { PerformFunction function(*this, progress, i); threadGroup.emplace_back(function); } for(std::thread& t : threadGroup) t.join(); progress.OnEndTask(*this); if(_resultSet != 0) { artifacts = *_resultSet; delete _resultSet; } delete[] _progressTaskCount; delete[] _progressTaskNo; _loopIndex.reset(); if(_exceptionOccured) throw std::runtime_error("An exception occured in one of the sub tasks of the (multi-threaded) \"For-each baseline\"-action: the RFI strategy will not continue."); } }
void SpatialCompositionAction::Perform(ArtifactSet &artifacts, ProgressListener &progress) { size_t imageCount = artifacts.ContaminatedData().ImageCount(); std::vector<Image2DPtr> images(imageCount); for(size_t p=0;p<imageCount;++p) images[p] = Image2D::CreateZeroImagePtr(artifacts.ContaminatedData().ImageWidth(), artifacts.ContaminatedData().ImageHeight()); std::string filename = artifacts.ImageSet()->File(); SpatialMSImageSet set(filename); ImageSetIndex *index = set.StartIndex(); size_t progressStep = 0, totalProgress = artifacts.ContaminatedData().ImageWidth() * artifacts.ContaminatedData().ImageHeight()/256; while(index->IsValid()) { TimeFrequencyData *data = set.LoadData(*index); SpatialMatrixMetaData metaData(set.SpatialMetaData(*index)); for(size_t p=0;p!=imageCount;++p) { switch(_operation) { case SumCrossCorrelationsOperation: images[p]->SetValue(metaData.TimeIndex(), metaData.ChannelIndex(), sumCrossCorrelations(data->GetImage(p))); break; case SumAutoCorrelationsOperation: images[p]->SetValue(metaData.TimeIndex(), metaData.ChannelIndex(), sumAutoCorrelations(data->GetImage(p))); break; case EigenvalueDecompositionOperation: { num_t value = eigenvalue(data->GetImage(p), data->GetImage(p+1)); images[p]->SetValue(metaData.TimeIndex(), metaData.ChannelIndex(), value); images[p+1]->SetValue(metaData.TimeIndex(), metaData.ChannelIndex(), 0.0); ++p; } break; case EigenvalueRemovalOperation: { std::pair<num_t, num_t> value = removeEigenvalue(data->GetImage(p), data->GetImage(p+1)); images[p]->SetValue(metaData.TimeIndex(), metaData.ChannelIndex(), value.first); images[p+1]->SetValue(metaData.TimeIndex(), metaData.ChannelIndex(), value.second); ++p; } break; } } delete data; index->Next(); ++progressStep; progress.OnProgress(*this, progressStep/256, totalProgress); } delete index; TimeFrequencyData newRevisedData = artifacts.RevisedData(); for(size_t p=0;p<imageCount;++p) newRevisedData.SetImage(p, images[p]); newRevisedData.SetMask(artifacts.RevisedData()); TimeFrequencyData *contaminatedData = TimeFrequencyData::CreateTFDataFromDiff(artifacts.ContaminatedData(), newRevisedData); contaminatedData->SetMask(artifacts.ContaminatedData()); artifacts.SetRevisedData(newRevisedData); artifacts.SetContaminatedData(*contaminatedData); delete contaminatedData; }