void PlotAction::plotAntennaFlagCounts(ArtifactSet &artifacts) { if(artifacts.AntennaFlagCountPlot() == 0) throw BadUsageException("No antenna flag count plot in the artifact set"); if(artifacts.HasMetaData() && artifacts.MetaData()->HasAntenna1() && artifacts.MetaData()->HasAntenna2()) { TimeFrequencyData &data = artifacts.ContaminatedData(); TimeFrequencyMetaDataCPtr meta = artifacts.MetaData(); artifacts.AntennaFlagCountPlot()->Add(data, meta); } else { AOLogger::Warn << "The strategy contains an action that makes an antenna plot, but the image set did not provide meta data.\n" "Plot will not be made.\n"; } }
void PlotAction::plotFrequencyPower(ArtifactSet &artifacts) { if(artifacts.FrequencyPowerPlot() == 0) throw BadUsageException("No frequency power plot in the artifact set"); TimeFrequencyData &data = artifacts.ContaminatedData(); TimeFrequencyMetaDataCPtr meta = artifacts.MetaData(); artifacts.FrequencyPowerPlot()->Add(data, meta); }
void PlotAction::plotAntennaFlagCounts(ArtifactSet &artifacts) { if(artifacts.AntennaFlagCountPlot() == 0) throw BadUsageException("No antenna flag count plot in the artifact set"); TimeFrequencyData &data = artifacts.ContaminatedData(); TimeFrequencyMetaDataCPtr meta = artifacts.MetaData(); artifacts.AntennaFlagCountPlot()->Add(data, meta); }
void PlotAction::plotSpectrumPerBaseline(ArtifactSet &artifacts) { if(artifacts.FrequencyPowerPlot() == 0) throw BadUsageException("No frequency power plot in the artifact set"); TimeFrequencyData &data = artifacts.ContaminatedData(); TimeFrequencyMetaDataCPtr meta = artifacts.MetaData(); artifacts.FrequencyPowerPlot()->SetLogYAxis(_logYAxis); artifacts.FrequencyPowerPlot()->StartNewLine(meta->Antenna1().name + " x " + meta->Antenna2().name); artifacts.FrequencyPowerPlot()->Add(data, meta); }
void TimeConvolutionAction::PerformFFTSincOperation(ArtifactSet &artifacts, Image2DPtr real, Image2DPtr imag) const { fftw_complex *fftIn = (fftw_complex*) fftw_malloc(sizeof(fftw_complex) * real->Width()), *fftOut = (fftw_complex*) fftw_malloc(sizeof(fftw_complex) * real->Width()); // FFTW plan routines are not thread safe, so lock. boost::mutex::scoped_lock lock(artifacts.IOMutex()); fftw_plan fftPlanForward = fftw_plan_dft_1d(real->Width(), fftIn, fftOut, FFTW_FORWARD, FFTW_MEASURE), fftPlanBackward = fftw_plan_dft_1d(real->Width(), fftIn, fftOut, FFTW_BACKWARD, FFTW_MEASURE); lock.unlock(); const size_t width = real->Width(); const BandInfo band = artifacts.MetaData()->Band(); for(unsigned y=0;y<real->Height();++y) { const numl_t sincScale = ActualSincScaleInSamples(artifacts, band.channels[y].frequencyHz); const numl_t limitFrequency = (numl_t) width / sincScale; if(y == real->Height()/2) { AOLogger::Debug << "Horizontal sinc scale: " << sincScale << " (filter scale: " << Angle::ToString(ActualSincScaleAsRaDecDist(artifacts, band.channels[y].frequencyHz)) << ")\n"; } if(sincScale > 1.0) { for(unsigned x=0;x<width;++x) { fftIn[x][0] = real->Value(x, y); fftIn[x][1] = imag->Value(x, y); } fftw_execute_dft(fftPlanForward, fftIn, fftOut); size_t filterIndexSize = (limitFrequency > 1.0) ? (size_t) ceil(limitFrequency/2.0) : 1; // Remove the high frequencies [filterIndexSize : n-filterIndexSize] for(size_t f=filterIndexSize;f<width - filterIndexSize;++f) { fftOut[f][0] = 0.0; fftOut[f][1] = 0.0; } fftw_execute_dft(fftPlanBackward, fftOut, fftIn); const double n = width; for(unsigned x=0;x<width;++x) { real->SetValue(x, y, fftIn[x][0] / n); imag->SetValue(x, y, fftIn[x][1] / n); } } } fftw_free(fftIn); fftw_free(fftOut); }
void ImagerAction::Perform(ArtifactSet &artifacts, ProgressListener &progress) { boost::mutex::scoped_lock lock(_imagerMutex); UVImager *imager = artifacts.Imager(); if(imager == 0) throw BadUsageException("No imager available to create image."); TimeFrequencyData &data = artifacts.ContaminatedData(); TimeFrequencyMetaDataCPtr metaData = artifacts.MetaData(); if(data.PolarisationCount() > 1) { TimeFrequencyData *tmp = data.CreateTFData(StokesIPolarisation); data = *tmp; delete tmp; } bool btPlaneImager = true; if(btPlaneImager) { typedef double ImagerNumeric; BaselineTimePlaneImager<ImagerNumeric> btImager; BandInfo band = metaData->Band(); Image2DCPtr inputReal = data.GetRealPart(), inputImag = data.GetImaginaryPart(); Mask2DCPtr mask = data.GetSingleMask(); size_t width = inputReal->Width(); for(size_t t=0;t!=width;++t) { UVW uvw = metaData->UVW()[t]; size_t channelCount = inputReal->Height(); std::vector<std::complex<ImagerNumeric> > data(channelCount); for(size_t ch=0;ch!=channelCount;++ch) { if(mask->Value(t, ch)) data[ch] = std::complex<ImagerNumeric>(0.0, 0.0); else data[ch] = std::complex<ImagerNumeric>(inputReal->Value(t, ch), inputImag->Value(t, ch)); } btImager.Image(uvw.u, uvw.v, uvw.w, band.channels[0].frequencyHz, band.channels[1].frequencyHz-band.channels[0].frequencyHz, channelCount, &(data[0]), imager->FTReal()); } } else { progress.OnStartTask(*this, 0, 1, "Imaging baseline"); for(size_t y=0;y<data.ImageHeight();++y) { imager->Image(data, metaData, y); progress.OnProgress(*this, y, data.ImageHeight()); } progress.OnEndTask(*this); } }
void PlotAction::plotBaselineRMS(ArtifactSet &artifacts) { if(artifacts.PolarizationStatistics() == 0) throw BadUsageException("No polarization statistics in the artifact set"); TimeFrequencyData &data = artifacts.ContaminatedData(); TimeFrequencyMetaDataCPtr metaData = artifacts.MetaData(); double rms = 0.0; for(unsigned i=0;i<data.PolarisationCount();++i) { TimeFrequencyData *polarisation = data.CreateTFDataFromPolarisationIndex(i); Mask2DCPtr mask = polarisation->GetSingleMask(); for(unsigned j=0;j<polarisation->ImageCount();++j) { Image2DCPtr image = polarisation->GetImage(j); rms += ThresholdTools::RMS(image, mask); } delete polarisation; } rms /= data.PolarisationCount(); ; AOLogger::Info << "RMS of " << metaData->Antenna1().name << " x " << metaData->Antenna2().name << ": " << rms << '\n'; }
void ForEachBaselineAction::Perform(ArtifactSet &artifacts, ProgressListener &progress) { if(!artifacts.HasImageSet()) { progress.OnStartTask(*this, 0, 1, "For each baseline (no image set)"); progress.OnEndTask(*this); AOLogger::Warn << "I executed a ForEachBaselineAction without an active imageset: something is\n" "likely wrong. Check your strategy and the input files.\n"; } else if(_selection == Current) { ActionBlock::Perform(artifacts, progress); } else { ImageSet *imageSet = artifacts.ImageSet(); MSImageSet *msImageSet = dynamic_cast<MSImageSet*>(imageSet); if(msImageSet != 0) { // Check memory usage ImageSetIndex *tempIndex = msImageSet->StartIndex(); size_t timeStepCount = msImageSet->ObservationTimesVector(*tempIndex).size(); delete tempIndex; size_t channelCount = msImageSet->GetBandInfo(0).channels.size(); size_t estMemorySizePerThread = 8/*bp complex*/ * 4 /*polarizations*/ * timeStepCount * channelCount * 3 /* approx copies of the data that will be made in memory*/; AOLogger::Debug << "Estimate of memory each thread will use: " << estMemorySizePerThread/(1024*1024) << " MB.\n"; size_t compThreadCount = _threadCount; if(compThreadCount > 0) --compThreadCount; if(estMemorySizePerThread * compThreadCount > 12ul*1024ul*1024ul*1024ul) { size_t maxThreads = (12ul * 1024ul * 1024ul * 1024ul) / estMemorySizePerThread; if(maxThreads < 1) maxThreads = 1; AOLogger::Warn << "WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING!\n" "This measurement set is TOO LARGE to be processed with " << _threadCount << " threads!\n" << _threadCount << " threads would require " << ((estMemorySizePerThread*compThreadCount)/(1024*1024)) << " MB of memory approximately.\n" "Number of threads that will actually be used: " << maxThreads << "\n" "This might hurt performance a lot!\n\n"; _threadCount = maxThreads; } } if(!_antennaeToSkip.empty()) { AOLogger::Debug << "The following antenna's will be skipped: "; for(std::set<size_t>::const_iterator i=_antennaeToSkip.begin();i!=_antennaeToSkip.end(); ++i) AOLogger::Debug << (*i) << ' '; AOLogger::Debug <<'\n'; } if(!_antennaeToInclude.empty()) { AOLogger::Debug << "Only the following antenna's will be included: "; for(std::set<size_t>::const_iterator i=_antennaeToInclude.begin();i!=_antennaeToInclude.end(); ++i) AOLogger::Debug << (*i) << ' '; AOLogger::Debug <<'\n'; } if(artifacts.MetaData() != 0) { _hasInitAntennae = true; if(artifacts.MetaData()->HasAntenna1()) _initAntenna1 = artifacts.MetaData()->Antenna1(); else _hasInitAntennae = false; if(artifacts.MetaData()->HasAntenna2()) _initAntenna2 = artifacts.MetaData()->Antenna2(); else _hasInitAntennae = false; } _artifacts = &artifacts; _initPartIndex = 0; _finishedBaselines = false; _baselineCount = 0; _baselineProgress = 0; _nextIndex = 0; // Count the baselines that are to be processed ImageSetIndex *iteratorIndex = imageSet->StartIndex(); while(iteratorIndex->IsValid()) { if(IsBaselineSelected(*iteratorIndex)) ++_baselineCount; iteratorIndex->Next(); } delete iteratorIndex; AOLogger::Debug << "Will process " << _baselineCount << " baselines.\n"; // Initialize thread data and threads _loopIndex = imageSet->StartIndex(); _progressTaskNo = new int[_threadCount]; _progressTaskCount = new int[_threadCount]; progress.OnStartTask(*this, 0, 1, "Initializing"); boost::thread_group threadGroup; ReaderFunction reader(*this); threadGroup.create_thread(reader); size_t mathThreads = mathThreadCount(); for(unsigned i=0;i<mathThreads;++i) { PerformFunction function(*this, progress, i); threadGroup.create_thread(function); } threadGroup.join_all(); progress.OnEndTask(*this); if(_resultSet != 0) { artifacts = *_resultSet; delete _resultSet; } delete[] _progressTaskCount; delete[] _progressTaskNo; delete _loopIndex; if(_exceptionOccured) throw std::runtime_error("An exception occured in one of the sub tasks of the (multi-threaded) \"For-each baseline\"-action: the RFI strategy will not continue."); } }
void ForEachBaselineAction::Perform(ArtifactSet &artifacts, ProgressListener &progress) { if(!artifacts.HasImageSet()) { progress.OnStartTask(*this, 0, 1, "For each baseline (no image set)"); progress.OnEndTask(*this); Logger::Warn << "I executed a ForEachBaselineAction without an active imageset: something is\n" "likely wrong. Check your strategy and the input files.\n"; } else if(_selection == Current) { ActionBlock::Perform(artifacts, progress); } else { ImageSet& imageSet = artifacts.ImageSet(); MSImageSet* msImageSet = dynamic_cast<MSImageSet*>(&imageSet); if(msImageSet != 0) { // Check memory usage std::unique_ptr<ImageSetIndex> tempIndex = msImageSet->StartIndex(); size_t timeStepCount = msImageSet->ObservationTimesVector(*tempIndex).size(); tempIndex.reset(); size_t channelCount = msImageSet->GetBandInfo(0).channels.size(); double estMemorySizePerThread = 8.0/*bp complex*/ * 4.0 /*polarizations*/ * double(timeStepCount) * double(channelCount) * 3.0 /* approx copies of the data that will be made in memory*/; Logger::Debug << "Estimate of memory each thread will use: " << memToStr(estMemorySizePerThread) << ".\n"; size_t compThreadCount = _threadCount; if(compThreadCount > 0) --compThreadCount; int64_t memSize = System::TotalMemory(); Logger::Debug << "Detected " << memToStr(memSize) << " of system memory.\n"; if(estMemorySizePerThread * double(compThreadCount) > memSize) { size_t maxThreads = size_t(memSize / estMemorySizePerThread); if(maxThreads < 1) maxThreads = 1; Logger::Warn << "This measurement set is TOO LARGE to be processed with " << _threadCount << " threads!\n" << _threadCount << " threads would require " << memToStr(estMemorySizePerThread*compThreadCount) << " of memory approximately.\n" "Number of threads that will actually be used: " << maxThreads << "\n" "This might hurt performance a lot!\n\n"; _threadCount = maxThreads; } } if(dynamic_cast<FilterBankSet*>(&imageSet) != nullptr && _threadCount != 1) { Logger::Info << "This is a Filterbank set -- disabling multi-threading\n"; _threadCount = 1; } if(!_antennaeToSkip.empty()) { Logger::Debug << "The following antennas will be skipped: "; for(std::set<size_t>::const_iterator i=_antennaeToSkip.begin();i!=_antennaeToSkip.end(); ++i) Logger::Debug << (*i) << ' '; Logger::Debug <<'\n'; } if(!_antennaeToInclude.empty()) { Logger::Debug << "Only the following antennas will be included: "; for(std::set<size_t>::const_iterator i=_antennaeToInclude.begin();i!=_antennaeToInclude.end(); ++i) Logger::Debug << (*i) << ' '; Logger::Debug <<'\n'; } if(artifacts.MetaData() != 0) { _hasInitAntennae = true; if(artifacts.MetaData()->HasAntenna1()) _initAntenna1 = artifacts.MetaData()->Antenna1(); else _hasInitAntennae = false; if(artifacts.MetaData()->HasAntenna2()) _initAntenna2 = artifacts.MetaData()->Antenna2(); else _hasInitAntennae = false; } _artifacts = &artifacts; _initPartIndex = 0; _finishedBaselines = false; _baselineCount = 0; _baselineProgress = 0; _nextIndex = 0; // Count the baselines that are to be processed std::unique_ptr<ImageSetIndex> iteratorIndex = imageSet.StartIndex(); while(iteratorIndex->IsValid()) { if(IsBaselineSelected(*iteratorIndex)) ++_baselineCount; iteratorIndex->Next(); } iteratorIndex.reset(); Logger::Debug << "Will process " << _baselineCount << " baselines.\n"; // Initialize thread data and threads _loopIndex = imageSet.StartIndex(); _progressTaskNo = new int[_threadCount]; _progressTaskCount = new int[_threadCount]; progress.OnStartTask(*this, 0, 1, "Initializing"); std::vector<std::thread> threadGroup; ReaderFunction reader(*this); threadGroup.emplace_back(reader); size_t mathThreads = mathThreadCount(); for(unsigned i=0;i<mathThreads;++i) { PerformFunction function(*this, progress, i); threadGroup.emplace_back(function); } for(std::thread& t : threadGroup) t.join(); progress.OnEndTask(*this); if(_resultSet != 0) { artifacts = *_resultSet; delete _resultSet; } delete[] _progressTaskCount; delete[] _progressTaskNo; _loopIndex.reset(); if(_exceptionOccured) throw std::runtime_error("An exception occured in one of the sub tasks of the (multi-threaded) \"For-each baseline\"-action: the RFI strategy will not continue."); } }