bool ForEachBaselineAction::IsBaselineSelected(ImageSetIndex &index)
{
ImageSet *imageSet = _artifacts->ImageSet();
MSImageSet *msImageSet = dynamic_cast<MSImageSet*>(imageSet);
size_t a1id, a2id;
if(msImageSet != 0)
{
a1id = msImageSet->GetAntenna1(index);
a2id = msImageSet->GetAntenna2(index);
} else {
a1id = 0;
a2id = 0;
}
if(_antennaeToSkip.count(a1id) != 0 || _antennaeToSkip.count(a2id) != 0)
return false;
if(!_antennaeToInclude.empty() && (_antennaeToInclude.count(a1id) == 0 && _antennaeToInclude.count(a2id) == 0))
return false;
// For SD/BHFits files, we want to select everything -- it's confusing
// if the default option "only flag cross correlations" would also
// hold for sdfits files.
if(dynamic_cast<FitsImageSet*>(imageSet)!=0 || dynamic_cast<BHFitsImageSet*>(imageSet)!=0)
return true;
switch(_selection)
{
case All:
return true;
case CrossCorrelations:
{
return a1id != a2id;
}
case AutoCorrelations:
return a1id == a2id;
case EqualToCurrent: {
if(!_hasInitAntennae)
throw BadUsageException("For each baseline over 'EqualToCurrent' with no current baseline");
throw BadUsageException("Not implemented");
}
case AutoCorrelationsOfCurrentAntennae:
if(!_hasInitAntennae)
throw BadUsageException("For each baseline over 'AutoCorrelationsOfCurrentAntennae' with no current baseline");
return a1id == a2id && (_initAntenna1.id == a1id || _initAntenna2.id == a1id);
default:
return false;
}
}
ImageSet *ImageSet::Create(const std::string &file, BaselineIOMode ioMode, bool readUVW)
{
if(IsFitsFile(file))
return new FitsImageSet(file);
else if(IsRCPRawFile(file))
return new RSPImageSet(file);
else if(IsTKPRawFile(file))
return new RawImageSet(file);
else if(IsRawDescFile(file))
return new RawDescImageSet(file);
else if(IsParmFile(file))
return new ParmImageSet(file);
else if(IsTimeFrequencyStatFile(file))
return new TimeFrequencyStatImageSet(file);
else if(IsNoiseStatFile(file))
return new NoiseStatImageSet(file);
else if(IsHarishFile(file))
return new HarishReader(file);
else {
MSImageSet *set = new MSImageSet(file, ioMode);
set->SetReadUVW(readUVW);
return set;
}
}
void ForEachBaselineAction::Perform(ArtifactSet &artifacts, ProgressListener &progress)
{
if(!artifacts.HasImageSet())
{
progress.OnStartTask(*this, 0, 1, "For each baseline (no image set)");
progress.OnEndTask(*this);
AOLogger::Warn <<
"I executed a ForEachBaselineAction without an active imageset: something is\n"
"likely wrong. Check your strategy and the input files.\n";
} else if(_selection == Current)
{
ActionBlock::Perform(artifacts, progress);
} else
{
ImageSet *imageSet = artifacts.ImageSet();
MSImageSet *msImageSet = dynamic_cast<MSImageSet*>(imageSet);
if(msImageSet != 0)
{
// Check memory usage
ImageSetIndex *tempIndex = msImageSet->StartIndex();
size_t timeStepCount = msImageSet->ObservationTimesVector(*tempIndex).size();
delete tempIndex;
size_t channelCount = msImageSet->GetBandInfo(0).channels.size();
size_t estMemorySizePerThread = 8/*bp complex*/ * 4 /*polarizations*/ * timeStepCount * channelCount * 3 /* approx copies of the data that will be made in memory*/;
AOLogger::Debug << "Estimate of memory each thread will use: " << estMemorySizePerThread/(1024*1024) << " MB.\n";
size_t compThreadCount = _threadCount;
if(compThreadCount > 0) --compThreadCount;
if(estMemorySizePerThread * compThreadCount > 12ul*1024ul*1024ul*1024ul)
{
size_t maxThreads = (12ul * 1024ul * 1024ul * 1024ul) / estMemorySizePerThread;
if(maxThreads < 1) maxThreads = 1;
AOLogger::Warn <<
"WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING!\n"
"This measurement set is TOO LARGE to be processed with " << _threadCount << " threads!\n" <<
_threadCount << " threads would require " << ((estMemorySizePerThread*compThreadCount)/(1024*1024)) << " MB of memory approximately.\n"
"Number of threads that will actually be used: " << maxThreads << "\n"
"This might hurt performance a lot!\n\n";
_threadCount = maxThreads;
}
}
if(!_antennaeToSkip.empty())
{
AOLogger::Debug << "The following antenna's will be skipped: ";
for(std::set<size_t>::const_iterator i=_antennaeToSkip.begin();i!=_antennaeToSkip.end(); ++i)
AOLogger::Debug << (*i) << ' ';
AOLogger::Debug <<'\n';
}
if(!_antennaeToInclude.empty())
{
AOLogger::Debug << "Only the following antenna's will be included: ";
for(std::set<size_t>::const_iterator i=_antennaeToInclude.begin();i!=_antennaeToInclude.end(); ++i)
AOLogger::Debug << (*i) << ' ';
AOLogger::Debug <<'\n';
}
if(artifacts.MetaData() != 0)
{
_hasInitAntennae = true;
if(artifacts.MetaData()->HasAntenna1())
_initAntenna1 = artifacts.MetaData()->Antenna1();
else
_hasInitAntennae = false;
if(artifacts.MetaData()->HasAntenna2())
_initAntenna2 = artifacts.MetaData()->Antenna2();
else
_hasInitAntennae = false;
}
_artifacts = &artifacts;
_initPartIndex = 0;
_finishedBaselines = false;
_baselineCount = 0;
_baselineProgress = 0;
_nextIndex = 0;
// Count the baselines that are to be processed
ImageSetIndex *iteratorIndex = imageSet->StartIndex();
while(iteratorIndex->IsValid())
{
if(IsBaselineSelected(*iteratorIndex))
++_baselineCount;
iteratorIndex->Next();
}
delete iteratorIndex;
AOLogger::Debug << "Will process " << _baselineCount << " baselines.\n";
// Initialize thread data and threads
_loopIndex = imageSet->StartIndex();
_progressTaskNo = new int[_threadCount];
_progressTaskCount = new int[_threadCount];
progress.OnStartTask(*this, 0, 1, "Initializing");
boost::thread_group threadGroup;
ReaderFunction reader(*this);
threadGroup.create_thread(reader);
size_t mathThreads = mathThreadCount();
for(unsigned i=0;i<mathThreads;++i)
{
PerformFunction function(*this, progress, i);
threadGroup.create_thread(function);
}
threadGroup.join_all();
progress.OnEndTask(*this);
if(_resultSet != 0)
{
artifacts = *_resultSet;
delete _resultSet;
}
delete[] _progressTaskCount;
delete[] _progressTaskNo;
delete _loopIndex;
if(_exceptionOccured)
throw std::runtime_error("An exception occured in one of the sub tasks of the (multi-threaded) \"For-each baseline\"-action: the RFI strategy will not continue.");
}
}
void ForEachBaselineAction::ReaderFunction::operator()()
{
Stopwatch watch(true);
bool finished = false;
size_t threadCount = _action.mathThreadCount();
size_t minRecommendedBufferSize, maxRecommendedBufferSize;
MSImageSet *msImageSet = dynamic_cast<MSImageSet*>(_action._artifacts->ImageSet());
if(msImageSet != 0)
{
minRecommendedBufferSize = msImageSet->Reader()->GetMinRecommendedBufferSize(threadCount);
maxRecommendedBufferSize = msImageSet->Reader()->GetMaxRecommendedBufferSize(threadCount) - _action.GetBaselinesInBufferCount();
} else {
minRecommendedBufferSize = 1;
maxRecommendedBufferSize = 2;
}
do {
watch.Pause();
_action.WaitForBufferAvailable(minRecommendedBufferSize);
size_t wantedCount = maxRecommendedBufferSize - _action.GetBaselinesInBufferCount();
size_t requestedCount = 0;
boost::mutex::scoped_lock lock(_action._artifacts->IOMutex());
watch.Start();
for(size_t i=0;i<wantedCount;++i)
{
ImageSetIndex *index = _action.GetNextIndex();
if(index != 0)
{
_action._artifacts->ImageSet()->AddReadRequest(*index);
++requestedCount;
delete index;
} else {
finished = true;
break;
}
}
if(requestedCount > 0)
{
_action._artifacts->ImageSet()->PerformReadRequests();
watch.Pause();
for(size_t i=0;i<requestedCount;++i)
{
BaselineData *baseline = _action._artifacts->ImageSet()->GetNextRequested();
boost::mutex::scoped_lock bufferLock(_action._mutex);
_action._baselineBuffer.push(baseline);
bufferLock.unlock();
}
}
lock.unlock();
_action._dataAvailable.notify_all();
watch.Start();
} while(!finished);
_action.SetFinishedBaselines();
_action._dataAvailable.notify_all();
watch.Pause();
AOLogger::Debug << "Time spent on reading: " << watch.ToString() << '\n';
}
void ForEachMSAction::Perform(ArtifactSet &artifacts, ProgressListener &progress)
{
unsigned taskIndex = 0;
FinishAll();
for(std::vector<std::string>::const_iterator i=_filenames.begin();i!=_filenames.end();++i)
{
std::string filename = *i;
progress.OnStartTask(*this, taskIndex, _filenames.size(), std::string("Processing measurement set ") + filename);
bool skip = false;
if(_skipIfAlreadyProcessed)
{
MeasurementSet set(filename);
if(set.HasRFIConsoleHistory())
{
skip = true;
AOLogger::Info << "Skipping " << filename << ",\n"
"because the set contains AOFlagger history and -skip-flagged was given.\n";
}
}
if(!skip)
{
std::auto_ptr<ImageSet> imageSet(ImageSet::Create(filename, _baselineIOMode, _readUVW));
bool isMS = dynamic_cast<MSImageSet*>(&*imageSet) != 0;
if(isMS)
{
MSImageSet *msImageSet = static_cast<MSImageSet*>(&*imageSet);
msImageSet->SetDataColumnName(_dataColumnName);
msImageSet->SetSubtractModel(_subtractModel);
}
imageSet->Initialize();
if(_loadOptimizedStrategy)
{
rfiStrategy::DefaultStrategy::TelescopeId telescopeId;
unsigned flags;
double frequency, timeResolution, frequencyResolution;
rfiStrategy::DefaultStrategy::DetermineSettings(*imageSet, telescopeId, flags, frequency, timeResolution, frequencyResolution);
RemoveAll();
rfiStrategy::DefaultStrategy::LoadFullStrategy(
*this,
telescopeId,
flags,
frequency,
timeResolution,
frequencyResolution
);
if(_threadCount != 0)
rfiStrategy::Strategy::SetThreadCount(*this, _threadCount);
}
std::auto_ptr<ImageSetIndex> index(imageSet->StartIndex());
artifacts.SetImageSet(&*imageSet);
artifacts.SetImageSetIndex(&*index);
InitializeAll();
ActionBlock::Perform(artifacts, progress);
FinishAll();
artifacts.SetNoImageSet();
index.reset();
imageSet.reset();
if(isMS)
writeHistory(*i);
}
progress.OnEndTask(*this);
++taskIndex;
}
InitializeAll();
}
void ForEachBaselineAction::ReaderFunction::operator()()
{
Stopwatch watch(true);
bool finished = false;
size_t threadCount = _action.mathThreadCount();
size_t minRecommendedBufferSize, maxRecommendedBufferSize;
MSImageSet* msImageSet = dynamic_cast<MSImageSet*>(&_action._artifacts->ImageSet());
if(msImageSet != nullptr)
{
minRecommendedBufferSize = msImageSet->Reader()->GetMinRecommendedBufferSize(threadCount);
maxRecommendedBufferSize = msImageSet->Reader()->GetMaxRecommendedBufferSize(threadCount) - _action.GetBaselinesInBufferCount();
} else {
minRecommendedBufferSize = 1;
maxRecommendedBufferSize = 2;
}
do {
watch.Pause();
_action.WaitForBufferAvailable(minRecommendedBufferSize);
size_t wantedCount = maxRecommendedBufferSize - _action.GetBaselinesInBufferCount();
size_t requestedCount = 0;
std::unique_lock<std::mutex> lock(_action._artifacts->IOMutex());
watch.Start();
for(size_t i=0;i<wantedCount;++i)
{
std::unique_ptr<ImageSetIndex> index = _action.GetNextIndex();
if(index != nullptr)
{
_action._artifacts->ImageSet().AddReadRequest(*index);
++requestedCount;
} else {
finished = true;
break;
}
}
if(requestedCount > 0)
{
_action._artifacts->ImageSet().PerformReadRequests();
watch.Pause();
for(size_t i=0;i<requestedCount;++i)
{
std::unique_ptr<BaselineData> baseline = _action._artifacts->ImageSet().GetNextRequested();
std::lock_guard<std::mutex> bufferLock(_action._mutex);
_action._baselineBuffer.emplace(std::move(baseline));
}
}
lock.unlock();
_action._dataAvailable.notify_all();
watch.Start();
} while(!finished);
_action.SetFinishedBaselines();
_action._dataAvailable.notify_all();
watch.Pause();
Logger::Debug << "Time spent on reading: " << watch.ToString() << '\n';
}
void ForEachBaselineAction::Perform(ArtifactSet &artifacts, ProgressListener &progress)
{
if(!artifacts.HasImageSet())
{
progress.OnStartTask(*this, 0, 1, "For each baseline (no image set)");
progress.OnEndTask(*this);
Logger::Warn <<
"I executed a ForEachBaselineAction without an active imageset: something is\n"
"likely wrong. Check your strategy and the input files.\n";
} else if(_selection == Current)
{
ActionBlock::Perform(artifacts, progress);
} else
{
ImageSet& imageSet = artifacts.ImageSet();
MSImageSet* msImageSet = dynamic_cast<MSImageSet*>(&imageSet);
if(msImageSet != 0)
{
// Check memory usage
std::unique_ptr<ImageSetIndex> tempIndex = msImageSet->StartIndex();
size_t timeStepCount = msImageSet->ObservationTimesVector(*tempIndex).size();
tempIndex.reset();
size_t channelCount = msImageSet->GetBandInfo(0).channels.size();
double estMemorySizePerThread =
8.0/*bp complex*/ * 4.0 /*polarizations*/ *
double(timeStepCount) * double(channelCount) *
3.0 /* approx copies of the data that will be made in memory*/;
Logger::Debug << "Estimate of memory each thread will use: " << memToStr(estMemorySizePerThread) << ".\n";
size_t compThreadCount = _threadCount;
if(compThreadCount > 0) --compThreadCount;
int64_t memSize = System::TotalMemory();
Logger::Debug << "Detected " << memToStr(memSize) << " of system memory.\n";
if(estMemorySizePerThread * double(compThreadCount) > memSize)
{
size_t maxThreads = size_t(memSize / estMemorySizePerThread);
if(maxThreads < 1) maxThreads = 1;
Logger::Warn <<
"This measurement set is TOO LARGE to be processed with " << _threadCount << " threads!\n" <<
_threadCount << " threads would require " << memToStr(estMemorySizePerThread*compThreadCount) << " of memory approximately.\n"
"Number of threads that will actually be used: " << maxThreads << "\n"
"This might hurt performance a lot!\n\n";
_threadCount = maxThreads;
}
}
if(dynamic_cast<FilterBankSet*>(&imageSet) != nullptr && _threadCount != 1)
{
Logger::Info << "This is a Filterbank set -- disabling multi-threading\n";
_threadCount = 1;
}
if(!_antennaeToSkip.empty())
{
Logger::Debug << "The following antennas will be skipped: ";
for(std::set<size_t>::const_iterator i=_antennaeToSkip.begin();i!=_antennaeToSkip.end(); ++i)
Logger::Debug << (*i) << ' ';
Logger::Debug <<'\n';
}
if(!_antennaeToInclude.empty())
{
Logger::Debug << "Only the following antennas will be included: ";
for(std::set<size_t>::const_iterator i=_antennaeToInclude.begin();i!=_antennaeToInclude.end(); ++i)
Logger::Debug << (*i) << ' ';
Logger::Debug <<'\n';
}
if(artifacts.MetaData() != 0)
{
_hasInitAntennae = true;
if(artifacts.MetaData()->HasAntenna1())
_initAntenna1 = artifacts.MetaData()->Antenna1();
else
_hasInitAntennae = false;
if(artifacts.MetaData()->HasAntenna2())
_initAntenna2 = artifacts.MetaData()->Antenna2();
else
_hasInitAntennae = false;
}
_artifacts = &artifacts;
_initPartIndex = 0;
_finishedBaselines = false;
_baselineCount = 0;
_baselineProgress = 0;
_nextIndex = 0;
// Count the baselines that are to be processed
std::unique_ptr<ImageSetIndex> iteratorIndex = imageSet.StartIndex();
while(iteratorIndex->IsValid())
{
if(IsBaselineSelected(*iteratorIndex))
++_baselineCount;
iteratorIndex->Next();
}
iteratorIndex.reset();
Logger::Debug << "Will process " << _baselineCount << " baselines.\n";
// Initialize thread data and threads
_loopIndex = imageSet.StartIndex();
_progressTaskNo = new int[_threadCount];
_progressTaskCount = new int[_threadCount];
progress.OnStartTask(*this, 0, 1, "Initializing");
std::vector<std::thread> threadGroup;
ReaderFunction reader(*this);
threadGroup.emplace_back(reader);
size_t mathThreads = mathThreadCount();
for(unsigned i=0;i<mathThreads;++i)
{
PerformFunction function(*this, progress, i);
threadGroup.emplace_back(function);
}
for(std::thread& t : threadGroup)
t.join();
progress.OnEndTask(*this);
if(_resultSet != 0)
{
artifacts = *_resultSet;
delete _resultSet;
}
delete[] _progressTaskCount;
delete[] _progressTaskNo;
_loopIndex.reset();
if(_exceptionOccured)
throw std::runtime_error("An exception occured in one of the sub tasks of the (multi-threaded) \"For-each baseline\"-action: the RFI strategy will not continue.");
}
}
