void TimeFrequencyImager::WriteNewFlagsPart(Mask2DCPtr newXX, Mask2DCPtr newXY, Mask2DCPtr newYX, Mask2DCPtr newYY, int antenna1, int antenna2, int spectralWindow, size_t timeOffset, size_t timeEnd, size_t leftBorder, size_t rightBorder)
{
initializePolarizations();
checkPolarizations();
size_t frequencyCount = _measurementSet->FrequencyCount();
std::map<double,size_t> observationTimes;
setObservationTimes(*_measurementSet, observationTimes);
casa::Table *table = _measurementSet->OpenTable(true);
casa::ScalarColumn<int> antenna1Column(*table, "ANTENNA1");
casa::ScalarColumn<int> antenna2Column(*table, "ANTENNA2");
casa::ScalarColumn<int> windowColumn(*table, "DATA_DESC_ID");
casa::ScalarColumn<double> timeColumn(*table, "TIME");
casa::ArrayColumn<bool> flagColumn(*table, "FLAG");
ScalarColumnIterator<int> antenna1Iter = ScalarColumnIterator<int>::First(antenna1Column);
ScalarColumnIterator<int> antenna2Iter = ScalarColumnIterator<int>::First(antenna2Column);
ScalarColumnIterator<int> windowIter = ScalarColumnIterator<int>::First(windowColumn);
ScalarColumnIterator<double> timeIter = ScalarColumnIterator<double>::First(timeColumn);
ArrayColumnIterator<bool> flagIter = ArrayColumnIterator<bool>::First(flagColumn);
if(frequencyCount != newXX->Height())
{
std::cerr << "The frequency count in the measurement set (" << frequencyCount << ") does not match the image!" << std::endl;
}
if(timeEnd - timeOffset != newXX->Width())
{
std::cerr << "The number of time scans to write in the measurement set (" << (timeEnd - timeOffset) << ") does not match the image (" << newXX->Width() << ") !" << std::endl;
}
size_t rowsWritten = 0;
for(size_t i=0;i<table->nrow();++i) {
if((*antenna1Iter) == (int) antenna1 &&
(*antenna2Iter) == (int) antenna2 &&
(*windowIter) == (int) spectralWindow)
{
double time = *timeIter;
size_t timeIndex = observationTimes.find(time)->second;
if(timeIndex >= timeOffset + leftBorder && timeIndex < timeEnd - rightBorder)
{
casa::Array<bool> flag = *flagIter;
casa::Array<bool>::iterator j = flag.begin();
for(size_t f=0;f<(size_t) frequencyCount;++f) {
if(_stokesIIndex >= 0)
{
if(_readStokesIDirectly) *j = newXX->Value(timeIndex - timeOffset, f);
++j;
}
if(_xxIndex >= 0)
{
if(_readXX) *j = newXX->Value(timeIndex - timeOffset, f);
++j;
}
if(_xyIndex >= 0)
{
if(_readXY) *j = newXY->Value(timeIndex - timeOffset, f);
++j;
}
if(_yxIndex >= 0)
{
if(_readYX) *j = newYX->Value(timeIndex - timeOffset, f);
++j;
}
if(_yyIndex >= 0)
{
if(_readYY) *j = newYY->Value(timeIndex - timeOffset, f);
++j;
}
}
flagIter.Set(flag);
++rowsWritten;
}
}
++antenna1Iter;
++antenna2Iter;
++timeIter;
++windowIter;
++flagIter;
}
AOLogger::Debug << "Rows written: " << rowsWritten << '\n';
delete table;
}
void actionCollect(const std::string &filename, enum CollectingMode mode, StatisticsCollection &statisticsCollection, HistogramCollection &histogramCollection, bool mwaChannels, size_t flaggedTimesteps, const std::set<size_t> &flaggedAntennae)
{
MeasurementSet *ms = new MeasurementSet(filename);
const unsigned polarizationCount = ms->PolarizationCount();
const unsigned bandCount = ms->BandCount();
const bool ignoreChannelZero = ms->IsChannelZeroRubish();
const std::string stationName = ms->GetStationName();
BandInfo *bands = new BandInfo[bandCount];
double **frequencies = new double*[bandCount];
unsigned totalChannels = 0;
for(unsigned b=0;b<bandCount;++b)
{
bands[b] = ms->GetBandInfo(b);
frequencies[b] = new double[bands[b].channels.size()];
totalChannels += bands[b].channels.size();
for(unsigned c=0;c<bands[b].channels.size();++c)
{
frequencies[b][c] = bands[b].channels[c].frequencyHz;
}
}
delete ms;
std::cout
<< "Polarizations: " << polarizationCount << '\n'
<< "Bands: " << bandCount << '\n'
<< "Channels/band: " << (totalChannels / bandCount) << '\n';
if(ignoreChannelZero)
std::cout << "Channel zero will be ignored, as this looks like a LOFAR data set with bad channel 0.\n";
else
std::cout << "Channel zero will be included in the statistics, as it seems that channel 0 is okay.\n";
// Initialize statisticscollection
statisticsCollection.SetPolarizationCount(polarizationCount);
if(mode == CollectDefault)
{
for(unsigned b=0;b<bandCount;++b)
{
if(ignoreChannelZero)
statisticsCollection.InitializeBand(b, (frequencies[b]+1), bands[b].channels.size()-1);
else
statisticsCollection.InitializeBand(b, frequencies[b], bands[b].channels.size());
}
}
// Initialize Histograms collection
histogramCollection.SetPolarizationCount(polarizationCount);
// get columns
casa::Table table(filename, casa::Table::Update);
const char *dataColumnName = "DATA";
casa::ROArrayColumn<casa::Complex> dataColumn(table, dataColumnName);
casa::ROArrayColumn<bool> flagColumn(table, "FLAG");
casa::ROScalarColumn<double> timeColumn(table, "TIME");
casa::ROScalarColumn<int> antenna1Column(table, "ANTENNA1");
casa::ROScalarColumn<int> antenna2Column(table, "ANTENNA2");
casa::ROScalarColumn<int> windowColumn(table, "DATA_DESC_ID");
std::cout << "Collecting statistics..." << std::endl;
size_t channelCount = bands[0].channels.size();
bool *correlatorFlags = new bool[channelCount];
bool *correlatorFlagsForBadAntenna = new bool[channelCount];
for(size_t ch=0; ch!=channelCount; ++ch)
{
correlatorFlags[ch] = false;
correlatorFlagsForBadAntenna[ch] = true;
}
if(mwaChannels)
{
if(channelCount%24 != 0)
std::cout << "MWA channels requested, but nr of channels not a multiply of 24. Ignoring.\n";
else {
size_t chanPerSb = channelCount/24;
for(size_t x=0;x!=24;++x)
{
correlatorFlags[x*chanPerSb] = true;
correlatorFlags[x*chanPerSb + chanPerSb/2] = true;
correlatorFlags[x*chanPerSb + chanPerSb-1] = true;
}
}
}
const unsigned nrow = table.nrow();
size_t timestepIndex = (size_t) -1;
double prevtime = -1.0;
for(unsigned row = 0; row!=nrow; ++row)
{
const double time = timeColumn(row);
const unsigned antenna1Index = antenna1Column(row);
const unsigned antenna2Index = antenna2Column(row);
const unsigned bandIndex = windowColumn(row);
if(time != prevtime)
{
++timestepIndex;
prevtime = time;
}
const BandInfo &band = bands[bandIndex];
const casa::Array<casa::Complex> dataArray = dataColumn(row);
const casa::Array<bool> flagArray = flagColumn(row);
std::vector<std::complex<float>* > samples(polarizationCount);
bool **isRFI = new bool*[polarizationCount];
for(unsigned p = 0; p < polarizationCount; ++p)
{
isRFI[p] = new bool[band.channels.size()];
samples[p] = new std::complex<float>[band.channels.size()];
}
const bool antennaIsFlagged =
flaggedAntennae.find(antenna1Index) != flaggedAntennae.end() ||
flaggedAntennae.find(antenna2Index) != flaggedAntennae.end();
casa::Array<casa::Complex>::const_iterator dataIter = dataArray.begin();
casa::Array<bool>::const_iterator flagIter = flagArray.begin();
const unsigned startChannel = ignoreChannelZero ? 1 : 0;
if(ignoreChannelZero)
{
for(unsigned p = 0; p < polarizationCount; ++p)
{
++dataIter;
++flagIter;
}
}
for(unsigned channel = startChannel ; channel<band.channels.size(); ++channel)
{
for(unsigned p = 0; p < polarizationCount; ++p)
{
samples[p][channel - startChannel] = *dataIter;
isRFI[p][channel - startChannel] = *flagIter;
++dataIter;
++flagIter;
}
}
for(unsigned p = 0; p < polarizationCount; ++p)
{
switch(mode)
{
case CollectDefault:
if(antennaIsFlagged || timestepIndex < flaggedTimesteps)
statisticsCollection.Add(antenna1Index, antenna2Index, time, bandIndex, p, &samples[p]->real(), &samples[p]->imag(), isRFI[p], correlatorFlagsForBadAntenna, band.channels.size() - startChannel, 2, 1, 1);
else
statisticsCollection.Add(antenna1Index, antenna2Index, time, bandIndex, p, &samples[p]->real(), &samples[p]->imag(), isRFI[p], correlatorFlags, band.channels.size() - startChannel, 2, 1, 1);
break;
case CollectHistograms:
histogramCollection.Add(antenna1Index, antenna2Index, p, samples[p], isRFI[p], band.channels.size() - startChannel);
break;
}
}
for(unsigned p = 0; p < polarizationCount; ++p)
{
delete[] isRFI[p];
delete[] samples[p];
}
delete[] isRFI;
reportProgress(row, nrow);
}
delete[] correlatorFlags;
delete[] correlatorFlagsForBadAntenna;
for(unsigned b=0;b<bandCount;++b)
delete[] frequencies[b];
delete[] frequencies;
delete[] bands;
std::cout << "100\n";
}
void TimeFrequencyImager::image(size_t antenna1Select, size_t antenna2Select, size_t spectralWindowSelect, size_t startIndex, size_t endIndex)
{
size_t timeCount = _observationTimes.size();
int frequencyCount = _measurementSet->FrequencyCount();
initializePolarizations();
checkPolarizations();
if(_sortedTable == 0)
{
casa::Table *rawTable = _measurementSet->OpenTable();
casa::Block<casa::String> names(4);
names[0] = "DATA_DESC_ID";
names[1] = "ANTENNA1";
names[2] = "ANTENNA2";
names[3] = "TIME";
_sortedTable = new casa::Table(rawTable->sort(names));
delete rawTable;
}
casa::Block<casa::String> selectionNames(3);
selectionNames[0] = "DATA_DESC_ID";
selectionNames[1] = "ANTENNA1";
selectionNames[2] = "ANTENNA2";
casa::TableIterator iter(*_sortedTable, selectionNames, casa::TableIterator::Ascending, casa::TableIterator::NoSort);
while(!iter.pastEnd())
{
casa::Table table = iter.table();
casa::ScalarColumn<int> antenna1(table, "ANTENNA1");
casa::ScalarColumn<int> antenna2(table, "ANTENNA2");
casa::ScalarColumn<int> windowColumn(table, "DATA_DESC_ID");
if(table.nrow() > 0 && windowColumn(0) == (int) spectralWindowSelect && antenna1(0) == (int) antenna1Select && antenna2(0) == (int) antenna2Select)
{
break;
} else {
iter.next();
}
}
if(iter.pastEnd())
{
throw std::runtime_error("Baseline not found");
}
casa::Table table = iter.table();
if(startIndex > timeCount)
{
std::cerr << "Warning: startIndex > timeCount" << std::endl;
}
if(endIndex > timeCount)
{
endIndex = timeCount;
std::cerr << "Warning: endIndex > timeCount" << std::endl;
}
size_t width = endIndex-startIndex;
if(width == 0 || frequencyCount == 0)
return;
ClearImages();
if(_readData) {
if(_realXX==0 && _readXX)
{
_realXX = Image2D::CreateZeroImagePtr(width, frequencyCount);
_imaginaryXX = Image2D::CreateZeroImagePtr(width, frequencyCount);
}
if(_realXY == 0 && _readXY)
{
_realXY = Image2D::CreateZeroImagePtr(width, frequencyCount);
_imaginaryXY = Image2D::CreateZeroImagePtr(width, frequencyCount);
}
if(_realYX == 0 && _readYX)
{
_realYX = Image2D::CreateZeroImagePtr(width, frequencyCount);
_imaginaryYX = Image2D::CreateZeroImagePtr(width, frequencyCount);
}
if(_realYY == 0 && _readYY)
{
_realYY = Image2D::CreateZeroImagePtr(width, frequencyCount);
_imaginaryYY = Image2D::CreateZeroImagePtr(width, frequencyCount);
}
if(_realStokesI == 0 && _readStokesI)
{
_realStokesI = Image2D::CreateZeroImagePtr(width, frequencyCount);
_imaginaryStokesI = Image2D::CreateZeroImagePtr(width, frequencyCount);
}
}
if(_readFlags) {
// The flags should be initialized to true, as this baseline might
// miss some time scans that other baselines do have, and these
// should be flagged.
if(_flagXX==0 && _readXX)
_flagXX = Mask2D::CreateSetMaskPtr<true>(width, frequencyCount);
if(_flagXY==0 && _readXY)
_flagXY = Mask2D::CreateSetMaskPtr<true>(width, frequencyCount);
if(_flagYX==0 && _readYX)
_flagYX = Mask2D::CreateSetMaskPtr<true>(width, frequencyCount);
if(_flagYY==0 && _readYY)
_flagYY = Mask2D::CreateSetMaskPtr<true>(width, frequencyCount);
if(_flagCombined==0 && _readStokesI)
_flagCombined = Mask2D::CreateSetMaskPtr<true>(width, frequencyCount);
}
_uvw.resize(width);
casa::ScalarColumn<int> antenna1(table, "ANTENNA1");
casa::ScalarColumn<int> antenna2(table, "ANTENNA2");
casa::ScalarColumn<int> windowColumn(table, "DATA_DESC_ID");
casa::ScalarColumn<double> timeColumn(table, "TIME");
casa::ArrayColumn<float> weightColumn(table, "WEIGHT");
casa::ArrayColumn<double> uvwColumn(table, "UVW");
casa::ArrayColumn<casa::Complex> *dataColumn = 0;
if(_readData)
dataColumn = CreateDataColumn(_dataKind, table);
ArrayColumnIterator<casa::Complex> *modelIter;
if(_dataKind == ResidualData) {
casa::ArrayColumn<casa::Complex> *modelColumn;
modelColumn = new casa::ArrayColumn<casa::Complex>(table, "MODEL_DATA");
modelIter = new ArrayColumnIterator<casa::Complex>(ArrayColumnIterator<casa::Complex>::First(*modelColumn));
} else {
modelIter = 0;
}
casa::ArrayColumn<bool> flagColumn(table, "FLAG");
ScalarColumnIterator<int> antenna1Iter = ScalarColumnIterator<int>::First(antenna1);
ScalarColumnIterator<int> antenna2Iter = ScalarColumnIterator<int>::First(antenna2);
ScalarColumnIterator<int> windowIter = ScalarColumnIterator<int>::First(windowColumn);
ScalarColumnIterator<double> timeIter = ScalarColumnIterator<double>::First(timeColumn);
ArrayColumnIterator<double> uvwIter = ArrayColumnIterator<double>::First(uvwColumn);
ArrayColumnIterator<float> weightIter = ArrayColumnIterator<float>::First(weightColumn);
ArrayColumnIterator<casa::Complex> dataIter =
ArrayColumnIterator<casa::Complex>::First(*dataColumn);
ArrayColumnIterator<bool> flagIter =
ArrayColumnIterator<bool>::First(flagColumn);
for(size_t i=0;i<table.nrow();++i) {
double time = *timeIter;
size_t timeIndex = _observationTimes.find(time)->second;
bool timeIsSelected = timeIndex>=startIndex && timeIndex<endIndex;
if(_readData && timeIsSelected) {
if(_dataKind == WeightData)
ReadWeights(timeIndex-startIndex, frequencyCount, *weightIter);
else if(modelIter == 0)
ReadTimeData(timeIndex-startIndex, frequencyCount, *dataIter, 0);
else {
const casa::Array<casa::Complex> model = **modelIter;
ReadTimeData(timeIndex-startIndex, frequencyCount, *dataIter, &model);
}
}
if(_readFlags && timeIsSelected) {
const casa::Array<bool> flag = *flagIter;
ReadTimeFlags(timeIndex-startIndex, frequencyCount, flag);
}
if(timeIsSelected) {
casa::Array<double> arr = *uvwIter;
casa::Array<double>::const_iterator i = arr.begin();
_uvw[timeIndex-startIndex].u = *i;
++i;
_uvw[timeIndex-startIndex].v = *i;
++i;
_uvw[timeIndex-startIndex].w = *i;
}
if(_readData)
{
++dataIter;
if(modelIter != 0)
++(*modelIter);
}
if(_readFlags)
{
++flagIter;
}
++weightIter;
++antenna1Iter;
++antenna2Iter;
++timeIter;
++uvwIter;
++windowIter;
}
if(dataColumn != 0)
delete dataColumn;
}