std::pair<TimeFrequencyData,TimeFrequencyMetaDataPtr> RSPReader::ReadSingleBeamlet(unsigned long timestepStart, unsigned long timestepEnd, unsigned beamletCount, unsigned beamletIndex)
{
std::pair<TimeFrequencyData,TimeFrequencyMetaDataPtr> data = ReadAllBeamlets(timestepStart, timestepEnd, beamletCount);
const unsigned width = timestepEnd - timestepStart;
Image2DPtr realX = Image2D::CreateZeroImagePtr(width, 1);
Image2DPtr imaginaryX = Image2D::CreateZeroImagePtr(width, 1);
Image2DPtr realY = Image2D::CreateZeroImagePtr(width, 1);
Image2DPtr imaginaryY = Image2D::CreateZeroImagePtr(width, 1);
Mask2DPtr mask = Mask2D::CreateUnsetMaskPtr(width, 1);
TimeFrequencyData allX = data.first.Make(Polarization::XX);
TimeFrequencyData allY = data.first.Make(Polarization::YY);
Image2DCPtr xr = allX.GetRealPart();
Image2DCPtr xi = allX.GetImaginaryPart();
Image2DCPtr yr = allY.GetRealPart();
Image2DCPtr yi = allY.GetImaginaryPart();
Mask2DCPtr maskWithBeamlets = data.first.GetSingleMask();
for(unsigned x=0;x<width;++x)
{
realX->SetValue(x, 0, xr->Value(x, beamletIndex));
imaginaryX->SetValue(x, 0, xi->Value(x, beamletIndex));
realY->SetValue(x, 0, yr->Value(x, beamletIndex));
imaginaryY->SetValue(x, 0, yi->Value(x, beamletIndex));
mask->SetValue(x, 0, maskWithBeamlets->Value(x, beamletIndex));
}
data.first = TimeFrequencyData(Polarization::XX, realX, imaginaryX, Polarization::YY, realY, imaginaryY);
data.first.SetGlobalMask(mask);
BandInfo band = data.second->Band();
band.channels[0] = data.second->Band().channels[beamletIndex];
band.channels.resize(1);
data.second->SetBand(band);
return data;
}
void HighPassFilter::elementWiseDivide(const Image2DPtr &leftHand, const Image2DCPtr &rightHand)
{
for(unsigned y=0;y<leftHand->Height();++y) {
for(unsigned x=0;x<leftHand->Width();++x) {
if(rightHand->Value(x, y) == 0.0)
leftHand->SetValue(x, y, 0.0);
else
leftHand->SetValue(x, y, leftHand->Value(x, y) / rightHand->Value(x, y));
}
}
}
void TimeConvolutionAction::PerformFFTSincOperation(ArtifactSet &artifacts, Image2DPtr real, Image2DPtr imag) const
{
fftw_complex
*fftIn = (fftw_complex*) fftw_malloc(sizeof(fftw_complex) * real->Width()),
*fftOut = (fftw_complex*) fftw_malloc(sizeof(fftw_complex) * real->Width());
// FFTW plan routines are not thread safe, so lock.
boost::mutex::scoped_lock lock(artifacts.IOMutex());
fftw_plan
fftPlanForward = fftw_plan_dft_1d(real->Width(), fftIn, fftOut, FFTW_FORWARD, FFTW_MEASURE),
fftPlanBackward = fftw_plan_dft_1d(real->Width(), fftIn, fftOut, FFTW_BACKWARD, FFTW_MEASURE);
lock.unlock();
const size_t width = real->Width();
const BandInfo band = artifacts.MetaData()->Band();
for(unsigned y=0;y<real->Height();++y)
{
const numl_t sincScale = ActualSincScaleInSamples(artifacts, band.channels[y].frequencyHz);
const numl_t limitFrequency = (numl_t) width / sincScale;
if(y == real->Height()/2)
{
AOLogger::Debug << "Horizontal sinc scale: " << sincScale << " (filter scale: " << Angle::ToString(ActualSincScaleAsRaDecDist(artifacts, band.channels[y].frequencyHz)) << ")\n";
}
if(sincScale > 1.0)
{
for(unsigned x=0;x<width;++x)
{
fftIn[x][0] = real->Value(x, y);
fftIn[x][1] = imag->Value(x, y);
}
fftw_execute_dft(fftPlanForward, fftIn, fftOut);
size_t filterIndexSize = (limitFrequency > 1.0) ? (size_t) ceil(limitFrequency/2.0) : 1;
// Remove the high frequencies [filterIndexSize : n-filterIndexSize]
for(size_t f=filterIndexSize;f<width - filterIndexSize;++f)
{
fftOut[f][0] = 0.0;
fftOut[f][1] = 0.0;
}
fftw_execute_dft(fftPlanBackward, fftOut, fftIn);
const double n = width;
for(unsigned x=0;x<width;++x)
{
real->SetValue(x, y, fftIn[x][0] / n);
imag->SetValue(x, y, fftIn[x][1] / n);
}
}
}
fftw_free(fftIn);
fftw_free(fftOut);
}
void HighPassFilter::setFlaggedValuesToZeroAndMakeWeights(const Image2DCPtr &inputImage, const Image2DPtr &outputImage, const Mask2DCPtr &inputMask, const Image2DPtr &weightsOutput)
{
const size_t width = inputImage->Width();
for(size_t y=0;y<inputImage->Height();++y)
{
for(size_t x=0;x<width;++x)
{
if(inputMask->Value(x, y) || !isfinite(inputImage->Value(x, y)))
{
outputImage->SetValue(x, y, 0.0);
weightsOutput->SetValue(x, y, 0.0);
} else {
outputImage->SetValue(x, y, inputImage->Value(x, y));
weightsOutput->SetValue(x, y, 1.0);
}
}
}
}
void ImageTile::LineThreshold(bool evaluateBaseline, long double mean, long double variance, bool convolve)
{
Image2DPtr input = Image2D::CreateEmptyImagePtr(_scanCount, _channelCount);
Mask2DPtr output = Mask2D::CreateSetMaskPtr<false>(_scanCount, _channelCount);
if(evaluateBaseline) {
for(unsigned channel = 0;channel<_channelCount;++channel)
for(unsigned scan = 0;scan<_scanCount;++scan)
input->SetValue(scan, channel, GetValueAt(channel, scan) - EvaluateBaselineFunction(scan, channel));
} else {
for(unsigned channel = 0;channel<_channelCount;++channel)
for(unsigned scan = 0;scan<_scanCount;++scan)
input->SetValue(scan, channel, GetValueAt(channel, scan) - mean);
}
ThresholdMitigater::SumThreshold(input, output, 1, _trigger * variance);
ThresholdMitigater::SumThreshold(input, output, 2, _trigger * variance * 1.6);
ThresholdMitigater::SumThreshold(input, output, 3, _trigger * variance * 2.2);
ThresholdMitigater::SumThreshold(input, output, 5, _trigger * variance * 3.0);
ThresholdMitigater::SumThreshold(input, output, 10, _trigger * variance * 5.0);
unsigned count = 0;
for(unsigned channel = 0;channel<_channelCount;++channel) {
for(unsigned scan = 0;scan<_scanCount;++scan) {
if(output->Value(scan, channel)) {
Window(scan, channel);
count++;
}
}
}
while(count*2 > _channelCount*_scanCount) {
size_t x = (size_t) (RNG::Uniform()*_scanCount);
size_t y = (size_t) (RNG::Uniform()*_channelCount);
if(_isWindowed[y][x]) {
count--;
_isWindowed[y][x]=false;
}
}
if(convolve)
ConvolveWindows();
}
num_t SpatialCompositionAction::eigenvalue(Image2DCPtr real, Image2DCPtr imaginary) const
{
try {
Image2DPtr
r = Image2D::CreateCopy(real),
i = Image2D::CreateCopy(imaginary);
for(size_t y=0;y<r->Height();++y)
{
for(size_t x=0;x<r->Width();++x)
{
if(!std::isfinite(r->Value(x,y))) r->SetValue(x, y, 0.0);
if(!std::isfinite(i->Value(x,y))) i->SetValue(x, y, 0.0);
}
}
if(r->ContainsOnlyZeros() && i->ContainsOnlyZeros()) return 0.0;
return Eigenvalue::Compute(r, i);
} catch(std::exception &e)
{
return std::numeric_limits<num_t>::quiet_NaN();
}
}
Image2DPtr Compress::Read(std::ifstream &stream, Image2DPtr image, Mask2DCPtr mask)
{
num_t max = 0.0, min = 0.0;
size_t width = 0, height = 0;
char mode = 0;
stream.read(reinterpret_cast<char*>(&max), sizeof(max));
stream.read(reinterpret_cast<char*>(&min), sizeof(min));
stream.read(reinterpret_cast<char*>(&width), sizeof(width));
stream.read(reinterpret_cast<char*>(&height), sizeof(height));
stream.read(&mode, sizeof(mode));
num_t normalizeFactor = (max - min) / (num_t) ((2<<22) + ((2<<22)-1));
for(unsigned y=0;y<height;++y)
{
for(unsigned x=0;x<width;++x)
{
if(!mask->Value(x, y))
{
int32_t value;
stream.read(reinterpret_cast<char*>(&value), 3);
value >>= 8;
image->SetValue(x, y, value / normalizeFactor + min);
}
}
}
void BHFitsImageSet::loadImageData(TimeFrequencyData &data, const TimeFrequencyMetaDataPtr &metaData, const BHFitsImageSetIndex &index)
{
std::vector<num_t> buffer(_width * _height);
_file->ReadCurrentImageData(0, &buffer[0], _width * _height);
int
rangeStart = _timeRanges[index._imageIndex].start,
rangeEnd = _timeRanges[index._imageIndex].end;
Image2DPtr image = Image2D::CreateZeroImagePtr(rangeEnd-rangeStart, _height);
std::vector<num_t>::const_iterator bufferPtr = buffer.begin() + _height*rangeStart;
for(int x=rangeStart; x!=rangeEnd; ++x)
{
for(int y=0; y!=_height; ++y)
{
image->SetValue(x-rangeStart, y, *bufferPtr);
++bufferPtr;
}
}
data = TimeFrequencyData(TimeFrequencyData::AmplitudePart, SinglePolarisation, image);
try {
FitsFile flagFile(flagFilePath());
flagFile.Open(FitsFile::ReadOnlyMode);
flagFile.ReadCurrentImageData(0, &buffer[0], _width * _height);
bufferPtr = buffer.begin() + _height*rangeStart;
Mask2DPtr mask = Mask2D::CreateUnsetMaskPtr(rangeEnd-rangeStart, _height);
for(int x=rangeStart; x!=rangeEnd; ++x)
{
for(int y=0; y!=_height; ++y)
{
bool flag = false;
if(*bufferPtr == 0.0)
flag = false;
else if(*bufferPtr == 1.0)
flag = true;
else std::runtime_error("Expecting a flag file with only ones and zeros, but this file contained other values.");
mask->SetValue(x-rangeStart, y, flag);
++bufferPtr;
}
}
data.SetGlobalMask(mask);
} catch(std::exception &)
{
// Flag file could not be read; probably does not exist. Ignore this, flags will be initialized to false.
}
double
frequencyDelta = _file->GetDoubleKeywordValue("CDELT1"),
timeDelta = _file->GetDoubleKeywordValue("CDELT2");
BandInfo band;
for(int ch=0; ch!=_height; ++ch)
{
ChannelInfo channel;
channel.frequencyHz = ch * frequencyDelta * 1000000.0;
band.channels.push_back(channel);
}
metaData->SetBand(band);
const int rangeWidth = rangeEnd-rangeStart;
std::vector<double> observationTimes(rangeWidth);
for(int t=0; t!=rangeWidth; ++t)
observationTimes[t] = (t + rangeStart) * timeDelta;
metaData->SetObservationTimes(observationTimes);
AntennaInfo antennaInfo;
antennaInfo.id = 0;
antennaInfo.name = RangeName(index._imageIndex);
antennaInfo.diameter = 0.0;
antennaInfo.mount = "Unknown";
antennaInfo.station = GetTelescopeName();
metaData->SetAntenna1(antennaInfo);
metaData->SetAntenna2(antennaInfo);
}
std::pair<TimeFrequencyData,TimeFrequencyMetaDataPtr> RSPReader::ReadAllBeamlets(unsigned long timestepStart, unsigned long timestepEnd, unsigned beamletCount)
{
const unsigned width = timestepEnd - timestepStart;
Image2DPtr realX = Image2D::CreateZeroImagePtr(width, beamletCount);
Image2DPtr imaginaryX = Image2D::CreateZeroImagePtr(width, beamletCount);
Image2DPtr realY = Image2D::CreateZeroImagePtr(width, beamletCount);
Image2DPtr imaginaryY = Image2D::CreateZeroImagePtr(width, beamletCount);
Mask2DPtr mask = Mask2D::CreateSetMaskPtr<true>(width, beamletCount);
std::ifstream file(_rawFile.c_str(), std::ios_base::binary | std::ios_base::in);
size_t frame = 0;
std::set<short> stations;
TimeFrequencyMetaDataPtr metaData = TimeFrequencyMetaDataPtr(new TimeFrequencyMetaData());
BandInfo band;
for(size_t i=0;i<beamletCount;++i)
{
ChannelInfo channel;
channel.frequencyHz = i+1;
channel.frequencyIndex = i;
band.channels.push_back(channel);
}
metaData->SetBand(band);
std::vector<double> observationTimes;
// Read a header and determine the reading start position
// Because timestepStart might fall within a block, the
RCPApplicationHeader firstHeader;
firstHeader.Read(file);
const unsigned long bytesPerFrame = beamletCount * firstHeader.nofBlocks * RCPBeamletData::SIZE + RCPApplicationHeader::SIZE;
const unsigned long startFrame = timestepStart / (unsigned long) firstHeader.nofBlocks;
const unsigned long startByte = startFrame * bytesPerFrame;
const unsigned long offsetFromStart = timestepStart - (startFrame * firstHeader.nofBlocks);
//Logger::Debug << "Seeking to " << startByte << " (timestepStart=" << timestepStart << ", offsetFromStart=" << offsetFromStart << ", startFrame=" << startFrame << ",bytesPerFrame=" << bytesPerFrame << ")\n";
file.seekg(startByte, std::ios_base::beg);
// Read the frames
unsigned long x=0;
while(x < width + offsetFromStart && file.good()) {
RCPApplicationHeader header;
header.Read(file);
if(header.versionId != 2)
{
std::stringstream s;
s << "Corrupted header found in frame " << frame << "!";
throw std::runtime_error(s.str());
}
if(stations.count(header.stationId)==0)
{
stations.insert(header.stationId);
AntennaInfo antenna;
std::stringstream s;
s << "LOFAR station with index " << header.stationId;
antenna.name = s.str();
metaData->SetAntenna1(antenna);
metaData->SetAntenna2(antenna);
}
for(size_t j=0;j<beamletCount;++j)
{
for(size_t i=0;i<header.nofBlocks;++i)
{
RCPBeamletData data;
data.Read(file);
if(i + x < width + offsetFromStart && i + x >= offsetFromStart)
{
const unsigned long pos = i + x - offsetFromStart;
realX->SetValue(pos, j, data.xr);
imaginaryX->SetValue(pos, j, data.xi);
realY->SetValue(pos, j, data.yr);
imaginaryY->SetValue(pos, j, data.yi);
mask->SetValue(pos, j, false);
}
}
}
x += header.nofBlocks;
++frame;
}
//Logger::Debug << "Read " << frame << " frames.\n";
for(unsigned long i=0;i<width;++i)
{
const unsigned long pos = i + timestepStart;
const double time =
(double) pos * (double) STATION_INTEGRATION_STEPS / (double) _clockSpeed;
observationTimes.push_back(time);
}
metaData->SetObservationTimes(observationTimes);
std::pair<TimeFrequencyData,TimeFrequencyMetaDataPtr> data;
data.first = TimeFrequencyData(Polarization::XX, realX, imaginaryX, Polarization::YY, realY, imaginaryY);
data.first.SetGlobalMask(mask);
data.second = metaData;
return data;
}
