void broadcast_spectra_info(struct spectra_info *s, int myid)
{
int arrsize;
// Broadcast the basic information first
MPI_Bcast(s, 1, spectra_info_type, 0, MPI_COMM_WORLD);
if (s->datatype == SUBBAND) {
arrsize = 1;
} else {
arrsize = s->num_files;
}
// Now allocate the arrays if we are not the master
if (myid > 0) {
s->start_subint = gen_ivect(arrsize);
s->num_subint = gen_ivect(arrsize);
s->start_spec = (long long *) malloc(sizeof(long long) * arrsize);
s->num_spec = (long long *) malloc(sizeof(long long) * arrsize);
s->num_pad = (long long *) malloc(sizeof(long long) * arrsize);
s->start_MJD = (long double *) malloc(sizeof(long double) * arrsize);
}
// Now broadcast all of the array information
MPI_Bcast(s->start_subint, arrsize, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(s->num_subint, arrsize, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(s->start_spec, arrsize, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
MPI_Bcast(s->num_spec, arrsize, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
MPI_Bcast(s->num_pad, arrsize, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
MPI_Bcast(s->start_MJD, arrsize, MPI_LONG_DOUBLE, 0, MPI_COMM_WORLD);
}
void fill_mask(double timesigma, double freqsigma, double mjd,
double dtint, double lofreq, double dfreq,
int numchan, int numint, int ptsperint,
int num_zap_chans, int *zap_chans, int num_zap_ints,
int *zap_ints, unsigned char **bytemask, mask * obsmask)
/* Fill a mask structure with the appropriate values */
{
int ii, jj, count;
obsmask->timesigma = timesigma;
obsmask->freqsigma = freqsigma;
obsmask->mjd = mjd;
obsmask->dtint = dtint;
obsmask->lofreq = lofreq;
obsmask->dfreq = dfreq;
obsmask->numchan = numchan;
obsmask->numint = numint;
obsmask->ptsperint = ptsperint;
obsmask->num_zap_chans = num_zap_chans;
if (obsmask->num_zap_chans) {
obsmask->zap_chans = gen_ivect(obsmask->num_zap_chans);
for (ii = 0; ii < obsmask->num_zap_chans; ii++)
obsmask->zap_chans[ii] = zap_chans[ii];
}
obsmask->num_zap_ints = num_zap_ints;
if (obsmask->num_zap_ints) {
obsmask->zap_ints = gen_ivect(obsmask->num_zap_ints);
for (ii = 0; ii < obsmask->num_zap_ints; ii++)
obsmask->zap_ints[ii] = zap_ints[ii];
}
obsmask->num_chans_per_int = gen_ivect(obsmask->numint);
obsmask->chans = (int **) malloc(obsmask->numint * sizeof(int *));
for (ii = 0; ii < obsmask->numint; ii++) {
count = 0;
/* Count the bad channels first */
for (jj = 0; jj < obsmask->numchan; jj++)
if ((bytemask[ii][jj] & BADDATA) |
(bytemask[ii][jj] & USERZAP))
count++;
obsmask->num_chans_per_int[ii] = count;
if (count) {
/* Now determine which channels */
count = 0;
obsmask->chans[ii] = gen_ivect(obsmask->num_chans_per_int[ii]);
for (jj = 0; jj < obsmask->numchan; jj++) {
if ((bytemask[ii][jj] & BADDATA) |
(bytemask[ii][jj] & USERZAP))
obsmask->chans[ii][count++] = jj;
}
}
}
}
void read_mask(char *maskfilenm, mask * obsmask)
/* Read the contents of a mask structure from a file */
{
FILE *infile;
int ii;
infile = chkfopen(maskfilenm, "rb");
chkfread(&(obsmask->timesigma), sizeof(double), 1, infile);
chkfread(&(obsmask->freqsigma), sizeof(double), 1, infile);
chkfread(&(obsmask->mjd), sizeof(double), 1, infile);
chkfread(&(obsmask->dtint), sizeof(double), 1, infile);
chkfread(&(obsmask->lofreq), sizeof(double), 1, infile);
chkfread(&(obsmask->dfreq), sizeof(double), 1, infile);
chkfread(&(obsmask->numchan), sizeof(int), 1, infile);
chkfread(&(obsmask->numint), sizeof(int), 1, infile);
chkfread(&(obsmask->ptsperint), sizeof(int), 1, infile);
chkfread(&(obsmask->num_zap_chans), sizeof(int), 1, infile);
if (obsmask->num_zap_chans) {
obsmask->zap_chans = gen_ivect(obsmask->num_zap_chans);
chkfread(obsmask->zap_chans, sizeof(int), obsmask->num_zap_chans, infile);
}
chkfread(&(obsmask->num_zap_ints), sizeof(int), 1, infile);
if (obsmask->num_zap_ints) {
obsmask->zap_ints = gen_ivect(obsmask->num_zap_ints);
chkfread(obsmask->zap_ints, sizeof(int), obsmask->num_zap_ints, infile);
}
obsmask->num_chans_per_int = gen_ivect(obsmask->numint);
chkfread(obsmask->num_chans_per_int, sizeof(int), obsmask->numint, infile);
obsmask->chans = (int **) malloc(obsmask->numint * sizeof(int *));
for (ii = 0; ii < obsmask->numint; ii++) {
if (obsmask->num_chans_per_int[ii] > 0 &&
obsmask->num_chans_per_int[ii] < obsmask->numchan) {
obsmask->chans[ii] = gen_ivect(obsmask->num_chans_per_int[ii]);
chkfread(obsmask->chans[ii], sizeof(int),
obsmask->num_chans_per_int[ii], infile);
} else if (obsmask->num_chans_per_int[ii] == obsmask->numchan) {
int jj;
obsmask->chans[ii] = gen_ivect(obsmask->num_chans_per_int[ii]);
for (jj = 0; jj < obsmask->numchan; jj++)
obsmask->chans[ii][jj] = jj;
}
}
fclose(infile);
}
void dedisp_subbands(unsigned char *data, unsigned char *lastdata,
int numpts, int numchan, double *dispdelays,
int numsubbands, float *result)
/* De-disperse a stretch of data with numpts * numchan points */
/* into numsubbands subbands. Each time point for each subband */
/* is a float in the result array. The result array order is */
/* subbands of increasing frequency together at each time pt. */
/* The delays (in bins) are in dispdelays for each channel. */
/* The input data and dispdelays are always in ascending */
/* frequency order. Input data are ordered in time, with the */
/* channels stored together at each time point. */
{
static int approx_mean, firsttime = 1, *delays, chan_per_subband;
static float *lastoffsets, *offsets; //, *bandpass;
int ii, jj, kk, ll, mm, chan;
if (firsttime) {
delays = gen_ivect(numchan);
/* Prep offsets (if required) to subtract from the raw values */
read_offsets(&lastoffsets, &offsets, numpts, numchan);
for (ii = 0; ii < numchan; ii++) {
if (dispdelays[ii] < 0.0) {
printf("\ndispdelays[%d] = %f is < 0.0 in dedisp_subbands().\n\n",
ii, dispdelays[ii]);
exit(-1);
}
delays[ii] = (int) (dispdelays[ii] + 0.5);
}
chan_per_subband = numchan / numsubbands;
approx_mean = 0;
firsttime = 0;
}
/* Initialize the result array */
for (ii = 0; ii < numpts * numsubbands; ii++)
result[ii] = approx_mean;
/* Read offsets (if required) to subtract from the raw values */
read_offsets(&lastoffsets, &offsets, numpts, numchan);
/* De-disperse into the subbands */
for (ii = 0; ii < numsubbands; ii++) {
chan = ii * chan_per_subband;
for (jj = 0; jj < chan_per_subband; jj++, chan++) {
kk = chan + delays[chan] * numchan;
for (ll = 0; ll < numpts - delays[chan] ; ll++, kk += numchan)
result[ll * numsubbands + ii] +=
(lastdata[kk] - lastoffsets[delays[chan]+ll]);
kk = chan;
for (mm = 0 ; ll < numpts ; ll++, kk += numchan, mm++)
result[ll * numsubbands + ii] += (data[kk] - offsets[mm]);
}
}
}
void dedisp(unsigned char *data, unsigned char *lastdata, int numpts,
int numchan, double *dispdelays, float *result)
/* De-disperse a stretch of data with numpts * numchan points. */
/* The delays (in bins) are in dispdelays for each channel. */
/* The result is returned in result. The input data and */
/* dispdelays are always in ascending frequency order. */
/* Input data are ordered in time, with the channels stored */
/* together at each time point. */
{
static int approx_mean, firsttime = 1, *delays;
static float *lastoffsets, *offsets; //, *bandpass;
int ii, jj, kk, ll;
if (firsttime) {
delays = gen_ivect(numchan);
/* Prep offsets (if required) to subtract from the raw values */
read_offsets(&lastoffsets, &offsets, numpts, numchan);
for (ii = 0; ii < numchan; ii++) {
if (dispdelays[ii] < 0.0) {
printf("\ndispdelays[%d] = %f is < 0.0 in dedisp().\n\n",
ii, dispdelays[ii]);
exit(-1);
}
delays[ii] = (int) (dispdelays[ii] + 0.5);
}
approx_mean = 0;
firsttime = 0;
}
/* Initialize the result array */
for (ii = 0; ii < numpts; ii++)
result[ii] = approx_mean;
/* Read offsets (if required) to subtract from the raw values */
read_offsets(&lastoffsets, &offsets, numpts, numchan);
/* De-disperse */
for (ii = 0; ii < numchan; ii++) {
jj = ii + delays[ii] * numchan;
for (kk = 0; kk < numpts - delays[ii]; kk++, jj += numchan)
result[kk] += (lastdata[jj] - lastoffsets[delays[ii]+kk]);
//result[kk] += (lastdata[jj] - lastoffsets[delays[ii]+kk]*bandpass[ii]);
jj = ii;
for (ll = 0 ; kk < numpts; kk++, jj += numchan, ll++)
result[kk] += (data[jj] - offsets[ll]);
//result[kk] += (data[jj] - offsets[ll]*bandpass[ii]);
}
}
int *get_hdr_int_arr(struct HEADERP *h, char *name, int *len)
{
struct HEADERVAL val;
int ii, *iptr, *iarr;
if (find_hdrval(h, name, &val)) {
printf("ERROR: Can't find '%s' in the WAPP header!\n", name);
exit(0);
}
iptr = (int *) val.value;
*len = val.key->alen;
/* Note that this needs to be freed! */
iarr = gen_ivect(*len);
for (ii = 0; ii < *len; ii++, iptr++) {
iarr[ii] = *iptr;
}
return iarr;
}
void correct_subbands_for_DM(double dm, prepfoldinfo * search,
double *ddprofs, foldstats * ddstats)
/* Calculate the DM delays and apply them to the subbands */
/* to create de-dispersed profiles. */
{
int ii, *dmdelays;
double *subbanddelays, hif, dopplerhif, hifdelay, rdphase;
rdphase = search->fold.p1 * search->proflen;
hif = search->lofreq + (search->numchan - 1.0) * search->chan_wid;
dopplerhif = doppler(hif, search->avgvoverc);
hifdelay = delay_from_dm(dm, dopplerhif);
subbanddelays = subband_delays(search->numchan, search->nsub, dm,
search->lofreq, search->chan_wid,
search->avgvoverc);
dmdelays = gen_ivect(search->nsub);
for (ii = 0; ii < search->nsub; ii++)
dmdelays[ii] =
NEAREST_INT((subbanddelays[ii] - hifdelay) * rdphase) % search->proflen;
vect_free(subbanddelays);
combine_subbands(search->rawfolds, search->stats, search->npart,
search->nsub, search->proflen, dmdelays, ddprofs, ddstats);
vect_free(dmdelays);
}
void read_PSRFITS_files(struct spectra_info *s)
// Read and convert PSRFITS information from a group of files
// and place the resulting info into a spectra_info structure.
{
int IMJD, SMJD, itmp, ii, status = 0;
double OFFS, dtmp;
long double MJDf;
char ctmp[80], comment[120];
s->datatype = PSRFITS;
s->fitsfiles = (fitsfile **)malloc(sizeof(fitsfile *) * s->num_files);
s->start_subint = gen_ivect(s->num_files);
s->num_subint = gen_ivect(s->num_files);
s->start_spec = (long long *)malloc(sizeof(long long) * s->num_files);
s->num_spec = (long long *)malloc(sizeof(long long) * s->num_files);
s->num_pad = (long long *)malloc(sizeof(long long) * s->num_files);
s->start_MJD = (long double *)malloc(sizeof(long double) * s->num_files);
s->N = 0;
s->num_beams = 1;
s->get_rawblock = &get_PSRFITS_rawblock;
s->offset_to_spectra = &offset_to_PSRFITS_spectra;
// By default, don't flip the band. But don't change
// the input value if it is aleady set to flip the band always
if (s->apply_flipband==-1) s->apply_flipband = 0;
// Step through the other files
for (ii = 0 ; ii < s->num_files ; ii++) {
// Is the file a PSRFITS file?
if (!is_PSRFITS(s->filenames[ii])) {
fprintf(stderr,
"\nError! File '%s' does not appear to be PSRFITS!\n",
s->filenames[ii]);
exit(1);
}
// Open the PSRFITS file
fits_open_file(&(s->fitsfiles[ii]), s->filenames[ii], READONLY, &status);
// Is the data in search mode?
fits_read_key(s->fitsfiles[ii], TSTRING, "OBS_MODE", ctmp, comment, &status);
// Quick fix for Parkes DFB data (SRCH? why????)...
if (strcmp("SRCH", ctmp)==0) {
strncpy(ctmp, "SEARCH", 40);
}
if (strcmp(ctmp, "SEARCH")) {
fprintf(stderr,
"\nError! File '%s' does not contain SEARCH-mode data!\n",
s->filenames[ii]);
exit(1);
}
// Now get the stuff we need from the primary HDU header
fits_read_key(s->fitsfiles[ii], TSTRING, "TELESCOP", ctmp, comment, &status); \
// Quick fix for MockSpec data...
if (strcmp("ARECIBO 305m", ctmp)==0) {
strncpy(ctmp, "Arecibo", 40);
}
// Quick fix for Parkes DFB data...
{
char newctmp[80];
// Copy ctmp first since strlower() is in-place
strcpy(newctmp, ctmp);
if (strcmp("parkes", strlower(remove_whitespace(newctmp)))==0) {
strncpy(ctmp, "Parkes", 40);
}
}
if (status) {
printf("Error %d reading key %s\n", status, "TELESCOP");
if (ii==0) s->telescope[0]='\0';
if (status==KEY_NO_EXIST) status=0;
} else {
if (ii==0) strncpy(s->telescope, ctmp, 40);
else if (strcmp(s->telescope, ctmp)!=0)
printf("Warning!: %s values don't match for files 0 and %d!\n",
"TELESCOP", ii);
}
get_hdr_string("OBSERVER", s->observer);
get_hdr_string("SRC_NAME", s->source);
get_hdr_string("FRONTEND", s->frontend);
get_hdr_string("BACKEND", s->backend);
get_hdr_string("PROJID", s->project_id);
get_hdr_string("DATE-OBS", s->date_obs);
get_hdr_string("FD_POLN", s->poln_type);
get_hdr_string("RA", s->ra_str);
get_hdr_string("DEC", s->dec_str);
get_hdr_double("OBSFREQ", s->fctr);
get_hdr_int("OBSNCHAN", s->orig_num_chan);
get_hdr_double("OBSBW", s->orig_df);
//get_hdr_double("CHAN_DM", s->chan_dm);
get_hdr_double("BMIN", s->beam_FWHM);
/* This is likely not in earlier versions of PSRFITS so */
/* treat it a bit differently */
fits_read_key(s->fitsfiles[ii], TDOUBLE, "CHAN_DM",
&(s->chan_dm), comment, &status);
if (status==KEY_NO_EXIST) {
status = 0;
s->chan_dm = 0.0;
}
// Don't use the macros unless you are using the struct!
fits_read_key(s->fitsfiles[ii], TINT, "STT_IMJD", &IMJD, comment, &status);
s->start_MJD[ii] = (long double) IMJD;
fits_read_key(s->fitsfiles[ii], TINT, "STT_SMJD", &SMJD, comment, &status);
fits_read_key(s->fitsfiles[ii], TDOUBLE, "STT_OFFS", &OFFS, comment, &status);
s->start_MJD[ii] += ((long double) SMJD + (long double) OFFS) / SECPERDAY;
// Are we tracking?
fits_read_key(s->fitsfiles[ii], TSTRING, "TRK_MODE", ctmp, comment, &status);
itmp = (strcmp("TRACK", ctmp)==0) ? 1 : 0;
if (ii==0) s->tracking = itmp;
else if (s->tracking != itmp)
printf("Warning!: TRK_MODE values don't match for files 0 and %d!\n", ii);
// Now switch to the SUBINT HDU header
fits_movnam_hdu(s->fitsfiles[ii], BINARY_TBL, "SUBINT", 0, &status);
get_hdr_double("TBIN", s->dt);
get_hdr_int("NCHAN", s->num_channels);
get_hdr_int("NPOL", s->num_polns);
get_hdr_string("POL_TYPE", s->poln_order);
fits_read_key(s->fitsfiles[ii], TINT, "NCHNOFFS", &itmp, comment, &status);
if (itmp > 0)
printf("Warning!: First freq channel is not 0 in file %d!\n", ii);
get_hdr_int("NSBLK", s->spectra_per_subint);
get_hdr_int("NBITS", s->bits_per_sample);
fits_read_key(s->fitsfiles[ii], TINT, "NAXIS2",
&(s->num_subint[ii]), comment, &status);
fits_read_key(s->fitsfiles[ii], TINT, "NSUBOFFS",
&(s->start_subint[ii]), comment, &status);
s->time_per_subint = s->dt * s->spectra_per_subint;
/* This is likely not in earlier versions of PSRFITS so */
/* treat it a bit differently */
fits_read_key(s->fitsfiles[ii], TFLOAT, "ZERO_OFF",
&(s->zero_offset), comment, &status);
if (status==KEY_NO_EXIST) {
status = 0;
s->zero_offset = 0.0;
}
s->zero_offset = fabs(s->zero_offset);
// Get the time offset column info and the offset for the 1st row
{
double offs_sub;
int colnum, anynull, numrows;
// Identify the OFFS_SUB column number
fits_get_colnum(s->fitsfiles[ii], 0, "OFFS_SUB", &colnum, &status);
if (status==COL_NOT_FOUND) {
printf("Warning!: Can't find the OFFS_SUB column!\n");
status = 0; // Reset status
} else {
if (ii==0) {
s->offs_sub_col = colnum;
} else if (colnum != s->offs_sub_col) {
printf("Warning!: OFFS_SUB column changes between files!\n");
}
}
// Read the OFFS_SUB column value for the 1st row
fits_read_col(s->fitsfiles[ii], TDOUBLE,
s->offs_sub_col, 1L, 1L, 1L,
0, &offs_sub, &anynull, &status);
numrows = (int)((offs_sub - 0.5 * s->time_per_subint) /
s->time_per_subint + 1e-7);
// Check to see if any rows have been deleted or are missing
if (numrows > s->start_subint[ii]) {
printf("Warning: NSUBOFFS reports %d previous rows\n"
" but OFFS_SUB implies %d. Using OFFS_SUB.\n"
" Will likely be able to correct for this.\n",
s->start_subint[ii], numrows);
}
s->start_subint[ii] = numrows;
}
// This is the MJD offset based on the starting subint number
MJDf = (s->time_per_subint * s->start_subint[ii]) / SECPERDAY;
// The start_MJD values should always be correct
s->start_MJD[ii] += MJDf;
// Compute the starting spectra from the times
MJDf = s->start_MJD[ii] - s->start_MJD[0];
if (MJDf < 0.0) {
fprintf(stderr, "Error!: File %d seems to be from before file 0!\n", ii);
exit(1);
}
s->start_spec[ii] = (long long)(MJDf * SECPERDAY / s->dt + 0.5);
// Now pull stuff from the other columns
{
float ftmp;
long repeat, width;
int colnum, anynull;
// Identify the data column and the data type
fits_get_colnum(s->fitsfiles[ii], 0, "DATA", &colnum, &status);
if (status==COL_NOT_FOUND) {
printf("Warning!: Can't find the DATA column!\n");
status = 0; // Reset status
} else {
if (ii==0) {
s->data_col = colnum;
fits_get_coltype(s->fitsfiles[ii], colnum, &(s->FITS_typecode),
&repeat, &width, &status);
} else if (colnum != s->data_col) {
printf("Warning!: DATA column changes between files!\n");
}
}
// Telescope azimuth
fits_get_colnum(s->fitsfiles[ii], 0, "TEL_AZ", &colnum, &status);
if (status==COL_NOT_FOUND) {
s->azimuth = 0.0;
status = 0; // Reset status
} else {
fits_read_col(s->fitsfiles[ii], TFLOAT, colnum,
1L, 1L, 1L, 0, &ftmp, &anynull, &status);
if (ii==0) s->azimuth = (double) ftmp;
}
// Telescope zenith angle
fits_get_colnum(s->fitsfiles[ii], 0, "TEL_ZEN", &colnum, &status);
if (status==COL_NOT_FOUND) {
s->zenith_ang = 0.0;
status = 0; // Reset status
} else {
fits_read_col(s->fitsfiles[ii], TFLOAT, colnum,
1L, 1L, 1L, 0, &ftmp, &anynull, &status);
if (ii==0) s->zenith_ang = (double) ftmp;
}
// Observing frequencies
fits_get_colnum(s->fitsfiles[ii], 0, "DAT_FREQ", &colnum, &status);
if (status==COL_NOT_FOUND) {
printf("Warning!: Can't find the channel freq column!\n");
status = 0; // Reset status
} else {
int jj;
float *freqs = (float *)malloc(sizeof(float) * s->num_channels);
fits_read_col(s->fitsfiles[ii], TFLOAT, colnum, 1L, 1L,
s->num_channels, 0, freqs, &anynull, &status);
if (ii==0) {
s->df = freqs[1]-freqs[0];
s->lo_freq = freqs[0];
s->hi_freq = freqs[s->num_channels-1];
// Now check that the channel spacing is the same throughout
for (jj = 0 ; jj < s->num_channels - 1 ; jj++) {
ftmp = freqs[jj+1] - freqs[jj];
if (fabs(ftmp - s->df) > 1e-7)
printf("Warning!: Channel spacing changes in file %d!\n", ii);
}
} else {
ftmp = fabs(s->df-(freqs[1]-freqs[0]));
if (ftmp > 1e-7)
printf("Warning!: Channel spacing changes between files!\n");
ftmp = fabs(s->lo_freq-freqs[0]);
if (ftmp > 1e-7)
printf("Warning!: Low channel changes between files!\n");
ftmp = fabs(s->hi_freq-freqs[s->num_channels-1]);
if (ftmp > 1e-7)
printf("Warning!: High channel changes between files!\n");
}
free(freqs);
}
// Data weights
fits_get_colnum(s->fitsfiles[ii], 0, "DAT_WTS", &colnum, &status);
if (status==COL_NOT_FOUND) {
printf("Warning!: Can't find the channel weights!\n");
status = 0; // Reset status
} else {
if (s->apply_weight < 0) { // Use the data to decide
int jj;
if (ii==0) {
s->dat_wts_col = colnum;
} else if (colnum != s->dat_wts_col) {
printf("Warning!: DAT_WTS column changes between files!\n");
}
float *fvec = (float *)malloc(sizeof(float) * s->num_channels);
fits_read_col(s->fitsfiles[ii], TFLOAT, s->dat_wts_col, 1L, 1L,
s->num_channels, 0, fvec, &anynull, &status);
for (jj = 0 ; jj < s->num_channels ; jj++) {
// If the weights are not 1, apply them
if (fvec[jj] != 1.0) {
s->apply_weight = 1;
break;
}
}
free(fvec);
}
if (s->apply_weight < 0) s->apply_weight = 0; // not needed
}
// Data offsets
fits_get_colnum(s->fitsfiles[ii], 0, "DAT_OFFS", &colnum, &status);
if (status==COL_NOT_FOUND) {
printf("Warning!: Can't find the channel offsets!\n");
status = 0; // Reset status
} else {
if (s->apply_offset < 0) { // Use the data to decide
int jj;
if (ii==0) {
s->dat_offs_col = colnum;
} else if (colnum != s->dat_offs_col) {
printf("Warning!: DAT_OFFS column changes between files!\n");
}
float *fvec = (float *)malloc(sizeof(float) *
s->num_channels * s->num_polns);
fits_read_col(s->fitsfiles[ii], TFLOAT, s->dat_offs_col, 1L, 1L,
s->num_channels * s->num_polns,
0, fvec, &anynull, &status);
for (jj = 0 ; jj < s->num_channels * s->num_polns ; jj++) {
// If the offsets are not 0, apply them
if (fvec[jj] != 0.0) {
s->apply_offset = 1;
break;
}
}
free(fvec);
}
if (s->apply_offset < 0) s->apply_offset = 0; // not needed
}
// Data scalings
fits_get_colnum(s->fitsfiles[ii], 0, "DAT_SCL", &colnum, &status);
if (status==COL_NOT_FOUND) {
printf("Warning!: Can't find the channel scalings!\n");
status = 0; // Reset status
} else {
if (s->apply_scale < 0) { // Use the data to decide
int jj;
if (ii==0) {
s->dat_scl_col = colnum;
} else if (colnum != s->dat_scl_col) {
printf("Warning!: DAT_SCL column changes between files!\n");
}
float *fvec = (float *)malloc(sizeof(float) *
s->num_channels * s->num_polns);
fits_read_col(s->fitsfiles[ii], TFLOAT, colnum, 1L, 1L,
s->num_channels * s->num_polns,
0, fvec, &anynull, &status);
for (jj = 0 ; jj < s->num_channels * s->num_polns ; jj++) {
// If the scales are not 1, apply them
if (fvec[jj] != 1.0) {
s->apply_scale = 1;
break;
}
}
free(fvec);
}
if (s->apply_scale < 0) s->apply_scale = 0; // not needed
}
}
// Compute the samples per file and the amount of padding
// that the _previous_ file has
s->num_pad[ii] = 0;
s->num_spec[ii] = s->spectra_per_subint * s->num_subint[ii];
if (ii > 0) {
if (s->start_spec[ii] > s->N) { // Need padding
s->num_pad[ii-1] = s->start_spec[ii] - s->N;
s->N += s->num_pad[ii-1];
}
}
s->N += s->num_spec[ii];
}
// Convert the position strings into degrees
{
int d, h, m;
double sec;
ra_dec_from_string(s->ra_str, &h, &m, &sec);
s->ra2000 = hms2rad(h, m, sec) * RADTODEG;
ra_dec_from_string(s->dec_str, &d, &m, &sec);
s->dec2000 = dms2rad(d, m, sec) * RADTODEG;
}
// Are the polarizations summed?
if ((strncmp("AA+BB", s->poln_order, 5)==0) ||
(strncmp("INTEN", s->poln_order, 5)==0))
s->summed_polns = 1;
else
s->summed_polns = 0;
// Calculate some others
s->T = s->N * s->dt;
s->orig_df /= (double) s->orig_num_chan;
s->samples_per_spectra = s->num_polns * s->num_channels;
// Note: the following is the number of bytes that will be in
// the returned array from CFITSIO.
// CFITSIO turns bits into bytes when FITS_typecode=1
// and we turn 2-bits or 4-bits into bytes if bits_per_sample < 8
if (s->bits_per_sample < 8)
s->bytes_per_spectra = s->samples_per_spectra;
else
s->bytes_per_spectra = (s->bits_per_sample * s->samples_per_spectra) / 8;
s->samples_per_subint = s->samples_per_spectra * s->spectra_per_subint;
s->bytes_per_subint = s->bytes_per_spectra * s->spectra_per_subint;
// Flip the band?
if (s->hi_freq < s->lo_freq) {
float ftmp = s->hi_freq;
s->hi_freq = s->lo_freq;
s->lo_freq = ftmp;
s->df *= -1.0;
s->apply_flipband = 1;
}
// Compute the bandwidth
s->BW = s->num_channels * s->df;
// Flip the bytes for Parkes FB_1BIT data
if (s->bits_per_sample==1 &&
strcmp(s->telescope, "Parkes")==0 &&
strcmp(s->backend, "FB_1BIT")==0) {
printf("Flipping bit ordering since Parkes FB_1BIT data.\n");
s->flip_bytes = 1;
} else {
s->flip_bytes = 0;
}
// Allocate the buffers
cdatabuffer = gen_bvect(s->bytes_per_subint);
// Following is twice as big because we use it as a ringbuffer too
fdatabuffer = gen_fvect(2 * s->spectra_per_subint * s->num_channels);
s->padvals = gen_fvect(s->num_channels);
for (ii = 0 ; ii < s->num_channels ; ii++)
s->padvals[ii] = 0.0;
offsets = gen_fvect(s->num_channels * s->num_polns);
scales = gen_fvect(s->num_channels * s->num_polns);
weights = gen_fvect(s->num_channels);
// Initialize these if we won't be reading them from the file
if (s->apply_offset==0)
for (ii = 0 ; ii < s->num_channels * s->num_polns ; ii++)
offsets[ii] = 0.0;
if (s->apply_scale==0)
for (ii = 0 ; ii < s->num_channels * s->num_polns ; ii++)
scales[ii] = 1.0;
if (s->apply_weight==0)
for (ii = 0 ; ii < s->num_channels ; ii++)
weights[ii] = 1.0;
}
void broadcast_mask(mask * obsmask, int myid)
{
int ii;
maskbase mbase;
if (myid == 0) {
mbase.timesigma = obsmask->timesigma;
mbase.freqsigma = obsmask->freqsigma;
mbase.mjd = obsmask->mjd;
mbase.dtint = obsmask->dtint;
mbase.lofreq = obsmask->lofreq;
mbase.dfreq = obsmask->dfreq;
mbase.numchan = obsmask->numchan;
mbase.numint = obsmask->numint;
mbase.ptsperint = obsmask->ptsperint;
mbase.num_zap_chans = obsmask->num_zap_chans;
mbase.num_zap_ints = obsmask->num_zap_ints;
}
MPI_Bcast(&mbase, 1, maskbase_type, 0, MPI_COMM_WORLD);
if (myid > 0) {
obsmask->timesigma = mbase.timesigma;
obsmask->freqsigma = mbase.freqsigma;
obsmask->mjd = mbase.mjd;
obsmask->dtint = mbase.dtint;
obsmask->lofreq = mbase.lofreq;
obsmask->dfreq = mbase.dfreq;
obsmask->numchan = mbase.numchan;
obsmask->numint = mbase.numint;
obsmask->ptsperint = mbase.ptsperint;
obsmask->num_zap_chans = mbase.num_zap_chans;
obsmask->num_zap_ints = mbase.num_zap_ints;
obsmask->zap_chans = gen_ivect(mbase.num_zap_chans);
obsmask->zap_ints = gen_ivect(mbase.num_zap_ints);
obsmask->num_chans_per_int = gen_ivect(mbase.numint);
obsmask->chans = (int **) malloc(mbase.numint * sizeof(int *));
}
MPI_Bcast(obsmask->zap_chans, mbase.num_zap_chans, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(obsmask->zap_ints, mbase.num_zap_ints, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(obsmask->num_chans_per_int, mbase.numint, MPI_INT, 0, MPI_COMM_WORLD);
for (ii = 0; ii < mbase.numint; ii++) {
if (obsmask->num_chans_per_int[ii] > 0 &&
obsmask->num_chans_per_int[ii] < obsmask->numchan) {
if (myid > 0)
obsmask->chans[ii] = gen_ivect(obsmask->num_chans_per_int[ii]);
MPI_Bcast(obsmask->chans[ii], obsmask->num_chans_per_int[ii],
MPI_INT, 0, MPI_COMM_WORLD);
//int jj;
//printf("%d: ", myid);
//for (jj=0; jj<obsmask->num_chans_per_int[ii]; jj++)
// printf("%d ", obsmask->chans[ii][jj]);
//printf("\n");
} else if (obsmask->num_chans_per_int[ii] == obsmask->numchan) {
int jj;
//printf("%d: all zapped\n", myid);
if (myid > 0) {
obsmask->chans[ii] = gen_ivect(obsmask->num_chans_per_int[ii]);
for (jj = 0; jj < obsmask->numchan; jj++)
obsmask->chans[ii][jj] = jj;
}
}
}
}
int main(int argc, char *argv[])
{
FILE *bytemaskfile;
float **dataavg = NULL, **datastd = NULL, **datapow = NULL;
float *chandata = NULL, powavg, powstd, powmax;
float inttime, norm, fracterror = RFI_FRACTERROR;
float *rawdata = NULL;
unsigned char **bytemask = NULL;
short *srawdata = NULL;
char *outfilenm, *statsfilenm, *maskfilenm;
char *bytemaskfilenm, *rfifilenm;
int numchan = 0, numint = 0, newper = 0, oldper = 0, good_padvals = 0;
int blocksperint, ptsperint = 0, ptsperblock = 0, padding = 0;
int numcands, candnum, numrfi = 0, numrfivect = NUM_RFI_VECT;
int ii, jj, kk, slen, numread = 0, insubs = 0;
int harmsum = RFI_NUMHARMSUM, lobin = RFI_LOBIN, numbetween = RFI_NUMBETWEEN;
double davg, dvar, freq;
struct spectra_info s;
presto_interptype interptype;
rfi *rfivect = NULL;
mask oldmask, newmask;
fftcand *cands;
infodata idata;
Cmdline *cmd;
/* Call usage() if we have no command line arguments */
if (argc == 1) {
Program = argv[0];
printf("\n");
usage();
exit(0);
}
/* Parse the command line using the excellent program Clig */
cmd = parseCmdline(argc, argv);
spectra_info_set_defaults(&s);
s.filenames = cmd->argv;
s.num_files = cmd->argc;
s.clip_sigma = cmd->clip;
// -1 causes the data to determine if we use weights, scales, &
// offsets for PSRFITS or flip the band for any data type where
// we can figure that out with the data
s.apply_flipband = (cmd->invertP) ? 1 : -1;
s.apply_weight = (cmd->noweightsP) ? 0 : -1;
s.apply_scale = (cmd->noscalesP) ? 0 : -1;
s.apply_offset = (cmd->nooffsetsP) ? 0 : -1;
s.remove_zerodm = (cmd->zerodmP) ? 1 : 0;
if (cmd->noclipP) {
cmd->clip = 0.0;
s.clip_sigma = 0.0;
}
if (cmd->ifsP) {
// 0 = default or summed, 1-4 are possible also
s.use_poln = cmd->ifs;
}
slen = strlen(cmd->outfile) + 20;
#ifdef DEBUG
showOptionValues();
#endif
printf("\n\n");
printf(" Pulsar Data RFI Finder\n");
printf(" by Scott M. Ransom\n\n");
/* The following is the root of all the output files */
outfilenm = (char *) calloc(slen, sizeof(char));
sprintf(outfilenm, "%s_rfifind", cmd->outfile);
/* And here are the output file names */
maskfilenm = (char *) calloc(slen, sizeof(char));
sprintf(maskfilenm, "%s.mask", outfilenm);
bytemaskfilenm = (char *) calloc(slen, sizeof(char));
sprintf(bytemaskfilenm, "%s.bytemask", outfilenm);
rfifilenm = (char *) calloc(slen, sizeof(char));
sprintf(rfifilenm, "%s.rfi", outfilenm);
statsfilenm = (char *) calloc(slen, sizeof(char));
sprintf(statsfilenm, "%s.stats", outfilenm);
sprintf(idata.name, "%s", outfilenm);
if (RAWDATA) {
if (cmd->filterbankP) s.datatype = SIGPROCFB;
else if (cmd->psrfitsP) s.datatype = PSRFITS;
else if (cmd->pkmbP) s.datatype = SCAMP;
else if (cmd->bcpmP) s.datatype = BPP;
else if (cmd->wappP) s.datatype = WAPP;
else if (cmd->spigotP) s.datatype = SPIGOT;
} else { // Attempt to auto-identify the data
identify_psrdatatype(&s, 1);
if (s.datatype==SIGPROCFB) cmd->filterbankP = 1;
else if (s.datatype==PSRFITS) cmd->psrfitsP = 1;
else if (s.datatype==SCAMP) cmd->pkmbP = 1;
else if (s.datatype==BPP) cmd->bcpmP = 1;
else if (s.datatype==WAPP) cmd->wappP = 1;
else if (s.datatype==SPIGOT) cmd->spigotP = 1;
else if (s.datatype==SUBBAND) insubs = 1;
else {
printf("Error: Unable to identify input data files. Please specify type.\n\n");
exit(1);
}
}
if (!cmd->nocomputeP) {
if (RAWDATA || insubs) {
char description[40];
psrdatatype_description(description, s.datatype);
if (s.num_files > 1)
printf("Reading %s data from %d files:\n", description, s.num_files);
else
printf("Reading %s data from 1 file:\n", description);
if (insubs) s.files = (FILE **)malloc(sizeof(FILE *) * s.num_files);
for (ii = 0; ii < s.num_files; ii++) {
printf(" '%s'\n", cmd->argv[ii]);
if (insubs) s.files[ii] = chkfopen(cmd->argv[ii], "rb");
}
printf("\n");
}
if (RAWDATA) {
read_rawdata_files(&s);
print_spectra_info_summary(&s);
spectra_info_to_inf(&s, &idata);
ptsperblock = s.spectra_per_subint;
numchan = s.num_channels;
idata.dm = 0.0;
}
if (insubs) {
/* Set-up values if we are using subbands */
char *tmpname, *root, *suffix;
if (split_root_suffix(s.filenames[0], &root, &suffix) == 0) {
printf("Error: The input filename (%s) must have a suffix!\n\n", s.filenames[0]);
exit(1);
}
if (strncmp(suffix, "sub", 3) == 0) {
tmpname = calloc(strlen(root) + 6, 1);
sprintf(tmpname, "%s.sub", root);
readinf(&idata, tmpname);
free(tmpname);
} else {
printf("\nThe input files (%s) must be subbands! (i.e. *.sub##)\n\n",
s.filenames[0]);
exit(1);
}
free(root);
free(suffix);
ptsperblock = 1;
/* Compensate for the fact that we have subbands and not channels */
idata.freq = idata.freq - 0.5 * idata.chan_wid +
0.5 * idata.chan_wid * (idata.num_chan / s.num_files);
idata.chan_wid = idata.num_chan / s.num_files * idata.chan_wid;
idata.num_chan = numchan = s.num_files;
idata.dm = 0.0;
sprintf(idata.name, "%s", outfilenm);
writeinf(&idata);
s.padvals = gen_fvect(s.num_files);
for (ii = 0 ; ii < s.num_files ; ii++)
s.padvals[ii] = 0.0;
}
/* Read an input mask if wanted */
if (cmd->maskfileP) {
read_mask(cmd->maskfile, &oldmask);
printf("Read old mask information from '%s'\n\n", cmd->maskfile);
good_padvals = determine_padvals(cmd->maskfile, &oldmask, s.padvals);
} else {
oldmask.numchan = oldmask.numint = 0;
}
/* The number of data points and blocks to work with at a time */
if (cmd->blocksP) {
blocksperint = cmd->blocks;
cmd->time = blocksperint * ptsperblock * idata.dt;
} else {
blocksperint = (int) (cmd->time / (ptsperblock * idata.dt) + 0.5);
}
ptsperint = blocksperint * ptsperblock;
numint = (long long) idata.N / ptsperint;
if ((long long) idata.N % ptsperint)
numint++;
inttime = ptsperint * idata.dt;
printf("Analyzing data sections of length %d points (%.6g sec).\n",
ptsperint, inttime);
{
int *factors, numfactors;
factors = get_prime_factors(ptsperint, &numfactors);
printf(" Prime factors are: ");
for (ii = 0; ii < numfactors; ii++)
printf("%d ", factors[ii]);
printf("\n");
if (factors[numfactors - 1] > 13) {
printf(" WARNING: The largest prime factor is pretty big! This will\n"
" cause the FFTs to take a long time to compute. I\n"
" recommend choosing a different -time value.\n");
}
printf("\n");
free(factors);
}
/* Allocate our workarrays */
if (RAWDATA)
rawdata = gen_fvect(idata.num_chan * ptsperblock * blocksperint);
else if (insubs)
srawdata = gen_svect(idata.num_chan * ptsperblock * blocksperint);
dataavg = gen_fmatrix(numint, numchan);
datastd = gen_fmatrix(numint, numchan);
datapow = gen_fmatrix(numint, numchan);
chandata = gen_fvect(ptsperint);
bytemask = gen_bmatrix(numint, numchan);
for (ii = 0; ii < numint; ii++)
for (jj = 0; jj < numchan; jj++)
bytemask[ii][jj] = GOODDATA;
rfivect = rfi_vector(rfivect, numchan, numint, 0, numrfivect);
if (numbetween == 2)
interptype = INTERBIN;
else
interptype = INTERPOLATE;
/* Main loop */
printf("Writing mask data to '%s'.\n", maskfilenm);
printf("Writing RFI data to '%s'.\n", rfifilenm);
printf("Writing statistics to '%s'.\n\n", statsfilenm);
printf("Massaging the data ...\n\n");
printf("Amount Complete = %3d%%", oldper);
fflush(stdout);
for (ii = 0; ii < numint; ii++) { /* Loop over the intervals */
newper = (int) ((float) ii / numint * 100.0 + 0.5);
if (newper > oldper) {
printf("\rAmount Complete = %3d%%", newper);
fflush(stdout);
oldper = newper;
}
/* Read a chunk of data */
if (RAWDATA)
numread = read_rawblocks(rawdata, blocksperint, &s, &padding);
else if (insubs)
numread = read_subband_rawblocks(s.files, s.num_files,
srawdata, blocksperint, &padding);
if (padding)
for (jj = 0; jj < numchan; jj++)
bytemask[ii][jj] |= PADDING;
for (jj = 0; jj < numchan; jj++) { /* Loop over the channels */
if (RAWDATA)
get_channel(chandata, jj, blocksperint, rawdata, &s);
else if (insubs)
get_subband(jj, chandata, srawdata, blocksperint);
/* Calculate the averages and standard deviations */
/* for each point in time. */
if (padding) {
dataavg[ii][jj] = 0.0;
datastd[ii][jj] = 0.0;
datapow[ii][jj] = 1.0;
} else {
avg_var(chandata, ptsperint, &davg, &dvar);
dataavg[ii][jj] = davg;
datastd[ii][jj] = sqrt(dvar);
realfft(chandata, ptsperint, -1);
numcands = 0;
norm = datastd[ii][jj] * datastd[ii][jj] * ptsperint;
if (norm == 0.0)
norm = (chandata[0] == 0.0) ? 1.0 : chandata[0];
cands = search_fft((fcomplex *) chandata, ptsperint / 2,
lobin, ptsperint / 2, harmsum,
numbetween, interptype, norm, cmd->freqsigma,
&numcands, &powavg, &powstd, &powmax);
datapow[ii][jj] = powmax;
/* Record the birdies */
if (numcands) {
for (kk = 0; kk < numcands; kk++) {
freq = cands[kk].r / inttime;
candnum = find_rfi(rfivect, numrfi, freq, RFI_FRACTERROR);
if (candnum >= 0) {
update_rfi(rfivect + candnum, freq, cands[kk].sig, jj, ii);
} else {
update_rfi(rfivect + numrfi, freq, cands[kk].sig, jj, ii);
numrfi++;
if (numrfi == numrfivect) {
numrfivect *= 2;
rfivect = rfi_vector(rfivect, numchan, numint,
numrfivect / 2, numrfivect);
}
}
}
free(cands);
}
}
}
}
printf("\rAmount Complete = 100%%\n");
/* Write the data to the output files */
write_rfifile(rfifilenm, rfivect, numrfi, numchan, numint,
ptsperint, lobin, numbetween, harmsum,
fracterror, cmd->freqsigma);
write_statsfile(statsfilenm, datapow[0], dataavg[0], datastd[0],
numchan, numint, ptsperint, lobin, numbetween);
} else { /* If "-nocompute" */
float freqsigma;
/* Read the data from the output files */
printf("Reading RFI data from '%s'.\n", rfifilenm);
printf("Reading statistics from '%s'.\n", statsfilenm);
readinf(&idata, outfilenm);
read_rfifile(rfifilenm, &rfivect, &numrfi, &numchan, &numint,
&ptsperint, &lobin, &numbetween, &harmsum,
&fracterror, &freqsigma);
numrfivect = numrfi;
read_statsfile(statsfilenm, &datapow, &dataavg, &datastd,
&numchan, &numint, &ptsperint, &lobin, &numbetween);
bytemask = gen_bmatrix(numint, numchan);
printf("Reading bytemask from '%s'.\n\n", bytemaskfilenm);
bytemaskfile = chkfopen(bytemaskfilenm, "rb");
chkfread(bytemask[0], numint * numchan, 1, bytemaskfile);
fclose(bytemaskfile);
for (ii = 0; ii < numint; ii++)
for (jj = 0; jj < numchan; jj++)
bytemask[ii][jj] &= PADDING; /* Clear all but the PADDING bits */
inttime = ptsperint * idata.dt;
}
/* Make the plots and set the mask */
{
int *zapints, *zapchan;
int numzapints = 0, numzapchan = 0;
if (cmd->zapintsstrP) {
zapints = ranges_to_ivect(cmd->zapintsstr, 0, numint - 1, &numzapints);
zapints = (int *) realloc(zapints, (size_t) (sizeof(int) * numint));
} else {
zapints = gen_ivect(numint);
}
if (cmd->zapchanstrP) {
zapchan = ranges_to_ivect(cmd->zapchanstr, 0, numchan - 1, &numzapchan);
zapchan = (int *) realloc(zapchan, (size_t) (sizeof(int) * numchan));
} else {
zapchan = gen_ivect(numchan);
}
rfifind_plot(numchan, numint, ptsperint, cmd->timesigma, cmd->freqsigma,
cmd->inttrigfrac, cmd->chantrigfrac,
dataavg, datastd, datapow, zapchan, numzapchan,
zapints, numzapints, &idata, bytemask,
&oldmask, &newmask, rfivect, numrfi,
cmd->rfixwinP, cmd->rfipsP, cmd->xwinP);
vect_free(zapints);
vect_free(zapchan);
}
/* Write the new mask and bytemask to the file */
write_mask(maskfilenm, &newmask);
bytemaskfile = chkfopen(bytemaskfilenm, "wb");
chkfwrite(bytemask[0], numint * numchan, 1, bytemaskfile);
fclose(bytemaskfile);
/* Determine the percent of good and bad data */
{
int numpad = 0, numbad = 0, numgood = 0;
for (ii = 0; ii < numint; ii++) {
for (jj = 0; jj < numchan; jj++) {
if (bytemask[ii][jj] == GOODDATA) {
numgood++;
} else {
if (bytemask[ii][jj] & PADDING)
numpad++;
else
numbad++;
}
}
}
printf("\nTotal number of intervals in the data: %d\n\n", numint * numchan);
printf(" Number of padded intervals: %7d (%6.3f%%)\n",
numpad, (float) numpad / (float) (numint * numchan) * 100.0);
printf(" Number of good intervals: %7d (%6.3f%%)\n",
numgood, (float) numgood / (float) (numint * numchan) * 100.0);
printf(" Number of bad intervals: %7d (%6.3f%%)\n\n",
numbad, (float) numbad / (float) (numint * numchan) * 100.0);
qsort(rfivect, numrfi, sizeof(rfi), compare_rfi_sigma);
printf(" Ten most significant birdies:\n");
printf("# Sigma Period(ms) Freq(Hz) Number \n");
printf("----------------------------------------------------\n");
for (ii = 0; ii < 10; ii++) {
double pperr;
char temp1[40], temp2[40];
if (rfivect[ii].freq_var == 0.0) {
pperr = 0.0;
sprintf(temp1, " %-14g", rfivect[ii].freq_avg);
sprintf(temp2, " %-14g", 1000.0 / rfivect[ii].freq_avg);
} else {
pperr = 1000.0 * sqrt(rfivect[ii].freq_var) /
(rfivect[ii].freq_avg * rfivect[ii].freq_avg);
nice_output_2(temp1, rfivect[ii].freq_avg, sqrt(rfivect[ii].freq_var),
-15);
nice_output_2(temp2, 1000.0 / rfivect[ii].freq_avg, pperr, -15);
}
printf("%-2d %-8.2f %13s %13s %-8d\n", ii + 1, rfivect[ii].sigma_avg,
temp2, temp1, rfivect[ii].numobs);
}
qsort(rfivect, numrfi, sizeof(rfi), compare_rfi_numobs);
printf("\n Ten most numerous birdies:\n");
printf("# Number Period(ms) Freq(Hz) Sigma \n");
printf("----------------------------------------------------\n");
for (ii = 0; ii < 10; ii++) {
double pperr;
char temp1[40], temp2[40];
if (rfivect[ii].freq_var == 0.0) {
pperr = 0.0;
sprintf(temp1, " %-14g", rfivect[ii].freq_avg);
sprintf(temp2, " %-14g", 1000.0 / rfivect[ii].freq_avg);
} else {
pperr = 1000.0 * sqrt(rfivect[ii].freq_var) /
(rfivect[ii].freq_avg * rfivect[ii].freq_avg);
nice_output_2(temp1, rfivect[ii].freq_avg, sqrt(rfivect[ii].freq_var),
-15);
nice_output_2(temp2, 1000.0 / rfivect[ii].freq_avg, pperr, -15);
}
printf("%-2d %-8d %13s %13s %-8.2f\n", ii + 1, rfivect[ii].numobs,
temp2, temp1, rfivect[ii].sigma_avg);
}
printf("\nDone.\n\n");
}
/* Close the files and cleanup */
free_rfi_vector(rfivect, numrfivect);
free_mask(newmask);
if (cmd->maskfileP)
free_mask(oldmask);
free(outfilenm);
free(statsfilenm);
free(bytemaskfilenm);
free(maskfilenm);
free(rfifilenm);
vect_free(dataavg[0]);
vect_free(dataavg);
vect_free(datastd[0]);
vect_free(datastd);
vect_free(datapow[0]);
vect_free(datapow);
vect_free(bytemask[0]);
vect_free(bytemask);
if (!cmd->nocomputeP) {
// Close all the raw files and free their vectors
close_rawfiles(&s);
vect_free(chandata);
if (insubs)
vect_free(srawdata);
else
vect_free(rawdata);
}
return (0);
}
static int get_data(float **outdata, int blocksperread,
struct spectra_info *s,
mask * obsmask, int *idispdts, int **offsets,
int *padding)
{
static int firsttime = 1, *maskchans = NULL, blocksize;
static int worklen, dsworklen;
static float *tempzz, *data1, *data2, *dsdata1 = NULL, *dsdata2 = NULL;
static float *currentdata, *lastdata, *currentdsdata, *lastdsdata;
static float *frawdata;
static double blockdt;
int totnumread = 0, gotblock = 0, numread = 0;
int ii, jj, tmppad = 0, nummasked = 0;
if (firsttime) {
if (cmd->maskfileP)
maskchans = gen_ivect(insubs ? s->num_files : s->num_channels);
worklen = s->spectra_per_subint * blocksperread;
dsworklen = worklen / cmd->downsamp;
blocksize = s->spectra_per_subint * cmd->nsub;
blockdt = s->spectra_per_subint * s->dt;
if (RAWDATA) {
frawdata = gen_fvect(2 * s->num_channels * s->spectra_per_subint);
// To initialize the data reading and prep_subbands routines
firsttime = 2;
}
data1 = gen_fvect(cmd->nsub * worklen);
data2 = gen_fvect(cmd->nsub * worklen);
currentdata = data1;
lastdata = data2;
if (cmd->downsamp > 1) {
dsdata1 = gen_fvect(cmd->nsub * dsworklen);
dsdata2 = gen_fvect(cmd->nsub * dsworklen);
currentdsdata = dsdata1;
lastdsdata = dsdata2;
} else {
currentdsdata = data1;
lastdsdata = data2;
}
{ // Make sure that our working blocks are long enough...
int testlen = insubs ? s->num_files : s->num_channels;
for (ii = 0; ii < testlen; ii++) {
if (idispdts[ii] > worklen)
printf("WARNING! (myid = %d): (idispdts[%d] = %d) > (worklen = %d)\n",
myid, ii, idispdts[ii], worklen);
}
for (ii = 0; ii < local_numdms; ii++) {
for (jj = 0; jj < cmd->nsub; jj++) {
if (offsets[ii][jj] > dsworklen)
printf("WARNING! (myid = %d): (offsets[%d][%d] = %d) > (dsworklen = %d)\n",
myid, ii, jj, offsets[ii][jj], dsworklen);
}
}
}
}
while (1) {
if (RAWDATA) {
for (ii = 0; ii < blocksperread; ii++) {
if (myid == 0) {
gotblock = s->get_rawblock(frawdata, s, &tmppad);
if (gotblock && !firsttime) totnumread += s->spectra_per_subint;
}
MPI_Bcast(&gotblock, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&tmppad, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(frawdata, s->num_channels * s->spectra_per_subint,
MPI_FLOAT, 0, MPI_COMM_WORLD);
if (myid > 0) {
if (gotblock) {
numread =
prep_subbands(currentdata + ii * blocksize,
frawdata, idispdts, cmd->nsub, s,
0, maskchans, &nummasked, obsmask);
if (!firsttime)
totnumread += numread;
} else {
*padding = 1;
for (jj = ii * blocksize; jj < (ii + 1) * blocksize; jj++)
currentdata[jj] = 0.0;
}
if (tmppad)
*padding = 1;
}
}
} else if (insubs) {
short *subsdata = NULL;
subsdata = gen_svect(SUBSBLOCKLEN * s->num_files);
for (ii = 0; ii < blocksperread; ii++) {
if (myid == 0)
numread = simple_read_subbands(s->files, s->num_files, subsdata);
MPI_Bcast(&numread, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(subsdata, SUBSBLOCKLEN * s->num_files, MPI_SHORT, 0,
MPI_COMM_WORLD);
convert_subbands(s->num_files, subsdata,
currentdata + ii * blocksize, blockdt,
maskchans, &nummasked, obsmask,
cmd->clip, s->padvals);
if (!firsttime)
totnumread += numread;
}
vect_free(subsdata);
}
/* Downsample the subband data if needed */
if (myid > 0) {
if (cmd->downsamp > 1) {
int kk, offset, dsoffset, index, dsindex;
float ftmp;
for (ii = 0; ii < dsworklen; ii++) {
dsoffset = ii * cmd->nsub;
offset = dsoffset * cmd->downsamp;
for (jj = 0; jj < cmd->nsub; jj++) {
dsindex = dsoffset + jj;
index = offset + jj;
currentdsdata[dsindex] = 0.0;
for (kk = 0, ftmp = 0.0; kk < cmd->downsamp; kk++) {
ftmp += currentdata[index];
index += cmd->nsub;
}
currentdsdata[dsindex] += ftmp / cmd->downsamp;
}
}
}
}
if (firsttime) {
SWAP(currentdata, lastdata);
SWAP(currentdsdata, lastdsdata);
firsttime -= 1;
} else
break;
}
if (myid > 0) {
for (ii = 0; ii < local_numdms; ii++)
float_dedisp(currentdsdata, lastdsdata, dsworklen,
cmd->nsub, offsets[ii], 0.0, outdata[ii]);
}
SWAP(currentdata, lastdata);
SWAP(currentdsdata, lastdsdata);
if (totnumread != worklen) {
if (cmd->maskfileP)
vect_free(maskchans);
vect_free(data1);
vect_free(data2);
if (RAWDATA) vect_free(frawdata);
if (cmd->downsamp > 1) {
vect_free(dsdata1);
vect_free(dsdata2);
}
}
return totnumread;
}
int main(int argc, char *argv[])
{
/* Any variable that begins with 't' means topocentric */
/* Any variable that begins with 'b' means barycentric */
FILE **outfiles = NULL;
float **outdata;
double dtmp, *dms, avgdm = 0.0, dsdt = 0, maxdm;
double *dispdt, tlotoa = 0.0, blotoa = 0.0, BW_ddelay = 0.0;
double max = -9.9E30, min = 9.9E30, var = 0.0, avg = 0.0;
double *btoa = NULL, *ttoa = NULL, avgvoverc = 0.0;
char obs[3], ephem[10], rastring[50], decstring[50];
long totnumtowrite, totwrote = 0, padwrote = 0, datawrote = 0;
int *idispdt, **offsets;
int ii, jj, numadded = 0, numremoved = 0, padding = 0, good_inputs = 1;
int numbarypts = 0, numread = 0, numtowrite = 0;
int padtowrite = 0, statnum = 0;
int numdiffbins = 0, *diffbins = NULL, *diffbinptr = NULL, good_padvals = 0;
double local_lodm;
char *datafilenm, *outpath, *outfilenm, *hostname;
struct spectra_info s;
infodata idata;
mask obsmask;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
#ifdef _OPENMP
omp_set_num_threads(1); // Explicitly turn off OpenMP
#endif
set_using_MPI();
{
FILE *hostfile;
char tmpname[100];
int retval;
hostfile = chkfopen("/etc/hostname", "r");
retval = fscanf(hostfile, "%s\n", tmpname);
if (retval==0) {
printf("Warning: error reading /etc/hostname on proc %d\n", myid);
}
hostname = (char *) calloc(strlen(tmpname) + 1, 1);
memcpy(hostname, tmpname, strlen(tmpname));
fclose(hostfile);
}
/* Call usage() if we have no command line arguments */
if (argc == 1) {
if (myid == 0) {
Program = argv[0];
usage();
}
MPI_Finalize();
exit(1);
}
make_maskbase_struct();
make_spectra_info_struct();
/* Parse the command line using the excellent program Clig */
cmd = parseCmdline(argc, argv);
spectra_info_set_defaults(&s);
// If we are zeroDMing, make sure that clipping is off.
if (cmd->zerodmP) cmd->noclipP = 1;
s.clip_sigma = cmd->clip;
if (cmd->noclipP) {
cmd->clip = 0.0;
s.clip_sigma = 0.0;
}
if (cmd->ifsP) {
// 0 = default or summed, 1-4 are possible also
s.use_poln = cmd->ifs + 1;
}
if (!cmd->numoutP)
cmd->numout = LONG_MAX;
#ifdef DEBUG
showOptionValues();
#endif
if (myid == 0) { /* Master node only */
printf("\n\n");
printf(" Parallel Pulsar Subband De-dispersion Routine\n");
printf(" by Scott M. Ransom\n\n");
s.filenames = cmd->argv;
s.num_files = cmd->argc;
s.clip_sigma = cmd->clip;
// -1 causes the data to determine if we use weights, scales, &
// offsets for PSRFITS or flip the band for any data type where
// we can figure that out with the data
s.apply_flipband = (cmd->invertP) ? 1 : -1;
s.apply_weight = (cmd->noweightsP) ? 0 : -1;
s.apply_scale = (cmd->noscalesP) ? 0 : -1;
s.apply_offset = (cmd->nooffsetsP) ? 0 : -1;
s.remove_zerodm = (cmd->zerodmP) ? 1 : 0;
if (RAWDATA) {
if (cmd->filterbankP) s.datatype = SIGPROCFB;
else if (cmd->psrfitsP) s.datatype = PSRFITS;
else if (cmd->pkmbP) s.datatype = SCAMP;
else if (cmd->bcpmP) s.datatype = BPP;
else if (cmd->wappP) s.datatype = WAPP;
else if (cmd->spigotP) s.datatype = SPIGOT;
} else { // Attempt to auto-identify the data
identify_psrdatatype(&s, 1);
if (s.datatype==SIGPROCFB) cmd->filterbankP = 1;
else if (s.datatype==PSRFITS) cmd->psrfitsP = 1;
else if (s.datatype==SCAMP) cmd->pkmbP = 1;
else if (s.datatype==BPP) cmd->bcpmP = 1;
else if (s.datatype==WAPP) cmd->wappP = 1;
else if (s.datatype==SPIGOT) cmd->spigotP = 1;
else if (s.datatype==SUBBAND) insubs = 1;
else {
printf("\nError: Unable to identify input data files. Please specify type.\n\n");
good_inputs = 0;
}
}
// So far we can only handle PSRFITS, filterbank, and subbands
if (s.datatype!=PSRFITS &&
s.datatype!=SIGPROCFB &&
s.datatype!=SUBBAND) good_inputs = 0;
// For subbanded data
if (!RAWDATA) s.files = (FILE **)malloc(sizeof(FILE *) * s.num_files);
if (good_inputs && (RAWDATA || insubs)) {
char description[40];
psrdatatype_description(description, s.datatype);
if (s.num_files > 1)
printf("Reading %s data from %d files:\n", description, s.num_files);
else
printf("Reading %s data from 1 file:\n", description);
for (ii = 0; ii < s.num_files; ii++) {
printf(" '%s'\n", cmd->argv[ii]);
if (insubs) s.files[ii] = chkfopen(s.filenames[ii], "rb");
}
printf("\n");
if (RAWDATA) {
read_rawdata_files(&s);
print_spectra_info_summary(&s);
spectra_info_to_inf(&s, &idata);
} else { // insubs
char *root, *suffix;
cmd->nsub = s.num_files;
s.N = chkfilelen(s.files[0], sizeof(short));
s.start_subint = gen_ivect(1);
s.num_subint = gen_ivect(1);
s.start_MJD = (long double *)malloc(sizeof(long double));
s.start_spec = (long long *)malloc(sizeof(long long));
s.num_spec = (long long *)malloc(sizeof(long long));
s.num_pad = (long long *)malloc(sizeof(long long));
s.start_spec[0] = 0L;
s.start_subint[0] = 0;
s.num_spec[0] = s.N;
s.num_subint[0] = s.N / SUBSBLOCKLEN;
s.num_pad[0] = 0L;
s.padvals = gen_fvect(s.num_files);
for (ii = 0 ; ii < ii ; ii++)
s.padvals[ii] = 0.0;
if (split_root_suffix(s.filenames[0], &root, &suffix) == 0) {
printf("\nError: The input filename (%s) must have a suffix!\n\n", s.filenames[0]);
exit(1);
}
if (strncmp(suffix, "sub", 3) == 0) {
char *tmpname;
tmpname = calloc(strlen(root) + 10, 1);
sprintf(tmpname, "%s.sub", root);
readinf(&idata, tmpname);
free(tmpname);
strncpy(s.telescope, idata.telescope, 40);
strncpy(s.backend, idata.instrument, 40);
strncpy(s.observer, idata.observer, 40);
strncpy(s.source, idata.object, 40);
s.ra2000 = hms2rad(idata.ra_h, idata.ra_m,
idata.ra_s) * RADTODEG;
s.dec2000 = dms2rad(idata.dec_d, idata.dec_m,
idata.dec_s) * RADTODEG;
ra_dec_to_string(s.ra_str,
idata.ra_h, idata.ra_m, idata.ra_s);
ra_dec_to_string(s.dec_str,
idata.dec_d, idata.dec_m, idata.dec_s);
s.num_channels = idata.num_chan;
s.start_MJD[0] = idata.mjd_i + idata.mjd_f;
s.dt = idata.dt;
s.T = s.N * s.dt;
s.lo_freq = idata.freq;
s.df = idata.chan_wid;
s.hi_freq = s.lo_freq + (s.num_channels - 1.0) * s.df;
s.BW = s.num_channels * s.df;
s.fctr = s.lo_freq - 0.5 * s.df + 0.5 * s.BW;
s.beam_FWHM = idata.fov / 3600.0;
s.spectra_per_subint = SUBSBLOCKLEN;
print_spectra_info_summary(&s);
} else {
printf("\nThe input files (%s) must be subbands! (i.e. *.sub##)\n\n",
cmd->argv[0]);
MPI_Finalize();
exit(1);
}
free(root);
free(suffix);
}
}
}
// If we don't have good input data, exit
MPI_Bcast(&good_inputs, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (!good_inputs) {
MPI_Finalize();
exit(1);
}
MPI_Bcast(&insubs, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (insubs)
cmd->nsub = cmd->argc;
/* Determine the output file names and open them */
local_numdms = cmd->numdms / (numprocs - 1);
dms = gen_dvect(local_numdms);
if (cmd->numdms % (numprocs - 1)) {
if (myid == 0)
printf
("\nThe number of DMs must be divisible by (the number of processors - 1).\n\n");
MPI_Finalize();
exit(1);
}
local_lodm = cmd->lodm + (myid - 1) * local_numdms * cmd->dmstep;
split_path_file(cmd->outfile, &outpath, &outfilenm);
datafilenm = (char *) calloc(strlen(outfilenm) + 20, 1);
if (myid > 0) {
if (chdir(outpath) == -1) {
printf("\nProcess %d on %s cannot chdir() to '%s'. Exiting.\n\n",
myid, hostname, outpath);
MPI_Finalize();
exit(1);
}
outfiles = (FILE **) malloc(local_numdms * sizeof(FILE *));
for (ii = 0; ii < local_numdms; ii++) {
dms[ii] = local_lodm + ii * cmd->dmstep;
avgdm += dms[ii];
sprintf(datafilenm, "%s_DM%.2f.dat", outfilenm, dms[ii]);
outfiles[ii] = chkfopen(datafilenm, "wb");
}
avgdm /= local_numdms;
}
// Broadcast the raw data information
broadcast_spectra_info(&s, myid);
if (myid > 0) {
spectra_info_to_inf(&s, &idata);
if (s.datatype==SIGPROCFB) cmd->filterbankP = 1;
else if (s.datatype==PSRFITS) cmd->psrfitsP = 1;
else if (s.datatype==SCAMP) cmd->pkmbP = 1;
else if (s.datatype==BPP) cmd->bcpmP = 1;
else if (s.datatype==WAPP) cmd->wappP = 1;
else if (s.datatype==SPIGOT) cmd->spigotP = 1;
else if (s.datatype==SUBBAND) insubs = 1;
}
s.filenames = cmd->argv;
/* Read an input mask if wanted */
if (myid > 0) {
int numpad = s.num_channels;
if (insubs)
numpad = s.num_files;
s.padvals = gen_fvect(numpad);
for (ii = 0 ; ii < numpad ; ii++)
s.padvals[ii] = 0.0;
}
if (cmd->maskfileP) {
if (myid == 0) {
read_mask(cmd->maskfile, &obsmask);
printf("Read mask information from '%s'\n\n", cmd->maskfile);
good_padvals = determine_padvals(cmd->maskfile, &obsmask, s.padvals);
}
broadcast_mask(&obsmask, myid);
MPI_Bcast(&good_padvals, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(s.padvals, obsmask.numchan, MPI_FLOAT, 0, MPI_COMM_WORLD);
} else {
obsmask.numchan = obsmask.numint = 0;
MPI_Bcast(&good_padvals, 1, MPI_INT, 0, MPI_COMM_WORLD);
}
// The number of topo to bary time points to generate with TEMPO
numbarypts = (int) (s.T * 1.1 / TDT + 5.5) + 1;
// Identify the TEMPO observatory code
{
char *outscope = (char *) calloc(40, sizeof(char));
telescope_to_tempocode(idata.telescope, outscope, obs);
free(outscope);
}
// Broadcast or calculate a few extra important values
if (insubs) avgdm = idata.dm;
idata.dm = avgdm;
dsdt = cmd->downsamp * idata.dt;
maxdm = cmd->lodm + cmd->numdms * cmd->dmstep;
BW_ddelay = delay_from_dm(maxdm, idata.freq) -
delay_from_dm(maxdm, idata.freq + (idata.num_chan-1) * idata.chan_wid);
blocksperread = ((int) (BW_ddelay / idata.dt) / s.spectra_per_subint + 1);
worklen = s.spectra_per_subint * blocksperread;
if (cmd->nsub > s.num_channels) {
printf
("Warning: The number of requested subbands (%d) is larger than the number of channels (%d).\n",
cmd->nsub, s.num_channels);
printf(" Re-setting the number of subbands to %d.\n\n", s.num_channels);
cmd->nsub = s.num_channels;
}
if (s.spectra_per_subint % cmd->downsamp) {
if (myid == 0) {
printf
("\nError: The downsample factor (%d) must be a factor of the\n",
cmd->downsamp);
printf(" blocklength (%d). Exiting.\n\n", s.spectra_per_subint);
}
MPI_Finalize();
exit(1);
}
tlotoa = idata.mjd_i + idata.mjd_f; /* Topocentric epoch */
if (cmd->numoutP)
totnumtowrite = cmd->numout;
else
totnumtowrite = (long) idata.N / cmd->downsamp;
if (cmd->nobaryP) { /* Main loop if we are not barycentering... */
/* Dispersion delays (in bins). The high freq gets no delay */
/* All other delays are positive fractions of bin length (dt) */
dispdt = subband_search_delays(s.num_channels, cmd->nsub, avgdm,
idata.freq, idata.chan_wid, 0.0);
idispdt = gen_ivect(s.num_channels);
for (ii = 0; ii < s.num_channels; ii++)
idispdt[ii] = NEAREST_LONG(dispdt[ii] / idata.dt);
vect_free(dispdt);
/* The subband dispersion delays (see note above) */
offsets = gen_imatrix(local_numdms, cmd->nsub);
for (ii = 0; ii < local_numdms; ii++) {
double *subdispdt;
subdispdt = subband_delays(s.num_channels, cmd->nsub, dms[ii],
idata.freq, idata.chan_wid, 0.0);
dtmp = subdispdt[cmd->nsub - 1];
for (jj = 0; jj < cmd->nsub; jj++)
offsets[ii][jj] = NEAREST_LONG((subdispdt[jj] - dtmp) / dsdt);
vect_free(subdispdt);
}
/* Allocate our data array and start getting data */
if (myid == 0) {
printf("De-dispersing using %d subbands.\n", cmd->nsub);
if (cmd->downsamp > 1)
printf("Downsampling by a factor of %d (new dt = %.10g)\n",
cmd->downsamp, dsdt);
printf("\n");
}
/* Print the nodes and the DMs they are handling */
print_dms(hostname, myid, numprocs, local_numdms, dms);
outdata = gen_fmatrix(local_numdms, worklen / cmd->downsamp);
numread = get_data(outdata, blocksperread, &s,
&obsmask, idispdt, offsets, &padding);
while (numread == worklen) {
numread /= cmd->downsamp;
if (myid == 0)
print_percent_complete(totwrote, totnumtowrite);
/* Write the latest chunk of data, but don't */
/* write more than cmd->numout points. */
numtowrite = numread;
if (cmd->numoutP && (totwrote + numtowrite) > cmd->numout)
numtowrite = cmd->numout - totwrote;
if (myid > 0) {
write_data(outfiles, local_numdms, outdata, 0, numtowrite);
/* Update the statistics */
if (!padding) {
for (ii = 0; ii < numtowrite; ii++)
update_stats(statnum + ii, outdata[0][ii], &min, &max, &avg, &var);
statnum += numtowrite;
}
}
totwrote += numtowrite;
/* Stop if we have written out all the data we need to */
if (cmd->numoutP && (totwrote == cmd->numout))
break;
numread = get_data(outdata, blocksperread, &s,
&obsmask, idispdt, offsets, &padding);
}
datawrote = totwrote;
} else { /* Main loop if we are barycentering... */
/* What ephemeris will we use? (Default is DE405) */
strcpy(ephem, "DE405");
/* Define the RA and DEC of the observation */
ra_dec_to_string(rastring, idata.ra_h, idata.ra_m, idata.ra_s);
ra_dec_to_string(decstring, idata.dec_d, idata.dec_m, idata.dec_s);
/* Allocate some arrays */
btoa = gen_dvect(numbarypts);
ttoa = gen_dvect(numbarypts);
for (ii = 0; ii < numbarypts; ii++)
ttoa[ii] = tlotoa + TDT * ii / SECPERDAY;
/* Call TEMPO for the barycentering */
if (myid == 0) {
double maxvoverc = -1.0, minvoverc = 1.0, *voverc = NULL;
printf("\nGenerating barycentric corrections...\n");
voverc = gen_dvect(numbarypts);
barycenter(ttoa, btoa, voverc, numbarypts, rastring, decstring, obs, ephem);
for (ii = 0; ii < numbarypts; ii++) {
if (voverc[ii] > maxvoverc)
maxvoverc = voverc[ii];
if (voverc[ii] < minvoverc)
minvoverc = voverc[ii];
avgvoverc += voverc[ii];
}
avgvoverc /= numbarypts;
vect_free(voverc);
printf(" Average topocentric velocity (c) = %.7g\n", avgvoverc);
printf(" Maximum topocentric velocity (c) = %.7g\n", maxvoverc);
printf(" Minimum topocentric velocity (c) = %.7g\n\n", minvoverc);
printf("De-dispersing using %d subbands.\n", cmd->nsub);
if (cmd->downsamp > 1) {
printf(" Downsample = %d\n", cmd->downsamp);
printf(" New sample dt = %.10g\n", dsdt);
}
printf("\n");
}
/* Print the nodes and the DMs they are handling */
print_dms(hostname, myid, numprocs, local_numdms, dms);
MPI_Bcast(btoa, numbarypts, MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Bcast(&avgvoverc, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
blotoa = btoa[0];
/* Dispersion delays (in bins). The high freq gets no delay */
/* All other delays are positive fractions of bin length (dt) */
dispdt = subband_search_delays(s.num_channels, cmd->nsub, avgdm,
idata.freq, idata.chan_wid, avgvoverc);
idispdt = gen_ivect(s.num_channels);
for (ii = 0; ii < s.num_channels; ii++)
idispdt[ii] = NEAREST_LONG(dispdt[ii] / idata.dt);
vect_free(dispdt);
/* The subband dispersion delays (see note above) */
offsets = gen_imatrix(local_numdms, cmd->nsub);
for (ii = 0; ii < local_numdms; ii++) {
double *subdispdt;
subdispdt = subband_delays(s.num_channels, cmd->nsub, dms[ii],
idata.freq, idata.chan_wid, avgvoverc);
dtmp = subdispdt[cmd->nsub - 1];
for (jj = 0; jj < cmd->nsub; jj++)
offsets[ii][jj] = NEAREST_LONG((subdispdt[jj] - dtmp) / dsdt);
vect_free(subdispdt);
}
/* Convert the bary TOAs to differences from the topo TOAs in */
/* units of bin length (dt) rounded to the nearest integer. */
dtmp = (btoa[0] - ttoa[0]);
for (ii = 0; ii < numbarypts; ii++)
btoa[ii] = ((btoa[ii] - ttoa[ii]) - dtmp) * SECPERDAY / dsdt;
/* Find the points where we need to add or remove bins */
{
int oldbin = 0, currentbin;
double lobin, hibin, calcpt;
numdiffbins = abs(NEAREST_LONG(btoa[numbarypts - 1])) + 1;
diffbins = gen_ivect(numdiffbins);
diffbinptr = diffbins;
for (ii = 1; ii < numbarypts; ii++) {
currentbin = NEAREST_LONG(btoa[ii]);
if (currentbin != oldbin) {
if (currentbin > 0) {
calcpt = oldbin + 0.5;
lobin = (ii - 1) * TDT / dsdt;
hibin = ii * TDT / dsdt;
} else {
calcpt = oldbin - 0.5;
lobin = -((ii - 1) * TDT / dsdt);
hibin = -(ii * TDT / dsdt);
}
while (fabs(calcpt) < fabs(btoa[ii])) {
/* Negative bin number means remove that bin */
/* Positive bin number means add a bin there */
*diffbinptr =
NEAREST_LONG(LININTERP
(calcpt, btoa[ii - 1], btoa[ii], lobin, hibin));
diffbinptr++;
calcpt = (currentbin > 0) ? calcpt + 1.0 : calcpt - 1.0;
}
oldbin = currentbin;
}
}
*diffbinptr = cmd->numout; /* Used as a marker */
}
diffbinptr = diffbins;
/* Now perform the barycentering */
outdata = gen_fmatrix(local_numdms, worklen / cmd->downsamp);
numread = get_data(outdata, blocksperread, &s,
&obsmask, idispdt, offsets, &padding);
while (numread == worklen) { /* Loop to read and write the data */
int numwritten = 0;
double block_avg, block_var;
numread /= cmd->downsamp;
/* Determine the approximate local average */
avg_var(outdata[0], numread, &block_avg, &block_var);
if (myid == 0)
print_percent_complete(totwrote, totnumtowrite);
/* Simply write the data if we don't have to add or */
/* remove any bins from this batch. */
/* OR write the amount of data up to cmd->numout or */
/* the next bin that will be added or removed. */
numtowrite = abs(*diffbinptr) - datawrote;
if (cmd->numoutP && (totwrote + numtowrite) > cmd->numout)
numtowrite = cmd->numout - totwrote;
if (numtowrite > numread)
numtowrite = numread;
if (myid > 0) {
write_data(outfiles, local_numdms, outdata, 0, numtowrite);
/* Update the statistics */
if (!padding) {
for (ii = 0; ii < numtowrite; ii++)
update_stats(statnum + ii, outdata[0][ii], &min, &max, &avg, &var);
statnum += numtowrite;
}
}
datawrote += numtowrite;
totwrote += numtowrite;
numwritten += numtowrite;
if ((datawrote == abs(*diffbinptr)) &&
(numwritten != numread) &&
(totwrote < cmd->numout)) { /* Add/remove a bin */
int skip, nextdiffbin;
skip = numtowrite;
/* Write the rest of the data after adding/removing a bin */
do {
if (*diffbinptr > 0) {
/* Add a bin */
if (myid > 0)
write_padding(outfiles, local_numdms, block_avg, 1);
numadded++;
totwrote++;
} else {
/* Remove a bin */
numremoved++;
datawrote++;
numwritten++;
skip++;
}
diffbinptr++;
/* Write the part after the diffbin */
numtowrite = numread - numwritten;
if (cmd->numoutP && (totwrote + numtowrite) > cmd->numout)
numtowrite = cmd->numout - totwrote;
nextdiffbin = abs(*diffbinptr) - datawrote;
if (numtowrite > nextdiffbin)
numtowrite = nextdiffbin;
if (myid > 0) {
write_data(outfiles, local_numdms, outdata, skip, numtowrite);
/* Update the statistics and counters */
if (!padding) {
for (ii = 0; ii < numtowrite; ii++)
update_stats(statnum + ii,
outdata[0][skip + ii], &min, &max, &avg, &var);
statnum += numtowrite;
}
}
numwritten += numtowrite;
datawrote += numtowrite;
totwrote += numtowrite;
skip += numtowrite;
/* Stop if we have written out all the data we need to */
if (cmd->numoutP && (totwrote == cmd->numout))
break;
} while (numwritten < numread);
}
/* Stop if we have written out all the data we need to */
if (cmd->numoutP && (totwrote == cmd->numout))
break;
numread = get_data(outdata, blocksperread, &s,
&obsmask, idispdt, offsets, &padding);
}
}
if (myid > 0) {
/* Calculate the amount of padding we need */
if (cmd->numoutP && (cmd->numout > totwrote))
padwrote = padtowrite = cmd->numout - totwrote;
/* Write the new info file for the output data */
idata.dt = dsdt;
update_infodata(&idata, totwrote, padtowrite, diffbins,
numdiffbins, cmd->downsamp);
for (ii = 0; ii < local_numdms; ii++) {
idata.dm = dms[ii];
if (!cmd->nobaryP) {
double baryepoch, barydispdt, baryhifreq;
baryhifreq = idata.freq + (s.num_channels - 1) * idata.chan_wid;
barydispdt = delay_from_dm(dms[ii], doppler(baryhifreq, avgvoverc));
baryepoch = blotoa - (barydispdt / SECPERDAY);
idata.bary = 1;
idata.mjd_i = (int) floor(baryepoch);
idata.mjd_f = baryepoch - idata.mjd_i;
}
sprintf(idata.name, "%s_DM%.2f", outfilenm, dms[ii]);
writeinf(&idata);
}
/* Set the padded points equal to the average data point */
if (idata.numonoff >= 1) {
int index, startpad, endpad;
for (ii = 0; ii < local_numdms; ii++) {
fclose(outfiles[ii]);
sprintf(datafilenm, "%s_DM%.2f.dat", outfilenm, dms[ii]);
outfiles[ii] = chkfopen(datafilenm, "rb+");
}
for (ii = 0; ii < idata.numonoff; ii++) {
index = 2 * ii;
startpad = idata.onoff[index + 1];
if (ii == idata.numonoff - 1)
endpad = idata.N - 1;
else
endpad = idata.onoff[index + 2];
for (jj = 0; jj < local_numdms; jj++)
chkfseek(outfiles[jj], (startpad + 1) * sizeof(float), SEEK_SET);
padtowrite = endpad - startpad;
write_padding(outfiles, local_numdms, avg, padtowrite);
}
}
}
/* Print simple stats and results */
var /= (datawrote - 1);
if (myid == 0)
print_percent_complete(1, 1);
if (myid == 1) {
printf("\n\nDone.\n\nSimple statistics of the output data:\n");
printf(" Data points written: %ld\n", totwrote);
if (padwrote)
printf(" Padding points written: %ld\n", padwrote);
if (!cmd->nobaryP) {
if (numadded)
printf(" Bins added for barycentering: %d\n", numadded);
if (numremoved)
printf(" Bins removed for barycentering: %d\n", numremoved);
}
printf(" Maximum value of data: %.2f\n", max);
printf(" Minimum value of data: %.2f\n", min);
printf(" Data average value: %.2f\n", avg);
printf(" Data standard deviation: %.2f\n", sqrt(var));
printf("\n");
}
/* Close the files and cleanup */
if (cmd->maskfileP)
free_mask(obsmask);
if (myid > 0) {
for (ii = 0; ii < local_numdms; ii++)
fclose(outfiles[ii]);
free(outfiles);
}
vect_free(outdata[0]);
vect_free(outdata);
vect_free(dms);
free(hostname);
vect_free(idispdt);
vect_free(offsets[0]);
vect_free(offsets);
free(datafilenm);
free(outfilenm);
free(outpath);
if (!cmd->nobaryP) {
vect_free(btoa);
vect_free(ttoa);
vect_free(diffbins);
}
MPI_Finalize();
return (0);
}
int main(int argc, char *argv[])
{
/* Any variable that begins with 't' means topocentric */
/* Any variable that begins with 'b' means barycentric */
FILE *outfile;
float *outdata = NULL;
double tdf = 0.0, dtmp = 0.0, barydispdt = 0.0, dsdt = 0.0;
double *dispdt, *tobsf = NULL, tlotoa = 0.0, blotoa = 0.0;
double max = -9.9E30, min = 9.9E30, var = 0.0, avg = 0.0;
char obs[3], ephem[10], *datafilenm, *outinfonm;
char rastring[50], decstring[50];
int numchan = 1, newper = 0, oldper = 0, nummasked = 0, useshorts = 0;
int numadded = 0, numremoved = 0, padding = 0, *maskchans = NULL, offset = 0;
long slen, ii, numbarypts = 0, worklen = 65536;
long numread = 0, numtowrite = 0, totwrote = 0, datawrote = 0;
long padwrote = 0, padtowrite = 0, statnum = 0;
int numdiffbins = 0, *diffbins = NULL, *diffbinptr = NULL, good_padvals = 0;
int *idispdt;
struct spectra_info s;
infodata idata;
Cmdline *cmd;
mask obsmask;
/* Call usage() if we have no command line arguments */
if (argc == 1) {
Program = argv[0];
printf("\n");
usage();
exit(0);
}
/* Parse the command line using the excellent program Clig */
cmd = parseCmdline(argc, argv);
spectra_info_set_defaults(&s);
s.filenames = cmd->argv;
s.num_files = cmd->argc;
// If we are zeroDMing, make sure that clipping is off.
if (cmd->zerodmP)
cmd->noclipP = 1;
s.clip_sigma = cmd->clip;
// -1 causes the data to determine if we use weights, scales, &
// offsets for PSRFITS or flip the band for any data type where
// we can figure that out with the data
s.apply_flipband = (cmd->invertP) ? 1 : -1;
s.apply_weight = (cmd->noweightsP) ? 0 : -1;
s.apply_scale = (cmd->noscalesP) ? 0 : -1;
s.apply_offset = (cmd->nooffsetsP) ? 0 : -1;
s.remove_zerodm = (cmd->zerodmP) ? 1 : 0;
if (cmd->noclipP) {
cmd->clip = 0.0;
s.clip_sigma = 0.0;
}
if (cmd->ifsP) {
// 0 = default or summed, 1-4 are possible also
s.use_poln = cmd->ifs + 1;
}
if (cmd->ncpus > 1) {
#ifdef _OPENMP
int maxcpus = omp_get_num_procs();
int openmp_numthreads = (cmd->ncpus <= maxcpus) ? cmd->ncpus : maxcpus;
// Make sure we are not dynamically setting the number of threads
omp_set_dynamic(0);
omp_set_num_threads(openmp_numthreads);
printf("Using %d threads with OpenMP\n\n", openmp_numthreads);
#endif
} else {
#ifdef _OPENMP
omp_set_num_threads(1); // Explicitly turn off OpenMP
#endif
}
#ifdef DEBUG
showOptionValues();
#endif
printf("\n\n");
printf(" Pulsar Data Preparation Routine\n");
printf(" Type conversion, de-dispersion, barycentering.\n");
printf(" by Scott M. Ransom\n\n");
if (RAWDATA) {
if (cmd->filterbankP)
s.datatype = SIGPROCFB;
else if (cmd->psrfitsP)
s.datatype = PSRFITS;
} else { // Attempt to auto-identify the data
identify_psrdatatype(&s, 1);
if (s.datatype == SIGPROCFB)
cmd->filterbankP = 1;
else if (s.datatype == PSRFITS)
cmd->psrfitsP = 1;
else if (s.datatype == SDAT)
useshorts = 1;
else if (s.datatype != DAT) {
printf
("Error: Unable to identify input data files. Please specify type.\n\n");
exit(1);
}
}
if (!RAWDATA) {
char *root, *suffix;
/* Split the filename into a rootname and a suffix */
if (split_root_suffix(s.filenames[0], &root, &suffix) == 0) {
printf("\nThe input filename (%s) must have a suffix!\n\n",
s.filenames[0]);
exit(1);
}
printf("Reading input data from '%s'.\n", s.filenames[0]);
printf("Reading information from '%s.inf'.\n\n", root);
/* Read the info file if available */
readinf(&idata, root);
free(root);
free(suffix);
s.files = (FILE **) malloc(sizeof(FILE *));
s.files[0] = chkfopen(s.filenames[0], "rb");
} else {
char description[40];
psrdatatype_description(description, s.datatype);
if (s.num_files > 1)
printf("Reading %s data from %d files:\n", description, s.num_files);
else
printf("Reading %s data from 1 file:\n", description);
for (ii = 0; ii < s.num_files; ii++) {
printf(" '%s'\n", cmd->argv[ii]);
}
printf("\n");
}
/* Determine the other file names and open the output data file */
slen = strlen(cmd->outfile) + 8;
datafilenm = (char *) calloc(slen, 1);
sprintf(datafilenm, "%s.dat", cmd->outfile);
outfile = chkfopen(datafilenm, "wb");
sprintf(idata.name, "%s", cmd->outfile);
outinfonm = (char *) calloc(slen, 1);
sprintf(outinfonm, "%s.inf", cmd->outfile);
if (RAWDATA) {
read_rawdata_files(&s);
if (cmd->ignorechanstrP) {
s.ignorechans = get_ignorechans(cmd->ignorechanstr, 0, s.num_channels-1,
&s.num_ignorechans, &s.ignorechans_str);
if (s.ignorechans_str==NULL) {
s.ignorechans_str = (char *)malloc(strlen(cmd->ignorechanstr)+1);
strcpy(s.ignorechans_str, cmd->ignorechanstr);
}
}
print_spectra_info_summary(&s);
spectra_info_to_inf(&s, &idata);
/* Finish setting up stuff common to all raw formats */
idata.dm = cmd->dm;
worklen = s.spectra_per_subint;
/* If we are offsetting into the file, change inf file start time */
if (cmd->start > 0.0 || cmd->offset > 0) {
if (cmd->start > 0.0) /* Offset in units of worklen */
cmd->offset = (long) (cmd->start *
idata.N / worklen) * worklen;
add_to_inf_epoch(&idata, cmd->offset * idata.dt);
offset_to_spectra(cmd->offset, &s);
printf("Offsetting into the input files by %ld spectra (%.6g sec)\n",
cmd->offset, cmd->offset * idata.dt);
}
if (cmd->maskfileP)
maskchans = gen_ivect(idata.num_chan);
/* Compare the size of the data to the size of output we request */
if (cmd->numoutP) {
dtmp = idata.N;
idata.N = cmd->numout;
writeinf(&idata);
idata.N = dtmp;
} else {
/* Set the output length to a good number if it wasn't requested */
cmd->numoutP = 1;
cmd->numout = choose_good_N((long long)(idata.N/cmd->downsamp));
writeinf(&idata);
printf("Setting a 'good' output length of %ld samples\n", cmd->numout);
}
/* The number of topo to bary time points to generate with TEMPO */
numbarypts = (long) (idata.dt * idata.N * 1.1 / TDT + 5.5) + 1;
// Identify the TEMPO observatory code
{
char *outscope = (char *) calloc(40, sizeof(char));
telescope_to_tempocode(idata.telescope, outscope, obs);
free(outscope);
}
}
/* Read an input mask if wanted */
if (cmd->maskfileP) {
read_mask(cmd->maskfile, &obsmask);
printf("Read mask information from '%s'\n\n", cmd->maskfile);
good_padvals = determine_padvals(cmd->maskfile, &obsmask, s.padvals);
} else {
obsmask.numchan = obsmask.numint = 0;
}
/* Determine our initialization data if we do _not_ have Parkes, */
/* Green Bank BCPM, or Arecibo WAPP data sets. */
if (!RAWDATA) {
/* If we will be barycentering... */
if (!cmd->nobaryP) {
/* The number of topo to bary time points to generate with TEMPO */
numbarypts = (long) (idata.dt * idata.N * 1.1 / TDT + 5.5) + 1;
// Identify the TEMPO observatory code
{
char *outscope = (char *) calloc(40, sizeof(char));
telescope_to_tempocode(idata.telescope, outscope, obs);
free(outscope);
}
}
/* The number of data points to work with at a time */
if (worklen > idata.N)
worklen = idata.N;
worklen = (long) (worklen / 1024) * 1024;
/* If we are offsetting into the file, change inf file start time */
if (cmd->start > 0.0 || cmd->offset > 0) {
if (cmd->start > 0.0) /* Offset in units of worklen */
cmd->offset = (long) (cmd->start *
idata.N / worklen) * worklen;
add_to_inf_epoch(&idata, cmd->offset * idata.dt);
printf("Offsetting into the input files by %ld samples (%.6g sec)\n",
cmd->offset, cmd->offset * idata.dt);
if (useshorts) {
chkfileseek(s.files[0], cmd->offset, sizeof(short), SEEK_SET);
} else {
chkfileseek(s.files[0], cmd->offset, sizeof(float), SEEK_SET);
}
}
/* Set the output length to a good number if it wasn't requested */
if (!cmd->numoutP) {
cmd->numoutP = 1;
cmd->numout = choose_good_N((long long)(idata.N/cmd->downsamp));
printf("Setting a 'good' output length of %ld samples\n", cmd->numout);
}
}
/* Check if we are downsampling */
dsdt = idata.dt * cmd->downsamp;
if (cmd->downsamp > 1) {
printf("Downsampling by a factor of %d\n", cmd->downsamp);
printf("New sample dt = %.10g\n\n", dsdt);
if (worklen % cmd->downsamp) {
printf("Error: The downsample factor (%d) must be a factor of the\n",
cmd->downsamp);
printf(" worklength (%ld). Exiting.\n\n", worklen);
exit(1);
}
}
printf("Writing output data to '%s'.\n", datafilenm);
printf("Writing information to '%s'.\n\n", outinfonm);
/* The topocentric epoch of the start of the data */
tlotoa = (double) idata.mjd_i + idata.mjd_f;
if (!strcmp(idata.band, "Radio") && RAWDATA) {
/* The topocentric spacing between channels */
tdf = idata.chan_wid;
numchan = idata.num_chan;
/* The topocentric observation frequencies */
tobsf = gen_dvect(numchan);
tobsf[0] = idata.freq;
for (ii = 0; ii < numchan; ii++)
tobsf[ii] = tobsf[0] + ii * tdf;
/* The dispersion delays (in time bins) */
dispdt = gen_dvect(numchan); // full float bins
idispdt = gen_ivect(numchan); // nearest integer bins
if (cmd->nobaryP) {
/* Determine our dispersion time delays for each channel */
for (ii = 0; ii < numchan; ii++)
dispdt[ii] = delay_from_dm(cmd->dm, tobsf[ii]);
/* The highest frequency channel gets no delay */
/* All other delays are positive fractions of bin length (dt) */
dtmp = dispdt[numchan - 1];
for (ii = 0; ii < numchan; ii++) {
dispdt[ii] = (dispdt[ii] - dtmp) / idata.dt;
idispdt[ii] = (int) (dispdt[ii] + 0.5);
}
worklen *= ((int) (fabs(dispdt[0])) / worklen) + 1;
}
} else { /* For unknown radio raw data (Why is this here?) */
tobsf = gen_dvect(numchan);
dispdt = gen_dvect(numchan);
idispdt = gen_ivect(numchan);
dispdt[0] = 0.0;
idispdt[0] = 0;
if (!strcmp(idata.band, "Radio")) {
tobsf[0] = idata.freq + (idata.num_chan - 1) * idata.chan_wid;
cmd->dm = idata.dm;
} else {
tobsf[0] = 0.0;
cmd->dm = 0.0;
}
}
if (cmd->nobaryP) { /* Main loop if we are not barycentering... */
/* Allocate our data array */
outdata = gen_fvect(worklen);
printf("Massaging the data ...\n\n");
printf("Amount Complete = 0%%");
do {
if (RAWDATA)
numread = read_psrdata(outdata, worklen, &s, idispdt, &padding,
maskchans, &nummasked, &obsmask);
else if (useshorts)
numread = read_shorts(s.files[0], outdata, worklen, numchan);
else
numread = read_floats(s.files[0], outdata, worklen, numchan);
if (numread == 0)
break;
/* Downsample if requested */
if (cmd->downsamp > 1)
numread = downsample(outdata, numread, cmd->downsamp);
/* Print percent complete */
newper = (int) ((float) totwrote / cmd->numout * 100.0) + 1;
if (newper > oldper) {
printf("\rAmount Complete = %3d%%", newper);
fflush(stdout);
oldper = newper;
}
/* Write the latest chunk of data, but don't */
/* write more than cmd->numout points. */
numtowrite = numread;
if ((totwrote + numtowrite) > cmd->numout)
numtowrite = cmd->numout - totwrote;
chkfwrite(outdata, sizeof(float), numtowrite, outfile);
totwrote += numtowrite;
/* Update the statistics */
if (!padding) {
for (ii = 0; ii < numtowrite; ii++)
update_stats(statnum + ii, outdata[ii], &min, &max, &avg, &var);
statnum += numtowrite;
}
/* Stop if we have written out all the data we need to */
if (totwrote == cmd->numout)
break;
} while (numread);
datawrote = totwrote;
} else { /* Main loop if we are barycentering... */
double avgvoverc = 0.0, maxvoverc = -1.0, minvoverc = 1.0, *voverc = NULL;
double *bobsf = NULL, *btoa = NULL, *ttoa = NULL;
/* What ephemeris will we use? (Default is DE405) */
strcpy(ephem, "DE405");
/* Define the RA and DEC of the observation */
ra_dec_to_string(rastring, idata.ra_h, idata.ra_m, idata.ra_s);
ra_dec_to_string(decstring, idata.dec_d, idata.dec_m, idata.dec_s);
/* Allocate some arrays */
bobsf = gen_dvect(numchan);
btoa = gen_dvect(numbarypts);
ttoa = gen_dvect(numbarypts);
voverc = gen_dvect(numbarypts);
for (ii = 0; ii < numbarypts; ii++)
ttoa[ii] = tlotoa + TDT * ii / SECPERDAY;
/* Call TEMPO for the barycentering */
printf("Generating barycentric corrections...\n");
barycenter(ttoa, btoa, voverc, numbarypts, rastring, decstring, obs, ephem);
for (ii = 0; ii < numbarypts; ii++) {
if (voverc[ii] > maxvoverc)
maxvoverc = voverc[ii];
if (voverc[ii] < minvoverc)
minvoverc = voverc[ii];
avgvoverc += voverc[ii];
}
avgvoverc /= numbarypts;
vect_free(voverc);
blotoa = btoa[0];
printf(" Average topocentric velocity (c) = %.7g\n", avgvoverc);
printf(" Maximum topocentric velocity (c) = %.7g\n", maxvoverc);
printf(" Minimum topocentric velocity (c) = %.7g\n\n", minvoverc);
printf("Collecting and barycentering %s...\n\n", cmd->argv[0]);
/* Determine the initial dispersion time delays for each channel */
for (ii = 0; ii < numchan; ii++) {
bobsf[ii] = doppler(tobsf[ii], avgvoverc);
dispdt[ii] = delay_from_dm(cmd->dm, bobsf[ii]);
}
/* The highest frequency channel gets no delay */
/* All other delays are positive fractions of bin length (dt) */
barydispdt = dispdt[numchan - 1];
for (ii = 0; ii < numchan; ii++) {
dispdt[ii] = (dispdt[ii] - barydispdt) / idata.dt;
idispdt[ii] = (int) (dispdt[ii] + 0.5);
}
if (RAWDATA)
worklen *= ((int) (dispdt[0]) / worklen) + 1;
/* If the data is de-dispersed radio data... */
if (!strcmp(idata.band, "Radio")) {
printf("The DM of %.2f at the barycentric observing freq of %.3f MHz\n",
idata.dm, bobsf[numchan - 1]);
printf(" causes a delay of %f seconds compared to infinite freq.\n",
barydispdt);
printf(" This delay is removed from the barycented times.\n\n");
}
printf("Topocentric epoch (at data start) is:\n");
printf(" %17.11f\n\n", tlotoa);
printf("Barycentric epoch (infinite obs freq at data start) is:\n");
printf(" %17.11f\n\n", blotoa - (barydispdt / SECPERDAY));
/* Convert the bary TOAs to differences from the topo TOAs in */
/* units of bin length (dsdt) rounded to the nearest integer. */
dtmp = (btoa[0] - ttoa[0]);
for (ii = 0; ii < numbarypts; ii++)
btoa[ii] = ((btoa[ii] - ttoa[ii]) - dtmp) * SECPERDAY / dsdt;
{ /* Find the points where we need to add or remove bins */
int oldbin = 0, currentbin;
double lobin, hibin, calcpt;
numdiffbins = abs(NEAREST_LONG(btoa[numbarypts - 1])) + 1;
diffbins = gen_ivect(numdiffbins);
diffbinptr = diffbins;
for (ii = 1; ii < numbarypts; ii++) {
currentbin = NEAREST_LONG(btoa[ii]);
if (currentbin != oldbin) {
if (currentbin > 0) {
calcpt = oldbin + 0.5;
lobin = (ii - 1) * TDT / dsdt;
hibin = ii * TDT / dsdt;
} else {
calcpt = oldbin - 0.5;
lobin = -((ii - 1) * TDT / dsdt);
hibin = -(ii * TDT / dsdt);
}
while (fabs(calcpt) < fabs(btoa[ii])) {
/* Negative bin number means remove that bin */
/* Positive bin number means add a bin there */
*diffbinptr = NEAREST_LONG(LININTERP(calcpt, btoa[ii - 1],
btoa[ii], lobin,
hibin));
diffbinptr++;
calcpt = (currentbin > 0) ? calcpt + 1.0 : calcpt - 1.0;
}
oldbin = currentbin;
}
}
*diffbinptr = cmd->numout; /* Used as a marker */
}
diffbinptr = diffbins;
/* Now perform the barycentering */
printf("Massaging the data...\n\n");
printf("Amount Complete = 0%%");
/* Allocate our data array */
outdata = gen_fvect(worklen);
do { /* Loop to read and write the data */
int numwritten = 0;
double block_avg, block_var;
if (RAWDATA)
numread = read_psrdata(outdata, worklen, &s, idispdt, &padding,
maskchans, &nummasked, &obsmask);
else if (useshorts)
numread = read_shorts(s.files[0], outdata, worklen, numchan);
else
numread = read_floats(s.files[0], outdata, worklen, numchan);
if (numread == 0)
break;
/* Downsample if requested */
if (cmd->downsamp > 1)
numread = downsample(outdata, numread, cmd->downsamp);
/* Determine the approximate local average */
avg_var(outdata, numread, &block_avg, &block_var);
/* Print percent complete */
newper = (int) ((float) totwrote / cmd->numout * 100.0) + 1;
if (newper > oldper) {
printf("\rAmount Complete = %3d%%", newper);
fflush(stdout);
oldper = newper;
}
/* Simply write the data if we don't have to add or */
/* remove any bins from this batch. */
/* OR write the amount of data up to cmd->numout or */
/* the next bin that will be added or removed. */
numtowrite = abs(*diffbinptr) - datawrote;
/* FIXME: numtowrite+totwrote can wrap! */
if ((totwrote + numtowrite) > cmd->numout)
numtowrite = cmd->numout - totwrote;
if (numtowrite > numread)
numtowrite = numread;
chkfwrite(outdata, sizeof(float), numtowrite, outfile);
datawrote += numtowrite;
totwrote += numtowrite;
numwritten += numtowrite;
/* Update the statistics */
if (!padding) {
for (ii = 0; ii < numtowrite; ii++)
update_stats(statnum + ii, outdata[ii], &min, &max, &avg, &var);
statnum += numtowrite;
}
if ((datawrote == abs(*diffbinptr)) && (numwritten != numread) && (totwrote < cmd->numout)) { /* Add/remove a bin */
float favg;
int skip, nextdiffbin;
skip = numtowrite;
do { /* Write the rest of the data after adding/removing a bin */
if (*diffbinptr > 0) {
/* Add a bin */
favg = (float) block_avg;
chkfwrite(&favg, sizeof(float), 1, outfile);
numadded++;
totwrote++;
} else {
/* Remove a bin */
numremoved++;
datawrote++;
numwritten++;
skip++;
}
diffbinptr++;
/* Write the part after the diffbin */
numtowrite = numread - numwritten;
if ((totwrote + numtowrite) > cmd->numout)
numtowrite = cmd->numout - totwrote;
nextdiffbin = abs(*diffbinptr) - datawrote;
if (numtowrite > nextdiffbin)
numtowrite = nextdiffbin;
chkfwrite(outdata + skip, sizeof(float), numtowrite, outfile);
numwritten += numtowrite;
datawrote += numtowrite;
totwrote += numtowrite;
/* Update the statistics and counters */
if (!padding) {
for (ii = 0; ii < numtowrite; ii++)
update_stats(statnum + ii, outdata[skip + ii],
&min, &max, &avg, &var);
statnum += numtowrite;
}
skip += numtowrite;
/* Stop if we have written out all the data we need to */
if (totwrote == cmd->numout)
break;
} while (numwritten < numread);
}
/* Stop if we have written out all the data we need to */
if (totwrote == cmd->numout)
break;
} while (numread);
/* Free the arrays used in barycentering */
vect_free(bobsf);
vect_free(btoa);
vect_free(ttoa);
}
/* Calculate what the amount of padding we need */
if (cmd->numout > totwrote)
padwrote = padtowrite = cmd->numout - totwrote;
/* Write the new info file for the output data */
if (!cmd->nobaryP) {
idata.bary = 1;
idata.mjd_i = (int) floor(blotoa - (barydispdt / SECPERDAY));
idata.mjd_f = blotoa - (barydispdt / SECPERDAY) - idata.mjd_i;
}
if (cmd->downsamp > 1)
idata.dt = dsdt;
update_infodata(&idata, totwrote, padtowrite, diffbins, numdiffbins);
writeinf(&idata);
/* Set the padded points equal to the average data point */
if (idata.numonoff >= 1) {
int jj, index, startpad, endpad;
for (ii = 0; ii < worklen; ii++)
outdata[ii] = avg;
fclose(outfile);
outfile = chkfopen(datafilenm, "rb+");
for (ii = 0; ii < idata.numonoff; ii++) {
index = 2 * ii;
startpad = idata.onoff[index + 1];
if (ii == idata.numonoff - 1)
endpad = idata.N - 1;
else
endpad = idata.onoff[index + 2];
chkfseek(outfile, (startpad + 1) * sizeof(float), SEEK_SET);
padtowrite = endpad - startpad;
for (jj = 0; jj < padtowrite / worklen; jj++)
chkfwrite(outdata, sizeof(float), worklen, outfile);
chkfwrite(outdata, sizeof(float), padtowrite % worklen, outfile);
}
}
vect_free(outdata);
// Close all the raw files and free their vectors
close_rawfiles(&s);
/* Print simple stats and results */
var /= (datawrote - 1);
/* Conver the '.dat' file to '.sdat' if requested */
if (cmd->shortsP) {
FILE *infile;
int safe_convert = 1, bufflen = 65536;
char *sdatafilenm;
float *fbuffer;
short *sbuffer;
offset = (int) (floor(avg));
if ((max - min) > (SHRT_MAX - SHRT_MIN)) {
if ((max - min) < 1.5 * (SHRT_MAX - SHRT_MIN)) {
printf("Warning: There is more dynamic range in the data\n"
" than can be handled perfectly:\n"
" max - min = %.2f - %.2f = %.2f\n"
" Clipping the low values...\n\n", max, min,
max - min);
offset = max - SHRT_MAX;
} else {
printf("Error: There is way too much dynamic range in the data:\n"
" max - min = %.2f - %.2f = %.2f\n"
" Not converting to shorts.\n\n", max, min, max - min);
safe_convert = 0;
}
}
if (safe_convert) {
fbuffer = gen_fvect(bufflen);
sbuffer = gen_svect(bufflen);
sdatafilenm = (char *) calloc(slen, 1);
sprintf(sdatafilenm, "%s.sdat", cmd->outfile);
printf("\n\nConverting floats in '%s' to shorts in '%s'.",
datafilenm, sdatafilenm);
fflush(NULL);
infile = chkfopen(datafilenm, "rb");
outfile = chkfopen(sdatafilenm, "wb");
while ((numread = chkfread(fbuffer, sizeof(float), bufflen, infile))) {
for (ii = 0; ii < numread; ii++)
sbuffer[ii] = (short) (fbuffer[ii] + 1e-20 - offset);
chkfwrite(sbuffer, sizeof(short), numread, outfile);
}
fclose(infile);
fclose(outfile);
remove(datafilenm);
vect_free(fbuffer);
vect_free(sbuffer);
}
}
printf("\n\nDone.\n\nSimple statistics of the output data:\n");
printf(" Data points written: %ld\n", totwrote);
if (padwrote)
printf(" Padding points written: %ld\n", padwrote);
if (!cmd->nobaryP) {
if (numadded)
printf(" Bins added for barycentering: %d\n", numadded);
if (numremoved)
printf(" Bins removed for barycentering: %d\n", numremoved);
}
printf(" Maximum value of data: %.2f\n", max);
printf(" Minimum value of data: %.2f\n", min);
printf(" Data average value: %.2f\n", avg);
printf(" Data standard deviation: %.2f\n", sqrt(var));
if (cmd->shortsP && offset != 0)
printf(" Offset applied to data: %d\n", -offset);
printf("\n");
/* Cleanup */
if (cmd->maskfileP) {
free_mask(obsmask);
vect_free(maskchans);
}
vect_free(tobsf);
vect_free(dispdt);
vect_free(idispdt);
free(outinfonm);
free(datafilenm);
if (!cmd->nobaryP)
vect_free(diffbins);
return (0);
}
static int get_data(float **outdata, int blocksperread,
struct spectra_info *s,
mask * obsmask, int *idispdts, int **offsets,
int *padding, short **subsdata)
{
static int firsttime = 1, *maskchans = NULL, blocksize;
static int worklen, dsworklen;
static float *tempzz, *data1, *data2, *dsdata1 = NULL, *dsdata2 = NULL;
static float *currentdata, *lastdata, *currentdsdata, *lastdsdata;
static double blockdt;
int totnumread = 0, numread = 0, ii, jj, tmppad = 0, nummasked = 0;
if (firsttime) {
if (cmd->maskfileP)
maskchans = gen_ivect(s->num_channels);
worklen = s->spectra_per_subint * blocksperread;
dsworklen = worklen / cmd->downsamp;
// Make sure that our working blocks are long enough...
for (ii = 0; ii < cmd->numdms; ii++) {
for (jj = 0; jj < cmd->nsub; jj++) {
if (offsets[ii][jj] > dsworklen)
printf
("WARNING!: (offsets[%d][%d] = %d) > (dsworklen = %d)\n",
ii, jj, offsets[ii][jj], dsworklen);
}
}
blocksize = s->spectra_per_subint * cmd->nsub;
blockdt = s->spectra_per_subint * s->dt;
data1 = gen_fvect(cmd->nsub * worklen);
data2 = gen_fvect(cmd->nsub * worklen);
currentdata = data1;
lastdata = data2;
if (cmd->downsamp > 1) {
dsdata1 = gen_fvect(cmd->nsub * dsworklen);
dsdata2 = gen_fvect(cmd->nsub * dsworklen);
currentdsdata = dsdata1;
lastdsdata = dsdata2;
} else {
currentdsdata = data1;
lastdsdata = data2;
}
}
while (1) {
if (RAWDATA || insubs) {
for (ii = 0; ii < blocksperread; ii++) {
if (RAWDATA)
numread = read_subbands(currentdata + ii * blocksize, idispdts,
cmd->nsub, s, 0, &tmppad,
maskchans, &nummasked, obsmask);
else if (insubs)
numread = read_PRESTO_subbands(s->files, s->num_files,
currentdata + ii * blocksize,
blockdt, maskchans, &nummasked,
obsmask, cmd->clip, s->padvals);
if (!firsttime)
totnumread += numread;
if (numread != s->spectra_per_subint) {
for (jj = ii * blocksize; jj < (ii + 1) * blocksize; jj++)
currentdata[jj] = 0.0;
}
if (tmppad)
*padding = 1;
}
}
/* Downsample the subband data if needed */
if (cmd->downsamp > 1) {
int kk, index;
float ftmp;
for (ii = 0; ii < dsworklen; ii++) {
const int dsoffset = ii * cmd->nsub;
const int offset = dsoffset * cmd->downsamp;
for (jj = 0; jj < cmd->nsub; jj++) {
const int dsindex = dsoffset + jj;
index = offset + jj;
currentdsdata[dsindex] = ftmp = 0.0;
for (kk = 0; kk < cmd->downsamp; kk++) {
ftmp += currentdata[index];
index += cmd->nsub;
}
currentdsdata[dsindex] += ftmp / cmd->downsamp;
}
}
}
if (firsttime) {
SWAP(currentdata, lastdata);
SWAP(currentdsdata, lastdsdata);
firsttime = 0;
} else
break;
}
if (!cmd->subP) {
for (ii = 0; ii < cmd->numdms; ii++)
float_dedisp(currentdsdata, lastdsdata, dsworklen,
cmd->nsub, offsets[ii], 0.0, outdata[ii]);
} else {
/* Input format is sub1[0], sub2[0], sub3[0], ..., sub1[1], sub2[1], sub3[1], ... */
float infloat;
for (ii = 0; ii < cmd->nsub; ii++) {
for (jj = 0; jj < dsworklen; jj++) {
infloat = lastdsdata[ii + (cmd->nsub * jj)];
subsdata[ii][jj] = (short) (infloat + 0.5);
//if ((float) subsdata[ii][jj] != infloat)
// printf
// ("Warning: We are incorrectly converting subband data! float = %f short = %d\n",
// infloat, subsdata[ii][jj]);
}
}
}
SWAP(currentdata, lastdata);
SWAP(currentdsdata, lastdsdata);
if (totnumread != worklen) {
if (cmd->maskfileP)
vect_free(maskchans);
vect_free(data1);
vect_free(data2);
if (cmd->downsamp > 1) {
vect_free(dsdata1);
vect_free(dsdata2);
}
}
return totnumread;
}
int check_mask(double starttime, double duration, mask * obsmask, int *maskchans)
/* Return value is the number of channels to mask. The */
/* channel numbers are placed in maskchans (which must */
/* have a length of numchan). If -1 is returned, all */
/* channels should be masked. */
{
int loint, hiint;
double endtime;
static int old_loint = -1, old_hiint = -1, old_numchan = 0;
/*
static int firsttime = 1;
if (firsttime){
int ii;
printf("\n\n numzapints = %d\n : ", obsmask->num_zap_ints);
for (ii=0; ii<obsmask->num_zap_ints; ii++)
printf("%d ", obsmask->zap_ints[ii]);
printf("\n\n numzapchans = %d\n : ", obsmask->num_zap_chans);
for (ii=0; ii<obsmask->num_zap_chans; ii++)
printf("%d ", obsmask->zap_chans[ii]);
printf("\n\n");
firsttime = 0;
}
*/
endtime = starttime + duration;
loint = (int) (starttime / obsmask->dtint);
hiint = (int) (endtime / obsmask->dtint);
/* Mask the same channels as for the last call */
if (loint == old_loint && hiint == old_hiint)
return old_numchan;
/* Make sure that we aren't past the last interval */
if (loint >= obsmask->numint)
loint = obsmask->numint - 1;
if (hiint >= obsmask->numint)
hiint = loint;
if ((loint >= obsmask->numint + 1) ||
(hiint >= obsmask->numint + 1)) {
printf("Warning!! Trying to use a mask interval well after the mask ends!\n");
}
/* Determine new channels to mask */
if (loint == hiint) {
old_loint = old_hiint = loint;
/* Check to see if this is an interval where we zap all the channels */
if (obsmask->num_zap_ints) {
if (find_num(loint, obsmask->zap_ints, obsmask->num_zap_ints)) {
old_numchan = -1;
return old_numchan;
}
}
/* Merge the overall channels to zap with the local channels to zap */
old_numchan = merge_no_dupes(obsmask->zap_chans,
obsmask->num_zap_chans,
obsmask->chans[loint],
obsmask->num_chans_per_int[loint], maskchans);
} else { /* We are straddling a rfifind interval boundary */
int *tmpchans;
old_loint = loint;
old_hiint = hiint;
/* Check to see if this is an interval where we zap all the channels */
if (obsmask->num_zap_ints) {
if (find_num(loint, obsmask->zap_ints, obsmask->num_zap_ints)) {
old_numchan = -1;
return old_numchan;
}
if (find_num(hiint, obsmask->zap_ints, obsmask->num_zap_ints)) {
old_numchan = -1;
return old_numchan;
}
}
/* Merge the overall channels to zap with the loint channels to zap */
if (obsmask->num_zap_chans) {
tmpchans = gen_ivect(obsmask->numchan);
old_numchan = merge_no_dupes(obsmask->zap_chans,
obsmask->num_zap_chans,
obsmask->chans[loint],
obsmask->num_chans_per_int[loint], tmpchans);
} else {
tmpchans = obsmask->zap_chans;
old_numchan = obsmask->num_zap_chans;
}
/* Merge the loint+overall channels to zap with the hiint channels to zap */
old_numchan = merge_no_dupes(tmpchans,
old_numchan,
obsmask->chans[hiint],
obsmask->num_chans_per_int[hiint], maskchans);
if (obsmask->num_zap_chans)
vect_free(tmpchans);
}
return old_numchan;
}