void calc_avgmedstd(float *arr, int numarr, float fraction,
int step, float *avg, float *med, float *std)
/* Calculates the median and middle-'fraction' std deviation */
/* and average of the array 'arr'. Values are returned in */
/* 'avg', 'med' and 'std'. The array is not modified. */
{
int ii, jj, len, start;
float *tmparr;
double davg, dstd;
len = (int) (numarr * fraction + 0.5);
if (len > numarr || len < 0) {
printf("fraction (%g) out-of-bounds in calc_avgmedstd()\n", fraction);
exit(1);
}
start = (numarr - len) / 2;
tmparr = gen_fvect(numarr);
for (ii = 0, jj = 0; ii < numarr; ii++, jj += step)
tmparr[ii] = arr[jj];
qsort(tmparr, numarr, sizeof(float), compare_floats);
avg_var(tmparr + start, len, &davg, &dstd);
*avg = (float) davg;
*med = tmparr[numarr / 2];
*std = sqrt(dstd);
vect_free(tmparr);
}
float estimate_offpulse_redchi2(double *inprofs, foldstats *stats,
int numparts, int numsubbands,
int proflen, int numtrials, double dofeff)
// Randomly offset each pulse profile in a .pfd data square or cube
// and combine them to estimate a "true" off-pulse level. Do this
// numtrials times in order to improve the statistics. Return the
// inverse of the average of the off-pulse reduced-chi^2 (i.e. the
// correction factor). dofeff is the effective number of DOF as
// returned by DOF_corr().
{
int ii, jj, kk, offset, trialnum, phsindex, statindex;
float *chis;
double chi_avg, chi_var, redchi;
double prof_avg, prof_var, *prof_ptr, *sumprof;
sumprof = gen_dvect(proflen);
chis = gen_fvect(numtrials);
for (trialnum = 0; trialnum < numtrials; trialnum++) {
// Initialize the summed profile
for (ii = 0; ii < proflen; ii++)
sumprof[ii] = 0.0;
prof_avg = 0.0;
prof_var = 0.0;
prof_ptr = inprofs;
for (ii = 0; ii < numparts; ii++) { // parts
for (jj = 0; jj < numsubbands; jj++) { // subbands
statindex = ii * numsubbands + jj;
offset = random() % proflen;
phsindex = 0;
for (kk = offset; kk < proflen; kk++, phsindex++) // phases
sumprof[phsindex] += prof_ptr[kk];
for (kk = 0; kk < offset; kk++, phsindex++) // phases
sumprof[phsindex] += prof_ptr[kk];
prof_ptr += proflen;
prof_avg += stats[statindex].prof_avg;
prof_var += stats[statindex].prof_var;
}
}
/* Calculate the current chi-squared */
redchi = chisqr(sumprof, proflen, prof_avg, prof_var) / dofeff;
chis[trialnum] = (float) redchi;
}
avg_var(chis, numtrials, &chi_avg, &chi_var);
vect_free(chis);
vect_free(sumprof);
return 1.0/chi_avg;
}
static dataview *get_dataview(int centern, int zoomlevel, datapart * dp)
{
int ii, jj, offset;
double tmpavg, tmpvar;
float *tmpchunk;
dataview *dv;
dv = (dataview *) malloc(sizeof(dataview));
dv->zoomlevel = zoomlevel;
dv->numsamps = (1 << (LOGMAXDISPNUM - zoomlevel));
if (dv->numsamps > dp->nn)
dv->numsamps = next2_to_n(dp->nn) / 2;
dv->chunklen = (zoomlevel < -LOGMINCHUNKLEN) ?
(1 << abs(zoomlevel)) : (1 << LOGMINCHUNKLEN);
dv->dispnum = (dv->numsamps > MAXDISPNUM) ? MAXDISPNUM : dv->numsamps;
if (DEBUGOUT)
printf("zoomlevel = %d numsamps = %d chunklen = %d dispnum %d nn = %d\n",
dv->zoomlevel, dv->numsamps, dv->chunklen, dv->dispnum, dp->nn);
dv->numchunks = dv->numsamps / dv->chunklen;
dv->centern = centern;
dv->lon = centern - dv->numsamps / 2;
dv->vdt = dv->chunklen * idata.dt;
dv->maxval = SMALLNUM;
dv->minval = LARGENUM;
if (dv->lon < 0) {
dv->lon = 0;
dv->centern = dv->lon + dv->numsamps / 2;
}
if (dv->lon + dv->numsamps >= dp->nn) {
dv->lon = dp->nn - dv->numsamps;
dv->centern = dv->lon + dv->numsamps / 2;
}
tmpchunk = gen_fvect(dv->chunklen);
for (ii = 0; ii < dv->numchunks; ii++) {
float tmpmin = LARGENUM, tmpmax = SMALLNUM, tmpval;
offset = dv->lon + ii * dv->chunklen;
memcpy(tmpchunk, dp->data + offset, sizeof(float) * dv->chunklen);
avg_var(dp->data + offset, dv->chunklen, &tmpavg, &tmpvar);
if (usemedian)
dv->avgmeds[ii] = median(tmpchunk, dv->chunklen);
else
dv->avgmeds[ii] = tmpavg;
dv->stds[ii] = sqrt(tmpvar);
for (jj = 0; jj < dv->chunklen; jj++, offset++) {
tmpval = dp->data[offset];
if (tmpval > tmpmax)
tmpmax = tmpval;
if (tmpval < tmpmin)
tmpmin = tmpval;
}
dv->maxs[ii] = tmpmax;
if (tmpmax > dv->maxval)
dv->maxval = tmpmax;
dv->mins[ii] = tmpmin;
if (tmpmin < dv->minval)
dv->minval = tmpmin;
}
vect_free(tmpchunk);
offset = dv->lon;
if (zoomlevel > 0) {
for (ii = 0, offset = dv->lon; ii < dv->numsamps; ii++, offset++)
*(dv->vals + ii) = *(dp->data + offset);
}
return dv;
}
int main(int argc, char *argv[])
{
FILE *bytemaskfile;
float **dataavg = NULL, **datastd = NULL, **datapow = NULL;
float *chandata = NULL, powavg, powstd, powmax;
float inttime, norm, fracterror = RFI_FRACTERROR;
float *rawdata = NULL;
unsigned char **bytemask = NULL;
short *srawdata = NULL;
char *outfilenm, *statsfilenm, *maskfilenm;
char *bytemaskfilenm, *rfifilenm;
int numchan = 0, numint = 0, newper = 0, oldper = 0, good_padvals = 0;
int blocksperint, ptsperint = 0, ptsperblock = 0, padding = 0;
int numcands, candnum, numrfi = 0, numrfivect = NUM_RFI_VECT;
int ii, jj, kk, slen, numread = 0, insubs = 0;
int harmsum = RFI_NUMHARMSUM, lobin = RFI_LOBIN, numbetween = RFI_NUMBETWEEN;
double davg, dvar, freq;
struct spectra_info s;
presto_interptype interptype;
rfi *rfivect = NULL;
mask oldmask, newmask;
fftcand *cands;
infodata idata;
Cmdline *cmd;
/* Call usage() if we have no command line arguments */
if (argc == 1) {
Program = argv[0];
printf("\n");
usage();
exit(0);
}
/* Parse the command line using the excellent program Clig */
cmd = parseCmdline(argc, argv);
spectra_info_set_defaults(&s);
s.filenames = cmd->argv;
s.num_files = cmd->argc;
s.clip_sigma = cmd->clip;
// -1 causes the data to determine if we use weights, scales, &
// offsets for PSRFITS or flip the band for any data type where
// we can figure that out with the data
s.apply_flipband = (cmd->invertP) ? 1 : -1;
s.apply_weight = (cmd->noweightsP) ? 0 : -1;
s.apply_scale = (cmd->noscalesP) ? 0 : -1;
s.apply_offset = (cmd->nooffsetsP) ? 0 : -1;
s.remove_zerodm = (cmd->zerodmP) ? 1 : 0;
if (cmd->noclipP) {
cmd->clip = 0.0;
s.clip_sigma = 0.0;
}
if (cmd->ifsP) {
// 0 = default or summed, 1-4 are possible also
s.use_poln = cmd->ifs;
}
slen = strlen(cmd->outfile) + 20;
#ifdef DEBUG
showOptionValues();
#endif
printf("\n\n");
printf(" Pulsar Data RFI Finder\n");
printf(" by Scott M. Ransom\n\n");
/* The following is the root of all the output files */
outfilenm = (char *) calloc(slen, sizeof(char));
sprintf(outfilenm, "%s_rfifind", cmd->outfile);
/* And here are the output file names */
maskfilenm = (char *) calloc(slen, sizeof(char));
sprintf(maskfilenm, "%s.mask", outfilenm);
bytemaskfilenm = (char *) calloc(slen, sizeof(char));
sprintf(bytemaskfilenm, "%s.bytemask", outfilenm);
rfifilenm = (char *) calloc(slen, sizeof(char));
sprintf(rfifilenm, "%s.rfi", outfilenm);
statsfilenm = (char *) calloc(slen, sizeof(char));
sprintf(statsfilenm, "%s.stats", outfilenm);
sprintf(idata.name, "%s", outfilenm);
if (RAWDATA) {
if (cmd->filterbankP) s.datatype = SIGPROCFB;
else if (cmd->psrfitsP) s.datatype = PSRFITS;
else if (cmd->pkmbP) s.datatype = SCAMP;
else if (cmd->bcpmP) s.datatype = BPP;
else if (cmd->wappP) s.datatype = WAPP;
else if (cmd->spigotP) s.datatype = SPIGOT;
} else { // Attempt to auto-identify the data
identify_psrdatatype(&s, 1);
if (s.datatype==SIGPROCFB) cmd->filterbankP = 1;
else if (s.datatype==PSRFITS) cmd->psrfitsP = 1;
else if (s.datatype==SCAMP) cmd->pkmbP = 1;
else if (s.datatype==BPP) cmd->bcpmP = 1;
else if (s.datatype==WAPP) cmd->wappP = 1;
else if (s.datatype==SPIGOT) cmd->spigotP = 1;
else if (s.datatype==SUBBAND) insubs = 1;
else {
printf("Error: Unable to identify input data files. Please specify type.\n\n");
exit(1);
}
}
if (!cmd->nocomputeP) {
if (RAWDATA || insubs) {
char description[40];
psrdatatype_description(description, s.datatype);
if (s.num_files > 1)
printf("Reading %s data from %d files:\n", description, s.num_files);
else
printf("Reading %s data from 1 file:\n", description);
if (insubs) s.files = (FILE **)malloc(sizeof(FILE *) * s.num_files);
for (ii = 0; ii < s.num_files; ii++) {
printf(" '%s'\n", cmd->argv[ii]);
if (insubs) s.files[ii] = chkfopen(cmd->argv[ii], "rb");
}
printf("\n");
}
if (RAWDATA) {
read_rawdata_files(&s);
print_spectra_info_summary(&s);
spectra_info_to_inf(&s, &idata);
ptsperblock = s.spectra_per_subint;
numchan = s.num_channels;
idata.dm = 0.0;
}
if (insubs) {
/* Set-up values if we are using subbands */
char *tmpname, *root, *suffix;
if (split_root_suffix(s.filenames[0], &root, &suffix) == 0) {
printf("Error: The input filename (%s) must have a suffix!\n\n", s.filenames[0]);
exit(1);
}
if (strncmp(suffix, "sub", 3) == 0) {
tmpname = calloc(strlen(root) + 6, 1);
sprintf(tmpname, "%s.sub", root);
readinf(&idata, tmpname);
free(tmpname);
} else {
printf("\nThe input files (%s) must be subbands! (i.e. *.sub##)\n\n",
s.filenames[0]);
exit(1);
}
free(root);
free(suffix);
ptsperblock = 1;
/* Compensate for the fact that we have subbands and not channels */
idata.freq = idata.freq - 0.5 * idata.chan_wid +
0.5 * idata.chan_wid * (idata.num_chan / s.num_files);
idata.chan_wid = idata.num_chan / s.num_files * idata.chan_wid;
idata.num_chan = numchan = s.num_files;
idata.dm = 0.0;
sprintf(idata.name, "%s", outfilenm);
writeinf(&idata);
s.padvals = gen_fvect(s.num_files);
for (ii = 0 ; ii < s.num_files ; ii++)
s.padvals[ii] = 0.0;
}
/* Read an input mask if wanted */
if (cmd->maskfileP) {
read_mask(cmd->maskfile, &oldmask);
printf("Read old mask information from '%s'\n\n", cmd->maskfile);
good_padvals = determine_padvals(cmd->maskfile, &oldmask, s.padvals);
} else {
oldmask.numchan = oldmask.numint = 0;
}
/* The number of data points and blocks to work with at a time */
if (cmd->blocksP) {
blocksperint = cmd->blocks;
cmd->time = blocksperint * ptsperblock * idata.dt;
} else {
blocksperint = (int) (cmd->time / (ptsperblock * idata.dt) + 0.5);
}
ptsperint = blocksperint * ptsperblock;
numint = (long long) idata.N / ptsperint;
if ((long long) idata.N % ptsperint)
numint++;
inttime = ptsperint * idata.dt;
printf("Analyzing data sections of length %d points (%.6g sec).\n",
ptsperint, inttime);
{
int *factors, numfactors;
factors = get_prime_factors(ptsperint, &numfactors);
printf(" Prime factors are: ");
for (ii = 0; ii < numfactors; ii++)
printf("%d ", factors[ii]);
printf("\n");
if (factors[numfactors - 1] > 13) {
printf(" WARNING: The largest prime factor is pretty big! This will\n"
" cause the FFTs to take a long time to compute. I\n"
" recommend choosing a different -time value.\n");
}
printf("\n");
free(factors);
}
/* Allocate our workarrays */
if (RAWDATA)
rawdata = gen_fvect(idata.num_chan * ptsperblock * blocksperint);
else if (insubs)
srawdata = gen_svect(idata.num_chan * ptsperblock * blocksperint);
dataavg = gen_fmatrix(numint, numchan);
datastd = gen_fmatrix(numint, numchan);
datapow = gen_fmatrix(numint, numchan);
chandata = gen_fvect(ptsperint);
bytemask = gen_bmatrix(numint, numchan);
for (ii = 0; ii < numint; ii++)
for (jj = 0; jj < numchan; jj++)
bytemask[ii][jj] = GOODDATA;
rfivect = rfi_vector(rfivect, numchan, numint, 0, numrfivect);
if (numbetween == 2)
interptype = INTERBIN;
else
interptype = INTERPOLATE;
/* Main loop */
printf("Writing mask data to '%s'.\n", maskfilenm);
printf("Writing RFI data to '%s'.\n", rfifilenm);
printf("Writing statistics to '%s'.\n\n", statsfilenm);
printf("Massaging the data ...\n\n");
printf("Amount Complete = %3d%%", oldper);
fflush(stdout);
for (ii = 0; ii < numint; ii++) { /* Loop over the intervals */
newper = (int) ((float) ii / numint * 100.0 + 0.5);
if (newper > oldper) {
printf("\rAmount Complete = %3d%%", newper);
fflush(stdout);
oldper = newper;
}
/* Read a chunk of data */
if (RAWDATA)
numread = read_rawblocks(rawdata, blocksperint, &s, &padding);
else if (insubs)
numread = read_subband_rawblocks(s.files, s.num_files,
srawdata, blocksperint, &padding);
if (padding)
for (jj = 0; jj < numchan; jj++)
bytemask[ii][jj] |= PADDING;
for (jj = 0; jj < numchan; jj++) { /* Loop over the channels */
if (RAWDATA)
get_channel(chandata, jj, blocksperint, rawdata, &s);
else if (insubs)
get_subband(jj, chandata, srawdata, blocksperint);
/* Calculate the averages and standard deviations */
/* for each point in time. */
if (padding) {
dataavg[ii][jj] = 0.0;
datastd[ii][jj] = 0.0;
datapow[ii][jj] = 1.0;
} else {
avg_var(chandata, ptsperint, &davg, &dvar);
dataavg[ii][jj] = davg;
datastd[ii][jj] = sqrt(dvar);
realfft(chandata, ptsperint, -1);
numcands = 0;
norm = datastd[ii][jj] * datastd[ii][jj] * ptsperint;
if (norm == 0.0)
norm = (chandata[0] == 0.0) ? 1.0 : chandata[0];
cands = search_fft((fcomplex *) chandata, ptsperint / 2,
lobin, ptsperint / 2, harmsum,
numbetween, interptype, norm, cmd->freqsigma,
&numcands, &powavg, &powstd, &powmax);
datapow[ii][jj] = powmax;
/* Record the birdies */
if (numcands) {
for (kk = 0; kk < numcands; kk++) {
freq = cands[kk].r / inttime;
candnum = find_rfi(rfivect, numrfi, freq, RFI_FRACTERROR);
if (candnum >= 0) {
update_rfi(rfivect + candnum, freq, cands[kk].sig, jj, ii);
} else {
update_rfi(rfivect + numrfi, freq, cands[kk].sig, jj, ii);
numrfi++;
if (numrfi == numrfivect) {
numrfivect *= 2;
rfivect = rfi_vector(rfivect, numchan, numint,
numrfivect / 2, numrfivect);
}
}
}
free(cands);
}
}
}
}
printf("\rAmount Complete = 100%%\n");
/* Write the data to the output files */
write_rfifile(rfifilenm, rfivect, numrfi, numchan, numint,
ptsperint, lobin, numbetween, harmsum,
fracterror, cmd->freqsigma);
write_statsfile(statsfilenm, datapow[0], dataavg[0], datastd[0],
numchan, numint, ptsperint, lobin, numbetween);
} else { /* If "-nocompute" */
float freqsigma;
/* Read the data from the output files */
printf("Reading RFI data from '%s'.\n", rfifilenm);
printf("Reading statistics from '%s'.\n", statsfilenm);
readinf(&idata, outfilenm);
read_rfifile(rfifilenm, &rfivect, &numrfi, &numchan, &numint,
&ptsperint, &lobin, &numbetween, &harmsum,
&fracterror, &freqsigma);
numrfivect = numrfi;
read_statsfile(statsfilenm, &datapow, &dataavg, &datastd,
&numchan, &numint, &ptsperint, &lobin, &numbetween);
bytemask = gen_bmatrix(numint, numchan);
printf("Reading bytemask from '%s'.\n\n", bytemaskfilenm);
bytemaskfile = chkfopen(bytemaskfilenm, "rb");
chkfread(bytemask[0], numint * numchan, 1, bytemaskfile);
fclose(bytemaskfile);
for (ii = 0; ii < numint; ii++)
for (jj = 0; jj < numchan; jj++)
bytemask[ii][jj] &= PADDING; /* Clear all but the PADDING bits */
inttime = ptsperint * idata.dt;
}
/* Make the plots and set the mask */
{
int *zapints, *zapchan;
int numzapints = 0, numzapchan = 0;
if (cmd->zapintsstrP) {
zapints = ranges_to_ivect(cmd->zapintsstr, 0, numint - 1, &numzapints);
zapints = (int *) realloc(zapints, (size_t) (sizeof(int) * numint));
} else {
zapints = gen_ivect(numint);
}
if (cmd->zapchanstrP) {
zapchan = ranges_to_ivect(cmd->zapchanstr, 0, numchan - 1, &numzapchan);
zapchan = (int *) realloc(zapchan, (size_t) (sizeof(int) * numchan));
} else {
zapchan = gen_ivect(numchan);
}
rfifind_plot(numchan, numint, ptsperint, cmd->timesigma, cmd->freqsigma,
cmd->inttrigfrac, cmd->chantrigfrac,
dataavg, datastd, datapow, zapchan, numzapchan,
zapints, numzapints, &idata, bytemask,
&oldmask, &newmask, rfivect, numrfi,
cmd->rfixwinP, cmd->rfipsP, cmd->xwinP);
vect_free(zapints);
vect_free(zapchan);
}
/* Write the new mask and bytemask to the file */
write_mask(maskfilenm, &newmask);
bytemaskfile = chkfopen(bytemaskfilenm, "wb");
chkfwrite(bytemask[0], numint * numchan, 1, bytemaskfile);
fclose(bytemaskfile);
/* Determine the percent of good and bad data */
{
int numpad = 0, numbad = 0, numgood = 0;
for (ii = 0; ii < numint; ii++) {
for (jj = 0; jj < numchan; jj++) {
if (bytemask[ii][jj] == GOODDATA) {
numgood++;
} else {
if (bytemask[ii][jj] & PADDING)
numpad++;
else
numbad++;
}
}
}
printf("\nTotal number of intervals in the data: %d\n\n", numint * numchan);
printf(" Number of padded intervals: %7d (%6.3f%%)\n",
numpad, (float) numpad / (float) (numint * numchan) * 100.0);
printf(" Number of good intervals: %7d (%6.3f%%)\n",
numgood, (float) numgood / (float) (numint * numchan) * 100.0);
printf(" Number of bad intervals: %7d (%6.3f%%)\n\n",
numbad, (float) numbad / (float) (numint * numchan) * 100.0);
qsort(rfivect, numrfi, sizeof(rfi), compare_rfi_sigma);
printf(" Ten most significant birdies:\n");
printf("# Sigma Period(ms) Freq(Hz) Number \n");
printf("----------------------------------------------------\n");
for (ii = 0; ii < 10; ii++) {
double pperr;
char temp1[40], temp2[40];
if (rfivect[ii].freq_var == 0.0) {
pperr = 0.0;
sprintf(temp1, " %-14g", rfivect[ii].freq_avg);
sprintf(temp2, " %-14g", 1000.0 / rfivect[ii].freq_avg);
} else {
pperr = 1000.0 * sqrt(rfivect[ii].freq_var) /
(rfivect[ii].freq_avg * rfivect[ii].freq_avg);
nice_output_2(temp1, rfivect[ii].freq_avg, sqrt(rfivect[ii].freq_var),
-15);
nice_output_2(temp2, 1000.0 / rfivect[ii].freq_avg, pperr, -15);
}
printf("%-2d %-8.2f %13s %13s %-8d\n", ii + 1, rfivect[ii].sigma_avg,
temp2, temp1, rfivect[ii].numobs);
}
qsort(rfivect, numrfi, sizeof(rfi), compare_rfi_numobs);
printf("\n Ten most numerous birdies:\n");
printf("# Number Period(ms) Freq(Hz) Sigma \n");
printf("----------------------------------------------------\n");
for (ii = 0; ii < 10; ii++) {
double pperr;
char temp1[40], temp2[40];
if (rfivect[ii].freq_var == 0.0) {
pperr = 0.0;
sprintf(temp1, " %-14g", rfivect[ii].freq_avg);
sprintf(temp2, " %-14g", 1000.0 / rfivect[ii].freq_avg);
} else {
pperr = 1000.0 * sqrt(rfivect[ii].freq_var) /
(rfivect[ii].freq_avg * rfivect[ii].freq_avg);
nice_output_2(temp1, rfivect[ii].freq_avg, sqrt(rfivect[ii].freq_var),
-15);
nice_output_2(temp2, 1000.0 / rfivect[ii].freq_avg, pperr, -15);
}
printf("%-2d %-8d %13s %13s %-8.2f\n", ii + 1, rfivect[ii].numobs,
temp2, temp1, rfivect[ii].sigma_avg);
}
printf("\nDone.\n\n");
}
/* Close the files and cleanup */
free_rfi_vector(rfivect, numrfivect);
free_mask(newmask);
if (cmd->maskfileP)
free_mask(oldmask);
free(outfilenm);
free(statsfilenm);
free(bytemaskfilenm);
free(maskfilenm);
free(rfifilenm);
vect_free(dataavg[0]);
vect_free(dataavg);
vect_free(datastd[0]);
vect_free(datastd);
vect_free(datapow[0]);
vect_free(datapow);
vect_free(bytemask[0]);
vect_free(bytemask);
if (!cmd->nocomputeP) {
// Close all the raw files and free their vectors
close_rawfiles(&s);
vect_free(chandata);
if (insubs)
vect_free(srawdata);
else
vect_free(rawdata);
}
return (0);
}
/* NEW Clipping Routine (uses channel running averages) */
int clip_times(float *rawdata, int ptsperblk, int numchan, float clip_sigma,
float *good_chan_levels)
// Perform time-domain clipping of rawdata. This is a 2D array with
// ptsperblk*numchan points, each of which is a float. The clipping
// is done at clip_sigma sigma above/below the running mean. The
// up-to-date running averages of the channels are returned in
// good_chan_levels (which must be pre-allocated).
{
static float *chan_running_avg;
static float running_avg = 0.0, running_std = 0.0;
static int blocksread = 0, firsttime = 1;
static long long current_point = 0;
static int numonoff = 0, onoffindex = 0;
static long long *onbins = NULL, *offbins = NULL;
float *zero_dm_block, *ftmp, *powptr;
double *chan_avg_temp;
float current_med, trigger;
double current_avg = 0.0, current_std = 0.0;
int ii, jj, clipit = 0, clipped = 0;
if (firsttime) {
chan_running_avg = gen_fvect(numchan);
firsttime = 0;
{ // This is experimental code to zap radar-filled data
char *envval = getenv("CLIPBINSFILE");
if (envval != NULL) {
FILE *onofffile = chkfopen(envval, "r");
numonoff = read_onoff_paris(onofffile, &onbins, &offbins);
fclose(onofffile);
printf("\nRead %d bin clipping pairs from '%s'.\n", numonoff,
envval);
//for (ii=0;ii<numonoff;ii++) printf("%lld %lld\n", onbins[ii], offbins[ii]);
}
}
}
chan_avg_temp = gen_dvect(numchan);
zero_dm_block = gen_fvect(ptsperblk);
ftmp = gen_fvect(ptsperblk);
/* Calculate the zero DM time series */
for (ii = 0; ii < ptsperblk; ii++) {
zero_dm_block[ii] = 0.0;
powptr = rawdata + ii * numchan;
for (jj = 0; jj < numchan; jj++)
zero_dm_block[ii] += *powptr++;
ftmp[ii] = zero_dm_block[ii];
}
avg_var(ftmp, ptsperblk, ¤t_avg, ¤t_std);
current_std = sqrt(current_std);
current_med = median(ftmp, ptsperblk);
/* Calculate the current standard deviation and mean */
/* but only for data points that are within a certain */
/* fraction of the median value. This removes the */
/* really strong RFI from the calculation. */
{
float lo_cutoff, hi_cutoff;
int numgoodpts = 0;
lo_cutoff = current_med - 3.0 * current_std;
hi_cutoff = current_med + 3.0 * current_std;;
for (jj = 0; jj < numchan; jj++)
chan_avg_temp[jj] = 0.0;
/* Find the "good" points */
for (ii = 0; ii < ptsperblk; ii++) {
if (zero_dm_block[ii] > lo_cutoff && zero_dm_block[ii] < hi_cutoff) {
ftmp[numgoodpts] = zero_dm_block[ii];
powptr = rawdata + ii * numchan;
for (jj = 0; jj < numchan; jj++)
chan_avg_temp[jj] += *powptr++;
numgoodpts++;
}
}
//printf("avg = %f med = %f std = %f numgoodpts = %d\n",
// current_avg, current_med, current_std, numgoodpts);
/* Calculate the current average and stddev */
if (numgoodpts < 1) {
current_avg = running_avg;
current_std = running_std;
for (jj = 0; jj < numchan; jj++)
chan_avg_temp[jj] = chan_running_avg[jj];
} else {
avg_var(ftmp, numgoodpts, ¤t_avg, ¤t_std);
current_std = sqrt(current_std);
for (jj = 0; jj < numchan; jj++)
chan_avg_temp[jj] /= numgoodpts;
}
}
/* Update a pseudo running average and stdev */
if (blocksread) {
running_avg = 0.9 * running_avg + 0.1 * current_avg;
running_std = 0.9 * running_std + 0.1 * current_std;
for (ii = 0; ii < numchan; ii++)
chan_running_avg[ii] =
0.9 * chan_running_avg[ii] + 0.1 * chan_avg_temp[ii];
} else {
running_avg = current_avg;
running_std = current_std;
for (ii = 0; ii < numchan; ii++)
chan_running_avg[ii] = chan_avg_temp[ii];
if (current_avg == 0.0)
printf("Warning: problem with clipping in first block!!!\n\n");
}
/* See if any points need clipping */
trigger = clip_sigma * running_std;
for (ii = 0; ii < ptsperblk; ii++) {
if (fabs(zero_dm_block[ii] - running_avg) > trigger) {
clipit = 1;
break;
}
}
/* or alternatively from the CLIPBINSFILE */
if (numonoff && ((current_point > onbins[onoffindex]
&& current_point <= offbins[onoffindex])
|| (current_point + ptsperblk > onbins[onoffindex]
&& current_point + ptsperblk <= offbins[onoffindex])
|| (current_point < onbins[onoffindex]
&& current_point + ptsperblk > offbins[onoffindex])))
clipit = 1;
/* Update the good channel levels */
for (ii = 0; ii < numchan; ii++)
good_chan_levels[ii] = chan_running_avg[ii];
/* Replace the bad channel data with channel median values */
/* that are scaled to equal the running_avg. */
if (clipit) {
for (ii = 0; ii < ptsperblk; ii++) {
if ((fabs(zero_dm_block[ii] - running_avg) > trigger) ||
(numonoff && (current_point > onbins[onoffindex]
&& current_point <= offbins[onoffindex]))) {
powptr = rawdata + ii * numchan;
for (jj = 0; jj < numchan; jj++)
*powptr++ = good_chan_levels[jj];
clipped++;
//fprintf(stderr, "zapping %lld\n", current_point);
}
current_point++;
if (numonoff && current_point > offbins[onoffindex]
&& onoffindex < numonoff - 1) {
while (current_point > offbins[onoffindex]
&& onoffindex < numonoff - 1)
onoffindex++;
//printf("updating index to %d\n", onoffindex);
}
}
} else {
current_point += ptsperblk;
if (numonoff && current_point > offbins[onoffindex]
&& onoffindex < numonoff - 1) {
while (current_point > offbins[onoffindex]
&& onoffindex < numonoff - 1)
onoffindex++;
//printf("updating index to %d\n", onoffindex);
}
}
blocksread++;
vect_free(chan_avg_temp);
vect_free(zero_dm_block);
vect_free(ftmp);
return clipped;
}
/* OLD Clipping Routine (uses channel medians) */
int old_clip_times(float *rawdata, int ptsperblk, int numchan, float clip_sigma,
float *good_chan_levels)
// Perform time-domain clipping of rawdata. This is a 2D array with
// ptsperblk*numchan points, each of which is a float. The clipping
// is done at clip_sigma sigma above/below the running mean. The
// up-to-date running averages of the channels are returned in
// good_chan_levels (which must be pre-allocated).
{
static float *median_chan_levels;
static float running_avg = 0.0, running_std = 0.0, median_sum = 0.0;
static int blocksread = 0, firsttime = 1;
float *zero_dm_block, *median_temp;
float current_med, trigger, running_wgt = 0.05;
double current_avg = 0.0, current_std = 0.0, scaling;
float *powptr;
int ii, jj, clipit = 0, clipped = 0;
if (firsttime) {
median_chan_levels = gen_fvect(numchan);
firsttime = 0;
}
zero_dm_block = gen_fvect(ptsperblk);
median_temp = gen_fvect(ptsperblk);
/* Calculate the zero DM time series */
for (ii = 0; ii < ptsperblk; ii++) {
zero_dm_block[ii] = 0.0;
powptr = rawdata + ii * numchan;
for (jj = 0; jj < numchan; jj++)
zero_dm_block[ii] += *powptr++;
median_temp[ii] = zero_dm_block[ii];
}
current_med = median(median_temp, ptsperblk);
/* Calculate the current standard deviation and mean */
/* but only for data points that are within a certain */
/* fraction of the median value. This removes the */
/* really strong RFI from the calculation. */
{
float lo_cutoff, hi_cutoff;
int numgoodpts = 0;
lo_cutoff = 0.7 * current_med;
hi_cutoff = 1.3 * current_med;
/* Find the "good" points */
for (ii = 0; ii < ptsperblk; ii++) {
if (zero_dm_block[ii] > lo_cutoff && zero_dm_block[ii] < hi_cutoff) {
median_temp[numgoodpts] = zero_dm_block[ii];
numgoodpts++;
}
}
/* Calculate the current average and stddev */
if (numgoodpts < 1) {
current_avg = running_avg;
current_std = running_std;
} else {
avg_var(median_temp, numgoodpts, ¤t_avg, ¤t_std);
current_std = sqrt(current_std);
}
}
/* Update a pseudo running average and stdev */
if (blocksread) {
running_avg =
(running_avg * (1.0 - running_wgt) + running_wgt * current_avg);
running_std =
(running_std * (1.0 - running_wgt) + running_wgt * current_std);
} else {
running_avg = current_avg;
running_std = current_std;
if (running_avg == 0.0 || current_avg == 0.0)
printf("BAD RFI IN BLOCK#1!!!\n\n");
}
/* See if any points need clipping */
trigger = clip_sigma * running_std;
for (ii = 0; ii < ptsperblk; ii++) {
if (fabs(zero_dm_block[ii] - running_avg) > trigger) {
clipit = 1;
break;
}
}
/* Calculate the channel medians if required */
if ((blocksread % BLOCKSTOSKIP == 0 && clipit == 0) || blocksread == 0) {
median_sum = 0.0;
for (ii = 0; ii < numchan; ii++) {
powptr = rawdata + ii;
for (jj = 0; jj < ptsperblk; jj++)
median_temp[jj] = *(powptr + jj * numchan);
median_chan_levels[ii] = median(median_temp, ptsperblk);
median_sum += median_chan_levels[ii];
}
}
/* Update the good channel levels */
scaling = running_avg / median_sum;
for (ii = 0; ii < numchan; ii++)
good_chan_levels[ii] = median_chan_levels[ii] * scaling;
/* Replace the bad channel data with channel median values */
/* that are scaled to equal the running_avg. */
if (clipit) {
for (ii = 0; ii < ptsperblk; ii++) {
if (fabs(zero_dm_block[ii] - running_avg) > trigger) {
powptr = rawdata + ii * numchan;
for (jj = 0; jj < numchan; jj++)
*powptr++ = good_chan_levels[jj];
clipped++;
}
}
}
blocksread++;
vect_free(zero_dm_block);
vect_free(median_temp);
return clipped;
}
int main(int argc, char *argv[])
{
/* Any variable that begins with 't' means topocentric */
/* Any variable that begins with 'b' means barycentric */
FILE **outfiles = NULL;
float **outdata;
double dtmp, *dms, avgdm = 0.0, dsdt = 0, maxdm;
double *dispdt, tlotoa = 0.0, blotoa = 0.0, BW_ddelay = 0.0;
double max = -9.9E30, min = 9.9E30, var = 0.0, avg = 0.0;
double *btoa = NULL, *ttoa = NULL, avgvoverc = 0.0;
char obs[3], ephem[10], rastring[50], decstring[50];
long totnumtowrite, totwrote = 0, padwrote = 0, datawrote = 0;
int *idispdt, **offsets;
int ii, jj, numadded = 0, numremoved = 0, padding = 0, good_inputs = 1;
int numbarypts = 0, numread = 0, numtowrite = 0;
int padtowrite = 0, statnum = 0;
int numdiffbins = 0, *diffbins = NULL, *diffbinptr = NULL, good_padvals = 0;
double local_lodm;
char *datafilenm, *outpath, *outfilenm, *hostname;
struct spectra_info s;
infodata idata;
mask obsmask;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
#ifdef _OPENMP
omp_set_num_threads(1); // Explicitly turn off OpenMP
#endif
set_using_MPI();
{
FILE *hostfile;
char tmpname[100];
int retval;
hostfile = chkfopen("/etc/hostname", "r");
retval = fscanf(hostfile, "%s\n", tmpname);
if (retval==0) {
printf("Warning: error reading /etc/hostname on proc %d\n", myid);
}
hostname = (char *) calloc(strlen(tmpname) + 1, 1);
memcpy(hostname, tmpname, strlen(tmpname));
fclose(hostfile);
}
/* Call usage() if we have no command line arguments */
if (argc == 1) {
if (myid == 0) {
Program = argv[0];
usage();
}
MPI_Finalize();
exit(1);
}
make_maskbase_struct();
make_spectra_info_struct();
/* Parse the command line using the excellent program Clig */
cmd = parseCmdline(argc, argv);
spectra_info_set_defaults(&s);
// If we are zeroDMing, make sure that clipping is off.
if (cmd->zerodmP) cmd->noclipP = 1;
s.clip_sigma = cmd->clip;
if (cmd->noclipP) {
cmd->clip = 0.0;
s.clip_sigma = 0.0;
}
if (cmd->ifsP) {
// 0 = default or summed, 1-4 are possible also
s.use_poln = cmd->ifs + 1;
}
if (!cmd->numoutP)
cmd->numout = LONG_MAX;
#ifdef DEBUG
showOptionValues();
#endif
if (myid == 0) { /* Master node only */
printf("\n\n");
printf(" Parallel Pulsar Subband De-dispersion Routine\n");
printf(" by Scott M. Ransom\n\n");
s.filenames = cmd->argv;
s.num_files = cmd->argc;
s.clip_sigma = cmd->clip;
// -1 causes the data to determine if we use weights, scales, &
// offsets for PSRFITS or flip the band for any data type where
// we can figure that out with the data
s.apply_flipband = (cmd->invertP) ? 1 : -1;
s.apply_weight = (cmd->noweightsP) ? 0 : -1;
s.apply_scale = (cmd->noscalesP) ? 0 : -1;
s.apply_offset = (cmd->nooffsetsP) ? 0 : -1;
s.remove_zerodm = (cmd->zerodmP) ? 1 : 0;
if (RAWDATA) {
if (cmd->filterbankP) s.datatype = SIGPROCFB;
else if (cmd->psrfitsP) s.datatype = PSRFITS;
else if (cmd->pkmbP) s.datatype = SCAMP;
else if (cmd->bcpmP) s.datatype = BPP;
else if (cmd->wappP) s.datatype = WAPP;
else if (cmd->spigotP) s.datatype = SPIGOT;
} else { // Attempt to auto-identify the data
identify_psrdatatype(&s, 1);
if (s.datatype==SIGPROCFB) cmd->filterbankP = 1;
else if (s.datatype==PSRFITS) cmd->psrfitsP = 1;
else if (s.datatype==SCAMP) cmd->pkmbP = 1;
else if (s.datatype==BPP) cmd->bcpmP = 1;
else if (s.datatype==WAPP) cmd->wappP = 1;
else if (s.datatype==SPIGOT) cmd->spigotP = 1;
else if (s.datatype==SUBBAND) insubs = 1;
else {
printf("\nError: Unable to identify input data files. Please specify type.\n\n");
good_inputs = 0;
}
}
// So far we can only handle PSRFITS, filterbank, and subbands
if (s.datatype!=PSRFITS &&
s.datatype!=SIGPROCFB &&
s.datatype!=SUBBAND) good_inputs = 0;
// For subbanded data
if (!RAWDATA) s.files = (FILE **)malloc(sizeof(FILE *) * s.num_files);
if (good_inputs && (RAWDATA || insubs)) {
char description[40];
psrdatatype_description(description, s.datatype);
if (s.num_files > 1)
printf("Reading %s data from %d files:\n", description, s.num_files);
else
printf("Reading %s data from 1 file:\n", description);
for (ii = 0; ii < s.num_files; ii++) {
printf(" '%s'\n", cmd->argv[ii]);
if (insubs) s.files[ii] = chkfopen(s.filenames[ii], "rb");
}
printf("\n");
if (RAWDATA) {
read_rawdata_files(&s);
print_spectra_info_summary(&s);
spectra_info_to_inf(&s, &idata);
} else { // insubs
char *root, *suffix;
cmd->nsub = s.num_files;
s.N = chkfilelen(s.files[0], sizeof(short));
s.start_subint = gen_ivect(1);
s.num_subint = gen_ivect(1);
s.start_MJD = (long double *)malloc(sizeof(long double));
s.start_spec = (long long *)malloc(sizeof(long long));
s.num_spec = (long long *)malloc(sizeof(long long));
s.num_pad = (long long *)malloc(sizeof(long long));
s.start_spec[0] = 0L;
s.start_subint[0] = 0;
s.num_spec[0] = s.N;
s.num_subint[0] = s.N / SUBSBLOCKLEN;
s.num_pad[0] = 0L;
s.padvals = gen_fvect(s.num_files);
for (ii = 0 ; ii < ii ; ii++)
s.padvals[ii] = 0.0;
if (split_root_suffix(s.filenames[0], &root, &suffix) == 0) {
printf("\nError: The input filename (%s) must have a suffix!\n\n", s.filenames[0]);
exit(1);
}
if (strncmp(suffix, "sub", 3) == 0) {
char *tmpname;
tmpname = calloc(strlen(root) + 10, 1);
sprintf(tmpname, "%s.sub", root);
readinf(&idata, tmpname);
free(tmpname);
strncpy(s.telescope, idata.telescope, 40);
strncpy(s.backend, idata.instrument, 40);
strncpy(s.observer, idata.observer, 40);
strncpy(s.source, idata.object, 40);
s.ra2000 = hms2rad(idata.ra_h, idata.ra_m,
idata.ra_s) * RADTODEG;
s.dec2000 = dms2rad(idata.dec_d, idata.dec_m,
idata.dec_s) * RADTODEG;
ra_dec_to_string(s.ra_str,
idata.ra_h, idata.ra_m, idata.ra_s);
ra_dec_to_string(s.dec_str,
idata.dec_d, idata.dec_m, idata.dec_s);
s.num_channels = idata.num_chan;
s.start_MJD[0] = idata.mjd_i + idata.mjd_f;
s.dt = idata.dt;
s.T = s.N * s.dt;
s.lo_freq = idata.freq;
s.df = idata.chan_wid;
s.hi_freq = s.lo_freq + (s.num_channels - 1.0) * s.df;
s.BW = s.num_channels * s.df;
s.fctr = s.lo_freq - 0.5 * s.df + 0.5 * s.BW;
s.beam_FWHM = idata.fov / 3600.0;
s.spectra_per_subint = SUBSBLOCKLEN;
print_spectra_info_summary(&s);
} else {
printf("\nThe input files (%s) must be subbands! (i.e. *.sub##)\n\n",
cmd->argv[0]);
MPI_Finalize();
exit(1);
}
free(root);
free(suffix);
}
}
}
// If we don't have good input data, exit
MPI_Bcast(&good_inputs, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (!good_inputs) {
MPI_Finalize();
exit(1);
}
MPI_Bcast(&insubs, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (insubs)
cmd->nsub = cmd->argc;
/* Determine the output file names and open them */
local_numdms = cmd->numdms / (numprocs - 1);
dms = gen_dvect(local_numdms);
if (cmd->numdms % (numprocs - 1)) {
if (myid == 0)
printf
("\nThe number of DMs must be divisible by (the number of processors - 1).\n\n");
MPI_Finalize();
exit(1);
}
local_lodm = cmd->lodm + (myid - 1) * local_numdms * cmd->dmstep;
split_path_file(cmd->outfile, &outpath, &outfilenm);
datafilenm = (char *) calloc(strlen(outfilenm) + 20, 1);
if (myid > 0) {
if (chdir(outpath) == -1) {
printf("\nProcess %d on %s cannot chdir() to '%s'. Exiting.\n\n",
myid, hostname, outpath);
MPI_Finalize();
exit(1);
}
outfiles = (FILE **) malloc(local_numdms * sizeof(FILE *));
for (ii = 0; ii < local_numdms; ii++) {
dms[ii] = local_lodm + ii * cmd->dmstep;
avgdm += dms[ii];
sprintf(datafilenm, "%s_DM%.2f.dat", outfilenm, dms[ii]);
outfiles[ii] = chkfopen(datafilenm, "wb");
}
avgdm /= local_numdms;
}
// Broadcast the raw data information
broadcast_spectra_info(&s, myid);
if (myid > 0) {
spectra_info_to_inf(&s, &idata);
if (s.datatype==SIGPROCFB) cmd->filterbankP = 1;
else if (s.datatype==PSRFITS) cmd->psrfitsP = 1;
else if (s.datatype==SCAMP) cmd->pkmbP = 1;
else if (s.datatype==BPP) cmd->bcpmP = 1;
else if (s.datatype==WAPP) cmd->wappP = 1;
else if (s.datatype==SPIGOT) cmd->spigotP = 1;
else if (s.datatype==SUBBAND) insubs = 1;
}
s.filenames = cmd->argv;
/* Read an input mask if wanted */
if (myid > 0) {
int numpad = s.num_channels;
if (insubs)
numpad = s.num_files;
s.padvals = gen_fvect(numpad);
for (ii = 0 ; ii < numpad ; ii++)
s.padvals[ii] = 0.0;
}
if (cmd->maskfileP) {
if (myid == 0) {
read_mask(cmd->maskfile, &obsmask);
printf("Read mask information from '%s'\n\n", cmd->maskfile);
good_padvals = determine_padvals(cmd->maskfile, &obsmask, s.padvals);
}
broadcast_mask(&obsmask, myid);
MPI_Bcast(&good_padvals, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(s.padvals, obsmask.numchan, MPI_FLOAT, 0, MPI_COMM_WORLD);
} else {
obsmask.numchan = obsmask.numint = 0;
MPI_Bcast(&good_padvals, 1, MPI_INT, 0, MPI_COMM_WORLD);
}
// The number of topo to bary time points to generate with TEMPO
numbarypts = (int) (s.T * 1.1 / TDT + 5.5) + 1;
// Identify the TEMPO observatory code
{
char *outscope = (char *) calloc(40, sizeof(char));
telescope_to_tempocode(idata.telescope, outscope, obs);
free(outscope);
}
// Broadcast or calculate a few extra important values
if (insubs) avgdm = idata.dm;
idata.dm = avgdm;
dsdt = cmd->downsamp * idata.dt;
maxdm = cmd->lodm + cmd->numdms * cmd->dmstep;
BW_ddelay = delay_from_dm(maxdm, idata.freq) -
delay_from_dm(maxdm, idata.freq + (idata.num_chan-1) * idata.chan_wid);
blocksperread = ((int) (BW_ddelay / idata.dt) / s.spectra_per_subint + 1);
worklen = s.spectra_per_subint * blocksperread;
if (cmd->nsub > s.num_channels) {
printf
("Warning: The number of requested subbands (%d) is larger than the number of channels (%d).\n",
cmd->nsub, s.num_channels);
printf(" Re-setting the number of subbands to %d.\n\n", s.num_channels);
cmd->nsub = s.num_channels;
}
if (s.spectra_per_subint % cmd->downsamp) {
if (myid == 0) {
printf
("\nError: The downsample factor (%d) must be a factor of the\n",
cmd->downsamp);
printf(" blocklength (%d). Exiting.\n\n", s.spectra_per_subint);
}
MPI_Finalize();
exit(1);
}
tlotoa = idata.mjd_i + idata.mjd_f; /* Topocentric epoch */
if (cmd->numoutP)
totnumtowrite = cmd->numout;
else
totnumtowrite = (long) idata.N / cmd->downsamp;
if (cmd->nobaryP) { /* Main loop if we are not barycentering... */
/* Dispersion delays (in bins). The high freq gets no delay */
/* All other delays are positive fractions of bin length (dt) */
dispdt = subband_search_delays(s.num_channels, cmd->nsub, avgdm,
idata.freq, idata.chan_wid, 0.0);
idispdt = gen_ivect(s.num_channels);
for (ii = 0; ii < s.num_channels; ii++)
idispdt[ii] = NEAREST_LONG(dispdt[ii] / idata.dt);
vect_free(dispdt);
/* The subband dispersion delays (see note above) */
offsets = gen_imatrix(local_numdms, cmd->nsub);
for (ii = 0; ii < local_numdms; ii++) {
double *subdispdt;
subdispdt = subband_delays(s.num_channels, cmd->nsub, dms[ii],
idata.freq, idata.chan_wid, 0.0);
dtmp = subdispdt[cmd->nsub - 1];
for (jj = 0; jj < cmd->nsub; jj++)
offsets[ii][jj] = NEAREST_LONG((subdispdt[jj] - dtmp) / dsdt);
vect_free(subdispdt);
}
/* Allocate our data array and start getting data */
if (myid == 0) {
printf("De-dispersing using %d subbands.\n", cmd->nsub);
if (cmd->downsamp > 1)
printf("Downsampling by a factor of %d (new dt = %.10g)\n",
cmd->downsamp, dsdt);
printf("\n");
}
/* Print the nodes and the DMs they are handling */
print_dms(hostname, myid, numprocs, local_numdms, dms);
outdata = gen_fmatrix(local_numdms, worklen / cmd->downsamp);
numread = get_data(outdata, blocksperread, &s,
&obsmask, idispdt, offsets, &padding);
while (numread == worklen) {
numread /= cmd->downsamp;
if (myid == 0)
print_percent_complete(totwrote, totnumtowrite);
/* Write the latest chunk of data, but don't */
/* write more than cmd->numout points. */
numtowrite = numread;
if (cmd->numoutP && (totwrote + numtowrite) > cmd->numout)
numtowrite = cmd->numout - totwrote;
if (myid > 0) {
write_data(outfiles, local_numdms, outdata, 0, numtowrite);
/* Update the statistics */
if (!padding) {
for (ii = 0; ii < numtowrite; ii++)
update_stats(statnum + ii, outdata[0][ii], &min, &max, &avg, &var);
statnum += numtowrite;
}
}
totwrote += numtowrite;
/* Stop if we have written out all the data we need to */
if (cmd->numoutP && (totwrote == cmd->numout))
break;
numread = get_data(outdata, blocksperread, &s,
&obsmask, idispdt, offsets, &padding);
}
datawrote = totwrote;
} else { /* Main loop if we are barycentering... */
/* What ephemeris will we use? (Default is DE405) */
strcpy(ephem, "DE405");
/* Define the RA and DEC of the observation */
ra_dec_to_string(rastring, idata.ra_h, idata.ra_m, idata.ra_s);
ra_dec_to_string(decstring, idata.dec_d, idata.dec_m, idata.dec_s);
/* Allocate some arrays */
btoa = gen_dvect(numbarypts);
ttoa = gen_dvect(numbarypts);
for (ii = 0; ii < numbarypts; ii++)
ttoa[ii] = tlotoa + TDT * ii / SECPERDAY;
/* Call TEMPO for the barycentering */
if (myid == 0) {
double maxvoverc = -1.0, minvoverc = 1.0, *voverc = NULL;
printf("\nGenerating barycentric corrections...\n");
voverc = gen_dvect(numbarypts);
barycenter(ttoa, btoa, voverc, numbarypts, rastring, decstring, obs, ephem);
for (ii = 0; ii < numbarypts; ii++) {
if (voverc[ii] > maxvoverc)
maxvoverc = voverc[ii];
if (voverc[ii] < minvoverc)
minvoverc = voverc[ii];
avgvoverc += voverc[ii];
}
avgvoverc /= numbarypts;
vect_free(voverc);
printf(" Average topocentric velocity (c) = %.7g\n", avgvoverc);
printf(" Maximum topocentric velocity (c) = %.7g\n", maxvoverc);
printf(" Minimum topocentric velocity (c) = %.7g\n\n", minvoverc);
printf("De-dispersing using %d subbands.\n", cmd->nsub);
if (cmd->downsamp > 1) {
printf(" Downsample = %d\n", cmd->downsamp);
printf(" New sample dt = %.10g\n", dsdt);
}
printf("\n");
}
/* Print the nodes and the DMs they are handling */
print_dms(hostname, myid, numprocs, local_numdms, dms);
MPI_Bcast(btoa, numbarypts, MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Bcast(&avgvoverc, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
blotoa = btoa[0];
/* Dispersion delays (in bins). The high freq gets no delay */
/* All other delays are positive fractions of bin length (dt) */
dispdt = subband_search_delays(s.num_channels, cmd->nsub, avgdm,
idata.freq, idata.chan_wid, avgvoverc);
idispdt = gen_ivect(s.num_channels);
for (ii = 0; ii < s.num_channels; ii++)
idispdt[ii] = NEAREST_LONG(dispdt[ii] / idata.dt);
vect_free(dispdt);
/* The subband dispersion delays (see note above) */
offsets = gen_imatrix(local_numdms, cmd->nsub);
for (ii = 0; ii < local_numdms; ii++) {
double *subdispdt;
subdispdt = subband_delays(s.num_channels, cmd->nsub, dms[ii],
idata.freq, idata.chan_wid, avgvoverc);
dtmp = subdispdt[cmd->nsub - 1];
for (jj = 0; jj < cmd->nsub; jj++)
offsets[ii][jj] = NEAREST_LONG((subdispdt[jj] - dtmp) / dsdt);
vect_free(subdispdt);
}
/* Convert the bary TOAs to differences from the topo TOAs in */
/* units of bin length (dt) rounded to the nearest integer. */
dtmp = (btoa[0] - ttoa[0]);
for (ii = 0; ii < numbarypts; ii++)
btoa[ii] = ((btoa[ii] - ttoa[ii]) - dtmp) * SECPERDAY / dsdt;
/* Find the points where we need to add or remove bins */
{
int oldbin = 0, currentbin;
double lobin, hibin, calcpt;
numdiffbins = abs(NEAREST_LONG(btoa[numbarypts - 1])) + 1;
diffbins = gen_ivect(numdiffbins);
diffbinptr = diffbins;
for (ii = 1; ii < numbarypts; ii++) {
currentbin = NEAREST_LONG(btoa[ii]);
if (currentbin != oldbin) {
if (currentbin > 0) {
calcpt = oldbin + 0.5;
lobin = (ii - 1) * TDT / dsdt;
hibin = ii * TDT / dsdt;
} else {
calcpt = oldbin - 0.5;
lobin = -((ii - 1) * TDT / dsdt);
hibin = -(ii * TDT / dsdt);
}
while (fabs(calcpt) < fabs(btoa[ii])) {
/* Negative bin number means remove that bin */
/* Positive bin number means add a bin there */
*diffbinptr =
NEAREST_LONG(LININTERP
(calcpt, btoa[ii - 1], btoa[ii], lobin, hibin));
diffbinptr++;
calcpt = (currentbin > 0) ? calcpt + 1.0 : calcpt - 1.0;
}
oldbin = currentbin;
}
}
*diffbinptr = cmd->numout; /* Used as a marker */
}
diffbinptr = diffbins;
/* Now perform the barycentering */
outdata = gen_fmatrix(local_numdms, worklen / cmd->downsamp);
numread = get_data(outdata, blocksperread, &s,
&obsmask, idispdt, offsets, &padding);
while (numread == worklen) { /* Loop to read and write the data */
int numwritten = 0;
double block_avg, block_var;
numread /= cmd->downsamp;
/* Determine the approximate local average */
avg_var(outdata[0], numread, &block_avg, &block_var);
if (myid == 0)
print_percent_complete(totwrote, totnumtowrite);
/* Simply write the data if we don't have to add or */
/* remove any bins from this batch. */
/* OR write the amount of data up to cmd->numout or */
/* the next bin that will be added or removed. */
numtowrite = abs(*diffbinptr) - datawrote;
if (cmd->numoutP && (totwrote + numtowrite) > cmd->numout)
numtowrite = cmd->numout - totwrote;
if (numtowrite > numread)
numtowrite = numread;
if (myid > 0) {
write_data(outfiles, local_numdms, outdata, 0, numtowrite);
/* Update the statistics */
if (!padding) {
for (ii = 0; ii < numtowrite; ii++)
update_stats(statnum + ii, outdata[0][ii], &min, &max, &avg, &var);
statnum += numtowrite;
}
}
datawrote += numtowrite;
totwrote += numtowrite;
numwritten += numtowrite;
if ((datawrote == abs(*diffbinptr)) &&
(numwritten != numread) &&
(totwrote < cmd->numout)) { /* Add/remove a bin */
int skip, nextdiffbin;
skip = numtowrite;
/* Write the rest of the data after adding/removing a bin */
do {
if (*diffbinptr > 0) {
/* Add a bin */
if (myid > 0)
write_padding(outfiles, local_numdms, block_avg, 1);
numadded++;
totwrote++;
} else {
/* Remove a bin */
numremoved++;
datawrote++;
numwritten++;
skip++;
}
diffbinptr++;
/* Write the part after the diffbin */
numtowrite = numread - numwritten;
if (cmd->numoutP && (totwrote + numtowrite) > cmd->numout)
numtowrite = cmd->numout - totwrote;
nextdiffbin = abs(*diffbinptr) - datawrote;
if (numtowrite > nextdiffbin)
numtowrite = nextdiffbin;
if (myid > 0) {
write_data(outfiles, local_numdms, outdata, skip, numtowrite);
/* Update the statistics and counters */
if (!padding) {
for (ii = 0; ii < numtowrite; ii++)
update_stats(statnum + ii,
outdata[0][skip + ii], &min, &max, &avg, &var);
statnum += numtowrite;
}
}
numwritten += numtowrite;
datawrote += numtowrite;
totwrote += numtowrite;
skip += numtowrite;
/* Stop if we have written out all the data we need to */
if (cmd->numoutP && (totwrote == cmd->numout))
break;
} while (numwritten < numread);
}
/* Stop if we have written out all the data we need to */
if (cmd->numoutP && (totwrote == cmd->numout))
break;
numread = get_data(outdata, blocksperread, &s,
&obsmask, idispdt, offsets, &padding);
}
}
if (myid > 0) {
/* Calculate the amount of padding we need */
if (cmd->numoutP && (cmd->numout > totwrote))
padwrote = padtowrite = cmd->numout - totwrote;
/* Write the new info file for the output data */
idata.dt = dsdt;
update_infodata(&idata, totwrote, padtowrite, diffbins,
numdiffbins, cmd->downsamp);
for (ii = 0; ii < local_numdms; ii++) {
idata.dm = dms[ii];
if (!cmd->nobaryP) {
double baryepoch, barydispdt, baryhifreq;
baryhifreq = idata.freq + (s.num_channels - 1) * idata.chan_wid;
barydispdt = delay_from_dm(dms[ii], doppler(baryhifreq, avgvoverc));
baryepoch = blotoa - (barydispdt / SECPERDAY);
idata.bary = 1;
idata.mjd_i = (int) floor(baryepoch);
idata.mjd_f = baryepoch - idata.mjd_i;
}
sprintf(idata.name, "%s_DM%.2f", outfilenm, dms[ii]);
writeinf(&idata);
}
/* Set the padded points equal to the average data point */
if (idata.numonoff >= 1) {
int index, startpad, endpad;
for (ii = 0; ii < local_numdms; ii++) {
fclose(outfiles[ii]);
sprintf(datafilenm, "%s_DM%.2f.dat", outfilenm, dms[ii]);
outfiles[ii] = chkfopen(datafilenm, "rb+");
}
for (ii = 0; ii < idata.numonoff; ii++) {
index = 2 * ii;
startpad = idata.onoff[index + 1];
if (ii == idata.numonoff - 1)
endpad = idata.N - 1;
else
endpad = idata.onoff[index + 2];
for (jj = 0; jj < local_numdms; jj++)
chkfseek(outfiles[jj], (startpad + 1) * sizeof(float), SEEK_SET);
padtowrite = endpad - startpad;
write_padding(outfiles, local_numdms, avg, padtowrite);
}
}
}
/* Print simple stats and results */
var /= (datawrote - 1);
if (myid == 0)
print_percent_complete(1, 1);
if (myid == 1) {
printf("\n\nDone.\n\nSimple statistics of the output data:\n");
printf(" Data points written: %ld\n", totwrote);
if (padwrote)
printf(" Padding points written: %ld\n", padwrote);
if (!cmd->nobaryP) {
if (numadded)
printf(" Bins added for barycentering: %d\n", numadded);
if (numremoved)
printf(" Bins removed for barycentering: %d\n", numremoved);
}
printf(" Maximum value of data: %.2f\n", max);
printf(" Minimum value of data: %.2f\n", min);
printf(" Data average value: %.2f\n", avg);
printf(" Data standard deviation: %.2f\n", sqrt(var));
printf("\n");
}
/* Close the files and cleanup */
if (cmd->maskfileP)
free_mask(obsmask);
if (myid > 0) {
for (ii = 0; ii < local_numdms; ii++)
fclose(outfiles[ii]);
free(outfiles);
}
vect_free(outdata[0]);
vect_free(outdata);
vect_free(dms);
free(hostname);
vect_free(idispdt);
vect_free(offsets[0]);
vect_free(offsets);
free(datafilenm);
free(outfilenm);
free(outpath);
if (!cmd->nobaryP) {
vect_free(btoa);
vect_free(ttoa);
vect_free(diffbins);
}
MPI_Finalize();
return (0);
}
int main(int argc, char *argv[])
{
Cmdline *cmd;
struct psrfits pfupper, pflower, pfo;
fitsfile *infits, *outfits;
char *pc1, *pc2;
char outfilename[200]; //Name of outfile if not specified on command line
int stat = 0, padding = 0, userN = 0, status;
// Call usage() if we have no command line arguments
if (argc == 1) {
Program = argv[0];
usage();
exit(0);
}
// Parse the command line using the excellent program Clig
cmd = parseCmdline(argc, argv);
pfupper.tot_rows = pfupper.N = pfupper.T = pfupper.status = 0; //Initialize upper band
pflower.tot_rows = pflower.N = pflower.T = pflower.status = 0; //Initialize lower band
pfupper.filenum = pflower.filenum = 1;
pfo.tot_rows = pfo.N = pfo.T = pfo.status = pfo.multifile = 0; //Initialize output
sprintf(pfupper.filename, cmd->argv[0]); //Copy filename specified on command line to
sprintf(pflower.filename, cmd->argv[0]); //upper and lower bands, will correct filenames shortly
if ((pc2 = strstr(pfupper.filename, "s1")) != NULL) //Upper contains s1, change to s0
strncpy(pc2, "s0", 2);
else if ((pc2 = strstr(pflower.filename, "s0")) != NULL) //Lower contains s0, change to s1
strncpy(pc2, "s1", 2);
else {
printf("Unable to determine which sideband is which\n");
exit(EXIT_FAILURE);
}
//Setting the name of the output file, setting as same name as input file, but removing s0/s1.
pc1 = strstr(pflower.filename, "s1");
pc2 = strrchr(pflower.filename, '.'); //At '.fits'
pc2--;
while ((pc2 >= pflower.filename) && isdigit(*pc2)) //Move through the digits to the separation char.
pc2--;
strncpy(outfilename, pflower.filename, pc1 - pflower.filename); //Copy everything up to s1 into outfilename
strncpy(outfilename + (pc1 - pflower.filename), pc1 + 2, pc2 - pc1 - 2); //Concatenate from after s1 to char before the separation char.
pc1 = outfilename + (pc2 - pflower.filename - 2);
*pc1 = 0;
int rv = psrfits_open(&pfupper); //Open upper band
if (rv) {
fits_report_error(stderr, rv);
exit(1);
}
rv = psrfits_open(&pflower); //Open lower band
if (rv) {
fits_report_error(stderr, rv);
exit(1);
}
pfo = pflower; //Copy all lower band variables into the output struct
if (!cmd->outputbasenameP)
sprintf(pfo.basefilename, basename(outfilename));
else
sprintf(pfo.basefilename, cmd->outputbasename);
pfo.filenum = 0;
sprintf(pfo.filename, "\0"); //Set filename to null so psrfits_open will create the filename for me
pfo.rownum = 1;
pfo.tot_rows = 0;
pfo.N = 0;
printf("lower rows_per_file=%d\n",pflower.rows_per_file);
printf("upper rows_per_file=%d\n",pfupper.rows_per_file);
if (pfupper.rows_per_file != pflower.rows_per_file) { //Sanity check for the two input frequency bands
fprintf(stderr, "rows_per_file in input files do not match!\n");
exit(1);
}
double upperfreqoflower, nextfromlower, lowerfreqofupper, numchandiff; //Used to find which frequencies to take from each band
double offsetfactor, scalefactor; //Factors which will be applied to offsets and scales
int upchanskip, lowchanskip; //Number of channels to skip in each banda
//Variables used to make code cleaner
int extrachanoffset, outoffset, upperoffset, numtocopyupper, loweroffset_skip,
loweroffset, numtocopylower, newuppernchan, newlowernchan;
double df = pflower.hdr.df;
int nchan = pflower.hdr.nchan;
int outnchan;
int npol = pflower.hdr.npol;
int nbits = pflower.hdr.nbits;
int nsblk = pflower.hdr.nsblk;
//Allocate memory for all upper and lower data
pflower.sub.dat_freqs = (double *) malloc(sizeof(double) * nchan);
pflower.sub.dat_weights = (float *) malloc(sizeof(float) * nchan);
pflower.sub.dat_offsets = (float *) malloc(sizeof(float) * nchan * npol);
pflower.sub.dat_scales = (float *) malloc(sizeof(float) * nchan * npol);
pflower.sub.rawdata = (unsigned char *) malloc(pflower.sub.bytes_per_subint);
pflower.sub.data = (unsigned char *) malloc(pflower.sub.bytes_per_subint*2);
pfupper.sub.dat_freqs = (double *) malloc(sizeof(double) * nchan);
pfupper.sub.dat_weights = (float *) malloc(sizeof(float) * nchan);
pfupper.sub.dat_offsets = (float *) malloc(sizeof(float) * nchan * npol);
pfupper.sub.dat_scales = (float *) malloc(sizeof(float) * nchan * npol);
pfupper.sub.rawdata = (unsigned char *) malloc(pfupper.sub.bytes_per_subint);
pfupper.sub.data = (unsigned char *) malloc(pfupper.sub.bytes_per_subint*2);
int firsttime = 1; //First time through do while loop
do {
print_percent_complete(pflower.rownum, pflower.rows_per_file,
pflower.rownum == 1 ? 1 : 0);
psrfits_read_subint(&pflower);
psrfits_read_subint(&pfupper);
if (firsttime) { //First time through loop, calculate factors for scales and offsets and number of channels to skip
firsttime = 0;
//Find the number of channels in the upper band which will be skipped
if (df < 0) { //Find channel order, low to high or high to low
upperfreqoflower = pflower.sub.dat_freqs[0]; //Highest frequency channel in lower band
lowerfreqofupper = pfupper.sub.dat_freqs[nchan - 1]; //Lowest frequency channel in upper band
} else {
upperfreqoflower = pflower.sub.dat_freqs[nchan - 1]; //Highest frequency channel in lower band
lowerfreqofupper = pfupper.sub.dat_freqs[0]; //Lowest frequency channel in upper band
}
nextfromlower = upperfreqoflower + fabs(df); //Second highest channel in lower band
numchandiff = (nextfromlower - lowerfreqofupper) / fabs(df); //Number of channels to skip in float form
int chanskip;
if (numchandiff > 0) { //Make sure there are channels which need to be skipped
if (numchandiff - (double) ((int) numchandiff) > .5) // See whether we need to round up integer channels to skip
chanskip = (int) numchandiff + 1;
else
chanskip = (int) numchandiff;
} else
chanskip = 0; //No need to skip any channels
if (chanskip % 2 == 1) { //Odd number of channels, give lower band the extra channel
upchanskip = chanskip / 2;
lowchanskip = chanskip / 2 + 1;
} else //Even number of channels to skip
upchanskip = lowchanskip = chanskip / 2;
if (upchanskip % 2 == 1) { //We want an even number of channels in upper band for 4-bit data to get copied correctly
++lowchanskip;
--upchanskip;
}
//Find new values given the number of channels skipped
pfo.hdr.nchan = outnchan = nchan + nchan - chanskip + 2; //New number of channels, plus 2 to make nchan=960 (many factors of 2)
pfo.hdr.BW = (double) outnchan *fabs(df); //New bandwidth
pfo.hdr.fctr = //New center frequency
(pflower.hdr.fctr - (double) (nchan / 2) * fabs(df)) + pfo.hdr.BW / 2.0;
pfo.sub.bytes_per_subint = //Calculate new number of bytes in each subint
outnchan * nsblk * nbits / 8 * npol;
//Allocate space for output data now that we know the new number of channels
pfo.sub.dat_freqs = (double *) malloc(sizeof(double) * outnchan);
pfo.sub.dat_weights = (float *) malloc(sizeof(float) * outnchan);
pfo.sub.dat_offsets = (float *) malloc(sizeof(float) * outnchan * npol);
pfo.sub.dat_scales = (float *) malloc(sizeof(float) * outnchan * npol);
pfo.sub.rawdata = (unsigned char *) malloc(pfo.sub.bytes_per_subint);
pfo.sub.data = (unsigned char *) malloc(pfo.sub.bytes_per_subint*2);
newuppernchan = nchan - upchanskip; //The number of channels to copy from the upper sideband.
newlowernchan = nchan - lowchanskip; //The number of channels to copy from the lower sideband.
extrachanoffset = 2; //Offset for 2 extra freq channels making nchan 960 in bytes
outoffset = (outnchan * npol); //Offset in each loop due to previously written data
upperoffset = (nchan * npol); //Offset in loop for upper band
numtocopyupper = (newuppernchan * npol); //Number of bytes to copy from upper band
loweroffset_skip = (lowchanskip * npol); //Number of bytes to skip when copying lower band due to
//having written upper band
loweroffset = //Number of bytes to skip due to having written previous lower band data
(nchan * npol);
numtocopylower = (newlowernchan * npol); //Number of bytes to copy from lower band
float upmean, upvar, lowmean, lowvar;
avg_var(pfupper.sub.dat_offsets + (nchan - upchanskip), //Find the mean and variance of the upper band's offsets
upchanskip, &upmean, &upvar);
printf("Upper offset stats: mean=%f variance=%f\n", upmean, upvar);
avg_var(pflower.sub.dat_offsets, lowchanskip, &lowmean, &lowvar); //Find the mean and variance of the lower band's offsets
printf("Lower offset stats: mean=%f variance=%f\n", lowmean, lowvar);
printf("Applying factor of %f to upper offsets\n", (lowmean / upmean));
offsetfactor = lowmean / upmean; //Set offset factor used to correct variance differences in the two bands
avg_var(pfupper.sub.dat_scales + (nchan - upchanskip), //Find the mean and var. of the upper band's scales
upchanskip, &upmean, &upvar);
printf("Upper scales stats: mean=%f variance=%f\n", upmean, upvar);
avg_var(pflower.sub.dat_scales, lowchanskip, &lowmean, &lowvar); //Find the mean and var. of the lower band's scales
printf("Lower scales stats: mean=%f variance=%f\n", lowmean, lowvar);
printf("Applying factor of %f to upper scales\n", (lowmean / upmean));
scalefactor = lowmean / upmean; //Set scale factor used to correct variance differences in the two bands
}
if (pflower.status == 0 && pfupper.status == 0) {
//Copy info from the lower band subint struct to the output file's subint struct
pfo.sub.tsubint = pflower.sub.tsubint;
pfo.sub.offs = pflower.sub.offs;
pfo.sub.lst = pflower.sub.lst;
pfo.sub.ra = pflower.sub.ra;
pfo.sub.dec = pflower.sub.dec;
pfo.sub.glon = pflower.sub.glon;
pfo.sub.glat = pflower.sub.glat;
pfo.sub.feed_ang = pflower.sub.feed_ang;
pfo.sub.pos_ang = pflower.sub.pos_ang;
pfo.sub.par_ang = pflower.sub.par_ang;
pfo.sub.tel_az = pflower.sub.tel_az;
pfo.sub.tel_zen = pflower.sub.tel_zen;
pfo.sub.FITS_typecode = pflower.sub.FITS_typecode;
//Create variables to reduce column width of lines below
double *dat_freqs = pfo.sub.dat_freqs;
double *udat_freqs = pfupper.sub.dat_freqs;
double *ldat_freqs = pflower.sub.dat_freqs;
float *dat_weights = pfo.sub.dat_weights;
float *udat_weights = pfupper.sub.dat_weights;
float *ldat_weights = pflower.sub.dat_weights;
float *dat_offsets = pfo.sub.dat_offsets;
float *udat_offsets = pfupper.sub.dat_offsets;
float *ldat_offsets = pflower.sub.dat_offsets;
float *dat_scales = pfo.sub.dat_scales;
float *udat_scales = pfupper.sub.dat_scales;
float *ldat_scales = pflower.sub.dat_scales;
unsigned char *data = pfo.sub.data;
unsigned char *udata = pfupper.sub.data;
unsigned char *ldata = pflower.sub.data;
if (df < 0) {
//Copy frequency labels
dat_freqs[1] = udat_freqs[0] + fabs(df); //Calculate the frequency labels
dat_freqs[0] = dat_freqs[1] + fabs(df); //for our two empty frequency channels
int newuppernchan = nchan - upchanskip; //The number of channels to copy from the upper band
int newlowernchan = nchan - lowchanskip; //The number of channels to copy from the lower band
memcpy(dat_freqs + 2, udat_freqs, sizeof(double) * newuppernchan); //Copy from the upper band, skipping first two chans.
memcpy(dat_freqs + newuppernchan + 2, //Copy from the lower band
ldat_freqs + lowchanskip, sizeof(double) * newlowernchan);
//Copy weights
dat_weights[0] = dat_weights[1] = 0; //Set the weights of first two channels to 0, so they shouldn't be used in calculations
memcpy(dat_weights + 2, udat_weights, //Copy weights from the upper band
sizeof(float) * newuppernchan);
memcpy(dat_weights + 2 + newuppernchan, //Copy weights from the lower band
ldat_weights + lowchanskip, sizeof(float) * newlowernchan);
//Copy offsets
dat_offsets[0] = dat_offsets[1] = //Set offsets of first two channels to the same as upper's first channel
udat_offsets[0]; //(shouldn't matter since they should be ignored)
int ii;
for (ii = 0; ii < newuppernchan; ++ii) //Apply offset factor to upper band
udat_offsets[ii] = udat_offsets[ii] * (offsetfactor);
memcpy(dat_offsets + 2 * npol, udat_offsets, //Copy upper offsets
sizeof(float) * newuppernchan * npol);
memcpy(dat_offsets + (newuppernchan + 2) * npol, //Copy lower offsets
ldat_offsets + lowchanskip, sizeof(float) * newlowernchan * npol);
//Copy scales
for (ii = 0; ii < newuppernchan; ++ii) //Apply scale factor to upper band
udat_scales[ii] = udat_scales[ii] * (scalefactor);
dat_scales[0] = dat_scales[1] = udat_scales[0];
memcpy(dat_scales + 2 * npol, udat_scales, //Copy upper scales
sizeof(float) * newuppernchan * npol);
memcpy(dat_scales + (newuppernchan + 2) * npol, //Copy lower scales
ldat_scales + lowchanskip, sizeof(float) * newlowernchan * npol);
//Copy the data
for (ii = 0; ii < nsblk; ++ii) { //Loop through data copying into place
memcpy(data + ii * outoffset + extrachanoffset,
udata + ii * upperoffset, numtocopyupper);
memcpy(data + ii * outoffset + extrachanoffset +
numtocopyupper,
ldata + ii * loweroffset + loweroffset_skip, numtocopylower);
}
psrfits_write_subint(&pfo);
} else {
}
}
} while (pfo.rownum <= pfo.rows_per_file && pfupper.status==0 && pflower.status==0);
printf("Closing file '%s'\n", pflower.filename);
fits_close_file(pfupper.fptr, &status);
printf("Closing file '%s'\n", pfupper.filename);
fits_close_file(pflower.fptr, &status);
if(pflower.status!=0||pfupper.status!=0)
{
fprintf(stderr,"An error occurred when combining the two Mock files!\n");
if(pflower.status==108||pfupper.status==108)
fprintf(stderr,"One or both of the files is incomplete.\n");
exit(1);
}
exit(0);
}
int main(int argc, char *argv[])
{
float maxpow = 0.0, inx = 0.0, iny = 0.0;
double centerr, offsetf;
int zoomlevel, maxzoom, minzoom, xid, psid;
char *rootfilenm, inchar;
fftpart *lofp;
fftview *fv;
if (argc == 1) {
printf("\nusage: explorefft fftfilename\n\n");
exit(0);
}
printf("\n\n");
printf(" Interactive FFT Explorer\n");
printf(" by Scott M. Ransom\n");
printf(" October, 2001\n");
print_help();
{
int hassuffix = 0;
char *suffix;
hassuffix = split_root_suffix(argv[1], &rootfilenm, &suffix);
if (hassuffix) {
if (strcmp(suffix, "fft") != 0) {
printf("\nInput file ('%s') must be a FFT file ('.fft')!\n\n", argv[1]);
free(suffix);
exit(0);
}
free(suffix);
} else {
printf("\nInput file ('%s') must be a FFT file ('.fft')!\n\n", argv[1]);
exit(0);
}
}
/* Read the info file */
readinf(&idata, rootfilenm);
if (strlen(remove_whitespace(idata.object)) > 0) {
printf("Examining %s data from '%s'.\n\n",
remove_whitespace(idata.object), argv[1]);
} else {
printf("Examining data from '%s'.\n\n", argv[1]);
}
N = idata.N;
T = idata.dt * idata.N;
#ifdef USEMMAP
printf("Memory mapping the input FFT. This may take a while...\n");
mmap_file = open(argv[1], O_RDONLY);
{
int rt;
struct stat buf;
rt = fstat(mmap_file, &buf);
if (rt == -1) {
perror("\nError in fstat() in explorefft.c");
printf("\n");
exit(-1);
}
Nfft = buf.st_size / sizeof(fcomplex);
}
lofp = get_fftpart(0, Nfft);
#else
{
int numamps;
fftfile = chkfopen(argv[1], "rb");
Nfft = chkfilelen(fftfile, sizeof(fcomplex));
numamps = (Nfft > MAXBINS) ? (int) MAXBINS : (int) Nfft;
lofp = get_fftpart(0, numamps);
}
#endif
/* Plot the initial data */
{
int initnumbins = INITIALNUMBINS;
if (initnumbins > Nfft) {
initnumbins = next2_to_n(Nfft) / 2;
zoomlevel = LOGDISPLAYNUM - (int) (log(initnumbins) / log(2.0));
minzoom = zoomlevel;
} else {
zoomlevel = LOGDISPLAYNUM - LOGINITIALNUMBINS;
minzoom = LOGDISPLAYNUM - LOGMAXBINS;
}
maxzoom = LOGDISPLAYNUM - LOGMINBINS;
centerr = initnumbins / 2;
}
fv = get_fftview(centerr, zoomlevel, lofp);
/* Prep the XWIN device for PGPLOT */
xid = cpgopen("/XWIN");
if (xid <= 0) {
free(fv);
#ifdef USEMMAP
close(mmap_file);
#else
fclose(fftfile);
#endif
free_fftpart(lofp);
exit(EXIT_FAILURE);
}
cpgscr(15, 0.4, 0.4, 0.4);
cpgask(0);
cpgpage();
offsetf = plot_fftview(fv, maxpow, 1.0, 0.0, 0);
do {
cpgcurs(&inx, &iny, &inchar);
if (DEBUGOUT)
printf("You pressed '%c'\n", inchar);
switch (inchar) {
case 'A': /* Zoom in */
case 'a':
centerr = (inx + offsetf) * T;
case 'I':
case 'i':
if (DEBUGOUT)
printf(" Zooming in (zoomlevel = %d)...\n", zoomlevel);
if (zoomlevel < maxzoom) {
zoomlevel++;
free(fv);
fv = get_fftview(centerr, zoomlevel, lofp);
cpgpage();
offsetf = plot_fftview(fv, maxpow, 1.0, 0.0, 0);
} else
printf(" Already at maximum zoom level (%d).\n", zoomlevel);
break;
case 'X': /* Zoom out */
case 'x':
case 'O':
case 'o':
if (DEBUGOUT)
printf(" Zooming out (zoomlevel = %d)...\n", zoomlevel);
if (zoomlevel > minzoom) {
zoomlevel--;
free(fv);
fv = get_fftview(centerr, zoomlevel, lofp);
cpgpage();
offsetf = plot_fftview(fv, maxpow, 1.0, 0.0, 0);
} else
printf(" Already at minimum zoom level (%d).\n", zoomlevel);
break;
case '<': /* Shift left 1 full screen */
centerr -= fv->numbins + fv->numbins / 8;
case ',': /* Shift left 1/8 screen */
if (DEBUGOUT)
printf(" Shifting left...\n");
centerr -= fv->numbins / 8;
{ /* Should probably get the previous chunk from the fftfile... */
double lowestr;
lowestr = 0.5 * fv->numbins;
if (centerr < lowestr)
centerr = lowestr;
}
free(fv);
fv = get_fftview(centerr, zoomlevel, lofp);
cpgpage();
offsetf = plot_fftview(fv, maxpow, 1.0, 0.0, 0);
break;
case '>': /* Shift right 1 full screen */
centerr += fv->numbins - fv->numbins / 8;
case '.': /* Shift right 1/8 screen */
if (DEBUGOUT)
printf(" Shifting right...\n");
centerr += fv->numbins / 8;
{ /* Should probably get the next chunk from the fftfile... */
double highestr;
highestr = lofp->rlo + lofp->numamps - 0.5 * fv->numbins;
if (centerr > highestr)
centerr = highestr;
}
free(fv);
fv = get_fftview(centerr, zoomlevel, lofp);
cpgpage();
offsetf = plot_fftview(fv, maxpow, 1.0, 0.0, 0);
break;
case '+': /* Increase height of powers */
case '=':
if (maxpow == 0.0) {
printf(" Auto-scaling is off.\n");
maxpow = 1.1 * fv->maxpow;
}
maxpow = 3.0 / 4.0 * maxpow;
cpgpage();
offsetf = plot_fftview(fv, maxpow, 1.0, 0.0, 0);
break;
case '-': /* Decrease height of powers */
case '_':
if (maxpow == 0.0) {
printf(" Auto-scaling is off.\n");
maxpow = 1.1 * fv->maxpow;
}
maxpow = 4.0 / 3.0 * maxpow;
cpgpage();
offsetf = plot_fftview(fv, maxpow, 1.0, 0.0, 0);
break;
case 'S': /* Auto-scale */
case 's':
if (maxpow == 0.0)
break;
else {
printf(" Auto-scaling is on.\n");
maxpow = 0.0;
cpgpage();
offsetf = plot_fftview(fv, maxpow, 1.0, 0.0, 0);
break;
}
case 'G': /* Goto a frequency */
case 'g':
{
char freqstr[50];
double freq = -1.0;
while (freq < 0.0) {
printf(" Enter the frequency (Hz) to go to:\n");
fgets(freqstr, 50, stdin);
freqstr[strlen(freqstr) - 1] = '\0';
freq = atof(freqstr);
}
offsetf = 0.0;
centerr = freq * T;
printf(" Moving to frequency %.15g.\n", freq);
free(fv);
fv = get_fftview(centerr, zoomlevel, lofp);
cpgpage();
offsetf = plot_fftview(fv, maxpow, 1.0, centerr, 2);
}
break;
case 'H': /* Show harmonics */
case 'h':
{
double retval;
retval = harmonic_loop(xid, centerr, zoomlevel, lofp);
if (retval > 0.0) {
offsetf = 0.0;
centerr = retval;
free(fv);
fv = get_fftview(centerr, zoomlevel, lofp);
cpgpage();
offsetf = plot_fftview(fv, maxpow, 1.0, centerr, 2);
}
}
break;
case '?': /* Print help screen */
print_help();
break;
case 'D': /* Show details about a selected point */
case 'd':
{
double newr;
printf(" Searching for peak near freq = %.7g Hz...\n", (inx + offsetf));
newr = find_peak(inx + offsetf, fv, lofp);
centerr = newr;
free(fv);
fv = get_fftview(centerr, zoomlevel, lofp);
cpgpage();
offsetf = plot_fftview(fv, maxpow, 1.0, centerr, 2);
}
break;
case 'L': /* Load a zaplist */
case 'l':
{
int ii, len;
char filename[200];
double *lobins, *hibins;
printf(" Enter the filename containing the zaplist to load:\n");
fgets(filename, 199, stdin);
len = strlen(filename) - 1;
filename[len] = '\0';
numzaplist = get_birdies(filename, T, 0.0, &lobins, &hibins);
lenzaplist = numzaplist + 20; /* Allow some room to add more */
if (lenzaplist)
free(zaplist);
zaplist = (bird *) malloc(sizeof(bird) * lenzaplist);
for (ii = 0; ii < numzaplist; ii++) {
zaplist[ii].lobin = lobins[ii];
zaplist[ii].hibin = hibins[ii];
}
vect_free(lobins);
vect_free(hibins);
printf("\n");
cpgpage();
offsetf = plot_fftview(fv, maxpow, 1.0, 0.0, 0);
}
break;
case 'Z': /* Add a birdie to a zaplist */
case 'z':
{
int badchoice = 2;
float lox, hix, loy, hiy;
double rs[2];
char choice;
if (numzaplist + 1 > lenzaplist) {
lenzaplist += 10;
zaplist = (bird *) realloc(zaplist, sizeof(bird) * lenzaplist);
}
cpgqwin(&lox, &hix, &loy, &hiy);
printf(" Click the left mouse button on the first frequency limit.\n");
while (badchoice) {
cpgcurs(&inx, &iny, &choice);
if (choice == 'A' || choice == 'a') {
rs[2 - badchoice] = ((double) inx + offsetf) * T;
cpgsave();
cpgsci(7);
cpgmove(inx, 0.0);
cpgdraw(inx, hiy);
cpgunsa();
badchoice--;
if (badchoice == 1)
printf
(" Click the left mouse button on the second frequency limit.\n");
} else {
printf(" Option not recognized.\n");
}
};
if (rs[1] > rs[0]) {
zaplist[numzaplist].lobin = rs[0];
zaplist[numzaplist].hibin = rs[1];
} else {
zaplist[numzaplist].lobin = rs[1];
zaplist[numzaplist].hibin = rs[0];
}
printf(" The new birdie has: f_avg = %.15g f_width = %.15g\n\n",
0.5 * (zaplist[numzaplist].hibin + zaplist[numzaplist].lobin) / T,
(zaplist[numzaplist].hibin - zaplist[numzaplist].lobin) / T);
numzaplist++;
qsort(zaplist, numzaplist, sizeof(bird), compare_birds);
cpgpage();
offsetf = plot_fftview(fv, maxpow, 1.0, 0.0, 0);
}
break;
case 'P': /* Print the current plot */
case 'p':
{
int len;
char filename[200];
printf(" Enter the filename to save the plot as:\n");
fgets(filename, 196, stdin);
len = strlen(filename) - 1;
filename[len + 0] = '/';
filename[len + 1] = 'P';
filename[len + 2] = 'S';
filename[len + 3] = '\0';
psid = cpgopen(filename);
cpgslct(psid);
cpgpap(10.25, 8.5 / 11.0);
cpgiden();
cpgscr(15, 0.8, 0.8, 0.8);
offsetf = plot_fftview(fv, maxpow, 1.0, 0.0, 0);
cpgclos();
cpgslct(xid);
cpgscr(15, 0.4, 0.4, 0.4);
filename[len] = '\0';
printf(" Wrote the plot to the file '%s'.\n", filename);
}
break;
case 'N': /* Changing power normalization */
case 'n':
{
float inx2 = 0.0, iny2 = 0.0;
char choice;
unsigned char badchoice = 1;
printf(" Specify the type of power normalization:\n"
" m,M : Median values determined locally\n"
" d,D : DC frequency amplitude\n"
" r,R : Raw powers (i.e. no normalization)\n"
" u,U : User specified interval (the average powers)\n");
while (badchoice) {
cpgcurs(&inx2, &iny2, &choice);
switch (choice) {
case 'M':
case 'm':
norm_const = 0.0;
maxpow = 0.0;
badchoice = 0;
printf
(" Using local median normalization. Autoscaling is on.\n");
break;
case 'D':
case 'd':
norm_const = 1.0 / r0;
maxpow = 0.0;
badchoice = 0;
printf
(" Using DC frequency (%f) normalization. Autoscaling is on.\n",
r0);
break;
case 'R':
case 'r':
norm_const = 1.0;
maxpow = 0.0;
badchoice = 0;
printf
(" Using raw powers (i.e. no normalization). Autoscaling is on.\n");
break;
case 'U':
case 'u':
{
char choice2;
float xx = inx, yy = iny;
int lor, hir, numr;
double avg, var;
printf
(" Use the left mouse button to select a left and right boundary\n"
" of a region to calculate the average power.\n");
do {
cpgcurs(&xx, &yy, &choice2);
} while (choice2 != 'A' && choice2 != 'a');
lor = (int) ((xx + offsetf) * T);
cpgsci(7);
cpgmove(xx, 0.0);
cpgdraw(xx, 10.0 * fv->maxpow);
do {
cpgcurs(&xx, &yy, &choice2);
} while (choice2 != 'A' && choice2 != 'a');
hir = (int) ((xx + offsetf) * T);
cpgmove(xx, 0.0);
cpgdraw(xx, 10.0 * fv->maxpow);
cpgsci(1);
if (lor > hir) {
int tempr;
tempr = hir;
hir = lor;
lor = tempr;
}
numr = hir - lor + 1;
avg_var(lofp->rawpowers + lor - lofp->rlo, numr, &avg, &var);
printf(" Selection has: average = %.5g\n"
" std dev = %.5g\n", avg, sqrt(var));
norm_const = 1.0 / avg;
maxpow = 0.0;
badchoice = 0;
printf
(" Using %.5g as the normalization constant. Autoscaling is on.\n",
avg);
break;
}
default:
printf(" Unrecognized choice '%c'.\n", choice);
break;
}
}
free(fv);
fv = get_fftview(centerr, zoomlevel, lofp);
cpgpage();
offsetf = plot_fftview(fv, maxpow, 1.0, 0.0, 0);
}
break;
case 'Q': /* Quit */
case 'q':
printf(" Quitting...\n");
free(fv);
cpgclos();
break;
default:
printf(" Unrecognized option '%c'.\n", inchar);
break;
}
} while (inchar != 'Q' && inchar != 'q');
free_fftpart(lofp);
#ifdef USEMMAP
close(mmap_file);
#else
fclose(fftfile);
#endif
if (lenzaplist)
free(zaplist);
printf("Done\n\n");
return 0;
}
fftcand *search_fft(fcomplex * fft, int numfft, int lobin, int hibin,
int numharmsum, int numbetween,
presto_interptype interptype,
float norm, float sigmacutoff, int *numcands,
float *powavg, float *powvar, float *powmax)
/* This routine searches a short FFT of 'numfft' complex freqs */
/* and returns a candidate vector of fftcand structures containing */
/* information about the best candidates found. */
/* The routine uses either interbinning or interpolation as well */
/* as harmonic summing during the search. */
/* The number of candidates returned is either 'numcands' if != 0, */
/* or is determined automatically by 'sigmacutoff' -- which */
/* takes into account the number of bins searched. */
/* The returned vector is sorted in order of decreasing power. */
/* Arguments: */
/* 'fft' is the FFT to search (complex valued) */
/* 'numfft' is the number of complex points in 'fft' */
/* 'lobin' is the lowest Fourier freq to search */
/* 'hibin' is the highest Fourier freq to search */
/* 'numharmsum' the number of harmonics to sum during the search */
/* 'numbetween' the points to interpolate per bin */
/* 'interptype' is either INTERBIN or INTERPOLATE. */
/* INTERBIN = (interbinning) is fast but less sensitive. */
/* NOTE: INTERBINNING is conducted by this routine! */
/* INTERPOLATE = (Fourier interpolation) is slower but more */
/* sensitive. */
/* NOTE: The interpolation is assumed to ALREADY have been */
/* completed by the calling function! The easiest */
/* way is by zero-padding to 2*numfft and FFTing. */
/* If you use this method, make sure numfft is the */
/* original length rather than the interpolated */
/* length and also make sure numbetween is correct. */
/* 'norm' is the normalization constant to multiply each power by */
/* 'sigmacutoff' if the number of candidates will be determined */
/* automatically, is the minimum Gaussian significance of */
/* candidates to keep -- taking into account the number of */
/* bins searched */
/* 'numcands' if !0, is the number of candates to return. */
/* if 0, is a return value giving the number of candidates. */
/* 'powavg' is a return value giving the average power level */
/* 'powvar' is a return value giving the power level variance */
/* 'powmax' is a return value giving the maximum power */
{
int ii, jj, offset, numtosearch, dynamic = 0;
int numspread = 0, nc = 0, startnc = 10;
float powargr, powargi, *fullpows = NULL, *sumpows, ftmp;
double twobypi, minpow = 0.0, tmpminsig = 0.0, dr, davg, dvar;
fftcand *cands, newcand;
fcomplex *spread;
/* Override the value of numbetween if interbinning */
if (interptype == INTERBIN)
numbetween = 2;
norm = 1.0 / norm;
*powmax = 0.0;
/* Decide if we will manage the number of candidates */
if (*numcands > 0)
startnc = *numcands;
else {
dynamic = 1;
minpow = power_for_sigma(sigmacutoff, 1, hibin - lobin);
}
cands = (fftcand *) malloc(startnc * sizeof(fftcand));
for (ii = 0; ii < startnc; ii++)
cands[ii].sig = 0.0;
/* Prep some other values we will need */
dr = 1.0 / (double) numbetween;
twobypi = 2.0 / PI;
numtosearch = numfft * numbetween;
/* Spread and interpolate the fft */
numspread = numfft * numbetween + 1;
if (interptype == INTERPOLATE) { /* INTERPOLATE */
spread = fft;
} else { /* INTERBIN */
spread = gen_cvect(numspread);
spread_with_pad(fft, numfft, spread, numspread, numbetween, 0);
for (ii = 1; ii < numtosearch; ii += 2) {
spread[ii].r = twobypi * (spread[ii - 1].r - spread[ii + 1].r);
spread[ii].i = twobypi * (spread[ii - 1].i - spread[ii + 1].i);
}
}
spread[0].r = spread[numtosearch].r = 1.0;
spread[0].i = spread[numtosearch].i = 0.0;
/* First generate the original powers in order to */
/* calculate the statistics. Yes, this is inefficient... */
fullpows = gen_fvect(numtosearch);
for (ii = lobin, jj = 0; ii < hibin; ii++, jj++) {
ftmp = POWER(fft[ii].r, fft[ii].i) * norm;
fullpows[jj] = ftmp;
if (ftmp > *powmax)
*powmax = ftmp;
}
avg_var(fullpows, hibin - lobin, &davg, &dvar);
*powavg = davg;
*powvar = dvar;
fullpows[0] = 1.0;
for (ii = 1; ii < numtosearch; ii++)
fullpows[ii] = POWER(spread[ii].r, spread[ii].i) * norm;
if (interptype == INTERBIN)
vect_free(spread);
/* Search the raw powers */
for (ii = lobin * numbetween; ii < hibin * numbetween; ii++) {
if (fullpows[ii] > minpow) {
newcand.r = dr * (double) ii;
newcand.p = fullpows[ii];
newcand.sig = candidate_sigma(fullpows[ii], 1, hibin - lobin);
newcand.nsum = 1;
cands[startnc - 1] = newcand;
tmpminsig = percolate_fftcands(cands, startnc);
if (dynamic) {
nc++;
if (nc == startnc) {
startnc *= 2;
cands = (fftcand *) realloc(cands, startnc * sizeof(fftcand));
for (jj = nc; jj < startnc; jj++)
cands[jj].sig = 0.0;
}
} else {
minpow = cands[startnc - 1].p;
if (nc < startnc)
nc++;
}
}
}
/* If needed, sum and search the harmonics */
if (numharmsum > 1) {
sumpows = gen_fvect(numtosearch);
memcpy(sumpows, fullpows, sizeof(float) * numtosearch);
for (ii = 2; ii <= numharmsum; ii++) {
offset = ii / 2;
if (dynamic)
minpow = power_for_sigma(sigmacutoff, ii, hibin - lobin);
else
minpow = power_for_sigma(tmpminsig, ii, hibin - lobin);
for (jj = lobin * numbetween; jj < numtosearch; jj++) {
sumpows[jj] += fullpows[(jj + offset) / ii];
if (sumpows[jj] > minpow) {
newcand.r = dr * (double) jj;
newcand.p = sumpows[jj];
newcand.sig = candidate_sigma(sumpows[jj], ii, hibin - lobin);
newcand.nsum = ii;
cands[startnc - 1] = newcand;
tmpminsig = percolate_fftcands(cands, startnc);
if (dynamic) {
nc++;
if (nc == startnc) {
startnc *= 2;
cands = (fftcand *) realloc(cands, startnc * sizeof(fftcand));
for (jj = nc; jj < startnc; jj++)
cands[jj].sig = 0.0;
}
} else {
minpow = power_for_sigma(tmpminsig, ii, hibin - lobin);
if (nc < startnc)
nc++;
}
}
}
}
vect_free(sumpows);
}
vect_free(fullpows);
/* Chop off the unused parts of the dynamic array */
if (dynamic)
cands = (fftcand *) realloc(cands, nc * sizeof(fftcand));
*numcands = nc;
return cands;
}
int main(int argc, char *argv[])
{
/* Any variable that begins with 't' means topocentric */
/* Any variable that begins with 'b' means barycentric */
FILE *outfile;
float *outdata = NULL;
double tdf = 0.0, dtmp = 0.0, barydispdt = 0.0, dsdt = 0.0;
double *dispdt, *tobsf = NULL, tlotoa = 0.0, blotoa = 0.0;
double max = -9.9E30, min = 9.9E30, var = 0.0, avg = 0.0;
char obs[3], ephem[10], *datafilenm, *outinfonm;
char rastring[50], decstring[50];
int numchan = 1, newper = 0, oldper = 0, nummasked = 0, useshorts = 0;
int numadded = 0, numremoved = 0, padding = 0, *maskchans = NULL, offset = 0;
long slen, ii, numbarypts = 0, worklen = 65536;
long numread = 0, numtowrite = 0, totwrote = 0, datawrote = 0;
long padwrote = 0, padtowrite = 0, statnum = 0;
int numdiffbins = 0, *diffbins = NULL, *diffbinptr = NULL, good_padvals = 0;
int *idispdt;
struct spectra_info s;
infodata idata;
Cmdline *cmd;
mask obsmask;
/* Call usage() if we have no command line arguments */
if (argc == 1) {
Program = argv[0];
printf("\n");
usage();
exit(0);
}
/* Parse the command line using the excellent program Clig */
cmd = parseCmdline(argc, argv);
spectra_info_set_defaults(&s);
s.filenames = cmd->argv;
s.num_files = cmd->argc;
// If we are zeroDMing, make sure that clipping is off.
if (cmd->zerodmP)
cmd->noclipP = 1;
s.clip_sigma = cmd->clip;
// -1 causes the data to determine if we use weights, scales, &
// offsets for PSRFITS or flip the band for any data type where
// we can figure that out with the data
s.apply_flipband = (cmd->invertP) ? 1 : -1;
s.apply_weight = (cmd->noweightsP) ? 0 : -1;
s.apply_scale = (cmd->noscalesP) ? 0 : -1;
s.apply_offset = (cmd->nooffsetsP) ? 0 : -1;
s.remove_zerodm = (cmd->zerodmP) ? 1 : 0;
if (cmd->noclipP) {
cmd->clip = 0.0;
s.clip_sigma = 0.0;
}
if (cmd->ifsP) {
// 0 = default or summed, 1-4 are possible also
s.use_poln = cmd->ifs + 1;
}
if (cmd->ncpus > 1) {
#ifdef _OPENMP
int maxcpus = omp_get_num_procs();
int openmp_numthreads = (cmd->ncpus <= maxcpus) ? cmd->ncpus : maxcpus;
// Make sure we are not dynamically setting the number of threads
omp_set_dynamic(0);
omp_set_num_threads(openmp_numthreads);
printf("Using %d threads with OpenMP\n\n", openmp_numthreads);
#endif
} else {
#ifdef _OPENMP
omp_set_num_threads(1); // Explicitly turn off OpenMP
#endif
}
#ifdef DEBUG
showOptionValues();
#endif
printf("\n\n");
printf(" Pulsar Data Preparation Routine\n");
printf(" Type conversion, de-dispersion, barycentering.\n");
printf(" by Scott M. Ransom\n\n");
if (RAWDATA) {
if (cmd->filterbankP)
s.datatype = SIGPROCFB;
else if (cmd->psrfitsP)
s.datatype = PSRFITS;
} else { // Attempt to auto-identify the data
identify_psrdatatype(&s, 1);
if (s.datatype == SIGPROCFB)
cmd->filterbankP = 1;
else if (s.datatype == PSRFITS)
cmd->psrfitsP = 1;
else if (s.datatype == SDAT)
useshorts = 1;
else if (s.datatype != DAT) {
printf
("Error: Unable to identify input data files. Please specify type.\n\n");
exit(1);
}
}
if (!RAWDATA) {
char *root, *suffix;
/* Split the filename into a rootname and a suffix */
if (split_root_suffix(s.filenames[0], &root, &suffix) == 0) {
printf("\nThe input filename (%s) must have a suffix!\n\n",
s.filenames[0]);
exit(1);
}
printf("Reading input data from '%s'.\n", s.filenames[0]);
printf("Reading information from '%s.inf'.\n\n", root);
/* Read the info file if available */
readinf(&idata, root);
free(root);
free(suffix);
s.files = (FILE **) malloc(sizeof(FILE *));
s.files[0] = chkfopen(s.filenames[0], "rb");
} else {
char description[40];
psrdatatype_description(description, s.datatype);
if (s.num_files > 1)
printf("Reading %s data from %d files:\n", description, s.num_files);
else
printf("Reading %s data from 1 file:\n", description);
for (ii = 0; ii < s.num_files; ii++) {
printf(" '%s'\n", cmd->argv[ii]);
}
printf("\n");
}
/* Determine the other file names and open the output data file */
slen = strlen(cmd->outfile) + 8;
datafilenm = (char *) calloc(slen, 1);
sprintf(datafilenm, "%s.dat", cmd->outfile);
outfile = chkfopen(datafilenm, "wb");
sprintf(idata.name, "%s", cmd->outfile);
outinfonm = (char *) calloc(slen, 1);
sprintf(outinfonm, "%s.inf", cmd->outfile);
if (RAWDATA) {
read_rawdata_files(&s);
if (cmd->ignorechanstrP) {
s.ignorechans = get_ignorechans(cmd->ignorechanstr, 0, s.num_channels-1,
&s.num_ignorechans, &s.ignorechans_str);
if (s.ignorechans_str==NULL) {
s.ignorechans_str = (char *)malloc(strlen(cmd->ignorechanstr)+1);
strcpy(s.ignorechans_str, cmd->ignorechanstr);
}
}
print_spectra_info_summary(&s);
spectra_info_to_inf(&s, &idata);
/* Finish setting up stuff common to all raw formats */
idata.dm = cmd->dm;
worklen = s.spectra_per_subint;
/* If we are offsetting into the file, change inf file start time */
if (cmd->start > 0.0 || cmd->offset > 0) {
if (cmd->start > 0.0) /* Offset in units of worklen */
cmd->offset = (long) (cmd->start *
idata.N / worklen) * worklen;
add_to_inf_epoch(&idata, cmd->offset * idata.dt);
offset_to_spectra(cmd->offset, &s);
printf("Offsetting into the input files by %ld spectra (%.6g sec)\n",
cmd->offset, cmd->offset * idata.dt);
}
if (cmd->maskfileP)
maskchans = gen_ivect(idata.num_chan);
/* Compare the size of the data to the size of output we request */
if (cmd->numoutP) {
dtmp = idata.N;
idata.N = cmd->numout;
writeinf(&idata);
idata.N = dtmp;
} else {
/* Set the output length to a good number if it wasn't requested */
cmd->numoutP = 1;
cmd->numout = choose_good_N((long long)(idata.N/cmd->downsamp));
writeinf(&idata);
printf("Setting a 'good' output length of %ld samples\n", cmd->numout);
}
/* The number of topo to bary time points to generate with TEMPO */
numbarypts = (long) (idata.dt * idata.N * 1.1 / TDT + 5.5) + 1;
// Identify the TEMPO observatory code
{
char *outscope = (char *) calloc(40, sizeof(char));
telescope_to_tempocode(idata.telescope, outscope, obs);
free(outscope);
}
}
/* Read an input mask if wanted */
if (cmd->maskfileP) {
read_mask(cmd->maskfile, &obsmask);
printf("Read mask information from '%s'\n\n", cmd->maskfile);
good_padvals = determine_padvals(cmd->maskfile, &obsmask, s.padvals);
} else {
obsmask.numchan = obsmask.numint = 0;
}
/* Determine our initialization data if we do _not_ have Parkes, */
/* Green Bank BCPM, or Arecibo WAPP data sets. */
if (!RAWDATA) {
/* If we will be barycentering... */
if (!cmd->nobaryP) {
/* The number of topo to bary time points to generate with TEMPO */
numbarypts = (long) (idata.dt * idata.N * 1.1 / TDT + 5.5) + 1;
// Identify the TEMPO observatory code
{
char *outscope = (char *) calloc(40, sizeof(char));
telescope_to_tempocode(idata.telescope, outscope, obs);
free(outscope);
}
}
/* The number of data points to work with at a time */
if (worklen > idata.N)
worklen = idata.N;
worklen = (long) (worklen / 1024) * 1024;
/* If we are offsetting into the file, change inf file start time */
if (cmd->start > 0.0 || cmd->offset > 0) {
if (cmd->start > 0.0) /* Offset in units of worklen */
cmd->offset = (long) (cmd->start *
idata.N / worklen) * worklen;
add_to_inf_epoch(&idata, cmd->offset * idata.dt);
printf("Offsetting into the input files by %ld samples (%.6g sec)\n",
cmd->offset, cmd->offset * idata.dt);
if (useshorts) {
chkfileseek(s.files[0], cmd->offset, sizeof(short), SEEK_SET);
} else {
chkfileseek(s.files[0], cmd->offset, sizeof(float), SEEK_SET);
}
}
/* Set the output length to a good number if it wasn't requested */
if (!cmd->numoutP) {
cmd->numoutP = 1;
cmd->numout = choose_good_N((long long)(idata.N/cmd->downsamp));
printf("Setting a 'good' output length of %ld samples\n", cmd->numout);
}
}
/* Check if we are downsampling */
dsdt = idata.dt * cmd->downsamp;
if (cmd->downsamp > 1) {
printf("Downsampling by a factor of %d\n", cmd->downsamp);
printf("New sample dt = %.10g\n\n", dsdt);
if (worklen % cmd->downsamp) {
printf("Error: The downsample factor (%d) must be a factor of the\n",
cmd->downsamp);
printf(" worklength (%ld). Exiting.\n\n", worklen);
exit(1);
}
}
printf("Writing output data to '%s'.\n", datafilenm);
printf("Writing information to '%s'.\n\n", outinfonm);
/* The topocentric epoch of the start of the data */
tlotoa = (double) idata.mjd_i + idata.mjd_f;
if (!strcmp(idata.band, "Radio") && RAWDATA) {
/* The topocentric spacing between channels */
tdf = idata.chan_wid;
numchan = idata.num_chan;
/* The topocentric observation frequencies */
tobsf = gen_dvect(numchan);
tobsf[0] = idata.freq;
for (ii = 0; ii < numchan; ii++)
tobsf[ii] = tobsf[0] + ii * tdf;
/* The dispersion delays (in time bins) */
dispdt = gen_dvect(numchan); // full float bins
idispdt = gen_ivect(numchan); // nearest integer bins
if (cmd->nobaryP) {
/* Determine our dispersion time delays for each channel */
for (ii = 0; ii < numchan; ii++)
dispdt[ii] = delay_from_dm(cmd->dm, tobsf[ii]);
/* The highest frequency channel gets no delay */
/* All other delays are positive fractions of bin length (dt) */
dtmp = dispdt[numchan - 1];
for (ii = 0; ii < numchan; ii++) {
dispdt[ii] = (dispdt[ii] - dtmp) / idata.dt;
idispdt[ii] = (int) (dispdt[ii] + 0.5);
}
worklen *= ((int) (fabs(dispdt[0])) / worklen) + 1;
}
} else { /* For unknown radio raw data (Why is this here?) */
tobsf = gen_dvect(numchan);
dispdt = gen_dvect(numchan);
idispdt = gen_ivect(numchan);
dispdt[0] = 0.0;
idispdt[0] = 0;
if (!strcmp(idata.band, "Radio")) {
tobsf[0] = idata.freq + (idata.num_chan - 1) * idata.chan_wid;
cmd->dm = idata.dm;
} else {
tobsf[0] = 0.0;
cmd->dm = 0.0;
}
}
if (cmd->nobaryP) { /* Main loop if we are not barycentering... */
/* Allocate our data array */
outdata = gen_fvect(worklen);
printf("Massaging the data ...\n\n");
printf("Amount Complete = 0%%");
do {
if (RAWDATA)
numread = read_psrdata(outdata, worklen, &s, idispdt, &padding,
maskchans, &nummasked, &obsmask);
else if (useshorts)
numread = read_shorts(s.files[0], outdata, worklen, numchan);
else
numread = read_floats(s.files[0], outdata, worklen, numchan);
if (numread == 0)
break;
/* Downsample if requested */
if (cmd->downsamp > 1)
numread = downsample(outdata, numread, cmd->downsamp);
/* Print percent complete */
newper = (int) ((float) totwrote / cmd->numout * 100.0) + 1;
if (newper > oldper) {
printf("\rAmount Complete = %3d%%", newper);
fflush(stdout);
oldper = newper;
}
/* Write the latest chunk of data, but don't */
/* write more than cmd->numout points. */
numtowrite = numread;
if ((totwrote + numtowrite) > cmd->numout)
numtowrite = cmd->numout - totwrote;
chkfwrite(outdata, sizeof(float), numtowrite, outfile);
totwrote += numtowrite;
/* Update the statistics */
if (!padding) {
for (ii = 0; ii < numtowrite; ii++)
update_stats(statnum + ii, outdata[ii], &min, &max, &avg, &var);
statnum += numtowrite;
}
/* Stop if we have written out all the data we need to */
if (totwrote == cmd->numout)
break;
} while (numread);
datawrote = totwrote;
} else { /* Main loop if we are barycentering... */
double avgvoverc = 0.0, maxvoverc = -1.0, minvoverc = 1.0, *voverc = NULL;
double *bobsf = NULL, *btoa = NULL, *ttoa = NULL;
/* What ephemeris will we use? (Default is DE405) */
strcpy(ephem, "DE405");
/* Define the RA and DEC of the observation */
ra_dec_to_string(rastring, idata.ra_h, idata.ra_m, idata.ra_s);
ra_dec_to_string(decstring, idata.dec_d, idata.dec_m, idata.dec_s);
/* Allocate some arrays */
bobsf = gen_dvect(numchan);
btoa = gen_dvect(numbarypts);
ttoa = gen_dvect(numbarypts);
voverc = gen_dvect(numbarypts);
for (ii = 0; ii < numbarypts; ii++)
ttoa[ii] = tlotoa + TDT * ii / SECPERDAY;
/* Call TEMPO for the barycentering */
printf("Generating barycentric corrections...\n");
barycenter(ttoa, btoa, voverc, numbarypts, rastring, decstring, obs, ephem);
for (ii = 0; ii < numbarypts; ii++) {
if (voverc[ii] > maxvoverc)
maxvoverc = voverc[ii];
if (voverc[ii] < minvoverc)
minvoverc = voverc[ii];
avgvoverc += voverc[ii];
}
avgvoverc /= numbarypts;
vect_free(voverc);
blotoa = btoa[0];
printf(" Average topocentric velocity (c) = %.7g\n", avgvoverc);
printf(" Maximum topocentric velocity (c) = %.7g\n", maxvoverc);
printf(" Minimum topocentric velocity (c) = %.7g\n\n", minvoverc);
printf("Collecting and barycentering %s...\n\n", cmd->argv[0]);
/* Determine the initial dispersion time delays for each channel */
for (ii = 0; ii < numchan; ii++) {
bobsf[ii] = doppler(tobsf[ii], avgvoverc);
dispdt[ii] = delay_from_dm(cmd->dm, bobsf[ii]);
}
/* The highest frequency channel gets no delay */
/* All other delays are positive fractions of bin length (dt) */
barydispdt = dispdt[numchan - 1];
for (ii = 0; ii < numchan; ii++) {
dispdt[ii] = (dispdt[ii] - barydispdt) / idata.dt;
idispdt[ii] = (int) (dispdt[ii] + 0.5);
}
if (RAWDATA)
worklen *= ((int) (dispdt[0]) / worklen) + 1;
/* If the data is de-dispersed radio data... */
if (!strcmp(idata.band, "Radio")) {
printf("The DM of %.2f at the barycentric observing freq of %.3f MHz\n",
idata.dm, bobsf[numchan - 1]);
printf(" causes a delay of %f seconds compared to infinite freq.\n",
barydispdt);
printf(" This delay is removed from the barycented times.\n\n");
}
printf("Topocentric epoch (at data start) is:\n");
printf(" %17.11f\n\n", tlotoa);
printf("Barycentric epoch (infinite obs freq at data start) is:\n");
printf(" %17.11f\n\n", blotoa - (barydispdt / SECPERDAY));
/* Convert the bary TOAs to differences from the topo TOAs in */
/* units of bin length (dsdt) rounded to the nearest integer. */
dtmp = (btoa[0] - ttoa[0]);
for (ii = 0; ii < numbarypts; ii++)
btoa[ii] = ((btoa[ii] - ttoa[ii]) - dtmp) * SECPERDAY / dsdt;
{ /* Find the points where we need to add or remove bins */
int oldbin = 0, currentbin;
double lobin, hibin, calcpt;
numdiffbins = abs(NEAREST_LONG(btoa[numbarypts - 1])) + 1;
diffbins = gen_ivect(numdiffbins);
diffbinptr = diffbins;
for (ii = 1; ii < numbarypts; ii++) {
currentbin = NEAREST_LONG(btoa[ii]);
if (currentbin != oldbin) {
if (currentbin > 0) {
calcpt = oldbin + 0.5;
lobin = (ii - 1) * TDT / dsdt;
hibin = ii * TDT / dsdt;
} else {
calcpt = oldbin - 0.5;
lobin = -((ii - 1) * TDT / dsdt);
hibin = -(ii * TDT / dsdt);
}
while (fabs(calcpt) < fabs(btoa[ii])) {
/* Negative bin number means remove that bin */
/* Positive bin number means add a bin there */
*diffbinptr = NEAREST_LONG(LININTERP(calcpt, btoa[ii - 1],
btoa[ii], lobin,
hibin));
diffbinptr++;
calcpt = (currentbin > 0) ? calcpt + 1.0 : calcpt - 1.0;
}
oldbin = currentbin;
}
}
*diffbinptr = cmd->numout; /* Used as a marker */
}
diffbinptr = diffbins;
/* Now perform the barycentering */
printf("Massaging the data...\n\n");
printf("Amount Complete = 0%%");
/* Allocate our data array */
outdata = gen_fvect(worklen);
do { /* Loop to read and write the data */
int numwritten = 0;
double block_avg, block_var;
if (RAWDATA)
numread = read_psrdata(outdata, worklen, &s, idispdt, &padding,
maskchans, &nummasked, &obsmask);
else if (useshorts)
numread = read_shorts(s.files[0], outdata, worklen, numchan);
else
numread = read_floats(s.files[0], outdata, worklen, numchan);
if (numread == 0)
break;
/* Downsample if requested */
if (cmd->downsamp > 1)
numread = downsample(outdata, numread, cmd->downsamp);
/* Determine the approximate local average */
avg_var(outdata, numread, &block_avg, &block_var);
/* Print percent complete */
newper = (int) ((float) totwrote / cmd->numout * 100.0) + 1;
if (newper > oldper) {
printf("\rAmount Complete = %3d%%", newper);
fflush(stdout);
oldper = newper;
}
/* Simply write the data if we don't have to add or */
/* remove any bins from this batch. */
/* OR write the amount of data up to cmd->numout or */
/* the next bin that will be added or removed. */
numtowrite = abs(*diffbinptr) - datawrote;
/* FIXME: numtowrite+totwrote can wrap! */
if ((totwrote + numtowrite) > cmd->numout)
numtowrite = cmd->numout - totwrote;
if (numtowrite > numread)
numtowrite = numread;
chkfwrite(outdata, sizeof(float), numtowrite, outfile);
datawrote += numtowrite;
totwrote += numtowrite;
numwritten += numtowrite;
/* Update the statistics */
if (!padding) {
for (ii = 0; ii < numtowrite; ii++)
update_stats(statnum + ii, outdata[ii], &min, &max, &avg, &var);
statnum += numtowrite;
}
if ((datawrote == abs(*diffbinptr)) && (numwritten != numread) && (totwrote < cmd->numout)) { /* Add/remove a bin */
float favg;
int skip, nextdiffbin;
skip = numtowrite;
do { /* Write the rest of the data after adding/removing a bin */
if (*diffbinptr > 0) {
/* Add a bin */
favg = (float) block_avg;
chkfwrite(&favg, sizeof(float), 1, outfile);
numadded++;
totwrote++;
} else {
/* Remove a bin */
numremoved++;
datawrote++;
numwritten++;
skip++;
}
diffbinptr++;
/* Write the part after the diffbin */
numtowrite = numread - numwritten;
if ((totwrote + numtowrite) > cmd->numout)
numtowrite = cmd->numout - totwrote;
nextdiffbin = abs(*diffbinptr) - datawrote;
if (numtowrite > nextdiffbin)
numtowrite = nextdiffbin;
chkfwrite(outdata + skip, sizeof(float), numtowrite, outfile);
numwritten += numtowrite;
datawrote += numtowrite;
totwrote += numtowrite;
/* Update the statistics and counters */
if (!padding) {
for (ii = 0; ii < numtowrite; ii++)
update_stats(statnum + ii, outdata[skip + ii],
&min, &max, &avg, &var);
statnum += numtowrite;
}
skip += numtowrite;
/* Stop if we have written out all the data we need to */
if (totwrote == cmd->numout)
break;
} while (numwritten < numread);
}
/* Stop if we have written out all the data we need to */
if (totwrote == cmd->numout)
break;
} while (numread);
/* Free the arrays used in barycentering */
vect_free(bobsf);
vect_free(btoa);
vect_free(ttoa);
}
/* Calculate what the amount of padding we need */
if (cmd->numout > totwrote)
padwrote = padtowrite = cmd->numout - totwrote;
/* Write the new info file for the output data */
if (!cmd->nobaryP) {
idata.bary = 1;
idata.mjd_i = (int) floor(blotoa - (barydispdt / SECPERDAY));
idata.mjd_f = blotoa - (barydispdt / SECPERDAY) - idata.mjd_i;
}
if (cmd->downsamp > 1)
idata.dt = dsdt;
update_infodata(&idata, totwrote, padtowrite, diffbins, numdiffbins);
writeinf(&idata);
/* Set the padded points equal to the average data point */
if (idata.numonoff >= 1) {
int jj, index, startpad, endpad;
for (ii = 0; ii < worklen; ii++)
outdata[ii] = avg;
fclose(outfile);
outfile = chkfopen(datafilenm, "rb+");
for (ii = 0; ii < idata.numonoff; ii++) {
index = 2 * ii;
startpad = idata.onoff[index + 1];
if (ii == idata.numonoff - 1)
endpad = idata.N - 1;
else
endpad = idata.onoff[index + 2];
chkfseek(outfile, (startpad + 1) * sizeof(float), SEEK_SET);
padtowrite = endpad - startpad;
for (jj = 0; jj < padtowrite / worklen; jj++)
chkfwrite(outdata, sizeof(float), worklen, outfile);
chkfwrite(outdata, sizeof(float), padtowrite % worklen, outfile);
}
}
vect_free(outdata);
// Close all the raw files and free their vectors
close_rawfiles(&s);
/* Print simple stats and results */
var /= (datawrote - 1);
/* Conver the '.dat' file to '.sdat' if requested */
if (cmd->shortsP) {
FILE *infile;
int safe_convert = 1, bufflen = 65536;
char *sdatafilenm;
float *fbuffer;
short *sbuffer;
offset = (int) (floor(avg));
if ((max - min) > (SHRT_MAX - SHRT_MIN)) {
if ((max - min) < 1.5 * (SHRT_MAX - SHRT_MIN)) {
printf("Warning: There is more dynamic range in the data\n"
" than can be handled perfectly:\n"
" max - min = %.2f - %.2f = %.2f\n"
" Clipping the low values...\n\n", max, min,
max - min);
offset = max - SHRT_MAX;
} else {
printf("Error: There is way too much dynamic range in the data:\n"
" max - min = %.2f - %.2f = %.2f\n"
" Not converting to shorts.\n\n", max, min, max - min);
safe_convert = 0;
}
}
if (safe_convert) {
fbuffer = gen_fvect(bufflen);
sbuffer = gen_svect(bufflen);
sdatafilenm = (char *) calloc(slen, 1);
sprintf(sdatafilenm, "%s.sdat", cmd->outfile);
printf("\n\nConverting floats in '%s' to shorts in '%s'.",
datafilenm, sdatafilenm);
fflush(NULL);
infile = chkfopen(datafilenm, "rb");
outfile = chkfopen(sdatafilenm, "wb");
while ((numread = chkfread(fbuffer, sizeof(float), bufflen, infile))) {
for (ii = 0; ii < numread; ii++)
sbuffer[ii] = (short) (fbuffer[ii] + 1e-20 - offset);
chkfwrite(sbuffer, sizeof(short), numread, outfile);
}
fclose(infile);
fclose(outfile);
remove(datafilenm);
vect_free(fbuffer);
vect_free(sbuffer);
}
}
printf("\n\nDone.\n\nSimple statistics of the output data:\n");
printf(" Data points written: %ld\n", totwrote);
if (padwrote)
printf(" Padding points written: %ld\n", padwrote);
if (!cmd->nobaryP) {
if (numadded)
printf(" Bins added for barycentering: %d\n", numadded);
if (numremoved)
printf(" Bins removed for barycentering: %d\n", numremoved);
}
printf(" Maximum value of data: %.2f\n", max);
printf(" Minimum value of data: %.2f\n", min);
printf(" Data average value: %.2f\n", avg);
printf(" Data standard deviation: %.2f\n", sqrt(var));
if (cmd->shortsP && offset != 0)
printf(" Offset applied to data: %d\n", -offset);
printf("\n");
/* Cleanup */
if (cmd->maskfileP) {
free_mask(obsmask);
vect_free(maskchans);
}
vect_free(tobsf);
vect_free(dispdt);
vect_free(idispdt);
free(outinfonm);
free(datafilenm);
if (!cmd->nobaryP)
vect_free(diffbins);
return (0);
}
/* NEW Clipping Routine (uses channel running averages) */
int new_clip_times(unsigned char *rawdata, int ptsperblk, int numchan,
float clip_sigma, unsigned char *good_chan_levels)
/* Perform time-domain clipping of rawdata. This is a 2D */
/* array with ptsperblk*numchan points, each of which is an */
/* unsigned char. The clipping is done at clip_sigma sigma */
/* above/below the running mean. The up-to-date running */
/* averages of the channels are returned in good_chan_levels */
/* (which must be pre-allocated). */
{
static float *chan_running_avg;
static float running_avg = 0.0, running_std = 0.0;
static int blocksread = 0, firsttime = 1;
static long long current_point = 0;
float *zero_dm_block, *median_temp;
double *chan_avg_temp;
float current_med, trigger;
double current_avg = 0.0, current_std = 0.0;
unsigned char *powptr;
int ii, jj, clipit = 0, clipped = 0;
if (firsttime) {
chan_running_avg = gen_fvect(numchan);
firsttime = 0;
}
chan_avg_temp = gen_dvect(numchan);
zero_dm_block = gen_fvect(ptsperblk);
median_temp = gen_fvect(ptsperblk);
/* Calculate the zero DM time series */
for (ii = 0; ii < ptsperblk; ii++) {
zero_dm_block[ii] = 0.0;
powptr = rawdata + ii * numchan;
for (jj = 0; jj < numchan; jj++)
zero_dm_block[ii] += *powptr++;
median_temp[ii] = zero_dm_block[ii];
}
current_med = median(median_temp, ptsperblk);
/* Calculate the current standard deviation and mean */
/* but only for data points that are within a certain */
/* fraction of the median value. This removes the */
/* really strong RFI from the calculation. */
{
float lo_cutoff, hi_cutoff;
int numgoodpts = 0;
lo_cutoff = 0.7 * current_med;
hi_cutoff = 1.3 * current_med;
for (jj = 0; jj < numchan; jj++)
chan_avg_temp[jj] = 0.0;
/* Find the "good" points */
for (ii = 0; ii < ptsperblk; ii++) {
if (zero_dm_block[ii] > lo_cutoff && zero_dm_block[ii] < hi_cutoff) {
median_temp[numgoodpts] = zero_dm_block[ii];
powptr = rawdata + ii * numchan;
for (jj = 0; jj < numchan; jj++)
chan_avg_temp[jj] += *powptr++;
numgoodpts++;
}
}
/* Calculate the current average and stddev */
if (numgoodpts < 1) {
current_avg = running_avg;
current_std = running_std;
for (jj = 0; jj < numchan; jj++)
chan_avg_temp[jj] = chan_running_avg[jj];
} else {
avg_var(median_temp, numgoodpts, ¤t_avg, ¤t_std);
current_std = sqrt(current_std);
for (jj = 0; jj < numchan; jj++)
chan_avg_temp[jj] /= numgoodpts;
}
}
/* Update a pseudo running average and stdev */
if (blocksread) {
running_avg = 0.9 * running_avg + 0.1 * current_avg;
running_std = 0.9 * running_std + 0.1 * current_std;
for (ii = 0; ii < numchan; ii++)
chan_running_avg[ii] = 0.9 * chan_running_avg[ii] + 0.1 * chan_avg_temp[ii];
} else {
running_avg = current_avg;
running_std = current_std;
for (ii = 0; ii < numchan; ii++)
chan_running_avg[ii] = chan_avg_temp[ii];
if (running_avg == 0.0 || current_avg == 0.0)
printf("BAD RFI IN BLOCK#1!!!\n\n");
}
/* See if any points need clipping */
trigger = clip_sigma * running_std;
for (ii = 0; ii < ptsperblk; ii++) {
if (fabs(zero_dm_block[ii] - running_avg) > trigger) {
clipit = 1;
break;
}
}
/* Update the good channel levels */
for (ii = 0; ii < numchan; ii++)
good_chan_levels[ii] = (unsigned char) (chan_running_avg[jj] + 0.5);
/* Replace the bad channel data with channel median values */
/* that are scaled to equal the running_avg. */
if (clipit) {
for (ii = 0; ii < ptsperblk; ii++) {
if (fabs(zero_dm_block[ii] - running_avg) > trigger) {
powptr = rawdata + ii * numchan;
for (jj = 0; jj < numchan; jj++)
*powptr++ = good_chan_levels[jj];
clipped++;
/* fprintf(stderr, "%lld\n", current_point); */
}
current_point++;
}
} else {
current_point += ptsperblk;
}
blocksread++;
free(chan_avg_temp);
free(zero_dm_block);
free(median_temp);
return clipped;
}