/* Read only part (N "spectra" in time) of the DATA column from the
* current subint from the set of files described by the psrfits
* struct. Put the data into the buffer "buffer". It is assumed that
* all files form a consistent set. Read automatically goes to the
* next file when one ends. Arrays should be allocated outside this
* routine. Counters are _not_ updated as they are in
* psrfits_read_subint().
*/
int psrfits_read_part_DATA(struct psrfits *pf, int N, int numunsigned,
float *fbuffer) {
struct hdrinfo *hdr = &(pf->hdr);
int colnum = 0, *status = &(pf->status);
// See if we need to move to next file
if (pf->rownum > pf->rows_per_file) {
printf("Closing file '%s'\n", pf->filename);
fits_close_file(pf->fptr, status);
pf->filenum++;
psrfits_open(pf);
if (*status==FILE_NOT_OPENED) {
printf("Finished with all input files.\n");
pf->filenum--;
*status = 1;
return *status;
}
}
int nivals = hdr->nchan * hdr->npol;
long long numdata = (long long) nivals * (long long) N;
long long bytes_to_read = (hdr->nbits * numdata) / 8L;
float *offsets = (float *)malloc(sizeof(float) * nivals);
float *scales = (float *)malloc(sizeof(float) * nivals);
unsigned char *buffer = (unsigned char *)malloc(numdata);
unsigned char *rawbuffer = buffer;
if (hdr->nbits==4) {
rawbuffer = (unsigned char *)malloc(bytes_to_read);
}
// Now read the data
fits_get_colnum(pf->fptr, 0, "DAT_OFFS", &colnum, status);
fits_read_col(pf->fptr, TFLOAT, colnum, pf->rownum, 1, nivals,
NULL, offsets, NULL, status);
fits_get_colnum(pf->fptr, 0, "DAT_SCL", &colnum, status);
fits_read_col(pf->fptr, TFLOAT, colnum, pf->rownum, 1, nivals,
NULL, scales, NULL, status);
fits_get_colnum(pf->fptr, 0, "DATA", &colnum, status);
fits_read_col(pf->fptr, TBYTE, colnum, pf->rownum, 1, bytes_to_read,
NULL, rawbuffer, NULL, status);
if (hdr->nbits==4) {
convert_4bit_to_8bit(rawbuffer, (unsigned char *)buffer, numdata);
free(rawbuffer);
}
// Now convert the 8-bit data to floats using the scales and offsets
apply_scales_and_offsets(hdr->nchan, hdr->npol, N, numunsigned,
scales, offsets, buffer, fbuffer);
free(offsets);
free(scales);
free(buffer);
// Complain on error
fits_report_error(stderr, *status);
return *status;
}
int main(int argc, char *argv[]) {
struct psrfits pf;
sprintf(pf.basefilename,
"/data2/demorest/parspec/parspec_test_B0329+54_0009");
pf.filenum=1;
int rv = psrfits_open(&pf);
pf.sub.dat_freqs = (float *)malloc(sizeof(float) * pf.hdr.nchan);
pf.sub.dat_weights = (float *)malloc(sizeof(float) * pf.hdr.nchan);
pf.sub.dat_offsets = (float *)malloc(sizeof(float)
* pf.hdr.nchan * pf.hdr.npol);
pf.sub.dat_scales = (float *)malloc(sizeof(float)
* pf.hdr.nchan * pf.hdr.npol);
pf.sub.data = (unsigned char *)malloc(pf.sub.bytes_per_subint);
while ((rv=psrfits_read_subint(&pf))==0) {
printf("Read subint (file %d, row %d/%d)\n",
pf.filenum, pf.rownum-1, pf.rows_per_file);
}
if (rv) { fits_report_error(stderr, rv); }
exit(0);
}
int main(int argc, char *argv[]) {
Cmdline *cmd;
struct psrfits pfi, pfo; // input and output
struct subband_info si;
int stat=0, padding=0, userN=0;
// Call usage() if we have no command line arguments
if (argc == 1) {
Program = argv[0];
usage();
exit(0);
}
// Parse the command line using the excellent program Clig
cmd = parseCmdline(argc, argv);
// Open the input PSRFITs files
psrfits_set_files(&pfi, cmd->argc, cmd->argv);
// Use the dynamic filename allocation
if (pfi.numfiles==0) pfi.filenum = cmd->startfile;
pfi.tot_rows = pfi.N = pfi.T = pfi.status = 0;
int rv = psrfits_open(&pfi);
if (rv) { fits_report_error(stderr, rv); exit(1); }
// Read the user weights if requested
si.userwgts = NULL;
if (cmd->wgtsfileP) {
read_weights(cmd->wgtsfile, &userN, &si.userwgts);
if (userN != pfi.hdr.nchan) {
printf("Error!: Input data has %d channels, but '%s' contains only %d weights!\n",
pfi.hdr.nchan, cmd->wgtsfile, userN);
exit(0);
}
printf("Overriding input channel weights with those in '%s'\n",
cmd->wgtsfile);
}
// Initialize the subbanding
// (including reading the first row of data and
// putting it in si->fbuffer)
init_subbanding(&pfi, &pfo, &si, cmd);
if (cmd->outputbasenameP)
strcpy(pfo.basefilename, cmd->outputbasename);
// Loop through the data
do {
// Put the overlapping parts from the next block into si->buffer
float *ptr = pfi.sub.fdata + si.buflen * si.bufwid;
if (padding==0)
stat = psrfits_read_part_DATA(&pfi, si.max_overlap, si.numunsigned, ptr);
if (stat || padding) { // Need to use padding since we ran out of data
printf("Adding a missing row (#%d) of padding to the subbands.\n",
pfi.tot_rows);
// Now fill the last part of si->fbuffer with the chan_avgs so that
// it acts like a correctly read block (or row)
fill_chans_with_avgs(si.max_overlap, si.bufwid,
ptr, si.chan_avgs);
}
//print_raw_chan_stats(pfi.sub.data, pfi.hdr.nsblk,
// pfi.hdr.nchan, pfi.hdr.npol);
// if the input data isn't 8 bit, unpack:
if (pfi.hdr.nbits == 2)
pf_unpack_2bit_to_8bit(&pfi, si.numunsigned);
else if (pfi.hdr.nbits == 4)
pf_unpack_4bit_to_8bit(&pfi, si.numunsigned);
if ((pfo.hdr.ds_time_fact == 1) &&
(pfo.hdr.ds_freq_fact == 1)) {
// No subbanding is needed, so just copy the float buffer
// This is useful if we are just changing the number of bits
// Could do it without a copy by simply exchanging pointers
// to the fdata buffers in pfo and pfi...
memcpy(pfo.sub.fdata, pfi.sub.fdata,
pfi.hdr.nsblk * pfi.hdr.npol * pfi.hdr.nchan * sizeof(float));
} else {
// Now create the subbanded row in the output buffer
make_subbands(&pfi, &si);
}
// Output only Stokes I (in place via floats)
if (pfo.hdr.onlyI && pfo.hdr.npol==4)
get_stokes_I(&pfo);
// Downsample in time (in place via floats)
if (pfo.hdr.ds_time_fact > 1)
downsample_time(&pfo);
// Compute new scales and offsets so that we can pack
// into 8-bits reliably
if (pfo.rownum == 1)
new_scales_and_offsets(&pfo, si.numunsigned, cmd);
// Convert the floats back to bytes in the output array
un_scale_and_offset_data(&pfo, si.numunsigned);
//print_raw_chan_stats(pfo.sub.data, pfo.hdr.nsblk / pfo.hdr.ds_time_fact,
// pfo.hdr.nchan / pfo.hdr.ds_freq_fact, pfo.hdr.npol);
// pack into 2 or 4 bits if needed
if (pfo.hdr.nbits == 2)
pf_pack_8bit_to_2bit(&pfo, si.numunsigned);
else if (pfo.hdr.nbits == 4)
pf_pack_8bit_to_4bit(&pfo, si.numunsigned);
// Write the new row to the output file
pfo.sub.offs = (pfo.tot_rows+0.5) * pfo.sub.tsubint;
psrfits_write_subint(&pfo);
// Break out of the loop here if stat is set
if (stat) break;
// shift the last part of the current row into the "last-row"
// part of the data buffer
memcpy(si.fbuffer, si.fbuffer + si.buflen * si.bufwid,
si.max_overlap * si.bufwid * sizeof(float));
// Read the next row (or padding)
padding = get_current_row(&pfi, &si);
// Set the new weights properly
new_weights(&pfi, &pfo);
} while (pfi.status == 0);
print_clips(&pfo);
rv = psrfits_close(&pfi);
if (rv>100) { fits_report_error(stderr, rv); }
rv = psrfits_close(&pfo);
if (rv>100) { fits_report_error(stderr, rv); }
exit(0);
}
int main(int argc, char *argv[])
{
Cmdline *cmd;
struct psrfits pfupper, pflower, pfo;
fitsfile *infits, *outfits;
char *pc1, *pc2;
char outfilename[200]; //Name of outfile if not specified on command line
int stat = 0, padding = 0, userN = 0, status;
// Call usage() if we have no command line arguments
if (argc == 1) {
Program = argv[0];
usage();
exit(0);
}
// Parse the command line using the excellent program Clig
cmd = parseCmdline(argc, argv);
pfupper.tot_rows = pfupper.N = pfupper.T = pfupper.status = 0; //Initialize upper band
pflower.tot_rows = pflower.N = pflower.T = pflower.status = 0; //Initialize lower band
pfupper.filenum = pflower.filenum = 1;
pfo.tot_rows = pfo.N = pfo.T = pfo.status = pfo.multifile = 0; //Initialize output
sprintf(pfupper.filename, cmd->argv[0]); //Copy filename specified on command line to
sprintf(pflower.filename, cmd->argv[0]); //upper and lower bands, will correct filenames shortly
if ((pc2 = strstr(pfupper.filename, "s1")) != NULL) //Upper contains s1, change to s0
strncpy(pc2, "s0", 2);
else if ((pc2 = strstr(pflower.filename, "s0")) != NULL) //Lower contains s0, change to s1
strncpy(pc2, "s1", 2);
else {
printf("Unable to determine which sideband is which\n");
exit(EXIT_FAILURE);
}
//Setting the name of the output file, setting as same name as input file, but removing s0/s1.
pc1 = strstr(pflower.filename, "s1");
pc2 = strrchr(pflower.filename, '.'); //At '.fits'
pc2--;
while ((pc2 >= pflower.filename) && isdigit(*pc2)) //Move through the digits to the separation char.
pc2--;
strncpy(outfilename, pflower.filename, pc1 - pflower.filename); //Copy everything up to s1 into outfilename
strncpy(outfilename + (pc1 - pflower.filename), pc1 + 2, pc2 - pc1 - 2); //Concatenate from after s1 to char before the separation char.
pc1 = outfilename + (pc2 - pflower.filename - 2);
*pc1 = 0;
int rv = psrfits_open(&pfupper); //Open upper band
if (rv) {
fits_report_error(stderr, rv);
exit(1);
}
rv = psrfits_open(&pflower); //Open lower band
if (rv) {
fits_report_error(stderr, rv);
exit(1);
}
pfo = pflower; //Copy all lower band variables into the output struct
if (!cmd->outputbasenameP)
sprintf(pfo.basefilename, basename(outfilename));
else
sprintf(pfo.basefilename, cmd->outputbasename);
pfo.filenum = 0;
sprintf(pfo.filename, "\0"); //Set filename to null so psrfits_open will create the filename for me
pfo.rownum = 1;
pfo.tot_rows = 0;
pfo.N = 0;
printf("lower rows_per_file=%d\n",pflower.rows_per_file);
printf("upper rows_per_file=%d\n",pfupper.rows_per_file);
if (pfupper.rows_per_file != pflower.rows_per_file) { //Sanity check for the two input frequency bands
fprintf(stderr, "rows_per_file in input files do not match!\n");
exit(1);
}
double upperfreqoflower, nextfromlower, lowerfreqofupper, numchandiff; //Used to find which frequencies to take from each band
double offsetfactor, scalefactor; //Factors which will be applied to offsets and scales
int upchanskip, lowchanskip; //Number of channels to skip in each banda
//Variables used to make code cleaner
int extrachanoffset, outoffset, upperoffset, numtocopyupper, loweroffset_skip,
loweroffset, numtocopylower, newuppernchan, newlowernchan;
double df = pflower.hdr.df;
int nchan = pflower.hdr.nchan;
int outnchan;
int npol = pflower.hdr.npol;
int nbits = pflower.hdr.nbits;
int nsblk = pflower.hdr.nsblk;
//Allocate memory for all upper and lower data
pflower.sub.dat_freqs = (double *) malloc(sizeof(double) * nchan);
pflower.sub.dat_weights = (float *) malloc(sizeof(float) * nchan);
pflower.sub.dat_offsets = (float *) malloc(sizeof(float) * nchan * npol);
pflower.sub.dat_scales = (float *) malloc(sizeof(float) * nchan * npol);
pflower.sub.rawdata = (unsigned char *) malloc(pflower.sub.bytes_per_subint);
pflower.sub.data = (unsigned char *) malloc(pflower.sub.bytes_per_subint*2);
pfupper.sub.dat_freqs = (double *) malloc(sizeof(double) * nchan);
pfupper.sub.dat_weights = (float *) malloc(sizeof(float) * nchan);
pfupper.sub.dat_offsets = (float *) malloc(sizeof(float) * nchan * npol);
pfupper.sub.dat_scales = (float *) malloc(sizeof(float) * nchan * npol);
pfupper.sub.rawdata = (unsigned char *) malloc(pfupper.sub.bytes_per_subint);
pfupper.sub.data = (unsigned char *) malloc(pfupper.sub.bytes_per_subint*2);
int firsttime = 1; //First time through do while loop
do {
print_percent_complete(pflower.rownum, pflower.rows_per_file,
pflower.rownum == 1 ? 1 : 0);
psrfits_read_subint(&pflower);
psrfits_read_subint(&pfupper);
if (firsttime) { //First time through loop, calculate factors for scales and offsets and number of channels to skip
firsttime = 0;
//Find the number of channels in the upper band which will be skipped
if (df < 0) { //Find channel order, low to high or high to low
upperfreqoflower = pflower.sub.dat_freqs[0]; //Highest frequency channel in lower band
lowerfreqofupper = pfupper.sub.dat_freqs[nchan - 1]; //Lowest frequency channel in upper band
} else {
upperfreqoflower = pflower.sub.dat_freqs[nchan - 1]; //Highest frequency channel in lower band
lowerfreqofupper = pfupper.sub.dat_freqs[0]; //Lowest frequency channel in upper band
}
nextfromlower = upperfreqoflower + fabs(df); //Second highest channel in lower band
numchandiff = (nextfromlower - lowerfreqofupper) / fabs(df); //Number of channels to skip in float form
int chanskip;
if (numchandiff > 0) { //Make sure there are channels which need to be skipped
if (numchandiff - (double) ((int) numchandiff) > .5) // See whether we need to round up integer channels to skip
chanskip = (int) numchandiff + 1;
else
chanskip = (int) numchandiff;
} else
chanskip = 0; //No need to skip any channels
if (chanskip % 2 == 1) { //Odd number of channels, give lower band the extra channel
upchanskip = chanskip / 2;
lowchanskip = chanskip / 2 + 1;
} else //Even number of channels to skip
upchanskip = lowchanskip = chanskip / 2;
if (upchanskip % 2 == 1) { //We want an even number of channels in upper band for 4-bit data to get copied correctly
++lowchanskip;
--upchanskip;
}
//Find new values given the number of channels skipped
pfo.hdr.nchan = outnchan = nchan + nchan - chanskip + 2; //New number of channels, plus 2 to make nchan=960 (many factors of 2)
pfo.hdr.BW = (double) outnchan *fabs(df); //New bandwidth
pfo.hdr.fctr = //New center frequency
(pflower.hdr.fctr - (double) (nchan / 2) * fabs(df)) + pfo.hdr.BW / 2.0;
pfo.sub.bytes_per_subint = //Calculate new number of bytes in each subint
outnchan * nsblk * nbits / 8 * npol;
//Allocate space for output data now that we know the new number of channels
pfo.sub.dat_freqs = (double *) malloc(sizeof(double) * outnchan);
pfo.sub.dat_weights = (float *) malloc(sizeof(float) * outnchan);
pfo.sub.dat_offsets = (float *) malloc(sizeof(float) * outnchan * npol);
pfo.sub.dat_scales = (float *) malloc(sizeof(float) * outnchan * npol);
pfo.sub.rawdata = (unsigned char *) malloc(pfo.sub.bytes_per_subint);
pfo.sub.data = (unsigned char *) malloc(pfo.sub.bytes_per_subint*2);
newuppernchan = nchan - upchanskip; //The number of channels to copy from the upper sideband.
newlowernchan = nchan - lowchanskip; //The number of channels to copy from the lower sideband.
extrachanoffset = 2; //Offset for 2 extra freq channels making nchan 960 in bytes
outoffset = (outnchan * npol); //Offset in each loop due to previously written data
upperoffset = (nchan * npol); //Offset in loop for upper band
numtocopyupper = (newuppernchan * npol); //Number of bytes to copy from upper band
loweroffset_skip = (lowchanskip * npol); //Number of bytes to skip when copying lower band due to
//having written upper band
loweroffset = //Number of bytes to skip due to having written previous lower band data
(nchan * npol);
numtocopylower = (newlowernchan * npol); //Number of bytes to copy from lower band
float upmean, upvar, lowmean, lowvar;
avg_var(pfupper.sub.dat_offsets + (nchan - upchanskip), //Find the mean and variance of the upper band's offsets
upchanskip, &upmean, &upvar);
printf("Upper offset stats: mean=%f variance=%f\n", upmean, upvar);
avg_var(pflower.sub.dat_offsets, lowchanskip, &lowmean, &lowvar); //Find the mean and variance of the lower band's offsets
printf("Lower offset stats: mean=%f variance=%f\n", lowmean, lowvar);
printf("Applying factor of %f to upper offsets\n", (lowmean / upmean));
offsetfactor = lowmean / upmean; //Set offset factor used to correct variance differences in the two bands
avg_var(pfupper.sub.dat_scales + (nchan - upchanskip), //Find the mean and var. of the upper band's scales
upchanskip, &upmean, &upvar);
printf("Upper scales stats: mean=%f variance=%f\n", upmean, upvar);
avg_var(pflower.sub.dat_scales, lowchanskip, &lowmean, &lowvar); //Find the mean and var. of the lower band's scales
printf("Lower scales stats: mean=%f variance=%f\n", lowmean, lowvar);
printf("Applying factor of %f to upper scales\n", (lowmean / upmean));
scalefactor = lowmean / upmean; //Set scale factor used to correct variance differences in the two bands
}
if (pflower.status == 0 && pfupper.status == 0) {
//Copy info from the lower band subint struct to the output file's subint struct
pfo.sub.tsubint = pflower.sub.tsubint;
pfo.sub.offs = pflower.sub.offs;
pfo.sub.lst = pflower.sub.lst;
pfo.sub.ra = pflower.sub.ra;
pfo.sub.dec = pflower.sub.dec;
pfo.sub.glon = pflower.sub.glon;
pfo.sub.glat = pflower.sub.glat;
pfo.sub.feed_ang = pflower.sub.feed_ang;
pfo.sub.pos_ang = pflower.sub.pos_ang;
pfo.sub.par_ang = pflower.sub.par_ang;
pfo.sub.tel_az = pflower.sub.tel_az;
pfo.sub.tel_zen = pflower.sub.tel_zen;
pfo.sub.FITS_typecode = pflower.sub.FITS_typecode;
//Create variables to reduce column width of lines below
double *dat_freqs = pfo.sub.dat_freqs;
double *udat_freqs = pfupper.sub.dat_freqs;
double *ldat_freqs = pflower.sub.dat_freqs;
float *dat_weights = pfo.sub.dat_weights;
float *udat_weights = pfupper.sub.dat_weights;
float *ldat_weights = pflower.sub.dat_weights;
float *dat_offsets = pfo.sub.dat_offsets;
float *udat_offsets = pfupper.sub.dat_offsets;
float *ldat_offsets = pflower.sub.dat_offsets;
float *dat_scales = pfo.sub.dat_scales;
float *udat_scales = pfupper.sub.dat_scales;
float *ldat_scales = pflower.sub.dat_scales;
unsigned char *data = pfo.sub.data;
unsigned char *udata = pfupper.sub.data;
unsigned char *ldata = pflower.sub.data;
if (df < 0) {
//Copy frequency labels
dat_freqs[1] = udat_freqs[0] + fabs(df); //Calculate the frequency labels
dat_freqs[0] = dat_freqs[1] + fabs(df); //for our two empty frequency channels
int newuppernchan = nchan - upchanskip; //The number of channels to copy from the upper band
int newlowernchan = nchan - lowchanskip; //The number of channels to copy from the lower band
memcpy(dat_freqs + 2, udat_freqs, sizeof(double) * newuppernchan); //Copy from the upper band, skipping first two chans.
memcpy(dat_freqs + newuppernchan + 2, //Copy from the lower band
ldat_freqs + lowchanskip, sizeof(double) * newlowernchan);
//Copy weights
dat_weights[0] = dat_weights[1] = 0; //Set the weights of first two channels to 0, so they shouldn't be used in calculations
memcpy(dat_weights + 2, udat_weights, //Copy weights from the upper band
sizeof(float) * newuppernchan);
memcpy(dat_weights + 2 + newuppernchan, //Copy weights from the lower band
ldat_weights + lowchanskip, sizeof(float) * newlowernchan);
//Copy offsets
dat_offsets[0] = dat_offsets[1] = //Set offsets of first two channels to the same as upper's first channel
udat_offsets[0]; //(shouldn't matter since they should be ignored)
int ii;
for (ii = 0; ii < newuppernchan; ++ii) //Apply offset factor to upper band
udat_offsets[ii] = udat_offsets[ii] * (offsetfactor);
memcpy(dat_offsets + 2 * npol, udat_offsets, //Copy upper offsets
sizeof(float) * newuppernchan * npol);
memcpy(dat_offsets + (newuppernchan + 2) * npol, //Copy lower offsets
ldat_offsets + lowchanskip, sizeof(float) * newlowernchan * npol);
//Copy scales
for (ii = 0; ii < newuppernchan; ++ii) //Apply scale factor to upper band
udat_scales[ii] = udat_scales[ii] * (scalefactor);
dat_scales[0] = dat_scales[1] = udat_scales[0];
memcpy(dat_scales + 2 * npol, udat_scales, //Copy upper scales
sizeof(float) * newuppernchan * npol);
memcpy(dat_scales + (newuppernchan + 2) * npol, //Copy lower scales
ldat_scales + lowchanskip, sizeof(float) * newlowernchan * npol);
//Copy the data
for (ii = 0; ii < nsblk; ++ii) { //Loop through data copying into place
memcpy(data + ii * outoffset + extrachanoffset,
udata + ii * upperoffset, numtocopyupper);
memcpy(data + ii * outoffset + extrachanoffset +
numtocopyupper,
ldata + ii * loweroffset + loweroffset_skip, numtocopylower);
}
psrfits_write_subint(&pfo);
} else {
}
}
} while (pfo.rownum <= pfo.rows_per_file && pfupper.status==0 && pflower.status==0);
printf("Closing file '%s'\n", pflower.filename);
fits_close_file(pfupper.fptr, &status);
printf("Closing file '%s'\n", pfupper.filename);
fits_close_file(pflower.fptr, &status);
if(pflower.status!=0||pfupper.status!=0)
{
fprintf(stderr,"An error occurred when combining the two Mock files!\n");
if(pflower.status==108||pfupper.status==108)
fprintf(stderr,"One or both of the files is incomplete.\n");
exit(1);
}
exit(0);
}
/* Read next subint from the set of files described
* by the psrfits struct. It is assumed that all files
* form a consistent set. Read automatically goes to the
* next file when one ends. Arrays should be allocated
* outside this routine.
*/
int psrfits_read_subint(struct psrfits *pf) {
struct hdrinfo *hdr = &(pf->hdr);
struct subint *sub = &(pf->sub);
int colnum = 0, *status = &(pf->status);
// See if we need to move to next file
if (pf->rownum > pf->rows_per_file) {
printf("Closing file '%s'\n", pf->filename);
fits_close_file(pf->fptr, status);
pf->filenum++;
psrfits_open(pf);
if (*status==FILE_NOT_OPENED) {
printf("Finished with all input files.\n");
pf->filenum--;
*status = 1;
return *status;
}
}
int mode = psrfits_obs_mode(hdr->obs_mode);
int nchan = hdr->nchan;
int nivals = hdr->nchan * hdr->npol;
int row = pf->rownum;
// TODO: bad! really need to base this on column names
fits_get_colnum(pf->fptr, 0, "TSUBINT", &colnum, status);
fits_read_col(pf->fptr, TDOUBLE, colnum, row, 1, 1, NULL, &(sub->tsubint),
NULL, status);
double last_offs = sub->offs;
fits_get_colnum(pf->fptr, 0, "OFFS_SUB", &colnum, status);
fits_read_col(pf->fptr, TDOUBLE, colnum, row, 1, 1, NULL, &(sub->offs),
NULL, status);
// Hack to fix wrapping in coherent data
if (pf->tot_rows > 0) {
double delta_offs = sub->offs - last_offs;
double wrap_offs = 4294967296L * hdr->dt;
if (delta_offs < -0.5*wrap_offs) {
sub->offs += wrap_offs;
fprintf(stderr, "Warning: detected likely counter wrap, attempting to fix it.\n");
}
}
fits_get_colnum(pf->fptr, 0, "LST_SUB", &colnum, status);
fits_read_col(pf->fptr, TDOUBLE, colnum, row, 1, 1, NULL, &(sub->lst),
NULL, status);
fits_get_colnum(pf->fptr, 0, "RA_SUB", &colnum, status);
fits_read_col(pf->fptr, TDOUBLE, colnum, row, 1, 1, NULL, &(sub->ra),
NULL, status);
fits_get_colnum(pf->fptr, 0, "DEC_SUB", &colnum, status);
fits_read_col(pf->fptr, TDOUBLE, colnum, row, 1, 1, NULL, &(sub->dec),
NULL, status);
fits_get_colnum(pf->fptr, 0, "GLON_SUB", &colnum, status);
fits_read_col(pf->fptr, TDOUBLE, colnum, row, 1, 1, NULL, &(sub->glon),
NULL, status);
fits_get_colnum(pf->fptr, 0, "GLAT_SUB", &colnum, status);
fits_read_col(pf->fptr, TDOUBLE, colnum, row, 1, 1, NULL, &(sub->glat),
NULL, status);
fits_get_colnum(pf->fptr, 0, "FD_ANG", &colnum, status);
fits_read_col(pf->fptr, TDOUBLE, colnum, row, 1, 1, NULL, &(sub->feed_ang),
NULL, status);
fits_get_colnum(pf->fptr, 0, "POS_ANG", &colnum, status);
fits_read_col(pf->fptr, TDOUBLE, colnum, row, 1, 1, NULL, &(sub->pos_ang),
NULL, status);
fits_get_colnum(pf->fptr, 0, "PAR_ANG", &colnum, status);
fits_read_col(pf->fptr, TDOUBLE, colnum, row, 1, 1, NULL, &(sub->par_ang),
NULL, status);
fits_get_colnum(pf->fptr, 0, "TEL_AZ", &colnum, status);
fits_read_col(pf->fptr, TDOUBLE, colnum, row, 1, 1, NULL, &(sub->tel_az),
NULL, status);
fits_get_colnum(pf->fptr, 0, "TEL_ZEN", &colnum, status);
fits_read_col(pf->fptr, TDOUBLE, colnum, row, 1, 1, NULL, &(sub->tel_zen),
NULL, status);
fits_get_colnum(pf->fptr, 0, "DAT_FREQ", &colnum, status);
fits_read_col(pf->fptr, TFLOAT, colnum, row, 1, nchan, NULL, sub->dat_freqs,
NULL, status);
fits_get_colnum(pf->fptr, 0, "DAT_WTS", &colnum, status);
fits_read_col(pf->fptr, TFLOAT, colnum, row, 1, nchan, NULL, sub->dat_weights,
NULL, status);
fits_get_colnum(pf->fptr, 0, "DAT_OFFS", &colnum, status);
fits_read_col(pf->fptr, TFLOAT, colnum, row, 1, nivals, NULL, sub->dat_offsets,
NULL, status);
fits_get_colnum(pf->fptr, 0, "DAT_SCL", &colnum, status);
fits_read_col(pf->fptr, TFLOAT, colnum, row, 1, nivals, NULL, sub->dat_scales,
NULL, status);
fits_get_colnum(pf->fptr, 0, "DATA", &colnum, status);
if (mode==SEARCH_MODE) {
if (hdr->nbits==32)
fits_read_col(pf->fptr, TFLOAT, colnum, row, 1,
sub->bytes_per_subint/4,
NULL, sub->rawdata, NULL, status);
else
fits_read_col(pf->fptr, TBYTE, colnum, row, 1,
sub->bytes_per_subint,
NULL, sub->rawdata, NULL, status);
if (hdr->nbits==4) pf_4bit_to_8bit(pf);
} else if (mode==FOLD_MODE) {
fits_read_col(pf->fptr, TFLOAT, colnum, row, 1, sub->bytes_per_subint,
NULL, sub->data, NULL, status);
}
// Complain on error
fits_report_error(stderr, *status);
// Update counters
if (!(*status)) {
pf->rownum++;
pf->tot_rows++;
pf->N += hdr->nsblk;
pf->T = pf->N * hdr->dt;
}
return *status;
}
int main(int argc, char *argv[]) {
/* Cmd line */
static struct option long_opts[] = {
{"output", 1, NULL, 'o'},
{"npulse", 1, NULL, 'n'},
{"nbin", 1, NULL, 'b'},
{"nthread", 1, NULL, 'j'},
{"initial", 1, NULL, 'i'},
{"final", 1, NULL, 'f'},
{"time", 1, NULL, 'T'},
{"length", 1, NULL, 'L'},
{"src", 1, NULL, 's'},
{"polyco", 1, NULL, 'p'},
{"parfile", 1, NULL, 'P'},
{"foldfreq",1, NULL, 'F'},
{"cal", 0, NULL, 'C'},
{"unsigned",0, NULL, 'u'},
{"quiet", 0, NULL, 'q'},
{"help", 0, NULL, 'h'},
{0,0,0,0}
};
int opt, opti;
int nbin=256, nthread=4, fnum_start=1, fnum_end=0;
int quiet=0, raw_signed=1, use_polycos=1, cal=0;
int npulse_per_file = 64;
double start_time=0.0, process_time=0.0;
double fold_frequency=0.0;
char output_base[256] = "";
char polyco_file[256] = "";
char par_file[256] = "";
char source[24]; source[0]='\0';
while ((opt=getopt_long(argc,argv,"o:n:b:j:i:f:T:L:s:p:P:F:Cuqh",long_opts,&opti))!=-1) {
switch (opt) {
case 'o':
strncpy(output_base, optarg, 255);
output_base[255]='\0';
break;
case 'n':
npulse_per_file = atoi(optarg);
break;
case 'b':
nbin = atoi(optarg);
break;
case 'j':
nthread = atoi(optarg);
break;
case 'i':
fnum_start = atoi(optarg);
break;
case 'f':
fnum_end = atoi(optarg);
break;
case 'T':
start_time = atof(optarg);
break;
case 'L':
process_time = atof(optarg);
break;
case 's':
strncpy(source, optarg, 24);
source[23]='\0';
break;
case 'p':
strncpy(polyco_file, optarg, 255);
polyco_file[255]='\0';
use_polycos = 1;
break;
case 'P':
strncpy(par_file, optarg, 255);
par_file[255] = '\0';
break;
case 'F':
fold_frequency = atof(optarg);
use_polycos = 0;
break;
case 'C':
cal = 1;
use_polycos = 0;
break;
case 'u':
raw_signed=0;
break;
case 'q':
quiet=1;
break;
case 'h':
default:
usage();
exit(0);
break;
}
}
if (optind==argc) {
usage();
exit(1);
}
/* If no polyco/par file given, default to polyco.dat */
if (use_polycos && (par_file[0]=='\0' && polyco_file[0]=='\0'))
sprintf(polyco_file, "polyco.dat");
/* Open first file */
struct psrfits pf;
strcpy(pf.basefilename, argv[optind]);
pf.filenum = fnum_start;
pf.tot_rows = pf.N = pf.T = pf.status = 0;
pf.hdr.chan_dm = 0.0; // What if folding data that has been partially de-dispersed?
pf.filename[0]='\0';
int rv = psrfits_open(&pf);
if (rv) { fits_report_error(stderr, rv); exit(1); }
/* Check any constraints */
if (pf.hdr.nbits!=8) {
fprintf(stderr, "Only implemented for 8-bit data (read nbits=%d).\n",
pf.hdr.nbits);
exit(1);
}
/* Check for calfreq */
if (cal) {
if (pf.hdr.cal_freq==0.0) {
if (fold_frequency==0.0) {
fprintf(stderr, "Error: Cal mode selected, but CAL_FREQ=0. "
"Set cal frequency with -F\n");
exit(1);
} else {
pf.hdr.cal_freq = fold_frequency;
}
} else {
fold_frequency = pf.hdr.cal_freq;
}
}
/* Set up output file */
struct psrfits pf_out;
memcpy(&pf_out, &pf, sizeof(struct psrfits));
if (source[0]!='\0') { strncpy(pf_out.hdr.source, source, 24); }
else { strncpy(source, pf.hdr.source, 24); source[23]='\0'; }
if (output_base[0]=='\0') {
/* Set up default output filename */
if (start_time>0.0)
sprintf(output_base, "%s_SP_%s_%5.5d_%5.5d_%4.4d_%3.3d%s",
pf_out.hdr.backend,
pf_out.hdr.source, pf_out.hdr.start_day,
(int)pf_out.hdr.start_sec, fnum_start,
(int)start_time,
cal ? "_cal" : "");
else
sprintf(output_base, "%s_SP_%s_%5.5d_%5.5d%s", pf_out.hdr.backend,
pf_out.hdr.source, pf_out.hdr.start_day,
(int)pf_out.hdr.start_sec, cal ? "_cal" : "");
}
strcpy(pf_out.basefilename, output_base);
if (cal) {
sprintf(pf_out.hdr.obs_mode, "CAL");
sprintf(pf_out.hdr.cal_mode, "SYNC");
} else
sprintf(pf_out.hdr.obs_mode, "PSR");
strncpy(pf_out.fold.parfile,par_file,255); pf_out.fold.parfile[255]='\0';
pf_out.fptr = NULL;
pf_out.filenum=0;
pf_out.status=0;
pf_out.hdr.nbin=nbin;
pf_out.sub.FITS_typecode = TFLOAT;
pf_out.sub.bytes_per_subint = sizeof(float) *
pf_out.hdr.nchan * pf_out.hdr.npol * pf_out.hdr.nbin;
pf_out.multifile = 1;
pf_out.quiet = 1;
pf_out.rows_per_file = npulse_per_file;
rv = psrfits_create(&pf_out);
if (rv) { fits_report_error(stderr, rv); exit(1); }
/* Alloc data buffers */
pf.sub.dat_freqs = (float *)malloc(sizeof(float) * pf.hdr.nchan);
pf_out.sub.dat_freqs = pf.sub.dat_freqs;
pf.sub.dat_weights = (float *)malloc(sizeof(float) * pf.hdr.nchan);
pf_out.sub.dat_weights = (float *)malloc(sizeof(float) * pf.hdr.nchan);
pf.sub.dat_offsets = (float *)malloc(sizeof(float)
* pf.hdr.nchan * pf.hdr.npol);
pf_out.sub.dat_offsets = (float *)malloc(sizeof(float)
* pf.hdr.nchan * pf.hdr.npol);
pf.sub.dat_scales = (float *)malloc(sizeof(float)
* pf.hdr.nchan * pf.hdr.npol);
pf_out.sub.dat_scales = (float *)malloc(sizeof(float)
* pf.hdr.nchan * pf.hdr.npol);
pf_out.sub.data = (unsigned char *)malloc(pf_out.sub.bytes_per_subint);
/* Output scale/offset */
int i, j, ipol, ichan;
float offset_uv=0.0;
// Extra cross-term offset for GUPPI
if (strcmp("GUPPI",pf.hdr.backend)==0) {
offset_uv=0.5;
fprintf(stderr, "Found backend=GUPPI, setting offset_uv=%f\n",
offset_uv);
}
// TODO: copy these from the input file
for (ipol=0; ipol<pf.hdr.npol; ipol++) {
for (ichan=0; ichan<pf.hdr.nchan; ichan++) {
float offs = 0.0;
if (ipol>1) offs = offset_uv;
pf_out.sub.dat_scales[ipol*pf.hdr.nchan + ichan] = 1.0;
pf_out.sub.dat_offsets[ipol*pf.hdr.nchan + ichan] = offs;
}
}
for (i=0; i<pf.hdr.nchan; i++) { pf_out.sub.dat_weights[i]=1.0; }
/* Read or make polycos */
int npc=0, ipc=0;
struct polyco *pc = NULL;
if (use_polycos) {
if (polyco_file[0]=='\0') {
/* Generate from par file */
npc = make_polycos(par_file, &pf.hdr, source, &pc);
if (npc<=0) {
fprintf(stderr, "Error generating polycos.\n");
exit(1);
}
printf("Auto-generated %d polycos, src=%s\n", npc, source);
} else {
/* Read from polyco file */
FILE *pcfile = fopen(polyco_file, "r");
if (pcfile==NULL) {
fprintf(stderr, "Couldn't open polyco file.\n");
exit(1);
}
npc = read_all_pc(pcfile, &pc);
if (npc==0) {
fprintf(stderr, "Error parsing polyco file.\n");
exit(1);
}
fclose(pcfile);
}
} else {
// Const fold period mode, generate a fake polyco?
pc = (struct polyco *)malloc(sizeof(struct polyco));
sprintf(pc[0].psr, "CONST");
pc[0].mjd = (int)pf.hdr.MJD_epoch;
pc[0].fmjd = fmod(pf.hdr.MJD_epoch,1.0);
pc[0].rphase = 0.0;
pc[0].f0 = fold_frequency;
pc[0].nsite = 0; // Does this matter?
pc[0].nmin = 24 * 60;
pc[0].nc = 1;
pc[0].rf = pf.hdr.fctr;
pc[0].c[0] = 0.0;
pc[0].used = 0;
npc = 1;
}
int *pc_written = (int *)malloc(sizeof(int) * npc);
for (i=0; i<npc; i++) pc_written[i]=0;
/* Set up fold buf */
struct foldbuf fb;
fb.nchan = pf.hdr.nchan;
fb.npol = pf.hdr.npol;
fb.nbin = pf_out.hdr.nbin;
malloc_foldbuf(&fb);
clear_foldbuf(&fb);
struct fold_args fargs;
fargs.data = (char *)malloc(sizeof(char)*pf.sub.bytes_per_subint);
fargs.fb = &fb;
fargs.nsamp = 1;
fargs.tsamp = pf.hdr.dt;
fargs.raw_signed = raw_signed;
/* Main loop */
rv=0;
int imjd;
double fmjd, fmjd0=0, fmjd_samp, fmjd_epoch;
long long cur_pulse=0, last_pulse=0;
double psr_freq=0.0;
int first_loop=1, first_data=1, sampcount=0, last_filenum=0;
int bytes_per_sample = pf.hdr.nchan * pf.hdr.npol;
signal(SIGINT, cc);
while (run) {
/* Read data block */
pf.sub.data = (unsigned char *)fargs.data;
rv = psrfits_read_subint(&pf);
if (rv) {
if (rv==FILE_NOT_OPENED) rv=0; // Don't complain on file not found
run=0;
break;
}
/* If we've passed final file, exit */
if (fnum_end && pf.filenum>fnum_end) { run=0; break; }
/* Get start date, etc */
imjd = (int)pf.hdr.MJD_epoch;
fmjd = (double)(pf.hdr.MJD_epoch - (long double)imjd);
fmjd += (pf.sub.offs-0.5*pf.sub.tsubint)/86400.0;
/* Select polyco set.
* We'll assume same one is valid for whole data block.
*/
if (use_polycos) {
ipc = select_pc(pc, npc, source, imjd, fmjd);
//ipc = select_pc(pc, npc, NULL, imjd, fmjd);
if (ipc<0) {
fprintf(stderr,
"No matching polycos (src=%s, imjd=%d, fmjd=%f)\n",
source, imjd, fmjd);
break;
}
} else {
ipc = 0;
}
pc[ipc].used = 1; // Mark this polyco set as used for folding
/* First time stuff */
if (first_loop) {
fmjd0 = fmjd;
psr_phase(&pc[ipc], imjd, fmjd, NULL, &last_pulse);
pf_out.sub.offs=0.0;
first_loop=0;
for (i=0; i<pf.hdr.nchan; i++) {
pf_out.sub.dat_weights[i]=pf.sub.dat_weights[i];
}
last_filenum = pf_out.filenum;
}
/* Check to see if its time to process data */
if (start_time>0.0) {
double cur_time = (fmjd - fmjd0) * 86400.0;
if (cur_time<start_time)
continue;
}
if (first_data) {
psr_phase(&pc[ipc], imjd, fmjd, NULL, &last_pulse);
first_data=0;
}
/* Check to see if we're done */
if (process_time>0.0) {
double cur_time = (fmjd - fmjd0) * 86400.0;
if (cur_time > start_time + process_time) {
run=0;
break;
}
}
/* for singlepulse: loop over samples, output a new subint
* whenever pulse number increases.
*/
for (i=0; i<pf.hdr.nsblk; i++) {
/* Keep track of timestamp */
// TODO also pointing stuff?
fmjd_samp = fmjd + i*pf.hdr.dt/86400.0;
pf_out.sub.offs += pf.sub.offs - 0.5*pf.sub.tsubint + i*pf.hdr.dt;
sampcount++;
/* Calc current pulse number */
psr_phase(&pc[ipc], imjd, fmjd_samp, &psr_freq, &cur_pulse);
/* TODO: deal with scale/offset? */
/* Fold this sample */
fargs.pc = &pc[ipc];
fargs.imjd = imjd;
fargs.fmjd = fmjd_samp;
rv = fold_8bit_power(fargs.pc,
fargs.imjd, fargs.fmjd,
fargs.data + i*bytes_per_sample,
fargs.nsamp, fargs.tsamp, fargs.raw_signed, fargs.fb);
if (rv!=0) {
fprintf(stderr, "Fold error.\n");
exit(1);
}
/* See if integration needs to be written, etc */
if (cur_pulse > last_pulse) {
/* Figure out timestamp */
pf_out.sub.offs /= (double)sampcount;
pf_out.sub.tsubint = 1.0/psr_freq;
fmjd_epoch = fmjd0 + pf_out.sub.offs/86400.0;
/* Transpose, output subint */
normalize_transpose_folds((float *)pf_out.sub.data, &fb);
psrfits_write_subint(&pf_out);
/* If file incremented, clear polyco flags */
if (pf_out.filenum > last_filenum)
for (j=0; j<npc; j++)
pc_written[j]=0;
/* Write this polyco if needed */
if (pc_written[ipc]==0) {
psrfits_write_polycos(&pf_out, pc, npc);
pc_written[ipc] = 1;
}
/* Check for write errors */
if (pf_out.status) {
fprintf(stderr, "Error writing subint.\n");
fits_report_error(stderr, pf_out.status);
exit(1);
}
/* Clear counters, avgs */
clear_foldbuf(&fb);
pf_out.sub.offs = 0.0;
sampcount=0;
last_pulse = cur_pulse;
last_filenum = pf_out.filenum;
}
}
/* Progress report */
if (!quiet) {
printf("\rFile %d %5.1f%%", pf.filenum,
100.0 * (float)(pf.rownum-1)/(float)pf.rows_per_file);
fflush(stdout);
}
}
psrfits_close(&pf_out);
psrfits_close(&pf);
if (rv) { fits_report_error(stderr, rv); }
exit(0);
}
int main(int argc, char *argv[])
{
int ii, ipol, nc = 0, ncnp = 0, gpubps = 0, status = 0, statsum = 0;
int fnum_start = 1, fnum_end = 0;
int numprocs, numbands, myid, baddata = 0, droppedrow = 0;
int *counts, *offsets;
unsigned char *tmpbuf = NULL;
struct psrfits pf;
struct {
double value;
int index;
} offs_in, offs_out;
char hostname[256];
char vegas_base_dir[256] = "\0";
char output_base[256] = "\0";
int starthpc = 0;
int reverse = 0;
MPI_Status mpistat;
/* Cmd line */
static struct option long_opts[] = {
{"output", 1, NULL, 'o'},
{"initial", 1, NULL, 'i'},
{"final", 1, NULL, 'f'},
{"vegas", 1, NULL, 'V'},
{"starthpc",1, NULL, 's'},
{"reverse" ,0, NULL, 'r'},
{0,0,0,0}
};
int opt, opti;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
numbands = numprocs - 1;
// Process the command line
while ((opt=getopt_long(argc,argv,"o:i:f:V:s:r",long_opts,&opti))!=-1) {
switch (opt) {
case 'o':
strncpy(output_base, optarg, 255);
output_base[255]='\0';
break;
case 'i':
fnum_start = atoi(optarg);
break;
case 'f':
fnum_end = atoi(optarg);
break;
case 'V':
strcpy(vegas_base_dir, optarg);
break;
case 's':
starthpc = atoi(optarg);
break;
case 'r':
reverse = 1;
break;
default:
if (myid==0) usage();
MPI_Finalize();
exit(0);
break;
}
}
if (optind==argc) {
if (myid==0) usage();
MPI_Finalize();
exit(1);
}
if (myid == 0) { // Master proc only
printf("\n\n");
printf(" MPI Search-mode PSRFITs Combiner\n");
printf(" by Scott M. Ransom\n\n");
}
// Determine the hostnames of the processes
{
if (gethostname(hostname, 255) < 0)
strcpy(hostname, "unknown");
MPI_Barrier(MPI_COMM_WORLD);
if (myid == 0) printf("\n");
fflush(NULL);
for (ii = 0 ; ii < numprocs ; ii++) {
MPI_Barrier(MPI_COMM_WORLD);
if (myid == ii)
printf("Process %3d is on machine %s\n", myid, hostname);
fflush(NULL);
MPI_Barrier(MPI_COMM_WORLD);
}
MPI_Barrier(MPI_COMM_WORLD);
fflush(NULL);
}
// Basefilenames for the GPU nodes
if (myid > 0) {
// Default to GUPPI mode
if (vegas_base_dir[0]=='\0')
sprintf(pf.basefilename, "/data/gpu/partial/%s/%s",
hostname, argv[optind]);
// VEGAS mode
else {
int hpcidx;
if (reverse)
hpcidx = starthpc - myid + 1;
else
hpcidx = myid + starthpc - 1;
sprintf(pf.basefilename, "%s/vegas-hpc%d-bdata1/%s",
vegas_base_dir, hpcidx, argv[optind]);
printf("**********: hostname = %s, myid = %d, datamnt = %d, basename=%s\n",
hostname, myid, hpcidx, pf.basefilename);
}
}
// Initialize some key parts of the PSRFITS structure
pf.tot_rows = pf.N = pf.T = pf.status = 0;
pf.filenum = fnum_start;
pf.filename[0] = '\0';
pf.filenames = NULL;
pf.numfiles = 0;
if (myid == 1) {
FILE *psrfitsfile;
char hdr[HDRLEN], filenm[200];
// Read the header info
sprintf(filenm, "%s_0001.fits", pf.basefilename);
psrfitsfile = fopen(filenm, "r");
fread(&hdr, 1, HDRLEN, psrfitsfile);
fclose(psrfitsfile);
// Send the header to the master proc
MPI_Send(hdr, HDRLEN, MPI_CHAR, 0, 0, MPI_COMM_WORLD);
} else if (myid == 0) {
FILE *psrfitsfile;
char hdr[HDRLEN], tmpfilenm[80];
// Receive the header info from proc 1
MPI_Recv(hdr, HDRLEN, MPI_CHAR, 1, 0, MPI_COMM_WORLD, &mpistat);
// Now write that header to a temp file
strcpy(tmpfilenm, "mpi_merge_psrfits.XXXXXX");
mkstemp(tmpfilenm);
psrfitsfile = fopen(tmpfilenm, "w");
fwrite(&hdr, 1, HDRLEN, psrfitsfile);
fclose(psrfitsfile);
pf.filenames = (char **)malloc(sizeof(char *));
pf.filenames[0] = tmpfilenm;
pf.basefilename[0]='\0';
pf.filenum = 0;
pf.numfiles = 1;
// And read the key information into a PSRFITS struct
status = psrfits_open(&pf);
status = psrfits_close(&pf);
free(pf.filenames);
remove(tmpfilenm);
// Now create the output PSTFITS file
if (output_base[0]=='\0') {
/* Set up default output filename */
strcpy(output_base, argv[optind]);
}
strcpy(pf.basefilename, output_base);
pf.multifile = 1;
pf.filenum = 0;
pf.numfiles = 0;
pf.filename[0] = '\0';
pf.filenames = NULL;
nc = pf.hdr.nchan;
ncnp = pf.hdr.nchan * pf.hdr.npol;
gpubps = pf.sub.bytes_per_subint;
pf.hdr.orig_nchan *= numbands;
pf.hdr.nchan *= numbands;
pf.hdr.fctr = pf.hdr.fctr - 0.5 * pf.hdr.BW + numbands/2.0 * pf.hdr.BW;
pf.hdr.BW *= numbands;
pf.sub.bytes_per_subint *= numbands;
long long filelen = 40 * (1L<<30); // In GB
pf.rows_per_file = filelen / pf.sub.bytes_per_subint;
status = psrfits_create(&pf);
// For in-memory transpose of data
tmpbuf = (unsigned char *)malloc(pf.sub.bytes_per_subint);
}
// Open the input PSRFITs files for real
if (myid > 0) {
status = psrfits_open(&pf);
nc = pf.hdr.nchan;
ncnp = pf.hdr.nchan * pf.hdr.npol;
gpubps = pf.sub.bytes_per_subint;
}
// Alloc data buffers for the PSRFITS files
pf.sub.dat_freqs = (float *)malloc(sizeof(float) * pf.hdr.nchan);
pf.sub.dat_weights = (float *)malloc(sizeof(float) * pf.hdr.nchan);
pf.sub.dat_offsets = (float *)malloc(sizeof(float) *
pf.hdr.nchan * pf.hdr.npol);
pf.sub.dat_scales = (float *)malloc(sizeof(float) *
pf.hdr.nchan * pf.hdr.npol);
pf.sub.data = (unsigned char *)malloc(pf.sub.bytes_per_subint);
pf.sub.rawdata = pf.sub.data;
// Counts and offsets for MPI_Gatherv
counts = (int *)malloc(sizeof(int) * numprocs);
offsets = (int *)malloc(sizeof(int) * numprocs);
counts[0] = offsets[0] = 0; // master sends nothing
// Now loop over the rows (i.e. subints)...
do {
MPI_Barrier(MPI_COMM_WORLD);
// Read the current subint from each of the "slave" nodes
if ((myid > 0) && (!baddata)) {
status = psrfits_read_subint(&pf);
if (status) {
pf.sub.offs = FLAG; // High value so it won't be min
if (pf.rownum > pf.rows_per_file) {
// Shouldn't be here unless opening of new file failed...
printf("Proc %d: Can't open next file. Setting status=114.\n", myid);
status = 114;
}
}
} else { // Root process
pf.sub.offs = FLAG; // High value so it won't be min
}
// Find the minimum value of OFFS_SUB to see if we dropped a row
offs_in.value = pf.sub.offs;
offs_in.index = myid;
MPI_Allreduce(&offs_in, &offs_out, 1, MPI_DOUBLE_INT,
MPI_MINLOC, MPI_COMM_WORLD);
// If all procs are returning the FLAG value, break.
if (offs_out.value==FLAG) break;
// Identify dropped rows
if ((myid > 0) && (!status) && (!baddata) &&
(pf.sub.offs > (offs_out.value + 0.1 * pf.sub.tsubint))) {
printf("Proc %d, row %d: Dropped a row. Filling with zeros.\n",
myid, pf.rownum);
droppedrow = 1;
}
if (myid > 0) {
// Ignore errors for moving past EOF (107), read errors (108)
// and missing files (114)
if (droppedrow ||
status==108 ||
((myid > 0) && (status==114 || status==107) && (!baddata))) {
if (status) printf("Proc %d, row %d: Ignoring CFITSIO error %d. Filling with zeros.\n", myid, pf.rownum, status);
// Set the data and the weights to all zeros
for (ii = 0 ; ii < pf.hdr.nchan ; ii++)
pf.sub.dat_weights[ii] = 0.0;
for (ii = 0 ; ii < pf.sub.bytes_per_subint ; ii++)
pf.sub.data[ii] = 0;
// And the scales and offsets to nominal values
for (ii = 0 ; ii < pf.hdr.nchan * pf.hdr.npol ; ii++) {
pf.sub.dat_offsets[ii] = 0.0;
pf.sub.dat_scales[ii] = 1.0;
}
// reset the status to 0 and allow going to next row
if (status==114 || status==107) {
baddata = 1;
}
if (status==108) { // Try reading the next row...
pf.rownum++;
pf.tot_rows++;
pf.N += pf.hdr.nsblk;
pf.T = pf.N * pf.hdr.dt;
}
if (droppedrow) { // We want to read the current row again...
pf.rownum--;
pf.tot_rows--;
pf.N -= pf.hdr.nsblk;
pf.T = pf.N * pf.hdr.dt;
droppedrow = 0; // reset
}
status = 0;
}
}
// If we've passed final file, exit
if (fnum_end && pf.filenum > fnum_end) break;
// Combine statuses of all nodes by summing....
MPI_Allreduce(&status, &statsum, 1,
MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (statsum) break;
if (myid == offs_out.index) { // Send all of the non-band-specific parts to master
MPI_Send(&pf.sub.tsubint, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD);
MPI_Send(&pf.sub.offs, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD);
MPI_Send(&pf.sub.lst, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD);
MPI_Send(&pf.sub.ra, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD);
MPI_Send(&pf.sub.dec, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD);
MPI_Send(&pf.sub.glon, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD);
MPI_Send(&pf.sub.glat, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD);
MPI_Send(&pf.sub.feed_ang, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD);
MPI_Send(&pf.sub.pos_ang, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD);
MPI_Send(&pf.sub.par_ang, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD);
MPI_Send(&pf.sub.tel_az, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD);
MPI_Send(&pf.sub.tel_zen, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD);
} else if (myid == 0) { // Receive all of the non-data parts
MPI_Recv(&pf.sub.tsubint, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat);
MPI_Recv(&pf.sub.offs, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat);
MPI_Recv(&pf.sub.lst, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat);
MPI_Recv(&pf.sub.ra, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat);
MPI_Recv(&pf.sub.dec, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat);
MPI_Recv(&pf.sub.glon, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat);
MPI_Recv(&pf.sub.glat, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat);
MPI_Recv(&pf.sub.feed_ang, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat);
MPI_Recv(&pf.sub.pos_ang, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat);
MPI_Recv(&pf.sub.par_ang, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat);
MPI_Recv(&pf.sub.tel_az, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat);
MPI_Recv(&pf.sub.tel_zen, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat);
}
// Now gather the vector quantities...
// Vectors of length nchan
for (ii = 1 ; ii < numprocs ; ii++) {
counts[ii] = nc;
offsets[ii] = (ii - 1) * nc;
}
status = MPI_Gatherv(pf.sub.dat_freqs, nc, MPI_FLOAT,
pf.sub.dat_freqs, counts, offsets, MPI_FLOAT,
0, MPI_COMM_WORLD);
status = MPI_Gatherv(pf.sub.dat_weights, nc, MPI_FLOAT,
pf.sub.dat_weights, counts, offsets, MPI_FLOAT,
0, MPI_COMM_WORLD);
// Vectors of length nchan * npol
for (ipol=0; ipol < pf.hdr.npol; ipol++) {
for (ii = 1 ; ii < numprocs ; ii++) {
counts[ii] = nc;
offsets[ii] = ipol*nc*numbands + (ii - 1) * nc;
}
status = MPI_Gatherv(pf.sub.dat_offsets+(ipol*nc), nc, MPI_FLOAT,
pf.sub.dat_offsets, counts, offsets,
MPI_FLOAT, 0, MPI_COMM_WORLD);
status = MPI_Gatherv(pf.sub.dat_scales+(ipol*nc), nc, MPI_FLOAT,
pf.sub.dat_scales, counts, offsets,
MPI_FLOAT, 0, MPI_COMM_WORLD);
}
// Vectors of length pf.sub.bytes_per_subint for the raw data
for (ii = 1 ; ii < numprocs ; ii++) {
counts[ii] = gpubps;
offsets[ii] = (ii - 1) * gpubps;
}
status = MPI_Gatherv(pf.sub.data, gpubps, MPI_UNSIGNED_CHAR,
tmpbuf, counts, offsets, MPI_UNSIGNED_CHAR,
0, MPI_COMM_WORLD);
// Reorder and write the new row to the output file
if (myid == 0) {
reorder_data(pf.sub.data, tmpbuf, numbands,
pf.hdr.nsblk, pf.hdr.npol, nc);
status = psrfits_write_subint(&pf);
}
} while (statsum == 0);
// Free the arrays
free(pf.sub.dat_freqs);
free(pf.sub.dat_weights);
free(pf.sub.dat_offsets);
free(pf.sub.dat_scales);
free(pf.sub.data);
if (myid == 0) free(tmpbuf);
free(counts);
free(offsets);
// Close the files and finalize things
status = psrfits_close(&pf);
MPI_Finalize();
exit(0);
}
int main(int argc, char *argv[])
{
int numfiles, ii, numrows, rownum, ichan, itsamp, datidx;
int spec_per_row, status, maxrows;
unsigned long int maxfilesize;
float offset, scale, datum, packdatum, maxval, fulltsubint;
float *datachunk;
FILE **infiles;
struct psrfits pfin, pfout;
Cmdline *cmd;
fitsfile *infits, *outfits;
char outfilename[128], templatename[128], tform[8];
char *pc1, *pc2;
int first = 1, dummy = 0, nclipped;
short int *inrowdata;
unsigned char *outrowdata;
if (argc == 1) {
Program = argv[0];
usage();
exit(1);
}
// Parse the command line using the excellent program Clig
cmd = parseCmdline(argc, argv);
numfiles = cmd->argc;
infiles = (FILE **) malloc(numfiles * sizeof(FILE *));
//Set the max. total size (in bytes) of all rows in an output file,
//leaving some room for PSRFITS header
maxfilesize = (unsigned long int)(cmd->numgb * GB);
maxfilesize = maxfilesize - 1000*KB;
//fprintf(stderr,"cmd->numgb: %f maxfilesize: %ld\n",cmd->numgb,maxfilesize);
#ifdef DEBUG
showOptionValues();
#endif
printf("\n PSRFITS 16-bit to 4-bit Conversion Code\n");
printf(" by J. Deneva, S. Ransom, & S. Chatterjee\n\n");
// Open the input files
status = 0; //fits_close segfaults if this is not initialized
printf("Reading input data from:\n");
for (ii = 0; ii < numfiles; ii++) {
printf(" '%s'\n", cmd->argv[ii]);
//Get the file basename and number from command-line argument
//(code taken from psrfits2fil)
pc2 = strrchr(cmd->argv[ii], '.'); // at .fits
*pc2 = 0; // terminate string
pc1 = pc2 - 1;
while ((pc1 >= cmd->argv[ii]) && isdigit(*pc1))
pc1--;
if (pc1 <= cmd->argv[ii]) { // need at least 1 char before filenum
puts("Illegal input filename. must have chars before the filenumber");
exit(1);
}
pc1++; // we were sitting on "." move to first digit
pfin.filenum = atoi(pc1);
pfin.fnamedigits = pc2 - pc1; // how many digits in filenumbering scheme.
*pc1 = 0; // null terminate the basefilename
strcpy(pfin.basefilename, cmd->argv[ii]);
pfin.initialized = 0; // set to 1 in psrfits_open()
pfin.status = 0;
//(end of code taken from psrfits2fil)
//Open the existing psrfits file
if (psrfits_open(&pfin, READONLY) != 0) {
fprintf(stderr, "error opening file\n");
fits_report_error(stderr, pfin.status);
exit(1);
}
// Create the subint arrays
if (first) {
pfin.sub.dat_freqs = (float *) malloc(sizeof(float) * pfin.hdr.nchan);
pfin.sub.dat_weights = (float *) malloc(sizeof(float) * pfin.hdr.nchan);
pfin.sub.dat_offsets =
(float *) malloc(sizeof(float) * pfin.hdr.nchan * pfin.hdr.npol);
pfin.sub.dat_scales =
(float *) malloc(sizeof(float) * pfin.hdr.nchan * pfin.hdr.npol);
//first is set to 0 after data buffer allocation further below
}
infits = pfin.fptr;
spec_per_row = pfin.hdr.nsblk;
fits_read_key(infits, TINT, "NAXIS2", &dummy, NULL, &status);
pfin.tot_rows = dummy;
numrows = dummy;
//If dealing with 1st input file, create output template
if (ii == 0) {
sprintf(templatename, "%s.template.fits",cmd->outfile);
fits_create_file(&outfits, templatename, &status);
//fprintf(stderr,"pfin.basefilename: %s\n", pfin.basefilename);
//fprintf(stderr,"status: %d\n", status);
//Instead of copying HDUs one by one, can move to the SUBINT
//HDU, and copy all the HDUs preceding it
fits_movnam_hdu(infits, BINARY_TBL, "SUBINT", 0, &status);
fits_copy_file(infits, outfits, 1, 0, 0, &status);
//Copy the SUBINT table header
fits_copy_header(infits, outfits, &status);
fits_flush_buffer(outfits, 0, &status);
//Set NAXIS2 in the output SUBINT table to 0 b/c we haven't
//written any rows yet
dummy = 0;
fits_update_key(outfits, TINT, "NAXIS2", &dummy, NULL, &status);
//Edit the NBITS key
if (DEBUG) {
dummy = 8;
fits_update_key(outfits, TINT, "NBITS", &dummy, NULL, &status);
} else {
fits_update_key(outfits, TINT, "NBITS", &(cmd->numbits), NULL,
&status);
}
//Edit the TFORM17 column: # of data bytes per row
//fits_get_colnum(outfits,1,"DATA",&dummy,&status);
if (DEBUG)
sprintf(tform, "%dB",
pfin.hdr.nsblk * pfin.hdr.nchan * pfin.hdr.npol);
else
sprintf(tform, "%dB", pfin.hdr.nsblk * pfin.hdr.nchan *
pfin.hdr.npol * cmd->numbits / 8);
fits_update_key(outfits, TSTRING, "TTYPE17", "DATA", NULL, &status);
fits_update_key(outfits, TSTRING, "TFORM17", tform, NULL, &status);
//Edit NAXIS1: row width in bytes
fits_read_key(outfits, TINT, "NAXIS1", &dummy, NULL, &status);
if (DEBUG) {
dummy = dummy - pfin.hdr.nsblk * pfin.hdr.nchan *
pfin.hdr.npol * (pfin.hdr.nbits - 8) / 8;
} else {
dummy = dummy - pfin.hdr.nsblk * pfin.hdr.nchan *
pfin.hdr.npol * (pfin.hdr.nbits - cmd->numbits) / 8;
}
fits_update_key(outfits, TINT, "NAXIS1", &dummy, NULL, &status);
//Set the max # of rows per file, based on the requested
//output file size
maxrows = maxfilesize / dummy;
//fprintf(stderr,"maxrows: %d\n",maxrows);
fits_close_file(outfits, &status);
rownum = 0;
}
while (psrfits_read_subint(&pfin, first) == 0) {
fprintf(stderr, "Working on row %d\n", ++rownum);
//If this is the first row, store the length of a full subint
if (ii == 0 && rownum == 1)
fulltsubint = pfin.sub.tsubint;
//If this is the last row and it's partial, drop it.
//(It's pfin.rownum-1 below because the rownum member of the psrfits struct seems to be intended to indicate at the *start* of what row we are, i.e. a row that has not yet been read. In contrast, pfout.rownum indicates how many rows have been written, i.e. at the *end* of what row we are in the output.)
if (pfin.rownum-1 == numrows && fabs(pfin.sub.tsubint - fulltsubint) > pfin.hdr.dt) {
fprintf(stderr,
"Dropping partial row of length %f s (full row is %f s)\n",
pfin.sub.tsubint, fulltsubint);
break;
}
//If we just read in the 1st row, or if we already wrote the last row in the current output file, create a new output file
if ((ii == 0 && rownum == 1) || pfout.rownum == maxrows) {
//Create new output file from the template
pfout.fnamedigits = pfin.fnamedigits;
if(ii == 0)
pfout.filenum = pfin.filenum;
else
pfout.filenum++;
sprintf(outfilename, "%s.%0*d.fits", cmd->outfile, pfout.fnamedigits, pfout.filenum);
fits_create_template(&outfits, outfilename, templatename, &status);
//fprintf(stderr,"After fits_create_template, status: %d\n",status);
fits_close_file(outfits, &status);
//Now reopen the file so that the pfout structure is initialized
pfout.status = 0;
pfout.initialized = 0;
sprintf(pfout.basefilename, "%s.", cmd->outfile);
if (psrfits_open(&pfout, READWRITE) != 0) {
fprintf(stderr, "error opening file\n");
fits_report_error(stderr, pfout.status);
exit(1);
}
outfits = pfout.fptr;
maxval = pow(2, cmd->numbits) - 1;
pfout.rows_per_file = maxrows;
//fprintf(stderr, "maxval: %f\n", maxval);
//fprintf(stderr, "pfout.rows_per_file: %d\n",pfout.rows_per_file);
//These are not initialized in psrfits_open but are needed
//in psrfits_write_subint (not obvious what are the corresponding
//fields in any of the psrfits table headers)
pfout.hdr.ds_freq_fact = 1;
pfout.hdr.ds_time_fact = 1;
}
//Copy the subint struct from pfin to pfout, but correct
//elements that are not the same
pfout.sub = pfin.sub; //this copies array pointers too
pfout.sub.bytes_per_subint =
pfin.sub.bytes_per_subint * pfout.hdr.nbits / pfin.hdr.nbits;
pfout.sub.dataBytesAlloced = pfout.sub.bytes_per_subint;
pfout.sub.FITS_typecode = TBYTE;
if (first) {
//Allocate scaling buffer and output buffer
datachunk = gen_fvect(spec_per_row);
outrowdata = gen_bvect(pfout.sub.bytes_per_subint);
first = 0;
}
pfout.sub.data = outrowdata;
inrowdata = (short int *) pfin.sub.data;
nclipped = 0;
// Loop over all the channels:
for (ichan = 0; ichan < pfout.hdr.nchan * pfout.hdr.npol; ichan++) {
// Populate datachunk[] by picking out all time samples for ichan
for (itsamp = 0; itsamp < spec_per_row; itsamp++)
datachunk[itsamp] = (float) (inrowdata[ichan + itsamp *
pfout.hdr.nchan *
pfout.hdr.npol]);
// Compute the statistics here, and put the offsets and scales in
// pf.sub.dat_offsets[] and pf.sub.dat_scales[]
if (rescale(datachunk, spec_per_row, cmd->numbits, &offset, &scale)
!= 0) {
printf("Rescale routine failed!\n");
return (-1);
}
pfout.sub.dat_offsets[ichan] = offset;
pfout.sub.dat_scales[ichan] = scale;
// Since we have the offset and scale ready, rescale the data:
for (itsamp = 0; itsamp < spec_per_row; itsamp++) {
datum = (scale == 0.0) ? 0.0 :
roundf((datachunk[itsamp] - offset) / scale);
if (datum < 0.0) {
datum = 0;
nclipped++;
} else if (datum > maxval) {
datum = maxval;
nclipped++;
}
inrowdata[ichan + itsamp * pfout.hdr.nchan * pfout.hdr.npol] =
(short int) datum;
}
// Now inrowdata[ichan] contains rescaled ints.
}
// Then do the conversion and store the
// results in pf.sub.data[]
if (cmd->numbits == 8 || DEBUG) {
for (itsamp = 0; itsamp < spec_per_row; itsamp++) {
datidx = itsamp * pfout.hdr.nchan * pfout.hdr.npol;
for (ichan = 0; ichan < pfout.hdr.nchan * pfout.hdr.npol;
ichan++, datidx++) {
pfout.sub.data[datidx] = (unsigned char) inrowdata[datidx];
}
}
} else if (cmd->numbits == 4) {
for (itsamp = 0; itsamp < spec_per_row; itsamp++) {
datidx = itsamp * pfout.hdr.nchan * pfout.hdr.npol;
for (ichan = 0; ichan < pfout.hdr.nchan * pfout.hdr.npol;
ichan += 2, datidx += 2) {
packdatum = inrowdata[datidx] * 16 + inrowdata[datidx + 1];
pfout.sub.data[datidx / 2] = (unsigned char) packdatum;
}
}
} else {
fprintf(stderr, "Only 4 or 8-bit output formats supported.\n");
fprintf(stderr, "Bits per sample requested: %d\n", cmd->numbits);
exit(1);
}
//pfout.sub.offs = (pfout.tot_rows+0.5) * pfout.sub.tsubint;
fprintf(stderr, "nclipped: %d fraction clipped: %f\n", nclipped,
(float) nclipped / (pfout.hdr.nchan * pfout.hdr.npol *
pfout.hdr.nsblk));
// Now write the row.
status = psrfits_write_subint(&pfout);
if (status) {
printf("\nError (%d) writing PSRFITS...\n\n", status);
break;
}
//If current output file has reached the max # of rows, close it
if (pfout.rownum == maxrows)
fits_close_file(outfits, &status);
}
//Close the files
fits_close_file(infits, &status);
}
fits_close_file(outfits, &status);
// Free the structure arrays too...
free(datachunk);
free(infiles);
free(pfin.sub.dat_freqs);
free(pfin.sub.dat_weights);
free(pfin.sub.dat_offsets);
free(pfin.sub.dat_scales);
free(pfin.sub.data);
free(pfout.sub.data);
free(pfin.sub.stat);
return 0;
}
int main(int argc, char *argv[]) {
/* Cmd line */
static struct option long_opts[] = {
{"output", 1, NULL, 'o'},
{"nbin", 1, NULL, 'b'},
{"tsub", 1, NULL, 't'},
{"nthread", 1, NULL, 'j'},
{"initial", 1, NULL, 'i'},
{"final", 1, NULL, 'f'},
{"src", 1, NULL, 's'},
{"polyco", 1, NULL, 'p'},
{"parfile", 1, NULL, 'P'},
{"foldfreq",1, NULL, 'F'},
{"cal", 0, NULL, 'C'},
{"unsigned",0, NULL, 'u'},
{"nunsigned",1, NULL, 'U'},
{"split", 1, NULL, 'S'},
{"apply", 0, NULL, 'A'},
{"quiet", 0, NULL, 'q'},
{"help", 0, NULL, 'h'},
{0,0,0,0}
};
int opt, opti;
int nbin=256, nthread=4, fnum_start=1, fnum_end=0;
int quiet=0, raw_signed=1, use_polycos=1, cal=0, apply_scale=0;
double split_size_gb = 1.0;
double tfold = 60.0;
double fold_frequency=0.0;
char output_base[256] = "";
char polyco_file[256] = "";
char par_file[256] = "";
char source[24]; source[0]='\0';
while ((opt=getopt_long(argc,argv,"o:b:t:j:i:f:s:p:P:F:CuU:S:Aqh",long_opts,&opti))!=-1) {
switch (opt) {
case 'o':
strncpy(output_base, optarg, 255);
output_base[255]='\0';
break;
case 'b':
nbin = atoi(optarg);
break;
case 't':
tfold = atof(optarg);
break;
case 'j':
nthread = atoi(optarg);
break;
case 'i':
fnum_start = atoi(optarg);
break;
case 'f':
fnum_end = atoi(optarg);
break;
case 's':
strncpy(source, optarg, 24);
source[23]='\0';
break;
case 'p':
strncpy(polyco_file, optarg, 255);
polyco_file[255]='\0';
use_polycos = 1;
break;
case 'P':
strncpy(par_file, optarg, 255);
par_file[255] = '\0';
break;
case 'F':
fold_frequency = atof(optarg);
use_polycos = 0;
break;
case 'C':
cal = 1;
use_polycos = 0;
break;
case 'u':
raw_signed=0;
break;
case 'U':
raw_signed = 4 - atoi(optarg);
break;
case 'S':
split_size_gb = atof(optarg);
break;
case 'A':
apply_scale = 1;
break;
case 'q':
quiet=1;
break;
case 'h':
default:
usage();
exit(0);
break;
}
}
if (optind==argc) {
usage();
exit(1);
}
/* If no polyco/par file given, default to polyco.dat */
if (use_polycos && (par_file[0]=='\0' && polyco_file[0]=='\0'))
sprintf(polyco_file, "polyco.dat");
/* Open first file */
struct psrfits pf;
sprintf(pf.basefilename, argv[optind]);
pf.filenum = fnum_start;
pf.tot_rows = pf.N = pf.T = pf.status = 0;
pf.hdr.chan_dm = 0.0; // What if folding data that has been partially de-dispersed?
pf.filename[0]='\0';
int rv = psrfits_open(&pf);
if (rv) { fits_report_error(stderr, rv); exit(1); }
/* Check any constraints */
if (pf.hdr.nbits!=8) {
fprintf(stderr, "Only implemented for 8-bit data (read nbits=%d).\n",
pf.hdr.nbits);
exit(1);
}
/* Check for calfreq */
if (cal) {
if (pf.hdr.cal_freq==0.0) {
if (fold_frequency==0.0) {
fprintf(stderr, "Error: Cal mode selected, but CAL_FREQ=0. "
"Set cal frequency with -F\n");
exit(1);
} else {
pf.hdr.cal_freq = fold_frequency;
}
} else {
fold_frequency = pf.hdr.cal_freq;
}
}
/* Set up output file */
struct psrfits pf_out;
memcpy(&pf_out, &pf, sizeof(struct psrfits));
if (source[0]!='\0') { strncpy(pf_out.hdr.source, source, 24); }
else { strncpy(source, pf.hdr.source, 24); source[23]='\0'; }
if (output_base[0]=='\0') {
/* Set up default output filename */
sprintf(output_base, "%s_%s_%5.5d_%5.5d%s", pf_out.hdr.backend,
pf_out.hdr.source, pf_out.hdr.start_day,
(int)pf_out.hdr.start_sec, cal ? "_cal" : "");
}
sprintf(pf_out.basefilename, output_base);
if (cal) {
sprintf(pf_out.hdr.obs_mode, "CAL");
sprintf(pf_out.hdr.cal_mode, "SYNC");
} else
sprintf(pf_out.hdr.obs_mode, "PSR");
strncpy(pf_out.fold.parfile,par_file,255); pf_out.fold.parfile[255]='\0';
pf_out.fptr = NULL;
pf_out.filenum=0;
pf_out.status=0;
pf_out.quiet=0;
pf_out.hdr.nbin=nbin;
pf_out.sub.FITS_typecode = TFLOAT;
pf_out.sub.bytes_per_subint = sizeof(float) *
pf_out.hdr.nchan * pf_out.hdr.npol * pf_out.hdr.nbin;
if (split_size_gb > 0.0) {
pf_out.multifile = 1;
pf_out.rows_per_file = (int) (split_size_gb * (1024.0*1024.0*1024.0)
/ (double)pf_out.sub.bytes_per_subint);
printf("Writing a maximum of %d subintegrations (~%.1f GB) per output file.\n",
pf_out.rows_per_file, split_size_gb);
} else {
pf_out.multifile = 0;
printf("Writing a single output file.\n");
}
rv = psrfits_create(&pf_out);
if (rv) { fits_report_error(stderr, rv); exit(1); }
/* Alloc data buffers */
pf.sub.dat_freqs = (float *)malloc(sizeof(float) * pf.hdr.nchan);
pf_out.sub.dat_freqs = pf.sub.dat_freqs;
pf.sub.dat_weights = (float *)malloc(sizeof(float) * pf.hdr.nchan);
pf_out.sub.dat_weights = (float *)malloc(sizeof(float) * pf.hdr.nchan);
pf.sub.dat_offsets = (float *)malloc(sizeof(float)
* pf.hdr.nchan * pf.hdr.npol);
pf_out.sub.dat_offsets = (float *)malloc(sizeof(float)
* pf.hdr.nchan * pf.hdr.npol);
pf.sub.dat_scales = (float *)malloc(sizeof(float)
* pf.hdr.nchan * pf.hdr.npol);
pf_out.sub.dat_scales = (float *)malloc(sizeof(float)
* pf.hdr.nchan * pf.hdr.npol);
pf_out.sub.data = (unsigned char *)malloc(pf_out.sub.bytes_per_subint);
/* Output scale/offset */
int i, ipol, ichan;
float offset_uv=0.0;
// Extra cross-term offset for GUPPI
if (strcmp("GUPPI",pf.hdr.backend)==0 && apply_scale==0) {
offset_uv=0.5;
fprintf(stderr, "Found backend=GUPPI, setting offset_uv=%f\n",
offset_uv);
}
// Initialize scale/output and weights.
// These get copied from the input file later during the main loop.
for (ipol=0; ipol<pf.hdr.npol; ipol++) {
for (ichan=0; ichan<pf.hdr.nchan; ichan++) {
float offs = 0.0;
if (ipol>1) offs = offset_uv;
pf_out.sub.dat_scales[ipol*pf.hdr.nchan + ichan] = 1.0;
pf_out.sub.dat_offsets[ipol*pf.hdr.nchan + ichan] = offs;
}
}
for (i=0; i<pf.hdr.nchan; i++) { pf_out.sub.dat_weights[i]=1.0; }
/* Read or make polycos */
int npc=0, ipc=0;
struct polyco *pc = NULL;
if (use_polycos) {
if (polyco_file[0]=='\0') {
/* Generate from par file */
npc = make_polycos(par_file, &pf.hdr, source, &pc);
if (npc<=0) {
fprintf(stderr, "Error generating polycos.\n");
exit(1);
}
printf("Auto-generated %d polycos, src=%s\n", npc, source);
} else {
/* Read from polyco file */
FILE *pcfile = fopen(polyco_file, "r");
if (pcfile==NULL) {
fprintf(stderr, "Couldn't open polyco file.\n");
exit(1);
}
npc = read_all_pc(pcfile, &pc);
if (npc==0) {
fprintf(stderr, "Error parsing polyco file.\n");
exit(1);
}
fclose(pcfile);
}
} else {
// Const fold period mode, generate a fake polyco?
pc = (struct polyco *)malloc(sizeof(struct polyco));
sprintf(pc[0].psr, "CONST");
pc[0].mjd = (int)pf.hdr.MJD_epoch;
pc[0].fmjd = fmod(pf.hdr.MJD_epoch,1.0);
pc[0].rphase = 0.0;
pc[0].f0 = fold_frequency;
pc[0].nsite = 0; // Does this matter?
pc[0].nmin = 24 * 60;
pc[0].nc = 1;
pc[0].rf = pf.hdr.fctr;
pc[0].c[0] = 0.0;
pc[0].used = 0;
npc = 1;
}
int *pc_written = (int *)malloc(sizeof(int) * npc);
for (i=0; i<npc; i++) pc_written[i]=0;
/* Alloc total fold buf */
struct foldbuf fb;
fb.nchan = pf.hdr.nchan;
fb.npol = pf.hdr.npol;
fb.nbin = pf_out.hdr.nbin;
malloc_foldbuf(&fb);
clear_foldbuf(&fb);
/* Set up thread management */
pthread_t *thread_id;
struct fold_args *fargs;
thread_id = (pthread_t *)malloc(sizeof(pthread_t) * nthread);
fargs = (struct fold_args *)malloc(sizeof(struct fold_args) * nthread);
for (i=0; i<nthread; i++) {
thread_id[i] = 0;
fargs[i].data = (char *)malloc(sizeof(char)*pf.sub.bytes_per_subint);
fargs[i].fb = (struct foldbuf *)malloc(sizeof(struct foldbuf));
fargs[i].fb->nbin = pf_out.hdr.nbin;
fargs[i].fb->nchan = pf.hdr.nchan;
fargs[i].fb->npol = pf.hdr.npol;
fargs[i].nsamp = pf.hdr.nsblk;
fargs[i].tsamp = pf.hdr.dt;
fargs[i].raw_signed=raw_signed;
malloc_foldbuf(fargs[i].fb);
clear_foldbuf(fargs[i].fb);
fargs[i].scale = (float *)malloc(sizeof(float)
* pf.hdr.nchan * pf.hdr.npol);
fargs[i].offset = (float *)malloc(sizeof(float)
* pf.hdr.nchan * pf.hdr.npol);
}
/* Main loop */
rv=0;
int imjd;
double fmjd, fmjd0=0, fmjd_next=0, fmjd_epoch;
double offs0=0, offs1=0;
//double phase=0.0, freq=1.0;
int first=1, subcount=0;
int cur_thread = 0;
signal(SIGINT, cc);
while (run) {
/* Read data block */
pf.sub.data = (unsigned char *)fargs[cur_thread].data;
rv = psrfits_read_subint(&pf);
if (rv) {
if (rv==FILE_NOT_OPENED) rv=0; // Don't complain on file not found
run=0;
break;
}
/* If we've passed final file, exit */
if (fnum_end && pf.filenum>fnum_end) { run=0; break; }
/* Get start date, etc */
imjd = (int)pf.hdr.MJD_epoch;
fmjd = (double)(pf.hdr.MJD_epoch - (long double)imjd);
fmjd += (pf.sub.offs-0.5*pf.sub.tsubint)/86400.0;
/* First time stuff */
if (first) {
fmjd0 = fmjd;
fmjd_next = fmjd + tfold/86400.0;
pf_out.sub.offs=0.0;
offs0 = pf.sub.offs - 0.5*pf.sub.tsubint;
offs1 = pf.sub.offs + 0.5*pf.sub.tsubint;
first=0;
for (i=0; i<pf.hdr.nchan; i++) {
pf_out.sub.dat_weights[i]=pf.sub.dat_weights[i];
}
}
/* Keep track of timestamp */
// TODO also pointing stuff.
pf_out.sub.offs += pf.sub.offs;
subcount++;
/* Update output block end time */
offs1 = pf.sub.offs + 0.5*pf.sub.tsubint;
/* Select polyco set */
if (use_polycos) {
ipc = select_pc(pc, npc, source, imjd, fmjd);
//ipc = select_pc(pc, npc, NULL, imjd, fmjd);
if (ipc<0) {
fprintf(stderr, "No matching polycos (src=%s, imjd=%d, fmjd=%f)\n",
source, imjd, fmjd);
break;
}
} else {
ipc = 0;
}
pc[ipc].used = 1; // Mark this polyco set as used for folding
/* Copy scale/offset from input to output if we're not applying it */
if (apply_scale==0) {
for (i=0; i<pf.hdr.nchan*pf.hdr.npol; i++) {
pf_out.sub.dat_scales[i] = pf.sub.dat_scales[i];
pf_out.sub.dat_offsets[i] = pf.sub.dat_offsets[i];
}
}
/* Fold this subint */
fargs[cur_thread].pc = &pc[ipc];
fargs[cur_thread].imjd = imjd;
fargs[cur_thread].fmjd = fmjd;
rv = pthread_create(&thread_id[cur_thread], NULL,
fold_8bit_power_thread, &fargs[cur_thread]);
if (rv) {
fprintf(stderr, "Thread creation error.\n");
exit(1);
}
if (apply_scale) {
for (i=0; i<pf.hdr.nchan*pf.hdr.npol; i++) {
fargs[cur_thread].scale[i] = pf.sub.dat_scales[i];
fargs[cur_thread].offset[i] = pf.sub.dat_offsets[i];
}
}
cur_thread++;
/* Combine thread results if needed */
if (cur_thread==nthread || fmjd>fmjd_next) {
/* Loop over active threads */
for (i=0; i<cur_thread; i++) {
/* Wait for thread to finish */
rv = pthread_join(thread_id[i], NULL);
if (rv) {
fprintf(stderr, "Thread join error.\n");
exit(1);
}
/* Apply scale and offset here */
if (apply_scale)
scale_offset_folds(fargs[i].fb, fargs[i].scale,
fargs[i].offset);
/* Combine its result into total fold */
accumulate_folds(&fb, fargs[i].fb);
/* Reset thread info */
clear_foldbuf(fargs[i].fb);
thread_id[i] = 0;
}
/* Reset active thread count */
cur_thread = 0;
}
/* See if integration needs to be written, etc */
if (fmjd > fmjd_next) {
/* Figure out timestamp */
pf_out.sub.offs /= (double)subcount;
pf_out.sub.tsubint = offs1 - offs0;
fmjd_epoch = fmjd0 + pf_out.sub.offs/86400.0;
/*
// Don't need this stuff if we set EPOCHS=MIDTIME
ipc = select_pc(pc, npc, pf.hdr.source, imjd, fmjd_epoch);
if (ipc<0) {
fprintf(stderr, "Polyco error, exiting.\n");
exit(1);
}
phase = psr_phase(&pc[ipc], imjd, fmjd_epoch, &freq);
phase = fmod(phase, 1.0);
pf_out.sub.offs -= phase/freq; // ref epoch needs 0 phase
*/
/* Transpose, output subint */
normalize_transpose_folds((float *)pf_out.sub.data, &fb);
int last_filenum = pf_out.filenum;
psrfits_write_subint(&pf_out);
/* Check for write errors */
if (pf_out.status) {
fprintf(stderr, "Error writing subint.\n");
fits_report_error(stderr, pf_out.status);
exit(1);
}
/* Check if we started a new file */
if (pf_out.filenum!=last_filenum) {
/* No polycos yet written to this file */
for (i=0; i<npc; i++) pc_written[i]=0;
}
/* Write the current polyco if needed */
if (pc_written[ipc]==0) {
psrfits_write_polycos(&pf_out, &pc[ipc], 1);
if (pf_out.status) {
fprintf(stderr, "Error writing polycos.\n");
fits_report_error(stderr, pf_out.status);
exit(1);
}
pc_written[ipc] = 1;
}
/* Clear counters, avgs */
clear_foldbuf(&fb);
pf_out.sub.offs = 0.0;
offs0 = pf.sub.offs - 0.5*pf.sub.tsubint;
subcount=0;
/* Set next output time */
fmjd_next = fmjd + tfold/86400.0;
}
/* Progress report */
if (!quiet) {
printf("\rFile %d %5.1f%%", pf.filenum,
100.0 * (float)(pf.rownum-1)/(float)pf.rows_per_file);
fflush(stdout);
}
}
/* Join any running threads */
for (i=0; i<cur_thread; i++)
if (thread_id[i]) pthread_join(thread_id[i], NULL);
/* Remove polyco table in cal mode */
if (cal) {
rv = psrfits_remove_polycos(&pf_out);
if (rv) { fits_report_error(stderr, rv); }
}
psrfits_close(&pf_out);
psrfits_close(&pf);
if (rv) { fits_report_error(stderr, rv); }
exit(0);
}