int main(int argc, char *argv[]) { struct psrfits pf; sprintf(pf.basefilename, "/data2/demorest/parspec/parspec_test_B0329+54_0009"); pf.filenum=1; int rv = psrfits_open(&pf); pf.sub.dat_freqs = (float *)malloc(sizeof(float) * pf.hdr.nchan); pf.sub.dat_weights = (float *)malloc(sizeof(float) * pf.hdr.nchan); pf.sub.dat_offsets = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); pf.sub.dat_scales = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); pf.sub.data = (unsigned char *)malloc(pf.sub.bytes_per_subint); while ((rv=psrfits_read_subint(&pf))==0) { printf("Read subint (file %d, row %d/%d)\n", pf.filenum, pf.rownum-1, pf.rows_per_file); } if (rv) { fits_report_error(stderr, rv); } exit(0); }
int get_current_row(struct psrfits *pfi, struct subband_info *si) { static int firsttime = 1, num_pad_blocks = 0; static double last_offs, row_duration; double diff_offs, dnum_blocks; if (firsttime) { row_duration = pfi->sub.tsubint; last_offs = pfi->sub.offs-row_duration; firsttime = 0; } print_percent_complete(pfi->rownum, pfi->rows_per_file, pfi->rownum==1 ? 1 : 0); #if 0 printf("row %d\n", pfi->rownum); #endif if (num_pad_blocks==0) { // Try to read the PSRFITS file // Read the current row of data psrfits_read_subint(pfi); diff_offs = pfi->sub.offs - last_offs; if (si->userwgts) // Always overwrite if using user weights memcpy(pfi->sub.dat_weights, si->userwgts, pfi->hdr.nchan * sizeof(float)); if (!TEST_CLOSE(diff_offs, row_duration) || pfi->status) { if (pfi->status) { // End of the files num_pad_blocks = 1; } else { // Missing row(s) dnum_blocks = diff_offs/row_duration - 1.0; num_pad_blocks = (int)(dnum_blocks + 1e-7); pfi->rownum--; // Will re-read when no more padding pfi->tot_rows--; // Only count "real" rows towards tot_rows #if 1 printf("At row %d, found %d dropped rows.\n", pfi->rownum, num_pad_blocks); printf("Adding a missing row (#%d) of padding to the subbands.\n", pfi->tot_rows); #endif pfi->N -= pfi->hdr.nsblk; // Will be re-added below for padding } // Now fill the main part of si->fbuffer with the chan_avgs so that // it acts like a correctly read block (or row) fill_chans_with_avgs(si->buflen, si->bufwid, pfi->sub.fdata, si->chan_avgs); } else { // Return the row from the file // Compute the float representations of the data scale_and_offset_data(pfi, si->numunsigned); // Determine channel statistics get_chan_stats(pfi, si); last_offs = pfi->sub.offs; return 0; } } // Return the same padding as before last_offs += row_duration; pfi->N += pfi->hdr.nsblk; pfi->T = pfi->N * pfi->hdr.dt; num_pad_blocks--; return num_pad_blocks; }
void init_subbanding(struct psrfits *pfi, struct psrfits *pfo, struct subband_info *si, Cmdline *cmd) { int ii, jj, kk, cindex; double lofreq, dtmp; // If -nsub is not set, do no subbanding if (!cmd->nsubP) cmd->nsub = pfi->hdr.nchan; // Don't change the number of output bits unless we explicitly ask to if (!cmd->outbitsP) cmd->outbits = pfi->hdr.nbits; si->nsub = cmd->nsub; si->nchan = pfi->hdr.nchan; si->npol = pfi->hdr.npol; si->numunsigned = si->npol; if (si->npol==4) { if (strncmp(pfi->hdr.poln_order, "AABBCRCI", 8)==0) si->numunsigned = 2; if (strncmp(pfi->hdr.poln_order, "IQUV", 4)==0) si->numunsigned = 1; } si->chan_per_sub = si->nchan / si->nsub; si->bufwid = si->nchan * si->npol; // Freq * polns si->buflen = pfi->hdr.nsblk; // Number of spectra in each row // Check the downsampling factor in time if (si->buflen % cmd->dstime) { fprintf(stderr, "Error!: %d spectra per row is not evenly divisible by -dstime of %d!\n", si->buflen, cmd->dstime); exit(1); } // Check the downsampling factor in frequency if (si->nchan % si->nsub) { fprintf(stderr, "Error! %d channels is not evenly divisible by %d subbands!\n", si->nchan, si->nsub); exit(1); } si->dm = cmd->dm; si->sub_df = pfi->hdr.df * si->chan_per_sub; si->sub_freqs = (float *)malloc(sizeof(float) * si->nsub); si->chan_delays = (double *)malloc(sizeof(double) * si->nchan); si->sub_delays = (double *)malloc(sizeof(double) * si->nsub); si->idelays = (int *)malloc(sizeof(int) * si->nchan); si->weights = (float *)malloc(sizeof(float) * si->nsub); si->offsets = (float *)malloc(sizeof(float) * si->nsub * si->npol); si->scales = (float *)malloc(sizeof(float) * si->nsub * si->npol); si->chan_avgs = (float *)malloc(sizeof(float) * si->bufwid); si->chan_stds = (float *)malloc(sizeof(float) * si->bufwid); /* Alloc data buffers for the input PSRFITS file */ pfi->sub.dat_freqs = (float *)malloc(sizeof(float) * pfi->hdr.nchan); pfi->sub.dat_weights = (float *)malloc(sizeof(float) * pfi->hdr.nchan); pfi->sub.dat_offsets = (float *)malloc(sizeof(float) * pfi->hdr.nchan * pfi->hdr.npol); pfi->sub.dat_scales = (float *)malloc(sizeof(float) * pfi->hdr.nchan * pfi->hdr.npol); pfi->sub.rawdata = (unsigned char *)malloc(pfi->sub.bytes_per_subint); if (pfi->hdr.nbits!=8) { pfi->sub.data = (unsigned char *)malloc(pfi->sub.bytes_per_subint * (8 / pfi->hdr.nbits)); } else { pfi->sub.data = pfi->sub.rawdata; } // Read the first row of data psrfits_read_subint(pfi); if (si->userwgts) // Always overwrite if using user weights memcpy(pfi->sub.dat_weights, si->userwgts, pfi->hdr.nchan * sizeof(float)); // Reset the read counters since we'll re-read pfi->rownum--; pfi->tot_rows--; pfi->N -= pfi->hdr.nsblk; // Compute the subband properties, DM delays and offsets lofreq = pfi->sub.dat_freqs[0] - pfi->hdr.df * 0.5; for (ii = 0, cindex = 0 ; ii < si->nsub ; ii++) { dtmp = lofreq + ((double)ii + 0.5) * si->sub_df; si->sub_freqs[ii] = dtmp; si->sub_delays[ii] = delay_from_dm(si->dm, dtmp); // Determine the dispersion delays and convert them // to offsets in units of sample times for (jj = 0 ; jj < si->chan_per_sub ; jj++, cindex++) { si->chan_delays[cindex] = delay_from_dm(si->dm, pfi->sub.dat_freqs[cindex]); si->chan_delays[cindex] -= si->sub_delays[ii]; si->idelays[cindex] = (int)rint(si->chan_delays[cindex] / pfi->hdr.dt); } } // Now determine the earliest and latest delays si->max_early = si->max_late = 0; for (ii = 0 ; ii < si->nchan ; ii++) { if (si->idelays[ii] < si->max_early) si->max_early = si->idelays[ii]; if (si->idelays[ii] > si->max_late) si->max_late = si->idelays[ii]; } si->max_overlap = abs(si->max_early) + si->max_late; // This buffer will hold the float-converted input data, plus the bits // of data from the previous and next blocks si->fbuffer = (float *)calloc((si->buflen + 2 * si->max_overlap) * si->bufwid, sizeof(float)); // The input data will be stored directly in the buffer space // So the following is really just an offset into the bigger buffer pfi->sub.fdata = si->fbuffer + si->max_overlap * si->bufwid; // Now start setting values for the output arrays *pfo = *pfi; // We are changing the number of bits in the data if (pfi->hdr.nbits != cmd->outbits) pfo->hdr.nbits = cmd->outbits; // Determine the length of the outputfiles to use if (cmd->filetimeP) { pfo->rows_per_file = 10 * \ (int) rint(0.1 * (cmd->filetime / pfi->sub.tsubint)); } else if (cmd->filelenP) { long long filelen; int bytes_per_subint; filelen = cmd->filelen * (1L<<30); // In GB bytes_per_subint = (pfo->hdr.nbits * pfo->hdr.nchan * pfo->hdr.npol * pfo->hdr.nsblk) / \ (8 * si->chan_per_sub * cmd->dstime * (cmd->onlyIP ? 4 : 1)); pfo->rows_per_file = filelen / bytes_per_subint; pfo->sub.bytes_per_subint = bytes_per_subint; } else { // By default, keep the filesize roughly constant pfo->rows_per_file = pfi->rows_per_file * si->chan_per_sub * cmd->dstime * (cmd->onlyIP ? 4 : 1) * pfi->hdr.nbits / pfo->hdr.nbits; } pfo->filenum = 0; // This causes the output files to be created pfo->filename[0] = '\0'; pfo->rownum = 1; pfo->tot_rows = 0; pfo->N = 0; // Set the "orig" values to those of the input file pfo->hdr.orig_nchan = pfi->hdr.nchan; pfo->hdr.orig_df = pfi->hdr.df; { char *inpath, *infile; split_path_file(pfi->basefilename, &inpath, &infile); sprintf(pfo->basefilename, "%s_subs", infile); free(inpath); free(infile); } // Reset different params pfo->sub.dat_freqs = si->sub_freqs; pfo->sub.dat_weights = si->weights; pfo->sub.dat_offsets = si->offsets; pfo->sub.dat_scales = si->scales; pfo->hdr.ds_freq_fact = si->chan_per_sub; pfo->hdr.ds_time_fact = cmd->dstime; pfo->hdr.onlyI = cmd->onlyIP; pfo->hdr.chan_dm = si->dm; pfo->sub.rawdata = (unsigned char *)malloc(si->nsub * si->npol * si->buflen); if (pfo->hdr.nbits!=8) { pfo->sub.data = (unsigned char *)malloc(si->nsub * si->npol * si->buflen * (8 / pfo->hdr.nbits)); } else { pfo->sub.data = pfo->sub.rawdata; } si->outfbuffer = (float *)calloc(si->nsub * si->npol * si->buflen, sizeof(float)); pfo->sub.fdata = si->outfbuffer; // Now re-read the first row (i.e. for "real" this time) get_current_row(pfi, si); // Set the new weights properly new_weights(pfi, pfo); // Now fill the first part of si->fbuffer with the chan_avgs so that // it acts like a previously read block (or row) fill_chans_with_avgs(si->max_overlap, si->bufwid, si->fbuffer, si->chan_avgs); }
int main(int argc, char *argv[]) { Cmdline *cmd; struct psrfits pfupper, pflower, pfo; fitsfile *infits, *outfits; char *pc1, *pc2; char outfilename[200]; //Name of outfile if not specified on command line int stat = 0, padding = 0, userN = 0, status; // Call usage() if we have no command line arguments if (argc == 1) { Program = argv[0]; usage(); exit(0); } // Parse the command line using the excellent program Clig cmd = parseCmdline(argc, argv); pfupper.tot_rows = pfupper.N = pfupper.T = pfupper.status = 0; //Initialize upper band pflower.tot_rows = pflower.N = pflower.T = pflower.status = 0; //Initialize lower band pfupper.filenum = pflower.filenum = 1; pfo.tot_rows = pfo.N = pfo.T = pfo.status = pfo.multifile = 0; //Initialize output sprintf(pfupper.filename, cmd->argv[0]); //Copy filename specified on command line to sprintf(pflower.filename, cmd->argv[0]); //upper and lower bands, will correct filenames shortly if ((pc2 = strstr(pfupper.filename, "s1")) != NULL) //Upper contains s1, change to s0 strncpy(pc2, "s0", 2); else if ((pc2 = strstr(pflower.filename, "s0")) != NULL) //Lower contains s0, change to s1 strncpy(pc2, "s1", 2); else { printf("Unable to determine which sideband is which\n"); exit(EXIT_FAILURE); } //Setting the name of the output file, setting as same name as input file, but removing s0/s1. pc1 = strstr(pflower.filename, "s1"); pc2 = strrchr(pflower.filename, '.'); //At '.fits' pc2--; while ((pc2 >= pflower.filename) && isdigit(*pc2)) //Move through the digits to the separation char. pc2--; strncpy(outfilename, pflower.filename, pc1 - pflower.filename); //Copy everything up to s1 into outfilename strncpy(outfilename + (pc1 - pflower.filename), pc1 + 2, pc2 - pc1 - 2); //Concatenate from after s1 to char before the separation char. pc1 = outfilename + (pc2 - pflower.filename - 2); *pc1 = 0; int rv = psrfits_open(&pfupper); //Open upper band if (rv) { fits_report_error(stderr, rv); exit(1); } rv = psrfits_open(&pflower); //Open lower band if (rv) { fits_report_error(stderr, rv); exit(1); } pfo = pflower; //Copy all lower band variables into the output struct if (!cmd->outputbasenameP) sprintf(pfo.basefilename, basename(outfilename)); else sprintf(pfo.basefilename, cmd->outputbasename); pfo.filenum = 0; sprintf(pfo.filename, "\0"); //Set filename to null so psrfits_open will create the filename for me pfo.rownum = 1; pfo.tot_rows = 0; pfo.N = 0; printf("lower rows_per_file=%d\n",pflower.rows_per_file); printf("upper rows_per_file=%d\n",pfupper.rows_per_file); if (pfupper.rows_per_file != pflower.rows_per_file) { //Sanity check for the two input frequency bands fprintf(stderr, "rows_per_file in input files do not match!\n"); exit(1); } double upperfreqoflower, nextfromlower, lowerfreqofupper, numchandiff; //Used to find which frequencies to take from each band double offsetfactor, scalefactor; //Factors which will be applied to offsets and scales int upchanskip, lowchanskip; //Number of channels to skip in each banda //Variables used to make code cleaner int extrachanoffset, outoffset, upperoffset, numtocopyupper, loweroffset_skip, loweroffset, numtocopylower, newuppernchan, newlowernchan; double df = pflower.hdr.df; int nchan = pflower.hdr.nchan; int outnchan; int npol = pflower.hdr.npol; int nbits = pflower.hdr.nbits; int nsblk = pflower.hdr.nsblk; //Allocate memory for all upper and lower data pflower.sub.dat_freqs = (double *) malloc(sizeof(double) * nchan); pflower.sub.dat_weights = (float *) malloc(sizeof(float) * nchan); pflower.sub.dat_offsets = (float *) malloc(sizeof(float) * nchan * npol); pflower.sub.dat_scales = (float *) malloc(sizeof(float) * nchan * npol); pflower.sub.rawdata = (unsigned char *) malloc(pflower.sub.bytes_per_subint); pflower.sub.data = (unsigned char *) malloc(pflower.sub.bytes_per_subint*2); pfupper.sub.dat_freqs = (double *) malloc(sizeof(double) * nchan); pfupper.sub.dat_weights = (float *) malloc(sizeof(float) * nchan); pfupper.sub.dat_offsets = (float *) malloc(sizeof(float) * nchan * npol); pfupper.sub.dat_scales = (float *) malloc(sizeof(float) * nchan * npol); pfupper.sub.rawdata = (unsigned char *) malloc(pfupper.sub.bytes_per_subint); pfupper.sub.data = (unsigned char *) malloc(pfupper.sub.bytes_per_subint*2); int firsttime = 1; //First time through do while loop do { print_percent_complete(pflower.rownum, pflower.rows_per_file, pflower.rownum == 1 ? 1 : 0); psrfits_read_subint(&pflower); psrfits_read_subint(&pfupper); if (firsttime) { //First time through loop, calculate factors for scales and offsets and number of channels to skip firsttime = 0; //Find the number of channels in the upper band which will be skipped if (df < 0) { //Find channel order, low to high or high to low upperfreqoflower = pflower.sub.dat_freqs[0]; //Highest frequency channel in lower band lowerfreqofupper = pfupper.sub.dat_freqs[nchan - 1]; //Lowest frequency channel in upper band } else { upperfreqoflower = pflower.sub.dat_freqs[nchan - 1]; //Highest frequency channel in lower band lowerfreqofupper = pfupper.sub.dat_freqs[0]; //Lowest frequency channel in upper band } nextfromlower = upperfreqoflower + fabs(df); //Second highest channel in lower band numchandiff = (nextfromlower - lowerfreqofupper) / fabs(df); //Number of channels to skip in float form int chanskip; if (numchandiff > 0) { //Make sure there are channels which need to be skipped if (numchandiff - (double) ((int) numchandiff) > .5) // See whether we need to round up integer channels to skip chanskip = (int) numchandiff + 1; else chanskip = (int) numchandiff; } else chanskip = 0; //No need to skip any channels if (chanskip % 2 == 1) { //Odd number of channels, give lower band the extra channel upchanskip = chanskip / 2; lowchanskip = chanskip / 2 + 1; } else //Even number of channels to skip upchanskip = lowchanskip = chanskip / 2; if (upchanskip % 2 == 1) { //We want an even number of channels in upper band for 4-bit data to get copied correctly ++lowchanskip; --upchanskip; } //Find new values given the number of channels skipped pfo.hdr.nchan = outnchan = nchan + nchan - chanskip + 2; //New number of channels, plus 2 to make nchan=960 (many factors of 2) pfo.hdr.BW = (double) outnchan *fabs(df); //New bandwidth pfo.hdr.fctr = //New center frequency (pflower.hdr.fctr - (double) (nchan / 2) * fabs(df)) + pfo.hdr.BW / 2.0; pfo.sub.bytes_per_subint = //Calculate new number of bytes in each subint outnchan * nsblk * nbits / 8 * npol; //Allocate space for output data now that we know the new number of channels pfo.sub.dat_freqs = (double *) malloc(sizeof(double) * outnchan); pfo.sub.dat_weights = (float *) malloc(sizeof(float) * outnchan); pfo.sub.dat_offsets = (float *) malloc(sizeof(float) * outnchan * npol); pfo.sub.dat_scales = (float *) malloc(sizeof(float) * outnchan * npol); pfo.sub.rawdata = (unsigned char *) malloc(pfo.sub.bytes_per_subint); pfo.sub.data = (unsigned char *) malloc(pfo.sub.bytes_per_subint*2); newuppernchan = nchan - upchanskip; //The number of channels to copy from the upper sideband. newlowernchan = nchan - lowchanskip; //The number of channels to copy from the lower sideband. extrachanoffset = 2; //Offset for 2 extra freq channels making nchan 960 in bytes outoffset = (outnchan * npol); //Offset in each loop due to previously written data upperoffset = (nchan * npol); //Offset in loop for upper band numtocopyupper = (newuppernchan * npol); //Number of bytes to copy from upper band loweroffset_skip = (lowchanskip * npol); //Number of bytes to skip when copying lower band due to //having written upper band loweroffset = //Number of bytes to skip due to having written previous lower band data (nchan * npol); numtocopylower = (newlowernchan * npol); //Number of bytes to copy from lower band float upmean, upvar, lowmean, lowvar; avg_var(pfupper.sub.dat_offsets + (nchan - upchanskip), //Find the mean and variance of the upper band's offsets upchanskip, &upmean, &upvar); printf("Upper offset stats: mean=%f variance=%f\n", upmean, upvar); avg_var(pflower.sub.dat_offsets, lowchanskip, &lowmean, &lowvar); //Find the mean and variance of the lower band's offsets printf("Lower offset stats: mean=%f variance=%f\n", lowmean, lowvar); printf("Applying factor of %f to upper offsets\n", (lowmean / upmean)); offsetfactor = lowmean / upmean; //Set offset factor used to correct variance differences in the two bands avg_var(pfupper.sub.dat_scales + (nchan - upchanskip), //Find the mean and var. of the upper band's scales upchanskip, &upmean, &upvar); printf("Upper scales stats: mean=%f variance=%f\n", upmean, upvar); avg_var(pflower.sub.dat_scales, lowchanskip, &lowmean, &lowvar); //Find the mean and var. of the lower band's scales printf("Lower scales stats: mean=%f variance=%f\n", lowmean, lowvar); printf("Applying factor of %f to upper scales\n", (lowmean / upmean)); scalefactor = lowmean / upmean; //Set scale factor used to correct variance differences in the two bands } if (pflower.status == 0 && pfupper.status == 0) { //Copy info from the lower band subint struct to the output file's subint struct pfo.sub.tsubint = pflower.sub.tsubint; pfo.sub.offs = pflower.sub.offs; pfo.sub.lst = pflower.sub.lst; pfo.sub.ra = pflower.sub.ra; pfo.sub.dec = pflower.sub.dec; pfo.sub.glon = pflower.sub.glon; pfo.sub.glat = pflower.sub.glat; pfo.sub.feed_ang = pflower.sub.feed_ang; pfo.sub.pos_ang = pflower.sub.pos_ang; pfo.sub.par_ang = pflower.sub.par_ang; pfo.sub.tel_az = pflower.sub.tel_az; pfo.sub.tel_zen = pflower.sub.tel_zen; pfo.sub.FITS_typecode = pflower.sub.FITS_typecode; //Create variables to reduce column width of lines below double *dat_freqs = pfo.sub.dat_freqs; double *udat_freqs = pfupper.sub.dat_freqs; double *ldat_freqs = pflower.sub.dat_freqs; float *dat_weights = pfo.sub.dat_weights; float *udat_weights = pfupper.sub.dat_weights; float *ldat_weights = pflower.sub.dat_weights; float *dat_offsets = pfo.sub.dat_offsets; float *udat_offsets = pfupper.sub.dat_offsets; float *ldat_offsets = pflower.sub.dat_offsets; float *dat_scales = pfo.sub.dat_scales; float *udat_scales = pfupper.sub.dat_scales; float *ldat_scales = pflower.sub.dat_scales; unsigned char *data = pfo.sub.data; unsigned char *udata = pfupper.sub.data; unsigned char *ldata = pflower.sub.data; if (df < 0) { //Copy frequency labels dat_freqs[1] = udat_freqs[0] + fabs(df); //Calculate the frequency labels dat_freqs[0] = dat_freqs[1] + fabs(df); //for our two empty frequency channels int newuppernchan = nchan - upchanskip; //The number of channels to copy from the upper band int newlowernchan = nchan - lowchanskip; //The number of channels to copy from the lower band memcpy(dat_freqs + 2, udat_freqs, sizeof(double) * newuppernchan); //Copy from the upper band, skipping first two chans. memcpy(dat_freqs + newuppernchan + 2, //Copy from the lower band ldat_freqs + lowchanskip, sizeof(double) * newlowernchan); //Copy weights dat_weights[0] = dat_weights[1] = 0; //Set the weights of first two channels to 0, so they shouldn't be used in calculations memcpy(dat_weights + 2, udat_weights, //Copy weights from the upper band sizeof(float) * newuppernchan); memcpy(dat_weights + 2 + newuppernchan, //Copy weights from the lower band ldat_weights + lowchanskip, sizeof(float) * newlowernchan); //Copy offsets dat_offsets[0] = dat_offsets[1] = //Set offsets of first two channels to the same as upper's first channel udat_offsets[0]; //(shouldn't matter since they should be ignored) int ii; for (ii = 0; ii < newuppernchan; ++ii) //Apply offset factor to upper band udat_offsets[ii] = udat_offsets[ii] * (offsetfactor); memcpy(dat_offsets + 2 * npol, udat_offsets, //Copy upper offsets sizeof(float) * newuppernchan * npol); memcpy(dat_offsets + (newuppernchan + 2) * npol, //Copy lower offsets ldat_offsets + lowchanskip, sizeof(float) * newlowernchan * npol); //Copy scales for (ii = 0; ii < newuppernchan; ++ii) //Apply scale factor to upper band udat_scales[ii] = udat_scales[ii] * (scalefactor); dat_scales[0] = dat_scales[1] = udat_scales[0]; memcpy(dat_scales + 2 * npol, udat_scales, //Copy upper scales sizeof(float) * newuppernchan * npol); memcpy(dat_scales + (newuppernchan + 2) * npol, //Copy lower scales ldat_scales + lowchanskip, sizeof(float) * newlowernchan * npol); //Copy the data for (ii = 0; ii < nsblk; ++ii) { //Loop through data copying into place memcpy(data + ii * outoffset + extrachanoffset, udata + ii * upperoffset, numtocopyupper); memcpy(data + ii * outoffset + extrachanoffset + numtocopyupper, ldata + ii * loweroffset + loweroffset_skip, numtocopylower); } psrfits_write_subint(&pfo); } else { } } } while (pfo.rownum <= pfo.rows_per_file && pfupper.status==0 && pflower.status==0); printf("Closing file '%s'\n", pflower.filename); fits_close_file(pfupper.fptr, &status); printf("Closing file '%s'\n", pfupper.filename); fits_close_file(pflower.fptr, &status); if(pflower.status!=0||pfupper.status!=0) { fprintf(stderr,"An error occurred when combining the two Mock files!\n"); if(pflower.status==108||pfupper.status==108) fprintf(stderr,"One or both of the files is incomplete.\n"); exit(1); } exit(0); }
int main(int argc, char *argv[]) { /* Cmd line */ static struct option long_opts[] = { {"output", 1, NULL, 'o'}, {"npulse", 1, NULL, 'n'}, {"nbin", 1, NULL, 'b'}, {"nthread", 1, NULL, 'j'}, {"initial", 1, NULL, 'i'}, {"final", 1, NULL, 'f'}, {"time", 1, NULL, 'T'}, {"length", 1, NULL, 'L'}, {"src", 1, NULL, 's'}, {"polyco", 1, NULL, 'p'}, {"parfile", 1, NULL, 'P'}, {"foldfreq",1, NULL, 'F'}, {"cal", 0, NULL, 'C'}, {"unsigned",0, NULL, 'u'}, {"quiet", 0, NULL, 'q'}, {"help", 0, NULL, 'h'}, {0,0,0,0} }; int opt, opti; int nbin=256, nthread=4, fnum_start=1, fnum_end=0; int quiet=0, raw_signed=1, use_polycos=1, cal=0; int npulse_per_file = 64; double start_time=0.0, process_time=0.0; double fold_frequency=0.0; char output_base[256] = ""; char polyco_file[256] = ""; char par_file[256] = ""; char source[24]; source[0]='\0'; while ((opt=getopt_long(argc,argv,"o:n:b:j:i:f:T:L:s:p:P:F:Cuqh",long_opts,&opti))!=-1) { switch (opt) { case 'o': strncpy(output_base, optarg, 255); output_base[255]='\0'; break; case 'n': npulse_per_file = atoi(optarg); break; case 'b': nbin = atoi(optarg); break; case 'j': nthread = atoi(optarg); break; case 'i': fnum_start = atoi(optarg); break; case 'f': fnum_end = atoi(optarg); break; case 'T': start_time = atof(optarg); break; case 'L': process_time = atof(optarg); break; case 's': strncpy(source, optarg, 24); source[23]='\0'; break; case 'p': strncpy(polyco_file, optarg, 255); polyco_file[255]='\0'; use_polycos = 1; break; case 'P': strncpy(par_file, optarg, 255); par_file[255] = '\0'; break; case 'F': fold_frequency = atof(optarg); use_polycos = 0; break; case 'C': cal = 1; use_polycos = 0; break; case 'u': raw_signed=0; break; case 'q': quiet=1; break; case 'h': default: usage(); exit(0); break; } } if (optind==argc) { usage(); exit(1); } /* If no polyco/par file given, default to polyco.dat */ if (use_polycos && (par_file[0]=='\0' && polyco_file[0]=='\0')) sprintf(polyco_file, "polyco.dat"); /* Open first file */ struct psrfits pf; strcpy(pf.basefilename, argv[optind]); pf.filenum = fnum_start; pf.tot_rows = pf.N = pf.T = pf.status = 0; pf.hdr.chan_dm = 0.0; // What if folding data that has been partially de-dispersed? pf.filename[0]='\0'; int rv = psrfits_open(&pf); if (rv) { fits_report_error(stderr, rv); exit(1); } /* Check any constraints */ if (pf.hdr.nbits!=8) { fprintf(stderr, "Only implemented for 8-bit data (read nbits=%d).\n", pf.hdr.nbits); exit(1); } /* Check for calfreq */ if (cal) { if (pf.hdr.cal_freq==0.0) { if (fold_frequency==0.0) { fprintf(stderr, "Error: Cal mode selected, but CAL_FREQ=0. " "Set cal frequency with -F\n"); exit(1); } else { pf.hdr.cal_freq = fold_frequency; } } else { fold_frequency = pf.hdr.cal_freq; } } /* Set up output file */ struct psrfits pf_out; memcpy(&pf_out, &pf, sizeof(struct psrfits)); if (source[0]!='\0') { strncpy(pf_out.hdr.source, source, 24); } else { strncpy(source, pf.hdr.source, 24); source[23]='\0'; } if (output_base[0]=='\0') { /* Set up default output filename */ if (start_time>0.0) sprintf(output_base, "%s_SP_%s_%5.5d_%5.5d_%4.4d_%3.3d%s", pf_out.hdr.backend, pf_out.hdr.source, pf_out.hdr.start_day, (int)pf_out.hdr.start_sec, fnum_start, (int)start_time, cal ? "_cal" : ""); else sprintf(output_base, "%s_SP_%s_%5.5d_%5.5d%s", pf_out.hdr.backend, pf_out.hdr.source, pf_out.hdr.start_day, (int)pf_out.hdr.start_sec, cal ? "_cal" : ""); } strcpy(pf_out.basefilename, output_base); if (cal) { sprintf(pf_out.hdr.obs_mode, "CAL"); sprintf(pf_out.hdr.cal_mode, "SYNC"); } else sprintf(pf_out.hdr.obs_mode, "PSR"); strncpy(pf_out.fold.parfile,par_file,255); pf_out.fold.parfile[255]='\0'; pf_out.fptr = NULL; pf_out.filenum=0; pf_out.status=0; pf_out.hdr.nbin=nbin; pf_out.sub.FITS_typecode = TFLOAT; pf_out.sub.bytes_per_subint = sizeof(float) * pf_out.hdr.nchan * pf_out.hdr.npol * pf_out.hdr.nbin; pf_out.multifile = 1; pf_out.quiet = 1; pf_out.rows_per_file = npulse_per_file; rv = psrfits_create(&pf_out); if (rv) { fits_report_error(stderr, rv); exit(1); } /* Alloc data buffers */ pf.sub.dat_freqs = (float *)malloc(sizeof(float) * pf.hdr.nchan); pf_out.sub.dat_freqs = pf.sub.dat_freqs; pf.sub.dat_weights = (float *)malloc(sizeof(float) * pf.hdr.nchan); pf_out.sub.dat_weights = (float *)malloc(sizeof(float) * pf.hdr.nchan); pf.sub.dat_offsets = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); pf_out.sub.dat_offsets = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); pf.sub.dat_scales = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); pf_out.sub.dat_scales = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); pf_out.sub.data = (unsigned char *)malloc(pf_out.sub.bytes_per_subint); /* Output scale/offset */ int i, j, ipol, ichan; float offset_uv=0.0; // Extra cross-term offset for GUPPI if (strcmp("GUPPI",pf.hdr.backend)==0) { offset_uv=0.5; fprintf(stderr, "Found backend=GUPPI, setting offset_uv=%f\n", offset_uv); } // TODO: copy these from the input file for (ipol=0; ipol<pf.hdr.npol; ipol++) { for (ichan=0; ichan<pf.hdr.nchan; ichan++) { float offs = 0.0; if (ipol>1) offs = offset_uv; pf_out.sub.dat_scales[ipol*pf.hdr.nchan + ichan] = 1.0; pf_out.sub.dat_offsets[ipol*pf.hdr.nchan + ichan] = offs; } } for (i=0; i<pf.hdr.nchan; i++) { pf_out.sub.dat_weights[i]=1.0; } /* Read or make polycos */ int npc=0, ipc=0; struct polyco *pc = NULL; if (use_polycos) { if (polyco_file[0]=='\0') { /* Generate from par file */ npc = make_polycos(par_file, &pf.hdr, source, &pc); if (npc<=0) { fprintf(stderr, "Error generating polycos.\n"); exit(1); } printf("Auto-generated %d polycos, src=%s\n", npc, source); } else { /* Read from polyco file */ FILE *pcfile = fopen(polyco_file, "r"); if (pcfile==NULL) { fprintf(stderr, "Couldn't open polyco file.\n"); exit(1); } npc = read_all_pc(pcfile, &pc); if (npc==0) { fprintf(stderr, "Error parsing polyco file.\n"); exit(1); } fclose(pcfile); } } else { // Const fold period mode, generate a fake polyco? pc = (struct polyco *)malloc(sizeof(struct polyco)); sprintf(pc[0].psr, "CONST"); pc[0].mjd = (int)pf.hdr.MJD_epoch; pc[0].fmjd = fmod(pf.hdr.MJD_epoch,1.0); pc[0].rphase = 0.0; pc[0].f0 = fold_frequency; pc[0].nsite = 0; // Does this matter? pc[0].nmin = 24 * 60; pc[0].nc = 1; pc[0].rf = pf.hdr.fctr; pc[0].c[0] = 0.0; pc[0].used = 0; npc = 1; } int *pc_written = (int *)malloc(sizeof(int) * npc); for (i=0; i<npc; i++) pc_written[i]=0; /* Set up fold buf */ struct foldbuf fb; fb.nchan = pf.hdr.nchan; fb.npol = pf.hdr.npol; fb.nbin = pf_out.hdr.nbin; malloc_foldbuf(&fb); clear_foldbuf(&fb); struct fold_args fargs; fargs.data = (char *)malloc(sizeof(char)*pf.sub.bytes_per_subint); fargs.fb = &fb; fargs.nsamp = 1; fargs.tsamp = pf.hdr.dt; fargs.raw_signed = raw_signed; /* Main loop */ rv=0; int imjd; double fmjd, fmjd0=0, fmjd_samp, fmjd_epoch; long long cur_pulse=0, last_pulse=0; double psr_freq=0.0; int first_loop=1, first_data=1, sampcount=0, last_filenum=0; int bytes_per_sample = pf.hdr.nchan * pf.hdr.npol; signal(SIGINT, cc); while (run) { /* Read data block */ pf.sub.data = (unsigned char *)fargs.data; rv = psrfits_read_subint(&pf); if (rv) { if (rv==FILE_NOT_OPENED) rv=0; // Don't complain on file not found run=0; break; } /* If we've passed final file, exit */ if (fnum_end && pf.filenum>fnum_end) { run=0; break; } /* Get start date, etc */ imjd = (int)pf.hdr.MJD_epoch; fmjd = (double)(pf.hdr.MJD_epoch - (long double)imjd); fmjd += (pf.sub.offs-0.5*pf.sub.tsubint)/86400.0; /* Select polyco set. * We'll assume same one is valid for whole data block. */ if (use_polycos) { ipc = select_pc(pc, npc, source, imjd, fmjd); //ipc = select_pc(pc, npc, NULL, imjd, fmjd); if (ipc<0) { fprintf(stderr, "No matching polycos (src=%s, imjd=%d, fmjd=%f)\n", source, imjd, fmjd); break; } } else { ipc = 0; } pc[ipc].used = 1; // Mark this polyco set as used for folding /* First time stuff */ if (first_loop) { fmjd0 = fmjd; psr_phase(&pc[ipc], imjd, fmjd, NULL, &last_pulse); pf_out.sub.offs=0.0; first_loop=0; for (i=0; i<pf.hdr.nchan; i++) { pf_out.sub.dat_weights[i]=pf.sub.dat_weights[i]; } last_filenum = pf_out.filenum; } /* Check to see if its time to process data */ if (start_time>0.0) { double cur_time = (fmjd - fmjd0) * 86400.0; if (cur_time<start_time) continue; } if (first_data) { psr_phase(&pc[ipc], imjd, fmjd, NULL, &last_pulse); first_data=0; } /* Check to see if we're done */ if (process_time>0.0) { double cur_time = (fmjd - fmjd0) * 86400.0; if (cur_time > start_time + process_time) { run=0; break; } } /* for singlepulse: loop over samples, output a new subint * whenever pulse number increases. */ for (i=0; i<pf.hdr.nsblk; i++) { /* Keep track of timestamp */ // TODO also pointing stuff? fmjd_samp = fmjd + i*pf.hdr.dt/86400.0; pf_out.sub.offs += pf.sub.offs - 0.5*pf.sub.tsubint + i*pf.hdr.dt; sampcount++; /* Calc current pulse number */ psr_phase(&pc[ipc], imjd, fmjd_samp, &psr_freq, &cur_pulse); /* TODO: deal with scale/offset? */ /* Fold this sample */ fargs.pc = &pc[ipc]; fargs.imjd = imjd; fargs.fmjd = fmjd_samp; rv = fold_8bit_power(fargs.pc, fargs.imjd, fargs.fmjd, fargs.data + i*bytes_per_sample, fargs.nsamp, fargs.tsamp, fargs.raw_signed, fargs.fb); if (rv!=0) { fprintf(stderr, "Fold error.\n"); exit(1); } /* See if integration needs to be written, etc */ if (cur_pulse > last_pulse) { /* Figure out timestamp */ pf_out.sub.offs /= (double)sampcount; pf_out.sub.tsubint = 1.0/psr_freq; fmjd_epoch = fmjd0 + pf_out.sub.offs/86400.0; /* Transpose, output subint */ normalize_transpose_folds((float *)pf_out.sub.data, &fb); psrfits_write_subint(&pf_out); /* If file incremented, clear polyco flags */ if (pf_out.filenum > last_filenum) for (j=0; j<npc; j++) pc_written[j]=0; /* Write this polyco if needed */ if (pc_written[ipc]==0) { psrfits_write_polycos(&pf_out, pc, npc); pc_written[ipc] = 1; } /* Check for write errors */ if (pf_out.status) { fprintf(stderr, "Error writing subint.\n"); fits_report_error(stderr, pf_out.status); exit(1); } /* Clear counters, avgs */ clear_foldbuf(&fb); pf_out.sub.offs = 0.0; sampcount=0; last_pulse = cur_pulse; last_filenum = pf_out.filenum; } } /* Progress report */ if (!quiet) { printf("\rFile %d %5.1f%%", pf.filenum, 100.0 * (float)(pf.rownum-1)/(float)pf.rows_per_file); fflush(stdout); } } psrfits_close(&pf_out); psrfits_close(&pf); if (rv) { fits_report_error(stderr, rv); } exit(0); }
int main(int argc, char *argv[]) { int ii, ipol, nc = 0, ncnp = 0, gpubps = 0, status = 0, statsum = 0; int fnum_start = 1, fnum_end = 0; int numprocs, numbands, myid, baddata = 0, droppedrow = 0; int *counts, *offsets; unsigned char *tmpbuf = NULL; struct psrfits pf; struct { double value; int index; } offs_in, offs_out; char hostname[256]; char vegas_base_dir[256] = "\0"; char output_base[256] = "\0"; int starthpc = 0; int reverse = 0; MPI_Status mpistat; /* Cmd line */ static struct option long_opts[] = { {"output", 1, NULL, 'o'}, {"initial", 1, NULL, 'i'}, {"final", 1, NULL, 'f'}, {"vegas", 1, NULL, 'V'}, {"starthpc",1, NULL, 's'}, {"reverse" ,0, NULL, 'r'}, {0,0,0,0} }; int opt, opti; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &numprocs); MPI_Comm_rank(MPI_COMM_WORLD, &myid); numbands = numprocs - 1; // Process the command line while ((opt=getopt_long(argc,argv,"o:i:f:V:s:r",long_opts,&opti))!=-1) { switch (opt) { case 'o': strncpy(output_base, optarg, 255); output_base[255]='\0'; break; case 'i': fnum_start = atoi(optarg); break; case 'f': fnum_end = atoi(optarg); break; case 'V': strcpy(vegas_base_dir, optarg); break; case 's': starthpc = atoi(optarg); break; case 'r': reverse = 1; break; default: if (myid==0) usage(); MPI_Finalize(); exit(0); break; } } if (optind==argc) { if (myid==0) usage(); MPI_Finalize(); exit(1); } if (myid == 0) { // Master proc only printf("\n\n"); printf(" MPI Search-mode PSRFITs Combiner\n"); printf(" by Scott M. Ransom\n\n"); } // Determine the hostnames of the processes { if (gethostname(hostname, 255) < 0) strcpy(hostname, "unknown"); MPI_Barrier(MPI_COMM_WORLD); if (myid == 0) printf("\n"); fflush(NULL); for (ii = 0 ; ii < numprocs ; ii++) { MPI_Barrier(MPI_COMM_WORLD); if (myid == ii) printf("Process %3d is on machine %s\n", myid, hostname); fflush(NULL); MPI_Barrier(MPI_COMM_WORLD); } MPI_Barrier(MPI_COMM_WORLD); fflush(NULL); } // Basefilenames for the GPU nodes if (myid > 0) { // Default to GUPPI mode if (vegas_base_dir[0]=='\0') sprintf(pf.basefilename, "/data/gpu/partial/%s/%s", hostname, argv[optind]); // VEGAS mode else { int hpcidx; if (reverse) hpcidx = starthpc - myid + 1; else hpcidx = myid + starthpc - 1; sprintf(pf.basefilename, "%s/vegas-hpc%d-bdata1/%s", vegas_base_dir, hpcidx, argv[optind]); printf("**********: hostname = %s, myid = %d, datamnt = %d, basename=%s\n", hostname, myid, hpcidx, pf.basefilename); } } // Initialize some key parts of the PSRFITS structure pf.tot_rows = pf.N = pf.T = pf.status = 0; pf.filenum = fnum_start; pf.filename[0] = '\0'; pf.filenames = NULL; pf.numfiles = 0; if (myid == 1) { FILE *psrfitsfile; char hdr[HDRLEN], filenm[200]; // Read the header info sprintf(filenm, "%s_0001.fits", pf.basefilename); psrfitsfile = fopen(filenm, "r"); fread(&hdr, 1, HDRLEN, psrfitsfile); fclose(psrfitsfile); // Send the header to the master proc MPI_Send(hdr, HDRLEN, MPI_CHAR, 0, 0, MPI_COMM_WORLD); } else if (myid == 0) { FILE *psrfitsfile; char hdr[HDRLEN], tmpfilenm[80]; // Receive the header info from proc 1 MPI_Recv(hdr, HDRLEN, MPI_CHAR, 1, 0, MPI_COMM_WORLD, &mpistat); // Now write that header to a temp file strcpy(tmpfilenm, "mpi_merge_psrfits.XXXXXX"); mkstemp(tmpfilenm); psrfitsfile = fopen(tmpfilenm, "w"); fwrite(&hdr, 1, HDRLEN, psrfitsfile); fclose(psrfitsfile); pf.filenames = (char **)malloc(sizeof(char *)); pf.filenames[0] = tmpfilenm; pf.basefilename[0]='\0'; pf.filenum = 0; pf.numfiles = 1; // And read the key information into a PSRFITS struct status = psrfits_open(&pf); status = psrfits_close(&pf); free(pf.filenames); remove(tmpfilenm); // Now create the output PSTFITS file if (output_base[0]=='\0') { /* Set up default output filename */ strcpy(output_base, argv[optind]); } strcpy(pf.basefilename, output_base); pf.multifile = 1; pf.filenum = 0; pf.numfiles = 0; pf.filename[0] = '\0'; pf.filenames = NULL; nc = pf.hdr.nchan; ncnp = pf.hdr.nchan * pf.hdr.npol; gpubps = pf.sub.bytes_per_subint; pf.hdr.orig_nchan *= numbands; pf.hdr.nchan *= numbands; pf.hdr.fctr = pf.hdr.fctr - 0.5 * pf.hdr.BW + numbands/2.0 * pf.hdr.BW; pf.hdr.BW *= numbands; pf.sub.bytes_per_subint *= numbands; long long filelen = 40 * (1L<<30); // In GB pf.rows_per_file = filelen / pf.sub.bytes_per_subint; status = psrfits_create(&pf); // For in-memory transpose of data tmpbuf = (unsigned char *)malloc(pf.sub.bytes_per_subint); } // Open the input PSRFITs files for real if (myid > 0) { status = psrfits_open(&pf); nc = pf.hdr.nchan; ncnp = pf.hdr.nchan * pf.hdr.npol; gpubps = pf.sub.bytes_per_subint; } // Alloc data buffers for the PSRFITS files pf.sub.dat_freqs = (float *)malloc(sizeof(float) * pf.hdr.nchan); pf.sub.dat_weights = (float *)malloc(sizeof(float) * pf.hdr.nchan); pf.sub.dat_offsets = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); pf.sub.dat_scales = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); pf.sub.data = (unsigned char *)malloc(pf.sub.bytes_per_subint); pf.sub.rawdata = pf.sub.data; // Counts and offsets for MPI_Gatherv counts = (int *)malloc(sizeof(int) * numprocs); offsets = (int *)malloc(sizeof(int) * numprocs); counts[0] = offsets[0] = 0; // master sends nothing // Now loop over the rows (i.e. subints)... do { MPI_Barrier(MPI_COMM_WORLD); // Read the current subint from each of the "slave" nodes if ((myid > 0) && (!baddata)) { status = psrfits_read_subint(&pf); if (status) { pf.sub.offs = FLAG; // High value so it won't be min if (pf.rownum > pf.rows_per_file) { // Shouldn't be here unless opening of new file failed... printf("Proc %d: Can't open next file. Setting status=114.\n", myid); status = 114; } } } else { // Root process pf.sub.offs = FLAG; // High value so it won't be min } // Find the minimum value of OFFS_SUB to see if we dropped a row offs_in.value = pf.sub.offs; offs_in.index = myid; MPI_Allreduce(&offs_in, &offs_out, 1, MPI_DOUBLE_INT, MPI_MINLOC, MPI_COMM_WORLD); // If all procs are returning the FLAG value, break. if (offs_out.value==FLAG) break; // Identify dropped rows if ((myid > 0) && (!status) && (!baddata) && (pf.sub.offs > (offs_out.value + 0.1 * pf.sub.tsubint))) { printf("Proc %d, row %d: Dropped a row. Filling with zeros.\n", myid, pf.rownum); droppedrow = 1; } if (myid > 0) { // Ignore errors for moving past EOF (107), read errors (108) // and missing files (114) if (droppedrow || status==108 || ((myid > 0) && (status==114 || status==107) && (!baddata))) { if (status) printf("Proc %d, row %d: Ignoring CFITSIO error %d. Filling with zeros.\n", myid, pf.rownum, status); // Set the data and the weights to all zeros for (ii = 0 ; ii < pf.hdr.nchan ; ii++) pf.sub.dat_weights[ii] = 0.0; for (ii = 0 ; ii < pf.sub.bytes_per_subint ; ii++) pf.sub.data[ii] = 0; // And the scales and offsets to nominal values for (ii = 0 ; ii < pf.hdr.nchan * pf.hdr.npol ; ii++) { pf.sub.dat_offsets[ii] = 0.0; pf.sub.dat_scales[ii] = 1.0; } // reset the status to 0 and allow going to next row if (status==114 || status==107) { baddata = 1; } if (status==108) { // Try reading the next row... pf.rownum++; pf.tot_rows++; pf.N += pf.hdr.nsblk; pf.T = pf.N * pf.hdr.dt; } if (droppedrow) { // We want to read the current row again... pf.rownum--; pf.tot_rows--; pf.N -= pf.hdr.nsblk; pf.T = pf.N * pf.hdr.dt; droppedrow = 0; // reset } status = 0; } } // If we've passed final file, exit if (fnum_end && pf.filenum > fnum_end) break; // Combine statuses of all nodes by summing.... MPI_Allreduce(&status, &statsum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (statsum) break; if (myid == offs_out.index) { // Send all of the non-band-specific parts to master MPI_Send(&pf.sub.tsubint, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); MPI_Send(&pf.sub.offs, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); MPI_Send(&pf.sub.lst, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); MPI_Send(&pf.sub.ra, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); MPI_Send(&pf.sub.dec, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); MPI_Send(&pf.sub.glon, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); MPI_Send(&pf.sub.glat, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); MPI_Send(&pf.sub.feed_ang, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); MPI_Send(&pf.sub.pos_ang, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); MPI_Send(&pf.sub.par_ang, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); MPI_Send(&pf.sub.tel_az, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); MPI_Send(&pf.sub.tel_zen, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD); } else if (myid == 0) { // Receive all of the non-data parts MPI_Recv(&pf.sub.tsubint, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat); MPI_Recv(&pf.sub.offs, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat); MPI_Recv(&pf.sub.lst, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat); MPI_Recv(&pf.sub.ra, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat); MPI_Recv(&pf.sub.dec, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat); MPI_Recv(&pf.sub.glon, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat); MPI_Recv(&pf.sub.glat, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat); MPI_Recv(&pf.sub.feed_ang, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat); MPI_Recv(&pf.sub.pos_ang, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat); MPI_Recv(&pf.sub.par_ang, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat); MPI_Recv(&pf.sub.tel_az, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat); MPI_Recv(&pf.sub.tel_zen, 1, MPI_DOUBLE, offs_out.index, 0, MPI_COMM_WORLD, &mpistat); } // Now gather the vector quantities... // Vectors of length nchan for (ii = 1 ; ii < numprocs ; ii++) { counts[ii] = nc; offsets[ii] = (ii - 1) * nc; } status = MPI_Gatherv(pf.sub.dat_freqs, nc, MPI_FLOAT, pf.sub.dat_freqs, counts, offsets, MPI_FLOAT, 0, MPI_COMM_WORLD); status = MPI_Gatherv(pf.sub.dat_weights, nc, MPI_FLOAT, pf.sub.dat_weights, counts, offsets, MPI_FLOAT, 0, MPI_COMM_WORLD); // Vectors of length nchan * npol for (ipol=0; ipol < pf.hdr.npol; ipol++) { for (ii = 1 ; ii < numprocs ; ii++) { counts[ii] = nc; offsets[ii] = ipol*nc*numbands + (ii - 1) * nc; } status = MPI_Gatherv(pf.sub.dat_offsets+(ipol*nc), nc, MPI_FLOAT, pf.sub.dat_offsets, counts, offsets, MPI_FLOAT, 0, MPI_COMM_WORLD); status = MPI_Gatherv(pf.sub.dat_scales+(ipol*nc), nc, MPI_FLOAT, pf.sub.dat_scales, counts, offsets, MPI_FLOAT, 0, MPI_COMM_WORLD); } // Vectors of length pf.sub.bytes_per_subint for the raw data for (ii = 1 ; ii < numprocs ; ii++) { counts[ii] = gpubps; offsets[ii] = (ii - 1) * gpubps; } status = MPI_Gatherv(pf.sub.data, gpubps, MPI_UNSIGNED_CHAR, tmpbuf, counts, offsets, MPI_UNSIGNED_CHAR, 0, MPI_COMM_WORLD); // Reorder and write the new row to the output file if (myid == 0) { reorder_data(pf.sub.data, tmpbuf, numbands, pf.hdr.nsblk, pf.hdr.npol, nc); status = psrfits_write_subint(&pf); } } while (statsum == 0); // Free the arrays free(pf.sub.dat_freqs); free(pf.sub.dat_weights); free(pf.sub.dat_offsets); free(pf.sub.dat_scales); free(pf.sub.data); if (myid == 0) free(tmpbuf); free(counts); free(offsets); // Close the files and finalize things status = psrfits_close(&pf); MPI_Finalize(); exit(0); }
int main(int argc, char *argv[]) { int numfiles, ii, numrows, rownum, ichan, itsamp, datidx; int spec_per_row, status, maxrows; unsigned long int maxfilesize; float offset, scale, datum, packdatum, maxval, fulltsubint; float *datachunk; FILE **infiles; struct psrfits pfin, pfout; Cmdline *cmd; fitsfile *infits, *outfits; char outfilename[128], templatename[128], tform[8]; char *pc1, *pc2; int first = 1, dummy = 0, nclipped; short int *inrowdata; unsigned char *outrowdata; if (argc == 1) { Program = argv[0]; usage(); exit(1); } // Parse the command line using the excellent program Clig cmd = parseCmdline(argc, argv); numfiles = cmd->argc; infiles = (FILE **) malloc(numfiles * sizeof(FILE *)); //Set the max. total size (in bytes) of all rows in an output file, //leaving some room for PSRFITS header maxfilesize = (unsigned long int)(cmd->numgb * GB); maxfilesize = maxfilesize - 1000*KB; //fprintf(stderr,"cmd->numgb: %f maxfilesize: %ld\n",cmd->numgb,maxfilesize); #ifdef DEBUG showOptionValues(); #endif printf("\n PSRFITS 16-bit to 4-bit Conversion Code\n"); printf(" by J. Deneva, S. Ransom, & S. Chatterjee\n\n"); // Open the input files status = 0; //fits_close segfaults if this is not initialized printf("Reading input data from:\n"); for (ii = 0; ii < numfiles; ii++) { printf(" '%s'\n", cmd->argv[ii]); //Get the file basename and number from command-line argument //(code taken from psrfits2fil) pc2 = strrchr(cmd->argv[ii], '.'); // at .fits *pc2 = 0; // terminate string pc1 = pc2 - 1; while ((pc1 >= cmd->argv[ii]) && isdigit(*pc1)) pc1--; if (pc1 <= cmd->argv[ii]) { // need at least 1 char before filenum puts("Illegal input filename. must have chars before the filenumber"); exit(1); } pc1++; // we were sitting on "." move to first digit pfin.filenum = atoi(pc1); pfin.fnamedigits = pc2 - pc1; // how many digits in filenumbering scheme. *pc1 = 0; // null terminate the basefilename strcpy(pfin.basefilename, cmd->argv[ii]); pfin.initialized = 0; // set to 1 in psrfits_open() pfin.status = 0; //(end of code taken from psrfits2fil) //Open the existing psrfits file if (psrfits_open(&pfin, READONLY) != 0) { fprintf(stderr, "error opening file\n"); fits_report_error(stderr, pfin.status); exit(1); } // Create the subint arrays if (first) { pfin.sub.dat_freqs = (float *) malloc(sizeof(float) * pfin.hdr.nchan); pfin.sub.dat_weights = (float *) malloc(sizeof(float) * pfin.hdr.nchan); pfin.sub.dat_offsets = (float *) malloc(sizeof(float) * pfin.hdr.nchan * pfin.hdr.npol); pfin.sub.dat_scales = (float *) malloc(sizeof(float) * pfin.hdr.nchan * pfin.hdr.npol); //first is set to 0 after data buffer allocation further below } infits = pfin.fptr; spec_per_row = pfin.hdr.nsblk; fits_read_key(infits, TINT, "NAXIS2", &dummy, NULL, &status); pfin.tot_rows = dummy; numrows = dummy; //If dealing with 1st input file, create output template if (ii == 0) { sprintf(templatename, "%s.template.fits",cmd->outfile); fits_create_file(&outfits, templatename, &status); //fprintf(stderr,"pfin.basefilename: %s\n", pfin.basefilename); //fprintf(stderr,"status: %d\n", status); //Instead of copying HDUs one by one, can move to the SUBINT //HDU, and copy all the HDUs preceding it fits_movnam_hdu(infits, BINARY_TBL, "SUBINT", 0, &status); fits_copy_file(infits, outfits, 1, 0, 0, &status); //Copy the SUBINT table header fits_copy_header(infits, outfits, &status); fits_flush_buffer(outfits, 0, &status); //Set NAXIS2 in the output SUBINT table to 0 b/c we haven't //written any rows yet dummy = 0; fits_update_key(outfits, TINT, "NAXIS2", &dummy, NULL, &status); //Edit the NBITS key if (DEBUG) { dummy = 8; fits_update_key(outfits, TINT, "NBITS", &dummy, NULL, &status); } else { fits_update_key(outfits, TINT, "NBITS", &(cmd->numbits), NULL, &status); } //Edit the TFORM17 column: # of data bytes per row //fits_get_colnum(outfits,1,"DATA",&dummy,&status); if (DEBUG) sprintf(tform, "%dB", pfin.hdr.nsblk * pfin.hdr.nchan * pfin.hdr.npol); else sprintf(tform, "%dB", pfin.hdr.nsblk * pfin.hdr.nchan * pfin.hdr.npol * cmd->numbits / 8); fits_update_key(outfits, TSTRING, "TTYPE17", "DATA", NULL, &status); fits_update_key(outfits, TSTRING, "TFORM17", tform, NULL, &status); //Edit NAXIS1: row width in bytes fits_read_key(outfits, TINT, "NAXIS1", &dummy, NULL, &status); if (DEBUG) { dummy = dummy - pfin.hdr.nsblk * pfin.hdr.nchan * pfin.hdr.npol * (pfin.hdr.nbits - 8) / 8; } else { dummy = dummy - pfin.hdr.nsblk * pfin.hdr.nchan * pfin.hdr.npol * (pfin.hdr.nbits - cmd->numbits) / 8; } fits_update_key(outfits, TINT, "NAXIS1", &dummy, NULL, &status); //Set the max # of rows per file, based on the requested //output file size maxrows = maxfilesize / dummy; //fprintf(stderr,"maxrows: %d\n",maxrows); fits_close_file(outfits, &status); rownum = 0; } while (psrfits_read_subint(&pfin, first) == 0) { fprintf(stderr, "Working on row %d\n", ++rownum); //If this is the first row, store the length of a full subint if (ii == 0 && rownum == 1) fulltsubint = pfin.sub.tsubint; //If this is the last row and it's partial, drop it. //(It's pfin.rownum-1 below because the rownum member of the psrfits struct seems to be intended to indicate at the *start* of what row we are, i.e. a row that has not yet been read. In contrast, pfout.rownum indicates how many rows have been written, i.e. at the *end* of what row we are in the output.) if (pfin.rownum-1 == numrows && fabs(pfin.sub.tsubint - fulltsubint) > pfin.hdr.dt) { fprintf(stderr, "Dropping partial row of length %f s (full row is %f s)\n", pfin.sub.tsubint, fulltsubint); break; } //If we just read in the 1st row, or if we already wrote the last row in the current output file, create a new output file if ((ii == 0 && rownum == 1) || pfout.rownum == maxrows) { //Create new output file from the template pfout.fnamedigits = pfin.fnamedigits; if(ii == 0) pfout.filenum = pfin.filenum; else pfout.filenum++; sprintf(outfilename, "%s.%0*d.fits", cmd->outfile, pfout.fnamedigits, pfout.filenum); fits_create_template(&outfits, outfilename, templatename, &status); //fprintf(stderr,"After fits_create_template, status: %d\n",status); fits_close_file(outfits, &status); //Now reopen the file so that the pfout structure is initialized pfout.status = 0; pfout.initialized = 0; sprintf(pfout.basefilename, "%s.", cmd->outfile); if (psrfits_open(&pfout, READWRITE) != 0) { fprintf(stderr, "error opening file\n"); fits_report_error(stderr, pfout.status); exit(1); } outfits = pfout.fptr; maxval = pow(2, cmd->numbits) - 1; pfout.rows_per_file = maxrows; //fprintf(stderr, "maxval: %f\n", maxval); //fprintf(stderr, "pfout.rows_per_file: %d\n",pfout.rows_per_file); //These are not initialized in psrfits_open but are needed //in psrfits_write_subint (not obvious what are the corresponding //fields in any of the psrfits table headers) pfout.hdr.ds_freq_fact = 1; pfout.hdr.ds_time_fact = 1; } //Copy the subint struct from pfin to pfout, but correct //elements that are not the same pfout.sub = pfin.sub; //this copies array pointers too pfout.sub.bytes_per_subint = pfin.sub.bytes_per_subint * pfout.hdr.nbits / pfin.hdr.nbits; pfout.sub.dataBytesAlloced = pfout.sub.bytes_per_subint; pfout.sub.FITS_typecode = TBYTE; if (first) { //Allocate scaling buffer and output buffer datachunk = gen_fvect(spec_per_row); outrowdata = gen_bvect(pfout.sub.bytes_per_subint); first = 0; } pfout.sub.data = outrowdata; inrowdata = (short int *) pfin.sub.data; nclipped = 0; // Loop over all the channels: for (ichan = 0; ichan < pfout.hdr.nchan * pfout.hdr.npol; ichan++) { // Populate datachunk[] by picking out all time samples for ichan for (itsamp = 0; itsamp < spec_per_row; itsamp++) datachunk[itsamp] = (float) (inrowdata[ichan + itsamp * pfout.hdr.nchan * pfout.hdr.npol]); // Compute the statistics here, and put the offsets and scales in // pf.sub.dat_offsets[] and pf.sub.dat_scales[] if (rescale(datachunk, spec_per_row, cmd->numbits, &offset, &scale) != 0) { printf("Rescale routine failed!\n"); return (-1); } pfout.sub.dat_offsets[ichan] = offset; pfout.sub.dat_scales[ichan] = scale; // Since we have the offset and scale ready, rescale the data: for (itsamp = 0; itsamp < spec_per_row; itsamp++) { datum = (scale == 0.0) ? 0.0 : roundf((datachunk[itsamp] - offset) / scale); if (datum < 0.0) { datum = 0; nclipped++; } else if (datum > maxval) { datum = maxval; nclipped++; } inrowdata[ichan + itsamp * pfout.hdr.nchan * pfout.hdr.npol] = (short int) datum; } // Now inrowdata[ichan] contains rescaled ints. } // Then do the conversion and store the // results in pf.sub.data[] if (cmd->numbits == 8 || DEBUG) { for (itsamp = 0; itsamp < spec_per_row; itsamp++) { datidx = itsamp * pfout.hdr.nchan * pfout.hdr.npol; for (ichan = 0; ichan < pfout.hdr.nchan * pfout.hdr.npol; ichan++, datidx++) { pfout.sub.data[datidx] = (unsigned char) inrowdata[datidx]; } } } else if (cmd->numbits == 4) { for (itsamp = 0; itsamp < spec_per_row; itsamp++) { datidx = itsamp * pfout.hdr.nchan * pfout.hdr.npol; for (ichan = 0; ichan < pfout.hdr.nchan * pfout.hdr.npol; ichan += 2, datidx += 2) { packdatum = inrowdata[datidx] * 16 + inrowdata[datidx + 1]; pfout.sub.data[datidx / 2] = (unsigned char) packdatum; } } } else { fprintf(stderr, "Only 4 or 8-bit output formats supported.\n"); fprintf(stderr, "Bits per sample requested: %d\n", cmd->numbits); exit(1); } //pfout.sub.offs = (pfout.tot_rows+0.5) * pfout.sub.tsubint; fprintf(stderr, "nclipped: %d fraction clipped: %f\n", nclipped, (float) nclipped / (pfout.hdr.nchan * pfout.hdr.npol * pfout.hdr.nsblk)); // Now write the row. status = psrfits_write_subint(&pfout); if (status) { printf("\nError (%d) writing PSRFITS...\n\n", status); break; } //If current output file has reached the max # of rows, close it if (pfout.rownum == maxrows) fits_close_file(outfits, &status); } //Close the files fits_close_file(infits, &status); } fits_close_file(outfits, &status); // Free the structure arrays too... free(datachunk); free(infiles); free(pfin.sub.dat_freqs); free(pfin.sub.dat_weights); free(pfin.sub.dat_offsets); free(pfin.sub.dat_scales); free(pfin.sub.data); free(pfout.sub.data); free(pfin.sub.stat); return 0; }
int main(int argc, char *argv[]) { /* Cmd line */ static struct option long_opts[] = { {"output", 1, NULL, 'o'}, {"nbin", 1, NULL, 'b'}, {"tsub", 1, NULL, 't'}, {"nthread", 1, NULL, 'j'}, {"initial", 1, NULL, 'i'}, {"final", 1, NULL, 'f'}, {"src", 1, NULL, 's'}, {"polyco", 1, NULL, 'p'}, {"parfile", 1, NULL, 'P'}, {"foldfreq",1, NULL, 'F'}, {"cal", 0, NULL, 'C'}, {"unsigned",0, NULL, 'u'}, {"nunsigned",1, NULL, 'U'}, {"split", 1, NULL, 'S'}, {"apply", 0, NULL, 'A'}, {"quiet", 0, NULL, 'q'}, {"help", 0, NULL, 'h'}, {0,0,0,0} }; int opt, opti; int nbin=256, nthread=4, fnum_start=1, fnum_end=0; int quiet=0, raw_signed=1, use_polycos=1, cal=0, apply_scale=0; double split_size_gb = 1.0; double tfold = 60.0; double fold_frequency=0.0; char output_base[256] = ""; char polyco_file[256] = ""; char par_file[256] = ""; char source[24]; source[0]='\0'; while ((opt=getopt_long(argc,argv,"o:b:t:j:i:f:s:p:P:F:CuU:S:Aqh",long_opts,&opti))!=-1) { switch (opt) { case 'o': strncpy(output_base, optarg, 255); output_base[255]='\0'; break; case 'b': nbin = atoi(optarg); break; case 't': tfold = atof(optarg); break; case 'j': nthread = atoi(optarg); break; case 'i': fnum_start = atoi(optarg); break; case 'f': fnum_end = atoi(optarg); break; case 's': strncpy(source, optarg, 24); source[23]='\0'; break; case 'p': strncpy(polyco_file, optarg, 255); polyco_file[255]='\0'; use_polycos = 1; break; case 'P': strncpy(par_file, optarg, 255); par_file[255] = '\0'; break; case 'F': fold_frequency = atof(optarg); use_polycos = 0; break; case 'C': cal = 1; use_polycos = 0; break; case 'u': raw_signed=0; break; case 'U': raw_signed = 4 - atoi(optarg); break; case 'S': split_size_gb = atof(optarg); break; case 'A': apply_scale = 1; break; case 'q': quiet=1; break; case 'h': default: usage(); exit(0); break; } } if (optind==argc) { usage(); exit(1); } /* If no polyco/par file given, default to polyco.dat */ if (use_polycos && (par_file[0]=='\0' && polyco_file[0]=='\0')) sprintf(polyco_file, "polyco.dat"); /* Open first file */ struct psrfits pf; sprintf(pf.basefilename, argv[optind]); pf.filenum = fnum_start; pf.tot_rows = pf.N = pf.T = pf.status = 0; pf.hdr.chan_dm = 0.0; // What if folding data that has been partially de-dispersed? pf.filename[0]='\0'; int rv = psrfits_open(&pf); if (rv) { fits_report_error(stderr, rv); exit(1); } /* Check any constraints */ if (pf.hdr.nbits!=8) { fprintf(stderr, "Only implemented for 8-bit data (read nbits=%d).\n", pf.hdr.nbits); exit(1); } /* Check for calfreq */ if (cal) { if (pf.hdr.cal_freq==0.0) { if (fold_frequency==0.0) { fprintf(stderr, "Error: Cal mode selected, but CAL_FREQ=0. " "Set cal frequency with -F\n"); exit(1); } else { pf.hdr.cal_freq = fold_frequency; } } else { fold_frequency = pf.hdr.cal_freq; } } /* Set up output file */ struct psrfits pf_out; memcpy(&pf_out, &pf, sizeof(struct psrfits)); if (source[0]!='\0') { strncpy(pf_out.hdr.source, source, 24); } else { strncpy(source, pf.hdr.source, 24); source[23]='\0'; } if (output_base[0]=='\0') { /* Set up default output filename */ sprintf(output_base, "%s_%s_%5.5d_%5.5d%s", pf_out.hdr.backend, pf_out.hdr.source, pf_out.hdr.start_day, (int)pf_out.hdr.start_sec, cal ? "_cal" : ""); } sprintf(pf_out.basefilename, output_base); if (cal) { sprintf(pf_out.hdr.obs_mode, "CAL"); sprintf(pf_out.hdr.cal_mode, "SYNC"); } else sprintf(pf_out.hdr.obs_mode, "PSR"); strncpy(pf_out.fold.parfile,par_file,255); pf_out.fold.parfile[255]='\0'; pf_out.fptr = NULL; pf_out.filenum=0; pf_out.status=0; pf_out.quiet=0; pf_out.hdr.nbin=nbin; pf_out.sub.FITS_typecode = TFLOAT; pf_out.sub.bytes_per_subint = sizeof(float) * pf_out.hdr.nchan * pf_out.hdr.npol * pf_out.hdr.nbin; if (split_size_gb > 0.0) { pf_out.multifile = 1; pf_out.rows_per_file = (int) (split_size_gb * (1024.0*1024.0*1024.0) / (double)pf_out.sub.bytes_per_subint); printf("Writing a maximum of %d subintegrations (~%.1f GB) per output file.\n", pf_out.rows_per_file, split_size_gb); } else { pf_out.multifile = 0; printf("Writing a single output file.\n"); } rv = psrfits_create(&pf_out); if (rv) { fits_report_error(stderr, rv); exit(1); } /* Alloc data buffers */ pf.sub.dat_freqs = (float *)malloc(sizeof(float) * pf.hdr.nchan); pf_out.sub.dat_freqs = pf.sub.dat_freqs; pf.sub.dat_weights = (float *)malloc(sizeof(float) * pf.hdr.nchan); pf_out.sub.dat_weights = (float *)malloc(sizeof(float) * pf.hdr.nchan); pf.sub.dat_offsets = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); pf_out.sub.dat_offsets = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); pf.sub.dat_scales = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); pf_out.sub.dat_scales = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); pf_out.sub.data = (unsigned char *)malloc(pf_out.sub.bytes_per_subint); /* Output scale/offset */ int i, ipol, ichan; float offset_uv=0.0; // Extra cross-term offset for GUPPI if (strcmp("GUPPI",pf.hdr.backend)==0 && apply_scale==0) { offset_uv=0.5; fprintf(stderr, "Found backend=GUPPI, setting offset_uv=%f\n", offset_uv); } // Initialize scale/output and weights. // These get copied from the input file later during the main loop. for (ipol=0; ipol<pf.hdr.npol; ipol++) { for (ichan=0; ichan<pf.hdr.nchan; ichan++) { float offs = 0.0; if (ipol>1) offs = offset_uv; pf_out.sub.dat_scales[ipol*pf.hdr.nchan + ichan] = 1.0; pf_out.sub.dat_offsets[ipol*pf.hdr.nchan + ichan] = offs; } } for (i=0; i<pf.hdr.nchan; i++) { pf_out.sub.dat_weights[i]=1.0; } /* Read or make polycos */ int npc=0, ipc=0; struct polyco *pc = NULL; if (use_polycos) { if (polyco_file[0]=='\0') { /* Generate from par file */ npc = make_polycos(par_file, &pf.hdr, source, &pc); if (npc<=0) { fprintf(stderr, "Error generating polycos.\n"); exit(1); } printf("Auto-generated %d polycos, src=%s\n", npc, source); } else { /* Read from polyco file */ FILE *pcfile = fopen(polyco_file, "r"); if (pcfile==NULL) { fprintf(stderr, "Couldn't open polyco file.\n"); exit(1); } npc = read_all_pc(pcfile, &pc); if (npc==0) { fprintf(stderr, "Error parsing polyco file.\n"); exit(1); } fclose(pcfile); } } else { // Const fold period mode, generate a fake polyco? pc = (struct polyco *)malloc(sizeof(struct polyco)); sprintf(pc[0].psr, "CONST"); pc[0].mjd = (int)pf.hdr.MJD_epoch; pc[0].fmjd = fmod(pf.hdr.MJD_epoch,1.0); pc[0].rphase = 0.0; pc[0].f0 = fold_frequency; pc[0].nsite = 0; // Does this matter? pc[0].nmin = 24 * 60; pc[0].nc = 1; pc[0].rf = pf.hdr.fctr; pc[0].c[0] = 0.0; pc[0].used = 0; npc = 1; } int *pc_written = (int *)malloc(sizeof(int) * npc); for (i=0; i<npc; i++) pc_written[i]=0; /* Alloc total fold buf */ struct foldbuf fb; fb.nchan = pf.hdr.nchan; fb.npol = pf.hdr.npol; fb.nbin = pf_out.hdr.nbin; malloc_foldbuf(&fb); clear_foldbuf(&fb); /* Set up thread management */ pthread_t *thread_id; struct fold_args *fargs; thread_id = (pthread_t *)malloc(sizeof(pthread_t) * nthread); fargs = (struct fold_args *)malloc(sizeof(struct fold_args) * nthread); for (i=0; i<nthread; i++) { thread_id[i] = 0; fargs[i].data = (char *)malloc(sizeof(char)*pf.sub.bytes_per_subint); fargs[i].fb = (struct foldbuf *)malloc(sizeof(struct foldbuf)); fargs[i].fb->nbin = pf_out.hdr.nbin; fargs[i].fb->nchan = pf.hdr.nchan; fargs[i].fb->npol = pf.hdr.npol; fargs[i].nsamp = pf.hdr.nsblk; fargs[i].tsamp = pf.hdr.dt; fargs[i].raw_signed=raw_signed; malloc_foldbuf(fargs[i].fb); clear_foldbuf(fargs[i].fb); fargs[i].scale = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); fargs[i].offset = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); } /* Main loop */ rv=0; int imjd; double fmjd, fmjd0=0, fmjd_next=0, fmjd_epoch; double offs0=0, offs1=0; //double phase=0.0, freq=1.0; int first=1, subcount=0; int cur_thread = 0; signal(SIGINT, cc); while (run) { /* Read data block */ pf.sub.data = (unsigned char *)fargs[cur_thread].data; rv = psrfits_read_subint(&pf); if (rv) { if (rv==FILE_NOT_OPENED) rv=0; // Don't complain on file not found run=0; break; } /* If we've passed final file, exit */ if (fnum_end && pf.filenum>fnum_end) { run=0; break; } /* Get start date, etc */ imjd = (int)pf.hdr.MJD_epoch; fmjd = (double)(pf.hdr.MJD_epoch - (long double)imjd); fmjd += (pf.sub.offs-0.5*pf.sub.tsubint)/86400.0; /* First time stuff */ if (first) { fmjd0 = fmjd; fmjd_next = fmjd + tfold/86400.0; pf_out.sub.offs=0.0; offs0 = pf.sub.offs - 0.5*pf.sub.tsubint; offs1 = pf.sub.offs + 0.5*pf.sub.tsubint; first=0; for (i=0; i<pf.hdr.nchan; i++) { pf_out.sub.dat_weights[i]=pf.sub.dat_weights[i]; } } /* Keep track of timestamp */ // TODO also pointing stuff. pf_out.sub.offs += pf.sub.offs; subcount++; /* Update output block end time */ offs1 = pf.sub.offs + 0.5*pf.sub.tsubint; /* Select polyco set */ if (use_polycos) { ipc = select_pc(pc, npc, source, imjd, fmjd); //ipc = select_pc(pc, npc, NULL, imjd, fmjd); if (ipc<0) { fprintf(stderr, "No matching polycos (src=%s, imjd=%d, fmjd=%f)\n", source, imjd, fmjd); break; } } else { ipc = 0; } pc[ipc].used = 1; // Mark this polyco set as used for folding /* Copy scale/offset from input to output if we're not applying it */ if (apply_scale==0) { for (i=0; i<pf.hdr.nchan*pf.hdr.npol; i++) { pf_out.sub.dat_scales[i] = pf.sub.dat_scales[i]; pf_out.sub.dat_offsets[i] = pf.sub.dat_offsets[i]; } } /* Fold this subint */ fargs[cur_thread].pc = &pc[ipc]; fargs[cur_thread].imjd = imjd; fargs[cur_thread].fmjd = fmjd; rv = pthread_create(&thread_id[cur_thread], NULL, fold_8bit_power_thread, &fargs[cur_thread]); if (rv) { fprintf(stderr, "Thread creation error.\n"); exit(1); } if (apply_scale) { for (i=0; i<pf.hdr.nchan*pf.hdr.npol; i++) { fargs[cur_thread].scale[i] = pf.sub.dat_scales[i]; fargs[cur_thread].offset[i] = pf.sub.dat_offsets[i]; } } cur_thread++; /* Combine thread results if needed */ if (cur_thread==nthread || fmjd>fmjd_next) { /* Loop over active threads */ for (i=0; i<cur_thread; i++) { /* Wait for thread to finish */ rv = pthread_join(thread_id[i], NULL); if (rv) { fprintf(stderr, "Thread join error.\n"); exit(1); } /* Apply scale and offset here */ if (apply_scale) scale_offset_folds(fargs[i].fb, fargs[i].scale, fargs[i].offset); /* Combine its result into total fold */ accumulate_folds(&fb, fargs[i].fb); /* Reset thread info */ clear_foldbuf(fargs[i].fb); thread_id[i] = 0; } /* Reset active thread count */ cur_thread = 0; } /* See if integration needs to be written, etc */ if (fmjd > fmjd_next) { /* Figure out timestamp */ pf_out.sub.offs /= (double)subcount; pf_out.sub.tsubint = offs1 - offs0; fmjd_epoch = fmjd0 + pf_out.sub.offs/86400.0; /* // Don't need this stuff if we set EPOCHS=MIDTIME ipc = select_pc(pc, npc, pf.hdr.source, imjd, fmjd_epoch); if (ipc<0) { fprintf(stderr, "Polyco error, exiting.\n"); exit(1); } phase = psr_phase(&pc[ipc], imjd, fmjd_epoch, &freq); phase = fmod(phase, 1.0); pf_out.sub.offs -= phase/freq; // ref epoch needs 0 phase */ /* Transpose, output subint */ normalize_transpose_folds((float *)pf_out.sub.data, &fb); int last_filenum = pf_out.filenum; psrfits_write_subint(&pf_out); /* Check for write errors */ if (pf_out.status) { fprintf(stderr, "Error writing subint.\n"); fits_report_error(stderr, pf_out.status); exit(1); } /* Check if we started a new file */ if (pf_out.filenum!=last_filenum) { /* No polycos yet written to this file */ for (i=0; i<npc; i++) pc_written[i]=0; } /* Write the current polyco if needed */ if (pc_written[ipc]==0) { psrfits_write_polycos(&pf_out, &pc[ipc], 1); if (pf_out.status) { fprintf(stderr, "Error writing polycos.\n"); fits_report_error(stderr, pf_out.status); exit(1); } pc_written[ipc] = 1; } /* Clear counters, avgs */ clear_foldbuf(&fb); pf_out.sub.offs = 0.0; offs0 = pf.sub.offs - 0.5*pf.sub.tsubint; subcount=0; /* Set next output time */ fmjd_next = fmjd + tfold/86400.0; } /* Progress report */ if (!quiet) { printf("\rFile %d %5.1f%%", pf.filenum, 100.0 * (float)(pf.rownum-1)/(float)pf.rows_per_file); fflush(stdout); } } /* Join any running threads */ for (i=0; i<cur_thread; i++) if (thread_id[i]) pthread_join(thread_id[i], NULL); /* Remove polyco table in cal mode */ if (cal) { rv = psrfits_remove_polycos(&pf_out); if (rv) { fits_report_error(stderr, rv); } } psrfits_close(&pf_out); psrfits_close(&pf); if (rv) { fits_report_error(stderr, rv); } exit(0); }