void decimate_header() /* includefile */
{
/* go no further here if not interested in header parameters */
if (headerless) return;
/* broadcast the header parameters to the output stream */
send_string("HEADER_START");
if (!strings_equal(source_name,"")) {
send_string("source_name");
send_string(source_name);
}
send_int("telescope_id",telescope_id);
send_int("machine_id",machine_id);
send_coords(src_raj,src_dej,az_start,za_start);
if (nchans/naddc==1) {
send_int("data_type",2);
// send_double("refdm",0.0);
send_double("refdm",refdm);
/*== MKEITH: A hack to stop the dminfo being lost when decimating after dedisperse! ==*/
} else {
send_int("data_type",1);
}
send_double("fch1",fch1);
send_double("foff",foff*(double)naddc);
send_int("nchans",nchans/naddc);
send_int("nbits",obits);
send_double ("tstart",tstart);
send_double("tsamp",tsamp*(double)naddt);
send_int("nifs",nifs);
send_int("barycentric",barycentric);
send_string("HEADER_END");
}
void filterbank_header(FILE *outptr) /* includefile */
{
int i,j;
output=outptr;
if (obits == -1) obits=nbits;
/* go no further here if not interested in header parameters */
if (headerless) return;
/* broadcast the header parameters to the output stream */
if (machine_id != 0) {
send_string("HEADER_START");
send_string("rawdatafile");
send_string(inpfile);
if (!strings_equal(source_name,"")) {
send_string("source_name");
send_string(source_name);
}
send_int("machine_id",machine_id);
send_int("telescope_id",telescope_id);
send_coords(src_raj,src_dej,az_start,za_start);
if (zerolagdump) {
/* time series data DM=0.0 */
send_int("data_type",2);
refdm=0.0;
send_double("refdm",refdm);
send_int("nchans",1);
} else {
/* filterbank data */
send_int("data_type",1);
send_double("fch1",fch1);
send_double("foff",foff);
send_int("nchans",nchans);
}
/* beam info */
send_int("nbeams",nbeams);
send_int("ibeam",ibeam);
/* number of bits per sample */
send_int("nbits",obits);
/* start time and sample interval */
send_double("tstart",tstart+(double)start_time/86400.0);
send_double("tsamp",tsamp);
if (sumifs) {
send_int("nifs",1);
} else {
j=0;
for (i=1;i<=nifs;i++) if (ifstream[i-1]=='Y') j++;
if (j==0) error_message("no valid IF streams selected!");
send_int("nifs",j);
}
send_string("HEADER_END");
}
}
main (int argc, char **argv)
{
int i=1, j, k, nfiles=0, *numbt, schans=0, nbytes, *nchan;
FILE *input[32];
char *block;
double *stamp, *frch1, *froff, *frmhz;
output=stdout;
/* print help if necessary */
if (argc<=1 || help_required(argv[1])) {
/*splice_help();*/
exit(0);
} else {
print_version(argv[0],argv[1]);
}
/* open up files */
while (i<argc) {
if (file_exists(argv[i])) {
input[nfiles]=open_file(argv[i],"rb");
nfiles++;
} else if (strings_equal(argv[i],"-o")) {
output=open_file(argv[++i],"wb");
}
i++;
}
/* read in headers and check time stamps */
stamp = (double *) malloc(nfiles*sizeof(double));
frch1 = (double *) malloc(nfiles*sizeof(double));
froff = (double *) malloc(nfiles*sizeof(double));
numbt = (int *) malloc(nfiles*sizeof(int));
nchan = (int *) malloc(nfiles*sizeof(int));
for (i=0; i<nfiles; i++) {
if (read_header(input[i])) {
stamp[i]=tstart;
frch1[i]=fch1;
froff[i]=foff;
numbt[i]=nbits;
nchan[i]=nchans;
schans+=nchans;
} else {
error_message("problem reading header parameters");
}
if (data_type != 1)
error_message("input data are not in filterbank format!");
if (stamp[i] != stamp[0])
error_message("start times in input files are not identical!");
if (numbt[i] != numbt[0])
error_message("number of bits per sample in input files not identical!");
if (i>0) {
if (frch1[i] > frch1[i-1])
error_message("input files not ordered in descending frequency!");
}
}
send_string("HEADER_START");
send_int("machine_id",machine_id);
send_int("telescope_id",telescope_id);
send_int("data_type",1);
send_string("FREQUENCY_START");
send_int("nchans",schans);
frmhz = (double *) malloc(sizeof(double)*schans);
k=0;
for (i=0; i<nfiles; i++) {
for (j=0; j<nchans; j++) {
frmhz[k]=frch1[i]+j*froff[i];
send_double("fchannel",frmhz[k++]);
}
}
send_string("FREQUENCY_END");
if (!strings_equal(source_name,"")) {
send_string("source_name");
send_string(source_name);
}
send_coords(src_raj,src_dej,az_start,za_start);
send_int("nbits",nbits);
send_double("tstart",tstart);
send_double("tsamp",tsamp);
send_int("nifs",nifs);
send_string("HEADER_END");
nbytes = nchans*nbits/8;
block = (char *) malloc(nbytes);
while (1) {
for (i=0; i<nfiles; i++) {
if (feof(input[i])) exit(0);
fread(block,nbytes,1,input[i]);
fwrite(block,nbytes,1,output);
}
}
}
main(int argc, char** argv)
{
int c,arg=1,i,j,k,nstep;
long long int fsizeusb,nsamp,nextract=0,nsampuse;
char fnameusb[180], fheader[180], headerline[80], junk[80];
char rastring[80], decstring[80], outputfile[180];
double ras, des;
int rah, ram, ded, dem, hh, mm, ssm;
double ss;
FILE *finfileusb,*flog,*headerfile;
short int *DATA1,*DATA4,temp;
unsigned char *DATA2,*DATA3;
char timeBuffer[80];
time_t now ;
if (argc <= 1)
{
printf("USAGE : gmrt2fb -f filename -r raj -d decj -m MJD -s timeSample -n nbits -w channelWidth -c first_channelFreq -t No of Channels \n");
printf("\n");
printf(" filename: Combined Data filename \n");
printf("\n");
exit(0);
}
if (file_exists(argv[1])) {
strcpy(fnameusb,argv[1]);
if((finfileusb = fopen(fnameusb,"rb"))== NULL) {
error_message("error opening GMRT data file");
}
strcpy(fheader,"");
strncat(fheader,fnameusb,strlen(fnameusb)-3);
strcat(fheader,"hdr");
if((headerfile = fopen(fheader,"rb"))== NULL) {
error_message("error opening GMRT data file");
}
while (fgets(headerline, 80, headerfile) != NULL) {
if (strstr(headerline,"Num Channels") != NULL) {
sscanf(headerline,"%s %s %s %d",junk,junk,junk,&nchans);
}
if (strstr(headerline,"Channel width") != NULL) {
sscanf(headerline,"%s %s %s %lf",junk,junk,junk,&foff);
foff*=-1.0;
}
if (strstr(headerline,"Frequency Ch.1") != NULL) {
sscanf(headerline,"%s %s %s %lf",junk,junk,junk,&fch1);
}
if (strstr(headerline,"Num bits/sample") != NULL) {
sscanf(headerline,"%s %s %s %d",junk,junk,junk,&nbits);
}
if (strstr(headerline,"MJD") != NULL) {
sscanf(headerline,"%s %s %lf",junk,junk,&tstart);
}
if (strstr(headerline,"UTC") != NULL) {
sscanf(headerline,"%s %s %d:%d:%lf",junk,junk,&hh,&mm,&ss);
tstart+=(double)hh/24.0+(double)mm/1440.0+ss/86400.0;
}
if (strstr(headerline,"Sampling Time") != NULL) {
sscanf(headerline,"%s %s %s %lf",junk,junk,junk,&tsamp);
tsamp*=1.0e-6;
}
if (strstr(headerline,"Coordinates") != NULL) {
sscanf(headerline,"%s %s %s %s",junk,junk,junk,decstring);
strncat(rastring,junk,strlen(junk)-1);
rah=atoi(strtok(rastring,":"));
ram=atoi(strtok(NULL,":"));
ras=atof(strtok(NULL,":"));
src_raj=ras+ram*100.0+rah*10000.0;
ded=atoi(strtok(decstring,":"));
dem=atoi(strtok(NULL,":"));
des=atof(strtok(NULL,":"));
src_dej=des+100.0*dem+10000.0*abs(ded);
}
if (strstr(headerline,"Source") != NULL) {
sscanf(headerline,"%s %s %s",junk,junk,source_name);
}
}
}
// flog=fopen("convert_sig_gmrt.log","a");
one_more_time:
now = time(0);
strcpy(timeBuffer, ctime(&now));
printf("\n\nStarting on %s\n",timeBuffer);
// fprintf(flog,"----------------------------\n");
// fprintf(flog,"Starting on %s\n",timeBuffer);
while((c = getopt(argc, argv, "f:r:d:m:s:n:w:c:t:")) != -1)
{
switch (c)
{
case 'f':
sscanf(optarg,"%s",fnameusb);
if((finfileusb = fopen(fnameusb,"rb"))== NULL)
{
printf("%s\n",fnameusb);
printf("Error opening file \n");
// fprintf(flog,"Error opening file \n");
// exit(0);
}
else
{
printf("\n opened file: %s \n",fnameusb);
// fprintf(flog,"\n opened file: %s \n",fnameusb);
}
}
switch (c)
{
case 'r':
sscanf(optarg,"%lf",&src_raj);
printf("testing %lf\n",src_raj);
}
switch (c)
{
case 'd':
sscanf(optarg,"%lf",&src_dej);
}
switch (c)
{
case 'm':
sscanf(optarg,"%lf",&tstart);
}
switch (c)
{
case 's':
sscanf(optarg,"%lf",&tsamp);
}
switch (c)
{
case 'n':
sscanf(optarg,"%d",&nbits);
}
switch (c)
{
case 'w':
sscanf(optarg,"%lf",&foff);
}
switch (c)
{
case 'c':
sscanf(optarg,"%lf",&fch1);
}
switch (c)
{
case 't':
sscanf(optarg,"%d",&nchans);
}
}
printf("Filling the header for sigproc file\n");
data_type=1;
nifs=1;
/* the earlier code (convertSigM) was built with these values :
machine_id=1; // this is the id used for analysing sigproc data at NCRA
telescope_id=3; // this Arecibo id is used for analysing sigproc data at NCRA for the time being..
*/
/* new values of machine_id and telescope_id : expected to work with sigproc3.7 and 4.1 */
machine_id=7; // new machine ID for GMRT
telescope_id=7; // new telescope ID for GMRT
az_start=0.0;
za_start=0.0;
printf("machine_id %d ; telescope_id %d \n",machine_id,telescope_id);
/******************************************/
/* Find the file size */
fseeko(finfileusb,0,SEEK_END);
fsizeusb = ftello(finfileusb);
fseeko(finfileusb,0,SEEK_SET);
printf("File size : %lld\n",fsizeusb);
// fprintf(flog,"File size : %lld\n",fsizeusb);
/* Move the file pointer */
printf("Location of file ptr: %lld\n",ftello(finfileusb));
/*
build a file name of the form:
source_MJD_secondssincemidnight_gmrt.fil
*/
ssm = (int) 86400.0*(tstart-floor(tstart));
sprintf(outputfile,"%s_%d_%d_gmrt.fil",source_name,(int)floor(tstart),ssm);
/*
strcpy(outputfile,"");
strncat(outputfile,fnameusb,strlen(fnameusb)-4);
strcat(outputfile,".fil");
*/
output = fopen(outputfile,"w");
printf("Outputfile: %s\n",outputfile);
// fprintf(flog,"Outputfile: %s\n",outputfile);
nsamp = fsizeusb;
if (nbits == 16) { // for 16bit data, read and write it as short int
DATA1 = malloc (2*nch*maxsamp * sizeof (short int *));
DATA4 = malloc (2*nch*maxsamp * sizeof (short int *));
nsamp = nsamp/1024; /* converting nsamp in units of time samples */
printf("Number of bytes: %lld, Number of time samples:%lld\n",1024*nsamp,nsamp);
// fprintf(flog,"Number of bytes: %lld, Number of time samples:%lld\n",1024*nsamp,nsamp);
}
else {// for 8bit data, read and write it as unsigned char
printf("no of bits %d \n",nbits);
DATA2 = malloc (2*nch*maxsamp * sizeof (unsigned char *));
DATA3 = malloc (2*nch*maxsamp * sizeof (unsigned char *));
nsamp = nsamp/512; /* converting nsamp in units of time samples */
printf("Number of bytes: %lld, Number of time samples:%lld\n",512*nsamp,nsamp);
// fprintf(flog,"Number of bytes: %lld, Number of time samples:%lld\n",512*nsamp,nsamp);
}
/* write out the header for the *fil files for sigproc that is needed*/
send_string("HEADER_START");
send_string("source_name");
send_string(source_name);
send_int("machine_id",machine_id);
send_int("telescope_id",telescope_id);
send_int("data_type",data_type);
send_coords(src_raj,src_dej,az_start,za_start);
send_double("fch1",fch1);
send_double("foff",foff);
send_int("nchans",nchans);
send_int("nbits",nbits);
send_double("tstart",tstart);
send_double("tsamp",tsamp);
send_int("nifs",nifs);
send_string("HEADER_END");
// End of the Sigproc header...
// fclose(output);
// exit(0);
nstep=5;
nsampuse = nsamp/nstep;
while(nsampuse > maxsamp)
{
nstep=nstep+10;
nsampuse = nsamp/nstep;
printf("Nsampuse: %lld\n",nsampuse);
}
printf("samples per block:%lld\n",nsampuse);
// fprintf(flog,"samples per block:%lld\n",nsampuse);
for(i=0;i<nstep;i++)
{
if (nbits == 16) fread(DATA4,sizeof(short),2*nch*nsampuse,finfileusb);
else fread(DATA2,sizeof(unsigned char),2*nch*nsampuse,finfileusb);
for(k=0;k<nsampuse;k++)
{
for(j=0;j<nchans;j++) {
if (nbits == 16) DATA1[k*nchans+j]=DATA4[k*nchans+(nchans-1-j)];
else DATA3[k*nchans+j]=DATA2[k*nchans+(nchans-1-j)];
}
}
if (nbits == 16) fwrite(DATA1,sizeof(short int),2*nch*nsampuse,output);
else fwrite(DATA3,sizeof(unsigned char),2*nch*nsampuse,output);
printf("STEP:%d out of %d steps writing %lld time samples\n",i+1,nstep,(i+1)*nsampuse);
// fprintf(flog,"STEP:%d out of %d steps writing %lld time samples\n",i+1,nstep,(i+1)*nsampuse);
}
// fprintf(flog,"Finished\n\n");
fclose(output);
fclose(finfileusb);
// fclose(flog);
}
main (int argc, char *argv[])
{
float clip_level, *blk, sum, mean, blksum, blksum2, blkmean, blkmean2;
unsigned char *cblk;
unsigned short *sblk;
float blksigma, dmean, dsigma;
int blksize, nread, nsblk, i, headersize, c, s, offset, obits;
long int isamp=0, seed=0, nclip=0;
if (argc<2 || help_required(argv[1])) {
//blanker_help();
exit(0);
}
print_version(argv[0],argv[1]);
if (!file_exists(argv[1]))
error_message("input file does not exist!");
strcpy(inpfile,argv[1]);
input=open_file(inpfile,"r");
strcpy(outfile,"stdout");
output=stdout;
nsblk=1024;
clip_level=6.0;
obits=32;
i=2;
while (i<argc) {
if (strings_equal(argv[i],"-c")) clip_level=atof(argv[++i]);
if (strings_equal(argv[i],"-s")) nsblk=atof(argv[++i]);
if (strings_equal(argv[i],"-b")) obits=atoi(argv[++i]);
i++;
}
if ((headersize=read_header(input))) {
if (nifs>1) error_message("blanker can only handle nifs=1");
send_string("HEADER_START");
if (!strings_equal(source_name,"")) {
send_string("source_name");
send_string(source_name);
}
send_int("telescope_id",telescope_id);
send_int("machine_id",machine_id);
send_coords(src_raj,src_dej,az_start,za_start);
send_int("data_type",data_type);
send_double("fch1",fch1);
send_double("foff",foff);
send_int("barycentric",barycentric);
send_int("nchans",nchans);
send_int("nbits",obits);
send_double ("tstart",tstart);
send_double("tsamp",tsamp);
send_int("nifs",nifs);
send_string("HEADER_END");
blksize=nchans*nsblk;
blk = (float *) malloc(blksize*sizeof(float));
cblk = (unsigned char *) malloc(blksize*sizeof(unsigned char));
sblk = (unsigned short *) malloc(blksize*sizeof(unsigned short));
while (nread=read_block(input,nbits,blk,blksize)) {
/* calculate mean and sigma of the whole block */
blksum=blksum2=0.0;
for (i=0; i<nread; i++) {
blksum+=blk[i];
blksum2+=blk[i]*blk[i];
}
blkmean=blksum/(float)nread;
blkmean2=blksum2/(float)nread;
blksigma=sqrt(blkmean2-blkmean*blkmean);
/* expected mean and sigma of a dedispersed sample in this block */
dmean=blkmean*(float)nchans;
dsigma=blksigma*sqrt((float)nchans);
/* now work through each sample in the block */
nsblk=nread/nchans;
offset=0;
for (s=0; s<nsblk; s++) {
isamp++;
sum=0.0;
for (c=0; c<nchans; c++) sum+=blk[offset+c];
if (abs(sum-dmean)>clip_level*dsigma) {
nclip++;
for (c=0; c<nchans; c++)
blk[offset+c]=gasdev(&seed)*blksigma+blkmean;
}
offset+=nchans;
}
/* write out the clipped data (32 bit!) */
if (obits == 32)
fwrite(blk,sizeof(float),nread,output);
} else if (obits == 8) {
for (i=0;i<nread;i++) cblk[i]=blk[i];
fwrite(cblk,sizeof(unsigned char),nread,output);
} else if (obits == 16) {
for (i=0;i<nread;i++) sblk[i]=blk[i];
fwrite(sblk,sizeof(unsigned short),nread,output);
} else {
error_message("only 8, 16 or 32 bit output modes supported");
}
}
void Octtree::find_cell_ranks( const boost::multi_array<Real,2>& coordinates, std::vector<Uint>& ranks )
{
ranks.resize(coordinates.size());
Handle< Elements > element_component;
Uint element_idx;
std::deque<Uint> missing_cells;
RealVector dummy(m_dim);
for(Uint i=0; i<coordinates.size(); ++i)
{
for (Uint d=0; d<m_dim; ++d)
dummy[d] = coordinates[i][d];
if( find_element(dummy,element_component,element_idx) )
{
ranks[i] = Comm::instance().rank();
}
else
{
ranks[i] = math::Consts::uint_max();
missing_cells.push_back(i);
}
}
std::vector<Real> send_coords(m_dim*missing_cells.size());
std::vector<Real> recv_coords;
Uint c(0);
boost_foreach(const Uint i, missing_cells)
{
for(Uint d=0; d<m_dim; ++d)
send_coords[c++]=coordinates[i][d];
}
for (Uint root=0; root<PE::Comm::instance().size(); ++root)
{
recv_coords.resize(0);
PE::Comm::instance().broadcast(send_coords,recv_coords,root,m_dim);
// size is only because it doesn't get resized for this rank
std::vector<Uint> send_found(missing_cells.size(),math::Consts::uint_max());
if (root!=Comm::instance().rank())
{
std::vector<RealVector> recv_coordinates(recv_coords.size()/m_dim) ;
boost_foreach(RealVector& realvec, recv_coordinates)
realvec.resize(m_dim);
c=0;
for (Uint i=0; i<recv_coordinates.size(); ++i)
{
for(Uint d=0; d<m_dim; ++d)
recv_coordinates[i][d]=recv_coords[c++];
}
send_found.resize(recv_coordinates.size());
for (Uint i=0; i<recv_coordinates.size(); ++i)
{
if( find_element(recv_coordinates[i],element_component,element_idx) )
{
send_found[i] = Comm::instance().rank();
}
else
send_found[i] = math::Consts::uint_max();
}
}
std::vector<Uint> recv_found(missing_cells.size()*Comm::instance().size());
PE::Comm::instance().gather(send_found,recv_found,root);
if( root==Comm::instance().rank())
{
const Uint stride = missing_cells.size();
for (Uint i=0; i<missing_cells.size(); ++i)
{
for(Uint p=0; p<Comm::instance().size(); ++p)
{
ranks[missing_cells[i]] = std::min(recv_found[i+p*stride] , ranks[missing_cells[i]]);
}
}
}
}
}
