void send_coords(double raj, double dej, double az, double za) /*includefile*/ { if ((raj != 0.0) || (raj != -1.0)) send_double("src_raj",raj); if ((dej != 0.0) || (dej != -1.0)) send_double("src_dej",dej); if ((az != 0.0) || (az != -1.0)) send_double("az_start",az); if ((za != 0.0) || (za != -1.0)) send_double("za_start",za); }
void decimate_header() /* includefile */ { /* go no further here if not interested in header parameters */ if (headerless) return; /* broadcast the header parameters to the output stream */ send_string("HEADER_START"); if (!strings_equal(source_name,"")) { send_string("source_name"); send_string(source_name); } send_int("telescope_id",telescope_id); send_int("machine_id",machine_id); send_coords(src_raj,src_dej,az_start,za_start); if (nchans/naddc==1) { send_int("data_type",2); // send_double("refdm",0.0); send_double("refdm",refdm); /*== MKEITH: A hack to stop the dminfo being lost when decimating after dedisperse! ==*/ } else { send_int("data_type",1); } send_double("fch1",fch1); send_double("foff",foff*(double)naddc); send_int("nchans",nchans/naddc); send_int("nbits",obits); send_double ("tstart",tstart); send_double("tsamp",tsamp*(double)naddt); send_int("nifs",nifs); send_int("barycentric",barycentric); send_string("HEADER_END"); }
void filterbank_header(FILE *outptr) /* includefile */ { int i,j; output=outptr; if (obits == -1) obits=nbits; /* go no further here if not interested in header parameters */ if (headerless) return; /* broadcast the header parameters to the output stream */ if (machine_id != 0) { send_string("HEADER_START"); send_string("rawdatafile"); send_string(inpfile); if (!strings_equal(source_name,"")) { send_string("source_name"); send_string(source_name); } send_int("machine_id",machine_id); send_int("telescope_id",telescope_id); send_coords(src_raj,src_dej,az_start,za_start); if (zerolagdump) { /* time series data DM=0.0 */ send_int("data_type",2); refdm=0.0; send_double("refdm",refdm); send_int("nchans",1); } else { /* filterbank data */ send_int("data_type",1); send_double("fch1",fch1); send_double("foff",foff); send_int("nchans",nchans); } /* beam info */ send_int("nbeams",nbeams); send_int("ibeam",ibeam); /* number of bits per sample */ send_int("nbits",obits); /* start time and sample interval */ send_double("tstart",tstart+(double)start_time/86400.0); send_double("tsamp",tsamp); if (sumifs) { send_int("nifs",1); } else { j=0; for (i=1;i<=nifs;i++) if (ifstream[i-1]=='Y') j++; if (j==0) error_message("no valid IF streams selected!"); send_int("nifs",j); } send_string("HEADER_END"); } }
main (int argc, char **argv) { int i=1, j, k, nfiles=0, *numbt, schans=0, nbytes, *nchan; FILE *input[32]; char *block; double *stamp, *frch1, *froff, *frmhz; output=stdout; /* print help if necessary */ if (argc<=1 || help_required(argv[1])) { /*splice_help();*/ exit(0); } else { print_version(argv[0],argv[1]); } /* open up files */ while (i<argc) { if (file_exists(argv[i])) { input[nfiles]=open_file(argv[i],"rb"); nfiles++; } else if (strings_equal(argv[i],"-o")) { output=open_file(argv[++i],"wb"); } i++; } /* read in headers and check time stamps */ stamp = (double *) malloc(nfiles*sizeof(double)); frch1 = (double *) malloc(nfiles*sizeof(double)); froff = (double *) malloc(nfiles*sizeof(double)); numbt = (int *) malloc(nfiles*sizeof(int)); nchan = (int *) malloc(nfiles*sizeof(int)); for (i=0; i<nfiles; i++) { if (read_header(input[i])) { stamp[i]=tstart; frch1[i]=fch1; froff[i]=foff; numbt[i]=nbits; nchan[i]=nchans; schans+=nchans; } else { error_message("problem reading header parameters"); } if (data_type != 1) error_message("input data are not in filterbank format!"); if (stamp[i] != stamp[0]) error_message("start times in input files are not identical!"); if (numbt[i] != numbt[0]) error_message("number of bits per sample in input files not identical!"); if (i>0) { if (frch1[i] > frch1[i-1]) error_message("input files not ordered in descending frequency!"); } } send_string("HEADER_START"); send_int("machine_id",machine_id); send_int("telescope_id",telescope_id); send_int("data_type",1); send_string("FREQUENCY_START"); send_int("nchans",schans); frmhz = (double *) malloc(sizeof(double)*schans); k=0; for (i=0; i<nfiles; i++) { for (j=0; j<nchans; j++) { frmhz[k]=frch1[i]+j*froff[i]; send_double("fchannel",frmhz[k++]); } } send_string("FREQUENCY_END"); if (!strings_equal(source_name,"")) { send_string("source_name"); send_string(source_name); } send_coords(src_raj,src_dej,az_start,za_start); send_int("nbits",nbits); send_double("tstart",tstart); send_double("tsamp",tsamp); send_int("nifs",nifs); send_string("HEADER_END"); nbytes = nchans*nbits/8; block = (char *) malloc(nbytes); while (1) { for (i=0; i<nfiles; i++) { if (feof(input[i])) exit(0); fread(block,nbytes,1,input[i]); fwrite(block,nbytes,1,output); } } }
main(int argc, char** argv) { int c,arg=1,i,j,k,nstep; long long int fsizeusb,nsamp,nextract=0,nsampuse; char fnameusb[180], fheader[180], headerline[80], junk[80]; char rastring[80], decstring[80], outputfile[180]; double ras, des; int rah, ram, ded, dem, hh, mm, ssm; double ss; FILE *finfileusb,*flog,*headerfile; short int *DATA1,*DATA4,temp; unsigned char *DATA2,*DATA3; char timeBuffer[80]; time_t now ; if (argc <= 1) { printf("USAGE : gmrt2fb -f filename -r raj -d decj -m MJD -s timeSample -n nbits -w channelWidth -c first_channelFreq -t No of Channels \n"); printf("\n"); printf(" filename: Combined Data filename \n"); printf("\n"); exit(0); } if (file_exists(argv[1])) { strcpy(fnameusb,argv[1]); if((finfileusb = fopen(fnameusb,"rb"))== NULL) { error_message("error opening GMRT data file"); } strcpy(fheader,""); strncat(fheader,fnameusb,strlen(fnameusb)-3); strcat(fheader,"hdr"); if((headerfile = fopen(fheader,"rb"))== NULL) { error_message("error opening GMRT data file"); } while (fgets(headerline, 80, headerfile) != NULL) { if (strstr(headerline,"Num Channels") != NULL) { sscanf(headerline,"%s %s %s %d",junk,junk,junk,&nchans); } if (strstr(headerline,"Channel width") != NULL) { sscanf(headerline,"%s %s %s %lf",junk,junk,junk,&foff); foff*=-1.0; } if (strstr(headerline,"Frequency Ch.1") != NULL) { sscanf(headerline,"%s %s %s %lf",junk,junk,junk,&fch1); } if (strstr(headerline,"Num bits/sample") != NULL) { sscanf(headerline,"%s %s %s %d",junk,junk,junk,&nbits); } if (strstr(headerline,"MJD") != NULL) { sscanf(headerline,"%s %s %lf",junk,junk,&tstart); } if (strstr(headerline,"UTC") != NULL) { sscanf(headerline,"%s %s %d:%d:%lf",junk,junk,&hh,&mm,&ss); tstart+=(double)hh/24.0+(double)mm/1440.0+ss/86400.0; } if (strstr(headerline,"Sampling Time") != NULL) { sscanf(headerline,"%s %s %s %lf",junk,junk,junk,&tsamp); tsamp*=1.0e-6; } if (strstr(headerline,"Coordinates") != NULL) { sscanf(headerline,"%s %s %s %s",junk,junk,junk,decstring); strncat(rastring,junk,strlen(junk)-1); rah=atoi(strtok(rastring,":")); ram=atoi(strtok(NULL,":")); ras=atof(strtok(NULL,":")); src_raj=ras+ram*100.0+rah*10000.0; ded=atoi(strtok(decstring,":")); dem=atoi(strtok(NULL,":")); des=atof(strtok(NULL,":")); src_dej=des+100.0*dem+10000.0*abs(ded); } if (strstr(headerline,"Source") != NULL) { sscanf(headerline,"%s %s %s",junk,junk,source_name); } } } // flog=fopen("convert_sig_gmrt.log","a"); one_more_time: now = time(0); strcpy(timeBuffer, ctime(&now)); printf("\n\nStarting on %s\n",timeBuffer); // fprintf(flog,"----------------------------\n"); // fprintf(flog,"Starting on %s\n",timeBuffer); while((c = getopt(argc, argv, "f:r:d:m:s:n:w:c:t:")) != -1) { switch (c) { case 'f': sscanf(optarg,"%s",fnameusb); if((finfileusb = fopen(fnameusb,"rb"))== NULL) { printf("%s\n",fnameusb); printf("Error opening file \n"); // fprintf(flog,"Error opening file \n"); // exit(0); } else { printf("\n opened file: %s \n",fnameusb); // fprintf(flog,"\n opened file: %s \n",fnameusb); } } switch (c) { case 'r': sscanf(optarg,"%lf",&src_raj); printf("testing %lf\n",src_raj); } switch (c) { case 'd': sscanf(optarg,"%lf",&src_dej); } switch (c) { case 'm': sscanf(optarg,"%lf",&tstart); } switch (c) { case 's': sscanf(optarg,"%lf",&tsamp); } switch (c) { case 'n': sscanf(optarg,"%d",&nbits); } switch (c) { case 'w': sscanf(optarg,"%lf",&foff); } switch (c) { case 'c': sscanf(optarg,"%lf",&fch1); } switch (c) { case 't': sscanf(optarg,"%d",&nchans); } } printf("Filling the header for sigproc file\n"); data_type=1; nifs=1; /* the earlier code (convertSigM) was built with these values : machine_id=1; // this is the id used for analysing sigproc data at NCRA telescope_id=3; // this Arecibo id is used for analysing sigproc data at NCRA for the time being.. */ /* new values of machine_id and telescope_id : expected to work with sigproc3.7 and 4.1 */ machine_id=7; // new machine ID for GMRT telescope_id=7; // new telescope ID for GMRT az_start=0.0; za_start=0.0; printf("machine_id %d ; telescope_id %d \n",machine_id,telescope_id); /******************************************/ /* Find the file size */ fseeko(finfileusb,0,SEEK_END); fsizeusb = ftello(finfileusb); fseeko(finfileusb,0,SEEK_SET); printf("File size : %lld\n",fsizeusb); // fprintf(flog,"File size : %lld\n",fsizeusb); /* Move the file pointer */ printf("Location of file ptr: %lld\n",ftello(finfileusb)); /* build a file name of the form: source_MJD_secondssincemidnight_gmrt.fil */ ssm = (int) 86400.0*(tstart-floor(tstart)); sprintf(outputfile,"%s_%d_%d_gmrt.fil",source_name,(int)floor(tstart),ssm); /* strcpy(outputfile,""); strncat(outputfile,fnameusb,strlen(fnameusb)-4); strcat(outputfile,".fil"); */ output = fopen(outputfile,"w"); printf("Outputfile: %s\n",outputfile); // fprintf(flog,"Outputfile: %s\n",outputfile); nsamp = fsizeusb; if (nbits == 16) { // for 16bit data, read and write it as short int DATA1 = malloc (2*nch*maxsamp * sizeof (short int *)); DATA4 = malloc (2*nch*maxsamp * sizeof (short int *)); nsamp = nsamp/1024; /* converting nsamp in units of time samples */ printf("Number of bytes: %lld, Number of time samples:%lld\n",1024*nsamp,nsamp); // fprintf(flog,"Number of bytes: %lld, Number of time samples:%lld\n",1024*nsamp,nsamp); } else {// for 8bit data, read and write it as unsigned char printf("no of bits %d \n",nbits); DATA2 = malloc (2*nch*maxsamp * sizeof (unsigned char *)); DATA3 = malloc (2*nch*maxsamp * sizeof (unsigned char *)); nsamp = nsamp/512; /* converting nsamp in units of time samples */ printf("Number of bytes: %lld, Number of time samples:%lld\n",512*nsamp,nsamp); // fprintf(flog,"Number of bytes: %lld, Number of time samples:%lld\n",512*nsamp,nsamp); } /* write out the header for the *fil files for sigproc that is needed*/ send_string("HEADER_START"); send_string("source_name"); send_string(source_name); send_int("machine_id",machine_id); send_int("telescope_id",telescope_id); send_int("data_type",data_type); send_coords(src_raj,src_dej,az_start,za_start); send_double("fch1",fch1); send_double("foff",foff); send_int("nchans",nchans); send_int("nbits",nbits); send_double("tstart",tstart); send_double("tsamp",tsamp); send_int("nifs",nifs); send_string("HEADER_END"); // End of the Sigproc header... // fclose(output); // exit(0); nstep=5; nsampuse = nsamp/nstep; while(nsampuse > maxsamp) { nstep=nstep+10; nsampuse = nsamp/nstep; printf("Nsampuse: %lld\n",nsampuse); } printf("samples per block:%lld\n",nsampuse); // fprintf(flog,"samples per block:%lld\n",nsampuse); for(i=0;i<nstep;i++) { if (nbits == 16) fread(DATA4,sizeof(short),2*nch*nsampuse,finfileusb); else fread(DATA2,sizeof(unsigned char),2*nch*nsampuse,finfileusb); for(k=0;k<nsampuse;k++) { for(j=0;j<nchans;j++) { if (nbits == 16) DATA1[k*nchans+j]=DATA4[k*nchans+(nchans-1-j)]; else DATA3[k*nchans+j]=DATA2[k*nchans+(nchans-1-j)]; } } if (nbits == 16) fwrite(DATA1,sizeof(short int),2*nch*nsampuse,output); else fwrite(DATA3,sizeof(unsigned char),2*nch*nsampuse,output); printf("STEP:%d out of %d steps writing %lld time samples\n",i+1,nstep,(i+1)*nsampuse); // fprintf(flog,"STEP:%d out of %d steps writing %lld time samples\n",i+1,nstep,(i+1)*nsampuse); } // fprintf(flog,"Finished\n\n"); fclose(output); fclose(finfileusb); // fclose(flog); }
main (int argc, char *argv[]) { float clip_level, *blk, sum, mean, blksum, blksum2, blkmean, blkmean2; unsigned char *cblk; unsigned short *sblk; float blksigma, dmean, dsigma; int blksize, nread, nsblk, i, headersize, c, s, offset, obits; long int isamp=0, seed=0, nclip=0; if (argc<2 || help_required(argv[1])) { //blanker_help(); exit(0); } print_version(argv[0],argv[1]); if (!file_exists(argv[1])) error_message("input file does not exist!"); strcpy(inpfile,argv[1]); input=open_file(inpfile,"r"); strcpy(outfile,"stdout"); output=stdout; nsblk=1024; clip_level=6.0; obits=32; i=2; while (i<argc) { if (strings_equal(argv[i],"-c")) clip_level=atof(argv[++i]); if (strings_equal(argv[i],"-s")) nsblk=atof(argv[++i]); if (strings_equal(argv[i],"-b")) obits=atoi(argv[++i]); i++; } if ((headersize=read_header(input))) { if (nifs>1) error_message("blanker can only handle nifs=1"); send_string("HEADER_START"); if (!strings_equal(source_name,"")) { send_string("source_name"); send_string(source_name); } send_int("telescope_id",telescope_id); send_int("machine_id",machine_id); send_coords(src_raj,src_dej,az_start,za_start); send_int("data_type",data_type); send_double("fch1",fch1); send_double("foff",foff); send_int("barycentric",barycentric); send_int("nchans",nchans); send_int("nbits",obits); send_double ("tstart",tstart); send_double("tsamp",tsamp); send_int("nifs",nifs); send_string("HEADER_END"); blksize=nchans*nsblk; blk = (float *) malloc(blksize*sizeof(float)); cblk = (unsigned char *) malloc(blksize*sizeof(unsigned char)); sblk = (unsigned short *) malloc(blksize*sizeof(unsigned short)); while (nread=read_block(input,nbits,blk,blksize)) { /* calculate mean and sigma of the whole block */ blksum=blksum2=0.0; for (i=0; i<nread; i++) { blksum+=blk[i]; blksum2+=blk[i]*blk[i]; } blkmean=blksum/(float)nread; blkmean2=blksum2/(float)nread; blksigma=sqrt(blkmean2-blkmean*blkmean); /* expected mean and sigma of a dedispersed sample in this block */ dmean=blkmean*(float)nchans; dsigma=blksigma*sqrt((float)nchans); /* now work through each sample in the block */ nsblk=nread/nchans; offset=0; for (s=0; s<nsblk; s++) { isamp++; sum=0.0; for (c=0; c<nchans; c++) sum+=blk[offset+c]; if (abs(sum-dmean)>clip_level*dsigma) { nclip++; for (c=0; c<nchans; c++) blk[offset+c]=gasdev(&seed)*blksigma+blkmean; } offset+=nchans; } /* write out the clipped data (32 bit!) */ if (obits == 32) fwrite(blk,sizeof(float),nread,output); } else if (obits == 8) { for (i=0;i<nread;i++) cblk[i]=blk[i]; fwrite(cblk,sizeof(unsigned char),nread,output); } else if (obits == 16) { for (i=0;i<nread;i++) sblk[i]=blk[i]; fwrite(sblk,sizeof(unsigned short),nread,output); } else { error_message("only 8, 16 or 32 bit output modes supported"); } }
int main (int argc, char *argv[]) { char headerless; double obstime; char string[80]; long seed=-1; float fval; unsigned char A,B,C,D; char help=0; uint_fast32_t i; /* set up default variables */ strcpy(inpfile,"stdin"); strcpy(outfile,"stdout"); headerless=0; machine_id=telescope_id=0; machine_id=10; telescope_id=4; nchans=1024; nbits=2; tstart=56000.0; tsamp=64.0; // microseconds.. will be converted to seconds later fch1=1581.804688; foff=-0.390625; nifs=1; nbeams=1; ibeam=1; obstime=270.0; output=stdout; help=getB("--help","-h",argc,argv,0); obstime=getF("--tobs","-T",argc,argv,obstime); telescope_id=getI("--tid","",argc,argv,telescope_id); machine_id=getI("--bid","",argc,argv,machine_id); tsamp=getF("--tsamp","-t",argc,argv,tsamp); tstart=getF("--mjd","-m",argc,argv,tstart); fch1=getF("--fch1","-F",argc,argv,fch1); foff=getF("--foff","-f",argc,argv,foff); nbits=getI("--nbits","-b",argc,argv,nbits); nchans=getI("--nchans","-c",argc,argv,nchans); seed=getI("--seed","-S",argc,argv,seed); char test_mode=getB("--test","-0",argc,argv,0); strcpy(outfile,getS("--out","-o",argc,argv,"stdout")); strcpy(source_name,getS("--name","-s",argc,argv,"FAKE")); getArgs(&argc,argv); if (help || argc > 1){ for(i=1; i < argc; i++)logerr("Unknown argument '%s'",argv[i]); fastfake_help(); } logmsg("FASTFAKE - M.Keith 2014"); time_t t0 = time(NULL); if (seed<0)seed=t0; mjk_rand_t *rnd = mjk_rand_init(seed); logmsg("tobs = %lfs",obstime); logmsg("tsamp = %lfus",tsamp); logmsg("mjdstart = %lf",tstart); logmsg("freq chan 1 = %lfMHz",fch1); logmsg("freq offset = %lfMHz",foff); logmsg("output nbits = %d",nbits); logmsg("random seed = %ld",seed); logmsg("output file = '%s'",outfile); tsamp*=1e-6; // convert tsamp to us if(STREQ(outfile,"stdout")){ output=stdout; } else{ output=fopen(outfile,"w"); } if (!headerless) { logmsg("write header"); send_string("HEADER_START"); send_string("source_name"); send_string(source_name); send_int("machine_id",machine_id); send_int("telescope_id",telescope_id); send_int("data_type",1); send_double("fch1",fch1); send_double("foff",foff); send_int("nchans",nchans); send_int("nbits",nbits); send_int("nbeams",nbeams); send_int("ibeam",ibeam); send_double("tstart",tstart); send_double("tsamp",tsamp); send_int("nifs",nifs); if (nbits==8){ send_char("signed",OSIGN); } send_string("HEADER_END"); } int bits_per_sample = (nbits * nchans * nifs); if (bits_per_sample % 8 != 0){ logerr("bits per sample is not a multiple of 8"); exit(1); } int bytes_per_sample = bits_per_sample / 8.0; uint64_t nsamples = (uint64_t)(obstime/tsamp+0.5); logmsg("Generate %lld samples, %.3lf GiB",nsamples,nsamples*bytes_per_sample/pow(2,30)); uint64_t onepercent = nsamples/100; int percent=0; unsigned char* buffer = (unsigned char*) malloc(sizeof(unsigned char)*bytes_per_sample); if (nbits==1 || nbits==2 || nbits==4 | nbits==8){ // integer samples for(uint64_t samp = 0; samp < nsamples; samp++){ if (samp%onepercent ==0){ double t1=(double)(time(NULL)-t0)+1e-3; double bytespersec = samp*bytes_per_sample/t1; fprintf(stderr,"Complete: % 3d%%. Sample: % 9lld Real time % 6.1lfs, Sim time % 6.1lfs. Speed % 4.2lfMiB/s\r",percent,samp,t1,(double)samp*tsamp,bytespersec/pow(2,20)); fflush(stderr); percent+=1; } mjk_rand_gauss_atleast(rnd,nchans); const int chanskip = 8/nbits; //#pragma omp parallel for schedule(dynamic,1) for(uint64_t chan = 0; chan < nchans; chan+=chanskip){ switch(nbits){ case 1: buffer[chan/8] = mjk_rand(rnd)&0xFF; break; case 2: fval = (mjk_rand_gauss(rnd)+1.5); fval = fmax(fval,0); fval = fmin(fval,3.0); A = (unsigned char)(fval)&0x3; fval = (mjk_rand_gauss(rnd)+1.5); fval = fmax(fval,0); fval = fmin(fval,3.0); B = (unsigned char)(fval)&0x3 << 2; fval = (mjk_rand_gauss(rnd)+1.5); fval = fmax(fval,0); fval = fmin(fval,3.0); C = (unsigned char)(fval)&0x3 << 4; fval = (mjk_rand_gauss(rnd)+1.5); fval = fmax(fval,0); fval = fmin(fval,3.0); D = (unsigned char)(fval)&0x3 << 6; buffer[chan/4]=A|B|C|D; break; case 4: fval = (mjk_rand_gauss(rnd)*3.0+7.5); fval = fmax(fval,0); fval = fmin(fval,15.0); A = (unsigned char)(fval)&0xF; fval = (mjk_rand_gauss(rnd)*3.0+7.5); fval = fmax(fval,0); fval = fmin(fval,15.0); B = ((unsigned char)(fval)&0xF)<<4; buffer[chan/2] = A|B; break; case 8: fval = (mjk_rand_gauss(rnd)*24.0+96.0); // more headroom. fval = fmax(fval,0); fval = fmin(fval,255.0); buffer[chan] = (unsigned char)fval; break; } } if(!test_mode)fwrite(buffer,sizeof(unsigned char),bytes_per_sample,output); } } else if (nbits==32){ // float samples float* fbuf = (float*)buffer; for(uint64_t samp = 0; samp < nsamples; samp++){ if (samp%onepercent ==0){ double t1=(double)(time(NULL)-t0)+1e-3; double bytespersec = samp*bytes_per_sample/t1; fprintf(stderr,"Complete: % 3d%%. Sample: % 9lld Real time % 6.1lfs, Sim time % 6.1lfs. Speed % 4.2lfMiB/s\r",percent,samp,t1,(double)samp*tsamp,bytespersec/pow(2,20)); fflush(stderr); percent+=1; } for(uint64_t chan = 0; chan < nchans; chan++){ fbuf[chan] = mjk_rand_gauss(rnd); } if(!test_mode)fwrite(buffer,sizeof(unsigned char),bytes_per_sample,output); } } fprintf(stderr,"\n"); free(buffer); mjk_rand_free(rnd); logmsg("Done!"); fclose(output); return 0; }