int *dmshift(double f1, double df, int nchans, int nbands, double dm, double refrf, double tsamp, double frequency_table[]) /*includefile*/ { int i, cpb, *shift; double fi; shift = (int *) malloc(nchans * sizeof(int)); fi=f1; cpb=nchans/nbands; if (frequency_table[0] != 0.0) f1=frequency_table[0]; for (i=0; i<nchans; i++) { if (refrf > 0.0) f1=refrf; if (frequency_table[0] != 0.0) fi=frequency_table[i]; shift[i]=(int)(dmdelay(fi,f1,dm)/tsamp); fi+=df; if (!((i+1)%cpb)) f1+=(double) cpb * df; } return shift; }
main(int argc, char **argv) { char message[80]; int i; float fmhz=430.0, bw=7.68, nch=128.0, dm=50.0, pms=150.0, wms=5.0, tms=0.08, tsc=0.0, mdm=0.0, nrm=100.0; float xdm, wef, snr, tdm, tdd, snrmax=0.0; FILE *input; if (argc > 1) { print_version(argv[0],argv[1]); if (help_required(argv[1])) { /*snrdm_help();*/ exit(0); } i=1; while (i<argc) { if (file_exists(argv[i])) { input=open_file(argv[i],"r"); } else if (strings_equal(argv[i],"-f")) { fmhz=atof(argv[++i]); } else if (strings_equal(argv[i],"-b")) { bw=atof(argv[++i]); } else if (strings_equal(argv[i],"-c")) { nch=atof(argv[++i]); } else if (strings_equal(argv[i],"-d")) { dm=atof(argv[++i]); } else if (strings_equal(argv[i],"-p")) { pms=atof(argv[++i]); } else if (strings_equal(argv[i],"-w")) { wms=atof(argv[++i]); } else if (strings_equal(argv[i],"-t")) { tms=atof(argv[++i]); } else if (strings_equal(argv[i],"-s")) { tsc=atof(argv[++i]); } else if (strings_equal(argv[i],"-m")) { mdm=atof(argv[++i]); } else if (strings_equal(argv[i],"-n")) { nrm=atof(argv[++i]); } else { sprintf(message,"command-line argument %s not recognized...",argv[i]); error_message(message); } i++; } } if (mdm == 0.0) mdm=2.0*dm; tdm=(float) dmdelay(fmhz-bw/nch,fmhz,dm)*1000.0; for (xdm=0.0;xdm<mdm;xdm+=0.1) { tdd=(float) dmdelay(fmhz-bw/2.0,fmhz+bw/2.0,abs(dm-xdm))*1000.0; wef=sqrt(wms*wms+tsc*tsc+tms*tms+tdm*tdm+tdd*tdd); snr=sqrt((pms-wef)/wef); snrmax=(snr>snrmax)?snr:snrmax; } printf("#Signal-to-noise/DM curve assuming following parameters:\n"); fprintf(stderr,"#Pulse Period %.3f ms\n",pms); fprintf(stderr,"#Intrinsic Width %.3f ms\n",wms); fprintf(stderr,"#Scattering Time %.3f ms\n",tsc); fprintf(stderr,"#True DM %.3f pc/cc\n",dm); fprintf(stderr,"#Centre Frequency %.3f MHz\n",fmhz); fprintf(stderr,"#Bandwidth %.3f MHz\n",bw); fprintf(stderr,"#Number of Channels %d\n",nch); printf("#START\n"); for (xdm=0.0;xdm<mdm;xdm+=0.1) { tdd=(float) dmdelay(fmhz-bw/2.0,fmhz+bw/2.0,abs(dm-xdm))*1000.0; wef=sqrt(wms*wms+tsc*tsc+tms*tms+tdm*tdm+tdd*tdd); if (wef>pms) snr=0.0; else snr=nrm*sqrt((pms-wef)/wef)/snrmax; printf("%.2f %f\n",xdm,snr); } printf("#STOP\n"); printf("#DONE\n"); }
// Initliase MDSM parameters, return poiinter to input buffer where // input data will be stored void initialiseMDSM(SURVEY* input_survey) { unsigned i, j, k; // Initialise survey survey = input_survey; // Initialise devices devices = call_initialise_devices(input_survey); // Calculate temporary DM-shifts, maxshift and nsamp per beam unsigned greatest_maxshift = 0; for (i = 0; i < survey -> nbeams; i++) { // Calculate temporary shifts BEAM *beam = &(survey -> beams[i]); beam -> dm_shifts = (float *) safeMalloc(survey -> nchans * sizeof(float)); for (j = 0; j < survey -> nchans; j++) beam -> dm_shifts[j] = dmdelay(beam -> fch1 + (beam -> foff * j), beam ->fch1); // Calculate maxshift float high_dm = survey -> lowdm + survey -> dmstep * (survey -> tdms - 1); unsigned maxshift = beam -> dm_shifts[survey -> nchans - 1] * high_dm / survey -> tsamp; greatest_maxshift = ( maxshift > greatest_maxshift) ? maxshift : greatest_maxshift; // Round up maxshift to the nearest multiple of 256 greatest_maxshift = (greatest_maxshift / 256) * 256 + 256; } // TEMPORARY: ASSIGN ALL BEAMS SAME MAXSHIFT for(i = 0; i < survey -> nbeams; i++) { BEAM *beam = &(survey -> beams[i]); beam -> maxshift = greatest_maxshift; } printf("============================================================================\n"); // Calculate global nsamp for all beams size_t inputsize, outputsize; survey -> nsamp = calculate_nsamp(greatest_maxshift, &inputsize, &outputsize, devices -> minTotalGlobalMem); // When beamforming, antenna data will be copied to the output buffer, so it's size // should be large enough to accommodate both if (survey -> apply_beamforming) { if (survey -> nantennas == 0) { printf("Number of antennas should not be 0!\n"); exit(0); } size_t beamforming_size = survey -> nantennas * survey -> nchans * survey -> nsamp * sizeof(unsigned char); outputsize = max(outputsize, beamforming_size); printf("Memory Required (per GPU): Input = %d MB; Output = %d MB\n", (int) (survey -> nbeams * inputsize / 1024 / 1024 / survey -> num_gpus), (int) (outputsize/1024/1024)); } else { printf("Memory Required (per beam): Input = %d MB; Output = %d MB\n", (int) (inputsize / 1024 / 1024), (int) (outputsize/1024/1024)); } // Allocate input buffer (MDSM_STAGES separate buffer to allow dumping to disk // during any iteration) input_buffer = (float **) safeMalloc(MDSM_STAGES * sizeof(float *)); for(i = 0; i < MDSM_STAGES; i++) input_buffer[i] = (float *) safeMalloc(survey -> nbeams * survey -> nsamp * survey -> nchans * sizeof(float)); // Allocate output buffer (one for each beam) output_buffer = (float **) safeMalloc(survey -> nbeams * sizeof(float *)); // Allocate antenna buffer if beamforming antenna_buffer = NULL; if (survey -> apply_beamforming) antenna_buffer = (unsigned char *) malloc(survey -> nantennas * survey -> nchans * survey -> nsamp * sizeof(unsigned char)); // Create writer buffer // TODO: Provide kernel suggestions for high speed I/O writer_buffer = (float *) safeMalloc(survey -> nbeams * survey -> nsamp * survey -> nchans * sizeof(float)); // Log parameters printf("Observation Params: ndms = %d, maxDM = %f\n", survey -> tdms, survey -> lowdm + survey -> dmstep * (survey -> tdms - 1)); printf("Observation Params: nchans = %d, nsamp = %d, tsamp = %f\n", survey -> nchans, survey -> nsamp, survey -> tsamp); printf("Beam Params:\n"); for(i = 0; i < survey -> nbeams; i++) { BEAM beam = survey -> beams[i]; printf(" Beam %d: fch1 = %f, foff = %f, maxshift = %d, gpuID = %d\n", beam.beam_id, beam.fch1, beam.foff, beam.maxshift, beam.gpu_id); } printf("\n"); if (survey -> apply_beamforming) printf("- Performing beamforming\n"); if (survey -> apply_rfi_clipper ) printf("- Clipping RFI\n"); printf("- Performing dedispersion\n"); if (survey -> apply_detrending ) printf("- Performing detrending\n"); if (survey -> apply_median_filter ) printf("- Performing median filtering\n"); if (survey -> dump_to_disk) printf("- Dump to disk mode enabled\n"); if (survey -> apply_clustering) printf("- Applying DBSCAN\n"); if (survey -> tbb_enabled) printf("- TBB mode enabled\n"); printf("\n"); printf("============================================================================\n"); // Initialise processing threads pthread_attr_init(&thread_attr); pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE); survey -> num_threads = survey -> nbeams; threads = (pthread_t *) calloc(sizeof(pthread_t), survey -> num_threads); threads_params = (THREAD_PARAMS **) safeMalloc(survey -> num_threads * sizeof(THREAD_PARAMS*)); for(i = 0; i < survey -> num_threads; i++) threads_params[i] = (THREAD_PARAMS *) safeMalloc(sizeof(THREAD_PARAMS)); // Initialise barriers if (pthread_barrier_init(&input_barrier, NULL, survey -> num_threads + 2)) { fprintf(stderr, "Unable to initialise input barrier\n"); exit(0); } if (pthread_barrier_init(&output_barrier, NULL, survey -> num_threads + 2)) { fprintf(stderr, "Unable to initialise output barrier\n"); exit(0); } // Create GPU objects and allocate beams/threads to them GPU **gpus = (GPU **) malloc(survey -> num_gpus * sizeof(GPU *)); for(i = 0; i < survey -> num_gpus; i++) { gpus[i] = (GPU *) malloc(sizeof(GPU)); gpus[i] -> device_id = survey -> gpu_ids[i]; gpus[i] -> num_threads = 0; gpus[i] -> primary_thread = 0; gpus[i] -> thread_ids = (unsigned *) malloc(8 * sizeof(unsigned)); } // Allocate GPUs to beams (split beams among GPUs, one beam cannot be processed on more than 1 GPU) for(i = 0; i < survey -> num_threads; i++) { unsigned gpu = (survey -> gpu_ids)[i % survey -> num_gpus]; // Update GPU properties for(j = 0; j < survey -> num_gpus; j++) { if (gpus[j] -> device_id == gpu) { if (gpus[j] -> num_threads == 0) gpus[j] -> primary_thread = i; threads_params[i] -> gpu_index = j; gpus[j] -> thread_ids[gpus[j] -> num_threads] = i; gpus[j] -> num_threads++; break; } } } // Initialise GPU barriers for(i = 0; i < survey -> num_gpus; i++) if (pthread_barrier_init(&(gpus[i] -> barrier), NULL, gpus[i] -> num_threads)) { fprintf(stderr, "Unable to initialise input barrier\n"); exit(0); } // Create output params and output file output_params.nthreads = survey -> num_threads; output_params.iterations = 3; output_params.maxiters = 2; output_params.output_buffer = output_buffer; output_params.input_buffer = input_buffer; output_params.stop = 0; output_params.rw_lock = &rw_lock; output_params.input_barrier = &input_barrier; output_params.output_barrier = &output_barrier; output_params.start = start; output_params.survey = survey; output_params.writer_mutex = &writer_mutex; output_params.writer_buffer = writer_buffer; output_params.writer_params = &writer_params; // Create output thread if (pthread_create(&output_thread, &thread_attr, process_output, (void *) &output_params)) { fprintf(stderr, "Error occured while creating output thread\n"); exit(0); } // Create threads and assign devices for(k = 0; k < survey -> num_threads; k++) { // Create THREAD_PARAMS for thread, based on input data and DEVICE_INFO // Input and output buffer pointers will point to beginning of beam threads_params[k] -> iterations = 2; threads_params[k] -> maxiters = 2; threads_params[k] -> stop = 0; threads_params[k] -> output = output_buffer; threads_params[k] -> input = input_buffer; threads_params[k] -> antenna_buffer = antenna_buffer; threads_params[k] -> thread_num = k; threads_params[k] -> num_threads = survey -> num_threads; threads_params[k] -> rw_lock = &rw_lock; threads_params[k] -> input_barrier = &input_barrier; threads_params[k] -> output_barrier = &output_barrier; threads_params[k] -> start = start; threads_params[k] -> survey = survey; threads_params[k] -> inputsize = inputsize; threads_params[k] -> outputsize = outputsize; threads_params[k] -> gpus = gpus; threads_params[k] -> cpu_threads = threads_params; // Create thread (using function in dedispersion_thread) if (pthread_create(&threads[k], &thread_attr, call_dedisperse, (void *) threads_params[k])) { fprintf(stderr, "Error occured while creating thread\n"); exit(0); } } // If we are writing to file, initialise and launch writer thread if (survey -> tbb_enabled || survey -> dump_to_disk) { // Initialiase and start the Data Writer thread, if required writer_params.survey = survey; writer_params.start = start; writer_params.stop = 0; writer_params.writer_buffer = writer_buffer; writer_params.writer_mutex = &writer_mutex; writer_params.create_new_file = false; writer_params.data_available = false; if (pthread_create(&writer_thread, &thread_attr, write_to_disk, (void *) &writer_params)) { fprintf(stderr, "Error occured while creating thread\n"); exit(0); } } // If we're writing all the incoming data stream, it takes precedence over TBB if (survey -> dump_to_disk) survey -> tbb_enabled = false; // Wait input barrier (for dedispersion_manager, first time) int ret = pthread_barrier_wait(&input_barrier); if (!(ret == 0 || ret == PTHREAD_BARRIER_SERIAL_THREAD)) { fprintf(stderr, "Error during barrier synchronisation\n"); exit(0); } }