int *dmshift(double f1, double df, int nchans, int nbands, double dm, double refrf, double tsamp, double frequency_table[]) /*includefile*/
{
int i, cpb, *shift;
double fi;
shift = (int *) malloc(nchans * sizeof(int));
fi=f1;
cpb=nchans/nbands;
if (frequency_table[0] != 0.0) f1=frequency_table[0];
for (i=0; i<nchans; i++) {
if (refrf > 0.0) f1=refrf;
if (frequency_table[0] != 0.0) fi=frequency_table[i];
shift[i]=(int)(dmdelay(fi,f1,dm)/tsamp);
fi+=df;
if (!((i+1)%cpb)) f1+=(double) cpb * df;
}
return shift;
}
main(int argc, char **argv)
{
char message[80];
int i;
float fmhz=430.0, bw=7.68, nch=128.0, dm=50.0, pms=150.0,
wms=5.0, tms=0.08, tsc=0.0, mdm=0.0, nrm=100.0;
float xdm, wef, snr, tdm, tdd, snrmax=0.0;
FILE *input;
if (argc > 1) {
print_version(argv[0],argv[1]);
if (help_required(argv[1])) {
/*snrdm_help();*/
exit(0);
}
i=1;
while (i<argc) {
if (file_exists(argv[i])) {
input=open_file(argv[i],"r");
} else if (strings_equal(argv[i],"-f")) {
fmhz=atof(argv[++i]);
} else if (strings_equal(argv[i],"-b")) {
bw=atof(argv[++i]);
} else if (strings_equal(argv[i],"-c")) {
nch=atof(argv[++i]);
} else if (strings_equal(argv[i],"-d")) {
dm=atof(argv[++i]);
} else if (strings_equal(argv[i],"-p")) {
pms=atof(argv[++i]);
} else if (strings_equal(argv[i],"-w")) {
wms=atof(argv[++i]);
} else if (strings_equal(argv[i],"-t")) {
tms=atof(argv[++i]);
} else if (strings_equal(argv[i],"-s")) {
tsc=atof(argv[++i]);
} else if (strings_equal(argv[i],"-m")) {
mdm=atof(argv[++i]);
} else if (strings_equal(argv[i],"-n")) {
nrm=atof(argv[++i]);
} else {
sprintf(message,"command-line argument %s not recognized...",argv[i]);
error_message(message);
}
i++;
}
}
if (mdm == 0.0) mdm=2.0*dm;
tdm=(float) dmdelay(fmhz-bw/nch,fmhz,dm)*1000.0;
for (xdm=0.0;xdm<mdm;xdm+=0.1) {
tdd=(float) dmdelay(fmhz-bw/2.0,fmhz+bw/2.0,abs(dm-xdm))*1000.0;
wef=sqrt(wms*wms+tsc*tsc+tms*tms+tdm*tdm+tdd*tdd);
snr=sqrt((pms-wef)/wef);
snrmax=(snr>snrmax)?snr:snrmax;
}
printf("#Signal-to-noise/DM curve assuming following parameters:\n");
fprintf(stderr,"#Pulse Period %.3f ms\n",pms);
fprintf(stderr,"#Intrinsic Width %.3f ms\n",wms);
fprintf(stderr,"#Scattering Time %.3f ms\n",tsc);
fprintf(stderr,"#True DM %.3f pc/cc\n",dm);
fprintf(stderr,"#Centre Frequency %.3f MHz\n",fmhz);
fprintf(stderr,"#Bandwidth %.3f MHz\n",bw);
fprintf(stderr,"#Number of Channels %d\n",nch);
printf("#START\n");
for (xdm=0.0;xdm<mdm;xdm+=0.1) {
tdd=(float) dmdelay(fmhz-bw/2.0,fmhz+bw/2.0,abs(dm-xdm))*1000.0;
wef=sqrt(wms*wms+tsc*tsc+tms*tms+tdm*tdm+tdd*tdd);
if (wef>pms)
snr=0.0;
else
snr=nrm*sqrt((pms-wef)/wef)/snrmax;
printf("%.2f %f\n",xdm,snr);
}
printf("#STOP\n");
printf("#DONE\n");
}
// Initliase MDSM parameters, return poiinter to input buffer where
// input data will be stored
void initialiseMDSM(SURVEY* input_survey)
{
unsigned i, j, k;
// Initialise survey
survey = input_survey;
// Initialise devices
devices = call_initialise_devices(input_survey);
// Calculate temporary DM-shifts, maxshift and nsamp per beam
unsigned greatest_maxshift = 0;
for (i = 0; i < survey -> nbeams; i++)
{
// Calculate temporary shifts
BEAM *beam = &(survey -> beams[i]);
beam -> dm_shifts = (float *) safeMalloc(survey -> nchans * sizeof(float));
for (j = 0; j < survey -> nchans; j++)
beam -> dm_shifts[j] = dmdelay(beam -> fch1 + (beam -> foff * j), beam ->fch1);
// Calculate maxshift
float high_dm = survey -> lowdm + survey -> dmstep * (survey -> tdms - 1);
unsigned maxshift = beam -> dm_shifts[survey -> nchans - 1] * high_dm / survey -> tsamp;
greatest_maxshift = ( maxshift > greatest_maxshift)
? maxshift : greatest_maxshift;
// Round up maxshift to the nearest multiple of 256
greatest_maxshift = (greatest_maxshift / 256) * 256 + 256;
}
// TEMPORARY: ASSIGN ALL BEAMS SAME MAXSHIFT
for(i = 0; i < survey -> nbeams; i++)
{
BEAM *beam = &(survey -> beams[i]);
beam -> maxshift = greatest_maxshift;
}
printf("============================================================================\n");
// Calculate global nsamp for all beams
size_t inputsize, outputsize;
survey -> nsamp = calculate_nsamp(greatest_maxshift, &inputsize, &outputsize,
devices -> minTotalGlobalMem);
// When beamforming, antenna data will be copied to the output buffer, so it's size
// should be large enough to accommodate both
if (survey -> apply_beamforming)
{
if (survey -> nantennas == 0)
{
printf("Number of antennas should not be 0!\n");
exit(0);
}
size_t beamforming_size = survey -> nantennas * survey -> nchans * survey -> nsamp * sizeof(unsigned char);
outputsize = max(outputsize, beamforming_size);
printf("Memory Required (per GPU): Input = %d MB; Output = %d MB\n",
(int) (survey -> nbeams * inputsize / 1024 / 1024 / survey -> num_gpus), (int) (outputsize/1024/1024));
}
else
{
printf("Memory Required (per beam): Input = %d MB; Output = %d MB\n",
(int) (inputsize / 1024 / 1024), (int) (outputsize/1024/1024));
}
// Allocate input buffer (MDSM_STAGES separate buffer to allow dumping to disk
// during any iteration)
input_buffer = (float **) safeMalloc(MDSM_STAGES * sizeof(float *));
for(i = 0; i < MDSM_STAGES; i++)
input_buffer[i] = (float *) safeMalloc(survey -> nbeams * survey -> nsamp *
survey -> nchans * sizeof(float));
// Allocate output buffer (one for each beam)
output_buffer = (float **) safeMalloc(survey -> nbeams * sizeof(float *));
// Allocate antenna buffer if beamforming
antenna_buffer = NULL;
if (survey -> apply_beamforming)
antenna_buffer = (unsigned char *) malloc(survey -> nantennas * survey -> nchans * survey -> nsamp * sizeof(unsigned char));
// Create writer buffer
// TODO: Provide kernel suggestions for high speed I/O
writer_buffer = (float *) safeMalloc(survey -> nbeams * survey -> nsamp *
survey -> nchans * sizeof(float));
// Log parameters
printf("Observation Params: ndms = %d, maxDM = %f\n", survey -> tdms,
survey -> lowdm + survey -> dmstep * (survey -> tdms - 1));
printf("Observation Params: nchans = %d, nsamp = %d, tsamp = %f\n",
survey -> nchans, survey -> nsamp, survey -> tsamp);
printf("Beam Params:\n");
for(i = 0; i < survey -> nbeams; i++)
{
BEAM beam = survey -> beams[i];
printf(" Beam %d: fch1 = %f, foff = %f, maxshift = %d, gpuID = %d\n",
beam.beam_id, beam.fch1, beam.foff, beam.maxshift, beam.gpu_id);
}
printf("\n");
if (survey -> apply_beamforming) printf("- Performing beamforming\n");
if (survey -> apply_rfi_clipper ) printf("- Clipping RFI\n");
printf("- Performing dedispersion\n");
if (survey -> apply_detrending ) printf("- Performing detrending\n");
if (survey -> apply_median_filter ) printf("- Performing median filtering\n");
if (survey -> dump_to_disk) printf("- Dump to disk mode enabled\n");
if (survey -> apply_clustering) printf("- Applying DBSCAN\n");
if (survey -> tbb_enabled) printf("- TBB mode enabled\n");
printf("\n");
printf("============================================================================\n");
// Initialise processing threads
pthread_attr_init(&thread_attr);
pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
survey -> num_threads = survey -> nbeams;
threads = (pthread_t *) calloc(sizeof(pthread_t), survey -> num_threads);
threads_params = (THREAD_PARAMS **) safeMalloc(survey -> num_threads * sizeof(THREAD_PARAMS*));
for(i = 0; i < survey -> num_threads; i++)
threads_params[i] = (THREAD_PARAMS *) safeMalloc(sizeof(THREAD_PARAMS));
// Initialise barriers
if (pthread_barrier_init(&input_barrier, NULL, survey -> num_threads + 2))
{ fprintf(stderr, "Unable to initialise input barrier\n"); exit(0); }
if (pthread_barrier_init(&output_barrier, NULL, survey -> num_threads + 2))
{ fprintf(stderr, "Unable to initialise output barrier\n"); exit(0); }
// Create GPU objects and allocate beams/threads to them
GPU **gpus = (GPU **) malloc(survey -> num_gpus * sizeof(GPU *));
for(i = 0; i < survey -> num_gpus; i++)
{
gpus[i] = (GPU *) malloc(sizeof(GPU));
gpus[i] -> device_id = survey -> gpu_ids[i];
gpus[i] -> num_threads = 0;
gpus[i] -> primary_thread = 0;
gpus[i] -> thread_ids = (unsigned *) malloc(8 * sizeof(unsigned));
}
// Allocate GPUs to beams (split beams among GPUs, one beam cannot be processed on more than 1 GPU)
for(i = 0; i < survey -> num_threads; i++)
{
unsigned gpu = (survey -> gpu_ids)[i % survey -> num_gpus];
// Update GPU properties
for(j = 0; j < survey -> num_gpus; j++)
{
if (gpus[j] -> device_id == gpu)
{
if (gpus[j] -> num_threads == 0) gpus[j] -> primary_thread = i;
threads_params[i] -> gpu_index = j;
gpus[j] -> thread_ids[gpus[j] -> num_threads] = i;
gpus[j] -> num_threads++;
break;
}
}
}
// Initialise GPU barriers
for(i = 0; i < survey -> num_gpus; i++)
if (pthread_barrier_init(&(gpus[i] -> barrier), NULL, gpus[i] -> num_threads))
{ fprintf(stderr, "Unable to initialise input barrier\n"); exit(0); }
// Create output params and output file
output_params.nthreads = survey -> num_threads;
output_params.iterations = 3;
output_params.maxiters = 2;
output_params.output_buffer = output_buffer;
output_params.input_buffer = input_buffer;
output_params.stop = 0;
output_params.rw_lock = &rw_lock;
output_params.input_barrier = &input_barrier;
output_params.output_barrier = &output_barrier;
output_params.start = start;
output_params.survey = survey;
output_params.writer_mutex = &writer_mutex;
output_params.writer_buffer = writer_buffer;
output_params.writer_params = &writer_params;
// Create output thread
if (pthread_create(&output_thread, &thread_attr, process_output, (void *) &output_params))
{ fprintf(stderr, "Error occured while creating output thread\n"); exit(0); }
// Create threads and assign devices
for(k = 0; k < survey -> num_threads; k++) {
// Create THREAD_PARAMS for thread, based on input data and DEVICE_INFO
// Input and output buffer pointers will point to beginning of beam
threads_params[k] -> iterations = 2;
threads_params[k] -> maxiters = 2;
threads_params[k] -> stop = 0;
threads_params[k] -> output = output_buffer;
threads_params[k] -> input = input_buffer;
threads_params[k] -> antenna_buffer = antenna_buffer;
threads_params[k] -> thread_num = k;
threads_params[k] -> num_threads = survey -> num_threads;
threads_params[k] -> rw_lock = &rw_lock;
threads_params[k] -> input_barrier = &input_barrier;
threads_params[k] -> output_barrier = &output_barrier;
threads_params[k] -> start = start;
threads_params[k] -> survey = survey;
threads_params[k] -> inputsize = inputsize;
threads_params[k] -> outputsize = outputsize;
threads_params[k] -> gpus = gpus;
threads_params[k] -> cpu_threads = threads_params;
// Create thread (using function in dedispersion_thread)
if (pthread_create(&threads[k], &thread_attr, call_dedisperse, (void *) threads_params[k]))
{ fprintf(stderr, "Error occured while creating thread\n"); exit(0); }
}
// If we are writing to file, initialise and launch writer thread
if (survey -> tbb_enabled || survey -> dump_to_disk)
{
// Initialiase and start the Data Writer thread, if required
writer_params.survey = survey;
writer_params.start = start;
writer_params.stop = 0;
writer_params.writer_buffer = writer_buffer;
writer_params.writer_mutex = &writer_mutex;
writer_params.create_new_file = false;
writer_params.data_available = false;
if (pthread_create(&writer_thread, &thread_attr, write_to_disk, (void *) &writer_params))
{ fprintf(stderr, "Error occured while creating thread\n"); exit(0); }
}
// If we're writing all the incoming data stream, it takes precedence over TBB
if (survey -> dump_to_disk)
survey -> tbb_enabled = false;
// Wait input barrier (for dedispersion_manager, first time)
int ret = pthread_barrier_wait(&input_barrier);
if (!(ret == 0 || ret == PTHREAD_BARRIER_SERIAL_THREAD))
{ fprintf(stderr, "Error during barrier synchronisation\n"); exit(0); }
}
