void vegas_sdfits_thread(void *_args) {
/* Get args */
struct vegas_thread_args *args = (struct vegas_thread_args *)_args;
pthread_cleanup_push((void *)vegas_thread_set_finished, args);
/* Set cpu affinity */
int rv = sched_setaffinity(0, sizeof(cpu_set_t), &args->cpuset);
if (rv<0) {
vegas_error("vegas_sdfits_thread", "Error setting cpu affinity.");
perror("sched_setaffinity");
}
/* Set priority */
rv=0;
if (args->priority != 0)
{
struct sched_param priority_param;
priority_param.sched_priority = args->priority;
rv = pthread_setschedparam(pthread_self(), SCHED_FIFO, &priority_param);
}
if (rv<0) {
vegas_error("vegas_sdfits_thread", "Error setting priority level.");
perror("set_priority");
}
/* Attach to status shared mem area */
struct vegas_status st;
rv = vegas_status_attach(&st);
if (rv!=VEGAS_OK) {
vegas_error("vegas_sdfits_thread",
"Error attaching to status shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)vegas_status_detach, &st);
pthread_cleanup_push((void *)set_exit_status, &st);
/* Init status */
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "init");
vegas_status_unlock_safe(&st);
/* Initialize some key parameters */
struct vegas_params gp;
struct sdfits sf;
sf.data_columns.data = NULL;
sf.filenum = 0;
sf.new_file = 1; // This is crucial
pthread_cleanup_push((void *)vegas_free_sdfits, &sf);
pthread_cleanup_push((void *)sdfits_close, &sf);
//pf.multifile = 0; // Use a single file for fold mode
sf.multifile = 1; // Use a multiple files for fold mode
sf.quiet = 0; // Print a message per each subint written
/* Attach to databuf shared mem */
struct vegas_databuf *db;
db = vegas_databuf_attach(args->input_buffer);
if (db==NULL) {
vegas_error("vegas_sdfits_thread",
"Error attaching to databuf shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)vegas_databuf_detach, db);
/* Loop */
int curblock=0, total_status=0, firsttime=1, run=1, got_packet_0=0, dataset=0;
char *ptr;
char tmpstr[256];
int scan_finished=0, old_filenum;
int num_exposures_written = 0;
int old_integ_num = -1;
signal(SIGINT, cc);
do {
/* Note waiting status */
vegas_status_lock_safe(&st);
if (got_packet_0)
sprintf(tmpstr, "waiting(%d)", curblock);
else
sprintf(tmpstr, "ready");
hputs(st.buf, STATUS_KEY, tmpstr);
vegas_status_unlock_safe(&st);
/* Wait for buf to have data */
rv = vegas_databuf_wait_filled(db, curblock);
if (rv!=0) {
// This is a big ol' kludge to avoid this process hanging
// due to thread synchronization problems.
sleep(1);
continue;
}
/* Note current block */
vegas_status_lock_safe(&st);
hputi4(st.buf, "DSKBLKIN", curblock);
vegas_status_unlock_safe(&st);
/* See how full databuf is */
total_status = vegas_databuf_total_status(db);
/* Read param structs for this block */
ptr = vegas_databuf_header(db, curblock);
if (firsttime) {
vegas_read_obs_params(ptr, &gp, &sf);
firsttime = 0;
} else {
vegas_read_subint_params(ptr, &gp, &sf);
}
/* Note waiting status */
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "writing");
vegas_status_unlock_safe(&st);
struct sdfits_data_columns* data_cols;
struct databuf_index* db_index;
db_index = (struct databuf_index*)(vegas_databuf_index(db, curblock));
/* Read the block index, writing each dataset to a SDFITS file */
for(dataset = 0; dataset < db_index->num_datasets; dataset++)
{
data_cols = (struct sdfits_data_columns*)(vegas_databuf_data(db, curblock) +
db_index->disk_buf[dataset].struct_offset);
sf.data_columns = *data_cols;
/* Write the data */
old_filenum = sf.filenum;
sdfits_write_subint(&sf);
/*Write new file number to shared memory*/
if(sf.filenum != old_filenum)
{
vegas_status_lock_safe(&st);
hputi4(st.buf, "FILENUM", sf.filenum);
vegas_status_unlock_safe(&st);
}
/* If a new integration number, increment the number of exposures written */
if(data_cols->integ_num != old_integ_num)
{
num_exposures_written += 1;
old_integ_num = data_cols->integ_num;
}
}
/* Indicate number of exposures written */
vegas_status_lock_safe(&st);
hputi4(st.buf, "DSKEXPWR", num_exposures_written);
vegas_status_unlock_safe(&st);
/* For debugging... */
if (gp.drop_frac > 0.0) {
printf("Block %d dropped %.3g%% of the packets\n",
sf.tot_rows, gp.drop_frac*100.0);
}
/* Mark as free */
vegas_databuf_set_free(db, curblock);
/* Go to next block */
curblock = (curblock + 1) % db->n_block;
/* Check for cancel */
pthread_testcancel();
} while (run && !scan_finished);
/* Cleanup */
pthread_exit(NULL);
pthread_cleanup_pop(0); /* Closes sdfits_close */
pthread_cleanup_pop(0); /* Closes vegas_free_sdfits */
pthread_cleanup_pop(0); /* Closes set_exit_status */
pthread_cleanup_pop(0); /* Closes set_finished */
pthread_cleanup_pop(0); /* Closes vegas_status_detach */
pthread_cleanup_pop(0); /* Closes vegas_databuf_detach */
}
void vegas_rawdisk_thread(void *_args) {
/* Set cpu affinity */
cpu_set_t cpuset, cpuset_orig;
sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig);
CPU_ZERO(&cpuset);
CPU_SET(6, &cpuset);
int rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
if (rv<0) {
vegas_error("vegas_rawdisk_thread", "Error setting cpu affinity.");
perror("sched_setaffinity");
}
/* Get args */
struct vegas_thread_args *args = (struct vegas_thread_args *)_args;
/* Set priority */
rv = setpriority(PRIO_PROCESS, 0, 0);
if (rv<0) {
vegas_error("vegas_rawdisk_thread", "Error setting priority level.");
perror("set_priority");
}
/* Attach to status shared mem area */
struct vegas_status st;
rv = vegas_status_attach(&st);
if (rv!=VEGAS_OK) {
vegas_error("vegas_rawdisk_thread",
"Error attaching to status shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)vegas_status_detach, &st);
pthread_cleanup_push((void *)set_exit_status, &st);
/* Init status */
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "init");
vegas_status_unlock_safe(&st);
/* Read in general parameters */
struct vegas_params gp;
struct sdfits sf;
pthread_cleanup_push((void *)vegas_free_sdfits, &sf);
/* Attach to databuf shared mem */
struct vegas_databuf *db;
db = vegas_databuf_attach(args->input_buffer);
if (db==NULL) {
vegas_error("vegas_rawdisk_thread",
"Error attaching to databuf shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)vegas_databuf_detach, db);
/* Init output file */
FILE *fraw = NULL;
pthread_cleanup_push((void *)safe_fclose, fraw);
/* Loop */
int blocksize=0;
int curblock=0, dataset;
int block_count=0, blocks_per_file=128, filenum=0;
int first=1;
char *ptr;
float *data_array;
struct databuf_index* db_index;
signal(SIGINT,cc);
while (run) {
/* Note waiting status */
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "waiting");
vegas_status_unlock_safe(&st);
/* Wait for buf to have data */
rv = vegas_databuf_wait_filled(db, curblock);
if (rv!=0) continue;
/* Read param struct and index for this block */
ptr = vegas_databuf_header(db, curblock);
db_index = (struct databuf_index*)(vegas_databuf_index(db, curblock));
/* If first time running */
if (first==1)
{
first = 0;
vegas_read_obs_params(ptr, &gp, &sf);
char fname[256];
sprintf(fname, "%s_%4d.raw", sf.basefilename, filenum);
fprintf(stderr, "Opening raw file '%s'\n", fname);
// TODO: check for file exist.
fraw = fopen(fname, "wb");
if (fraw==NULL) {
vegas_error("vegas_rawdisk_thread", "Error opening file.");
pthread_exit(NULL);
}
}
else
vegas_read_subint_params(ptr, &gp, &sf);
/* See if we need to open next file */
if (block_count >= blocks_per_file) {
fclose(fraw);
filenum++;
char fname[256];
sprintf(fname, "%s_%4d.raw", sf.basefilename, filenum);
fprintf(stderr, "Opening raw file '%s'\n", fname);
fraw = fopen(fname, "wb");
if (fraw==NULL) {
vegas_error("vegas_rawdisk_thread", "Error opening file.");
pthread_exit(NULL);
}
block_count=0;
}
/* Get full data block size */
hgeti4(ptr, "BLOCSIZE", &blocksize);
/* Note writing status and current block */
vegas_status_lock_safe(&st);
hputi4(st.buf, "DSKBLKIN", curblock);
hputs(st.buf, STATUS_KEY, "writing");
vegas_status_unlock_safe(&st);
/* Write all data arrays to disk */
for(dataset = 0; dataset < db_index->num_datasets; dataset++)
{
data_array = (float*)(vegas_databuf_data(db, curblock) +
db_index->disk_buf[dataset].array_offset);
rv = fwrite(data_array, 4, (size_t)(db_index->array_size/4), fraw);
if (rv != db_index->array_size/4) {
vegas_error("vegas_rawdisk_thread",
"Error writing data.");
}
}
/* Increment block counter */
block_count++;
/* flush output */
fflush(fraw);
/* Mark as free */
vegas_databuf_set_free(db, curblock);
/* Go to next block */
curblock = (curblock + 1) % db->n_block;
/* Check for cancel */
pthread_testcancel();
}
pthread_exit(NULL);
pthread_cleanup_pop(0); /* Closes fclose */
pthread_cleanup_pop(0); /* Closes vegas_databuf_detach */
pthread_cleanup_pop(0); /* Closes vegas_free_psrfits */
pthread_cleanup_pop(0); /* Closes set_exit_status */
pthread_cleanup_pop(0); /* Closes vegas_status_detach */
}
void vegas_null_thread(void *_args) {
int rv;
/* Set cpu affinity */
cpu_set_t cpuset, cpuset_orig;
sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig);
CPU_ZERO(&cpuset);
CPU_SET(6, &cpuset);
rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
if (rv<0) {
vegas_error("vegas_null_thread", "Error setting cpu affinity.");
perror("sched_setaffinity");
}
/* Set priority */
rv = setpriority(PRIO_PROCESS, 0, 0);
if (rv<0) {
vegas_error("vegas_null_thread", "Error setting priority level.");
perror("set_priority");
}
/* Get args */
struct vegas_thread_args *args = (struct vegas_thread_args *)_args;
/* Attach to status shared mem area */
struct vegas_status st;
rv = vegas_status_attach(&st);
if (rv!=VEGAS_OK) {
vegas_error("vegas_null_thread",
"Error attaching to status shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)vegas_status_detach, &st);
pthread_cleanup_push((void *)set_exit_status, &st);
/* Init status */
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "init");
vegas_status_unlock_safe(&st);
/* Attach to databuf shared mem */
struct vegas_databuf *db;
db = vegas_databuf_attach(args->input_buffer);
if (db==NULL) {
vegas_error("vegas_null_thread",
"Error attaching to databuf shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)vegas_databuf_detach, db);
/* Loop */
char *ptr;
struct vegas_params gp;
#if FITS_TYPE == PSRFITS
struct psrfits pf;
pf.sub.dat_freqs = NULL;
pf.sub.dat_weights = NULL;
pf.sub.dat_offsets = NULL;
pf.sub.dat_scales = NULL;
pthread_cleanup_push((void *)vegas_free_psrfits, &pf);
#else
struct sdfits pf;
pthread_cleanup_push((void *)vegas_free_sdfits, &pf);
#endif
int curblock=0;
signal(SIGINT,cc);
while (run) {
/* Note waiting status */
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "waiting");
vegas_status_unlock_safe(&st);
/* Wait for buf to have data */
rv = vegas_databuf_wait_filled(db, curblock);
if (rv!=0) {
//sleep(1);
continue;
}
/* Note waiting status, current block */
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "discarding");
hputi4(st.buf, "DSKBLKIN", curblock);
vegas_status_unlock_safe(&st);
/* Get params */
ptr = vegas_databuf_header(db, curblock);
vegas_read_obs_params(ptr, &gp, &pf);
/* Output if data was lost */
#if FITS_TYPE == PSRFITS
if (gp.n_dropped!=0 &&
(gp.packetindex==0 || strcmp(pf.hdr.obs_mode,"SEARCH"))) {
printf("Block beginning with pktidx=%lld dropped %d packets\n",
gp.packetindex, gp.n_dropped);
fflush(stdout);
}
#else
if (gp.num_pkts_dropped!=0 && gp.num_pkts_rcvd!=0) {
printf("Block received %d packets and dropped %d packets\n",
gp.num_pkts_rcvd, gp.num_pkts_dropped);
fflush(stdout);
}
#endif
/* Mark as free */
vegas_databuf_set_free(db, curblock);
/* Go to next block */
curblock = (curblock + 1) % db->n_block;
/* Check for cancel */
pthread_testcancel();
}
pthread_exit(NULL);
pthread_cleanup_pop(0); /* Closes set_exit_status */
pthread_cleanup_pop(0); /* Closes vegas_free_psrfits */
pthread_cleanup_pop(0); /* Closes vegas_status_detach */
pthread_cleanup_pop(0); /* Closes vegas_databuf_detach */
}
void vegas_pfb_thread(void *_args) {
/* Get args */
struct vegas_thread_args *args = (struct vegas_thread_args *)_args;
int rv;
/* Set cpu affinity */
cpu_set_t cpuset, cpuset_orig;
sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig);
//CPU_ZERO(&cpuset);
CPU_CLR(13, &cpuset);
CPU_SET(11, &cpuset);
rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
if (rv<0) {
vegas_error("vegas_pfb_thread", "Error setting cpu affinity.");
perror("sched_setaffinity");
}
/* Set priority */
rv = setpriority(PRIO_PROCESS, 0, args->priority);
if (rv<0) {
vegas_error("vegas_pfb_thread", "Error setting priority level.");
perror("set_priority");
}
/* Attach to status shared mem area */
struct vegas_status st;
rv = vegas_status_attach(&st);
if (rv!=VEGAS_OK) {
vegas_error("vegas_pfb_thread",
"Error attaching to status shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)vegas_status_detach, &st);
pthread_cleanup_push((void *)set_exit_status, &st);
pthread_cleanup_push((void *)vegas_thread_set_finished, args);
/* Init status */
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "init");
vegas_status_unlock_safe(&st);
/* Init structs */
struct vegas_params gp;
struct sdfits sf;
pthread_cleanup_push((void *)vegas_free_sdfits, &sf);
/* Attach to databuf shared mem */
struct vegas_databuf *db_in, *db_out;
db_in = vegas_databuf_attach(args->input_buffer);
if (db_in==NULL) {
char msg[256];
sprintf(msg, "Error attaching to databuf(%d) shared memory.",
args->input_buffer);
vegas_error("vegas_pfb_thread", msg);
pthread_exit(NULL);
}
pthread_cleanup_push((void *)vegas_databuf_detach, db_in);
db_out = vegas_databuf_attach(args->output_buffer);
if (db_out==NULL) {
char msg[256];
sprintf(msg, "Error attaching to databuf(%d) shared memory.",
args->output_buffer);
vegas_error("vegas_pfb_thread", msg);
pthread_exit(NULL);
}
pthread_cleanup_push((void *)vegas_databuf_detach, db_out);
/* Loop */
char *hdr_in = NULL;
int curblock_in=0;
int first=1;
int acc_len = 0;
int nchan = 0;
int nsubband = 0;
signal(SIGINT,cc);
vegas_status_lock_safe(&st);
if (hgeti4(st.buf, "NCHAN", &nchan)==0) {
fprintf(stderr, "ERROR: %s not in status shm!\n", "NCHAN");
}
if (hgeti4(st.buf, "NSUBBAND", &nsubband)==0) {
fprintf(stderr, "ERROR: %s not in status shm!\n", "NSUBBAND");
}
vegas_status_unlock_safe(&st);
if (EXIT_SUCCESS != init_gpu(db_in->block_size,
db_out->block_size,
nsubband,
nchan))
{
(void) fprintf(stderr, "ERROR: GPU initialisation failed!\n");
run = 0;
}
while (run) {
/* Note waiting status */
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "waiting");
vegas_status_unlock_safe(&st);
/* Wait for buf to have data */
rv = vegas_databuf_wait_filled(db_in, curblock_in);
if (rv!=0) continue;
/* Note waiting status, current input block */
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "processing");
hputi4(st.buf, "PFBBLKIN", curblock_in);
vegas_status_unlock_safe(&st);
hdr_in = vegas_databuf_header(db_in, curblock_in);
/* Get params */
if (first)
{
vegas_read_obs_params(hdr_in, &gp, &sf);
/* Read required exposure from status shared memory, and calculate
corresponding accumulation length */
acc_len = (abs(sf.hdr.chan_bw) * sf.hdr.hwexposr);
}
vegas_read_subint_params(hdr_in, &gp, &sf);
/* Call PFB function */
do_pfb(db_in, curblock_in, db_out, first, st, acc_len);
/* Mark input block as free */
vegas_databuf_set_free(db_in, curblock_in);
/* Go to next input block */
curblock_in = (curblock_in + 1) % db_in->n_block;
/* Check for cancel */
pthread_testcancel();
if (first) {
first=0;
}
}
run=0;
//cudaThreadExit();
pthread_exit(NULL);
cleanup_gpu();
pthread_cleanup_pop(0); /* Closes vegas_databuf_detach(out) */
pthread_cleanup_pop(0); /* Closes vegas_databuf_detach(in) */
pthread_cleanup_pop(0); /* Closes vegas_free_sdfits */
pthread_cleanup_pop(0); /* Closes vegas_thread_set_finished */
pthread_cleanup_pop(0); /* Closes set_exit_status */
pthread_cleanup_pop(0); /* Closes vegas_status_detach */
}
