// Method to initialize the block_info_t structure
static inline void initialize_block_info(block_info_t * binfo) {
// If already initialized, return
if (binfo->initialized) {
return;
}
// Initialize our XID
binfo->self_xid = -1;
hashpipe_status_lock_safe(st_p);
hgeti4(st_p->buf, "XID", &binfo->self_xid);
hashpipe_status_unlock_safe(st_p);
// Initialize packet counters
int i;
for (i = 0; i < N_INPUT_BLOCKS; i++) {
binfo->packet_count[i] = 0;
}
// Initialize rest
binfo->mcnt_start = 0;
binfo->block_i = 0;
binfo->initialized = 0;
}
/* This thread is passed a single arg, pointer
* to the vegas_udp_params struct. This thread should
* be cancelled and restarted if any hardware params
* change, as this potentially affects packet size, etc.
*/
void *vegas_net_thread(void *_args) {
/* Get arguments */
struct vegas_thread_args *args = (struct vegas_thread_args *)_args;
int rv;
/* Set cpu affinity */
cpu_set_t cpuset, cpuset_orig;
sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig);
//CPU_ZERO(&cpuset);
CPU_SET(13, &cpuset);
rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
if (rv<0) {
vegas_error("vegas_net_thread", "Error setting cpu affinity.");
perror("sched_setaffinity");
}
/* Set priority */
rv = setpriority(PRIO_PROCESS, 0, args->priority);
if (rv<0) {
vegas_error("vegas_net_thread", "Error setting priority level.");
perror("set_priority");
}
/* Attach to status shared mem area */
struct vegas_status st;
rv = vegas_status_attach(&st);
if (rv!=VEGAS_OK) {
vegas_error("vegas_net_thread",
"Error attaching to status shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)vegas_status_detach, &st);
pthread_cleanup_push((void *)set_exit_status, &st);
/* Init status, read info */
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "init");
vegas_status_unlock_safe(&st);
/* Read in general parameters */
struct vegas_params gp;
struct sdfits pf;
char status_buf[VEGAS_STATUS_SIZE];
vegas_status_lock_safe(&st);
memcpy(status_buf, st.buf, VEGAS_STATUS_SIZE);
vegas_status_unlock_safe(&st);
vegas_read_obs_params(status_buf, &gp, &pf);
pthread_cleanup_push((void *)vegas_free_sdfits, &pf);
/* Read network params */
struct vegas_udp_params up;
vegas_read_net_params(status_buf, &up);
/* Attach to databuf shared mem */
struct vegas_databuf *db;
db = vegas_databuf_attach(args->output_buffer);
if (db==NULL) {
vegas_error("vegas_net_thread",
"Error attaching to databuf shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)vegas_databuf_detach, db);
/* Time parameters */
double meas_stt_mjd=0.0;
double meas_stt_offs=0.0;
/* See which packet format to use */
int nchan=0, npol=0;
nchan = pf.hdr.nchan;
npol = pf.hdr.npol;
/* Figure out size of data in each packet, number of packets
* per block, etc. Changing packet size during an obs is not
* recommended.
*/
int block_size;
struct vegas_udp_packet p;
size_t heap_size, spead_hdr_size;
unsigned int heaps_per_block, packets_per_heap;
char bw_mode[16];
if (hgets(status_buf, "BW_MODE", 16, bw_mode))
{
if(strncmp(bw_mode, "high", 4) == 0)
{
heap_size = sizeof(struct freq_spead_heap) + nchan*4*sizeof(int);
spead_hdr_size = sizeof(struct freq_spead_heap);
packets_per_heap = nchan*4*sizeof(int) / PAYLOAD_SIZE;
}
else if(strncmp(bw_mode, "low", 3) == 0)
{
heap_size = sizeof(struct time_spead_heap) + PAYLOAD_SIZE;
spead_hdr_size = sizeof(struct time_spead_heap);
packets_per_heap = 1;
}
else
vegas_error("vegas_net_thread", "Unsupported bandwidth mode");
}
else
vegas_error("vegas_net_thread", "BW_MODE not set");
if (hgeti4(status_buf, "BLOCSIZE", &block_size)==0) {
block_size = db->block_size;
hputi4(status_buf, "BLOCSIZE", block_size);
} else {
if (block_size > db->block_size) {
vegas_error("vegas_net_thread", "BLOCSIZE > databuf block_size");
block_size = db->block_size;
hputi4(status_buf, "BLOCSIZE", block_size);
}
}
heaps_per_block = (block_size - MAX_HEAPS_PER_BLK*spead_hdr_size) /
(heap_size - spead_hdr_size);
/* List of databuf blocks currently in use */
unsigned i;
const int nblock = 2;
struct datablock_stats blocks[nblock];
for (i=0; i<nblock; i++)
init_block(&blocks[i], db, heap_size, spead_hdr_size, heaps_per_block);
/* Convenience names for first/last blocks in set */
struct datablock_stats *fblock, *lblock;
fblock = &blocks[0];
lblock = &blocks[nblock-1];
/* Misc counters, etc */
char *curdata=NULL, *curheader=NULL, *curindex=NULL;
unsigned int heap_cntr=0, last_heap_cntr=2048, nextblock_heap_cntr=0;
unsigned int heap_offset;
unsigned int seq_num=0, last_seq_num=1050;
int heap_cntr_diff, seq_num_diff;
unsigned int obs_started = 0;
unsigned long long npacket_total, npacket_this_block=0, ndropped_total;
double drop_frac_avg=0.0;
const double drop_lpf = 0.25;
prev_heap_cntr = 0;
prev_heap_offset = 0;
char msg[256];
/* Give all the threads a chance to start before opening network socket */
sleep(1);
/* Set up UDP socket */
rv = vegas_udp_init(&up);
if (rv!=VEGAS_OK) {
vegas_error("vegas_net_thread",
"Error opening UDP socket.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)vegas_udp_close, &up);
/* Main loop */
unsigned force_new_block=0, waiting=-1;
signal(SIGINT,cc);
while (run) {
/* Wait for data */
rv = vegas_udp_wait(&up);
if (rv!=VEGAS_OK) {
if (rv==VEGAS_TIMEOUT) {
/* Set "waiting" flag */
if (waiting!=1) {
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "waiting");
vegas_status_unlock_safe(&st);
waiting=1;
}
continue;
} else {
vegas_error("vegas_net_thread",
"vegas_udp_wait returned error");
perror("vegas_udp_wait");
pthread_exit(NULL);
}
}
/* Read packet */
rv = vegas_udp_recv(&up, &p, bw_mode);
if (rv!=VEGAS_OK) {
if (rv==VEGAS_ERR_PACKET) {
#ifdef DEBUG_NET
vegas_warn("vegas_net_thread", "Incorrect pkt size");
#endif
continue;
} else {
vegas_error("vegas_net_thread",
"vegas_udp_recv returned error");
perror("vegas_udp_recv");
pthread_exit(NULL);
}
}
/* Update status if needed */
if (waiting!=0) {
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "receiving");
vegas_status_unlock_safe(&st);
waiting=0;
}
/* Check seq num diff */
heap_cntr = vegas_spead_packet_heap_cntr(&p);
heap_offset = vegas_spead_packet_heap_offset(&p);
seq_num = vegas_spead_packet_seq_num(heap_cntr, heap_offset, packets_per_heap);
heap_cntr_diff = heap_cntr - last_heap_cntr;
seq_num_diff = (int)(seq_num - last_seq_num);
last_seq_num = seq_num;
last_heap_cntr = heap_cntr;
if (seq_num_diff<=0) {
if (seq_num_diff<-1024)
{
force_new_block=1;
obs_started = 1;
#ifdef DEBUG_NET
printf("Debug: observation started\n");
#endif
}
else if (seq_num_diff==0) {
sprintf(msg, "Received duplicate packet (seq_num=%d)", seq_num);
vegas_warn("vegas_net_thread", msg);
}
else {
#ifdef DEBUG_NET
sprintf(msg, "out of order packet. Diff = %d", seq_num_diff);
vegas_warn("vegas_net_thread", msg);
#endif
continue; /* No going backwards */
}
} else {
force_new_block=0;
npacket_total += seq_num_diff;
ndropped_total += seq_num_diff - 1;
npacket_this_block += seq_num_diff;
fblock->pkts_dropped += seq_num_diff - 1;
#ifdef DEBUG_NET
if(seq_num_diff > 1)
{
sprintf(msg, "Missing packet. seq_num_diff = %d", seq_num_diff);
vegas_warn("vegas_net_thread", msg);
}
#endif
}
/* If obs has not started, ignore this packet */
if(!obs_started)
{
fblock->pkts_dropped = 0;
npacket_total = 0;
ndropped_total = 0;
npacket_this_block = 0;
continue;
}
/* Determine if we go to next block */
if (heap_cntr>=nextblock_heap_cntr || force_new_block)
{
/* Update drop stats */
if (npacket_this_block > 0)
drop_frac_avg = (1.0-drop_lpf)*drop_frac_avg
+ drop_lpf *
(double)fblock->pkts_dropped / (double)npacket_this_block;
vegas_status_lock_safe(&st);
hputi8(st.buf, "NPKT", npacket_total);
hputi8(st.buf, "NDROP", ndropped_total);
hputr8(st.buf, "DROPAVG", drop_frac_avg);
hputr8(st.buf, "DROPTOT",
npacket_total ?
(double)ndropped_total/(double)npacket_total
: 0.0);
hputi4(st.buf, "NETBLKOU", fblock->block_idx);
vegas_status_unlock_safe(&st);
/* Finalize first block, and push it off the list.
* Then grab next available block.
*/
if (fblock->block_idx>=0) finalize_block(fblock);
block_stack_push(blocks, nblock);
increment_block(lblock, heap_cntr);
curdata = vegas_databuf_data(db, lblock->block_idx);
curheader = vegas_databuf_header(db, lblock->block_idx);
curindex = vegas_databuf_index(db, lblock->block_idx);
nextblock_heap_cntr = lblock->heap_idx + heaps_per_block;
npacket_this_block = 0;
/* If new obs started, reset total counters, get start
* time. Start time is rounded to nearest integer
* second, with warning if we're off that by more
* than 100ms. Any current blocks on the stack
* are also finalized/reset */
if (force_new_block) {
/* Reset stats */
npacket_total=0;
ndropped_total=0;
npacket_this_block = 0;
/* Get obs start time */
get_current_mjd_double(&meas_stt_mjd);
printf("vegas_net_thread: got start packet at MJD %f", meas_stt_mjd);
meas_stt_offs = meas_stt_mjd*24*60*60 - floor(meas_stt_mjd*24*60*60);
if(meas_stt_offs > 0.1 && meas_stt_offs < 0.9)
{
char msg[256];
sprintf(msg,
"Second fraction = %3.1f ms > +/-100 ms",
meas_stt_offs*1e3);
vegas_warn("vegas_net_thread", msg);
}
vegas_status_lock_safe(&st);
hputnr8(st.buf, "M_STTMJD", 8, meas_stt_mjd);
hputr8(st.buf, "M_STTOFF", meas_stt_offs);
vegas_status_unlock_safe(&st);
/* Warn if 1st packet number is not zero */
if (seq_num!=0) {
char msg[256];
sprintf(msg, "First packet number is not 0 (seq_num=%d)", seq_num);
vegas_warn("vegas_net_thread", msg);
}
}
/* Read current status shared mem */
vegas_status_lock_safe(&st);
memcpy(status_buf, st.buf, VEGAS_STATUS_SIZE);
vegas_status_unlock_safe(&st);
/* Wait for new block to be free, then clear it
* if necessary and fill its header with new values.
*/
while ((rv=vegas_databuf_wait_free(db, lblock->block_idx))
!= VEGAS_OK) {
if (rv==VEGAS_TIMEOUT) {
waiting=1;
vegas_warn("vegas_net_thread", "timeout while waiting for output block\n");
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "blocked");
vegas_status_unlock_safe(&st);
continue;
} else {
vegas_error("vegas_net_thread",
"error waiting for free databuf");
run=0;
pthread_exit(NULL);
break;
}
}
memcpy(curheader, status_buf, VEGAS_STATUS_SIZE);
memset(curdata, 0, block_size);
memset(curindex, 0, db->index_size);
}
/* Copy packet into any blocks where it belongs.
* The "write packets" functions also update drop stats
* for blocks, etc.
*/
int nblocks = 0;
for (i=0; i<nblock; i++)
{
if ((blocks[i].block_idx>=0) && (block_heap_check(&blocks[i],heap_cntr)==0))
{
if (nblocks > 0) {
printf("vegas_net_thread: Warning! Adding packet to more than one block! heap_cntr= %d, block = %d",heap_cntr,i);
}
nblocks++;
write_spead_packet_to_block(&blocks[i], &p, heap_cntr,
heap_offset, packets_per_heap, bw_mode);
}
}
/* Will exit if thread has been cancelled */
pthread_testcancel();
}
pthread_exit(NULL);
/* Have to close all push's */
pthread_cleanup_pop(0); /* Closes push(vegas_udp_close) */
pthread_cleanup_pop(0); /* Closes set_exit_status */
pthread_cleanup_pop(0); /* Closes vegas_free_psrfits */
pthread_cleanup_pop(0); /* Closes vegas_status_detach */
pthread_cleanup_pop(0); /* Closes vegas_databuf_detach */
}
static void *run(hashpipe_thread_args_t * args, int doCPU)
{
// Local aliases to shorten access to args fields
paper_gpu_input_databuf_t *db_in = (paper_gpu_input_databuf_t *)args->ibuf;
paper_output_databuf_t *db_out = (paper_output_databuf_t *)args->obuf;
hashpipe_status_t st = args->st;
const char * status_key = args->thread_desc->skey;
#ifdef DEBUG_SEMS
fprintf(stderr, "s/tid %lu/ GPU/\n", pthread_self());
#endif
// Init integration control status variables
int gpu_dev = 0;
hashpipe_status_lock_safe(&st);
hputs(st.buf, "INTSTAT", "off");
hputi8(st.buf, "INTSYNC", 0);
hputi4(st.buf, "INTCOUNT", N_SUB_BLOCKS_PER_INPUT_BLOCK);
hputi8(st.buf, "GPUDUMPS", 0);
hgeti4(st.buf, "GPUDEV", &gpu_dev); // No change if not found
hputi4(st.buf, "GPUDEV", gpu_dev);
hashpipe_status_unlock_safe(&st);
/* Loop */
int rv;
char integ_status[17];
uint64_t start_mcount, last_mcount=0;
uint64_t gpu_dumps=0;
int int_count; // Number of blocks to integrate per dump
int xgpu_error = 0;
int curblock_in=0;
int curblock_out=0;
struct timespec start, stop;
uint64_t elapsed_gpu_ns = 0;
uint64_t gpu_block_count = 0;
// Initialize context to point at first input and output memory blocks.
// This seems redundant since we do this just before calling
// xgpuCudaXengine, but we need to pass something in for array_h and
// matrix_x to prevent xgpuInit from allocating memory.
XGPUContext context;
context.array_h = (ComplexInput *)db_in->block[0].data;
context.array_len = (db_in->header.n_block * sizeof(paper_gpu_input_block_t) - sizeof(paper_input_header_t)) / sizeof(ComplexInput);
context.matrix_h = (Complex *)db_out->block[0].data;
context.matrix_len = (db_out->header.n_block * sizeof(paper_output_block_t) - sizeof(paper_output_header_t)) / sizeof(Complex);
xgpu_error = xgpuInit(&context, gpu_dev);
if (XGPU_OK != xgpu_error) {
fprintf(stderr, "ERROR: xGPU initialization failed (error code %d)\n", xgpu_error);
return THREAD_ERROR;
}
while (run_threads()) {
// Note waiting status,
// query integrating status
// and, if armed, start count
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "waiting");
hgets(st.buf, "INTSTAT", 16, integ_status);
hgeti8(st.buf, "INTSYNC", (long long*)&start_mcount);
hashpipe_status_unlock_safe(&st);
// Wait for new input block to be filled
while ((rv=hashpipe_databuf_wait_filled((hashpipe_databuf_t *)db_in, curblock_in)) != HASHPIPE_OK) {
if (rv==HASHPIPE_TIMEOUT) {
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "blocked_in");
hashpipe_status_unlock_safe(&st);
continue;
} else {
hashpipe_error(__FUNCTION__, "error waiting for filled databuf");
pthread_exit(NULL);
break;
}
}
// Got a new data block, update status and determine how to handle it
hashpipe_status_lock_safe(&st);
hputi4(st.buf, "GPUBLKIN", curblock_in);
hputu8(st.buf, "GPUMCNT", db_in->block[curblock_in].header.mcnt);
hashpipe_status_unlock_safe(&st);
// If integration status "off"
if(!strcmp(integ_status, "off")) {
// Mark input block as free and advance
hashpipe_databuf_set_free((hashpipe_databuf_t *)db_in, curblock_in);
curblock_in = (curblock_in + 1) % db_in->header.n_block;
// Skip to next input buffer
continue;
}
// If integration status is "start"
if(!strcmp(integ_status, "start")) {
// If buffer mcount < start_mcount (i.e. not there yet)
if(db_in->block[curblock_in].header.mcnt < start_mcount) {
// Drop input buffer
// Mark input block as free and advance
hashpipe_databuf_set_free((hashpipe_databuf_t *)db_in, curblock_in);
curblock_in = (curblock_in + 1) % db_in->header.n_block;
// Skip to next input buffer
continue;
// Else if mcount == start_mcount (time to start)
} else if(db_in->block[curblock_in].header.mcnt == start_mcount) {
// Set integration status to "on"
// Read integration count (INTCOUNT)
fprintf(stderr, "--- integration on ---\n");
strcpy(integ_status, "on");
hashpipe_status_lock_safe(&st);
hputs(st.buf, "INTSTAT", integ_status);
hgeti4(st.buf, "INTCOUNT", &int_count);
hashpipe_status_unlock_safe(&st);
// Compute last mcount
last_mcount = start_mcount + (int_count-1) * N_SUB_BLOCKS_PER_INPUT_BLOCK;
// Else (missed starting mcount)
} else {
// Handle missed start of integration
// TODO!
fprintf(stderr, "--- mcnt=%06lx > start_mcnt=%06lx ---\n",
db_in->block[curblock_in].header.mcnt, start_mcount);
}
}
// Integration status is "on" or "stop"
// Note processing status
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "processing gpu");
hashpipe_status_unlock_safe(&st);
// Setup for current chunk
context.input_offset = curblock_in * sizeof(paper_gpu_input_block_t) / sizeof(ComplexInput);
context.output_offset = curblock_out * sizeof(paper_output_block_t) / sizeof(Complex);
// Call CUDA X engine function
int doDump = 0;
// Dump if this is the last block or we are doing both CPU and GPU
// (GPU and CPU test mode always dumps every input block)
if(db_in->block[curblock_in].header.mcnt >= last_mcount || doCPU) {
doDump = 1;
// Check whether we missed the end of integration. If we get a block
// whose mcnt is greater than last_mcount, then for some reason (e.g.
// networking problems) we didn't see a block whose mcnt was
// last_mcount. This should "never" happen, but it has been seen to
// occur when the 10 GbE links have many errors.
if(db_in->block[curblock_in].header.mcnt > last_mcount) {
// Can't do much error recovery, so just log it.
fprintf(stderr, "--- mcnt=%06lx > last_mcnt=%06lx ---\n",
db_in->block[curblock_in].header.mcnt, last_mcount);
}
// Wait for new output block to be free
while ((rv=paper_output_databuf_wait_free(db_out, curblock_out)) != HASHPIPE_OK) {
if (rv==HASHPIPE_TIMEOUT) {
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "blocked gpu out");
hashpipe_status_unlock_safe(&st);
continue;
} else {
hashpipe_error(__FUNCTION__, "error waiting for free databuf");
pthread_exit(NULL);
break;
}
}
}
clock_gettime(CLOCK_MONOTONIC, &start);
xgpuCudaXengine(&context, doDump ? SYNCOP_DUMP : SYNCOP_SYNC_TRANSFER);
clock_gettime(CLOCK_MONOTONIC, &stop);
elapsed_gpu_ns += ELAPSED_NS(start, stop);
gpu_block_count++;
if(doDump) {
clock_gettime(CLOCK_MONOTONIC, &start);
xgpuClearDeviceIntegrationBuffer(&context);
clock_gettime(CLOCK_MONOTONIC, &stop);
elapsed_gpu_ns += ELAPSED_NS(start, stop);
// TODO Maybe need to subtract all or half the integration time here
// depending on recevier's expectations.
db_out->block[curblock_out].header.mcnt = last_mcount;
// If integration status if "stop"
if(!strcmp(integ_status, "stop")) {
// Set integration status to "off"
strcpy(integ_status, "off");
hashpipe_status_lock_safe(&st);
hputs(st.buf, "INTSTAT", integ_status);
hashpipe_status_unlock_safe(&st);
} else {
// Advance last_mcount for end of next integration
last_mcount += int_count * N_SUB_BLOCKS_PER_INPUT_BLOCK;
}
// Mark output block as full and advance
paper_output_databuf_set_filled(db_out, curblock_out);
curblock_out = (curblock_out + 1) % db_out->header.n_block;
// TODO Need to handle or at least check for overflow!
// Update GPU dump counter and GPU Gbps
gpu_dumps++;
hashpipe_status_lock_safe(&st);
hputi8(st.buf, "GPUDUMPS", gpu_dumps);
hputr4(st.buf, "GPUGBPS", (float)(8*N_FLUFFED_BYTES_PER_BLOCK*gpu_block_count)/elapsed_gpu_ns);
hashpipe_status_unlock_safe(&st);
// Start new average
elapsed_gpu_ns = 0;
gpu_block_count = 0;
}
if(doCPU) {
/* Note waiting status */
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "waiting");
hashpipe_status_unlock_safe(&st);
// Wait for new output block to be free
while ((rv=paper_output_databuf_wait_free(db_out, curblock_out)) != HASHPIPE_OK) {
if (rv==HASHPIPE_TIMEOUT) {
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "blocked cpu out");
hashpipe_status_unlock_safe(&st);
continue;
} else {
hashpipe_error(__FUNCTION__, "error waiting for free databuf");
pthread_exit(NULL);
break;
}
}
// Note "processing cpu" status, current input block
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "processing cpu");
hashpipe_status_unlock_safe(&st);
/*
* Call CPU X engine function
*/
xgpuOmpXengine((Complex *)db_out->block[curblock_out].data, context.array_h);
// Mark output block as full and advance
paper_output_databuf_set_filled(db_out, curblock_out);
curblock_out = (curblock_out + 1) % db_out->header.n_block;
// TODO Need to handle or at least check for overflow!
}
// Mark input block as free and advance
hashpipe_databuf_set_free((hashpipe_databuf_t *)db_in, curblock_in);
curblock_in = (curblock_in + 1) % db_in->header.n_block;
/* Check for cancel */
pthread_testcancel();
}
xgpuFree(&context);
// Thread success!
return NULL;
}
int hget (const char *buffer, const char *key, int *val)
{
return hgeti4 (buffer, key, val);
}
static void *run(hashpipe_thread_args_t * args)
{
s6_input_databuf_t *db = (s6_input_databuf_t *)args->obuf;
hashpipe_status_t *p_st = &(args->st);
hashpipe_status_t st = args->st;
const char * status_key = args->thread_desc->skey;
//s6_input_block_t fake_data_block;
/* Main loop */
int i, rv;
uint64_t mcnt = 0;
uint64_t num_coarse_chan = N_COARSE_CHAN;
uint64_t *data;
int block_idx = 0;
int error_count = 0, max_error_count = 0;
float error, max_error = 0.0;
int gen_fake = 0;
hashpipe_status_lock_safe(&st);
//hashpipe_status_lock_safe(p_st);
hputi4(st.buf, "NUMCCHAN", N_COARSE_CHAN);
hputi4(st.buf, "NUMFCHAN", N_FINE_CHAN);
hputi4(st.buf, "NUMBBEAM", N_BYTES_PER_BEAM);
hputi4(st.buf, "NUMBBLOC", sizeof(s6_input_block_t));
hputi4(st.buf, "THRESHLD", POWER_THRESH);
hgeti4(st.buf, "GENFAKE", &gen_fake);
hashpipe_status_unlock_safe(&st);
//hashpipe_status_unlock_safe(p_st);
time_t t, prior_t;
prior_t = time(&prior_t);
while (run_threads()) {
hashpipe_status_lock_safe(&st);
//hashpipe_status_lock_safe(p_st);
hputi4(st.buf, "NETBKOUT", block_idx);
hputs(st.buf, status_key, "waiting");
hashpipe_status_unlock_safe(&st);
//hashpipe_status_unlock_safe(p_st);
t = time(&t);
fprintf(stderr, "elapsed seconds for block %d : %ld\n", block_idx, t - prior_t);
prior_t = t;
// Wait for data
struct timespec sleep_dur, rem_sleep_dur;
sleep_dur.tv_sec = 1;
sleep_dur.tv_nsec = 0;
//fprintf(stderr, "fake net thread sleeping for %7.5f seconds\n",
// sleep_dur.tv_sec + (double)sleep_dur.tv_nsec/1000000000.0);
nanosleep(&sleep_dur, &rem_sleep_dur);
/* Wait for new block to be free, then clear it
* if necessary and fill its header with new values.
*/
while ((rv=s6_input_databuf_wait_free(db, block_idx))
!= HASHPIPE_OK) {
if (rv==HASHPIPE_TIMEOUT) {
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "blocked");
hashpipe_status_unlock_safe(&st);
continue;
} else {
hashpipe_error(__FUNCTION__, "error waiting for free databuf");
pthread_exit(NULL);
break;
}
}
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "receiving");
hashpipe_status_unlock_safe(&st);
// populate block header
db->block[block_idx].header.mcnt = mcnt;
db->block[block_idx].header.coarse_chan_id = 321;
db->block[block_idx].header.num_coarse_chan = num_coarse_chan;
memset(db->block[block_idx].header.missed_pkts, 0, sizeof(uint64_t) * N_BEAM_SLOTS);
if(gen_fake) {
gen_fake = 0;
// gen fake data for all beams, all blocks
// TODO vary data by beam
fprintf(stderr, "generating fake data to block 0 beam 0...");
gen_fake_data(&(db->block[0].data[0]));
fprintf(stderr, " done\n");
fprintf(stderr, "copying to block 0 beam");
for(int beam_i = 1; beam_i < N_BEAMS; beam_i++) {
fprintf(stderr, " %d", beam_i);
memcpy((void *)&db->block[0].data[beam_i*N_BYTES_PER_BEAM/sizeof(uint64_t)],
(void *)&db->block[0].data[0],
N_BYTES_PER_BEAM);
}
fprintf(stderr, " done\n");
fprintf(stderr, "copying to block");
for(int block_i = 1; block_i < N_INPUT_BLOCKS; block_i++) {
fprintf(stderr, " %d", block_i);
memcpy((void *)&db->block[block_i].data[0],
(void *)&db->block[0].data[0],
N_DATA_BYTES_PER_BLOCK);
}
fprintf(stderr, " done\n");
}
hashpipe_status_lock_safe(&st);
hputr4(st.buf, "NETMXERR", max_error);
hputi4(st.buf, "NETERCNT", error_count);
hputi4(st.buf, "NETMXECT", max_error_count);
hashpipe_status_unlock_safe(&st);
// Mark block as full
s6_input_databuf_set_filled(db, block_idx);
// Setup for next block
block_idx = (block_idx + 1) % db->header.n_block;
mcnt++;
// uncomment the following to test dynamic setting of num_coarse_chan
//num_coarse_chan--;
/* Will exit if thread has been cancelled */
pthread_testcancel();
}
// Thread success!
return THREAD_OK;
}
void guppi_rawdisk_thread(void *_args) {
/* Set cpu affinity */
cpu_set_t cpuset, cpuset_orig;
sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig);
CPU_ZERO(&cpuset);
CPU_SET(1, &cpuset);
int rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
if (rv<0) {
guppi_error("guppi_rawdisk_thread", "Error setting cpu affinity.");
perror("sched_setaffinity");
}
/* Get args */
struct guppi_thread_args *args = (struct guppi_thread_args *)_args;
/* Set priority */
rv = setpriority(PRIO_PROCESS, 0, 0);
if (rv<0) {
guppi_error("guppi_rawdisk_thread", "Error setting priority level.");
perror("set_priority");
}
/* Attach to status shared mem area */
struct guppi_status st;
rv = guppi_status_attach(&st);
if (rv!=GUPPI_OK) {
guppi_error("guppi_rawdisk_thread",
"Error attaching to status shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)guppi_status_detach, &st);
pthread_cleanup_push((void *)set_exit_status, &st);
/* Init status */
guppi_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "init");
guppi_status_unlock_safe(&st);
/* Read in general parameters */
struct guppi_params gp;
#if FITS_TYPE == PSRFITS
struct sdfits pf;
pf.sub.dat_freqs = NULL;
pf.sub.dat_weights = NULL;
pf.sub.dat_offsets = NULL;
pf.sub.dat_scales = NULL;
pthread_cleanup_push((void *)guppi_free_psrfits, &pf);
#else
struct sdfits pf;
pthread_cleanup_push((void *)guppi_free_sdfits, &pf);
#endif
/* Attach to databuf shared mem */
struct guppi_databuf *db;
db = guppi_databuf_attach(args->input_buffer);
if (db==NULL) {
guppi_error("guppi_rawdisk_thread",
"Error attaching to databuf shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)guppi_databuf_detach, db);
/* Init output file */
FILE *fraw = NULL;
pthread_cleanup_push((void *)safe_fclose, fraw);
/* Pointers for quantization params */
double *mean = NULL;
double *std = NULL;
printf("casper: raw disk thread created and running\n");
/* Loop */
int packetidx=0, npacket=0, ndrop=0, packetsize=0, blocksize=0;
int orig_blocksize=0;
int curblock=0;
int block_count=0, blocks_per_file=128, filenum=0;
int got_packet_0=0, first=1;
int requantize = 0;
char *ptr, *hend;
signal(SIGINT,cc);
while (run_threads) {
/* Note waiting status */
guppi_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "waiting");
guppi_status_unlock_safe(&st);
/* Wait for buf to have data */
rv = guppi_databuf_wait_filled(db, curblock);
if (rv!=0) continue;
printf("casper: raw disk thread rcvd data.\n");
/* Read param struct for this block */
ptr = guppi_databuf_header(db, curblock);
if (first) {
guppi_read_obs_params(ptr, &gp, &pf);
first = 0;
} else {
guppi_read_subint_params(ptr, &gp, &pf);
}
/* Parse packet size, npacket from header */
hgeti4(ptr, "PKTIDX", &packetidx);
hgeti4(ptr, "PKTSIZE", &packetsize);
hgeti4(ptr, "NPKT", &npacket);
hgeti4(ptr, "NDROP", &ndrop);
#if FITS_TYPE == PSR_FITS
/* Check for re-quantization flag */
int nbits_req = 0;
if (hgeti4(ptr, "NBITSREQ", &nbits_req) == 0) {
/* Param not present, don't requantize */
requantize = 0;
} else {
/* Param is present */
if (nbits_req==8)
requantize = 0;
else if (nbits_req==2)
requantize = 1;
else
/* Invalid selection for requested nbits
* .. die or ignore?
*/
requantize = 0;
}
#endif
/* Set up data ptr for quant routines */
#if FITS_TYPE == PSR_FITS
pf.sub.data = (unsigned char *)guppi_databuf_data(db, curblock);
#else
pf.data_columns.data = (unsigned char *)guppi_databuf_data(db, curblock);
#endif
/* Wait for packet 0 before starting write */
if (got_packet_0==0 && packetidx==0 && gp.stt_valid==1) {
got_packet_0 = 1;
guppi_read_obs_params(ptr, &gp, &pf);
#if FITS_TYPE == PSR_FITS
orig_blocksize = pf.sub.bytes_per_subint;
#endif
char fname[256];
sprintf(fname, "%s.%4.4d.raw", pf.basefilename, filenum);
fprintf(stderr, "Opening raw file '%s'\n", fname);
// TODO: check for file exist.
fraw = fopen(fname, "w");
if (fraw==NULL) {
guppi_error("guppi_rawdisk_thread", "Error opening file.");
pthread_exit(NULL);
}
#if FITS_TYPE == PSR_FITS
/* Determine scaling factors for quantization if appropriate */
if (requantize) {
mean = (double *)realloc(mean,
pf.hdr.rcvr_polns * pf.hdr.nchan * sizeof(double));
std = (double *)realloc(std,
pf.hdr.rcvr_polns * pf.hdr.nchan * sizeof(double));
compute_stat(&pf, mean, std);
fprintf(stderr, "Computed 2-bit stats\n");
}
#endif
}
/* See if we need to open next file */
if (block_count >= blocks_per_file) {
fclose(fraw);
filenum++;
char fname[256];
sprintf(fname, "%s.%4.4d.raw", pf.basefilename, filenum);
fprintf(stderr, "Opening raw file '%s'\n", fname);
fraw = fopen(fname, "w");
if (fraw==NULL) {
guppi_error("guppi_rawdisk_thread", "Error opening file.");
pthread_exit(NULL);
}
block_count=0;
}
/* See how full databuf is */
//total_status = guppi_databuf_total_status(db);
/* Requantize from 8 bits to 2 bits if necessary.
* See raw_quant.c for more usage examples.
*/
#if FITS_TYPE == PSR_FITS
if (requantize && got_packet_0) {
pf.sub.bytes_per_subint = orig_blocksize;
/* Does the quantization in-place */
quantize_2bit(&pf, mean, std);
/* Update some parameters for output */
hputi4(ptr, "BLOCSIZE", pf.sub.bytes_per_subint);
hputi4(ptr, "NBITS", pf.hdr.nbits);
}
#endif
/* Get full data block size */
hgeti4(ptr, "BLOCSIZE", &blocksize);
/* If we got packet 0, write data to disk */
if (got_packet_0) {
/* Note waiting status */
guppi_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "writing");
guppi_status_unlock_safe(&st);
/* Write header to file */
hend = ksearch(ptr, "END");
for (ptr=ptr; ptr<=hend; ptr+=80) {
fwrite(ptr, 80, 1, fraw);
}
/* Write data */
printf("block size: %d\n", blocksize);
ptr = guppi_databuf_data(db, curblock);
rv = fwrite(ptr, 1, (size_t)blocksize, fraw);
if (rv != blocksize) {
guppi_error("guppi_rawdisk_thread",
"Error writing data.");
}
/* Increment counter */
block_count++;
/* flush output */
fflush(fraw);
}
/* Mark as free */
guppi_databuf_set_free(db, curblock);
/* Go to next block */
curblock = (curblock + 1) % db->n_block;
/* Check for cancel */
pthread_testcancel();
}
pthread_exit(NULL);
pthread_cleanup_pop(0); /* Closes fclose */
pthread_cleanup_pop(0); /* Closes guppi_databuf_detach */
pthread_cleanup_pop(0); /* Closes guppi_free_psrfits */
pthread_cleanup_pop(0); /* Closes set_exit_status */
pthread_cleanup_pop(0); /* Closes guppi_status_detach */
}
static void *run(hashpipe_thread_args_t * args)
{
// Local aliases to shorten access to args fields
// Our output buffer happens to be a paper_input_databuf
hashpipe_status_t st = args->st;
const char * status_key = args->thread_desc->skey;
st_p = &st; // allow global (this source file) access to the status buffer
// Get inital value for crc32 function
uint32_t init_crc = crc32(0,0,0);
// Flag that holds off the crc thread
int holdoff = 1;
// Force ourself into the hold off state
hashpipe_status_lock_safe(&st);
hputi4(st.buf, "CRCHOLD", 1);
hashpipe_status_unlock_safe(&st);
while(holdoff) {
// We're not in any hurry to startup
sleep(1);
hashpipe_status_lock_safe(&st);
// Look for CRCHOLD value
hgeti4(st.buf, "CRCHOLD", &holdoff);
if(!holdoff) {
// Done holding, so delete the key
hdel(st.buf, "CRCHOLD");
}
hashpipe_status_unlock_safe(&st);
}
/* Read network params */
struct hashpipe_udp_params up = {
.bindhost = "0.0.0.0",
.bindport = 8511,
.packet_size = 8200
};
hashpipe_status_lock_safe(&st);
// Get info from status buffer if present (no change if not present)
hgets(st.buf, "BINDHOST", 80, up.bindhost);
hgeti4(st.buf, "BINDPORT", &up.bindport);
// Store bind host/port info etc in status buffer
hputs(st.buf, "BINDHOST", up.bindhost);
hputi4(st.buf, "BINDPORT", up.bindport);
hputu4(st.buf, "CRCPKOK", 0);
hputu4(st.buf, "CRCPKERR", 0);
hputs(st.buf, status_key, "running");
hashpipe_status_unlock_safe(&st);
struct hashpipe_udp_packet p;
/* Give all the threads a chance to start before opening network socket */
sleep(1);
/* Set up UDP socket */
int rv = hashpipe_udp_init(&up);
if (rv!=HASHPIPE_OK) {
hashpipe_error("paper_crc_thread",
"Error opening UDP socket.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)hashpipe_udp_close, &up);
/* Main loop */
uint64_t packet_count = 0;
uint64_t good_count = 0;
uint64_t error_count = 0;
uint64_t elapsed_wait_ns = 0;
uint64_t elapsed_recv_ns = 0;
uint64_t elapsed_proc_ns = 0;
float ns_per_wait = 0.0;
float ns_per_recv = 0.0;
float ns_per_proc = 0.0;
struct timespec start, stop;
struct timespec recv_start, recv_stop;
packet_header_t hdr;
while (run_threads()) {
/* Read packet */
clock_gettime(CLOCK_MONOTONIC, &recv_start);
do {
clock_gettime(CLOCK_MONOTONIC, &start);
p.packet_size = recv(up.sock, p.data, HASHPIPE_MAX_PACKET_SIZE, 0);
clock_gettime(CLOCK_MONOTONIC, &recv_stop);
} while (p.packet_size == -1 && (errno == EAGAIN || errno == EWOULDBLOCK) && run_threads());
// Break out of loop if stopping
if(!run_threads()) break;
// Increment packet count
packet_count++;
// Check CRC
if(crc32(init_crc, (/*const?*/ uint8_t *)p.data, p.packet_size) == 0xffffffff) {
// CRC OK! Increment good counter
good_count++;
} else {
// CRC error! Increment error counter
error_count++;
// Log message
get_header(&p, &hdr);
hashpipe_warn("paper_crc", "CRC error mcnt %llu ; fid %u ; xid %u",
hdr.mcnt, hdr.fid, hdr.xid);
}
clock_gettime(CLOCK_MONOTONIC, &stop);
elapsed_wait_ns += ELAPSED_NS(recv_start, start);
elapsed_recv_ns += ELAPSED_NS(start, recv_stop);
elapsed_proc_ns += ELAPSED_NS(recv_stop, stop);
if(packet_count % 1000 == 0) {
// Compute stats
get_header(&p, &hdr);
ns_per_wait = (float)elapsed_wait_ns / packet_count;
ns_per_recv = (float)elapsed_recv_ns / packet_count;
ns_per_proc = (float)elapsed_proc_ns / packet_count;
// Update status
hashpipe_status_lock_busywait_safe(&st);
hputu8(st.buf, "CRCMCNT", hdr.mcnt);
// Gbps = bits_per_packet / ns_per_packet
// (N_BYTES_PER_PACKET excludes header, so +8 for the header)
hputr4(st.buf, "CRCGBPS", 8*(N_BYTES_PER_PACKET+8)/(ns_per_recv+ns_per_proc));
hputr4(st.buf, "CRCWATNS", ns_per_wait);
hputr4(st.buf, "CRCRECNS", ns_per_recv);
hputr4(st.buf, "CRCPRCNS", ns_per_proc);
// TODO Provide some way to recognize request to zero out the
// CRCERR and CRCOK fields.
hputu8(st.buf, "CRCPKOK", good_count);
hputu8(st.buf, "CRCPKERR", error_count);
hashpipe_status_unlock_safe(&st);
// Start new average
elapsed_wait_ns = 0;
elapsed_recv_ns = 0;
elapsed_proc_ns = 0;
packet_count = 0;
}
/* Will exit if thread has been cancelled */
pthread_testcancel();
}
/* Have to close all push's */
pthread_cleanup_pop(1); /* Closes push(hashpipe_udp_close) */
return NULL;
}
static hashpipe_thread_desc_t crc_thread = {
name: "paper_crc_thread",
skey: "CRCSTAT",
init: NULL,
run: run,
ibuf_desc: {NULL},
void vegas_rawdisk_thread(void *_args) {
/* Set cpu affinity */
cpu_set_t cpuset, cpuset_orig;
sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig);
CPU_ZERO(&cpuset);
CPU_SET(6, &cpuset);
int rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
if (rv<0) {
vegas_error("vegas_rawdisk_thread", "Error setting cpu affinity.");
perror("sched_setaffinity");
}
/* Get args */
struct vegas_thread_args *args = (struct vegas_thread_args *)_args;
/* Set priority */
rv = setpriority(PRIO_PROCESS, 0, 0);
if (rv<0) {
vegas_error("vegas_rawdisk_thread", "Error setting priority level.");
perror("set_priority");
}
/* Attach to status shared mem area */
struct vegas_status st;
rv = vegas_status_attach(&st);
if (rv!=VEGAS_OK) {
vegas_error("vegas_rawdisk_thread",
"Error attaching to status shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)vegas_status_detach, &st);
pthread_cleanup_push((void *)set_exit_status, &st);
/* Init status */
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "init");
vegas_status_unlock_safe(&st);
/* Read in general parameters */
struct vegas_params gp;
struct sdfits sf;
pthread_cleanup_push((void *)vegas_free_sdfits, &sf);
/* Attach to databuf shared mem */
struct vegas_databuf *db;
db = vegas_databuf_attach(args->input_buffer);
if (db==NULL) {
vegas_error("vegas_rawdisk_thread",
"Error attaching to databuf shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)vegas_databuf_detach, db);
/* Init output file */
FILE *fraw = NULL;
pthread_cleanup_push((void *)safe_fclose, fraw);
/* Loop */
int blocksize=0;
int curblock=0, dataset;
int block_count=0, blocks_per_file=128, filenum=0;
int first=1;
char *ptr;
float *data_array;
struct databuf_index* db_index;
signal(SIGINT,cc);
while (run) {
/* Note waiting status */
vegas_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "waiting");
vegas_status_unlock_safe(&st);
/* Wait for buf to have data */
rv = vegas_databuf_wait_filled(db, curblock);
if (rv!=0) continue;
/* Read param struct and index for this block */
ptr = vegas_databuf_header(db, curblock);
db_index = (struct databuf_index*)(vegas_databuf_index(db, curblock));
/* If first time running */
if (first==1)
{
first = 0;
vegas_read_obs_params(ptr, &gp, &sf);
char fname[256];
sprintf(fname, "%s_%4d.raw", sf.basefilename, filenum);
fprintf(stderr, "Opening raw file '%s'\n", fname);
// TODO: check for file exist.
fraw = fopen(fname, "wb");
if (fraw==NULL) {
vegas_error("vegas_rawdisk_thread", "Error opening file.");
pthread_exit(NULL);
}
}
else
vegas_read_subint_params(ptr, &gp, &sf);
/* See if we need to open next file */
if (block_count >= blocks_per_file) {
fclose(fraw);
filenum++;
char fname[256];
sprintf(fname, "%s_%4d.raw", sf.basefilename, filenum);
fprintf(stderr, "Opening raw file '%s'\n", fname);
fraw = fopen(fname, "wb");
if (fraw==NULL) {
vegas_error("vegas_rawdisk_thread", "Error opening file.");
pthread_exit(NULL);
}
block_count=0;
}
/* Get full data block size */
hgeti4(ptr, "BLOCSIZE", &blocksize);
/* Note writing status and current block */
vegas_status_lock_safe(&st);
hputi4(st.buf, "DSKBLKIN", curblock);
hputs(st.buf, STATUS_KEY, "writing");
vegas_status_unlock_safe(&st);
/* Write all data arrays to disk */
for(dataset = 0; dataset < db_index->num_datasets; dataset++)
{
data_array = (float*)(vegas_databuf_data(db, curblock) +
db_index->disk_buf[dataset].array_offset);
rv = fwrite(data_array, 4, (size_t)(db_index->array_size/4), fraw);
if (rv != db_index->array_size/4) {
vegas_error("vegas_rawdisk_thread",
"Error writing data.");
}
}
/* Increment block counter */
block_count++;
/* flush output */
fflush(fraw);
/* Mark as free */
vegas_databuf_set_free(db, curblock);
/* Go to next block */
curblock = (curblock + 1) % db->n_block;
/* Check for cancel */
pthread_testcancel();
}
pthread_exit(NULL);
pthread_cleanup_pop(0); /* Closes fclose */
pthread_cleanup_pop(0); /* Closes vegas_databuf_detach */
pthread_cleanup_pop(0); /* Closes vegas_free_psrfits */
pthread_cleanup_pop(0); /* Closes set_exit_status */
pthread_cleanup_pop(0); /* Closes vegas_status_detach */
}
/* This thread is passed a single arg, pointer
* to the guppi_udp_params struct. This thread should
* be cancelled and restarted if any hardware params
* change, as this potentially affects packet size, etc.
*/
void *guppi_fake_net_thread(void *_args) {
/* Get arguments */
struct guppi_thread_args *args = (struct guppi_thread_args *)_args;
/* Set cpu affinity */
cpu_set_t cpuset, cpuset_orig;
sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig);
CPU_ZERO(&cpuset);
//CPU_SET(2, &cpuset);
CPU_SET(3, &cpuset);
int rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
if (rv<0) {
guppi_error("guppi_fake_net_thread", "Error setting cpu affinity.");
perror("sched_setaffinity");
}
/* Set priority */
rv = setpriority(PRIO_PROCESS, 0, args->priority);
if (rv<0) {
guppi_error("guppi_fake_net_thread", "Error setting priority level.");
perror("set_priority");
}
/* Attach to status shared mem area */
struct guppi_status st;
rv = guppi_status_attach(&st);
if (rv!=GUPPI_OK) {
guppi_error("guppi_fake_net_thread",
"Error attaching to status shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)guppi_status_detach, &st);
pthread_cleanup_push((void *)set_exit_status, &st);
/* Init status, read info */
guppi_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "init");
guppi_status_unlock_safe(&st);
/* Read in general parameters */
struct guppi_params gp;
#if FITS_TYPE == PSRFITS
struct psrfits pf;
pf.sub.dat_freqs = NULL;
pf.sub.dat_weights = NULL;
pf.sub.dat_offsets = NULL;
pf.sub.dat_scales = NULL;
#else
struct sdfits pf;
#endif
char status_buf[GUPPI_STATUS_SIZE];
guppi_status_lock_safe(&st);
memcpy(status_buf, st.buf, GUPPI_STATUS_SIZE);
guppi_status_unlock_safe(&st);
guppi_read_obs_params(status_buf, &gp, &pf);
#if FITS_TYPE == PSRFITS
pthread_cleanup_push((void *)guppi_free_psrfits, &pf);
#else
pthread_cleanup_push((void *)guppi_free_sdfits, &pf);
#endif
/* Attach to databuf shared mem */
struct guppi_databuf *db;
db = guppi_databuf_attach(args->output_buffer);
if (db==NULL) {
guppi_error("guppi_fake_net_thread",
"Error attaching to databuf shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)guppi_databuf_detach, db);
/* Time parameters */
int stt_imjd=0, stt_smjd=0;
double stt_offs=0.0;
/* Figure out size of data in each packet, number of packets
* per block, etc. Changing packet size during an obs is not
* recommended.
*/
int block_size;
if (hgeti4(status_buf, "BLOCSIZE", &block_size)==0) {
block_size = db->block_size;
hputi4(status_buf, "BLOCSIZE", block_size);
} else {
if (block_size > db->block_size) {
guppi_error("guppi_net_thread", "BLOCSIZE > databuf block_size");
block_size = db->block_size;
hputi4(status_buf, "BLOCSIZE", block_size);
}
}
unsigned heaps_per_block = block_size / sizeof(struct freq_spead_heap);
/* List of databuf blocks currently in use */
unsigned i;
const int nblock = 2;
struct fake_datablock_stats blocks[nblock];
for (i=0; i<nblock; i++)
fake_init_block(&blocks[i], db, sizeof(struct freq_spead_heap), heaps_per_block);
/* Convenience names for first/last blocks in set */
struct fake_datablock_stats *fblock, *lblock;
fblock = &blocks[0];
lblock = &blocks[nblock-1];
/* Misc counters, etc */
char *curdata=NULL, *curheader=NULL, *curindex=NULL;
int first_time = 1;
int heap_cntr = 0, next_block_heap_cntr = heaps_per_block;
/* Main loop */
unsigned force_new_block=0, waiting=-1;
signal(SIGINT,cc);
while (run_threads) {
/* Wait for data */
struct timespec sleep_dur, rem_sleep_dur;
sleep_dur.tv_sec = 0;
sleep_dur.tv_nsec = 2e6;
nanosleep(&sleep_dur, &rem_sleep_dur);
/* Update status if needed */
if (waiting!=0) {
guppi_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "receiving");
guppi_status_unlock_safe(&st);
waiting=0;
}
/* Convert packet format if needed */
if (first_time)
{
first_time = 0;
force_new_block=1;
}
else
force_new_block=0;
/* Determine if we go to next block */
if ((heap_cntr>=next_block_heap_cntr) || force_new_block) {
printf("casper: going to next shared memory block\n");
/* Update drop stats */
guppi_status_lock_safe(&st);
hputr8(st.buf, "DROPAVG", 0.0);
hputr8(st.buf, "DROPTOT", 0.0);
hputr8(st.buf, "DROPBLK", 0.0);
guppi_status_unlock_safe(&st);
/* Finalize first block, and push it off the list.
* Then grab next available block.
*/
if (fblock->block_idx>=0) fake_finalize_block(fblock);
fake_block_stack_push(blocks, nblock);
fake_increment_block(lblock, heap_cntr);
curdata = guppi_databuf_data(db, lblock->block_idx);
curheader = guppi_databuf_header(db, lblock->block_idx);
curindex = guppi_databuf_index(db, lblock->block_idx);
next_block_heap_cntr = lblock->heap_idx + heaps_per_block;
/* If new obs started, reset total counters, get start
* time. Start time is rounded to nearest integer
* second, with warning if we're off that by more
* than 100ms. Any current blocks on the stack
* are also finalized/reset */
if (force_new_block) {
/* Get obs start time */
get_current_mjd(&stt_imjd, &stt_smjd, &stt_offs);
if (stt_offs>0.5) { stt_smjd+=1; stt_offs-=1.0; }
stt_offs = 0.0;
/* Flush any current buffers */
for (i=0; i<nblock-1; i++) {
if (blocks[i].block_idx>=0)
fake_finalize_block(&blocks[i]);
fake_reset_block(&blocks[i]);
}
}
/* Read/update current status shared mem */
guppi_status_lock_safe(&st);
if (stt_imjd!=0) {
#if 1
hputi4(st.buf, "STT_IMJD", stt_imjd);
hputi4(st.buf, "STT_SMJD", stt_smjd);
hputr8(st.buf, "STT_OFFS", stt_offs);
#endif
hputi4(st.buf, "STTVALID", 1);
} else {
hputi4(st.buf, "STTVALID", 0);
}
memcpy(status_buf, st.buf, GUPPI_STATUS_SIZE);
guppi_status_unlock_safe(&st);
/* Wait for new block to be free, then clear it
* if necessary and fill its header with new values.
*/
while ((rv=guppi_databuf_wait_free(db, lblock->block_idx))
!= GUPPI_OK) {
if (rv==GUPPI_TIMEOUT) {
waiting=1;
guppi_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "blocked");
guppi_status_unlock_safe(&st);
continue;
} else {
guppi_error("guppi_fake_net_thread",
"error waiting for free databuf");
run_threads=0;
pthread_exit(NULL);
break;
}
}
memcpy(curheader, status_buf, GUPPI_STATUS_SIZE);
memset(curdata, 0, block_size);
memset(curindex, 0, db->index_size);
}
/*Write fake data to block */
write_fake_heap_to_block(lblock, heap_cntr);
heap_cntr++;
/* Will exit if thread has been cancelled */
pthread_testcancel();
}
pthread_exit(NULL);
/* Have to close all push's */
pthread_cleanup_pop(0); /* Closes set_exit_status */
pthread_cleanup_pop(0); /* Closes guppi_free_psrfits */
pthread_cleanup_pop(0); /* Closes guppi_status_detach */
pthread_cleanup_pop(0); /* Closes guppi_databuf_detach */
}
// Run method for the thread
// It is meant to do the following:
// (1) Initialize status buffer
// (2) Set up network parameters and socket
// (3) Start main loop
// (3a) Receive packet on socket
// (3b) Error check packet (packet size, etc)
// (3c) Call process_packet on received packet
// (4) Terminate thread cleanly
static void *run(hashpipe_thread_args_t * args) {
fprintf(stdout, "N_INPUTS = %d\n", N_INPUTS);
fprintf(stdout, "N_CHAN = %d\n", N_CHAN);
fprintf(stdout, "N_CHAN_PER_X = %d\n", N_CHAN_PER_X);
fprintf(stdout, "N_CHAN_PER_PACKET = %d\n", N_CHAN_PER_PACKET);
fprintf(stdout, "N_TIME_PER_PACKET = %d\n", N_TIME_PER_PACKET);
fprintf(stdout, "N_TIME_PER_BLOCK = %d\n", N_TIME_PER_BLOCK);
fprintf(stdout, "N_BYTES_PER_BLOCK = %d\n", N_BYTES_PER_BLOCK);
fprintf(stdout, "N_BYTES_PER_PACKET = %d\n", N_BYTES_PER_PACKET);
fprintf(stdout, "N_PACKETS_PER_BLOCK = %d\n", N_PACKETS_PER_BLOCK);
fprintf(stdout, "N_COR_MATRIX = %d\n", N_COR_MATRIX);
// Local aliases to shorten access to args fields
// Our output buffer happens to be a paper_input_databuf
flag_input_databuf_t *db = (flag_input_databuf_t *)args->obuf;
hashpipe_status_t st = args->st;
const char * status_key = args->thread_desc->skey;
st_p = &st; // allow global (this source file) access to the status buffer
/* Read network params */
fprintf(stdout, "Setting up network parameters\n");
struct hashpipe_udp_params up = {
.bindhost = "0.0.0.0",
.bindport = 8511,
.packet_size = 8008
};
hashpipe_status_lock_safe(&st);
// Get info from status buffer if present (no change if not present)
hgets(st.buf, "BINDHOST", 80, up.bindhost);
hgeti4(st.buf, "BINDPORT", &up.bindport);
// Store bind host/port info etc in status buffer
hputs(st.buf, "BINDHOST", up.bindhost);
hputi4(st.buf, "BINDPORT", up.bindport);
hputu4(st.buf, "MISSEDFE", 0);
hputu4(st.buf, "MISSEDPK", 0);
hputs(st.buf, status_key, "running");
hashpipe_status_unlock_safe(&st);
struct hashpipe_udp_packet p;
/* Give all the threads a chance to start before opening network socket */
int netready = 0;
int corready = 0;
int checkready = 0;
while (!netready) {
sleep(1);
// Check the correlator to see if it's ready yet
hashpipe_status_lock_safe(&st);
hgeti4(st.buf, "CORREADY", &corready);
hgeti4(st.buf, "SAVEREADY", &checkready);
hashpipe_status_unlock_safe(&st);
if (!corready) {
continue;
}
//if (!checkready) {
// continue;
//}
// Check the other threads to see if they're ready yet
// TBD
// If we get here, then all threads are initialized
netready = 1;
}
sleep(3);
/* Set up UDP socket */
fprintf(stderr, "NET: BINDHOST = %s\n", up.bindhost);
fprintf(stderr, "NET: BINDPORT = %d\n", up.bindport);
int rv = hashpipe_udp_init(&up);
if (rv!=HASHPIPE_OK) {
hashpipe_error("paper_net_thread",
"Error opening UDP socket.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)hashpipe_udp_close, &up);
// Initialize first few blocks in the buffer
int i;
for (i = 0; i < 2; i++) {
// Wait until block semaphore is free
if (flag_input_databuf_wait_free(db, i) != HASHPIPE_OK) {
if (errno == EINTR) { // Interrupt occurred
hashpipe_error(__FUNCTION__, "waiting for free block interrupted\n");
pthread_exit(NULL);
} else {
hashpipe_error(__FUNCTION__, "error waiting for free block\n");
pthread_exit(NULL);
}
}
initialize_block(db, i*Nm);
}
// Set correlator to "start" state
hashpipe_status_lock_safe(&st);
hputs(st.buf, "INTSTAT", "start");
hashpipe_status_unlock_safe(&st);
/* Main loop */
uint64_t packet_count = 0;
fprintf(stdout, "Net: Starting Thread!\n");
while (run_threads()) {
// Get packet
do {
p.packet_size = recv(up.sock, p.data, HASHPIPE_MAX_PACKET_SIZE, 0);
} while (p.packet_size == -1 && (errno == EAGAIN || errno == EWOULDBLOCK) && run_threads());
if(!run_threads()) break;
if (up.packet_size != p.packet_size && up.packet_size != p.packet_size-8) {
// If an error was returned instead of a valid packet size
if (p.packet_size == -1) {
fprintf(stderr, "uh oh!\n");
// Log error and exit
hashpipe_error("paper_net_thread",
"hashpipe_udp_recv returned error");
perror("hashpipe_udp_recv");
pthread_exit(NULL);
} else {
// Log warning and ignore wrongly sized packet
hashpipe_warn("paper_net_thread", "Incorrect pkt size (%d)", p.packet_size);
continue;
}
}
packet_count++;
process_packet(db, &p);
/* Will exit if thread has been cancelled */
pthread_testcancel();
}
pthread_cleanup_pop(1); /* Closes push(hashpipe_udp_close) */
hashpipe_status_lock_busywait_safe(&st);
hputs(st.buf, status_key, "terminated");
hashpipe_status_unlock_safe(&st);
return NULL;
}
static hashpipe_thread_desc_t net_thread = {
name: "flag_net_thread",
skey: "NETSTAT",
init: NULL,
run: run,
ibuf_desc: {NULL},
void vegas_pfb_thread(void *_args) {
/* Get args */
struct guppi_thread_args *args = (struct guppi_thread_args *)_args;
int rv;
/* Set cpu affinity */
cpu_set_t cpuset, cpuset_orig;
sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig);
//CPU_ZERO(&cpuset);
CPU_CLR(13, &cpuset);
CPU_SET(11, &cpuset);
rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
if (rv<0) {
guppi_error("vegas_pfb_thread", "Error setting cpu affinity.");
perror("sched_setaffinity");
}
/* Set priority */
rv = setpriority(PRIO_PROCESS, 0, args->priority);
if (rv<0) {
guppi_error("vegas_pfb_thread", "Error setting priority level.");
perror("set_priority");
}
/* Attach to status shared mem area */
struct guppi_status st;
rv = guppi_status_attach(&st);
if (rv!=GUPPI_OK) {
guppi_error("vegas_pfb_thread",
"Error attaching to status shared memory.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)guppi_status_detach, &st);
pthread_cleanup_push((void *)set_exit_status, &st);
pthread_cleanup_push((void *)guppi_thread_set_finished, args);
/* Init status */
guppi_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "init");
guppi_status_unlock_safe(&st);
/* Init structs */
struct guppi_params gp;
struct sdfits sf;
pthread_cleanup_push((void *)guppi_free_sdfits, &sf);
/* Attach to databuf shared mem */
struct guppi_databuf *db_in, *db_out;
db_in = guppi_databuf_attach(args->input_buffer);
if (db_in==NULL) {
char msg[256];
sprintf(msg, "Error attaching to databuf(%d) shared memory.",
args->input_buffer);
guppi_error("vegas_pfb_thread", msg);
pthread_exit(NULL);
}
pthread_cleanup_push((void *)guppi_databuf_detach, db_in);
db_out = guppi_databuf_attach(args->output_buffer);
if (db_out==NULL) {
char msg[256];
sprintf(msg, "Error attaching to databuf(%d) shared memory.",
args->output_buffer);
guppi_error("vegas_pfb_thread", msg);
pthread_exit(NULL);
}
pthread_cleanup_push((void *)guppi_databuf_detach, db_out);
/* Loop */
char *hdr_in = NULL;
int curblock_in=0;
int first=1;
int acc_len = 0;
int nchan = 0;
int nsubband = 0;
signal(SIGINT,cc);
guppi_status_lock_safe(&st);
if (hgeti4(st.buf, "NCHAN", &nchan)==0) {
fprintf(stderr, "ERROR: %s not in status shm!\n", "NCHAN");
}
if (hgeti4(st.buf, "NSUBBAND", &nsubband)==0) {
fprintf(stderr, "ERROR: %s not in status shm!\n", "NSUBBAND");
}
guppi_status_unlock_safe(&st);
if (EXIT_SUCCESS != init_gpu(db_in->block_size,
db_out->block_size,
nsubband,
nchan))
{
(void) fprintf(stderr, "ERROR: GPU initialisation failed!\n");
run = 0;
}
while (run) {
/* Note waiting status */
guppi_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "waiting");
guppi_status_unlock_safe(&st);
/* Wait for buf to have data */
rv = guppi_databuf_wait_filled(db_in, curblock_in);
if (rv!=0) continue;
/* Note waiting status, current input block */
guppi_status_lock_safe(&st);
hputs(st.buf, STATUS_KEY, "processing");
hputi4(st.buf, "PFBBLKIN", curblock_in);
guppi_status_unlock_safe(&st);
hdr_in = guppi_databuf_header(db_in, curblock_in);
/* Get params */
if (first)
{
guppi_read_obs_params(hdr_in, &gp, &sf);
/* Read required exposure from status shared memory, and calculate
corresponding accumulation length */
acc_len = (sf.hdr.chan_bw * sf.hdr.hwexposr);
}
guppi_read_subint_params(hdr_in, &gp, &sf);
/* Call PFB function */
do_pfb(db_in, curblock_in, db_out, first, st, acc_len);
/* Mark input block as free */
guppi_databuf_set_free(db_in, curblock_in);
/* Go to next input block */
curblock_in = (curblock_in + 1) % db_in->n_block;
/* Check for cancel */
pthread_testcancel();
if (first) {
first=0;
}
}
run=0;
//cudaThreadExit();
pthread_exit(NULL);
cleanup_gpu();
pthread_cleanup_pop(0); /* Closes guppi_databuf_detach(out) */
pthread_cleanup_pop(0); /* Closes guppi_databuf_detach(in) */
pthread_cleanup_pop(0); /* Closes guppi_free_sdfits */
pthread_cleanup_pop(0); /* Closes guppi_thread_set_finished */
pthread_cleanup_pop(0); /* Closes set_exit_status */
pthread_cleanup_pop(0); /* Closes guppi_status_detach */
}
int main(int argc, char *argv[]) {
static struct option long_opts[] = {
{"help", 0, NULL, 'h'},
{"null", 0, NULL, 'n'},
{"ds", 0, NULL, 'D'},
{0,0,0,0}
};
int use_null_thread = 0;
int ds = 0;
int opt, opti;
while ((opt=getopt_long(argc,argv,"hnD",long_opts,&opti))!=-1) {
switch (opt) {
case 'n':
use_null_thread = 1;
break;
case 'D':
ds = 1;
break;
default:
case 'h':
usage();
exit(0);
break;
}
}
int twogpu = 1;
// -- Ids --
thread_args net_args, dedisp_args, disk_args;
thread_args net_args_2, dedisp_args_2, disk_args_2;
thread_args_init(&net_args);
thread_args_init(&dedisp_args);
thread_args_init(&disk_args);
net_args.output_buffer = 1;
dedisp_args.input_buffer = net_args.output_buffer;
dedisp_args.output_buffer = 2;
disk_args.input_buffer = dedisp_args.output_buffer;
disk_args.output_buffer = 3;
net_args.gpu_id = dedisp_args.gpu_id = disk_args.gpu_id = 1;
net_args.priority = 15;
dedisp_args.priority = 20;
disk_args.priority = 20;
// -- Attach to status shared mem --
status stat;
databuf *dbuf_net=NULL, *dbuf_fold=NULL;
int rv = status_attach(&stat, 1);
if (rv!=OK) {
log_error("nuppi_daq_dedisp", "Error connecting to status shm");
exit(1);
}
// -- Init mutex for PSRFITS template --
rv = pthread_mutex_init(&lock_psrfits_tpl, NULL);
if (rv != 0) {
log_error("nuppi_daq_dedisp", "Mutex initialization failed");
exit(1);
}
// -- Read status shm, init RUN and read filename --
char basename[256];
status_lock_safe(&stat);
hgets(stat.buf, "BASENAME", 256, basename);
hputi4(stat.buf, "RUN", 1);
status_unlock_safe(&stat);
run=1;
signal(SIGINT, cc);
// -- Create or just attach to net shared mem --
dbuf_net = databuf_attach(net_args.output_buffer);
if (dbuf_net==NULL)
dbuf_net = databuf_create(8, 256*1024*1024, net_args.output_buffer);
if (dbuf_net==NULL) {
log_error("nuppi_daq_dedisp", "Error connecting to databuf net shm");
exit(1);
}
databuf_clear(dbuf_net);
// -- Create or just attach to fold shared mem --
dbuf_fold = databuf_attach(dedisp_args.output_buffer);
if (dbuf_fold==NULL)
dbuf_fold = databuf_create(8, 128*1024*1024, dedisp_args.output_buffer);
if (dbuf_fold==NULL) {
log_error("nuppi_daq_dedisp", "Error connecting to databuf fold shm");
exit(1);
}
databuf_clear(dbuf_fold);
/* Launch net thread */
pthread_t net_thread_id;
rv = pthread_create(&net_thread_id, NULL, net_thread, (void *)&net_args);
if (rv) {
log_error("nuppi_daq_dedisp", "Error creating net thread");
perror("pthread_create");
exit(1);
}
/* Launch dedisp thread */
pthread_t dedisp_thread_id;
if (ds)
rv = pthread_create(&dedisp_thread_id, NULL, dedisp_ds_thread, (void *)&dedisp_args);
else
rv = pthread_create(&dedisp_thread_id, NULL, dedisp_thread, (void *)&dedisp_args);
if (rv) {
log_error("nuppi_daq_dedisp", "Error creating dedisp thread");
perror("pthread_create");
exit(1);
}
/* Launch psrfits/null thread */
pthread_t disk_thread_id=0;
if (use_null_thread)
rv = pthread_create(&disk_thread_id, NULL, null_thread, (void *)&disk_args);
else
rv = pthread_create(&disk_thread_id, NULL, psrfits_thread, (void *)&disk_args);
if (rv) {
log_error("nuppi_daq_dedisp", "Error creating psrfits thread");
perror("pthread_create");
exit(1);
}
// -- Second GPU --
pthread_t net_thread_id_2;
pthread_t dedisp_thread_id_2;
pthread_t disk_thread_id_2=0;
if(twogpu) {
sleep(2);
// -- Ids --
thread_args_init(&net_args_2);
thread_args_init(&dedisp_args_2);
thread_args_init(&disk_args_2);
net_args_2.output_buffer = 5;
dedisp_args_2.input_buffer = net_args_2.output_buffer;
dedisp_args_2.output_buffer = 6;
disk_args_2.input_buffer = dedisp_args_2.output_buffer;
disk_args_2.output_buffer = 7;
net_args_2.gpu_id = dedisp_args_2.gpu_id = disk_args_2.gpu_id = 2;
net_args_2.priority = 20;
dedisp_args_2.priority = 20;
disk_args_2.priority = 20;
databuf *dbuf_net_2=NULL, *dbuf_fold_2=NULL;
// -- Create or just attach to net shared mem --
dbuf_net_2 = databuf_attach(net_args_2.output_buffer);
if (dbuf_net_2==NULL)
dbuf_net_2 = databuf_create(8, 256*1024*1024, net_args_2.output_buffer);
if (dbuf_net_2==NULL) {
log_error("nuppi_daq_dedisp", "Error connecting to databuf net shm for GPU #2");
exit(1);
}
databuf_clear(dbuf_net_2);
// -- Create or just attach to fold shared mem --
dbuf_fold_2 = databuf_attach(dedisp_args_2.output_buffer);
if (dbuf_fold_2==NULL)
dbuf_fold_2 = databuf_create(8, 128*1024*1024, dedisp_args_2.output_buffer);
if (dbuf_fold_2==NULL) {
log_error("nuppi_daq_dedisp", "Error connecting to databuf fold shm for GPU #2");
exit(1);
}
databuf_clear(dbuf_fold_2);
/* Launch net thread */
rv = pthread_create(&net_thread_id_2, NULL, net_thread, (void *)&net_args_2);
if (rv) {
log_error("nuppi_daq_dedisp", "Error creating net thread 2");
perror("pthread_create");
exit(1);
}
/* Launch dedisp thread */
if (ds)
rv = pthread_create(&dedisp_thread_id_2, NULL, dedisp_ds_thread, (void *)&dedisp_args_2);
else
rv = pthread_create(&dedisp_thread_id_2, NULL, dedisp_thread, (void *)&dedisp_args_2);
if (rv) {
log_error("nuppi_daq_dedisp", "Error creating dedisp thread 2");
perror("pthread_create");
exit(1);
}
/* Launch psrfits/null thread */
if (use_null_thread)
rv = pthread_create(&disk_thread_id_2, NULL, null_thread, (void *)&disk_args_2);
else
rv = pthread_create(&disk_thread_id_2, NULL, psrfits_thread, (void *)&disk_args_2);
if (rv) {
log_error("nuppi_daq_dedisp", "Error creating psrfits thread 2");
perror("pthread_create");
exit(1);
}
}
/* Alt loop, wait for run=0 */
while (run) {
sleep(1);
// Read the RUN keyword in the first status shm, to look for a stop order
status_lock_safe(&stat);
hgeti4(stat.buf, "RUN", &run);
status_unlock_safe(&stat);
if (run == 0) log_info("nuppi_daq_dedisp", "Caught RUN = 0 signal for end of observation");
if (disk_args.finished) run=0;
}
/* Clean up */
pthread_cancel(dedisp_thread_id);
pthread_cancel(net_thread_id);
pthread_cancel(disk_thread_id);
pthread_kill(dedisp_thread_id,SIGINT);
pthread_kill(net_thread_id,SIGINT);
pthread_kill(disk_thread_id,SIGINT);
if(twogpu) {
pthread_cancel(dedisp_thread_id_2);
pthread_cancel(net_thread_id_2);
pthread_cancel(disk_thread_id_2);
pthread_kill(dedisp_thread_id_2,SIGINT);
pthread_kill(net_thread_id_2,SIGINT);
pthread_kill(disk_thread_id_2,SIGINT);
}
// -- Join threads --
pthread_join(net_thread_id,NULL);
log_info("nuppi_daq_dedisp", "Joined net thread");
if(twogpu) {
pthread_join(net_thread_id_2,NULL);
log_info("nuppi_daq_dedisp", "Joined net thread 2");
}
pthread_join(dedisp_thread_id,NULL);
log_info("nuppi_daq_dedisp", "Joined dedisp thread");
if(twogpu) {
pthread_join(dedisp_thread_id_2,NULL);
log_info("nuppi_daq_dedisp", "Joined dedisp thread 2");
}
pthread_join(disk_thread_id,NULL);
log_info("nuppi_daq_dedisp", "Joined disk thread");
if(twogpu) {
pthread_join(disk_thread_id_2,NULL);
log_info("nuppi_daq_dedisp", "Joined disk thread 2");
}
// -- Destroy args --
thread_args_destroy(&net_args);
thread_args_destroy(&dedisp_args);
thread_args_destroy(&disk_args);
thread_args_destroy(&net_args_2);
thread_args_destroy(&dedisp_args_2);
thread_args_destroy(&disk_args_2);
pthread_mutex_destroy(&lock_psrfits_tpl);
char cmd[128], hostname[128];
gethostname(hostname, 127);
sprintf(cmd, "mv %s /home/pulsar/data/%s-%s.log", LOG_FILENAME, basename, hostname);
log_info("nuppi_daq_dedisp", cmd);
system(cmd);
exit(0);
}