/* Update block header info, set filled status */ void fake_finalize_block(struct fake_datablock_stats *d) { char *header = guppi_databuf_header(d->db, d->block_idx); hputi4(header, "PKTIDX", d->heap_idx); hputi4(header, "PKTSIZE", d->heap_size); hputi4(header, "NPKT", 0); hputi4(header, "NDROP", 0); guppi_databuf_set_filled(d->db, d->block_idx); }
/* Update block header info, set filled status */ void finalize_block(struct datablock_stats *d) { char *header = vegas_databuf_header(d->db, d->block_idx); hputi4(header, "HEAPIDX", d->heap_idx); hputi4(header, "HEAPSIZE", d->heap_size); hputi4(header, "NHEAPS", d->nheaps); hputi4(header, "NDROP", d->pkts_dropped); struct databuf_index* index = (struct databuf_index*) vegas_databuf_index(d->db, d->block_idx); index->num_heaps = d->nheaps; index->heap_size = d->heap_size; vegas_databuf_set_filled(d->db, d->block_idx); }
/* This thread is passed a single arg, pointer * to the vegas_udp_params struct. This thread should * be cancelled and restarted if any hardware params * change, as this potentially affects packet size, etc. */ void *vegas_net_thread(void *_args) { /* Get arguments */ struct vegas_thread_args *args = (struct vegas_thread_args *)_args; int rv; /* Set cpu affinity */ cpu_set_t cpuset, cpuset_orig; sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig); //CPU_ZERO(&cpuset); CPU_SET(13, &cpuset); rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); if (rv<0) { vegas_error("vegas_net_thread", "Error setting cpu affinity."); perror("sched_setaffinity"); } /* Set priority */ rv = setpriority(PRIO_PROCESS, 0, args->priority); if (rv<0) { vegas_error("vegas_net_thread", "Error setting priority level."); perror("set_priority"); } /* Attach to status shared mem area */ struct vegas_status st; rv = vegas_status_attach(&st); if (rv!=VEGAS_OK) { vegas_error("vegas_net_thread", "Error attaching to status shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)vegas_status_detach, &st); pthread_cleanup_push((void *)set_exit_status, &st); /* Init status, read info */ vegas_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "init"); vegas_status_unlock_safe(&st); /* Read in general parameters */ struct vegas_params gp; struct sdfits pf; char status_buf[VEGAS_STATUS_SIZE]; vegas_status_lock_safe(&st); memcpy(status_buf, st.buf, VEGAS_STATUS_SIZE); vegas_status_unlock_safe(&st); vegas_read_obs_params(status_buf, &gp, &pf); pthread_cleanup_push((void *)vegas_free_sdfits, &pf); /* Read network params */ struct vegas_udp_params up; vegas_read_net_params(status_buf, &up); /* Attach to databuf shared mem */ struct vegas_databuf *db; db = vegas_databuf_attach(args->output_buffer); if (db==NULL) { vegas_error("vegas_net_thread", "Error attaching to databuf shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)vegas_databuf_detach, db); /* Time parameters */ double meas_stt_mjd=0.0; double meas_stt_offs=0.0; /* See which packet format to use */ int nchan=0, npol=0; nchan = pf.hdr.nchan; npol = pf.hdr.npol; /* Figure out size of data in each packet, number of packets * per block, etc. Changing packet size during an obs is not * recommended. */ int block_size; struct vegas_udp_packet p; size_t heap_size, spead_hdr_size; unsigned int heaps_per_block, packets_per_heap; char bw_mode[16]; if (hgets(status_buf, "BW_MODE", 16, bw_mode)) { if(strncmp(bw_mode, "high", 4) == 0) { heap_size = sizeof(struct freq_spead_heap) + nchan*4*sizeof(int); spead_hdr_size = sizeof(struct freq_spead_heap); packets_per_heap = nchan*4*sizeof(int) / PAYLOAD_SIZE; } else if(strncmp(bw_mode, "low", 3) == 0) { heap_size = sizeof(struct time_spead_heap) + PAYLOAD_SIZE; spead_hdr_size = sizeof(struct time_spead_heap); packets_per_heap = 1; } else vegas_error("vegas_net_thread", "Unsupported bandwidth mode"); } else vegas_error("vegas_net_thread", "BW_MODE not set"); if (hgeti4(status_buf, "BLOCSIZE", &block_size)==0) { block_size = db->block_size; hputi4(status_buf, "BLOCSIZE", block_size); } else { if (block_size > db->block_size) { vegas_error("vegas_net_thread", "BLOCSIZE > databuf block_size"); block_size = db->block_size; hputi4(status_buf, "BLOCSIZE", block_size); } } heaps_per_block = (block_size - MAX_HEAPS_PER_BLK*spead_hdr_size) / (heap_size - spead_hdr_size); /* List of databuf blocks currently in use */ unsigned i; const int nblock = 2; struct datablock_stats blocks[nblock]; for (i=0; i<nblock; i++) init_block(&blocks[i], db, heap_size, spead_hdr_size, heaps_per_block); /* Convenience names for first/last blocks in set */ struct datablock_stats *fblock, *lblock; fblock = &blocks[0]; lblock = &blocks[nblock-1]; /* Misc counters, etc */ char *curdata=NULL, *curheader=NULL, *curindex=NULL; unsigned int heap_cntr=0, last_heap_cntr=2048, nextblock_heap_cntr=0; unsigned int heap_offset; unsigned int seq_num=0, last_seq_num=1050; int heap_cntr_diff, seq_num_diff; unsigned int obs_started = 0; unsigned long long npacket_total, npacket_this_block=0, ndropped_total; double drop_frac_avg=0.0; const double drop_lpf = 0.25; prev_heap_cntr = 0; prev_heap_offset = 0; char msg[256]; /* Give all the threads a chance to start before opening network socket */ sleep(1); /* Set up UDP socket */ rv = vegas_udp_init(&up); if (rv!=VEGAS_OK) { vegas_error("vegas_net_thread", "Error opening UDP socket."); pthread_exit(NULL); } pthread_cleanup_push((void *)vegas_udp_close, &up); /* Main loop */ unsigned force_new_block=0, waiting=-1; signal(SIGINT,cc); while (run) { /* Wait for data */ rv = vegas_udp_wait(&up); if (rv!=VEGAS_OK) { if (rv==VEGAS_TIMEOUT) { /* Set "waiting" flag */ if (waiting!=1) { vegas_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "waiting"); vegas_status_unlock_safe(&st); waiting=1; } continue; } else { vegas_error("vegas_net_thread", "vegas_udp_wait returned error"); perror("vegas_udp_wait"); pthread_exit(NULL); } } /* Read packet */ rv = vegas_udp_recv(&up, &p, bw_mode); if (rv!=VEGAS_OK) { if (rv==VEGAS_ERR_PACKET) { #ifdef DEBUG_NET vegas_warn("vegas_net_thread", "Incorrect pkt size"); #endif continue; } else { vegas_error("vegas_net_thread", "vegas_udp_recv returned error"); perror("vegas_udp_recv"); pthread_exit(NULL); } } /* Update status if needed */ if (waiting!=0) { vegas_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "receiving"); vegas_status_unlock_safe(&st); waiting=0; } /* Check seq num diff */ heap_cntr = vegas_spead_packet_heap_cntr(&p); heap_offset = vegas_spead_packet_heap_offset(&p); seq_num = vegas_spead_packet_seq_num(heap_cntr, heap_offset, packets_per_heap); heap_cntr_diff = heap_cntr - last_heap_cntr; seq_num_diff = (int)(seq_num - last_seq_num); last_seq_num = seq_num; last_heap_cntr = heap_cntr; if (seq_num_diff<=0) { if (seq_num_diff<-1024) { force_new_block=1; obs_started = 1; #ifdef DEBUG_NET printf("Debug: observation started\n"); #endif } else if (seq_num_diff==0) { sprintf(msg, "Received duplicate packet (seq_num=%d)", seq_num); vegas_warn("vegas_net_thread", msg); } else { #ifdef DEBUG_NET sprintf(msg, "out of order packet. Diff = %d", seq_num_diff); vegas_warn("vegas_net_thread", msg); #endif continue; /* No going backwards */ } } else { force_new_block=0; npacket_total += seq_num_diff; ndropped_total += seq_num_diff - 1; npacket_this_block += seq_num_diff; fblock->pkts_dropped += seq_num_diff - 1; #ifdef DEBUG_NET if(seq_num_diff > 1) { sprintf(msg, "Missing packet. seq_num_diff = %d", seq_num_diff); vegas_warn("vegas_net_thread", msg); } #endif } /* If obs has not started, ignore this packet */ if(!obs_started) { fblock->pkts_dropped = 0; npacket_total = 0; ndropped_total = 0; npacket_this_block = 0; continue; } /* Determine if we go to next block */ if (heap_cntr>=nextblock_heap_cntr || force_new_block) { /* Update drop stats */ if (npacket_this_block > 0) drop_frac_avg = (1.0-drop_lpf)*drop_frac_avg + drop_lpf * (double)fblock->pkts_dropped / (double)npacket_this_block; vegas_status_lock_safe(&st); hputi8(st.buf, "NPKT", npacket_total); hputi8(st.buf, "NDROP", ndropped_total); hputr8(st.buf, "DROPAVG", drop_frac_avg); hputr8(st.buf, "DROPTOT", npacket_total ? (double)ndropped_total/(double)npacket_total : 0.0); hputi4(st.buf, "NETBLKOU", fblock->block_idx); vegas_status_unlock_safe(&st); /* Finalize first block, and push it off the list. * Then grab next available block. */ if (fblock->block_idx>=0) finalize_block(fblock); block_stack_push(blocks, nblock); increment_block(lblock, heap_cntr); curdata = vegas_databuf_data(db, lblock->block_idx); curheader = vegas_databuf_header(db, lblock->block_idx); curindex = vegas_databuf_index(db, lblock->block_idx); nextblock_heap_cntr = lblock->heap_idx + heaps_per_block; npacket_this_block = 0; /* If new obs started, reset total counters, get start * time. Start time is rounded to nearest integer * second, with warning if we're off that by more * than 100ms. Any current blocks on the stack * are also finalized/reset */ if (force_new_block) { /* Reset stats */ npacket_total=0; ndropped_total=0; npacket_this_block = 0; /* Get obs start time */ get_current_mjd_double(&meas_stt_mjd); printf("vegas_net_thread: got start packet at MJD %f", meas_stt_mjd); meas_stt_offs = meas_stt_mjd*24*60*60 - floor(meas_stt_mjd*24*60*60); if(meas_stt_offs > 0.1 && meas_stt_offs < 0.9) { char msg[256]; sprintf(msg, "Second fraction = %3.1f ms > +/-100 ms", meas_stt_offs*1e3); vegas_warn("vegas_net_thread", msg); } vegas_status_lock_safe(&st); hputnr8(st.buf, "M_STTMJD", 8, meas_stt_mjd); hputr8(st.buf, "M_STTOFF", meas_stt_offs); vegas_status_unlock_safe(&st); /* Warn if 1st packet number is not zero */ if (seq_num!=0) { char msg[256]; sprintf(msg, "First packet number is not 0 (seq_num=%d)", seq_num); vegas_warn("vegas_net_thread", msg); } } /* Read current status shared mem */ vegas_status_lock_safe(&st); memcpy(status_buf, st.buf, VEGAS_STATUS_SIZE); vegas_status_unlock_safe(&st); /* Wait for new block to be free, then clear it * if necessary and fill its header with new values. */ while ((rv=vegas_databuf_wait_free(db, lblock->block_idx)) != VEGAS_OK) { if (rv==VEGAS_TIMEOUT) { waiting=1; vegas_warn("vegas_net_thread", "timeout while waiting for output block\n"); vegas_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "blocked"); vegas_status_unlock_safe(&st); continue; } else { vegas_error("vegas_net_thread", "error waiting for free databuf"); run=0; pthread_exit(NULL); break; } } memcpy(curheader, status_buf, VEGAS_STATUS_SIZE); memset(curdata, 0, block_size); memset(curindex, 0, db->index_size); } /* Copy packet into any blocks where it belongs. * The "write packets" functions also update drop stats * for blocks, etc. */ int nblocks = 0; for (i=0; i<nblock; i++) { if ((blocks[i].block_idx>=0) && (block_heap_check(&blocks[i],heap_cntr)==0)) { if (nblocks > 0) { printf("vegas_net_thread: Warning! Adding packet to more than one block! heap_cntr= %d, block = %d",heap_cntr,i); } nblocks++; write_spead_packet_to_block(&blocks[i], &p, heap_cntr, heap_offset, packets_per_heap, bw_mode); } } /* Will exit if thread has been cancelled */ pthread_testcancel(); } pthread_exit(NULL); /* Have to close all push's */ pthread_cleanup_pop(0); /* Closes push(vegas_udp_close) */ pthread_cleanup_pop(0); /* Closes set_exit_status */ pthread_cleanup_pop(0); /* Closes vegas_free_psrfits */ pthread_cleanup_pop(0); /* Closes vegas_status_detach */ pthread_cleanup_pop(0); /* Closes vegas_databuf_detach */ }
static void *run(void * _args) { // Cast _args struct guppi_thread_args *args = (struct guppi_thread_args *)_args; #ifdef DEBUG_SEMS fprintf(stderr, "s/tid %lu/ FLUFf/\n", pthread_self()); #endif THREAD_RUN_BEGIN(args); THREAD_RUN_SET_AFFINITY_PRIORITY(args); /* Attach to status shared mem area */ THREAD_RUN_ATTACH_STATUS(args->instance_id, st); /* Attach to paper_input_databuf */ THREAD_RUN_ATTACH_DATABUF(args->instance_id, paper_input_databuf, db_in, args->input_buffer); /* Attach to paper_gpu_input_databuf */ THREAD_RUN_ATTACH_DATABUF(args->instance_id, paper_gpu_input_databuf, db_out, args->output_buffer); // Init status variables guppi_status_lock_safe(&st); hputi8(st.buf, "FLUFMCNT", 0); guppi_status_unlock_safe(&st); /* Loop */ int rv; int curblock_in=0; int curblock_out=0; struct timespec start, finish; while (run_threads) { // Note waiting status, // query integrating status // and, if armed, start count guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "waiting"); guppi_status_unlock_safe(&st); // Wait for new input block to be filled while ((rv=paper_input_databuf_wait_filled(db_in, curblock_in)) != GUPPI_OK) { if (rv==GUPPI_TIMEOUT) { guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "blocked_in"); guppi_status_unlock_safe(&st); continue; } else { guppi_error(__FUNCTION__, "error waiting for filled databuf"); run_threads=0; pthread_exit(NULL); break; } } // Wait for new gpu_input block (our output block) to be free while ((rv=paper_gpu_input_databuf_wait_free(db_out, curblock_out)) != GUPPI_OK) { if (rv==GUPPI_TIMEOUT) { guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "blocked gpu input"); guppi_status_unlock_safe(&st); continue; } else { guppi_error(__FUNCTION__, "error waiting for free databuf"); run_threads=0; pthread_exit(NULL); break; } } // Got a new data block, update status guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "fluffing"); hputi4(st.buf, "FLUFBKIN", curblock_in); hputu8(st.buf, "FLUFMCNT", db_in->block[curblock_in].header.mcnt); guppi_status_unlock_safe(&st); // Copy header and call fluff function clock_gettime(CLOCK_MONOTONIC, &start); memcpy(&db_out->block[curblock_out].header, &db_in->block[curblock_in].header, sizeof(paper_input_header_t)); paper_fluff(db_in->block[curblock_in].data, db_out->block[curblock_out].data); clock_gettime(CLOCK_MONOTONIC, &finish); // Note processing time guppi_status_lock_safe(&st); // Bits per fluff / ns per fluff = Gbps hputr4(st.buf, "FLUFGBPS", (float)(8*N_BYTES_PER_BLOCK)/ELAPSED_NS(start,finish)); guppi_status_unlock_safe(&st); // Mark input block as free and advance paper_input_databuf_set_free(db_in, curblock_in); curblock_in = (curblock_in + 1) % db_in->header.n_block; // Mark output block as full and advance paper_gpu_input_databuf_set_filled(db_out, curblock_out); curblock_out = (curblock_out + 1) % db_out->header.n_block; /* Check for cancel */ pthread_testcancel(); } run_threads=0; // Have to close all pushes THREAD_RUN_DETACH_DATAUF; THREAD_RUN_DETACH_DATAUF; THREAD_RUN_DETACH_STATUS; THREAD_RUN_END; // Thread success! return NULL; }
static void *run(hashpipe_thread_args_t * args, int doCPU) { // Local aliases to shorten access to args fields paper_gpu_input_databuf_t *db_in = (paper_gpu_input_databuf_t *)args->ibuf; paper_output_databuf_t *db_out = (paper_output_databuf_t *)args->obuf; hashpipe_status_t st = args->st; const char * status_key = args->thread_desc->skey; #ifdef DEBUG_SEMS fprintf(stderr, "s/tid %lu/ GPU/\n", pthread_self()); #endif // Init integration control status variables int gpu_dev = 0; hashpipe_status_lock_safe(&st); hputs(st.buf, "INTSTAT", "off"); hputi8(st.buf, "INTSYNC", 0); hputi4(st.buf, "INTCOUNT", N_SUB_BLOCKS_PER_INPUT_BLOCK); hputi8(st.buf, "GPUDUMPS", 0); hgeti4(st.buf, "GPUDEV", &gpu_dev); // No change if not found hputi4(st.buf, "GPUDEV", gpu_dev); hashpipe_status_unlock_safe(&st); /* Loop */ int rv; char integ_status[17]; uint64_t start_mcount, last_mcount=0; uint64_t gpu_dumps=0; int int_count; // Number of blocks to integrate per dump int xgpu_error = 0; int curblock_in=0; int curblock_out=0; struct timespec start, stop; uint64_t elapsed_gpu_ns = 0; uint64_t gpu_block_count = 0; // Initialize context to point at first input and output memory blocks. // This seems redundant since we do this just before calling // xgpuCudaXengine, but we need to pass something in for array_h and // matrix_x to prevent xgpuInit from allocating memory. XGPUContext context; context.array_h = (ComplexInput *)db_in->block[0].data; context.array_len = (db_in->header.n_block * sizeof(paper_gpu_input_block_t) - sizeof(paper_input_header_t)) / sizeof(ComplexInput); context.matrix_h = (Complex *)db_out->block[0].data; context.matrix_len = (db_out->header.n_block * sizeof(paper_output_block_t) - sizeof(paper_output_header_t)) / sizeof(Complex); xgpu_error = xgpuInit(&context, gpu_dev); if (XGPU_OK != xgpu_error) { fprintf(stderr, "ERROR: xGPU initialization failed (error code %d)\n", xgpu_error); return THREAD_ERROR; } while (run_threads()) { // Note waiting status, // query integrating status // and, if armed, start count hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "waiting"); hgets(st.buf, "INTSTAT", 16, integ_status); hgeti8(st.buf, "INTSYNC", (long long*)&start_mcount); hashpipe_status_unlock_safe(&st); // Wait for new input block to be filled while ((rv=hashpipe_databuf_wait_filled((hashpipe_databuf_t *)db_in, curblock_in)) != HASHPIPE_OK) { if (rv==HASHPIPE_TIMEOUT) { hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "blocked_in"); hashpipe_status_unlock_safe(&st); continue; } else { hashpipe_error(__FUNCTION__, "error waiting for filled databuf"); pthread_exit(NULL); break; } } // Got a new data block, update status and determine how to handle it hashpipe_status_lock_safe(&st); hputi4(st.buf, "GPUBLKIN", curblock_in); hputu8(st.buf, "GPUMCNT", db_in->block[curblock_in].header.mcnt); hashpipe_status_unlock_safe(&st); // If integration status "off" if(!strcmp(integ_status, "off")) { // Mark input block as free and advance hashpipe_databuf_set_free((hashpipe_databuf_t *)db_in, curblock_in); curblock_in = (curblock_in + 1) % db_in->header.n_block; // Skip to next input buffer continue; } // If integration status is "start" if(!strcmp(integ_status, "start")) { // If buffer mcount < start_mcount (i.e. not there yet) if(db_in->block[curblock_in].header.mcnt < start_mcount) { // Drop input buffer // Mark input block as free and advance hashpipe_databuf_set_free((hashpipe_databuf_t *)db_in, curblock_in); curblock_in = (curblock_in + 1) % db_in->header.n_block; // Skip to next input buffer continue; // Else if mcount == start_mcount (time to start) } else if(db_in->block[curblock_in].header.mcnt == start_mcount) { // Set integration status to "on" // Read integration count (INTCOUNT) fprintf(stderr, "--- integration on ---\n"); strcpy(integ_status, "on"); hashpipe_status_lock_safe(&st); hputs(st.buf, "INTSTAT", integ_status); hgeti4(st.buf, "INTCOUNT", &int_count); hashpipe_status_unlock_safe(&st); // Compute last mcount last_mcount = start_mcount + (int_count-1) * N_SUB_BLOCKS_PER_INPUT_BLOCK; // Else (missed starting mcount) } else { // Handle missed start of integration // TODO! fprintf(stderr, "--- mcnt=%06lx > start_mcnt=%06lx ---\n", db_in->block[curblock_in].header.mcnt, start_mcount); } } // Integration status is "on" or "stop" // Note processing status hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "processing gpu"); hashpipe_status_unlock_safe(&st); // Setup for current chunk context.input_offset = curblock_in * sizeof(paper_gpu_input_block_t) / sizeof(ComplexInput); context.output_offset = curblock_out * sizeof(paper_output_block_t) / sizeof(Complex); // Call CUDA X engine function int doDump = 0; // Dump if this is the last block or we are doing both CPU and GPU // (GPU and CPU test mode always dumps every input block) if(db_in->block[curblock_in].header.mcnt >= last_mcount || doCPU) { doDump = 1; // Check whether we missed the end of integration. If we get a block // whose mcnt is greater than last_mcount, then for some reason (e.g. // networking problems) we didn't see a block whose mcnt was // last_mcount. This should "never" happen, but it has been seen to // occur when the 10 GbE links have many errors. if(db_in->block[curblock_in].header.mcnt > last_mcount) { // Can't do much error recovery, so just log it. fprintf(stderr, "--- mcnt=%06lx > last_mcnt=%06lx ---\n", db_in->block[curblock_in].header.mcnt, last_mcount); } // Wait for new output block to be free while ((rv=paper_output_databuf_wait_free(db_out, curblock_out)) != HASHPIPE_OK) { if (rv==HASHPIPE_TIMEOUT) { hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "blocked gpu out"); hashpipe_status_unlock_safe(&st); continue; } else { hashpipe_error(__FUNCTION__, "error waiting for free databuf"); pthread_exit(NULL); break; } } } clock_gettime(CLOCK_MONOTONIC, &start); xgpuCudaXengine(&context, doDump ? SYNCOP_DUMP : SYNCOP_SYNC_TRANSFER); clock_gettime(CLOCK_MONOTONIC, &stop); elapsed_gpu_ns += ELAPSED_NS(start, stop); gpu_block_count++; if(doDump) { clock_gettime(CLOCK_MONOTONIC, &start); xgpuClearDeviceIntegrationBuffer(&context); clock_gettime(CLOCK_MONOTONIC, &stop); elapsed_gpu_ns += ELAPSED_NS(start, stop); // TODO Maybe need to subtract all or half the integration time here // depending on recevier's expectations. db_out->block[curblock_out].header.mcnt = last_mcount; // If integration status if "stop" if(!strcmp(integ_status, "stop")) { // Set integration status to "off" strcpy(integ_status, "off"); hashpipe_status_lock_safe(&st); hputs(st.buf, "INTSTAT", integ_status); hashpipe_status_unlock_safe(&st); } else { // Advance last_mcount for end of next integration last_mcount += int_count * N_SUB_BLOCKS_PER_INPUT_BLOCK; } // Mark output block as full and advance paper_output_databuf_set_filled(db_out, curblock_out); curblock_out = (curblock_out + 1) % db_out->header.n_block; // TODO Need to handle or at least check for overflow! // Update GPU dump counter and GPU Gbps gpu_dumps++; hashpipe_status_lock_safe(&st); hputi8(st.buf, "GPUDUMPS", gpu_dumps); hputr4(st.buf, "GPUGBPS", (float)(8*N_FLUFFED_BYTES_PER_BLOCK*gpu_block_count)/elapsed_gpu_ns); hashpipe_status_unlock_safe(&st); // Start new average elapsed_gpu_ns = 0; gpu_block_count = 0; } if(doCPU) { /* Note waiting status */ hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "waiting"); hashpipe_status_unlock_safe(&st); // Wait for new output block to be free while ((rv=paper_output_databuf_wait_free(db_out, curblock_out)) != HASHPIPE_OK) { if (rv==HASHPIPE_TIMEOUT) { hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "blocked cpu out"); hashpipe_status_unlock_safe(&st); continue; } else { hashpipe_error(__FUNCTION__, "error waiting for free databuf"); pthread_exit(NULL); break; } } // Note "processing cpu" status, current input block hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "processing cpu"); hashpipe_status_unlock_safe(&st); /* * Call CPU X engine function */ xgpuOmpXengine((Complex *)db_out->block[curblock_out].data, context.array_h); // Mark output block as full and advance paper_output_databuf_set_filled(db_out, curblock_out); curblock_out = (curblock_out + 1) % db_out->header.n_block; // TODO Need to handle or at least check for overflow! } // Mark input block as free and advance hashpipe_databuf_set_free((hashpipe_databuf_t *)db_in, curblock_in); curblock_in = (curblock_in + 1) % db_in->header.n_block; /* Check for cancel */ pthread_testcancel(); } xgpuFree(&context); // Thread success! return NULL; }
int main(int argc, char *argv[]) { static struct option long_opts[] = { {"help", 0, NULL, 'h'}, {"null", 0, NULL, 'n'}, {"ds", 0, NULL, 'D'}, {0,0,0,0} }; int use_null_thread = 0; int ds = 0; int opt, opti; while ((opt=getopt_long(argc,argv,"hnD",long_opts,&opti))!=-1) { switch (opt) { case 'n': use_null_thread = 1; break; case 'D': ds = 1; break; default: case 'h': usage(); exit(0); break; } } int twogpu = 1; // -- Ids -- thread_args net_args, dedisp_args, disk_args; thread_args net_args_2, dedisp_args_2, disk_args_2; thread_args_init(&net_args); thread_args_init(&dedisp_args); thread_args_init(&disk_args); net_args.output_buffer = 1; dedisp_args.input_buffer = net_args.output_buffer; dedisp_args.output_buffer = 2; disk_args.input_buffer = dedisp_args.output_buffer; disk_args.output_buffer = 3; net_args.gpu_id = dedisp_args.gpu_id = disk_args.gpu_id = 1; net_args.priority = 15; dedisp_args.priority = 20; disk_args.priority = 20; // -- Attach to status shared mem -- status stat; databuf *dbuf_net=NULL, *dbuf_fold=NULL; int rv = status_attach(&stat, 1); if (rv!=OK) { log_error("nuppi_daq_dedisp", "Error connecting to status shm"); exit(1); } // -- Init mutex for PSRFITS template -- rv = pthread_mutex_init(&lock_psrfits_tpl, NULL); if (rv != 0) { log_error("nuppi_daq_dedisp", "Mutex initialization failed"); exit(1); } // -- Read status shm, init RUN and read filename -- char basename[256]; status_lock_safe(&stat); hgets(stat.buf, "BASENAME", 256, basename); hputi4(stat.buf, "RUN", 1); status_unlock_safe(&stat); run=1; signal(SIGINT, cc); // -- Create or just attach to net shared mem -- dbuf_net = databuf_attach(net_args.output_buffer); if (dbuf_net==NULL) dbuf_net = databuf_create(8, 256*1024*1024, net_args.output_buffer); if (dbuf_net==NULL) { log_error("nuppi_daq_dedisp", "Error connecting to databuf net shm"); exit(1); } databuf_clear(dbuf_net); // -- Create or just attach to fold shared mem -- dbuf_fold = databuf_attach(dedisp_args.output_buffer); if (dbuf_fold==NULL) dbuf_fold = databuf_create(8, 128*1024*1024, dedisp_args.output_buffer); if (dbuf_fold==NULL) { log_error("nuppi_daq_dedisp", "Error connecting to databuf fold shm"); exit(1); } databuf_clear(dbuf_fold); /* Launch net thread */ pthread_t net_thread_id; rv = pthread_create(&net_thread_id, NULL, net_thread, (void *)&net_args); if (rv) { log_error("nuppi_daq_dedisp", "Error creating net thread"); perror("pthread_create"); exit(1); } /* Launch dedisp thread */ pthread_t dedisp_thread_id; if (ds) rv = pthread_create(&dedisp_thread_id, NULL, dedisp_ds_thread, (void *)&dedisp_args); else rv = pthread_create(&dedisp_thread_id, NULL, dedisp_thread, (void *)&dedisp_args); if (rv) { log_error("nuppi_daq_dedisp", "Error creating dedisp thread"); perror("pthread_create"); exit(1); } /* Launch psrfits/null thread */ pthread_t disk_thread_id=0; if (use_null_thread) rv = pthread_create(&disk_thread_id, NULL, null_thread, (void *)&disk_args); else rv = pthread_create(&disk_thread_id, NULL, psrfits_thread, (void *)&disk_args); if (rv) { log_error("nuppi_daq_dedisp", "Error creating psrfits thread"); perror("pthread_create"); exit(1); } // -- Second GPU -- pthread_t net_thread_id_2; pthread_t dedisp_thread_id_2; pthread_t disk_thread_id_2=0; if(twogpu) { sleep(2); // -- Ids -- thread_args_init(&net_args_2); thread_args_init(&dedisp_args_2); thread_args_init(&disk_args_2); net_args_2.output_buffer = 5; dedisp_args_2.input_buffer = net_args_2.output_buffer; dedisp_args_2.output_buffer = 6; disk_args_2.input_buffer = dedisp_args_2.output_buffer; disk_args_2.output_buffer = 7; net_args_2.gpu_id = dedisp_args_2.gpu_id = disk_args_2.gpu_id = 2; net_args_2.priority = 20; dedisp_args_2.priority = 20; disk_args_2.priority = 20; databuf *dbuf_net_2=NULL, *dbuf_fold_2=NULL; // -- Create or just attach to net shared mem -- dbuf_net_2 = databuf_attach(net_args_2.output_buffer); if (dbuf_net_2==NULL) dbuf_net_2 = databuf_create(8, 256*1024*1024, net_args_2.output_buffer); if (dbuf_net_2==NULL) { log_error("nuppi_daq_dedisp", "Error connecting to databuf net shm for GPU #2"); exit(1); } databuf_clear(dbuf_net_2); // -- Create or just attach to fold shared mem -- dbuf_fold_2 = databuf_attach(dedisp_args_2.output_buffer); if (dbuf_fold_2==NULL) dbuf_fold_2 = databuf_create(8, 128*1024*1024, dedisp_args_2.output_buffer); if (dbuf_fold_2==NULL) { log_error("nuppi_daq_dedisp", "Error connecting to databuf fold shm for GPU #2"); exit(1); } databuf_clear(dbuf_fold_2); /* Launch net thread */ rv = pthread_create(&net_thread_id_2, NULL, net_thread, (void *)&net_args_2); if (rv) { log_error("nuppi_daq_dedisp", "Error creating net thread 2"); perror("pthread_create"); exit(1); } /* Launch dedisp thread */ if (ds) rv = pthread_create(&dedisp_thread_id_2, NULL, dedisp_ds_thread, (void *)&dedisp_args_2); else rv = pthread_create(&dedisp_thread_id_2, NULL, dedisp_thread, (void *)&dedisp_args_2); if (rv) { log_error("nuppi_daq_dedisp", "Error creating dedisp thread 2"); perror("pthread_create"); exit(1); } /* Launch psrfits/null thread */ if (use_null_thread) rv = pthread_create(&disk_thread_id_2, NULL, null_thread, (void *)&disk_args_2); else rv = pthread_create(&disk_thread_id_2, NULL, psrfits_thread, (void *)&disk_args_2); if (rv) { log_error("nuppi_daq_dedisp", "Error creating psrfits thread 2"); perror("pthread_create"); exit(1); } } /* Alt loop, wait for run=0 */ while (run) { sleep(1); // Read the RUN keyword in the first status shm, to look for a stop order status_lock_safe(&stat); hgeti4(stat.buf, "RUN", &run); status_unlock_safe(&stat); if (run == 0) log_info("nuppi_daq_dedisp", "Caught RUN = 0 signal for end of observation"); if (disk_args.finished) run=0; } /* Clean up */ pthread_cancel(dedisp_thread_id); pthread_cancel(net_thread_id); pthread_cancel(disk_thread_id); pthread_kill(dedisp_thread_id,SIGINT); pthread_kill(net_thread_id,SIGINT); pthread_kill(disk_thread_id,SIGINT); if(twogpu) { pthread_cancel(dedisp_thread_id_2); pthread_cancel(net_thread_id_2); pthread_cancel(disk_thread_id_2); pthread_kill(dedisp_thread_id_2,SIGINT); pthread_kill(net_thread_id_2,SIGINT); pthread_kill(disk_thread_id_2,SIGINT); } // -- Join threads -- pthread_join(net_thread_id,NULL); log_info("nuppi_daq_dedisp", "Joined net thread"); if(twogpu) { pthread_join(net_thread_id_2,NULL); log_info("nuppi_daq_dedisp", "Joined net thread 2"); } pthread_join(dedisp_thread_id,NULL); log_info("nuppi_daq_dedisp", "Joined dedisp thread"); if(twogpu) { pthread_join(dedisp_thread_id_2,NULL); log_info("nuppi_daq_dedisp", "Joined dedisp thread 2"); } pthread_join(disk_thread_id,NULL); log_info("nuppi_daq_dedisp", "Joined disk thread"); if(twogpu) { pthread_join(disk_thread_id_2,NULL); log_info("nuppi_daq_dedisp", "Joined disk thread 2"); } // -- Destroy args -- thread_args_destroy(&net_args); thread_args_destroy(&dedisp_args); thread_args_destroy(&disk_args); thread_args_destroy(&net_args_2); thread_args_destroy(&dedisp_args_2); thread_args_destroy(&disk_args_2); pthread_mutex_destroy(&lock_psrfits_tpl); char cmd[128], hostname[128]; gethostname(hostname, 127); sprintf(cmd, "mv %s /home/pulsar/data/%s-%s.log", LOG_FILENAME, basename, hostname); log_info("nuppi_daq_dedisp", cmd); system(cmd); exit(0); }
int main(int argc, char *argv[]) { int instance_id = 0; hashpipe_status_t *s; /* Loop over cmd line to fill in params */ static struct option long_opts[] = { {"help", 0, NULL, 'h'}, {"shmkey", 1, NULL, 'K'}, {"key", 1, NULL, 'k'}, {"get", 1, NULL, 'g'}, {"string", 1, NULL, 's'}, {"float", 1, NULL, 'f'}, {"double", 1, NULL, 'd'}, {"int", 1, NULL, 'i'}, {"verbose", 0, NULL, 'v'}, {"clear", 0, NULL, 'C'}, {"del", 0, NULL, 'D'}, {"query", 1, NULL, 'Q'}, {"instance", 1, NULL, 'I'}, {0,0,0,0} }; int opt,opti; char *key=NULL; char value[81]; float flttmp; double dbltmp; int inttmp; int verbose=0, clear=0; char keyfile[1000]; while ((opt=getopt_long(argc,argv,"hk:g:s:f:d:i:vCDQ:K:I:",long_opts,&opti))!=-1) { switch (opt) { case 'K': // Keyfile snprintf(keyfile, sizeof(keyfile), "HASHPIPE_KEYFILE=%s", optarg); keyfile[sizeof(keyfile)-1] = '\0'; putenv(keyfile); break; case 'I': instance_id = atoi(optarg); break; case 'k': key = optarg; break; case 'Q': s = get_status_buffer(instance_id); hashpipe_status_lock(s); hgets(s->buf, optarg, 80, value); hashpipe_status_unlock(s); value[80] = '\0'; printf("%s\n", value); break; case 'g': s = get_status_buffer(instance_id); hashpipe_status_lock(s); hgetr8(s->buf, optarg, &dbltmp); hashpipe_status_unlock(s); printf("%g\n", dbltmp); break; case 's': if (key) { s = get_status_buffer(instance_id); hashpipe_status_lock(s); hputs(s->buf, key, optarg); hashpipe_status_unlock(s); } break; case 'f': flttmp = atof(optarg); if (key) { s = get_status_buffer(instance_id); hashpipe_status_lock(s); hputr4(s->buf, key, flttmp); hashpipe_status_unlock(s); } break; case 'd': dbltmp = atof(optarg); if (key) { s = get_status_buffer(instance_id); hashpipe_status_lock(s); hputr8(s->buf, key, dbltmp); hashpipe_status_unlock(s); } break; case 'i': inttmp = atoi(optarg); if (key) { s = get_status_buffer(instance_id); hashpipe_status_lock(s); hputi4(s->buf, key, inttmp); hashpipe_status_unlock(s); } break; case 'D': if (key) { s = get_status_buffer(instance_id); hashpipe_status_lock(s); hdel(s->buf, key); hashpipe_status_unlock(s); } break; case 'C': clear=1; break; case 'v': verbose=1; break; case 'h': usage(); return 0; case '?': // Command line parsing error default: usage(); exit(1); break; } } s = get_status_buffer(instance_id); /* If verbose, print out buffer */ if (verbose) { hashpipe_status_lock(s); printf("%s\n", s->buf); hashpipe_status_unlock(s); } if (clear) hashpipe_status_clear(s); exit(0); }
void guppi_psrfits_thread(void *_args) { /* Get args */ struct guppi_thread_args *args = (struct guppi_thread_args *)_args; pthread_cleanup_push((void *)guppi_thread_set_finished, args); /* Set cpu affinity */ cpu_set_t cpuset, cpuset_orig; sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig); CPU_ZERO(&cpuset); CPU_SET(1, &cpuset); int rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); if (rv<0) { guppi_error("guppi_psrfits_thread", "Error setting cpu affinity."); perror("sched_setaffinity"); } /* Set priority */ rv = setpriority(PRIO_PROCESS, 0, args->priority); if (rv<0) { guppi_error("guppi_psrfits_thread", "Error setting priority level."); perror("set_priority"); } /* Attach to status shared mem area */ struct guppi_status st; rv = guppi_status_attach(&st); if (rv!=GUPPI_OK) { guppi_error("guppi_psrfits_thread", "Error attaching to status shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_status_detach, &st); pthread_cleanup_push((void *)set_exit_status, &st); /* Init status */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "init"); guppi_status_unlock_safe(&st); /* Initialize some key parameters */ struct guppi_params gp; struct psrfits pf; pf.sub.data = NULL; pf.sub.dat_freqs = pf.sub.dat_weights = NULL; pf.sub.dat_offsets = pf.sub.dat_scales = NULL; pf.hdr.chan_dm = 0.0; pf.filenum = 0; // This is crucial pthread_cleanup_push((void *)guppi_free_psrfits, &pf); pthread_cleanup_push((void *)psrfits_close, &pf); //pf.multifile = 0; // Use a single file for fold mode pf.multifile = 1; // Use a multiple files for fold mode pf.quiet = 0; // Print a message per each subint written /* Attach to databuf shared mem */ struct guppi_databuf *db; db = guppi_databuf_attach(args->input_buffer); if (db==NULL) { guppi_error("guppi_psrfits_thread", "Error attaching to databuf shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_databuf_detach, db); /* Loop */ int curblock=0, total_status=0, firsttime=1, run=1, got_packet_0=0; int mode=SEARCH_MODE; char *ptr; char tmpstr[256]; struct foldbuf fb; struct polyco pc[64]; memset(pc, 0, sizeof(pc)); int n_polyco_written=0; float *fold_output_array = NULL; int scan_finished=0; signal(SIGINT, cc); do { /* Note waiting status */ guppi_status_lock_safe(&st); if (got_packet_0) sprintf(tmpstr, "waiting(%d)", curblock); else sprintf(tmpstr, "ready"); hputs(st.buf, STATUS_KEY, tmpstr); guppi_status_unlock_safe(&st); /* Wait for buf to have data */ rv = guppi_databuf_wait_filled(db, curblock); if (rv!=0) { // This is a big ol' kludge to avoid this process hanging // due to thread synchronization problems. sleep(1); continue; } /* Note current block */ guppi_status_lock_safe(&st); hputi4(st.buf, "CURBLOCK", curblock); guppi_status_unlock_safe(&st); /* See how full databuf is */ total_status = guppi_databuf_total_status(db); /* Read param structs for this block */ ptr = guppi_databuf_header(db, curblock); if (firsttime) { guppi_read_obs_params(ptr, &gp, &pf); firsttime = 0; } else { guppi_read_subint_params(ptr, &gp, &pf); } /* Find out what mode this data is in */ mode = psrfits_obs_mode(pf.hdr.obs_mode); /* Check if we got both packet 0 and a valid observation * start time. If so, flag writing to start. */ if (got_packet_0==0 && gp.packetindex==0 && gp.stt_valid==1) { got_packet_0 = 1; guppi_read_obs_params(ptr, &gp, &pf); guppi_update_ds_params(&pf); memset(pc, 0, sizeof(pc)); n_polyco_written=0; } /* If actual observation has started, write the data */ if (got_packet_0) { /* Note waiting status */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "writing"); guppi_status_unlock_safe(&st); /* Get the pointer to the current data */ if (mode==FOLD_MODE) { fb.nchan = pf.hdr.nchan; fb.npol = pf.hdr.npol; fb.nbin = pf.hdr.nbin; fb.data = (float *)guppi_databuf_data(db, curblock); fb.count = (unsigned *)(guppi_databuf_data(db, curblock) + foldbuf_data_size(&fb)); fold_output_array = (float *)realloc(fold_output_array, sizeof(float) * pf.hdr.nbin * pf.hdr.nchan * pf.hdr.npol); pf.sub.data = (unsigned char *)fold_output_array; pf.fold.pc = (struct polyco *)(guppi_databuf_data(db,curblock) + foldbuf_data_size(&fb) + foldbuf_count_size(&fb)); } else pf.sub.data = (unsigned char *)guppi_databuf_data(db, curblock); /* Set the DC and Nyquist channels explicitly to zero */ /* because of the "FFT Problem" that splits DC power */ /* into those two bins. */ zero_end_chans(&pf); /* Output only Stokes I (in place) */ if (pf.hdr.onlyI && pf.hdr.npol==4) get_stokes_I(&pf); /* Downsample in frequency (in place) */ if (pf.hdr.ds_freq_fact > 1) downsample_freq(&pf); /* Downsample in time (in place) */ if (pf.hdr.ds_time_fact > 1) downsample_time(&pf); /* Folded data needs a transpose */ if (mode==FOLD_MODE) normalize_transpose_folds(fold_output_array, &fb); /* Write the data */ int last_filenum = pf.filenum; psrfits_write_subint(&pf); /* Any actions that need to be taken when a new file * is created. */ if (pf.filenum!=last_filenum) { /* No polycos yet written to the new file */ n_polyco_written=0; } /* Write the polycos if needed */ int write_pc=0, i, j; for (i=0; i<pf.fold.n_polyco_sets; i++) { if (pf.fold.pc[i].used==0) continue; int new_pc=1; for (j=0; j<n_polyco_written; j++) { if (polycos_differ(&pf.fold.pc[i], &pc[j])==0) { new_pc=0; break; } } if (new_pc || n_polyco_written==0) { pc[n_polyco_written] = pf.fold.pc[i]; n_polyco_written++; write_pc=1; } else { pf.fold.pc[i].used = 0; // Already have this one } } if (write_pc) psrfits_write_polycos(&pf, pf.fold.pc, pf.fold.n_polyco_sets); /* Is the scan complete? */ if ((pf.hdr.scanlen > 0.0) && (pf.T > pf.hdr.scanlen)) scan_finished = 1; /* For debugging... */ if (gp.drop_frac > 0.0) { printf("Block %d dropped %.3g%% of the packets\n", pf.tot_rows, gp.drop_frac*100.0); } } /* Mark as free */ guppi_databuf_set_free(db, curblock); /* Go to next block */ curblock = (curblock + 1) % db->n_block; /* Check for cancel */ pthread_testcancel(); } while (run && !scan_finished); /* Cleanup */ if (fold_output_array!=NULL) free(fold_output_array); pthread_exit(NULL); pthread_cleanup_pop(0); /* Closes psrfits_close */ pthread_cleanup_pop(0); /* Closes guppi_free_psrfits */ pthread_cleanup_pop(0); /* Closes set_exit_status */ pthread_cleanup_pop(0); /* Closes set_finished */ pthread_cleanup_pop(0); /* Closes guppi_status_detach */ pthread_cleanup_pop(0); /* Closes guppi_databuf_detach */ }
void guppi_null_thread(void *_args) { int rv; /* Set cpu affinity */ cpu_set_t cpuset, cpuset_orig; sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig); CPU_ZERO(&cpuset); CPU_SET(6, &cpuset); rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); if (rv<0) { guppi_error("guppi_null_thread", "Error setting cpu affinity."); perror("sched_setaffinity"); } /* Set priority */ rv = setpriority(PRIO_PROCESS, 0, 0); if (rv<0) { guppi_error("guppi_null_thread", "Error setting priority level."); perror("set_priority"); } /* Get args */ struct guppi_thread_args *args = (struct guppi_thread_args *)_args; /* Attach to status shared mem area */ struct guppi_status st; rv = guppi_status_attach(&st); if (rv!=GUPPI_OK) { guppi_error("guppi_null_thread", "Error attaching to status shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_status_detach, &st); pthread_cleanup_push((void *)set_exit_status, &st); /* Init status */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "init"); guppi_status_unlock_safe(&st); /* Attach to databuf shared mem */ struct guppi_databuf *db; db = guppi_databuf_attach(args->input_buffer); if (db==NULL) { guppi_error("guppi_null_thread", "Error attaching to databuf shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_databuf_detach, db); /* Loop */ char *ptr; struct guppi_params gp; #if FITS_TYPE == PSRFITS struct psrfits pf; pf.sub.dat_freqs = NULL; pf.sub.dat_weights = NULL; pf.sub.dat_offsets = NULL; pf.sub.dat_scales = NULL; pthread_cleanup_push((void *)guppi_free_psrfits, &pf); #else struct sdfits pf; pthread_cleanup_push((void *)guppi_free_sdfits, &pf); #endif int curblock=0; signal(SIGINT,cc); while (run_threads) { /* Note waiting status */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "waiting"); guppi_status_unlock_safe(&st); /* Wait for buf to have data */ rv = guppi_databuf_wait_filled(db, curblock); if (rv!=0) { //sleep(1); continue; } /* Note waiting status, current block */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "discarding"); hputi4(st.buf, "DSKBLKIN", curblock); guppi_status_unlock_safe(&st); /* Get params */ ptr = guppi_databuf_header(db, curblock); guppi_read_obs_params(ptr, &gp, &pf); /* Output if data was lost */ #if FITS_TYPE == PSRFITS if (gp.n_dropped!=0 && (gp.packetindex==0 || strcmp(pf.hdr.obs_mode,"SEARCH"))) { printf("Block beginning with pktidx=%lld dropped %d packets\n", gp.packetindex, gp.n_dropped); fflush(stdout); } #else if (gp.num_pkts_dropped!=0 && gp.num_pkts_rcvd!=0) { printf("Block received %d packets and dropped %d packets\n", gp.num_pkts_rcvd, gp.num_pkts_dropped); fflush(stdout); } #endif /* Mark as free */ guppi_databuf_set_free(db, curblock); /* Go to next block */ curblock = (curblock + 1) % db->n_block; /* Check for cancel */ pthread_testcancel(); } pthread_exit(NULL); pthread_cleanup_pop(0); /* Closes set_exit_status */ pthread_cleanup_pop(0); /* Closes guppi_free_psrfits */ pthread_cleanup_pop(0); /* Closes guppi_status_detach */ pthread_cleanup_pop(0); /* Closes guppi_databuf_detach */ }
static void *run(void * _args) { // Cast _args struct guppi_thread_args *args = (struct guppi_thread_args *)_args; THREAD_RUN_BEGIN(args); THREAD_RUN_SET_AFFINITY_PRIORITY(args); THREAD_RUN_ATTACH_STATUS(args->instance_id, st); // Attach to paper_ouput_databuf THREAD_RUN_ATTACH_DATABUF(args->instance_id, paper_output_databuf, db, args->input_buffer); // Setup socket and message structures int sockfd; unsigned int xengine_id = 0; struct timespec packet_delay = { .tv_sec = 0, .tv_nsec = PACKET_DELAY_NS }; guppi_status_lock_safe(&st); hgetu4(st.buf, "XID", &xengine_id); // No change if not found hputu4(st.buf, "XID", xengine_id); hputu4(st.buf, "OUTDUMPS", 0); guppi_status_unlock_safe(&st); pkt_t pkt; pkt.hdr.header = HEADER; pkt.hdr.instids = INSTIDS(xengine_id); pkt.hdr.pktinfo = PKTINFO(BYTES_PER_PACKET); pkt.hdr.heaplen = HEAPLEN; // TODO Get catcher hostname and port from somewhere #ifndef CATCHER_PORT #define CATCHER_PORT 7148 #endif #define stringify2(x) #x #define stringify(x) stringify2(x) // Open socket sockfd = open_udp_socket("catcher", stringify(CATCHER_PORT)); if(sockfd == -1) { guppi_error(__FUNCTION__, "error opening socket"); run_threads=0; pthread_exit(NULL); } #ifdef TEST_INDEX_CALCS int i, j; for(i=0; i<32; i++) { for(j=i; j<32; j++) { regtile_index(2*i, 2*j); } } for(i=0; i<32; i++) { for(j=i; j<32; j++) { casper_index(2*i, 2*j); } } run_threads=0; #endif /* Main loop */ int rv; int casper_chan, gpu_chan; int baseline; unsigned int dumps = 0; int block_idx = 0; struct timespec start, stop; signal(SIGINT,cc); signal(SIGTERM,cc); while (run_threads) { guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "waiting"); guppi_status_unlock_safe(&st); // Wait for new block to be filled while ((rv=paper_output_databuf_wait_filled(db, block_idx)) != GUPPI_OK) { if (rv==GUPPI_TIMEOUT) { guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "blocked"); guppi_status_unlock_safe(&st); continue; } else { guppi_error(__FUNCTION__, "error waiting for filled databuf"); run_threads=0; pthread_exit(NULL); break; } } clock_gettime(CLOCK_MONOTONIC, &start); // Note processing status, current input block guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "processing"); hputi4(st.buf, "OUTBLKIN", block_idx); guppi_status_unlock_safe(&st); // Update header's timestamp for this dump pkt.hdr.timestamp = TIMESTAMP(db->block[block_idx].header.mcnt * N_TIME_PER_PACKET * 2 * N_CHAN_TOTAL / 128); // Init header's offset for this dump uint32_t nbytes = 0; pkt.hdr.offset = OFFSET(nbytes); // Unpack and convert in packet sized chunks float * pf_re = db->block[block_idx].data; float * pf_im = db->block[block_idx].data + xgpu_info.matLength; pktdata_t * p_out = pkt.data; for(casper_chan=0; casper_chan<N_CHAN_PER_X; casper_chan++) { // De-interleave the channels gpu_chan = (casper_chan/Nc) + ((casper_chan%Nc)*Nx); for(baseline=0; baseline<CASPER_CHAN_LENGTH; baseline++) { off_t idx_regtile = idx_map[baseline]; pktdata_t re = CONVERT(pf_re[gpu_chan*REGTILE_CHAN_LENGTH+idx_regtile]); pktdata_t im = CONVERT(pf_im[gpu_chan*REGTILE_CHAN_LENGTH+idx_regtile]); *p_out++ = re; *p_out++ = -im; // Conjugate data to match downstream expectations nbytes += 2*sizeof(pktdata_t); if(nbytes % BYTES_PER_PACKET == 0) { int bytes_sent = send(sockfd, &pkt, sizeof(pkt.hdr)+BYTES_PER_PACKET, 0); if(bytes_sent == -1) { // Send all packets even if cactcher is not listening (i.e. we // we get a connection refused error), but abort sending this // dump if we get any other error. if(errno != ECONNREFUSED) { perror("send"); // Update stats guppi_status_lock_safe(&st); hputu4(st.buf, "OUTDUMPS", ++dumps); hputr4(st.buf, "OUTSECS", 0.0); hputr4(st.buf, "OUTMBPS", 0.0); guppi_status_unlock_safe(&st); // Break out of both for loops goto done_sending; } } else if(bytes_sent != sizeof(pkt.hdr)+BYTES_PER_PACKET) { printf("only sent %d of %lu bytes!!!\n", bytes_sent, sizeof(pkt.hdr)+BYTES_PER_PACKET); } // Delay to prevent overflowing network TX queue nanosleep(&packet_delay, NULL); // Setup for next packet p_out = pkt.data; // Update header's byte_offset for this chunk pkt.hdr.offset = OFFSET(nbytes); } } } clock_gettime(CLOCK_MONOTONIC, &stop); guppi_status_lock_safe(&st); hputu4(st.buf, "OUTDUMPS", ++dumps); hputr4(st.buf, "OUTSECS", (float)ELAPSED_NS(start,stop)/1e9); hputr4(st.buf, "OUTMBPS", (1e3*8*bytes_per_dump)/ELAPSED_NS(start,stop)); guppi_status_unlock_safe(&st); done_sending: // Mark block as free paper_output_databuf_set_free(db, block_idx); // Setup for next block block_idx = (block_idx + 1) % db->header.n_block; /* Will exit if thread has been cancelled */ pthread_testcancel(); } // Have to close all pushes THREAD_RUN_DETACH_DATAUF; THREAD_RUN_DETACH_STATUS; THREAD_RUN_END; // Thread success! return NULL; } static pipeline_thread_module_t module = { name: "paper_gpu_output_thread", type: PIPELINE_OUTPUT_THREAD, init: init, run: run };
/* This thread is passed a single arg, pointer * to the guppi_udp_params struct. This thread should * be cancelled and restarted if any hardware params * change, as this potentially affects packet size, etc. */ void *guppi_fake_net_thread(void *_args) { /* Get arguments */ struct guppi_thread_args *args = (struct guppi_thread_args *)_args; /* Set cpu affinity */ cpu_set_t cpuset, cpuset_orig; sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig); CPU_ZERO(&cpuset); //CPU_SET(2, &cpuset); CPU_SET(3, &cpuset); int rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); if (rv<0) { guppi_error("guppi_fake_net_thread", "Error setting cpu affinity."); perror("sched_setaffinity"); } /* Set priority */ rv = setpriority(PRIO_PROCESS, 0, args->priority); if (rv<0) { guppi_error("guppi_fake_net_thread", "Error setting priority level."); perror("set_priority"); } /* Attach to status shared mem area */ struct guppi_status st; rv = guppi_status_attach(&st); if (rv!=GUPPI_OK) { guppi_error("guppi_fake_net_thread", "Error attaching to status shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_status_detach, &st); pthread_cleanup_push((void *)set_exit_status, &st); /* Init status, read info */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "init"); guppi_status_unlock_safe(&st); /* Read in general parameters */ struct guppi_params gp; #if FITS_TYPE == PSRFITS struct psrfits pf; pf.sub.dat_freqs = NULL; pf.sub.dat_weights = NULL; pf.sub.dat_offsets = NULL; pf.sub.dat_scales = NULL; #else struct sdfits pf; #endif char status_buf[GUPPI_STATUS_SIZE]; guppi_status_lock_safe(&st); memcpy(status_buf, st.buf, GUPPI_STATUS_SIZE); guppi_status_unlock_safe(&st); guppi_read_obs_params(status_buf, &gp, &pf); #if FITS_TYPE == PSRFITS pthread_cleanup_push((void *)guppi_free_psrfits, &pf); #else pthread_cleanup_push((void *)guppi_free_sdfits, &pf); #endif /* Attach to databuf shared mem */ struct guppi_databuf *db; db = guppi_databuf_attach(args->output_buffer); if (db==NULL) { guppi_error("guppi_fake_net_thread", "Error attaching to databuf shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_databuf_detach, db); /* Time parameters */ int stt_imjd=0, stt_smjd=0; double stt_offs=0.0; /* Figure out size of data in each packet, number of packets * per block, etc. Changing packet size during an obs is not * recommended. */ int block_size; if (hgeti4(status_buf, "BLOCSIZE", &block_size)==0) { block_size = db->block_size; hputi4(status_buf, "BLOCSIZE", block_size); } else { if (block_size > db->block_size) { guppi_error("guppi_net_thread", "BLOCSIZE > databuf block_size"); block_size = db->block_size; hputi4(status_buf, "BLOCSIZE", block_size); } } unsigned heaps_per_block = block_size / sizeof(struct freq_spead_heap); /* List of databuf blocks currently in use */ unsigned i; const int nblock = 2; struct fake_datablock_stats blocks[nblock]; for (i=0; i<nblock; i++) fake_init_block(&blocks[i], db, sizeof(struct freq_spead_heap), heaps_per_block); /* Convenience names for first/last blocks in set */ struct fake_datablock_stats *fblock, *lblock; fblock = &blocks[0]; lblock = &blocks[nblock-1]; /* Misc counters, etc */ char *curdata=NULL, *curheader=NULL, *curindex=NULL; int first_time = 1; int heap_cntr = 0, next_block_heap_cntr = heaps_per_block; /* Main loop */ unsigned force_new_block=0, waiting=-1; signal(SIGINT,cc); while (run_threads) { /* Wait for data */ struct timespec sleep_dur, rem_sleep_dur; sleep_dur.tv_sec = 0; sleep_dur.tv_nsec = 2e6; nanosleep(&sleep_dur, &rem_sleep_dur); /* Update status if needed */ if (waiting!=0) { guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "receiving"); guppi_status_unlock_safe(&st); waiting=0; } /* Convert packet format if needed */ if (first_time) { first_time = 0; force_new_block=1; } else force_new_block=0; /* Determine if we go to next block */ if ((heap_cntr>=next_block_heap_cntr) || force_new_block) { printf("casper: going to next shared memory block\n"); /* Update drop stats */ guppi_status_lock_safe(&st); hputr8(st.buf, "DROPAVG", 0.0); hputr8(st.buf, "DROPTOT", 0.0); hputr8(st.buf, "DROPBLK", 0.0); guppi_status_unlock_safe(&st); /* Finalize first block, and push it off the list. * Then grab next available block. */ if (fblock->block_idx>=0) fake_finalize_block(fblock); fake_block_stack_push(blocks, nblock); fake_increment_block(lblock, heap_cntr); curdata = guppi_databuf_data(db, lblock->block_idx); curheader = guppi_databuf_header(db, lblock->block_idx); curindex = guppi_databuf_index(db, lblock->block_idx); next_block_heap_cntr = lblock->heap_idx + heaps_per_block; /* If new obs started, reset total counters, get start * time. Start time is rounded to nearest integer * second, with warning if we're off that by more * than 100ms. Any current blocks on the stack * are also finalized/reset */ if (force_new_block) { /* Get obs start time */ get_current_mjd(&stt_imjd, &stt_smjd, &stt_offs); if (stt_offs>0.5) { stt_smjd+=1; stt_offs-=1.0; } stt_offs = 0.0; /* Flush any current buffers */ for (i=0; i<nblock-1; i++) { if (blocks[i].block_idx>=0) fake_finalize_block(&blocks[i]); fake_reset_block(&blocks[i]); } } /* Read/update current status shared mem */ guppi_status_lock_safe(&st); if (stt_imjd!=0) { #if 1 hputi4(st.buf, "STT_IMJD", stt_imjd); hputi4(st.buf, "STT_SMJD", stt_smjd); hputr8(st.buf, "STT_OFFS", stt_offs); #endif hputi4(st.buf, "STTVALID", 1); } else { hputi4(st.buf, "STTVALID", 0); } memcpy(status_buf, st.buf, GUPPI_STATUS_SIZE); guppi_status_unlock_safe(&st); /* Wait for new block to be free, then clear it * if necessary and fill its header with new values. */ while ((rv=guppi_databuf_wait_free(db, lblock->block_idx)) != GUPPI_OK) { if (rv==GUPPI_TIMEOUT) { waiting=1; guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "blocked"); guppi_status_unlock_safe(&st); continue; } else { guppi_error("guppi_fake_net_thread", "error waiting for free databuf"); run_threads=0; pthread_exit(NULL); break; } } memcpy(curheader, status_buf, GUPPI_STATUS_SIZE); memset(curdata, 0, block_size); memset(curindex, 0, db->index_size); } /*Write fake data to block */ write_fake_heap_to_block(lblock, heap_cntr); heap_cntr++; /* Will exit if thread has been cancelled */ pthread_testcancel(); } pthread_exit(NULL); /* Have to close all push's */ pthread_cleanup_pop(0); /* Closes set_exit_status */ pthread_cleanup_pop(0); /* Closes guppi_free_psrfits */ pthread_cleanup_pop(0); /* Closes guppi_status_detach */ pthread_cleanup_pop(0); /* Closes guppi_databuf_detach */ }
// Run method for the thread // It is meant to do the following: // (1) Initialize status buffer // (2) Set up network parameters and socket // (3) Start main loop // (3a) Receive packet on socket // (3b) Error check packet (packet size, etc) // (3c) Call process_packet on received packet // (4) Terminate thread cleanly static void *run(hashpipe_thread_args_t * args) { fprintf(stdout, "N_INPUTS = %d\n", N_INPUTS); fprintf(stdout, "N_CHAN = %d\n", N_CHAN); fprintf(stdout, "N_CHAN_PER_X = %d\n", N_CHAN_PER_X); fprintf(stdout, "N_CHAN_PER_PACKET = %d\n", N_CHAN_PER_PACKET); fprintf(stdout, "N_TIME_PER_PACKET = %d\n", N_TIME_PER_PACKET); fprintf(stdout, "N_TIME_PER_BLOCK = %d\n", N_TIME_PER_BLOCK); fprintf(stdout, "N_BYTES_PER_BLOCK = %d\n", N_BYTES_PER_BLOCK); fprintf(stdout, "N_BYTES_PER_PACKET = %d\n", N_BYTES_PER_PACKET); fprintf(stdout, "N_PACKETS_PER_BLOCK = %d\n", N_PACKETS_PER_BLOCK); fprintf(stdout, "N_COR_MATRIX = %d\n", N_COR_MATRIX); // Local aliases to shorten access to args fields // Our output buffer happens to be a paper_input_databuf flag_input_databuf_t *db = (flag_input_databuf_t *)args->obuf; hashpipe_status_t st = args->st; const char * status_key = args->thread_desc->skey; st_p = &st; // allow global (this source file) access to the status buffer /* Read network params */ fprintf(stdout, "Setting up network parameters\n"); struct hashpipe_udp_params up = { .bindhost = "0.0.0.0", .bindport = 8511, .packet_size = 8008 }; hashpipe_status_lock_safe(&st); // Get info from status buffer if present (no change if not present) hgets(st.buf, "BINDHOST", 80, up.bindhost); hgeti4(st.buf, "BINDPORT", &up.bindport); // Store bind host/port info etc in status buffer hputs(st.buf, "BINDHOST", up.bindhost); hputi4(st.buf, "BINDPORT", up.bindport); hputu4(st.buf, "MISSEDFE", 0); hputu4(st.buf, "MISSEDPK", 0); hputs(st.buf, status_key, "running"); hashpipe_status_unlock_safe(&st); struct hashpipe_udp_packet p; /* Give all the threads a chance to start before opening network socket */ int netready = 0; int corready = 0; int checkready = 0; while (!netready) { sleep(1); // Check the correlator to see if it's ready yet hashpipe_status_lock_safe(&st); hgeti4(st.buf, "CORREADY", &corready); hgeti4(st.buf, "SAVEREADY", &checkready); hashpipe_status_unlock_safe(&st); if (!corready) { continue; } //if (!checkready) { // continue; //} // Check the other threads to see if they're ready yet // TBD // If we get here, then all threads are initialized netready = 1; } sleep(3); /* Set up UDP socket */ fprintf(stderr, "NET: BINDHOST = %s\n", up.bindhost); fprintf(stderr, "NET: BINDPORT = %d\n", up.bindport); int rv = hashpipe_udp_init(&up); if (rv!=HASHPIPE_OK) { hashpipe_error("paper_net_thread", "Error opening UDP socket."); pthread_exit(NULL); } pthread_cleanup_push((void *)hashpipe_udp_close, &up); // Initialize first few blocks in the buffer int i; for (i = 0; i < 2; i++) { // Wait until block semaphore is free if (flag_input_databuf_wait_free(db, i) != HASHPIPE_OK) { if (errno == EINTR) { // Interrupt occurred hashpipe_error(__FUNCTION__, "waiting for free block interrupted\n"); pthread_exit(NULL); } else { hashpipe_error(__FUNCTION__, "error waiting for free block\n"); pthread_exit(NULL); } } initialize_block(db, i*Nm); } // Set correlator to "start" state hashpipe_status_lock_safe(&st); hputs(st.buf, "INTSTAT", "start"); hashpipe_status_unlock_safe(&st); /* Main loop */ uint64_t packet_count = 0; fprintf(stdout, "Net: Starting Thread!\n"); while (run_threads()) { // Get packet do { p.packet_size = recv(up.sock, p.data, HASHPIPE_MAX_PACKET_SIZE, 0); } while (p.packet_size == -1 && (errno == EAGAIN || errno == EWOULDBLOCK) && run_threads()); if(!run_threads()) break; if (up.packet_size != p.packet_size && up.packet_size != p.packet_size-8) { // If an error was returned instead of a valid packet size if (p.packet_size == -1) { fprintf(stderr, "uh oh!\n"); // Log error and exit hashpipe_error("paper_net_thread", "hashpipe_udp_recv returned error"); perror("hashpipe_udp_recv"); pthread_exit(NULL); } else { // Log warning and ignore wrongly sized packet hashpipe_warn("paper_net_thread", "Incorrect pkt size (%d)", p.packet_size); continue; } } packet_count++; process_packet(db, &p); /* Will exit if thread has been cancelled */ pthread_testcancel(); } pthread_cleanup_pop(1); /* Closes push(hashpipe_udp_close) */ hashpipe_status_lock_busywait_safe(&st); hputs(st.buf, status_key, "terminated"); hashpipe_status_unlock_safe(&st); return NULL; } static hashpipe_thread_desc_t net_thread = { name: "flag_net_thread", skey: "NETSTAT", init: NULL, run: run, ibuf_desc: {NULL},
void vegas_pfb_thread(void *_args) { /* Get args */ struct guppi_thread_args *args = (struct guppi_thread_args *)_args; int rv; /* Set cpu affinity */ cpu_set_t cpuset, cpuset_orig; sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig); //CPU_ZERO(&cpuset); CPU_CLR(13, &cpuset); CPU_SET(11, &cpuset); rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); if (rv<0) { guppi_error("vegas_pfb_thread", "Error setting cpu affinity."); perror("sched_setaffinity"); } /* Set priority */ rv = setpriority(PRIO_PROCESS, 0, args->priority); if (rv<0) { guppi_error("vegas_pfb_thread", "Error setting priority level."); perror("set_priority"); } /* Attach to status shared mem area */ struct guppi_status st; rv = guppi_status_attach(&st); if (rv!=GUPPI_OK) { guppi_error("vegas_pfb_thread", "Error attaching to status shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_status_detach, &st); pthread_cleanup_push((void *)set_exit_status, &st); pthread_cleanup_push((void *)guppi_thread_set_finished, args); /* Init status */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "init"); guppi_status_unlock_safe(&st); /* Init structs */ struct guppi_params gp; struct sdfits sf; pthread_cleanup_push((void *)guppi_free_sdfits, &sf); /* Attach to databuf shared mem */ struct guppi_databuf *db_in, *db_out; db_in = guppi_databuf_attach(args->input_buffer); if (db_in==NULL) { char msg[256]; sprintf(msg, "Error attaching to databuf(%d) shared memory.", args->input_buffer); guppi_error("vegas_pfb_thread", msg); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_databuf_detach, db_in); db_out = guppi_databuf_attach(args->output_buffer); if (db_out==NULL) { char msg[256]; sprintf(msg, "Error attaching to databuf(%d) shared memory.", args->output_buffer); guppi_error("vegas_pfb_thread", msg); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_databuf_detach, db_out); /* Loop */ char *hdr_in = NULL; int curblock_in=0; int first=1; int acc_len = 0; int nchan = 0; int nsubband = 0; signal(SIGINT,cc); guppi_status_lock_safe(&st); if (hgeti4(st.buf, "NCHAN", &nchan)==0) { fprintf(stderr, "ERROR: %s not in status shm!\n", "NCHAN"); } if (hgeti4(st.buf, "NSUBBAND", &nsubband)==0) { fprintf(stderr, "ERROR: %s not in status shm!\n", "NSUBBAND"); } guppi_status_unlock_safe(&st); if (EXIT_SUCCESS != init_gpu(db_in->block_size, db_out->block_size, nsubband, nchan)) { (void) fprintf(stderr, "ERROR: GPU initialisation failed!\n"); run = 0; } while (run) { /* Note waiting status */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "waiting"); guppi_status_unlock_safe(&st); /* Wait for buf to have data */ rv = guppi_databuf_wait_filled(db_in, curblock_in); if (rv!=0) continue; /* Note waiting status, current input block */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "processing"); hputi4(st.buf, "PFBBLKIN", curblock_in); guppi_status_unlock_safe(&st); hdr_in = guppi_databuf_header(db_in, curblock_in); /* Get params */ if (first) { guppi_read_obs_params(hdr_in, &gp, &sf); /* Read required exposure from status shared memory, and calculate corresponding accumulation length */ acc_len = (sf.hdr.chan_bw * sf.hdr.hwexposr); } guppi_read_subint_params(hdr_in, &gp, &sf); /* Call PFB function */ do_pfb(db_in, curblock_in, db_out, first, st, acc_len); /* Mark input block as free */ guppi_databuf_set_free(db_in, curblock_in); /* Go to next input block */ curblock_in = (curblock_in + 1) % db_in->n_block; /* Check for cancel */ pthread_testcancel(); if (first) { first=0; } } run=0; //cudaThreadExit(); pthread_exit(NULL); cleanup_gpu(); pthread_cleanup_pop(0); /* Closes guppi_databuf_detach(out) */ pthread_cleanup_pop(0); /* Closes guppi_databuf_detach(in) */ pthread_cleanup_pop(0); /* Closes guppi_free_sdfits */ pthread_cleanup_pop(0); /* Closes guppi_thread_set_finished */ pthread_cleanup_pop(0); /* Closes set_exit_status */ pthread_cleanup_pop(0); /* Closes guppi_status_detach */ }
int main(int argc, char *argv[]) { struct guppi_params gf; struct psrfits pf; char buf[32768]; char quantfilename[250]; //file name for quantized file int filepos=0; size_t rv=0; int by=0; FILE *fil = NULL; //input file FILE *quantfil = NULL; //quantized file int x,y,z; int a,b,c; int sample; double running_sum; double running_sum_sq; double mean[32][2]; //shouldn't be more than 32 dual pol channels in a file for our obs double std[32][2]; int vflag=0; //verbose if(argc < 2) { print_usage(argv); exit(1); } opterr = 0; while ((c = getopt (argc, argv, "Vvi:o:")) != -1) switch (c) { case 'v': vflag = 1; break; case 'V': vflag = 2; break; case 'i': sprintf(pf.basefilename, optarg); fil = fopen(pf.basefilename, "rb"); break; case 'o': sprintf(quantfilename, optarg); if(strcmp(quantfilename, "stdout")==0) { quantfil = stdout; } else { quantfil = fopen(quantfilename, "wb"); } break; case '?': if (optopt == 'i' || optopt == 'o') fprintf (stderr, "Option -%c requires an argument.\n", optopt); else if (isprint (optopt)) fprintf (stderr, "Unknown option `-%c'.\n", optopt); else fprintf (stderr, "Unknown option character `\\x%x'.\n", optopt); return 1; default: abort (); } pf.filenum=1; pf.sub.dat_freqs = (float *)malloc(sizeof(float) * pf.hdr.nchan); pf.sub.dat_weights = (float *)malloc(sizeof(float) * pf.hdr.nchan); pf.sub.dat_offsets = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); pf.sub.dat_scales = (float *)malloc(sizeof(float) * pf.hdr.nchan * pf.hdr.npol); pf.sub.data = (unsigned char *)malloc(pf.sub.bytes_per_subint); if(!fil || !quantfil) { fprintf(stderr, "must specify input/output files\n"); print_usage(argv); exit(1); } filepos=0; while(fread(buf, sizeof(char), 32768, fil)==32768) { fseek(fil, -32768, SEEK_CUR); //printf("lhead: %d", lhead0); if(vflag>=1) fprintf(stderr, "length: %d\n", gethlength(buf)); guppi_read_obs_params(buf, &gf, &pf); printf("%d\n", pf.hdr.nbits); if(vflag>=1) fprintf(stderr, "size %d\n",pf.sub.bytes_per_subint + gethlength(buf)); by = by + pf.sub.bytes_per_subint + gethlength(buf); if(vflag>=1) fprintf(stderr, "mjd %Lf\n", pf.hdr.MJD_epoch); if(vflag>=1) fprintf(stderr, "zen: %f\n\n", pf.sub.tel_zen); if (pf.sub.data) free(pf.sub.data); pf.sub.data = (unsigned char *)malloc(pf.sub.bytes_per_subint); fseek(fil, gethlength(buf), SEEK_CUR); rv=fread(pf.sub.data, sizeof(char), pf.sub.bytes_per_subint, fil); if((long int)rv == pf.sub.bytes_per_subint){ if(vflag>=1) fprintf(stderr, "%i\n", filepos); if(vflag>=1) fprintf(stderr, "pos: %ld %d\n", ftell(fil),feof(fil)); if(filepos == 0) { /* calulcate mean and rms for each channel-polarization */ /* we'll treat the real and imaginary parts identically - considering them as 2 samples/period) */ /* for(x=0;x < pf.hdr.nchan; x = x + 1) { for(y=0;y<pf.hdr.rcvr_polns;y=y+1) { running_sum = 0; running_sum_sq = 0; for(z=0;z < pf.sub.bytes_per_subint/pf.hdr.nchan; z = z + (pf.hdr.rcvr_polns * 2)){ //pol 0, real imag //real sample = (int) ((signed char) pf.sub.data[(x * pf.sub.bytes_per_subint/pf.hdr.nchan) + z + (y * 2)]); running_sum = running_sum + (double) sample; //imag sample = (int) ((signed char) pf.sub.data[(x * pf.sub.bytes_per_subint/pf.hdr.nchan) + z + (y * 2) + 1]); running_sum = running_sum + (double) sample; } mean[x][y] = running_sum / (double) (pf.sub.bytes_per_subint/(pf.hdr.nchan * pf.hdr.rcvr_polns) ); for(z=0;z < pf.sub.bytes_per_subint/pf.hdr.nchan; z = z + (pf.hdr.rcvr_polns * 2)){ //sample = (int) ((signed char) pf.sub.data[(x * pf.sub.bytes_per_subint/pf.hdr.nchan) + z]); //real sample = (int) ((signed char) pf.sub.data[(x * pf.sub.bytes_per_subint/pf.hdr.nchan) + z + (y * 2)]); running_sum_sq = running_sum_sq + pow( ((double) sample - mean[x][y]) , 2); //imag sample = (int) ((signed char) pf.sub.data[(x * pf.sub.bytes_per_subint/pf.hdr.nchan) + z + (y * 2) + 1]); running_sum_sq = running_sum_sq + pow( ((double) sample - mean[x][y]) , 2); } std[x][y] = pow(running_sum_sq / ((double) (pf.sub.bytes_per_subint/(pf.hdr.nchan*pf.hdr.rcvr_polns)) - 1), 0.5); if(vflag>=1) fprintf(stderr, "chan %d pol %d mean %f\n", x,y,mean[x][y]); if(vflag>=1) fprintf(stderr, "chan %d pol %d std %f\n", x,y,std[x][y]); } } */ for(x=0;x<32;x++){ std[x][0] = 19; mean[x][0] = 0; std[x][1] = 19; mean[x][1] = 0; } } quantize_2bit_o(&pf, mean, std); hputi4 (buf, "BLOCSIZE", pf.sub.bytes_per_subint); hputi4 (buf,"NBITS",pf.hdr.nbits); fwrite(buf, sizeof(char), gethlength(buf), quantfil); //write header /* bytes_per_subint now updated to be the proper length */ fwrite(pf.sub.data, sizeof(char), pf.sub.bytes_per_subint, quantfil); //write data filepos++; //pol, time, frequency // for(x=0; x < pf.sub.bytes_per_subint; x = x + 1) { // fprintf(stderr, "old: %d", pf.sub.data[x]); // if(pf.sub.data[x] > 127) { pf.sub.data[x] = pf.sub.data[x] - 256; // fprintf(stderr, "new: %d", pf.sub.data[x]); } // } /* for(x=0; x < pf.sub.bytes_per_subint/pf.hdr.nchan; x = x + (4 * sampsper)) { power=0; for(z=0;z<sampsper;z=z+4){ for(y=0;y<4;y++) { sample = (int) pf.sub.data[x+y+(z*4)]; if(sample > 127) sample = sample - 256; power = power + pow((double) sample, 2); } } //printf("%d, %d\n", pf.sub.data[x], pf.sub.data[x+1]); printf("%f\n",power); } */ } else { if(vflag>=1) fprintf(stderr, "only read %ld bytes...\n", (long int) rv); } } if(vflag>=1) fprintf(stderr, "bytes: %d\n",by); if(vflag>=1) fprintf(stderr, "pos: %ld %d\n", ftell(fil),feof(fil)); //fread(buf, sizeof(char), 32768, fil); //guppi_read_obs_params(buf, &gf, &pf); //printf("mjd %llf\n", pf.hdr.MJD_epoch); //printf("zen: %f", pf.sub.tel_zen); //while ((rv=psrfits_read_subint(&pf))==0) { // printf("Read subint (file %d, row %d/%d)\n", // pf.filenum, pf.rownum-1, pf.rows_per_file); //} //if (rv) { fits_report_error(stderr, rv); } fclose(quantfil); fclose(fil); exit(0); }
void vegas_rawdisk_thread(void *_args) { /* Set cpu affinity */ cpu_set_t cpuset, cpuset_orig; sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig); CPU_ZERO(&cpuset); CPU_SET(6, &cpuset); int rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); if (rv<0) { vegas_error("vegas_rawdisk_thread", "Error setting cpu affinity."); perror("sched_setaffinity"); } /* Get args */ struct vegas_thread_args *args = (struct vegas_thread_args *)_args; /* Set priority */ rv = setpriority(PRIO_PROCESS, 0, 0); if (rv<0) { vegas_error("vegas_rawdisk_thread", "Error setting priority level."); perror("set_priority"); } /* Attach to status shared mem area */ struct vegas_status st; rv = vegas_status_attach(&st); if (rv!=VEGAS_OK) { vegas_error("vegas_rawdisk_thread", "Error attaching to status shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)vegas_status_detach, &st); pthread_cleanup_push((void *)set_exit_status, &st); /* Init status */ vegas_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "init"); vegas_status_unlock_safe(&st); /* Read in general parameters */ struct vegas_params gp; struct sdfits sf; pthread_cleanup_push((void *)vegas_free_sdfits, &sf); /* Attach to databuf shared mem */ struct vegas_databuf *db; db = vegas_databuf_attach(args->input_buffer); if (db==NULL) { vegas_error("vegas_rawdisk_thread", "Error attaching to databuf shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)vegas_databuf_detach, db); /* Init output file */ FILE *fraw = NULL; pthread_cleanup_push((void *)safe_fclose, fraw); /* Loop */ int blocksize=0; int curblock=0, dataset; int block_count=0, blocks_per_file=128, filenum=0; int first=1; char *ptr; float *data_array; struct databuf_index* db_index; signal(SIGINT,cc); while (run) { /* Note waiting status */ vegas_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "waiting"); vegas_status_unlock_safe(&st); /* Wait for buf to have data */ rv = vegas_databuf_wait_filled(db, curblock); if (rv!=0) continue; /* Read param struct and index for this block */ ptr = vegas_databuf_header(db, curblock); db_index = (struct databuf_index*)(vegas_databuf_index(db, curblock)); /* If first time running */ if (first==1) { first = 0; vegas_read_obs_params(ptr, &gp, &sf); char fname[256]; sprintf(fname, "%s_%4d.raw", sf.basefilename, filenum); fprintf(stderr, "Opening raw file '%s'\n", fname); // TODO: check for file exist. fraw = fopen(fname, "wb"); if (fraw==NULL) { vegas_error("vegas_rawdisk_thread", "Error opening file."); pthread_exit(NULL); } } else vegas_read_subint_params(ptr, &gp, &sf); /* See if we need to open next file */ if (block_count >= blocks_per_file) { fclose(fraw); filenum++; char fname[256]; sprintf(fname, "%s_%4d.raw", sf.basefilename, filenum); fprintf(stderr, "Opening raw file '%s'\n", fname); fraw = fopen(fname, "wb"); if (fraw==NULL) { vegas_error("vegas_rawdisk_thread", "Error opening file."); pthread_exit(NULL); } block_count=0; } /* Get full data block size */ hgeti4(ptr, "BLOCSIZE", &blocksize); /* Note writing status and current block */ vegas_status_lock_safe(&st); hputi4(st.buf, "DSKBLKIN", curblock); hputs(st.buf, STATUS_KEY, "writing"); vegas_status_unlock_safe(&st); /* Write all data arrays to disk */ for(dataset = 0; dataset < db_index->num_datasets; dataset++) { data_array = (float*)(vegas_databuf_data(db, curblock) + db_index->disk_buf[dataset].array_offset); rv = fwrite(data_array, 4, (size_t)(db_index->array_size/4), fraw); if (rv != db_index->array_size/4) { vegas_error("vegas_rawdisk_thread", "Error writing data."); } } /* Increment block counter */ block_count++; /* flush output */ fflush(fraw); /* Mark as free */ vegas_databuf_set_free(db, curblock); /* Go to next block */ curblock = (curblock + 1) % db->n_block; /* Check for cancel */ pthread_testcancel(); } pthread_exit(NULL); pthread_cleanup_pop(0); /* Closes fclose */ pthread_cleanup_pop(0); /* Closes vegas_databuf_detach */ pthread_cleanup_pop(0); /* Closes vegas_free_psrfits */ pthread_cleanup_pop(0); /* Closes set_exit_status */ pthread_cleanup_pop(0); /* Closes vegas_status_detach */ }
void vegas_sdfits_thread(void *_args) { /* Get args */ struct vegas_thread_args *args = (struct vegas_thread_args *)_args; pthread_cleanup_push((void *)vegas_thread_set_finished, args); /* Set cpu affinity */ int rv = sched_setaffinity(0, sizeof(cpu_set_t), &args->cpuset); if (rv<0) { vegas_error("vegas_sdfits_thread", "Error setting cpu affinity."); perror("sched_setaffinity"); } /* Set priority */ rv=0; if (args->priority != 0) { struct sched_param priority_param; priority_param.sched_priority = args->priority; rv = pthread_setschedparam(pthread_self(), SCHED_FIFO, &priority_param); } if (rv<0) { vegas_error("vegas_sdfits_thread", "Error setting priority level."); perror("set_priority"); } /* Attach to status shared mem area */ struct vegas_status st; rv = vegas_status_attach(&st); if (rv!=VEGAS_OK) { vegas_error("vegas_sdfits_thread", "Error attaching to status shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)vegas_status_detach, &st); pthread_cleanup_push((void *)set_exit_status, &st); /* Init status */ vegas_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "init"); vegas_status_unlock_safe(&st); /* Initialize some key parameters */ struct vegas_params gp; struct sdfits sf; sf.data_columns.data = NULL; sf.filenum = 0; sf.new_file = 1; // This is crucial pthread_cleanup_push((void *)vegas_free_sdfits, &sf); pthread_cleanup_push((void *)sdfits_close, &sf); //pf.multifile = 0; // Use a single file for fold mode sf.multifile = 1; // Use a multiple files for fold mode sf.quiet = 0; // Print a message per each subint written /* Attach to databuf shared mem */ struct vegas_databuf *db; db = vegas_databuf_attach(args->input_buffer); if (db==NULL) { vegas_error("vegas_sdfits_thread", "Error attaching to databuf shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)vegas_databuf_detach, db); /* Loop */ int curblock=0, total_status=0, firsttime=1, run=1, got_packet_0=0, dataset=0; char *ptr; char tmpstr[256]; int scan_finished=0, old_filenum; int num_exposures_written = 0; int old_integ_num = -1; signal(SIGINT, cc); do { /* Note waiting status */ vegas_status_lock_safe(&st); if (got_packet_0) sprintf(tmpstr, "waiting(%d)", curblock); else sprintf(tmpstr, "ready"); hputs(st.buf, STATUS_KEY, tmpstr); vegas_status_unlock_safe(&st); /* Wait for buf to have data */ rv = vegas_databuf_wait_filled(db, curblock); if (rv!=0) { // This is a big ol' kludge to avoid this process hanging // due to thread synchronization problems. sleep(1); continue; } /* Note current block */ vegas_status_lock_safe(&st); hputi4(st.buf, "DSKBLKIN", curblock); vegas_status_unlock_safe(&st); /* See how full databuf is */ total_status = vegas_databuf_total_status(db); /* Read param structs for this block */ ptr = vegas_databuf_header(db, curblock); if (firsttime) { vegas_read_obs_params(ptr, &gp, &sf); firsttime = 0; } else { vegas_read_subint_params(ptr, &gp, &sf); } /* Note waiting status */ vegas_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "writing"); vegas_status_unlock_safe(&st); struct sdfits_data_columns* data_cols; struct databuf_index* db_index; db_index = (struct databuf_index*)(vegas_databuf_index(db, curblock)); /* Read the block index, writing each dataset to a SDFITS file */ for(dataset = 0; dataset < db_index->num_datasets; dataset++) { data_cols = (struct sdfits_data_columns*)(vegas_databuf_data(db, curblock) + db_index->disk_buf[dataset].struct_offset); sf.data_columns = *data_cols; /* Write the data */ old_filenum = sf.filenum; sdfits_write_subint(&sf); /*Write new file number to shared memory*/ if(sf.filenum != old_filenum) { vegas_status_lock_safe(&st); hputi4(st.buf, "FILENUM", sf.filenum); vegas_status_unlock_safe(&st); } /* If a new integration number, increment the number of exposures written */ if(data_cols->integ_num != old_integ_num) { num_exposures_written += 1; old_integ_num = data_cols->integ_num; } } /* Indicate number of exposures written */ vegas_status_lock_safe(&st); hputi4(st.buf, "DSKEXPWR", num_exposures_written); vegas_status_unlock_safe(&st); /* For debugging... */ if (gp.drop_frac > 0.0) { printf("Block %d dropped %.3g%% of the packets\n", sf.tot_rows, gp.drop_frac*100.0); } /* Mark as free */ vegas_databuf_set_free(db, curblock); /* Go to next block */ curblock = (curblock + 1) % db->n_block; /* Check for cancel */ pthread_testcancel(); } while (run && !scan_finished); /* Cleanup */ pthread_exit(NULL); pthread_cleanup_pop(0); /* Closes sdfits_close */ pthread_cleanup_pop(0); /* Closes vegas_free_sdfits */ pthread_cleanup_pop(0); /* Closes set_exit_status */ pthread_cleanup_pop(0); /* Closes set_finished */ pthread_cleanup_pop(0); /* Closes vegas_status_detach */ pthread_cleanup_pop(0); /* Closes vegas_databuf_detach */ }
static void *run(hashpipe_thread_args_t * args) { // Local aliases to shorten access to args fields // Our output buffer happens to be a paper_input_databuf hashpipe_status_t st = args->st; const char * status_key = args->thread_desc->skey; st_p = &st; // allow global (this source file) access to the status buffer // Get inital value for crc32 function uint32_t init_crc = crc32(0,0,0); // Flag that holds off the crc thread int holdoff = 1; // Force ourself into the hold off state hashpipe_status_lock_safe(&st); hputi4(st.buf, "CRCHOLD", 1); hashpipe_status_unlock_safe(&st); while(holdoff) { // We're not in any hurry to startup sleep(1); hashpipe_status_lock_safe(&st); // Look for CRCHOLD value hgeti4(st.buf, "CRCHOLD", &holdoff); if(!holdoff) { // Done holding, so delete the key hdel(st.buf, "CRCHOLD"); } hashpipe_status_unlock_safe(&st); } /* Read network params */ struct hashpipe_udp_params up = { .bindhost = "0.0.0.0", .bindport = 8511, .packet_size = 8200 }; hashpipe_status_lock_safe(&st); // Get info from status buffer if present (no change if not present) hgets(st.buf, "BINDHOST", 80, up.bindhost); hgeti4(st.buf, "BINDPORT", &up.bindport); // Store bind host/port info etc in status buffer hputs(st.buf, "BINDHOST", up.bindhost); hputi4(st.buf, "BINDPORT", up.bindport); hputu4(st.buf, "CRCPKOK", 0); hputu4(st.buf, "CRCPKERR", 0); hputs(st.buf, status_key, "running"); hashpipe_status_unlock_safe(&st); struct hashpipe_udp_packet p; /* Give all the threads a chance to start before opening network socket */ sleep(1); /* Set up UDP socket */ int rv = hashpipe_udp_init(&up); if (rv!=HASHPIPE_OK) { hashpipe_error("paper_crc_thread", "Error opening UDP socket."); pthread_exit(NULL); } pthread_cleanup_push((void *)hashpipe_udp_close, &up); /* Main loop */ uint64_t packet_count = 0; uint64_t good_count = 0; uint64_t error_count = 0; uint64_t elapsed_wait_ns = 0; uint64_t elapsed_recv_ns = 0; uint64_t elapsed_proc_ns = 0; float ns_per_wait = 0.0; float ns_per_recv = 0.0; float ns_per_proc = 0.0; struct timespec start, stop; struct timespec recv_start, recv_stop; packet_header_t hdr; while (run_threads()) { /* Read packet */ clock_gettime(CLOCK_MONOTONIC, &recv_start); do { clock_gettime(CLOCK_MONOTONIC, &start); p.packet_size = recv(up.sock, p.data, HASHPIPE_MAX_PACKET_SIZE, 0); clock_gettime(CLOCK_MONOTONIC, &recv_stop); } while (p.packet_size == -1 && (errno == EAGAIN || errno == EWOULDBLOCK) && run_threads()); // Break out of loop if stopping if(!run_threads()) break; // Increment packet count packet_count++; // Check CRC if(crc32(init_crc, (/*const?*/ uint8_t *)p.data, p.packet_size) == 0xffffffff) { // CRC OK! Increment good counter good_count++; } else { // CRC error! Increment error counter error_count++; // Log message get_header(&p, &hdr); hashpipe_warn("paper_crc", "CRC error mcnt %llu ; fid %u ; xid %u", hdr.mcnt, hdr.fid, hdr.xid); } clock_gettime(CLOCK_MONOTONIC, &stop); elapsed_wait_ns += ELAPSED_NS(recv_start, start); elapsed_recv_ns += ELAPSED_NS(start, recv_stop); elapsed_proc_ns += ELAPSED_NS(recv_stop, stop); if(packet_count % 1000 == 0) { // Compute stats get_header(&p, &hdr); ns_per_wait = (float)elapsed_wait_ns / packet_count; ns_per_recv = (float)elapsed_recv_ns / packet_count; ns_per_proc = (float)elapsed_proc_ns / packet_count; // Update status hashpipe_status_lock_busywait_safe(&st); hputu8(st.buf, "CRCMCNT", hdr.mcnt); // Gbps = bits_per_packet / ns_per_packet // (N_BYTES_PER_PACKET excludes header, so +8 for the header) hputr4(st.buf, "CRCGBPS", 8*(N_BYTES_PER_PACKET+8)/(ns_per_recv+ns_per_proc)); hputr4(st.buf, "CRCWATNS", ns_per_wait); hputr4(st.buf, "CRCRECNS", ns_per_recv); hputr4(st.buf, "CRCPRCNS", ns_per_proc); // TODO Provide some way to recognize request to zero out the // CRCERR and CRCOK fields. hputu8(st.buf, "CRCPKOK", good_count); hputu8(st.buf, "CRCPKERR", error_count); hashpipe_status_unlock_safe(&st); // Start new average elapsed_wait_ns = 0; elapsed_recv_ns = 0; elapsed_proc_ns = 0; packet_count = 0; } /* Will exit if thread has been cancelled */ pthread_testcancel(); } /* Have to close all push's */ pthread_cleanup_pop(1); /* Closes push(hashpipe_udp_close) */ return NULL; } static hashpipe_thread_desc_t crc_thread = { name: "paper_crc_thread", skey: "CRCSTAT", init: NULL, run: run, ibuf_desc: {NULL},
static void *run(hashpipe_thread_args_t * args) { // Local aliases to shorten access to args fields // Our input buffer is a paper_input_databuf // Our output buffer is a paper_gpu_input_databuf paper_input_databuf_t *db_in = (paper_input_databuf_t *)args->ibuf; paper_gpu_input_databuf_t *db_out = (paper_gpu_input_databuf_t *)args->obuf; hashpipe_status_t st = args->st; const char * status_key = args->thread_desc->skey; #ifdef DEBUG_SEMS fprintf(stderr, "s/tid %lu/ FLUFf/\n", pthread_self()); #endif // Init status variables hashpipe_status_lock_safe(&st); hputi8(st.buf, "FLUFMCNT", 0); hashpipe_status_unlock_safe(&st); /* Loop */ int rv; int curblock_in=0; int curblock_out=0; float gbps, min_gbps; struct timespec start, finish; while (run_threads()) { // Note waiting status, // query integrating status // and, if armed, start count hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "waiting"); hashpipe_status_unlock_safe(&st); // Wait for new input block to be filled while ((rv=paper_input_databuf_wait_filled(db_in, curblock_in)) != HASHPIPE_OK) { if (rv==HASHPIPE_TIMEOUT) { hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "blocked_in"); hashpipe_status_unlock_safe(&st); continue; } else { hashpipe_error(__FUNCTION__, "error waiting for filled databuf"); pthread_exit(NULL); break; } } // Wait for new gpu_input block (our output block) to be free while ((rv=paper_gpu_input_databuf_wait_free(db_out, curblock_out)) != HASHPIPE_OK) { if (rv==HASHPIPE_TIMEOUT) { hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "blocked gpu input"); hashpipe_status_unlock_safe(&st); continue; } else { hashpipe_error(__FUNCTION__, "error waiting for free databuf"); pthread_exit(NULL); break; } } // Got a new data block, update status hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "fluffing"); hputi4(st.buf, "FLUFBKIN", curblock_in); hputu8(st.buf, "FLUFMCNT", db_in->block[curblock_in].header.mcnt); hashpipe_status_unlock_safe(&st); // Copy header and call fluff function clock_gettime(CLOCK_MONOTONIC, &start); memcpy(&db_out->block[curblock_out].header, &db_in->block[curblock_in].header, sizeof(paper_input_header_t)); paper_fluff(db_in->block[curblock_in].data, db_out->block[curblock_out].data); clock_gettime(CLOCK_MONOTONIC, &finish); // Note processing time hashpipe_status_lock_safe(&st); // Bits per fluff / ns per fluff = Gbps hgetr4(st.buf, "FLUFMING", &min_gbps); gbps = (float)(8*N_BYTES_PER_BLOCK)/ELAPSED_NS(start,finish); hputr4(st.buf, "FLUFGBPS", gbps); if(min_gbps == 0 || gbps < min_gbps) { hputr4(st.buf, "FLUFMING", gbps); } hashpipe_status_unlock_safe(&st); // Mark input block as free and advance paper_input_databuf_set_free(db_in, curblock_in); curblock_in = (curblock_in + 1) % db_in->header.n_block; // Mark output block as full and advance paper_gpu_input_databuf_set_filled(db_out, curblock_out); curblock_out = (curblock_out + 1) % db_out->header.n_block; /* Check for cancel */ pthread_testcancel(); } // Thread success! return NULL; }
void guppi_rawdisk_thread(void *_args) { /* Set cpu affinity */ cpu_set_t cpuset, cpuset_orig; sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig); CPU_ZERO(&cpuset); CPU_SET(1, &cpuset); int rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); if (rv<0) { guppi_error("guppi_rawdisk_thread", "Error setting cpu affinity."); perror("sched_setaffinity"); } /* Get args */ struct guppi_thread_args *args = (struct guppi_thread_args *)_args; /* Set priority */ rv = setpriority(PRIO_PROCESS, 0, 0); if (rv<0) { guppi_error("guppi_rawdisk_thread", "Error setting priority level."); perror("set_priority"); } /* Attach to status shared mem area */ struct guppi_status st; rv = guppi_status_attach(&st); if (rv!=GUPPI_OK) { guppi_error("guppi_rawdisk_thread", "Error attaching to status shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_status_detach, &st); pthread_cleanup_push((void *)set_exit_status, &st); /* Init status */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "init"); guppi_status_unlock_safe(&st); /* Read in general parameters */ struct guppi_params gp; #if FITS_TYPE == PSRFITS struct sdfits pf; pf.sub.dat_freqs = NULL; pf.sub.dat_weights = NULL; pf.sub.dat_offsets = NULL; pf.sub.dat_scales = NULL; pthread_cleanup_push((void *)guppi_free_psrfits, &pf); #else struct sdfits pf; pthread_cleanup_push((void *)guppi_free_sdfits, &pf); #endif /* Attach to databuf shared mem */ struct guppi_databuf *db; db = guppi_databuf_attach(args->input_buffer); if (db==NULL) { guppi_error("guppi_rawdisk_thread", "Error attaching to databuf shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_databuf_detach, db); /* Init output file */ FILE *fraw = NULL; pthread_cleanup_push((void *)safe_fclose, fraw); /* Pointers for quantization params */ double *mean = NULL; double *std = NULL; printf("casper: raw disk thread created and running\n"); /* Loop */ int packetidx=0, npacket=0, ndrop=0, packetsize=0, blocksize=0; int orig_blocksize=0; int curblock=0; int block_count=0, blocks_per_file=128, filenum=0; int got_packet_0=0, first=1; int requantize = 0; char *ptr, *hend; signal(SIGINT,cc); while (run_threads) { /* Note waiting status */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "waiting"); guppi_status_unlock_safe(&st); /* Wait for buf to have data */ rv = guppi_databuf_wait_filled(db, curblock); if (rv!=0) continue; printf("casper: raw disk thread rcvd data.\n"); /* Read param struct for this block */ ptr = guppi_databuf_header(db, curblock); if (first) { guppi_read_obs_params(ptr, &gp, &pf); first = 0; } else { guppi_read_subint_params(ptr, &gp, &pf); } /* Parse packet size, npacket from header */ hgeti4(ptr, "PKTIDX", &packetidx); hgeti4(ptr, "PKTSIZE", &packetsize); hgeti4(ptr, "NPKT", &npacket); hgeti4(ptr, "NDROP", &ndrop); #if FITS_TYPE == PSR_FITS /* Check for re-quantization flag */ int nbits_req = 0; if (hgeti4(ptr, "NBITSREQ", &nbits_req) == 0) { /* Param not present, don't requantize */ requantize = 0; } else { /* Param is present */ if (nbits_req==8) requantize = 0; else if (nbits_req==2) requantize = 1; else /* Invalid selection for requested nbits * .. die or ignore? */ requantize = 0; } #endif /* Set up data ptr for quant routines */ #if FITS_TYPE == PSR_FITS pf.sub.data = (unsigned char *)guppi_databuf_data(db, curblock); #else pf.data_columns.data = (unsigned char *)guppi_databuf_data(db, curblock); #endif /* Wait for packet 0 before starting write */ if (got_packet_0==0 && packetidx==0 && gp.stt_valid==1) { got_packet_0 = 1; guppi_read_obs_params(ptr, &gp, &pf); #if FITS_TYPE == PSR_FITS orig_blocksize = pf.sub.bytes_per_subint; #endif char fname[256]; sprintf(fname, "%s.%4.4d.raw", pf.basefilename, filenum); fprintf(stderr, "Opening raw file '%s'\n", fname); // TODO: check for file exist. fraw = fopen(fname, "w"); if (fraw==NULL) { guppi_error("guppi_rawdisk_thread", "Error opening file."); pthread_exit(NULL); } #if FITS_TYPE == PSR_FITS /* Determine scaling factors for quantization if appropriate */ if (requantize) { mean = (double *)realloc(mean, pf.hdr.rcvr_polns * pf.hdr.nchan * sizeof(double)); std = (double *)realloc(std, pf.hdr.rcvr_polns * pf.hdr.nchan * sizeof(double)); compute_stat(&pf, mean, std); fprintf(stderr, "Computed 2-bit stats\n"); } #endif } /* See if we need to open next file */ if (block_count >= blocks_per_file) { fclose(fraw); filenum++; char fname[256]; sprintf(fname, "%s.%4.4d.raw", pf.basefilename, filenum); fprintf(stderr, "Opening raw file '%s'\n", fname); fraw = fopen(fname, "w"); if (fraw==NULL) { guppi_error("guppi_rawdisk_thread", "Error opening file."); pthread_exit(NULL); } block_count=0; } /* See how full databuf is */ //total_status = guppi_databuf_total_status(db); /* Requantize from 8 bits to 2 bits if necessary. * See raw_quant.c for more usage examples. */ #if FITS_TYPE == PSR_FITS if (requantize && got_packet_0) { pf.sub.bytes_per_subint = orig_blocksize; /* Does the quantization in-place */ quantize_2bit(&pf, mean, std); /* Update some parameters for output */ hputi4(ptr, "BLOCSIZE", pf.sub.bytes_per_subint); hputi4(ptr, "NBITS", pf.hdr.nbits); } #endif /* Get full data block size */ hgeti4(ptr, "BLOCSIZE", &blocksize); /* If we got packet 0, write data to disk */ if (got_packet_0) { /* Note waiting status */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "writing"); guppi_status_unlock_safe(&st); /* Write header to file */ hend = ksearch(ptr, "END"); for (ptr=ptr; ptr<=hend; ptr+=80) { fwrite(ptr, 80, 1, fraw); } /* Write data */ printf("block size: %d\n", blocksize); ptr = guppi_databuf_data(db, curblock); rv = fwrite(ptr, 1, (size_t)blocksize, fraw); if (rv != blocksize) { guppi_error("guppi_rawdisk_thread", "Error writing data."); } /* Increment counter */ block_count++; /* flush output */ fflush(fraw); } /* Mark as free */ guppi_databuf_set_free(db, curblock); /* Go to next block */ curblock = (curblock + 1) % db->n_block; /* Check for cancel */ pthread_testcancel(); } pthread_exit(NULL); pthread_cleanup_pop(0); /* Closes fclose */ pthread_cleanup_pop(0); /* Closes guppi_databuf_detach */ pthread_cleanup_pop(0); /* Closes guppi_free_psrfits */ pthread_cleanup_pop(0); /* Closes set_exit_status */ pthread_cleanup_pop(0); /* Closes guppi_status_detach */ }
int main(int argc, char *argv[]) { int rv; struct vegas_status s; rv = vegas_status_attach(&s); if (rv!=VEGAS_OK) { fprintf(stderr, "Error connecting to shared mem.\n"); perror(NULL); exit(1); } vegas_status_lock(&s); /* Loop over cmd line to fill in params */ static struct option long_opts[] = { {"key", 1, NULL, 'k'}, {"get", 1, NULL, 'g'}, {"string", 1, NULL, 's'}, {"float", 1, NULL, 'f'}, {"double", 1, NULL, 'd'}, {"int", 1, NULL, 'i'}, {"quiet", 0, NULL, 'q'}, {"clear", 0, NULL, 'C'}, {"del", 0, NULL, 'D'}, {0,0,0,0} }; int opt,opti; char *key=NULL; float flttmp; double dbltmp; int inttmp; int quiet=0, clear=0; while ((opt=getopt_long(argc,argv,"k:g:s:f:d:i:qCD",long_opts,&opti))!=-1) { switch (opt) { case 'k': key = optarg; break; case 'g': hgetr8(s.buf, optarg, &dbltmp); printf("%g\n", dbltmp); break; case 's': if (key) hputs(s.buf, key, optarg); break; case 'f': flttmp = atof(optarg); if (key) hputr4(s.buf, key, flttmp); break; case 'd': dbltmp = atof(optarg); if (key) hputr8(s.buf, key, dbltmp); break; case 'i': inttmp = atoi(optarg); if (key) hputi4(s.buf, key, inttmp); break; case 'D': if (key) hdel(s.buf, key); break; case 'C': clear=1; break; case 'q': quiet=1; break; default: break; } } /* If not quiet, print out buffer */ if (!quiet) { printf(s.buf); printf("\n"); } vegas_status_unlock(&s); if (clear) vegas_status_clear(&s); exit(0); }
static void *run(hashpipe_thread_args_t * args) { s6_input_databuf_t *db = (s6_input_databuf_t *)args->obuf; hashpipe_status_t *p_st = &(args->st); hashpipe_status_t st = args->st; const char * status_key = args->thread_desc->skey; //s6_input_block_t fake_data_block; /* Main loop */ int i, rv; uint64_t mcnt = 0; uint64_t num_coarse_chan = N_COARSE_CHAN; uint64_t *data; int block_idx = 0; int error_count = 0, max_error_count = 0; float error, max_error = 0.0; int gen_fake = 0; hashpipe_status_lock_safe(&st); //hashpipe_status_lock_safe(p_st); hputi4(st.buf, "NUMCCHAN", N_COARSE_CHAN); hputi4(st.buf, "NUMFCHAN", N_FINE_CHAN); hputi4(st.buf, "NUMBBEAM", N_BYTES_PER_BEAM); hputi4(st.buf, "NUMBBLOC", sizeof(s6_input_block_t)); hputi4(st.buf, "THRESHLD", POWER_THRESH); hgeti4(st.buf, "GENFAKE", &gen_fake); hashpipe_status_unlock_safe(&st); //hashpipe_status_unlock_safe(p_st); time_t t, prior_t; prior_t = time(&prior_t); while (run_threads()) { hashpipe_status_lock_safe(&st); //hashpipe_status_lock_safe(p_st); hputi4(st.buf, "NETBKOUT", block_idx); hputs(st.buf, status_key, "waiting"); hashpipe_status_unlock_safe(&st); //hashpipe_status_unlock_safe(p_st); t = time(&t); fprintf(stderr, "elapsed seconds for block %d : %ld\n", block_idx, t - prior_t); prior_t = t; // Wait for data struct timespec sleep_dur, rem_sleep_dur; sleep_dur.tv_sec = 1; sleep_dur.tv_nsec = 0; //fprintf(stderr, "fake net thread sleeping for %7.5f seconds\n", // sleep_dur.tv_sec + (double)sleep_dur.tv_nsec/1000000000.0); nanosleep(&sleep_dur, &rem_sleep_dur); /* Wait for new block to be free, then clear it * if necessary and fill its header with new values. */ while ((rv=s6_input_databuf_wait_free(db, block_idx)) != HASHPIPE_OK) { if (rv==HASHPIPE_TIMEOUT) { hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "blocked"); hashpipe_status_unlock_safe(&st); continue; } else { hashpipe_error(__FUNCTION__, "error waiting for free databuf"); pthread_exit(NULL); break; } } hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "receiving"); hashpipe_status_unlock_safe(&st); // populate block header db->block[block_idx].header.mcnt = mcnt; db->block[block_idx].header.coarse_chan_id = 321; db->block[block_idx].header.num_coarse_chan = num_coarse_chan; memset(db->block[block_idx].header.missed_pkts, 0, sizeof(uint64_t) * N_BEAM_SLOTS); if(gen_fake) { gen_fake = 0; // gen fake data for all beams, all blocks // TODO vary data by beam fprintf(stderr, "generating fake data to block 0 beam 0..."); gen_fake_data(&(db->block[0].data[0])); fprintf(stderr, " done\n"); fprintf(stderr, "copying to block 0 beam"); for(int beam_i = 1; beam_i < N_BEAMS; beam_i++) { fprintf(stderr, " %d", beam_i); memcpy((void *)&db->block[0].data[beam_i*N_BYTES_PER_BEAM/sizeof(uint64_t)], (void *)&db->block[0].data[0], N_BYTES_PER_BEAM); } fprintf(stderr, " done\n"); fprintf(stderr, "copying to block"); for(int block_i = 1; block_i < N_INPUT_BLOCKS; block_i++) { fprintf(stderr, " %d", block_i); memcpy((void *)&db->block[block_i].data[0], (void *)&db->block[0].data[0], N_DATA_BYTES_PER_BLOCK); } fprintf(stderr, " done\n"); } hashpipe_status_lock_safe(&st); hputr4(st.buf, "NETMXERR", max_error); hputi4(st.buf, "NETERCNT", error_count); hputi4(st.buf, "NETMXECT", max_error_count); hashpipe_status_unlock_safe(&st); // Mark block as full s6_input_databuf_set_filled(db, block_idx); // Setup for next block block_idx = (block_idx + 1) % db->header.n_block; mcnt++; // uncomment the following to test dynamic setting of num_coarse_chan //num_coarse_chan--; /* Will exit if thread has been cancelled */ pthread_testcancel(); } // Thread success! return THREAD_OK; }
/* The main CPU accumulator thread */ void guppi_accum_thread(void *_args) { float **accumulator; //indexed accumulator[accum_id][chan][subband][stokes] char accum_dirty[NUM_SW_STATES]; struct sdfits_data_columns data_cols[NUM_SW_STATES]; int payload_type; int i, j, k, rv; /* Get arguments */ struct guppi_thread_args *args = (struct guppi_thread_args *)_args; /* Set cpu affinity */ cpu_set_t cpuset, cpuset_orig; sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig); //CPU_ZERO(&cpuset); CPU_CLR(13, &cpuset); CPU_SET(9, &cpuset); rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); if (rv<0) { guppi_error("guppi_accum_thread", "Error setting cpu affinity."); perror("sched_setaffinity"); } /* Set priority */ rv = setpriority(PRIO_PROCESS, 0, args->priority); if (rv<0) { guppi_error("guppi_accum_thread", "Error setting priority level."); perror("set_priority"); } /* Attach to status shared mem area */ struct guppi_status st; rv = guppi_status_attach(&st); if (rv!=GUPPI_OK) { guppi_error("guppi_accum_thread", "Error attaching to status shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_status_detach, &st); pthread_cleanup_push((void *)set_exit_status, &st); pthread_cleanup_push((void *)guppi_thread_set_finished, args); /* Init status */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "init"); guppi_status_unlock_safe(&st); /* Read in general parameters */ struct guppi_params gp; struct sdfits sf; pthread_cleanup_push((void *)guppi_free_sdfits, &sf); /* Attach to databuf shared mem */ struct guppi_databuf *db_in, *db_out; db_in = guppi_databuf_attach(args->input_buffer); char errmsg[256]; if (db_in==NULL) { sprintf(errmsg, "Error attaching to input databuf(%d) shared memory.", args->input_buffer); guppi_error("guppi_accum_thread", errmsg); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_databuf_detach, db_in); db_out = guppi_databuf_attach(args->output_buffer); if (db_out==NULL) { sprintf(errmsg, "Error attaching to output databuf(%d) shared memory.", args->output_buffer); guppi_error("guppi_accum_thread", errmsg); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_databuf_detach, db_out); /* Determine high/low bandwidth mode */ char bw_mode[16]; if (hgets(st.buf, "BW_MODE", 16, bw_mode)) { if(strncmp(bw_mode, "high", 4) == 0) payload_type = INT_PAYLOAD; else if(strncmp(bw_mode, "low", 3) == 0) payload_type = FLOAT_PAYLOAD; else guppi_error("guppi_net_thread", "Unsupported bandwidth mode"); } else guppi_error("guppi_net_thread", "BW_MODE not set"); /* Read nchan and nsubband from status shared memory */ guppi_read_obs_params(st.buf, &gp, &sf); /* Allocate memory for vector accumulators */ create_accumulators(&accumulator, sf.hdr.nchan, sf.hdr.nsubband); pthread_cleanup_push((void *)destroy_accumulators, accumulator); /* Clear the vector accumulators */ for(i = 0; i < NUM_SW_STATES; i++) accum_dirty[i] = 1; reset_accumulators(accumulator, data_cols, accum_dirty, sf.hdr.nsubband, sf.hdr.nchan); /* Loop */ int curblock_in=0, curblock_out=0; int first=1; float reqd_exposure=0; double accum_time=0; int integ_num; float pfb_rate; int heap, accumid, struct_offset, array_offset; char *hdr_in=NULL, *hdr_out=NULL; struct databuf_index *index_in, *index_out; int nblock_int=0, npacket=0, n_pkt_drop=0, n_heap_drop=0; signal(SIGINT,cc); while (run) { /* Note waiting status */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "waiting"); guppi_status_unlock_safe(&st); /* Wait for buf to have data */ rv = guppi_databuf_wait_filled(db_in, curblock_in); if (rv!=0) continue; /* Note waiting status and current block*/ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "accumulating"); hputi4(st.buf, "ACCBLKIN", curblock_in); guppi_status_unlock_safe(&st); /* Read param struct for this block */ hdr_in = guppi_databuf_header(db_in, curblock_in); if (first) guppi_read_obs_params(hdr_in, &gp, &sf); else guppi_read_subint_params(hdr_in, &gp, &sf); /* Do any first time stuff: first time code runs, not first time process this block */ if (first) { /* Set up first output header. This header is copied from block to block each time a new block is created */ hdr_out = guppi_databuf_header(db_out, curblock_out); memcpy(hdr_out, guppi_databuf_header(db_in, curblock_in), GUPPI_STATUS_SIZE); /* Read required exposure and PFB rate from status shared memory */ reqd_exposure = sf.data_columns.exposure; pfb_rate = sf.hdr.efsampfr / (2 * sf.hdr.nchan); /* Initialise the index in the output block */ index_out = (struct databuf_index*)guppi_databuf_index(db_out, curblock_out); index_out->num_datasets = 0; index_out->array_size = sf.hdr.nsubband * sf.hdr.nchan * NUM_STOKES * 4; first=0; } /* Loop through each spectrum (heap) in input buffer */ index_in = (struct databuf_index*)guppi_databuf_index(db_in, curblock_in); for(heap = 0; heap < index_in->num_heaps; heap++) { /* If invalid, record it and move to next heap */ if(!index_in->cpu_gpu_buf[heap].heap_valid) { n_heap_drop++; continue; } /* Read in heap from buffer */ char* heap_addr = (char*)(guppi_databuf_data(db_in, curblock_in) + sizeof(struct freq_spead_heap) * heap); struct freq_spead_heap* freq_heap = (struct freq_spead_heap*)(heap_addr); char* payload_addr = (char*)(guppi_databuf_data(db_in, curblock_in) + sizeof(struct freq_spead_heap) * MAX_HEAPS_PER_BLK + (index_in->heap_size - sizeof(struct freq_spead_heap)) * heap ); int *i_payload = (int*)(payload_addr); float *f_payload = (float*)(payload_addr); accumid = freq_heap->status_bits & 0x7; /*Debug: print heap */ /* printf("%d, %d, %d, %d, %d, %d\n", freq_heap->time_cntr, freq_heap->spectrum_cntr, freq_heap->integ_size, freq_heap->mode, freq_heap->status_bits, freq_heap->payload_data_off); */ /* If we have accumulated for long enough, write vectors to output block */ if(accum_time >= reqd_exposure) { for(i = 0; i < NUM_SW_STATES; i++) { /*If a particular accumulator is dirty, write it to output buffer */ if(accum_dirty[i]) { /*If insufficient space, first mark block as filled and request new block*/ index_out = (struct databuf_index*)(guppi_databuf_index(db_out, curblock_out)); if( (index_out->num_datasets+1) * (index_out->array_size + sizeof(struct sdfits_data_columns)) > db_out->block_size) { printf("Accumulator finished with output block %d\n", curblock_out); /* Write block number to status buffer */ guppi_status_lock_safe(&st); hputi4(st.buf, "ACCBLKOU", curblock_out); guppi_status_unlock_safe(&st); /* Update packet count and loss fields in output header */ hputi4(hdr_out, "NBLOCK", nblock_int); hputi4(hdr_out, "NPKT", npacket); hputi4(hdr_out, "NPKTDROP", n_pkt_drop); hputi4(hdr_out, "NHPDROP", n_heap_drop); /* Close out current integration */ guppi_databuf_set_filled(db_out, curblock_out); /* Wait for next output buf */ curblock_out = (curblock_out + 1) % db_out->n_block; guppi_databuf_wait_free(db_out, curblock_out); while ((rv=guppi_databuf_wait_free(db_out, curblock_out)) != GUPPI_OK) { if (rv==GUPPI_TIMEOUT) { guppi_warn("guppi_accum_thread", "timeout while waiting for output block"); continue; } else { guppi_error("guppi_accum_thread", "error waiting for free databuf"); run=0; pthread_exit(NULL); break; } } hdr_out = guppi_databuf_header(db_out, curblock_out); memcpy(hdr_out, guppi_databuf_header(db_in, curblock_in), GUPPI_STATUS_SIZE); /* Initialise the index in new output block */ index_out = (struct databuf_index*)guppi_databuf_index(db_out, curblock_out); index_out->num_datasets = 0; index_out->array_size = sf.hdr.nsubband * sf.hdr.nchan * NUM_STOKES * 4; nblock_int=0; npacket=0; n_pkt_drop=0; n_heap_drop=0; } /*Update index for output buffer*/ index_out = (struct databuf_index*)(guppi_databuf_index(db_out, curblock_out)); if(index_out->num_datasets == 0) struct_offset = 0; else struct_offset = index_out->disk_buf[index_out->num_datasets-1].array_offset + index_out->array_size; array_offset = struct_offset + sizeof(struct sdfits_data_columns); index_out->disk_buf[index_out->num_datasets].struct_offset = struct_offset; index_out->disk_buf[index_out->num_datasets].array_offset = array_offset; /*Copy sdfits_data_columns struct to disk buffer */ memcpy(guppi_databuf_data(db_out, curblock_out) + struct_offset, &data_cols[i], sizeof(struct sdfits_data_columns)); /*Copy data array to disk buffer */ memcpy(guppi_databuf_data(db_out, curblock_out) + array_offset, accumulator[i], index_out->array_size); /*Update SDFITS data_columns pointer to data array */ ((struct sdfits_data_columns*) (guppi_databuf_data(db_out, curblock_out) + struct_offset))->data = (unsigned char*)(guppi_databuf_data(db_out, curblock_out) + array_offset); index_out->num_datasets = index_out->num_datasets + 1; } } accum_time = 0; integ_num += 1; reset_accumulators(accumulator, data_cols, accum_dirty, sf.hdr.nsubband, sf.hdr.nchan); } /* Only add spectrum to accumulator if blanking bit is low */ if((freq_heap->status_bits & 0x08) == 0) { /* Fill in data columns header fields */ if(!accum_dirty[accumid]) { /*Record SPEAD header fields*/ data_cols[accumid].time = index_in->cpu_gpu_buf[heap].heap_rcvd_mjd; data_cols[accumid].time_counter = freq_heap->time_cntr; data_cols[accumid].integ_num = integ_num; data_cols[accumid].sttspec = freq_heap->spectrum_cntr; data_cols[accumid].accumid = accumid; /* Fill in rest of fields from status buffer */ strcpy(data_cols[accumid].object, sf.data_columns.object); data_cols[accumid].azimuth = sf.data_columns.azimuth; data_cols[accumid].elevation = sf.data_columns.elevation; data_cols[accumid].bmaj = sf.data_columns.bmaj; data_cols[accumid].bmin = sf.data_columns.bmin; data_cols[accumid].bpa = sf.data_columns.bpa; data_cols[accumid].centre_freq_idx = sf.data_columns.centre_freq_idx; data_cols[accumid].ra = sf.data_columns.ra; data_cols[accumid].dec = sf.data_columns.dec; data_cols[accumid].exposure = 0.0; for(i = 0; i < NUM_SW_STATES; i++) data_cols[accumid].centre_freq[i] = sf.data_columns.centre_freq[i]; accum_dirty[accumid] = 1; } data_cols[accumid].exposure += (float)(freq_heap->integ_size)/pfb_rate; data_cols[accumid].stpspec = freq_heap->spectrum_cntr; /* Add spectrum to appropriate vector accumulator (high-bw mode) */ if(payload_type == INT_PAYLOAD) { for(i = 0; i < sf.hdr.nchan; i++) { for(j = 0; j < sf.hdr.nsubband; j++) { for(k = 0; k < NUM_STOKES; k++) { accumulator[accumid] [i*sf.hdr.nsubband*NUM_STOKES + j*NUM_STOKES + k] += (float)i_payload[i*sf.hdr.nsubband*NUM_STOKES + j*NUM_STOKES + k]; } } } } /* Add spectrum to appropriate vector accumulator (low-bw mode) */ else { for(i = 0; i < sf.hdr.nchan; i++) { for(j = 0; j < sf.hdr.nsubband; j++) { for(k = 0; k < NUM_STOKES; k++) { accumulator[accumid] [i*sf.hdr.nsubband*NUM_STOKES + j*NUM_STOKES + k] += f_payload[i*sf.hdr.nsubband*NUM_STOKES + j*NUM_STOKES + k]; } } } } } accum_time += (double)freq_heap->integ_size / pfb_rate; } /* Update packet count and loss fields from input header */ nblock_int++; npacket += gp.num_pkts_rcvd; n_pkt_drop += gp.num_pkts_dropped; /* Done with current input block */ guppi_databuf_set_free(db_in, curblock_in); curblock_in = (curblock_in + 1) % db_in->n_block; /* Check for cancel */ pthread_testcancel(); } pthread_exit(NULL); pthread_cleanup_pop(0); /* Closes set_exit_status */ pthread_cleanup_pop(0); /* Closes set_finished */ pthread_cleanup_pop(0); /* Closes guppi_free_sdfits */ pthread_cleanup_pop(0); /* Closes ? */ pthread_cleanup_pop(0); /* Closes destroy_accumulators */ pthread_cleanup_pop(0); /* Closes guppi_status_detach */ pthread_cleanup_pop(0); /* Closes guppi_databuf_detach */ }
static void *run(void * _args) { // Cast _args struct guppi_thread_args *args = (struct guppi_thread_args *)_args; #ifdef DEBUG_SEMS fprintf(stderr, "s/tid %lu/NET/' <<.\n", pthread_self()); #endif THREAD_RUN_BEGIN(args); THREAD_RUN_SET_AFFINITY_PRIORITY(args); THREAD_RUN_ATTACH_STATUS(args->instance_id, st); st_p = &st; // allow global (this source file) access to the status buffer /* Attach to paper_input_databuf */ THREAD_RUN_ATTACH_DATABUF(args->instance_id, paper_input_databuf, db, args->output_buffer); /* Read in general parameters */ struct guppi_params gp; struct sdfits pf; char status_buf[GUPPI_STATUS_SIZE]; guppi_status_lock_busywait_safe(st_p); memcpy(status_buf, st_p->buf, GUPPI_STATUS_SIZE); guppi_status_unlock_safe(st_p); guppi_read_obs_params(status_buf, &gp, &pf); pthread_cleanup_push((void *)guppi_free_sdfits, &pf); /* Read network params */ struct guppi_udp_params up; //guppi_read_net_params(status_buf, &up); paper_read_net_params(status_buf, &up); // Store bind host/port info etc in status buffer guppi_status_lock_busywait_safe(&st); hputs(st.buf, "BINDHOST", up.bindhost); hputi4(st.buf, "BINDPORT", up.bindport); hputu4(st.buf, "MISSEDFE", 0); hputu4(st.buf, "MISSEDPK", 0); hputs(st.buf, STATUS_KEY, "running"); guppi_status_unlock_safe(&st); struct guppi_udp_packet p; /* Give all the threads a chance to start before opening network socket */ sleep(1); #ifndef TIMING_TEST /* Set up UDP socket */ int rv = guppi_udp_init(&up); if (rv!=GUPPI_OK) { guppi_error("guppi_net_thread", "Error opening UDP socket."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_udp_close, &up); #endif /* Main loop */ uint64_t packet_count = 0; uint64_t elapsed_wait_ns = 0; uint64_t elapsed_recv_ns = 0; uint64_t elapsed_proc_ns = 0; float ns_per_wait = 0.0; float ns_per_recv = 0.0; float ns_per_proc = 0.0; struct timespec start, stop; struct timespec recv_start, recv_stop; signal(SIGINT,cc); while (run_threads) { #ifndef TIMING_TEST /* Read packet */ clock_gettime(CLOCK_MONOTONIC, &recv_start); do { clock_gettime(CLOCK_MONOTONIC, &start); p.packet_size = recv(up.sock, p.data, GUPPI_MAX_PACKET_SIZE, 0); clock_gettime(CLOCK_MONOTONIC, &recv_stop); } while (p.packet_size == -1 && (errno == EAGAIN || errno == EWOULDBLOCK) && run_threads); if(!run_threads) break; if (up.packet_size != p.packet_size) { if (p.packet_size != -1) { #ifdef DEBUG_NET guppi_warn("guppi_net_thread", "Incorrect pkt size"); #endif continue; } else { guppi_error("guppi_net_thread", "guppi_udp_recv returned error"); perror("guppi_udp_recv"); pthread_exit(NULL); } } #endif packet_count++; // Copy packet into any blocks where it belongs. const uint64_t mcnt = write_paper_packet_to_blocks((paper_input_databuf_t *)db, &p); clock_gettime(CLOCK_MONOTONIC, &stop); elapsed_wait_ns += ELAPSED_NS(recv_start, start); elapsed_recv_ns += ELAPSED_NS(start, recv_stop); elapsed_proc_ns += ELAPSED_NS(recv_stop, stop); if(mcnt != -1) { // Update status ns_per_wait = (float)elapsed_wait_ns / packet_count; ns_per_recv = (float)elapsed_recv_ns / packet_count; ns_per_proc = (float)elapsed_proc_ns / packet_count; guppi_status_lock_busywait_safe(&st); hputu8(st.buf, "NETMCNT", mcnt); // Gbps = bits_per_packet / ns_per_packet // (N_BYTES_PER_PACKET excludes header, so +8 for the header) hputr4(st.buf, "NETGBPS", 8*(N_BYTES_PER_PACKET+8)/(ns_per_recv+ns_per_proc)); hputr4(st.buf, "NETWATNS", ns_per_wait); hputr4(st.buf, "NETRECNS", ns_per_recv); hputr4(st.buf, "NETPRCNS", ns_per_proc); guppi_status_unlock_safe(&st); // Start new average elapsed_wait_ns = 0; elapsed_recv_ns = 0; elapsed_proc_ns = 0; packet_count = 0; } #if defined TIMING_TEST || defined NET_TIMING_TEST #define END_LOOP_COUNT (1*1000*1000) static int loop_count=0; static struct timespec tt_start, tt_stop; if(loop_count == 0) { clock_gettime(CLOCK_MONOTONIC, &tt_start); } //if(loop_count == 1000000) run_threads = 0; if(loop_count == END_LOOP_COUNT) { clock_gettime(CLOCK_MONOTONIC, &tt_stop); int64_t elapsed = ELAPSED_NS(tt_start, tt_stop); printf("processed %d packets in %.6f ms (%.3f us per packet)\n", END_LOOP_COUNT, elapsed/1e6, elapsed/1e3/END_LOOP_COUNT); exit(0); } loop_count++; #endif /* Will exit if thread has been cancelled */ pthread_testcancel(); } /* Have to close all push's */ #ifndef TIMING_TEST pthread_cleanup_pop(0); /* Closes push(guppi_udp_close) */ #endif pthread_cleanup_pop(0); /* Closes guppi_free_psrfits */ THREAD_RUN_DETACH_DATAUF; THREAD_RUN_DETACH_STATUS; THREAD_RUN_END; return NULL; }