/* Generate a fake heap and write it to shared mem block */ void write_fake_heap_to_block(struct fake_datablock_stats *d, int heap_cntr) { int block_heap_idx = heap_cntr - d->heap_idx; int i; float spectrum_value; //Update the heap index first struct databuf_index* index = (struct databuf_index*)guppi_databuf_index(d->db, d->block_idx); index->num_heaps = index->num_heaps + 1; index->heap_size = d->heap_size; index->cpu_gpu_buf[block_heap_idx].heap_cntr = heap_cntr; index->cpu_gpu_buf[block_heap_idx].heap_valid = 1; //Create speed_heap at correct location in block struct freq_spead_heap* fake_heap; char *heap_addr = guppi_databuf_data(d->db, d->block_idx) + block_heap_idx*d->heap_size; fake_heap = (struct freq_spead_heap*)(heap_addr); float *payload = (float*)(heap_addr + sizeof(struct freq_spead_heap)); //Populate necessary fields fake_heap->time_cntr = heap_cntr * 10; fake_heap->spectrum_cntr = heap_cntr; fake_heap->integ_size = 100; fake_heap->mode = 1; fake_heap->status_bits = 0; fake_heap->payload_data_off = 48; for(i = 0; i < SPEAD_SPECTRUM_SZ; i++) { spectrum_value = (float)i; memcpy(&(payload[i*4]), &spectrum_value, 4); memcpy(&(payload[i*4 + 1]), &spectrum_value, 4); memcpy(&(payload[i*4 + 2]), &spectrum_value, 4); memcpy(&(payload[i*4 + 3]), &spectrum_value, 4); } /* Update counters */ d->nheaps++; d->last_heap = heap_cntr; }
/* The main CPU accumulator thread */ void guppi_accum_thread(void *_args) { float **accumulator; //indexed accumulator[accum_id][chan][subband][stokes] char accum_dirty[NUM_SW_STATES]; struct sdfits_data_columns data_cols[NUM_SW_STATES]; int payload_type; int i, j, k, rv; /* Get arguments */ struct guppi_thread_args *args = (struct guppi_thread_args *)_args; /* Set cpu affinity */ cpu_set_t cpuset, cpuset_orig; sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig); //CPU_ZERO(&cpuset); CPU_CLR(13, &cpuset); CPU_SET(9, &cpuset); rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); if (rv<0) { guppi_error("guppi_accum_thread", "Error setting cpu affinity."); perror("sched_setaffinity"); } /* Set priority */ rv = setpriority(PRIO_PROCESS, 0, args->priority); if (rv<0) { guppi_error("guppi_accum_thread", "Error setting priority level."); perror("set_priority"); } /* Attach to status shared mem area */ struct guppi_status st; rv = guppi_status_attach(&st); if (rv!=GUPPI_OK) { guppi_error("guppi_accum_thread", "Error attaching to status shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_status_detach, &st); pthread_cleanup_push((void *)set_exit_status, &st); pthread_cleanup_push((void *)guppi_thread_set_finished, args); /* Init status */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "init"); guppi_status_unlock_safe(&st); /* Read in general parameters */ struct guppi_params gp; struct sdfits sf; pthread_cleanup_push((void *)guppi_free_sdfits, &sf); /* Attach to databuf shared mem */ struct guppi_databuf *db_in, *db_out; db_in = guppi_databuf_attach(args->input_buffer); char errmsg[256]; if (db_in==NULL) { sprintf(errmsg, "Error attaching to input databuf(%d) shared memory.", args->input_buffer); guppi_error("guppi_accum_thread", errmsg); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_databuf_detach, db_in); db_out = guppi_databuf_attach(args->output_buffer); if (db_out==NULL) { sprintf(errmsg, "Error attaching to output databuf(%d) shared memory.", args->output_buffer); guppi_error("guppi_accum_thread", errmsg); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_databuf_detach, db_out); /* Determine high/low bandwidth mode */ char bw_mode[16]; if (hgets(st.buf, "BW_MODE", 16, bw_mode)) { if(strncmp(bw_mode, "high", 4) == 0) payload_type = INT_PAYLOAD; else if(strncmp(bw_mode, "low", 3) == 0) payload_type = FLOAT_PAYLOAD; else guppi_error("guppi_net_thread", "Unsupported bandwidth mode"); } else guppi_error("guppi_net_thread", "BW_MODE not set"); /* Read nchan and nsubband from status shared memory */ guppi_read_obs_params(st.buf, &gp, &sf); /* Allocate memory for vector accumulators */ create_accumulators(&accumulator, sf.hdr.nchan, sf.hdr.nsubband); pthread_cleanup_push((void *)destroy_accumulators, accumulator); /* Clear the vector accumulators */ for(i = 0; i < NUM_SW_STATES; i++) accum_dirty[i] = 1; reset_accumulators(accumulator, data_cols, accum_dirty, sf.hdr.nsubband, sf.hdr.nchan); /* Loop */ int curblock_in=0, curblock_out=0; int first=1; float reqd_exposure=0; double accum_time=0; int integ_num; float pfb_rate; int heap, accumid, struct_offset, array_offset; char *hdr_in=NULL, *hdr_out=NULL; struct databuf_index *index_in, *index_out; int nblock_int=0, npacket=0, n_pkt_drop=0, n_heap_drop=0; signal(SIGINT,cc); while (run) { /* Note waiting status */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "waiting"); guppi_status_unlock_safe(&st); /* Wait for buf to have data */ rv = guppi_databuf_wait_filled(db_in, curblock_in); if (rv!=0) continue; /* Note waiting status and current block*/ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "accumulating"); hputi4(st.buf, "ACCBLKIN", curblock_in); guppi_status_unlock_safe(&st); /* Read param struct for this block */ hdr_in = guppi_databuf_header(db_in, curblock_in); if (first) guppi_read_obs_params(hdr_in, &gp, &sf); else guppi_read_subint_params(hdr_in, &gp, &sf); /* Do any first time stuff: first time code runs, not first time process this block */ if (first) { /* Set up first output header. This header is copied from block to block each time a new block is created */ hdr_out = guppi_databuf_header(db_out, curblock_out); memcpy(hdr_out, guppi_databuf_header(db_in, curblock_in), GUPPI_STATUS_SIZE); /* Read required exposure and PFB rate from status shared memory */ reqd_exposure = sf.data_columns.exposure; pfb_rate = sf.hdr.efsampfr / (2 * sf.hdr.nchan); /* Initialise the index in the output block */ index_out = (struct databuf_index*)guppi_databuf_index(db_out, curblock_out); index_out->num_datasets = 0; index_out->array_size = sf.hdr.nsubband * sf.hdr.nchan * NUM_STOKES * 4; first=0; } /* Loop through each spectrum (heap) in input buffer */ index_in = (struct databuf_index*)guppi_databuf_index(db_in, curblock_in); for(heap = 0; heap < index_in->num_heaps; heap++) { /* If invalid, record it and move to next heap */ if(!index_in->cpu_gpu_buf[heap].heap_valid) { n_heap_drop++; continue; } /* Read in heap from buffer */ char* heap_addr = (char*)(guppi_databuf_data(db_in, curblock_in) + sizeof(struct freq_spead_heap) * heap); struct freq_spead_heap* freq_heap = (struct freq_spead_heap*)(heap_addr); char* payload_addr = (char*)(guppi_databuf_data(db_in, curblock_in) + sizeof(struct freq_spead_heap) * MAX_HEAPS_PER_BLK + (index_in->heap_size - sizeof(struct freq_spead_heap)) * heap ); int *i_payload = (int*)(payload_addr); float *f_payload = (float*)(payload_addr); accumid = freq_heap->status_bits & 0x7; /*Debug: print heap */ /* printf("%d, %d, %d, %d, %d, %d\n", freq_heap->time_cntr, freq_heap->spectrum_cntr, freq_heap->integ_size, freq_heap->mode, freq_heap->status_bits, freq_heap->payload_data_off); */ /* If we have accumulated for long enough, write vectors to output block */ if(accum_time >= reqd_exposure) { for(i = 0; i < NUM_SW_STATES; i++) { /*If a particular accumulator is dirty, write it to output buffer */ if(accum_dirty[i]) { /*If insufficient space, first mark block as filled and request new block*/ index_out = (struct databuf_index*)(guppi_databuf_index(db_out, curblock_out)); if( (index_out->num_datasets+1) * (index_out->array_size + sizeof(struct sdfits_data_columns)) > db_out->block_size) { printf("Accumulator finished with output block %d\n", curblock_out); /* Write block number to status buffer */ guppi_status_lock_safe(&st); hputi4(st.buf, "ACCBLKOU", curblock_out); guppi_status_unlock_safe(&st); /* Update packet count and loss fields in output header */ hputi4(hdr_out, "NBLOCK", nblock_int); hputi4(hdr_out, "NPKT", npacket); hputi4(hdr_out, "NPKTDROP", n_pkt_drop); hputi4(hdr_out, "NHPDROP", n_heap_drop); /* Close out current integration */ guppi_databuf_set_filled(db_out, curblock_out); /* Wait for next output buf */ curblock_out = (curblock_out + 1) % db_out->n_block; guppi_databuf_wait_free(db_out, curblock_out); while ((rv=guppi_databuf_wait_free(db_out, curblock_out)) != GUPPI_OK) { if (rv==GUPPI_TIMEOUT) { guppi_warn("guppi_accum_thread", "timeout while waiting for output block"); continue; } else { guppi_error("guppi_accum_thread", "error waiting for free databuf"); run=0; pthread_exit(NULL); break; } } hdr_out = guppi_databuf_header(db_out, curblock_out); memcpy(hdr_out, guppi_databuf_header(db_in, curblock_in), GUPPI_STATUS_SIZE); /* Initialise the index in new output block */ index_out = (struct databuf_index*)guppi_databuf_index(db_out, curblock_out); index_out->num_datasets = 0; index_out->array_size = sf.hdr.nsubband * sf.hdr.nchan * NUM_STOKES * 4; nblock_int=0; npacket=0; n_pkt_drop=0; n_heap_drop=0; } /*Update index for output buffer*/ index_out = (struct databuf_index*)(guppi_databuf_index(db_out, curblock_out)); if(index_out->num_datasets == 0) struct_offset = 0; else struct_offset = index_out->disk_buf[index_out->num_datasets-1].array_offset + index_out->array_size; array_offset = struct_offset + sizeof(struct sdfits_data_columns); index_out->disk_buf[index_out->num_datasets].struct_offset = struct_offset; index_out->disk_buf[index_out->num_datasets].array_offset = array_offset; /*Copy sdfits_data_columns struct to disk buffer */ memcpy(guppi_databuf_data(db_out, curblock_out) + struct_offset, &data_cols[i], sizeof(struct sdfits_data_columns)); /*Copy data array to disk buffer */ memcpy(guppi_databuf_data(db_out, curblock_out) + array_offset, accumulator[i], index_out->array_size); /*Update SDFITS data_columns pointer to data array */ ((struct sdfits_data_columns*) (guppi_databuf_data(db_out, curblock_out) + struct_offset))->data = (unsigned char*)(guppi_databuf_data(db_out, curblock_out) + array_offset); index_out->num_datasets = index_out->num_datasets + 1; } } accum_time = 0; integ_num += 1; reset_accumulators(accumulator, data_cols, accum_dirty, sf.hdr.nsubband, sf.hdr.nchan); } /* Only add spectrum to accumulator if blanking bit is low */ if((freq_heap->status_bits & 0x08) == 0) { /* Fill in data columns header fields */ if(!accum_dirty[accumid]) { /*Record SPEAD header fields*/ data_cols[accumid].time = index_in->cpu_gpu_buf[heap].heap_rcvd_mjd; data_cols[accumid].time_counter = freq_heap->time_cntr; data_cols[accumid].integ_num = integ_num; data_cols[accumid].sttspec = freq_heap->spectrum_cntr; data_cols[accumid].accumid = accumid; /* Fill in rest of fields from status buffer */ strcpy(data_cols[accumid].object, sf.data_columns.object); data_cols[accumid].azimuth = sf.data_columns.azimuth; data_cols[accumid].elevation = sf.data_columns.elevation; data_cols[accumid].bmaj = sf.data_columns.bmaj; data_cols[accumid].bmin = sf.data_columns.bmin; data_cols[accumid].bpa = sf.data_columns.bpa; data_cols[accumid].centre_freq_idx = sf.data_columns.centre_freq_idx; data_cols[accumid].ra = sf.data_columns.ra; data_cols[accumid].dec = sf.data_columns.dec; data_cols[accumid].exposure = 0.0; for(i = 0; i < NUM_SW_STATES; i++) data_cols[accumid].centre_freq[i] = sf.data_columns.centre_freq[i]; accum_dirty[accumid] = 1; } data_cols[accumid].exposure += (float)(freq_heap->integ_size)/pfb_rate; data_cols[accumid].stpspec = freq_heap->spectrum_cntr; /* Add spectrum to appropriate vector accumulator (high-bw mode) */ if(payload_type == INT_PAYLOAD) { for(i = 0; i < sf.hdr.nchan; i++) { for(j = 0; j < sf.hdr.nsubband; j++) { for(k = 0; k < NUM_STOKES; k++) { accumulator[accumid] [i*sf.hdr.nsubband*NUM_STOKES + j*NUM_STOKES + k] += (float)i_payload[i*sf.hdr.nsubband*NUM_STOKES + j*NUM_STOKES + k]; } } } } /* Add spectrum to appropriate vector accumulator (low-bw mode) */ else { for(i = 0; i < sf.hdr.nchan; i++) { for(j = 0; j < sf.hdr.nsubband; j++) { for(k = 0; k < NUM_STOKES; k++) { accumulator[accumid] [i*sf.hdr.nsubband*NUM_STOKES + j*NUM_STOKES + k] += f_payload[i*sf.hdr.nsubband*NUM_STOKES + j*NUM_STOKES + k]; } } } } } accum_time += (double)freq_heap->integ_size / pfb_rate; } /* Update packet count and loss fields from input header */ nblock_int++; npacket += gp.num_pkts_rcvd; n_pkt_drop += gp.num_pkts_dropped; /* Done with current input block */ guppi_databuf_set_free(db_in, curblock_in); curblock_in = (curblock_in + 1) % db_in->n_block; /* Check for cancel */ pthread_testcancel(); } pthread_exit(NULL); pthread_cleanup_pop(0); /* Closes set_exit_status */ pthread_cleanup_pop(0); /* Closes set_finished */ pthread_cleanup_pop(0); /* Closes guppi_free_sdfits */ pthread_cleanup_pop(0); /* Closes ? */ pthread_cleanup_pop(0); /* Closes destroy_accumulators */ pthread_cleanup_pop(0); /* Closes guppi_status_detach */ pthread_cleanup_pop(0); /* Closes guppi_databuf_detach */ }
/* This thread is passed a single arg, pointer * to the guppi_udp_params struct. This thread should * be cancelled and restarted if any hardware params * change, as this potentially affects packet size, etc. */ void *guppi_fake_net_thread(void *_args) { /* Get arguments */ struct guppi_thread_args *args = (struct guppi_thread_args *)_args; /* Set cpu affinity */ cpu_set_t cpuset, cpuset_orig; sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig); CPU_ZERO(&cpuset); //CPU_SET(2, &cpuset); CPU_SET(3, &cpuset); int rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); if (rv<0) { guppi_error("guppi_fake_net_thread", "Error setting cpu affinity."); perror("sched_setaffinity"); } /* Set priority */ rv = setpriority(PRIO_PROCESS, 0, args->priority); if (rv<0) { guppi_error("guppi_fake_net_thread", "Error setting priority level."); perror("set_priority"); } /* Attach to status shared mem area */ struct guppi_status st; rv = guppi_status_attach(&st); if (rv!=GUPPI_OK) { guppi_error("guppi_fake_net_thread", "Error attaching to status shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_status_detach, &st); pthread_cleanup_push((void *)set_exit_status, &st); /* Init status, read info */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "init"); guppi_status_unlock_safe(&st); /* Read in general parameters */ struct guppi_params gp; #if FITS_TYPE == PSRFITS struct psrfits pf; pf.sub.dat_freqs = NULL; pf.sub.dat_weights = NULL; pf.sub.dat_offsets = NULL; pf.sub.dat_scales = NULL; #else struct sdfits pf; #endif char status_buf[GUPPI_STATUS_SIZE]; guppi_status_lock_safe(&st); memcpy(status_buf, st.buf, GUPPI_STATUS_SIZE); guppi_status_unlock_safe(&st); guppi_read_obs_params(status_buf, &gp, &pf); #if FITS_TYPE == PSRFITS pthread_cleanup_push((void *)guppi_free_psrfits, &pf); #else pthread_cleanup_push((void *)guppi_free_sdfits, &pf); #endif /* Attach to databuf shared mem */ struct guppi_databuf *db; db = guppi_databuf_attach(args->output_buffer); if (db==NULL) { guppi_error("guppi_fake_net_thread", "Error attaching to databuf shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_databuf_detach, db); /* Time parameters */ int stt_imjd=0, stt_smjd=0; double stt_offs=0.0; /* Figure out size of data in each packet, number of packets * per block, etc. Changing packet size during an obs is not * recommended. */ int block_size; if (hgeti4(status_buf, "BLOCSIZE", &block_size)==0) { block_size = db->block_size; hputi4(status_buf, "BLOCSIZE", block_size); } else { if (block_size > db->block_size) { guppi_error("guppi_net_thread", "BLOCSIZE > databuf block_size"); block_size = db->block_size; hputi4(status_buf, "BLOCSIZE", block_size); } } unsigned heaps_per_block = block_size / sizeof(struct freq_spead_heap); /* List of databuf blocks currently in use */ unsigned i; const int nblock = 2; struct fake_datablock_stats blocks[nblock]; for (i=0; i<nblock; i++) fake_init_block(&blocks[i], db, sizeof(struct freq_spead_heap), heaps_per_block); /* Convenience names for first/last blocks in set */ struct fake_datablock_stats *fblock, *lblock; fblock = &blocks[0]; lblock = &blocks[nblock-1]; /* Misc counters, etc */ char *curdata=NULL, *curheader=NULL, *curindex=NULL; int first_time = 1; int heap_cntr = 0, next_block_heap_cntr = heaps_per_block; /* Main loop */ unsigned force_new_block=0, waiting=-1; signal(SIGINT,cc); while (run_threads) { /* Wait for data */ struct timespec sleep_dur, rem_sleep_dur; sleep_dur.tv_sec = 0; sleep_dur.tv_nsec = 2e6; nanosleep(&sleep_dur, &rem_sleep_dur); /* Update status if needed */ if (waiting!=0) { guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "receiving"); guppi_status_unlock_safe(&st); waiting=0; } /* Convert packet format if needed */ if (first_time) { first_time = 0; force_new_block=1; } else force_new_block=0; /* Determine if we go to next block */ if ((heap_cntr>=next_block_heap_cntr) || force_new_block) { printf("casper: going to next shared memory block\n"); /* Update drop stats */ guppi_status_lock_safe(&st); hputr8(st.buf, "DROPAVG", 0.0); hputr8(st.buf, "DROPTOT", 0.0); hputr8(st.buf, "DROPBLK", 0.0); guppi_status_unlock_safe(&st); /* Finalize first block, and push it off the list. * Then grab next available block. */ if (fblock->block_idx>=0) fake_finalize_block(fblock); fake_block_stack_push(blocks, nblock); fake_increment_block(lblock, heap_cntr); curdata = guppi_databuf_data(db, lblock->block_idx); curheader = guppi_databuf_header(db, lblock->block_idx); curindex = guppi_databuf_index(db, lblock->block_idx); next_block_heap_cntr = lblock->heap_idx + heaps_per_block; /* If new obs started, reset total counters, get start * time. Start time is rounded to nearest integer * second, with warning if we're off that by more * than 100ms. Any current blocks on the stack * are also finalized/reset */ if (force_new_block) { /* Get obs start time */ get_current_mjd(&stt_imjd, &stt_smjd, &stt_offs); if (stt_offs>0.5) { stt_smjd+=1; stt_offs-=1.0; } stt_offs = 0.0; /* Flush any current buffers */ for (i=0; i<nblock-1; i++) { if (blocks[i].block_idx>=0) fake_finalize_block(&blocks[i]); fake_reset_block(&blocks[i]); } } /* Read/update current status shared mem */ guppi_status_lock_safe(&st); if (stt_imjd!=0) { #if 1 hputi4(st.buf, "STT_IMJD", stt_imjd); hputi4(st.buf, "STT_SMJD", stt_smjd); hputr8(st.buf, "STT_OFFS", stt_offs); #endif hputi4(st.buf, "STTVALID", 1); } else { hputi4(st.buf, "STTVALID", 0); } memcpy(status_buf, st.buf, GUPPI_STATUS_SIZE); guppi_status_unlock_safe(&st); /* Wait for new block to be free, then clear it * if necessary and fill its header with new values. */ while ((rv=guppi_databuf_wait_free(db, lblock->block_idx)) != GUPPI_OK) { if (rv==GUPPI_TIMEOUT) { waiting=1; guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "blocked"); guppi_status_unlock_safe(&st); continue; } else { guppi_error("guppi_fake_net_thread", "error waiting for free databuf"); run_threads=0; pthread_exit(NULL); break; } } memcpy(curheader, status_buf, GUPPI_STATUS_SIZE); memset(curdata, 0, block_size); memset(curindex, 0, db->index_size); } /*Write fake data to block */ write_fake_heap_to_block(lblock, heap_cntr); heap_cntr++; /* Will exit if thread has been cancelled */ pthread_testcancel(); } pthread_exit(NULL); /* Have to close all push's */ pthread_cleanup_pop(0); /* Closes set_exit_status */ pthread_cleanup_pop(0); /* Closes guppi_free_psrfits */ pthread_cleanup_pop(0); /* Closes guppi_status_detach */ pthread_cleanup_pop(0); /* Closes guppi_databuf_detach */ }