// Method to calculate the buffer address for packet payload
// Also verifies FID and XID of packets
static inline int calc_block_indices(block_info_t * binfo, packet_header_t * pkt_header) {
// Verify FID and XID
if (pkt_header->fid >= Nf) {
hashpipe_error(__FUNCTION__, "packet FID %u out of range (0-%d)", pkt_header->fid, Nf-1);
return -1;
}
else if (pkt_header->xid != binfo->self_xid && binfo->self_xid != -1) {
hashpipe_error(__FUNCTION__, "unexpected packet XID %d (expected %d)", pkt_header->xid, binfo->self_xid);
return -1;
}
binfo->m = pkt_header->mcnt % Nm;
binfo->f = pkt_header->fid;
return 0;
}
//----------------------------------------------------------
int write_hits_header(etfits_t * etf, int beampol, size_t nhits, size_t missed_pkts) {
//----------------------------------------------------------
#define TFIELDS 4
int * status_p = &(etf->status);
*status_p = 0;
int tbltype = BINARY_TBL;
long long naxis2 = 0;
//const int tfields = 3;
// TODO check chan types!
const char *ttype[TFIELDS] = {"DETPOW ", "MEANPOW ", "COARCHAN", "FINECHAN"};
const char *tform[TFIELDS] = {"1E", "1E", "1U", "1V"}; // cfitsio format codes
// 32-bit floats 16-bit unint 32-bit uint
if(etf->integration_cnt == 0 && etf->beampol_cnt == 0) {
// at start of file go to the template created HDU for this set of beampols
if(! *status_p) fits_movnam_hdu(etf->fptr, BINARY_TBL, (char *)"ETHITS", 0, status_p);
} else {
// otherwise create new HDU for this set of beampols
if(! *status_p) fits_create_tbl(etf->fptr, BINARY_TBL, 0, TFIELDS, (char **)&ttype, (char **)&tform, NULL, (char *)"ETHITS", status_p);
}
if(! *status_p) fits_update_key(etf->fptr, TINT, "TIME", &(etf->hits_hdr[beampol].time), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "RA", &(etf->hits_hdr[beampol].ra), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "DEC", &(etf->hits_hdr[beampol].dec), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "BEAMPOL", &(etf->hits_hdr[beampol].beampol), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "NHITS", &nhits, NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "MISSEDPK",&missed_pkts, NULL, status_p);
if (*status_p) {
hashpipe_error(__FUNCTION__, "Error writing hits header");
fits_report_error(stderr, *status_p);
}
}
//----------------------------------------------------------
int write_primary_header(etfits_t * etf) {
//----------------------------------------------------------
int * status_p = &(etf->status);
*status_p = 0;
int itmp;
char ctmp[40];
//fprintf(stderr, "writing primary header\n");
if(! *status_p) fits_get_system_time(ctmp, &itmp, status_p); // date the file was written
if(! *status_p) fits_movabs_hdu(etf->fptr, 1, NULL, status_p); // go to primary HDU
if(! *status_p) fits_update_key(etf->fptr, TSTRING, "DATE", ctmp, NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TSTRING, "TELESCOP", &(etf->primary_hdr.receiver), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "NSUBBAND", &(etf->primary_hdr.n_subband), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "NCHAN", &(etf->primary_hdr.n_chan), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "NINPUTS", &(etf->primary_hdr.n_inputs), NULL, status_p);
// TODO not yet implemented
//if(! *status_p) fits_update_key(etf->fptr, TINT, "BANDWID", &(etf->primary_hdr.bandwidth), NULL, status_p);
//if(! *status_p) fits_update_key(etf->fptr, TINT, "CHAN_BW", &(etf->primary_hdr.chan_bandwidth), NULL, status_p);
//if(! *status_p) its_update_key(etf->fptr, TINT, "FREQRES", &(etf->primary_hdr.freq_res), NULL, status_p); // redundant w/ CHAN_BW?
//if(! *status_p) fits_flush_file(etf->fptr, status_p);
if (*status_p) {
hashpipe_error(__FUNCTION__, "Error updating primary header");
//fprintf(stderr, "Error updating primary header.\n");
fits_report_error(stderr, *status_p);
}
return *status_p;
}
// Method to process a received packet
// Processing involves the following
// (1) header extraction
// (2) block population (output buffer data type is a block)
// (3) buffer population (if block is filled)
static inline uint64_t process_packet(flag_input_databuf_t * db, struct hashpipe_udp_packet *p) {
static block_info_t binfo;
packet_header_t pkt_header;
// Parse packet header
get_header(p, &pkt_header);
uint64_t pkt_mcnt = pkt_header.mcnt;
uint64_t cur_mcnt = binfo.mcnt_start;
int dest_block_idx = get_block_idx(pkt_mcnt);
// fprintf(stdout, "Block idx = %d\n", dest_block_idx);
// Check mcnt to see if packet belongs in current block, next, or the one after
uint64_t pkt_mcnt_dist = pkt_mcnt - cur_mcnt;
if (pkt_mcnt_dist >= 2*Nm) { // 2nd next block (Current block + 2)
set_block_filled(db, &binfo);
// Advance mcnt_start to next block
cur_mcnt += Nm;
binfo.mcnt_start += Nm;
binfo.block_i = (binfo.block_i + 1) % N_INPUT_BLOCKS;
// Initialize next block
flag_input_databuf_wait_free(db, dest_block_idx);
initialize_block(db, pkt_mcnt);
// Reset packet counter for this block
binfo.packet_count[dest_block_idx] = 0;
}
// Increment packet count for block
binfo.packet_count[dest_block_idx]++;
// Validate FID and XID
// Calculate "m" and "f" which index the buffer for writing packet payload
if (calc_block_indices(&binfo, &pkt_header) == -1) {
hashpipe_error(__FUNCTION__, "invalid FID and XID in header");
return -1;
}
// Calculate starting points for writing packet payload into buffer
// POSSIBLE RACE CONDITION!!!! Need to lock db->block access with semaphore
uint64_t * dest_p = db->block[dest_block_idx].data + flag_input_databuf_idx(binfo.m, binfo.f, 0, 0);
const uint64_t * payload_p = (uint64_t *)(p->data+8);
// Copy data into buffer
memcpy(dest_p, payload_p, N_BYTES_PER_PACKET);
// print_pkt_header(&pkt_header);
return pkt_mcnt;
}
//----------------------------------------------------------
int check_for_file_roll(etfits_t *etf) {
//----------------------------------------------------------
// checks if we need to roll over to a new file
int * status_p = &(etf->status);
struct stat st;
fits_flush_file(etf->fptr, status_p); // flush to get true size
if(stat(etf->filename_working, &st) == -1) {
hashpipe_error(__FUNCTION__, "Error getting etfits file size");
} else {
//fprintf(stderr, "size of %s is %ld\n", etf->filename, st.st_size);
if(st.st_size >= etf->max_file_size) {
etf->file_num++;
etf->new_file = 1;
}
}
return *status_p;
}
//----------------------------------------------------------
int etfits_close(etfits_t *etf) {
//----------------------------------------------------------
int * status_p = &(etf->status);
*status_p = 0;
int rv;
fits_close_file(etf->fptr, status_p);
// TODO test for successful file close
etf->file_open = 0;
rv = rename((const char *)etf->filename_working, (const char *)etf->filename_fits);
if(rv) {
hashpipe_error(__FUNCTION__, "file rename error : %d %s", errno, strerror(errno));
} else {
hashpipe_info(__FUNCTION__, "Done. %s %ld data rows into %s (status = %d).\n",
etf->mode=='r' ? "Read" : "Wrote",
etf->tot_rows, etf->filename_fits, *status_p);
}
return *status_p;
}
static void *run_method(hashpipe_thread_args_t * args)
{
int rv = 0;
easy_in_output_databuf_t *db_out = (easy_in_output_databuf_t *)args->obuf;
hashpipe_status_t st = args->st;
const char * status_key = args->thread_desc->skey;
int idx_data = 0;
char data = 'a';
while (run_threads())
{
while ((rv=hashpipe_databuf_wait_free((hashpipe_databuf_t *)db_out, idx_data)) != HASHPIPE_OK) {
if (rv==HASHPIPE_TIMEOUT) {
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "blocked_in");
hashpipe_status_unlock_safe(&st);
continue;
} else {
hashpipe_error(__FUNCTION__, "error waiting for free databuf");
pthread_exit(NULL);
break;
}
}
#ifdef DEBUG
fprintf(stdout,"easy_in_thread:\n");
fprintf(stdout,"\tcount = %d\n",db_out->count);
fprintf(stdout,"\tdata[%d] = %c\n",idx_data,'a' + (char)(db_out->count % 26));
#endif
db_out->data[idx_data] = 'a' + (char)(db_out->count % 26);
db_out->count++;
hashpipe_databuf_set_filled((hashpipe_databuf_t *)db_out, idx_data);
idx_data = (idx_data + 1) % db_out->header.n_block;
pthread_testcancel();
}
// Thread success!
return NULL;
}
static void *run(hashpipe_thread_args_t * args)
{
s6_input_databuf_t *db = (s6_input_databuf_t *)args->obuf;
hashpipe_status_t *p_st = &(args->st);
hashpipe_status_t st = args->st;
const char * status_key = args->thread_desc->skey;
//s6_input_block_t fake_data_block;
/* Main loop */
int i, rv;
uint64_t mcnt = 0;
uint64_t num_coarse_chan = N_COARSE_CHAN;
uint64_t *data;
int block_idx = 0;
int error_count = 0, max_error_count = 0;
float error, max_error = 0.0;
int gen_fake = 0;
hashpipe_status_lock_safe(&st);
//hashpipe_status_lock_safe(p_st);
hputi4(st.buf, "NUMCCHAN", N_COARSE_CHAN);
hputi4(st.buf, "NUMFCHAN", N_FINE_CHAN);
hputi4(st.buf, "NUMBBEAM", N_BYTES_PER_BEAM);
hputi4(st.buf, "NUMBBLOC", sizeof(s6_input_block_t));
hputi4(st.buf, "THRESHLD", POWER_THRESH);
hgeti4(st.buf, "GENFAKE", &gen_fake);
hashpipe_status_unlock_safe(&st);
//hashpipe_status_unlock_safe(p_st);
time_t t, prior_t;
prior_t = time(&prior_t);
while (run_threads()) {
hashpipe_status_lock_safe(&st);
//hashpipe_status_lock_safe(p_st);
hputi4(st.buf, "NETBKOUT", block_idx);
hputs(st.buf, status_key, "waiting");
hashpipe_status_unlock_safe(&st);
//hashpipe_status_unlock_safe(p_st);
t = time(&t);
fprintf(stderr, "elapsed seconds for block %d : %ld\n", block_idx, t - prior_t);
prior_t = t;
// Wait for data
struct timespec sleep_dur, rem_sleep_dur;
sleep_dur.tv_sec = 1;
sleep_dur.tv_nsec = 0;
//fprintf(stderr, "fake net thread sleeping for %7.5f seconds\n",
// sleep_dur.tv_sec + (double)sleep_dur.tv_nsec/1000000000.0);
nanosleep(&sleep_dur, &rem_sleep_dur);
/* Wait for new block to be free, then clear it
* if necessary and fill its header with new values.
*/
while ((rv=s6_input_databuf_wait_free(db, block_idx))
!= HASHPIPE_OK) {
if (rv==HASHPIPE_TIMEOUT) {
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "blocked");
hashpipe_status_unlock_safe(&st);
continue;
} else {
hashpipe_error(__FUNCTION__, "error waiting for free databuf");
pthread_exit(NULL);
break;
}
}
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "receiving");
hashpipe_status_unlock_safe(&st);
// populate block header
db->block[block_idx].header.mcnt = mcnt;
db->block[block_idx].header.coarse_chan_id = 321;
db->block[block_idx].header.num_coarse_chan = num_coarse_chan;
memset(db->block[block_idx].header.missed_pkts, 0, sizeof(uint64_t) * N_BEAM_SLOTS);
if(gen_fake) {
gen_fake = 0;
// gen fake data for all beams, all blocks
// TODO vary data by beam
fprintf(stderr, "generating fake data to block 0 beam 0...");
gen_fake_data(&(db->block[0].data[0]));
fprintf(stderr, " done\n");
fprintf(stderr, "copying to block 0 beam");
for(int beam_i = 1; beam_i < N_BEAMS; beam_i++) {
fprintf(stderr, " %d", beam_i);
memcpy((void *)&db->block[0].data[beam_i*N_BYTES_PER_BEAM/sizeof(uint64_t)],
(void *)&db->block[0].data[0],
N_BYTES_PER_BEAM);
}
fprintf(stderr, " done\n");
fprintf(stderr, "copying to block");
for(int block_i = 1; block_i < N_INPUT_BLOCKS; block_i++) {
fprintf(stderr, " %d", block_i);
memcpy((void *)&db->block[block_i].data[0],
(void *)&db->block[0].data[0],
N_DATA_BYTES_PER_BLOCK);
}
fprintf(stderr, " done\n");
}
hashpipe_status_lock_safe(&st);
hputr4(st.buf, "NETMXERR", max_error);
hputi4(st.buf, "NETERCNT", error_count);
hputi4(st.buf, "NETMXECT", max_error_count);
hashpipe_status_unlock_safe(&st);
// Mark block as full
s6_input_databuf_set_filled(db, block_idx);
// Setup for next block
block_idx = (block_idx + 1) % db->header.n_block;
mcnt++;
// uncomment the following to test dynamic setting of num_coarse_chan
//num_coarse_chan--;
/* Will exit if thread has been cancelled */
pthread_testcancel();
}
// Thread success!
return THREAD_OK;
}
static void *run(hashpipe_thread_args_t * args)
{
// Local aliases to shorten access to args fields
// Our output buffer happens to be a paper_input_databuf
hashpipe_status_t st = args->st;
const char * status_key = args->thread_desc->skey;
st_p = &st; // allow global (this source file) access to the status buffer
// Get inital value for crc32 function
uint32_t init_crc = crc32(0,0,0);
// Flag that holds off the crc thread
int holdoff = 1;
// Force ourself into the hold off state
hashpipe_status_lock_safe(&st);
hputi4(st.buf, "CRCHOLD", 1);
hashpipe_status_unlock_safe(&st);
while(holdoff) {
// We're not in any hurry to startup
sleep(1);
hashpipe_status_lock_safe(&st);
// Look for CRCHOLD value
hgeti4(st.buf, "CRCHOLD", &holdoff);
if(!holdoff) {
// Done holding, so delete the key
hdel(st.buf, "CRCHOLD");
}
hashpipe_status_unlock_safe(&st);
}
/* Read network params */
struct hashpipe_udp_params up = {
.bindhost = "0.0.0.0",
.bindport = 8511,
.packet_size = 8200
};
hashpipe_status_lock_safe(&st);
// Get info from status buffer if present (no change if not present)
hgets(st.buf, "BINDHOST", 80, up.bindhost);
hgeti4(st.buf, "BINDPORT", &up.bindport);
// Store bind host/port info etc in status buffer
hputs(st.buf, "BINDHOST", up.bindhost);
hputi4(st.buf, "BINDPORT", up.bindport);
hputu4(st.buf, "CRCPKOK", 0);
hputu4(st.buf, "CRCPKERR", 0);
hputs(st.buf, status_key, "running");
hashpipe_status_unlock_safe(&st);
struct hashpipe_udp_packet p;
/* Give all the threads a chance to start before opening network socket */
sleep(1);
/* Set up UDP socket */
int rv = hashpipe_udp_init(&up);
if (rv!=HASHPIPE_OK) {
hashpipe_error("paper_crc_thread",
"Error opening UDP socket.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)hashpipe_udp_close, &up);
/* Main loop */
uint64_t packet_count = 0;
uint64_t good_count = 0;
uint64_t error_count = 0;
uint64_t elapsed_wait_ns = 0;
uint64_t elapsed_recv_ns = 0;
uint64_t elapsed_proc_ns = 0;
float ns_per_wait = 0.0;
float ns_per_recv = 0.0;
float ns_per_proc = 0.0;
struct timespec start, stop;
struct timespec recv_start, recv_stop;
packet_header_t hdr;
while (run_threads()) {
/* Read packet */
clock_gettime(CLOCK_MONOTONIC, &recv_start);
do {
clock_gettime(CLOCK_MONOTONIC, &start);
p.packet_size = recv(up.sock, p.data, HASHPIPE_MAX_PACKET_SIZE, 0);
clock_gettime(CLOCK_MONOTONIC, &recv_stop);
} while (p.packet_size == -1 && (errno == EAGAIN || errno == EWOULDBLOCK) && run_threads());
// Break out of loop if stopping
if(!run_threads()) break;
// Increment packet count
packet_count++;
// Check CRC
if(crc32(init_crc, (/*const?*/ uint8_t *)p.data, p.packet_size) == 0xffffffff) {
// CRC OK! Increment good counter
good_count++;
} else {
// CRC error! Increment error counter
error_count++;
// Log message
get_header(&p, &hdr);
hashpipe_warn("paper_crc", "CRC error mcnt %llu ; fid %u ; xid %u",
hdr.mcnt, hdr.fid, hdr.xid);
}
clock_gettime(CLOCK_MONOTONIC, &stop);
elapsed_wait_ns += ELAPSED_NS(recv_start, start);
elapsed_recv_ns += ELAPSED_NS(start, recv_stop);
elapsed_proc_ns += ELAPSED_NS(recv_stop, stop);
if(packet_count % 1000 == 0) {
// Compute stats
get_header(&p, &hdr);
ns_per_wait = (float)elapsed_wait_ns / packet_count;
ns_per_recv = (float)elapsed_recv_ns / packet_count;
ns_per_proc = (float)elapsed_proc_ns / packet_count;
// Update status
hashpipe_status_lock_busywait_safe(&st);
hputu8(st.buf, "CRCMCNT", hdr.mcnt);
// Gbps = bits_per_packet / ns_per_packet
// (N_BYTES_PER_PACKET excludes header, so +8 for the header)
hputr4(st.buf, "CRCGBPS", 8*(N_BYTES_PER_PACKET+8)/(ns_per_recv+ns_per_proc));
hputr4(st.buf, "CRCWATNS", ns_per_wait);
hputr4(st.buf, "CRCRECNS", ns_per_recv);
hputr4(st.buf, "CRCPRCNS", ns_per_proc);
// TODO Provide some way to recognize request to zero out the
// CRCERR and CRCOK fields.
hputu8(st.buf, "CRCPKOK", good_count);
hputu8(st.buf, "CRCPKERR", error_count);
hashpipe_status_unlock_safe(&st);
// Start new average
elapsed_wait_ns = 0;
elapsed_recv_ns = 0;
elapsed_proc_ns = 0;
packet_count = 0;
}
/* Will exit if thread has been cancelled */
pthread_testcancel();
}
/* Have to close all push's */
pthread_cleanup_pop(1); /* Closes push(hashpipe_udp_close) */
return NULL;
}
static hashpipe_thread_desc_t crc_thread = {
name: "paper_crc_thread",
skey: "CRCSTAT",
init: NULL,
run: run,
ibuf_desc: {NULL},
// Run method for the thread
static void *run(hashpipe_thread_args_t * args) {
// Local aliases to shorten access to args fields
// Our output buffer happens to be a paper_input_databuf
grating_input_databuf_t *db_in = (grating_input_databuf_t *)args->ibuf;
grating_gpu_input_databuf_t * db_out = (grating_gpu_input_databuf_t *)args->obuf;
hashpipe_status_t st = args->st;
const char * status_key = args->thread_desc->skey;
st_p = &st; // allow global (this source file) access to the status buffer
// Set thread to "start" state
hashpipe_status_lock_safe(&st);
hputs(st.buf, "TRANREADY", "start");
hashpipe_status_unlock_safe(&st);
int rv;
int curblock_in = 0;
int curblock_out = 0;
int mcnt;
fprintf(stdout, "Tra: Starting Thread!\n");
while (run_threads()) {
while ((rv=grating_input_databuf_wait_filled(db_in, curblock_in)) != HASHPIPE_OK) {
if (rv==HASHPIPE_TIMEOUT) {
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "waiting for filled block");
hashpipe_status_unlock_safe(&st);
}
else {
hashpipe_error(__FUNCTION__, "error waiting for filled databuf block");
pthread_exit(NULL);
break;
}
}
while ((rv=grating_gpu_input_databuf_wait_free(db_out, curblock_out)) != HASHPIPE_OK) {
if (rv==HASHPIPE_TIMEOUT) {
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "waiting for free block");
hashpipe_status_unlock_safe(&st);
}
else {
hashpipe_error(__FUNCTION__, "error waiting for free databuf block");
pthread_exit(NULL);
break;
}
}
mcnt = db_in->block[curblock_in].header.mcnt_start;
int m;
int f;
int t;
int c;
uint64_t * in_p;
uint64_t * out_p;
uint64_t * block_in_p = db_in->block[curblock_in].data;
uint64_t * block_out_p = db_out->block[curblock_out].data;
for (m = 0; m < Nm; m++) {
for (t = 0; t < Nt; t++) {
for (f = 0; f < Nf; f++) {
for (c = 0; c < Nc; c++) {
in_p = block_in_p + grating_input_databuf_idx(m,f,t,c);
out_p = block_out_p + grating_gpu_input_databuf_idx(m,f,t,c);
memcpy(out_p, in_p, 128/8);
}
}
}
}
db_out->block[curblock_out].header.mcnt = mcnt;
grating_gpu_input_databuf_set_filled(db_out, curblock_out);
curblock_out = (curblock_out + 1) % db_out->header.n_block;
grating_input_databuf_set_free(db_in, curblock_in);
curblock_in = (curblock_in + 1) % db_in->header.n_block;
/* Will exit if thread has been cancelled */
pthread_testcancel();
}
return NULL;
}
// Run method for the thread
// It is meant to do the following:
// (1) Initialize status buffer
// (2) Set up network parameters and socket
// (3) Start main loop
// (3a) Receive packet on socket
// (3b) Error check packet (packet size, etc)
// (3c) Call process_packet on received packet
// (4) Terminate thread cleanly
static void *run(hashpipe_thread_args_t * args) {
fprintf(stdout, "N_INPUTS = %d\n", N_INPUTS);
fprintf(stdout, "N_CHAN = %d\n", N_CHAN);
fprintf(stdout, "N_CHAN_PER_X = %d\n", N_CHAN_PER_X);
fprintf(stdout, "N_CHAN_PER_PACKET = %d\n", N_CHAN_PER_PACKET);
fprintf(stdout, "N_TIME_PER_PACKET = %d\n", N_TIME_PER_PACKET);
fprintf(stdout, "N_TIME_PER_BLOCK = %d\n", N_TIME_PER_BLOCK);
fprintf(stdout, "N_BYTES_PER_BLOCK = %d\n", N_BYTES_PER_BLOCK);
fprintf(stdout, "N_BYTES_PER_PACKET = %d\n", N_BYTES_PER_PACKET);
fprintf(stdout, "N_PACKETS_PER_BLOCK = %d\n", N_PACKETS_PER_BLOCK);
fprintf(stdout, "N_COR_MATRIX = %d\n", N_COR_MATRIX);
// Local aliases to shorten access to args fields
// Our output buffer happens to be a paper_input_databuf
flag_input_databuf_t *db = (flag_input_databuf_t *)args->obuf;
hashpipe_status_t st = args->st;
const char * status_key = args->thread_desc->skey;
st_p = &st; // allow global (this source file) access to the status buffer
/* Read network params */
fprintf(stdout, "Setting up network parameters\n");
struct hashpipe_udp_params up = {
.bindhost = "0.0.0.0",
.bindport = 8511,
.packet_size = 8008
};
hashpipe_status_lock_safe(&st);
// Get info from status buffer if present (no change if not present)
hgets(st.buf, "BINDHOST", 80, up.bindhost);
hgeti4(st.buf, "BINDPORT", &up.bindport);
// Store bind host/port info etc in status buffer
hputs(st.buf, "BINDHOST", up.bindhost);
hputi4(st.buf, "BINDPORT", up.bindport);
hputu4(st.buf, "MISSEDFE", 0);
hputu4(st.buf, "MISSEDPK", 0);
hputs(st.buf, status_key, "running");
hashpipe_status_unlock_safe(&st);
struct hashpipe_udp_packet p;
/* Give all the threads a chance to start before opening network socket */
int netready = 0;
int corready = 0;
int checkready = 0;
while (!netready) {
sleep(1);
// Check the correlator to see if it's ready yet
hashpipe_status_lock_safe(&st);
hgeti4(st.buf, "CORREADY", &corready);
hgeti4(st.buf, "SAVEREADY", &checkready);
hashpipe_status_unlock_safe(&st);
if (!corready) {
continue;
}
//if (!checkready) {
// continue;
//}
// Check the other threads to see if they're ready yet
// TBD
// If we get here, then all threads are initialized
netready = 1;
}
sleep(3);
/* Set up UDP socket */
fprintf(stderr, "NET: BINDHOST = %s\n", up.bindhost);
fprintf(stderr, "NET: BINDPORT = %d\n", up.bindport);
int rv = hashpipe_udp_init(&up);
if (rv!=HASHPIPE_OK) {
hashpipe_error("paper_net_thread",
"Error opening UDP socket.");
pthread_exit(NULL);
}
pthread_cleanup_push((void *)hashpipe_udp_close, &up);
// Initialize first few blocks in the buffer
int i;
for (i = 0; i < 2; i++) {
// Wait until block semaphore is free
if (flag_input_databuf_wait_free(db, i) != HASHPIPE_OK) {
if (errno == EINTR) { // Interrupt occurred
hashpipe_error(__FUNCTION__, "waiting for free block interrupted\n");
pthread_exit(NULL);
} else {
hashpipe_error(__FUNCTION__, "error waiting for free block\n");
pthread_exit(NULL);
}
}
initialize_block(db, i*Nm);
}
// Set correlator to "start" state
hashpipe_status_lock_safe(&st);
hputs(st.buf, "INTSTAT", "start");
hashpipe_status_unlock_safe(&st);
/* Main loop */
uint64_t packet_count = 0;
fprintf(stdout, "Net: Starting Thread!\n");
while (run_threads()) {
// Get packet
do {
p.packet_size = recv(up.sock, p.data, HASHPIPE_MAX_PACKET_SIZE, 0);
} while (p.packet_size == -1 && (errno == EAGAIN || errno == EWOULDBLOCK) && run_threads());
if(!run_threads()) break;
if (up.packet_size != p.packet_size && up.packet_size != p.packet_size-8) {
// If an error was returned instead of a valid packet size
if (p.packet_size == -1) {
fprintf(stderr, "uh oh!\n");
// Log error and exit
hashpipe_error("paper_net_thread",
"hashpipe_udp_recv returned error");
perror("hashpipe_udp_recv");
pthread_exit(NULL);
} else {
// Log warning and ignore wrongly sized packet
hashpipe_warn("paper_net_thread", "Incorrect pkt size (%d)", p.packet_size);
continue;
}
}
packet_count++;
process_packet(db, &p);
/* Will exit if thread has been cancelled */
pthread_testcancel();
}
pthread_cleanup_pop(1); /* Closes push(hashpipe_udp_close) */
hashpipe_status_lock_busywait_safe(&st);
hputs(st.buf, status_key, "terminated");
hashpipe_status_unlock_safe(&st);
return NULL;
}
static hashpipe_thread_desc_t net_thread = {
name: "flag_net_thread",
skey: "NETSTAT",
init: NULL,
run: run,
ibuf_desc: {NULL},
//----------------------------------------------------------
int write_etfits(s6_output_databuf_t *db, int block_idx, etfits_t *etf, scram_t *scram_p) {
//----------------------------------------------------------
int row, rv;
int nchan, nivals, nsubband;
char* temp_str;
double temp_dbl;
size_t nhits;
int * status_p = &(etf->status);
*status_p = 0;
scram_t scram;
extern const char *receiver[];
// Create the initial file or change to a new one if needed.
if (etf->new_run || etf->new_file) {
etf->new_file = 0;
if (!etf->new_run) {
if(etf->file_open) {
etfits_close(etf);
}
etf->integration_cnt = 0;
}
// TODO update code versions
etf->primary_hdr.n_subband = db->block[block_idx].header.num_coarse_chan;
etf->primary_hdr.n_chan = N_FINE_CHAN;
etf->primary_hdr.n_inputs = N_BEAMS * N_POLS_PER_BEAM;
strncpy(etf->primary_hdr.receiver, receiver[scram_p->receiver], sizeof(etf->primary_hdr.receiver));
// TODO not yet implemented
//etf->primary_hdr.bandwidth = ;
//etf->primary_hdr.chan_bandwidth = ;
//etf->primary_hdr.freq_res = ;
etfits_create(etf);
if(*status_p) {
hashpipe_error(__FUNCTION__, "Error creating/initializing new etfits file");
//fprintf(stderr, "Error creating/initializing new etfits file.\n");
fits_report_error(stderr, *status_p);
exit(1);
}
write_primary_header(etf);
etf->file_cnt++;
}
// populate hits header data
// TODO maybe I should do away with this and write directly to the header
// from scram in write_hits_header()
for(int i=0; i < N_BEAMS*N_POLS_PER_BEAM; i++) {
etf->hits_hdr[i].time = (time_t)s6_seti_ao_timeMS2unixtime(scram_p->AGCTIME, scram_p->AGCSTIME);
etf->hits_hdr[i].ra = scram_p->ra_by_beam[int(floor(i/N_POLS_PER_BEAM))];
etf->hits_hdr[i].dec = scram_p->dec_by_beam[int(floor(i/N_POLS_PER_BEAM))];
etf->hits_hdr[i].beampol = i;
}
if(! *status_p) write_integration_header(etf, scram_p);
if(! *status_p) nhits = write_hits(db, block_idx, etf);
etf->integration_cnt++;
// Now update some key values if no CFITSIO errors
if (! *status_p) {
etf->tot_rows += nhits;
etf->N += 1;
*status_p = check_for_file_roll(etf);
}
if(*status_p) {
hashpipe_error(__FUNCTION__, "FITS error, exiting");
//fprintf(stderr, "FITS error, exiting.\n");
exit(1);
}
return *status_p;
}
//----------------------------------------------------------
int write_hits(s6_output_databuf_t *db, int block_idx, etfits_t *etf) {
//----------------------------------------------------------
long firstrow, firstelem, colnum;
size_t nrows, hit_i, nhits_this_input;
int cur_beam, cur_input, cur_beampol;
size_t nhits = 0;
int * status_p = &(etf->status);
*status_p = 0;
etf->beampol_cnt = 0;
//fprintf(stderr, "writing hits\n");
//std::vector<hits_t> hits;
float det_pow[MAXHITS];
float mean_pow[MAXHITS];
int coarse_chan[MAXHITS];
int fine_chan[MAXHITS];
firstrow = 1;
firstelem = 1;
for(int beam=0; beam < N_BEAMS; beam++) {
// TODO - this goes through the output block for each beam twice. We could cut
// this in half if we have a set of arrays for each pol (consuming twice the
// memory).
for(int input=0; input < N_POLS_PER_BEAM; input++) {
int beampol = beam * N_POLS_PER_BEAM + input;
int hit_j=0;
nhits_this_input=0;
for(int hit_i=0; hit_i < (size_t)db->block[block_idx].header.nhits[beam]; hit_i++) {
if(db->block[block_idx].pol[beam][hit_i] == input) {
nhits_this_input++;
det_pow[hit_j] = db->block[block_idx].power[beam][hit_i];
mean_pow[hit_j] = db->block[block_idx].baseline[beam][hit_i];
coarse_chan[hit_j] = db->block[block_idx].coarse_chan[beam][hit_i];
fine_chan[hit_j] = db->block[block_idx].fine_chan[beam][hit_i];
hit_j++;
}
}
write_hits_header(etf,
beampol,
nhits_this_input,
(size_t)db->block[block_idx].header.missed_pkts[beam]);
nhits += nhits_this_input;
// write the hits for this input
colnum = 1;
if(! *status_p) fits_write_col(etf->fptr, TFLOAT, colnum, firstrow, firstelem, nhits_this_input, det_pow, status_p);
colnum = 2;
if(! *status_p) fits_write_col(etf->fptr, TFLOAT, colnum, firstrow, firstelem, nhits_this_input, mean_pow, status_p);
colnum = 3;
if(! *status_p) fits_write_col(etf->fptr, TINT, colnum, firstrow, firstelem, nhits_this_input, coarse_chan, status_p);
colnum = 4;
if(! *status_p) fits_write_col(etf->fptr, TINT, colnum, firstrow, firstelem, nhits_this_input, fine_chan, status_p);
etf->beampol_cnt++;
}
}
if (*status_p) {
hashpipe_error(__FUNCTION__, "Error writing hits");
//fprintf(stderr, "Error writing hits.\n");
fits_report_error(stderr, *status_p);
}
// non-standard return
if(*status_p) {
return -1;
} else {
return nhits;
}
}
//----------------------------------------------------------
int write_integration_header(etfits_t * etf, scram_t *scram) {
//----------------------------------------------------------
int * status_p = &(etf->status);
*status_p = 0;
//fprintf(stderr, "writing integration header\n");
if(etf->integration_cnt == 0) {
// go to the template created HDU
if(! *status_p) fits_movnam_hdu(etf->fptr, BINARY_TBL, (char *)"AOSCRAM", 0, status_p);
} else {
// create new HDU
if(! *status_p) fits_create_tbl(etf->fptr, BINARY_TBL, 0, 0, NULL, NULL, NULL, (char *)"AOSCRAM", status_p);
}
if(! *status_p) fits_update_key(etf->fptr, TSTRING, "EXTNAME", (char *)"AOSCRAM", NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "COARCHID", &scram->coarse_chan_id, NULL, status_p);
// observatory (scram) data
if(! *status_p) fits_update_key(etf->fptr, TINT, "PNTSTIME", &(scram->PNTSTIME), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "PNTRA", &(scram->PNTRA), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "PNTDEC", &(scram->PNTDEC), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "PNTMJD", &(scram->PNTMJD), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "PNTAZCOR", &(scram->PNTAZCOR), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "PNTZACOR", &(scram->PNTZACOR), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "AGCSTIME", &(scram->AGCSTIME), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "AGCAZ", &(scram->AGCAZ), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "AGCZA", &(scram->AGCZA), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "AGCTIME", &(scram->AGCTIME), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "AGCLST", &(scram->AGCLST), NULL, status_p); // TODO
if(! *status_p) fits_update_key(etf->fptr, TINT, "ALFSTIME", &(scram->ALFSTIME), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "ALFBIAS1", &(scram->ALFBIAS1), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "ALFBIAS2", &(scram->ALFBIAS2), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ALFMOPOS", &(scram->ALFMOPOS), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "IF1STIME", &(scram->IF1STIME), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "IF1SYNHZ", &(scram->IF1SYNHZ), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "IF1SYNDB", &(scram->IF1SYNDB), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "IF1RFFRQ", &(scram->IF1RFFRQ), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "IF1IFFRQ", &(scram->IF1IFFRQ), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "IF1ALFFB", &(scram->IF1ALFFB), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "IF2STIME", &(scram->IF2STIME), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "IF2ALFON", &(scram->IF2ALFON), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "IF2SYNHZ", &(scram->IF2SYNHZ), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "IF2SIGSR", &(scram->IF2SIGSR), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "TTSTIME", &(scram->TTSTIME), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "TTTURENC", &(scram->TTTURENC), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "TTTURDEG", &(scram->TTTURDEG), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "CLOCKTIM", &(scram->CLOCKTIM), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "CLOCKFRQ", &(scram->CLOCKFRQ), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "CLOCKDBM", &(scram->CLOCKDBM), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "CLOCKLOC", &(scram->CLOCKLOC), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "BIRDITIM", &(scram->BIRDITIM), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "BIRDIFRQ", &(scram->BIRDIFRQ), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "BIRDIDBM", &(scram->BIRDIDBM), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "BIRDILOC", &(scram->BIRDILOC), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TINT, "ADCRMSTM", &(scram->ADCRMSTM), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS01", &(scram->ADC1RMS[0]), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS02", &(scram->ADC1RMS[1]), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS03", &(scram->ADC1RMS[2]), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS04", &(scram->ADC1RMS[3]), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS05", &(scram->ADC1RMS[4]), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS06", &(scram->ADC1RMS[5]), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS07", &(scram->ADC1RMS[6]), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS08", &(scram->ADC1RMS[7]), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS09", &(scram->ADC2RMS[0]), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS10", &(scram->ADC2RMS[1]), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS11", &(scram->ADC2RMS[2]), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS12", &(scram->ADC2RMS[3]), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS13", &(scram->ADC2RMS[4]), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS14", &(scram->ADC2RMS[5]), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS15", &(scram->ADC2RMS[6]), NULL, status_p);
if(! *status_p) fits_update_key(etf->fptr, TDOUBLE, "ADCRMS16", &(scram->ADC2RMS[7]), NULL, status_p);
#if 0
fits_update_key(etf->fptr, TINT, "MISSEDPK", &(scram.MISSEDPK), NULL, status_p); // missed packets per input per second
#endif
//if(! *status_p) fits_flush_file(etf->fptr, status_p);
if (*status_p) {
hashpipe_error(__FUNCTION__, "Error writing integration header");
//fprintf(stderr, "Error writing integration header.\n");
fits_report_error(stderr, *status_p);
}
return *status_p;
}
static void *run(hashpipe_thread_args_t * args, int doCPU)
{
// Local aliases to shorten access to args fields
paper_gpu_input_databuf_t *db_in = (paper_gpu_input_databuf_t *)args->ibuf;
paper_output_databuf_t *db_out = (paper_output_databuf_t *)args->obuf;
hashpipe_status_t st = args->st;
const char * status_key = args->thread_desc->skey;
#ifdef DEBUG_SEMS
fprintf(stderr, "s/tid %lu/ GPU/\n", pthread_self());
#endif
// Init integration control status variables
int gpu_dev = 0;
hashpipe_status_lock_safe(&st);
hputs(st.buf, "INTSTAT", "off");
hputi8(st.buf, "INTSYNC", 0);
hputi4(st.buf, "INTCOUNT", N_SUB_BLOCKS_PER_INPUT_BLOCK);
hputi8(st.buf, "GPUDUMPS", 0);
hgeti4(st.buf, "GPUDEV", &gpu_dev); // No change if not found
hputi4(st.buf, "GPUDEV", gpu_dev);
hashpipe_status_unlock_safe(&st);
/* Loop */
int rv;
char integ_status[17];
uint64_t start_mcount, last_mcount=0;
uint64_t gpu_dumps=0;
int int_count; // Number of blocks to integrate per dump
int xgpu_error = 0;
int curblock_in=0;
int curblock_out=0;
struct timespec start, stop;
uint64_t elapsed_gpu_ns = 0;
uint64_t gpu_block_count = 0;
// Initialize context to point at first input and output memory blocks.
// This seems redundant since we do this just before calling
// xgpuCudaXengine, but we need to pass something in for array_h and
// matrix_x to prevent xgpuInit from allocating memory.
XGPUContext context;
context.array_h = (ComplexInput *)db_in->block[0].data;
context.array_len = (db_in->header.n_block * sizeof(paper_gpu_input_block_t) - sizeof(paper_input_header_t)) / sizeof(ComplexInput);
context.matrix_h = (Complex *)db_out->block[0].data;
context.matrix_len = (db_out->header.n_block * sizeof(paper_output_block_t) - sizeof(paper_output_header_t)) / sizeof(Complex);
xgpu_error = xgpuInit(&context, gpu_dev);
if (XGPU_OK != xgpu_error) {
fprintf(stderr, "ERROR: xGPU initialization failed (error code %d)\n", xgpu_error);
return THREAD_ERROR;
}
while (run_threads()) {
// Note waiting status,
// query integrating status
// and, if armed, start count
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "waiting");
hgets(st.buf, "INTSTAT", 16, integ_status);
hgeti8(st.buf, "INTSYNC", (long long*)&start_mcount);
hashpipe_status_unlock_safe(&st);
// Wait for new input block to be filled
while ((rv=hashpipe_databuf_wait_filled((hashpipe_databuf_t *)db_in, curblock_in)) != HASHPIPE_OK) {
if (rv==HASHPIPE_TIMEOUT) {
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "blocked_in");
hashpipe_status_unlock_safe(&st);
continue;
} else {
hashpipe_error(__FUNCTION__, "error waiting for filled databuf");
pthread_exit(NULL);
break;
}
}
// Got a new data block, update status and determine how to handle it
hashpipe_status_lock_safe(&st);
hputi4(st.buf, "GPUBLKIN", curblock_in);
hputu8(st.buf, "GPUMCNT", db_in->block[curblock_in].header.mcnt);
hashpipe_status_unlock_safe(&st);
// If integration status "off"
if(!strcmp(integ_status, "off")) {
// Mark input block as free and advance
hashpipe_databuf_set_free((hashpipe_databuf_t *)db_in, curblock_in);
curblock_in = (curblock_in + 1) % db_in->header.n_block;
// Skip to next input buffer
continue;
}
// If integration status is "start"
if(!strcmp(integ_status, "start")) {
// If buffer mcount < start_mcount (i.e. not there yet)
if(db_in->block[curblock_in].header.mcnt < start_mcount) {
// Drop input buffer
// Mark input block as free and advance
hashpipe_databuf_set_free((hashpipe_databuf_t *)db_in, curblock_in);
curblock_in = (curblock_in + 1) % db_in->header.n_block;
// Skip to next input buffer
continue;
// Else if mcount == start_mcount (time to start)
} else if(db_in->block[curblock_in].header.mcnt == start_mcount) {
// Set integration status to "on"
// Read integration count (INTCOUNT)
fprintf(stderr, "--- integration on ---\n");
strcpy(integ_status, "on");
hashpipe_status_lock_safe(&st);
hputs(st.buf, "INTSTAT", integ_status);
hgeti4(st.buf, "INTCOUNT", &int_count);
hashpipe_status_unlock_safe(&st);
// Compute last mcount
last_mcount = start_mcount + (int_count-1) * N_SUB_BLOCKS_PER_INPUT_BLOCK;
// Else (missed starting mcount)
} else {
// Handle missed start of integration
// TODO!
fprintf(stderr, "--- mcnt=%06lx > start_mcnt=%06lx ---\n",
db_in->block[curblock_in].header.mcnt, start_mcount);
}
}
// Integration status is "on" or "stop"
// Note processing status
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "processing gpu");
hashpipe_status_unlock_safe(&st);
// Setup for current chunk
context.input_offset = curblock_in * sizeof(paper_gpu_input_block_t) / sizeof(ComplexInput);
context.output_offset = curblock_out * sizeof(paper_output_block_t) / sizeof(Complex);
// Call CUDA X engine function
int doDump = 0;
// Dump if this is the last block or we are doing both CPU and GPU
// (GPU and CPU test mode always dumps every input block)
if(db_in->block[curblock_in].header.mcnt >= last_mcount || doCPU) {
doDump = 1;
// Check whether we missed the end of integration. If we get a block
// whose mcnt is greater than last_mcount, then for some reason (e.g.
// networking problems) we didn't see a block whose mcnt was
// last_mcount. This should "never" happen, but it has been seen to
// occur when the 10 GbE links have many errors.
if(db_in->block[curblock_in].header.mcnt > last_mcount) {
// Can't do much error recovery, so just log it.
fprintf(stderr, "--- mcnt=%06lx > last_mcnt=%06lx ---\n",
db_in->block[curblock_in].header.mcnt, last_mcount);
}
// Wait for new output block to be free
while ((rv=paper_output_databuf_wait_free(db_out, curblock_out)) != HASHPIPE_OK) {
if (rv==HASHPIPE_TIMEOUT) {
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "blocked gpu out");
hashpipe_status_unlock_safe(&st);
continue;
} else {
hashpipe_error(__FUNCTION__, "error waiting for free databuf");
pthread_exit(NULL);
break;
}
}
}
clock_gettime(CLOCK_MONOTONIC, &start);
xgpuCudaXengine(&context, doDump ? SYNCOP_DUMP : SYNCOP_SYNC_TRANSFER);
clock_gettime(CLOCK_MONOTONIC, &stop);
elapsed_gpu_ns += ELAPSED_NS(start, stop);
gpu_block_count++;
if(doDump) {
clock_gettime(CLOCK_MONOTONIC, &start);
xgpuClearDeviceIntegrationBuffer(&context);
clock_gettime(CLOCK_MONOTONIC, &stop);
elapsed_gpu_ns += ELAPSED_NS(start, stop);
// TODO Maybe need to subtract all or half the integration time here
// depending on recevier's expectations.
db_out->block[curblock_out].header.mcnt = last_mcount;
// If integration status if "stop"
if(!strcmp(integ_status, "stop")) {
// Set integration status to "off"
strcpy(integ_status, "off");
hashpipe_status_lock_safe(&st);
hputs(st.buf, "INTSTAT", integ_status);
hashpipe_status_unlock_safe(&st);
} else {
// Advance last_mcount for end of next integration
last_mcount += int_count * N_SUB_BLOCKS_PER_INPUT_BLOCK;
}
// Mark output block as full and advance
paper_output_databuf_set_filled(db_out, curblock_out);
curblock_out = (curblock_out + 1) % db_out->header.n_block;
// TODO Need to handle or at least check for overflow!
// Update GPU dump counter and GPU Gbps
gpu_dumps++;
hashpipe_status_lock_safe(&st);
hputi8(st.buf, "GPUDUMPS", gpu_dumps);
hputr4(st.buf, "GPUGBPS", (float)(8*N_FLUFFED_BYTES_PER_BLOCK*gpu_block_count)/elapsed_gpu_ns);
hashpipe_status_unlock_safe(&st);
// Start new average
elapsed_gpu_ns = 0;
gpu_block_count = 0;
}
if(doCPU) {
/* Note waiting status */
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "waiting");
hashpipe_status_unlock_safe(&st);
// Wait for new output block to be free
while ((rv=paper_output_databuf_wait_free(db_out, curblock_out)) != HASHPIPE_OK) {
if (rv==HASHPIPE_TIMEOUT) {
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "blocked cpu out");
hashpipe_status_unlock_safe(&st);
continue;
} else {
hashpipe_error(__FUNCTION__, "error waiting for free databuf");
pthread_exit(NULL);
break;
}
}
// Note "processing cpu" status, current input block
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "processing cpu");
hashpipe_status_unlock_safe(&st);
/*
* Call CPU X engine function
*/
xgpuOmpXengine((Complex *)db_out->block[curblock_out].data, context.array_h);
// Mark output block as full and advance
paper_output_databuf_set_filled(db_out, curblock_out);
curblock_out = (curblock_out + 1) % db_out->header.n_block;
// TODO Need to handle or at least check for overflow!
}
// Mark input block as free and advance
hashpipe_databuf_set_free((hashpipe_databuf_t *)db_in, curblock_in);
curblock_in = (curblock_in + 1) % db_in->header.n_block;
/* Check for cancel */
pthread_testcancel();
}
xgpuFree(&context);
// Thread success!
return NULL;
}
static void *run(hashpipe_thread_args_t * args)
{
// Local aliases to shorten access to args fields
// Our input buffer is a paper_input_databuf
// Our output buffer is a paper_gpu_input_databuf
paper_input_databuf_t *db_in = (paper_input_databuf_t *)args->ibuf;
paper_gpu_input_databuf_t *db_out = (paper_gpu_input_databuf_t *)args->obuf;
hashpipe_status_t st = args->st;
const char * status_key = args->thread_desc->skey;
#ifdef DEBUG_SEMS
fprintf(stderr, "s/tid %lu/ FLUFf/\n", pthread_self());
#endif
// Init status variables
hashpipe_status_lock_safe(&st);
hputi8(st.buf, "FLUFMCNT", 0);
hashpipe_status_unlock_safe(&st);
/* Loop */
int rv;
int curblock_in=0;
int curblock_out=0;
float gbps, min_gbps;
struct timespec start, finish;
while (run_threads()) {
// Note waiting status,
// query integrating status
// and, if armed, start count
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "waiting");
hashpipe_status_unlock_safe(&st);
// Wait for new input block to be filled
while ((rv=paper_input_databuf_wait_filled(db_in, curblock_in)) != HASHPIPE_OK) {
if (rv==HASHPIPE_TIMEOUT) {
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "blocked_in");
hashpipe_status_unlock_safe(&st);
continue;
} else {
hashpipe_error(__FUNCTION__, "error waiting for filled databuf");
pthread_exit(NULL);
break;
}
}
// Wait for new gpu_input block (our output block) to be free
while ((rv=paper_gpu_input_databuf_wait_free(db_out, curblock_out)) != HASHPIPE_OK) {
if (rv==HASHPIPE_TIMEOUT) {
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "blocked gpu input");
hashpipe_status_unlock_safe(&st);
continue;
} else {
hashpipe_error(__FUNCTION__, "error waiting for free databuf");
pthread_exit(NULL);
break;
}
}
// Got a new data block, update status
hashpipe_status_lock_safe(&st);
hputs(st.buf, status_key, "fluffing");
hputi4(st.buf, "FLUFBKIN", curblock_in);
hputu8(st.buf, "FLUFMCNT", db_in->block[curblock_in].header.mcnt);
hashpipe_status_unlock_safe(&st);
// Copy header and call fluff function
clock_gettime(CLOCK_MONOTONIC, &start);
memcpy(&db_out->block[curblock_out].header, &db_in->block[curblock_in].header, sizeof(paper_input_header_t));
paper_fluff(db_in->block[curblock_in].data, db_out->block[curblock_out].data);
clock_gettime(CLOCK_MONOTONIC, &finish);
// Note processing time
hashpipe_status_lock_safe(&st);
// Bits per fluff / ns per fluff = Gbps
hgetr4(st.buf, "FLUFMING", &min_gbps);
gbps = (float)(8*N_BYTES_PER_BLOCK)/ELAPSED_NS(start,finish);
hputr4(st.buf, "FLUFGBPS", gbps);
if(min_gbps == 0 || gbps < min_gbps) {
hputr4(st.buf, "FLUFMING", gbps);
}
hashpipe_status_unlock_safe(&st);
// Mark input block as free and advance
paper_input_databuf_set_free(db_in, curblock_in);
curblock_in = (curblock_in + 1) % db_in->header.n_block;
// Mark output block as full and advance
paper_gpu_input_databuf_set_filled(db_out, curblock_out);
curblock_out = (curblock_out + 1) % db_out->header.n_block;
/* Check for cancel */
pthread_testcancel();
}
// Thread success!
return NULL;
}
//----------------------------------------------------------
int etfits_create(etfits_t * etf) {
//----------------------------------------------------------
int * status_p = &(etf->status);
*status_p = 0;
struct tm tm_now;
time_t time_now;
char file_name_str[256];
// TODO enclose all init code in a do-as-needed block
// Initialize the key variables if needed
if (etf->new_run == 1) { // first time writing this run
etf->new_run = 0;
etf->status = 0;
etf->N = 0L;
etf->T = 0.0;
etf->mode = 'w';
// Create the output directory if needed
char datadir[1024];
strncpy(datadir, etf->basefilename, 1023);
char *last_slash = strrchr(datadir, '/');
if (last_slash!=NULL && last_slash!=datadir) {
*last_slash = '\0';
printf("Using directory '%s' for output.\n", datadir);
char cmd[1024];
sprintf(cmd, "mkdir -m 1777 -p %s", datadir);
system(cmd);
}
} // end first time writing this run
etf->tot_rows = 0; // count rows per file
// Form file name
time(&time_now);
localtime_r(&time_now, &tm_now);
sprintf(file_name_str, "%s_%04d_%04d%02d%02d_%02d%02d%02d",
etf->primary_hdr.receiver,
etf->file_chan,
1900+tm_now.tm_year, 1+tm_now.tm_mon, tm_now.tm_mday,
tm_now.tm_hour, tm_now.tm_min, tm_now.tm_sec);
sprintf(etf->filename_working, "%s_%s_%s.working", etf->basefilename, etf->hostname, file_name_str);
sprintf(etf->filename_fits, "%s_%s_%s.fits", etf->basefilename, etf->hostname, file_name_str);
// Create basic FITS file from our template
char template_file[1024];
//printf("Opening file '%s'\n", etf->filename);
sprintf(template_file, "%s/%s", etf->s6_dir, ETFITS_TEMPLATE);
if(! *status_p) fits_create_template(&(etf->fptr), etf->filename_working, template_file, status_p);
// Check to see if file was successfully created
if (*status_p) {
hashpipe_error(__FUNCTION__, "Error creating sdfits file from template");
//fprintf(stderr, "Error creating sdfits file from template.\n");
fits_report_error(stderr, *status_p);
} else {
etf->file_open = 1; // successful file open
}
return *status_p;
}
