VOID handle_connection(connection_data *data) { char *cmd = 0; int res; connection_context _c, *c = &_c; cmd = malloc(MAX_COMMAND_LENGTH); if (!cmd) { hprintf(data->pipe, "error: unable to allocate buffer for command\n"); return; } ZeroMemory(cmd, MAX_COMMAND_LENGTH); ZeroMemory(c, sizeof(connection_context)); c->pipe = data->pipe; c->cmd = cmd; c->conn_number = data->conn_number; free(data); /* FIXME make wait for end of process or ctrl_pipe input */ while (1) { res = hgets(cmd, MAX_COMMAND_LENGTH, c->pipe); if (res <= 0) { SvcDebugOut("Error reading from pipe(%08X)\n", (int) c->pipe->h); goto finish; } SvcDebugOut("Retrieved line: \"%s\"\n", (int)cmd); CMD_ITEM *ci; for (ci = cmd_table; ci->name; ++ci) { if (strstr(cmd, ci->name) != cmd) continue; char c = cmd[strlen(ci->name)]; if (!c || (c == ' ')) break; } if (ci->name) { if (!ci->func(c)) goto finish; } else hprintf(c->pipe, "error Ignoring unknown command (%s)\n", cmd); } finish: FlushFileBuffers(c->pipe->h); DisconnectNamedPipe(c->pipe->h); CloseHandle(c->pipe->h); CloseHandle(c->pipe->o.hEvent); free(c->pipe); free(cmd); }
VALUE rb_hps_hgets(VALUE self, VALUE vkey) { int rc; char val[HASHPIPE_STATUS_RECORD_SIZE]; hashpipe_status_t *s; char save = '\0'; char * key = StringValueCStr(vkey); if(strlen(key) > 8) { rb_warning("key '%s' truncated to 8 characters", key); save = key[8]; key[8] = '\0'; } Data_Get_HPStruct_Ensure_Attached(self, s); rc = hgets(s->buf, key, HASHPIPE_STATUS_RECORD_SIZE, val); val[HASHPIPE_STATUS_RECORD_SIZE-1] = '\0'; if(save) key[8] = save; return rc ? rb_str_new_cstr(val) : Qnil; }
/* The main CPU accumulator thread */ void guppi_accum_thread(void *_args) { float **accumulator; //indexed accumulator[accum_id][chan][subband][stokes] char accum_dirty[NUM_SW_STATES]; struct sdfits_data_columns data_cols[NUM_SW_STATES]; int payload_type; int i, j, k, rv; /* Get arguments */ struct guppi_thread_args *args = (struct guppi_thread_args *)_args; /* Set cpu affinity */ cpu_set_t cpuset, cpuset_orig; sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig); //CPU_ZERO(&cpuset); CPU_CLR(13, &cpuset); CPU_SET(9, &cpuset); rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); if (rv<0) { guppi_error("guppi_accum_thread", "Error setting cpu affinity."); perror("sched_setaffinity"); } /* Set priority */ rv = setpriority(PRIO_PROCESS, 0, args->priority); if (rv<0) { guppi_error("guppi_accum_thread", "Error setting priority level."); perror("set_priority"); } /* Attach to status shared mem area */ struct guppi_status st; rv = guppi_status_attach(&st); if (rv!=GUPPI_OK) { guppi_error("guppi_accum_thread", "Error attaching to status shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_status_detach, &st); pthread_cleanup_push((void *)set_exit_status, &st); pthread_cleanup_push((void *)guppi_thread_set_finished, args); /* Init status */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "init"); guppi_status_unlock_safe(&st); /* Read in general parameters */ struct guppi_params gp; struct sdfits sf; pthread_cleanup_push((void *)guppi_free_sdfits, &sf); /* Attach to databuf shared mem */ struct guppi_databuf *db_in, *db_out; db_in = guppi_databuf_attach(args->input_buffer); char errmsg[256]; if (db_in==NULL) { sprintf(errmsg, "Error attaching to input databuf(%d) shared memory.", args->input_buffer); guppi_error("guppi_accum_thread", errmsg); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_databuf_detach, db_in); db_out = guppi_databuf_attach(args->output_buffer); if (db_out==NULL) { sprintf(errmsg, "Error attaching to output databuf(%d) shared memory.", args->output_buffer); guppi_error("guppi_accum_thread", errmsg); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_databuf_detach, db_out); /* Determine high/low bandwidth mode */ char bw_mode[16]; if (hgets(st.buf, "BW_MODE", 16, bw_mode)) { if(strncmp(bw_mode, "high", 4) == 0) payload_type = INT_PAYLOAD; else if(strncmp(bw_mode, "low", 3) == 0) payload_type = FLOAT_PAYLOAD; else guppi_error("guppi_net_thread", "Unsupported bandwidth mode"); } else guppi_error("guppi_net_thread", "BW_MODE not set"); /* Read nchan and nsubband from status shared memory */ guppi_read_obs_params(st.buf, &gp, &sf); /* Allocate memory for vector accumulators */ create_accumulators(&accumulator, sf.hdr.nchan, sf.hdr.nsubband); pthread_cleanup_push((void *)destroy_accumulators, accumulator); /* Clear the vector accumulators */ for(i = 0; i < NUM_SW_STATES; i++) accum_dirty[i] = 1; reset_accumulators(accumulator, data_cols, accum_dirty, sf.hdr.nsubband, sf.hdr.nchan); /* Loop */ int curblock_in=0, curblock_out=0; int first=1; float reqd_exposure=0; double accum_time=0; int integ_num; float pfb_rate; int heap, accumid, struct_offset, array_offset; char *hdr_in=NULL, *hdr_out=NULL; struct databuf_index *index_in, *index_out; int nblock_int=0, npacket=0, n_pkt_drop=0, n_heap_drop=0; signal(SIGINT,cc); while (run) { /* Note waiting status */ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "waiting"); guppi_status_unlock_safe(&st); /* Wait for buf to have data */ rv = guppi_databuf_wait_filled(db_in, curblock_in); if (rv!=0) continue; /* Note waiting status and current block*/ guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "accumulating"); hputi4(st.buf, "ACCBLKIN", curblock_in); guppi_status_unlock_safe(&st); /* Read param struct for this block */ hdr_in = guppi_databuf_header(db_in, curblock_in); if (first) guppi_read_obs_params(hdr_in, &gp, &sf); else guppi_read_subint_params(hdr_in, &gp, &sf); /* Do any first time stuff: first time code runs, not first time process this block */ if (first) { /* Set up first output header. This header is copied from block to block each time a new block is created */ hdr_out = guppi_databuf_header(db_out, curblock_out); memcpy(hdr_out, guppi_databuf_header(db_in, curblock_in), GUPPI_STATUS_SIZE); /* Read required exposure and PFB rate from status shared memory */ reqd_exposure = sf.data_columns.exposure; pfb_rate = sf.hdr.efsampfr / (2 * sf.hdr.nchan); /* Initialise the index in the output block */ index_out = (struct databuf_index*)guppi_databuf_index(db_out, curblock_out); index_out->num_datasets = 0; index_out->array_size = sf.hdr.nsubband * sf.hdr.nchan * NUM_STOKES * 4; first=0; } /* Loop through each spectrum (heap) in input buffer */ index_in = (struct databuf_index*)guppi_databuf_index(db_in, curblock_in); for(heap = 0; heap < index_in->num_heaps; heap++) { /* If invalid, record it and move to next heap */ if(!index_in->cpu_gpu_buf[heap].heap_valid) { n_heap_drop++; continue; } /* Read in heap from buffer */ char* heap_addr = (char*)(guppi_databuf_data(db_in, curblock_in) + sizeof(struct freq_spead_heap) * heap); struct freq_spead_heap* freq_heap = (struct freq_spead_heap*)(heap_addr); char* payload_addr = (char*)(guppi_databuf_data(db_in, curblock_in) + sizeof(struct freq_spead_heap) * MAX_HEAPS_PER_BLK + (index_in->heap_size - sizeof(struct freq_spead_heap)) * heap ); int *i_payload = (int*)(payload_addr); float *f_payload = (float*)(payload_addr); accumid = freq_heap->status_bits & 0x7; /*Debug: print heap */ /* printf("%d, %d, %d, %d, %d, %d\n", freq_heap->time_cntr, freq_heap->spectrum_cntr, freq_heap->integ_size, freq_heap->mode, freq_heap->status_bits, freq_heap->payload_data_off); */ /* If we have accumulated for long enough, write vectors to output block */ if(accum_time >= reqd_exposure) { for(i = 0; i < NUM_SW_STATES; i++) { /*If a particular accumulator is dirty, write it to output buffer */ if(accum_dirty[i]) { /*If insufficient space, first mark block as filled and request new block*/ index_out = (struct databuf_index*)(guppi_databuf_index(db_out, curblock_out)); if( (index_out->num_datasets+1) * (index_out->array_size + sizeof(struct sdfits_data_columns)) > db_out->block_size) { printf("Accumulator finished with output block %d\n", curblock_out); /* Write block number to status buffer */ guppi_status_lock_safe(&st); hputi4(st.buf, "ACCBLKOU", curblock_out); guppi_status_unlock_safe(&st); /* Update packet count and loss fields in output header */ hputi4(hdr_out, "NBLOCK", nblock_int); hputi4(hdr_out, "NPKT", npacket); hputi4(hdr_out, "NPKTDROP", n_pkt_drop); hputi4(hdr_out, "NHPDROP", n_heap_drop); /* Close out current integration */ guppi_databuf_set_filled(db_out, curblock_out); /* Wait for next output buf */ curblock_out = (curblock_out + 1) % db_out->n_block; guppi_databuf_wait_free(db_out, curblock_out); while ((rv=guppi_databuf_wait_free(db_out, curblock_out)) != GUPPI_OK) { if (rv==GUPPI_TIMEOUT) { guppi_warn("guppi_accum_thread", "timeout while waiting for output block"); continue; } else { guppi_error("guppi_accum_thread", "error waiting for free databuf"); run=0; pthread_exit(NULL); break; } } hdr_out = guppi_databuf_header(db_out, curblock_out); memcpy(hdr_out, guppi_databuf_header(db_in, curblock_in), GUPPI_STATUS_SIZE); /* Initialise the index in new output block */ index_out = (struct databuf_index*)guppi_databuf_index(db_out, curblock_out); index_out->num_datasets = 0; index_out->array_size = sf.hdr.nsubband * sf.hdr.nchan * NUM_STOKES * 4; nblock_int=0; npacket=0; n_pkt_drop=0; n_heap_drop=0; } /*Update index for output buffer*/ index_out = (struct databuf_index*)(guppi_databuf_index(db_out, curblock_out)); if(index_out->num_datasets == 0) struct_offset = 0; else struct_offset = index_out->disk_buf[index_out->num_datasets-1].array_offset + index_out->array_size; array_offset = struct_offset + sizeof(struct sdfits_data_columns); index_out->disk_buf[index_out->num_datasets].struct_offset = struct_offset; index_out->disk_buf[index_out->num_datasets].array_offset = array_offset; /*Copy sdfits_data_columns struct to disk buffer */ memcpy(guppi_databuf_data(db_out, curblock_out) + struct_offset, &data_cols[i], sizeof(struct sdfits_data_columns)); /*Copy data array to disk buffer */ memcpy(guppi_databuf_data(db_out, curblock_out) + array_offset, accumulator[i], index_out->array_size); /*Update SDFITS data_columns pointer to data array */ ((struct sdfits_data_columns*) (guppi_databuf_data(db_out, curblock_out) + struct_offset))->data = (unsigned char*)(guppi_databuf_data(db_out, curblock_out) + array_offset); index_out->num_datasets = index_out->num_datasets + 1; } } accum_time = 0; integ_num += 1; reset_accumulators(accumulator, data_cols, accum_dirty, sf.hdr.nsubband, sf.hdr.nchan); } /* Only add spectrum to accumulator if blanking bit is low */ if((freq_heap->status_bits & 0x08) == 0) { /* Fill in data columns header fields */ if(!accum_dirty[accumid]) { /*Record SPEAD header fields*/ data_cols[accumid].time = index_in->cpu_gpu_buf[heap].heap_rcvd_mjd; data_cols[accumid].time_counter = freq_heap->time_cntr; data_cols[accumid].integ_num = integ_num; data_cols[accumid].sttspec = freq_heap->spectrum_cntr; data_cols[accumid].accumid = accumid; /* Fill in rest of fields from status buffer */ strcpy(data_cols[accumid].object, sf.data_columns.object); data_cols[accumid].azimuth = sf.data_columns.azimuth; data_cols[accumid].elevation = sf.data_columns.elevation; data_cols[accumid].bmaj = sf.data_columns.bmaj; data_cols[accumid].bmin = sf.data_columns.bmin; data_cols[accumid].bpa = sf.data_columns.bpa; data_cols[accumid].centre_freq_idx = sf.data_columns.centre_freq_idx; data_cols[accumid].ra = sf.data_columns.ra; data_cols[accumid].dec = sf.data_columns.dec; data_cols[accumid].exposure = 0.0; for(i = 0; i < NUM_SW_STATES; i++) data_cols[accumid].centre_freq[i] = sf.data_columns.centre_freq[i]; accum_dirty[accumid] = 1; } data_cols[accumid].exposure += (float)(freq_heap->integ_size)/pfb_rate; data_cols[accumid].stpspec = freq_heap->spectrum_cntr; /* Add spectrum to appropriate vector accumulator (high-bw mode) */ if(payload_type == INT_PAYLOAD) { for(i = 0; i < sf.hdr.nchan; i++) { for(j = 0; j < sf.hdr.nsubband; j++) { for(k = 0; k < NUM_STOKES; k++) { accumulator[accumid] [i*sf.hdr.nsubband*NUM_STOKES + j*NUM_STOKES + k] += (float)i_payload[i*sf.hdr.nsubband*NUM_STOKES + j*NUM_STOKES + k]; } } } } /* Add spectrum to appropriate vector accumulator (low-bw mode) */ else { for(i = 0; i < sf.hdr.nchan; i++) { for(j = 0; j < sf.hdr.nsubband; j++) { for(k = 0; k < NUM_STOKES; k++) { accumulator[accumid] [i*sf.hdr.nsubband*NUM_STOKES + j*NUM_STOKES + k] += f_payload[i*sf.hdr.nsubband*NUM_STOKES + j*NUM_STOKES + k]; } } } } } accum_time += (double)freq_heap->integ_size / pfb_rate; } /* Update packet count and loss fields from input header */ nblock_int++; npacket += gp.num_pkts_rcvd; n_pkt_drop += gp.num_pkts_dropped; /* Done with current input block */ guppi_databuf_set_free(db_in, curblock_in); curblock_in = (curblock_in + 1) % db_in->n_block; /* Check for cancel */ pthread_testcancel(); } pthread_exit(NULL); pthread_cleanup_pop(0); /* Closes set_exit_status */ pthread_cleanup_pop(0); /* Closes set_finished */ pthread_cleanup_pop(0); /* Closes guppi_free_sdfits */ pthread_cleanup_pop(0); /* Closes ? */ pthread_cleanup_pop(0); /* Closes destroy_accumulators */ pthread_cleanup_pop(0); /* Closes guppi_status_detach */ pthread_cleanup_pop(0); /* Closes guppi_databuf_detach */ }
int main(int argc, char *argv[]) { int instance_id = 0; hashpipe_status_t *s; /* Loop over cmd line to fill in params */ static struct option long_opts[] = { {"help", 0, NULL, 'h'}, {"shmkey", 1, NULL, 'K'}, {"key", 1, NULL, 'k'}, {"get", 1, NULL, 'g'}, {"string", 1, NULL, 's'}, {"float", 1, NULL, 'f'}, {"double", 1, NULL, 'd'}, {"int", 1, NULL, 'i'}, {"verbose", 0, NULL, 'v'}, {"clear", 0, NULL, 'C'}, {"del", 0, NULL, 'D'}, {"query", 1, NULL, 'Q'}, {"instance", 1, NULL, 'I'}, {0,0,0,0} }; int opt,opti; char *key=NULL; char value[81]; float flttmp; double dbltmp; int inttmp; int verbose=0, clear=0; char keyfile[1000]; while ((opt=getopt_long(argc,argv,"hk:g:s:f:d:i:vCDQ:K:I:",long_opts,&opti))!=-1) { switch (opt) { case 'K': // Keyfile snprintf(keyfile, sizeof(keyfile), "HASHPIPE_KEYFILE=%s", optarg); keyfile[sizeof(keyfile)-1] = '\0'; putenv(keyfile); break; case 'I': instance_id = atoi(optarg); break; case 'k': key = optarg; break; case 'Q': s = get_status_buffer(instance_id); hashpipe_status_lock(s); hgets(s->buf, optarg, 80, value); hashpipe_status_unlock(s); value[80] = '\0'; printf("%s\n", value); break; case 'g': s = get_status_buffer(instance_id); hashpipe_status_lock(s); hgetr8(s->buf, optarg, &dbltmp); hashpipe_status_unlock(s); printf("%g\n", dbltmp); break; case 's': if (key) { s = get_status_buffer(instance_id); hashpipe_status_lock(s); hputs(s->buf, key, optarg); hashpipe_status_unlock(s); } break; case 'f': flttmp = atof(optarg); if (key) { s = get_status_buffer(instance_id); hashpipe_status_lock(s); hputr4(s->buf, key, flttmp); hashpipe_status_unlock(s); } break; case 'd': dbltmp = atof(optarg); if (key) { s = get_status_buffer(instance_id); hashpipe_status_lock(s); hputr8(s->buf, key, dbltmp); hashpipe_status_unlock(s); } break; case 'i': inttmp = atoi(optarg); if (key) { s = get_status_buffer(instance_id); hashpipe_status_lock(s); hputi4(s->buf, key, inttmp); hashpipe_status_unlock(s); } break; case 'D': if (key) { s = get_status_buffer(instance_id); hashpipe_status_lock(s); hdel(s->buf, key); hashpipe_status_unlock(s); } break; case 'C': clear=1; break; case 'v': verbose=1; break; case 'h': usage(); return 0; case '?': // Command line parsing error default: usage(); exit(1); break; } } s = get_status_buffer(instance_id); /* If verbose, print out buffer */ if (verbose) { hashpipe_status_lock(s); printf("%s\n", s->buf); hashpipe_status_unlock(s); } if (clear) hashpipe_status_clear(s); exit(0); }
static void *http_request(void* arg) { WEBBLK *webblk; int authok = !http_serv.httpauth; char line[HTTP_PATH_LENGTH]; char *url = NULL; char *pointer; char *strtok_str = NULL; CGITAB *cgient; int content_length = 0; int sock = (int) (uintptr_t) arg; if(!(webblk = malloc(sizeof(WEBBLK)))) http_exit(webblk); memset(webblk,0,sizeof(WEBBLK)); webblk->sock = sock; while (hgets(line, sizeof(line), webblk->sock)) { if (*line == '\r' || *line == '\n') break; if((pointer = strtok_r(line," \t\r\n",&strtok_str))) { if(!strcasecmp(pointer,"GET")) { if((pointer = strtok_r(NULL," \t\r\n",&strtok_str))) { webblk->request_type = REQTYPE_GET; url = strdup(pointer); } } else if(!strcasecmp(pointer,"POST")) { if((pointer = strtok_r(NULL," \t\r\n",&strtok_str))) { webblk->request_type = REQTYPE_POST; url = strdup(pointer); } } else if(!strcasecmp(pointer,"PUT")) { http_error(webblk,"400 Bad Request", "", "This server does not accept PUT requests"); } else if(!strcasecmp(pointer,"Authorization:")) { if((pointer = strtok_r(NULL," \t\r\n",&strtok_str))) authok = http_authenticate(webblk,pointer, strtok_r(NULL," \t\r\n",&strtok_str)); } else if(!strcasecmp(pointer,"Cookie:")) { if((pointer = strtok_r(NULL,"\r\n",&strtok_str))) http_interpret_variable_string(webblk, pointer, VARTYPE_COOKIE); } else if(!strcasecmp(pointer,"Content-Length:")) { if((pointer = strtok_r(NULL," \t\r\n",&strtok_str))) content_length = atoi(pointer); } } } webblk->request = url; if(webblk->request_type == REQTYPE_POST && content_length != 0) { char *post_arg; if((pointer = post_arg = malloc(content_length + 1))) { int i; for(i = 0; i < content_length; i++) { *pointer = hgetc(webblk->sock); if(*pointer != '\n' && *pointer != '\r') pointer++; } *pointer = '\0'; http_interpret_variable_string(webblk, post_arg, VARTYPE_POST); free(post_arg); } } if (!authok) { http_error(webblk, "401 Authorization Required", "WWW-Authenticate: Basic realm=\"HERCULES\"\n", "You must be authenticated to use this service"); } if (!url) { http_error(webblk,"400 Bad Request", "", "You must specify a GET or POST request"); } /* anything following a ? in the URL is part of the get arguments */ if ((pointer=strchr(url,'?'))) { *pointer++ = 0; http_interpret_variable_string(webblk, pointer, VARTYPE_GET); } while(url[0] == '/' && url[1] == '/') url++; webblk->baseurl = url; if(!strcasecmp("/",url)) url = HTTP_WELCOME; if(strncasecmp("/cgi-bin/",url,9)) http_download(webblk,url); else url += 9; while(*url == '/') url++; #if 0 http_dump_cgi_variables(webblk); #endif for(cgient = cgidir; cgient->path; cgient++) { if(!strcmp(cgient->path, url)) { char tbuf[80]; hprintf(webblk->sock,"HTTP/1.0 200 OK\nConnection: close\n"); hprintf(webblk->sock,"Date: %s\n", http_timestring(tbuf,sizeof(tbuf),time(NULL))); (cgient->cgibin) (webblk); http_exit(webblk); } } #if defined(OPTION_DYNAMIC_LOAD) { zz_cgibin dyncgi; if( (dyncgi = HDL_FINDSYM(webblk->baseurl)) ) { char tbuf[80]; hprintf(webblk->sock,"HTTP/1.0 200 OK\nConnection: close\n"); hprintf(webblk->sock,"Date: %s\n", http_timestring(tbuf,sizeof(tbuf),time(NULL))); dyncgi(webblk); http_exit(webblk); } } #endif /*defined(OPTION_DYNAMIC_LOAD)*/ http_error(webblk, "404 File Not Found","", "The requested file was not found"); return NULL; }
/* This thread is passed a single arg, pointer * to the vegas_udp_params struct. This thread should * be cancelled and restarted if any hardware params * change, as this potentially affects packet size, etc. */ void *vegas_net_thread(void *_args) { /* Get arguments */ struct vegas_thread_args *args = (struct vegas_thread_args *)_args; int rv; /* Set cpu affinity */ cpu_set_t cpuset, cpuset_orig; sched_getaffinity(0, sizeof(cpu_set_t), &cpuset_orig); //CPU_ZERO(&cpuset); CPU_SET(13, &cpuset); rv = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); if (rv<0) { vegas_error("vegas_net_thread", "Error setting cpu affinity."); perror("sched_setaffinity"); } /* Set priority */ rv = setpriority(PRIO_PROCESS, 0, args->priority); if (rv<0) { vegas_error("vegas_net_thread", "Error setting priority level."); perror("set_priority"); } /* Attach to status shared mem area */ struct vegas_status st; rv = vegas_status_attach(&st); if (rv!=VEGAS_OK) { vegas_error("vegas_net_thread", "Error attaching to status shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)vegas_status_detach, &st); pthread_cleanup_push((void *)set_exit_status, &st); /* Init status, read info */ vegas_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "init"); vegas_status_unlock_safe(&st); /* Read in general parameters */ struct vegas_params gp; struct sdfits pf; char status_buf[VEGAS_STATUS_SIZE]; vegas_status_lock_safe(&st); memcpy(status_buf, st.buf, VEGAS_STATUS_SIZE); vegas_status_unlock_safe(&st); vegas_read_obs_params(status_buf, &gp, &pf); pthread_cleanup_push((void *)vegas_free_sdfits, &pf); /* Read network params */ struct vegas_udp_params up; vegas_read_net_params(status_buf, &up); /* Attach to databuf shared mem */ struct vegas_databuf *db; db = vegas_databuf_attach(args->output_buffer); if (db==NULL) { vegas_error("vegas_net_thread", "Error attaching to databuf shared memory."); pthread_exit(NULL); } pthread_cleanup_push((void *)vegas_databuf_detach, db); /* Time parameters */ double meas_stt_mjd=0.0; double meas_stt_offs=0.0; /* See which packet format to use */ int nchan=0, npol=0; nchan = pf.hdr.nchan; npol = pf.hdr.npol; /* Figure out size of data in each packet, number of packets * per block, etc. Changing packet size during an obs is not * recommended. */ int block_size; struct vegas_udp_packet p; size_t heap_size, spead_hdr_size; unsigned int heaps_per_block, packets_per_heap; char bw_mode[16]; if (hgets(status_buf, "BW_MODE", 16, bw_mode)) { if(strncmp(bw_mode, "high", 4) == 0) { heap_size = sizeof(struct freq_spead_heap) + nchan*4*sizeof(int); spead_hdr_size = sizeof(struct freq_spead_heap); packets_per_heap = nchan*4*sizeof(int) / PAYLOAD_SIZE; } else if(strncmp(bw_mode, "low", 3) == 0) { heap_size = sizeof(struct time_spead_heap) + PAYLOAD_SIZE; spead_hdr_size = sizeof(struct time_spead_heap); packets_per_heap = 1; } else vegas_error("vegas_net_thread", "Unsupported bandwidth mode"); } else vegas_error("vegas_net_thread", "BW_MODE not set"); if (hgeti4(status_buf, "BLOCSIZE", &block_size)==0) { block_size = db->block_size; hputi4(status_buf, "BLOCSIZE", block_size); } else { if (block_size > db->block_size) { vegas_error("vegas_net_thread", "BLOCSIZE > databuf block_size"); block_size = db->block_size; hputi4(status_buf, "BLOCSIZE", block_size); } } heaps_per_block = (block_size - MAX_HEAPS_PER_BLK*spead_hdr_size) / (heap_size - spead_hdr_size); /* List of databuf blocks currently in use */ unsigned i; const int nblock = 2; struct datablock_stats blocks[nblock]; for (i=0; i<nblock; i++) init_block(&blocks[i], db, heap_size, spead_hdr_size, heaps_per_block); /* Convenience names for first/last blocks in set */ struct datablock_stats *fblock, *lblock; fblock = &blocks[0]; lblock = &blocks[nblock-1]; /* Misc counters, etc */ char *curdata=NULL, *curheader=NULL, *curindex=NULL; unsigned int heap_cntr=0, last_heap_cntr=2048, nextblock_heap_cntr=0; unsigned int heap_offset; unsigned int seq_num=0, last_seq_num=1050; int heap_cntr_diff, seq_num_diff; unsigned int obs_started = 0; unsigned long long npacket_total, npacket_this_block=0, ndropped_total; double drop_frac_avg=0.0; const double drop_lpf = 0.25; prev_heap_cntr = 0; prev_heap_offset = 0; char msg[256]; /* Give all the threads a chance to start before opening network socket */ sleep(1); /* Set up UDP socket */ rv = vegas_udp_init(&up); if (rv!=VEGAS_OK) { vegas_error("vegas_net_thread", "Error opening UDP socket."); pthread_exit(NULL); } pthread_cleanup_push((void *)vegas_udp_close, &up); /* Main loop */ unsigned force_new_block=0, waiting=-1; signal(SIGINT,cc); while (run) { /* Wait for data */ rv = vegas_udp_wait(&up); if (rv!=VEGAS_OK) { if (rv==VEGAS_TIMEOUT) { /* Set "waiting" flag */ if (waiting!=1) { vegas_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "waiting"); vegas_status_unlock_safe(&st); waiting=1; } continue; } else { vegas_error("vegas_net_thread", "vegas_udp_wait returned error"); perror("vegas_udp_wait"); pthread_exit(NULL); } } /* Read packet */ rv = vegas_udp_recv(&up, &p, bw_mode); if (rv!=VEGAS_OK) { if (rv==VEGAS_ERR_PACKET) { #ifdef DEBUG_NET vegas_warn("vegas_net_thread", "Incorrect pkt size"); #endif continue; } else { vegas_error("vegas_net_thread", "vegas_udp_recv returned error"); perror("vegas_udp_recv"); pthread_exit(NULL); } } /* Update status if needed */ if (waiting!=0) { vegas_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "receiving"); vegas_status_unlock_safe(&st); waiting=0; } /* Check seq num diff */ heap_cntr = vegas_spead_packet_heap_cntr(&p); heap_offset = vegas_spead_packet_heap_offset(&p); seq_num = vegas_spead_packet_seq_num(heap_cntr, heap_offset, packets_per_heap); heap_cntr_diff = heap_cntr - last_heap_cntr; seq_num_diff = (int)(seq_num - last_seq_num); last_seq_num = seq_num; last_heap_cntr = heap_cntr; if (seq_num_diff<=0) { if (seq_num_diff<-1024) { force_new_block=1; obs_started = 1; #ifdef DEBUG_NET printf("Debug: observation started\n"); #endif } else if (seq_num_diff==0) { sprintf(msg, "Received duplicate packet (seq_num=%d)", seq_num); vegas_warn("vegas_net_thread", msg); } else { #ifdef DEBUG_NET sprintf(msg, "out of order packet. Diff = %d", seq_num_diff); vegas_warn("vegas_net_thread", msg); #endif continue; /* No going backwards */ } } else { force_new_block=0; npacket_total += seq_num_diff; ndropped_total += seq_num_diff - 1; npacket_this_block += seq_num_diff; fblock->pkts_dropped += seq_num_diff - 1; #ifdef DEBUG_NET if(seq_num_diff > 1) { sprintf(msg, "Missing packet. seq_num_diff = %d", seq_num_diff); vegas_warn("vegas_net_thread", msg); } #endif } /* If obs has not started, ignore this packet */ if(!obs_started) { fblock->pkts_dropped = 0; npacket_total = 0; ndropped_total = 0; npacket_this_block = 0; continue; } /* Determine if we go to next block */ if (heap_cntr>=nextblock_heap_cntr || force_new_block) { /* Update drop stats */ if (npacket_this_block > 0) drop_frac_avg = (1.0-drop_lpf)*drop_frac_avg + drop_lpf * (double)fblock->pkts_dropped / (double)npacket_this_block; vegas_status_lock_safe(&st); hputi8(st.buf, "NPKT", npacket_total); hputi8(st.buf, "NDROP", ndropped_total); hputr8(st.buf, "DROPAVG", drop_frac_avg); hputr8(st.buf, "DROPTOT", npacket_total ? (double)ndropped_total/(double)npacket_total : 0.0); hputi4(st.buf, "NETBLKOU", fblock->block_idx); vegas_status_unlock_safe(&st); /* Finalize first block, and push it off the list. * Then grab next available block. */ if (fblock->block_idx>=0) finalize_block(fblock); block_stack_push(blocks, nblock); increment_block(lblock, heap_cntr); curdata = vegas_databuf_data(db, lblock->block_idx); curheader = vegas_databuf_header(db, lblock->block_idx); curindex = vegas_databuf_index(db, lblock->block_idx); nextblock_heap_cntr = lblock->heap_idx + heaps_per_block; npacket_this_block = 0; /* If new obs started, reset total counters, get start * time. Start time is rounded to nearest integer * second, with warning if we're off that by more * than 100ms. Any current blocks on the stack * are also finalized/reset */ if (force_new_block) { /* Reset stats */ npacket_total=0; ndropped_total=0; npacket_this_block = 0; /* Get obs start time */ get_current_mjd_double(&meas_stt_mjd); printf("vegas_net_thread: got start packet at MJD %f", meas_stt_mjd); meas_stt_offs = meas_stt_mjd*24*60*60 - floor(meas_stt_mjd*24*60*60); if(meas_stt_offs > 0.1 && meas_stt_offs < 0.9) { char msg[256]; sprintf(msg, "Second fraction = %3.1f ms > +/-100 ms", meas_stt_offs*1e3); vegas_warn("vegas_net_thread", msg); } vegas_status_lock_safe(&st); hputnr8(st.buf, "M_STTMJD", 8, meas_stt_mjd); hputr8(st.buf, "M_STTOFF", meas_stt_offs); vegas_status_unlock_safe(&st); /* Warn if 1st packet number is not zero */ if (seq_num!=0) { char msg[256]; sprintf(msg, "First packet number is not 0 (seq_num=%d)", seq_num); vegas_warn("vegas_net_thread", msg); } } /* Read current status shared mem */ vegas_status_lock_safe(&st); memcpy(status_buf, st.buf, VEGAS_STATUS_SIZE); vegas_status_unlock_safe(&st); /* Wait for new block to be free, then clear it * if necessary and fill its header with new values. */ while ((rv=vegas_databuf_wait_free(db, lblock->block_idx)) != VEGAS_OK) { if (rv==VEGAS_TIMEOUT) { waiting=1; vegas_warn("vegas_net_thread", "timeout while waiting for output block\n"); vegas_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "blocked"); vegas_status_unlock_safe(&st); continue; } else { vegas_error("vegas_net_thread", "error waiting for free databuf"); run=0; pthread_exit(NULL); break; } } memcpy(curheader, status_buf, VEGAS_STATUS_SIZE); memset(curdata, 0, block_size); memset(curindex, 0, db->index_size); } /* Copy packet into any blocks where it belongs. * The "write packets" functions also update drop stats * for blocks, etc. */ int nblocks = 0; for (i=0; i<nblock; i++) { if ((blocks[i].block_idx>=0) && (block_heap_check(&blocks[i],heap_cntr)==0)) { if (nblocks > 0) { printf("vegas_net_thread: Warning! Adding packet to more than one block! heap_cntr= %d, block = %d",heap_cntr,i); } nblocks++; write_spead_packet_to_block(&blocks[i], &p, heap_cntr, heap_offset, packets_per_heap, bw_mode); } } /* Will exit if thread has been cancelled */ pthread_testcancel(); } pthread_exit(NULL); /* Have to close all push's */ pthread_cleanup_pop(0); /* Closes push(vegas_udp_close) */ pthread_cleanup_pop(0); /* Closes set_exit_status */ pthread_cleanup_pop(0); /* Closes vegas_free_psrfits */ pthread_cleanup_pop(0); /* Closes vegas_status_detach */ pthread_cleanup_pop(0); /* Closes vegas_databuf_detach */ }
static void *run(hashpipe_thread_args_t * args, int doCPU) { // Local aliases to shorten access to args fields paper_gpu_input_databuf_t *db_in = (paper_gpu_input_databuf_t *)args->ibuf; paper_output_databuf_t *db_out = (paper_output_databuf_t *)args->obuf; hashpipe_status_t st = args->st; const char * status_key = args->thread_desc->skey; #ifdef DEBUG_SEMS fprintf(stderr, "s/tid %lu/ GPU/\n", pthread_self()); #endif // Init integration control status variables int gpu_dev = 0; hashpipe_status_lock_safe(&st); hputs(st.buf, "INTSTAT", "off"); hputi8(st.buf, "INTSYNC", 0); hputi4(st.buf, "INTCOUNT", N_SUB_BLOCKS_PER_INPUT_BLOCK); hputi8(st.buf, "GPUDUMPS", 0); hgeti4(st.buf, "GPUDEV", &gpu_dev); // No change if not found hputi4(st.buf, "GPUDEV", gpu_dev); hashpipe_status_unlock_safe(&st); /* Loop */ int rv; char integ_status[17]; uint64_t start_mcount, last_mcount=0; uint64_t gpu_dumps=0; int int_count; // Number of blocks to integrate per dump int xgpu_error = 0; int curblock_in=0; int curblock_out=0; struct timespec start, stop; uint64_t elapsed_gpu_ns = 0; uint64_t gpu_block_count = 0; // Initialize context to point at first input and output memory blocks. // This seems redundant since we do this just before calling // xgpuCudaXengine, but we need to pass something in for array_h and // matrix_x to prevent xgpuInit from allocating memory. XGPUContext context; context.array_h = (ComplexInput *)db_in->block[0].data; context.array_len = (db_in->header.n_block * sizeof(paper_gpu_input_block_t) - sizeof(paper_input_header_t)) / sizeof(ComplexInput); context.matrix_h = (Complex *)db_out->block[0].data; context.matrix_len = (db_out->header.n_block * sizeof(paper_output_block_t) - sizeof(paper_output_header_t)) / sizeof(Complex); xgpu_error = xgpuInit(&context, gpu_dev); if (XGPU_OK != xgpu_error) { fprintf(stderr, "ERROR: xGPU initialization failed (error code %d)\n", xgpu_error); return THREAD_ERROR; } while (run_threads()) { // Note waiting status, // query integrating status // and, if armed, start count hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "waiting"); hgets(st.buf, "INTSTAT", 16, integ_status); hgeti8(st.buf, "INTSYNC", (long long*)&start_mcount); hashpipe_status_unlock_safe(&st); // Wait for new input block to be filled while ((rv=hashpipe_databuf_wait_filled((hashpipe_databuf_t *)db_in, curblock_in)) != HASHPIPE_OK) { if (rv==HASHPIPE_TIMEOUT) { hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "blocked_in"); hashpipe_status_unlock_safe(&st); continue; } else { hashpipe_error(__FUNCTION__, "error waiting for filled databuf"); pthread_exit(NULL); break; } } // Got a new data block, update status and determine how to handle it hashpipe_status_lock_safe(&st); hputi4(st.buf, "GPUBLKIN", curblock_in); hputu8(st.buf, "GPUMCNT", db_in->block[curblock_in].header.mcnt); hashpipe_status_unlock_safe(&st); // If integration status "off" if(!strcmp(integ_status, "off")) { // Mark input block as free and advance hashpipe_databuf_set_free((hashpipe_databuf_t *)db_in, curblock_in); curblock_in = (curblock_in + 1) % db_in->header.n_block; // Skip to next input buffer continue; } // If integration status is "start" if(!strcmp(integ_status, "start")) { // If buffer mcount < start_mcount (i.e. not there yet) if(db_in->block[curblock_in].header.mcnt < start_mcount) { // Drop input buffer // Mark input block as free and advance hashpipe_databuf_set_free((hashpipe_databuf_t *)db_in, curblock_in); curblock_in = (curblock_in + 1) % db_in->header.n_block; // Skip to next input buffer continue; // Else if mcount == start_mcount (time to start) } else if(db_in->block[curblock_in].header.mcnt == start_mcount) { // Set integration status to "on" // Read integration count (INTCOUNT) fprintf(stderr, "--- integration on ---\n"); strcpy(integ_status, "on"); hashpipe_status_lock_safe(&st); hputs(st.buf, "INTSTAT", integ_status); hgeti4(st.buf, "INTCOUNT", &int_count); hashpipe_status_unlock_safe(&st); // Compute last mcount last_mcount = start_mcount + (int_count-1) * N_SUB_BLOCKS_PER_INPUT_BLOCK; // Else (missed starting mcount) } else { // Handle missed start of integration // TODO! fprintf(stderr, "--- mcnt=%06lx > start_mcnt=%06lx ---\n", db_in->block[curblock_in].header.mcnt, start_mcount); } } // Integration status is "on" or "stop" // Note processing status hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "processing gpu"); hashpipe_status_unlock_safe(&st); // Setup for current chunk context.input_offset = curblock_in * sizeof(paper_gpu_input_block_t) / sizeof(ComplexInput); context.output_offset = curblock_out * sizeof(paper_output_block_t) / sizeof(Complex); // Call CUDA X engine function int doDump = 0; // Dump if this is the last block or we are doing both CPU and GPU // (GPU and CPU test mode always dumps every input block) if(db_in->block[curblock_in].header.mcnt >= last_mcount || doCPU) { doDump = 1; // Check whether we missed the end of integration. If we get a block // whose mcnt is greater than last_mcount, then for some reason (e.g. // networking problems) we didn't see a block whose mcnt was // last_mcount. This should "never" happen, but it has been seen to // occur when the 10 GbE links have many errors. if(db_in->block[curblock_in].header.mcnt > last_mcount) { // Can't do much error recovery, so just log it. fprintf(stderr, "--- mcnt=%06lx > last_mcnt=%06lx ---\n", db_in->block[curblock_in].header.mcnt, last_mcount); } // Wait for new output block to be free while ((rv=paper_output_databuf_wait_free(db_out, curblock_out)) != HASHPIPE_OK) { if (rv==HASHPIPE_TIMEOUT) { hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "blocked gpu out"); hashpipe_status_unlock_safe(&st); continue; } else { hashpipe_error(__FUNCTION__, "error waiting for free databuf"); pthread_exit(NULL); break; } } } clock_gettime(CLOCK_MONOTONIC, &start); xgpuCudaXengine(&context, doDump ? SYNCOP_DUMP : SYNCOP_SYNC_TRANSFER); clock_gettime(CLOCK_MONOTONIC, &stop); elapsed_gpu_ns += ELAPSED_NS(start, stop); gpu_block_count++; if(doDump) { clock_gettime(CLOCK_MONOTONIC, &start); xgpuClearDeviceIntegrationBuffer(&context); clock_gettime(CLOCK_MONOTONIC, &stop); elapsed_gpu_ns += ELAPSED_NS(start, stop); // TODO Maybe need to subtract all or half the integration time here // depending on recevier's expectations. db_out->block[curblock_out].header.mcnt = last_mcount; // If integration status if "stop" if(!strcmp(integ_status, "stop")) { // Set integration status to "off" strcpy(integ_status, "off"); hashpipe_status_lock_safe(&st); hputs(st.buf, "INTSTAT", integ_status); hashpipe_status_unlock_safe(&st); } else { // Advance last_mcount for end of next integration last_mcount += int_count * N_SUB_BLOCKS_PER_INPUT_BLOCK; } // Mark output block as full and advance paper_output_databuf_set_filled(db_out, curblock_out); curblock_out = (curblock_out + 1) % db_out->header.n_block; // TODO Need to handle or at least check for overflow! // Update GPU dump counter and GPU Gbps gpu_dumps++; hashpipe_status_lock_safe(&st); hputi8(st.buf, "GPUDUMPS", gpu_dumps); hputr4(st.buf, "GPUGBPS", (float)(8*N_FLUFFED_BYTES_PER_BLOCK*gpu_block_count)/elapsed_gpu_ns); hashpipe_status_unlock_safe(&st); // Start new average elapsed_gpu_ns = 0; gpu_block_count = 0; } if(doCPU) { /* Note waiting status */ hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "waiting"); hashpipe_status_unlock_safe(&st); // Wait for new output block to be free while ((rv=paper_output_databuf_wait_free(db_out, curblock_out)) != HASHPIPE_OK) { if (rv==HASHPIPE_TIMEOUT) { hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "blocked cpu out"); hashpipe_status_unlock_safe(&st); continue; } else { hashpipe_error(__FUNCTION__, "error waiting for free databuf"); pthread_exit(NULL); break; } } // Note "processing cpu" status, current input block hashpipe_status_lock_safe(&st); hputs(st.buf, status_key, "processing cpu"); hashpipe_status_unlock_safe(&st); /* * Call CPU X engine function */ xgpuOmpXengine((Complex *)db_out->block[curblock_out].data, context.array_h); // Mark output block as full and advance paper_output_databuf_set_filled(db_out, curblock_out); curblock_out = (curblock_out + 1) % db_out->header.n_block; // TODO Need to handle or at least check for overflow! } // Mark input block as free and advance hashpipe_databuf_set_free((hashpipe_databuf_t *)db_in, curblock_in); curblock_in = (curblock_in + 1) % db_in->header.n_block; /* Check for cancel */ pthread_testcancel(); } xgpuFree(&context); // Thread success! return NULL; }
int hget (const char *buffer, const char *key, char *val) { // Also a bit hacky.. assumes 80 chars have been reserved for val // could do this better with string. return hgets (buffer, key, 80, val); }
static void *run(hashpipe_thread_args_t * args) { // Local aliases to shorten access to args fields // Our output buffer happens to be a paper_input_databuf hashpipe_status_t st = args->st; const char * status_key = args->thread_desc->skey; st_p = &st; // allow global (this source file) access to the status buffer // Get inital value for crc32 function uint32_t init_crc = crc32(0,0,0); // Flag that holds off the crc thread int holdoff = 1; // Force ourself into the hold off state hashpipe_status_lock_safe(&st); hputi4(st.buf, "CRCHOLD", 1); hashpipe_status_unlock_safe(&st); while(holdoff) { // We're not in any hurry to startup sleep(1); hashpipe_status_lock_safe(&st); // Look for CRCHOLD value hgeti4(st.buf, "CRCHOLD", &holdoff); if(!holdoff) { // Done holding, so delete the key hdel(st.buf, "CRCHOLD"); } hashpipe_status_unlock_safe(&st); } /* Read network params */ struct hashpipe_udp_params up = { .bindhost = "0.0.0.0", .bindport = 8511, .packet_size = 8200 }; hashpipe_status_lock_safe(&st); // Get info from status buffer if present (no change if not present) hgets(st.buf, "BINDHOST", 80, up.bindhost); hgeti4(st.buf, "BINDPORT", &up.bindport); // Store bind host/port info etc in status buffer hputs(st.buf, "BINDHOST", up.bindhost); hputi4(st.buf, "BINDPORT", up.bindport); hputu4(st.buf, "CRCPKOK", 0); hputu4(st.buf, "CRCPKERR", 0); hputs(st.buf, status_key, "running"); hashpipe_status_unlock_safe(&st); struct hashpipe_udp_packet p; /* Give all the threads a chance to start before opening network socket */ sleep(1); /* Set up UDP socket */ int rv = hashpipe_udp_init(&up); if (rv!=HASHPIPE_OK) { hashpipe_error("paper_crc_thread", "Error opening UDP socket."); pthread_exit(NULL); } pthread_cleanup_push((void *)hashpipe_udp_close, &up); /* Main loop */ uint64_t packet_count = 0; uint64_t good_count = 0; uint64_t error_count = 0; uint64_t elapsed_wait_ns = 0; uint64_t elapsed_recv_ns = 0; uint64_t elapsed_proc_ns = 0; float ns_per_wait = 0.0; float ns_per_recv = 0.0; float ns_per_proc = 0.0; struct timespec start, stop; struct timespec recv_start, recv_stop; packet_header_t hdr; while (run_threads()) { /* Read packet */ clock_gettime(CLOCK_MONOTONIC, &recv_start); do { clock_gettime(CLOCK_MONOTONIC, &start); p.packet_size = recv(up.sock, p.data, HASHPIPE_MAX_PACKET_SIZE, 0); clock_gettime(CLOCK_MONOTONIC, &recv_stop); } while (p.packet_size == -1 && (errno == EAGAIN || errno == EWOULDBLOCK) && run_threads()); // Break out of loop if stopping if(!run_threads()) break; // Increment packet count packet_count++; // Check CRC if(crc32(init_crc, (/*const?*/ uint8_t *)p.data, p.packet_size) == 0xffffffff) { // CRC OK! Increment good counter good_count++; } else { // CRC error! Increment error counter error_count++; // Log message get_header(&p, &hdr); hashpipe_warn("paper_crc", "CRC error mcnt %llu ; fid %u ; xid %u", hdr.mcnt, hdr.fid, hdr.xid); } clock_gettime(CLOCK_MONOTONIC, &stop); elapsed_wait_ns += ELAPSED_NS(recv_start, start); elapsed_recv_ns += ELAPSED_NS(start, recv_stop); elapsed_proc_ns += ELAPSED_NS(recv_stop, stop); if(packet_count % 1000 == 0) { // Compute stats get_header(&p, &hdr); ns_per_wait = (float)elapsed_wait_ns / packet_count; ns_per_recv = (float)elapsed_recv_ns / packet_count; ns_per_proc = (float)elapsed_proc_ns / packet_count; // Update status hashpipe_status_lock_busywait_safe(&st); hputu8(st.buf, "CRCMCNT", hdr.mcnt); // Gbps = bits_per_packet / ns_per_packet // (N_BYTES_PER_PACKET excludes header, so +8 for the header) hputr4(st.buf, "CRCGBPS", 8*(N_BYTES_PER_PACKET+8)/(ns_per_recv+ns_per_proc)); hputr4(st.buf, "CRCWATNS", ns_per_wait); hputr4(st.buf, "CRCRECNS", ns_per_recv); hputr4(st.buf, "CRCPRCNS", ns_per_proc); // TODO Provide some way to recognize request to zero out the // CRCERR and CRCOK fields. hputu8(st.buf, "CRCPKOK", good_count); hputu8(st.buf, "CRCPKERR", error_count); hashpipe_status_unlock_safe(&st); // Start new average elapsed_wait_ns = 0; elapsed_recv_ns = 0; elapsed_proc_ns = 0; packet_count = 0; } /* Will exit if thread has been cancelled */ pthread_testcancel(); } /* Have to close all push's */ pthread_cleanup_pop(1); /* Closes push(hashpipe_udp_close) */ return NULL; } static hashpipe_thread_desc_t crc_thread = { name: "paper_crc_thread", skey: "CRCSTAT", init: NULL, run: run, ibuf_desc: {NULL},
// Run method for the thread // It is meant to do the following: // (1) Initialize status buffer // (2) Set up network parameters and socket // (3) Start main loop // (3a) Receive packet on socket // (3b) Error check packet (packet size, etc) // (3c) Call process_packet on received packet // (4) Terminate thread cleanly static void *run(hashpipe_thread_args_t * args) { fprintf(stdout, "N_INPUTS = %d\n", N_INPUTS); fprintf(stdout, "N_CHAN = %d\n", N_CHAN); fprintf(stdout, "N_CHAN_PER_X = %d\n", N_CHAN_PER_X); fprintf(stdout, "N_CHAN_PER_PACKET = %d\n", N_CHAN_PER_PACKET); fprintf(stdout, "N_TIME_PER_PACKET = %d\n", N_TIME_PER_PACKET); fprintf(stdout, "N_TIME_PER_BLOCK = %d\n", N_TIME_PER_BLOCK); fprintf(stdout, "N_BYTES_PER_BLOCK = %d\n", N_BYTES_PER_BLOCK); fprintf(stdout, "N_BYTES_PER_PACKET = %d\n", N_BYTES_PER_PACKET); fprintf(stdout, "N_PACKETS_PER_BLOCK = %d\n", N_PACKETS_PER_BLOCK); fprintf(stdout, "N_COR_MATRIX = %d\n", N_COR_MATRIX); // Local aliases to shorten access to args fields // Our output buffer happens to be a paper_input_databuf flag_input_databuf_t *db = (flag_input_databuf_t *)args->obuf; hashpipe_status_t st = args->st; const char * status_key = args->thread_desc->skey; st_p = &st; // allow global (this source file) access to the status buffer /* Read network params */ fprintf(stdout, "Setting up network parameters\n"); struct hashpipe_udp_params up = { .bindhost = "0.0.0.0", .bindport = 8511, .packet_size = 8008 }; hashpipe_status_lock_safe(&st); // Get info from status buffer if present (no change if not present) hgets(st.buf, "BINDHOST", 80, up.bindhost); hgeti4(st.buf, "BINDPORT", &up.bindport); // Store bind host/port info etc in status buffer hputs(st.buf, "BINDHOST", up.bindhost); hputi4(st.buf, "BINDPORT", up.bindport); hputu4(st.buf, "MISSEDFE", 0); hputu4(st.buf, "MISSEDPK", 0); hputs(st.buf, status_key, "running"); hashpipe_status_unlock_safe(&st); struct hashpipe_udp_packet p; /* Give all the threads a chance to start before opening network socket */ int netready = 0; int corready = 0; int checkready = 0; while (!netready) { sleep(1); // Check the correlator to see if it's ready yet hashpipe_status_lock_safe(&st); hgeti4(st.buf, "CORREADY", &corready); hgeti4(st.buf, "SAVEREADY", &checkready); hashpipe_status_unlock_safe(&st); if (!corready) { continue; } //if (!checkready) { // continue; //} // Check the other threads to see if they're ready yet // TBD // If we get here, then all threads are initialized netready = 1; } sleep(3); /* Set up UDP socket */ fprintf(stderr, "NET: BINDHOST = %s\n", up.bindhost); fprintf(stderr, "NET: BINDPORT = %d\n", up.bindport); int rv = hashpipe_udp_init(&up); if (rv!=HASHPIPE_OK) { hashpipe_error("paper_net_thread", "Error opening UDP socket."); pthread_exit(NULL); } pthread_cleanup_push((void *)hashpipe_udp_close, &up); // Initialize first few blocks in the buffer int i; for (i = 0; i < 2; i++) { // Wait until block semaphore is free if (flag_input_databuf_wait_free(db, i) != HASHPIPE_OK) { if (errno == EINTR) { // Interrupt occurred hashpipe_error(__FUNCTION__, "waiting for free block interrupted\n"); pthread_exit(NULL); } else { hashpipe_error(__FUNCTION__, "error waiting for free block\n"); pthread_exit(NULL); } } initialize_block(db, i*Nm); } // Set correlator to "start" state hashpipe_status_lock_safe(&st); hputs(st.buf, "INTSTAT", "start"); hashpipe_status_unlock_safe(&st); /* Main loop */ uint64_t packet_count = 0; fprintf(stdout, "Net: Starting Thread!\n"); while (run_threads()) { // Get packet do { p.packet_size = recv(up.sock, p.data, HASHPIPE_MAX_PACKET_SIZE, 0); } while (p.packet_size == -1 && (errno == EAGAIN || errno == EWOULDBLOCK) && run_threads()); if(!run_threads()) break; if (up.packet_size != p.packet_size && up.packet_size != p.packet_size-8) { // If an error was returned instead of a valid packet size if (p.packet_size == -1) { fprintf(stderr, "uh oh!\n"); // Log error and exit hashpipe_error("paper_net_thread", "hashpipe_udp_recv returned error"); perror("hashpipe_udp_recv"); pthread_exit(NULL); } else { // Log warning and ignore wrongly sized packet hashpipe_warn("paper_net_thread", "Incorrect pkt size (%d)", p.packet_size); continue; } } packet_count++; process_packet(db, &p); /* Will exit if thread has been cancelled */ pthread_testcancel(); } pthread_cleanup_pop(1); /* Closes push(hashpipe_udp_close) */ hashpipe_status_lock_busywait_safe(&st); hputs(st.buf, status_key, "terminated"); hashpipe_status_unlock_safe(&st); return NULL; } static hashpipe_thread_desc_t net_thread = { name: "flag_net_thread", skey: "NETSTAT", init: NULL, run: run, ibuf_desc: {NULL},
int main(int argc, char *argv[]) { static struct option long_opts[] = { {"help", 0, NULL, 'h'}, {"null", 0, NULL, 'n'}, {"ds", 0, NULL, 'D'}, {0,0,0,0} }; int use_null_thread = 0; int ds = 0; int opt, opti; while ((opt=getopt_long(argc,argv,"hnD",long_opts,&opti))!=-1) { switch (opt) { case 'n': use_null_thread = 1; break; case 'D': ds = 1; break; default: case 'h': usage(); exit(0); break; } } int twogpu = 1; // -- Ids -- thread_args net_args, dedisp_args, disk_args; thread_args net_args_2, dedisp_args_2, disk_args_2; thread_args_init(&net_args); thread_args_init(&dedisp_args); thread_args_init(&disk_args); net_args.output_buffer = 1; dedisp_args.input_buffer = net_args.output_buffer; dedisp_args.output_buffer = 2; disk_args.input_buffer = dedisp_args.output_buffer; disk_args.output_buffer = 3; net_args.gpu_id = dedisp_args.gpu_id = disk_args.gpu_id = 1; net_args.priority = 15; dedisp_args.priority = 20; disk_args.priority = 20; // -- Attach to status shared mem -- status stat; databuf *dbuf_net=NULL, *dbuf_fold=NULL; int rv = status_attach(&stat, 1); if (rv!=OK) { log_error("nuppi_daq_dedisp", "Error connecting to status shm"); exit(1); } // -- Init mutex for PSRFITS template -- rv = pthread_mutex_init(&lock_psrfits_tpl, NULL); if (rv != 0) { log_error("nuppi_daq_dedisp", "Mutex initialization failed"); exit(1); } // -- Read status shm, init RUN and read filename -- char basename[256]; status_lock_safe(&stat); hgets(stat.buf, "BASENAME", 256, basename); hputi4(stat.buf, "RUN", 1); status_unlock_safe(&stat); run=1; signal(SIGINT, cc); // -- Create or just attach to net shared mem -- dbuf_net = databuf_attach(net_args.output_buffer); if (dbuf_net==NULL) dbuf_net = databuf_create(8, 256*1024*1024, net_args.output_buffer); if (dbuf_net==NULL) { log_error("nuppi_daq_dedisp", "Error connecting to databuf net shm"); exit(1); } databuf_clear(dbuf_net); // -- Create or just attach to fold shared mem -- dbuf_fold = databuf_attach(dedisp_args.output_buffer); if (dbuf_fold==NULL) dbuf_fold = databuf_create(8, 128*1024*1024, dedisp_args.output_buffer); if (dbuf_fold==NULL) { log_error("nuppi_daq_dedisp", "Error connecting to databuf fold shm"); exit(1); } databuf_clear(dbuf_fold); /* Launch net thread */ pthread_t net_thread_id; rv = pthread_create(&net_thread_id, NULL, net_thread, (void *)&net_args); if (rv) { log_error("nuppi_daq_dedisp", "Error creating net thread"); perror("pthread_create"); exit(1); } /* Launch dedisp thread */ pthread_t dedisp_thread_id; if (ds) rv = pthread_create(&dedisp_thread_id, NULL, dedisp_ds_thread, (void *)&dedisp_args); else rv = pthread_create(&dedisp_thread_id, NULL, dedisp_thread, (void *)&dedisp_args); if (rv) { log_error("nuppi_daq_dedisp", "Error creating dedisp thread"); perror("pthread_create"); exit(1); } /* Launch psrfits/null thread */ pthread_t disk_thread_id=0; if (use_null_thread) rv = pthread_create(&disk_thread_id, NULL, null_thread, (void *)&disk_args); else rv = pthread_create(&disk_thread_id, NULL, psrfits_thread, (void *)&disk_args); if (rv) { log_error("nuppi_daq_dedisp", "Error creating psrfits thread"); perror("pthread_create"); exit(1); } // -- Second GPU -- pthread_t net_thread_id_2; pthread_t dedisp_thread_id_2; pthread_t disk_thread_id_2=0; if(twogpu) { sleep(2); // -- Ids -- thread_args_init(&net_args_2); thread_args_init(&dedisp_args_2); thread_args_init(&disk_args_2); net_args_2.output_buffer = 5; dedisp_args_2.input_buffer = net_args_2.output_buffer; dedisp_args_2.output_buffer = 6; disk_args_2.input_buffer = dedisp_args_2.output_buffer; disk_args_2.output_buffer = 7; net_args_2.gpu_id = dedisp_args_2.gpu_id = disk_args_2.gpu_id = 2; net_args_2.priority = 20; dedisp_args_2.priority = 20; disk_args_2.priority = 20; databuf *dbuf_net_2=NULL, *dbuf_fold_2=NULL; // -- Create or just attach to net shared mem -- dbuf_net_2 = databuf_attach(net_args_2.output_buffer); if (dbuf_net_2==NULL) dbuf_net_2 = databuf_create(8, 256*1024*1024, net_args_2.output_buffer); if (dbuf_net_2==NULL) { log_error("nuppi_daq_dedisp", "Error connecting to databuf net shm for GPU #2"); exit(1); } databuf_clear(dbuf_net_2); // -- Create or just attach to fold shared mem -- dbuf_fold_2 = databuf_attach(dedisp_args_2.output_buffer); if (dbuf_fold_2==NULL) dbuf_fold_2 = databuf_create(8, 128*1024*1024, dedisp_args_2.output_buffer); if (dbuf_fold_2==NULL) { log_error("nuppi_daq_dedisp", "Error connecting to databuf fold shm for GPU #2"); exit(1); } databuf_clear(dbuf_fold_2); /* Launch net thread */ rv = pthread_create(&net_thread_id_2, NULL, net_thread, (void *)&net_args_2); if (rv) { log_error("nuppi_daq_dedisp", "Error creating net thread 2"); perror("pthread_create"); exit(1); } /* Launch dedisp thread */ if (ds) rv = pthread_create(&dedisp_thread_id_2, NULL, dedisp_ds_thread, (void *)&dedisp_args_2); else rv = pthread_create(&dedisp_thread_id_2, NULL, dedisp_thread, (void *)&dedisp_args_2); if (rv) { log_error("nuppi_daq_dedisp", "Error creating dedisp thread 2"); perror("pthread_create"); exit(1); } /* Launch psrfits/null thread */ if (use_null_thread) rv = pthread_create(&disk_thread_id_2, NULL, null_thread, (void *)&disk_args_2); else rv = pthread_create(&disk_thread_id_2, NULL, psrfits_thread, (void *)&disk_args_2); if (rv) { log_error("nuppi_daq_dedisp", "Error creating psrfits thread 2"); perror("pthread_create"); exit(1); } } /* Alt loop, wait for run=0 */ while (run) { sleep(1); // Read the RUN keyword in the first status shm, to look for a stop order status_lock_safe(&stat); hgeti4(stat.buf, "RUN", &run); status_unlock_safe(&stat); if (run == 0) log_info("nuppi_daq_dedisp", "Caught RUN = 0 signal for end of observation"); if (disk_args.finished) run=0; } /* Clean up */ pthread_cancel(dedisp_thread_id); pthread_cancel(net_thread_id); pthread_cancel(disk_thread_id); pthread_kill(dedisp_thread_id,SIGINT); pthread_kill(net_thread_id,SIGINT); pthread_kill(disk_thread_id,SIGINT); if(twogpu) { pthread_cancel(dedisp_thread_id_2); pthread_cancel(net_thread_id_2); pthread_cancel(disk_thread_id_2); pthread_kill(dedisp_thread_id_2,SIGINT); pthread_kill(net_thread_id_2,SIGINT); pthread_kill(disk_thread_id_2,SIGINT); } // -- Join threads -- pthread_join(net_thread_id,NULL); log_info("nuppi_daq_dedisp", "Joined net thread"); if(twogpu) { pthread_join(net_thread_id_2,NULL); log_info("nuppi_daq_dedisp", "Joined net thread 2"); } pthread_join(dedisp_thread_id,NULL); log_info("nuppi_daq_dedisp", "Joined dedisp thread"); if(twogpu) { pthread_join(dedisp_thread_id_2,NULL); log_info("nuppi_daq_dedisp", "Joined dedisp thread 2"); } pthread_join(disk_thread_id,NULL); log_info("nuppi_daq_dedisp", "Joined disk thread"); if(twogpu) { pthread_join(disk_thread_id_2,NULL); log_info("nuppi_daq_dedisp", "Joined disk thread 2"); } // -- Destroy args -- thread_args_destroy(&net_args); thread_args_destroy(&dedisp_args); thread_args_destroy(&disk_args); thread_args_destroy(&net_args_2); thread_args_destroy(&dedisp_args_2); thread_args_destroy(&disk_args_2); pthread_mutex_destroy(&lock_psrfits_tpl); char cmd[128], hostname[128]; gethostname(hostname, 127); sprintf(cmd, "mv %s /home/pulsar/data/%s-%s.log", LOG_FILENAME, basename, hostname); log_info("nuppi_daq_dedisp", cmd); system(cmd); exit(0); }