int set_block_filled(paper_input_databuf_t *paper_input_databuf_p, block_info_t *binfo, int block_i) { static int last_filled = -1; uint32_t block_missed_pkt_cnt=0, block_missed_mod_cnt, block_missed_feng, missed_pkt_cnt=0; if(binfo->block_active[block_i]) { // if all packets are accounted for, mark this block as good if(binfo->block_active[block_i] == N_PACKETS_PER_BLOCK) { paper_input_databuf_p->block[block_i].header.good_data = 1; } last_filled = (last_filled+1) % N_INPUT_BLOCKS; if(last_filled != block_i) { printf("block %d being marked filled, but expected block %d!\n", block_i, last_filled); #ifdef DIE_ON_OUT_OF_SEQ_FILL die(paper_input_databuf_p, binfo); #else binfo->initialized = 0; return 0; #endif } if(paper_input_databuf_set_filled(paper_input_databuf_p, block_i) != GUPPI_OK) { guppi_error(__FUNCTION__, "error waiting for databuf filled call"); run_threads=0; pthread_exit(NULL); return 0; } block_missed_pkt_cnt = N_PACKETS_PER_BLOCK - binfo->block_active[block_i]; // If we missed more than N_PACKETS_PER_BLOCK_PER_F, then assume we // are missing one or more F engines. Any missed packets beyond an // integer multiple of N_PACKETS_PER_BLOCK_PER_F will be considered // as dropped packets. block_missed_feng = block_missed_pkt_cnt / N_PACKETS_PER_BLOCK_PER_F; block_missed_mod_cnt = block_missed_pkt_cnt % N_PACKETS_PER_BLOCK_PER_F; guppi_status_lock_busywait_safe(st_p); hputu4(st_p->buf, "NETBKOUT", block_i); hputu4(st_p->buf, "MISSEDFE", block_missed_feng); if(block_missed_mod_cnt) { // Increment MISSEDPK by number of missed packets for this block hgetu4(st_p->buf, "MISSEDPK", &missed_pkt_cnt); missed_pkt_cnt += block_missed_mod_cnt; hputu4(st_p->buf, "MISSEDPK", missed_pkt_cnt); // fprintf(stderr, "got %d packets instead of %d\n", // binfo->block_active[block_i], N_PACKETS_PER_BLOCK); } guppi_status_unlock_safe(st_p); binfo->block_active[block_i] = 0; } return block_missed_pkt_cnt; }
static void *run(hashpipe_thread_args_t * args) { // Local aliases to shorten access to args fields // Our output buffer happens to be a paper_input_databuf hashpipe_status_t st = args->st; const char * status_key = args->thread_desc->skey; st_p = &st; // allow global (this source file) access to the status buffer // Get inital value for crc32 function uint32_t init_crc = crc32(0,0,0); // Flag that holds off the crc thread int holdoff = 1; // Force ourself into the hold off state hashpipe_status_lock_safe(&st); hputi4(st.buf, "CRCHOLD", 1); hashpipe_status_unlock_safe(&st); while(holdoff) { // We're not in any hurry to startup sleep(1); hashpipe_status_lock_safe(&st); // Look for CRCHOLD value hgeti4(st.buf, "CRCHOLD", &holdoff); if(!holdoff) { // Done holding, so delete the key hdel(st.buf, "CRCHOLD"); } hashpipe_status_unlock_safe(&st); } /* Read network params */ struct hashpipe_udp_params up = { .bindhost = "0.0.0.0", .bindport = 8511, .packet_size = 8200 }; hashpipe_status_lock_safe(&st); // Get info from status buffer if present (no change if not present) hgets(st.buf, "BINDHOST", 80, up.bindhost); hgeti4(st.buf, "BINDPORT", &up.bindport); // Store bind host/port info etc in status buffer hputs(st.buf, "BINDHOST", up.bindhost); hputi4(st.buf, "BINDPORT", up.bindport); hputu4(st.buf, "CRCPKOK", 0); hputu4(st.buf, "CRCPKERR", 0); hputs(st.buf, status_key, "running"); hashpipe_status_unlock_safe(&st); struct hashpipe_udp_packet p; /* Give all the threads a chance to start before opening network socket */ sleep(1); /* Set up UDP socket */ int rv = hashpipe_udp_init(&up); if (rv!=HASHPIPE_OK) { hashpipe_error("paper_crc_thread", "Error opening UDP socket."); pthread_exit(NULL); } pthread_cleanup_push((void *)hashpipe_udp_close, &up); /* Main loop */ uint64_t packet_count = 0; uint64_t good_count = 0; uint64_t error_count = 0; uint64_t elapsed_wait_ns = 0; uint64_t elapsed_recv_ns = 0; uint64_t elapsed_proc_ns = 0; float ns_per_wait = 0.0; float ns_per_recv = 0.0; float ns_per_proc = 0.0; struct timespec start, stop; struct timespec recv_start, recv_stop; packet_header_t hdr; while (run_threads()) { /* Read packet */ clock_gettime(CLOCK_MONOTONIC, &recv_start); do { clock_gettime(CLOCK_MONOTONIC, &start); p.packet_size = recv(up.sock, p.data, HASHPIPE_MAX_PACKET_SIZE, 0); clock_gettime(CLOCK_MONOTONIC, &recv_stop); } while (p.packet_size == -1 && (errno == EAGAIN || errno == EWOULDBLOCK) && run_threads()); // Break out of loop if stopping if(!run_threads()) break; // Increment packet count packet_count++; // Check CRC if(crc32(init_crc, (/*const?*/ uint8_t *)p.data, p.packet_size) == 0xffffffff) { // CRC OK! Increment good counter good_count++; } else { // CRC error! Increment error counter error_count++; // Log message get_header(&p, &hdr); hashpipe_warn("paper_crc", "CRC error mcnt %llu ; fid %u ; xid %u", hdr.mcnt, hdr.fid, hdr.xid); } clock_gettime(CLOCK_MONOTONIC, &stop); elapsed_wait_ns += ELAPSED_NS(recv_start, start); elapsed_recv_ns += ELAPSED_NS(start, recv_stop); elapsed_proc_ns += ELAPSED_NS(recv_stop, stop); if(packet_count % 1000 == 0) { // Compute stats get_header(&p, &hdr); ns_per_wait = (float)elapsed_wait_ns / packet_count; ns_per_recv = (float)elapsed_recv_ns / packet_count; ns_per_proc = (float)elapsed_proc_ns / packet_count; // Update status hashpipe_status_lock_busywait_safe(&st); hputu8(st.buf, "CRCMCNT", hdr.mcnt); // Gbps = bits_per_packet / ns_per_packet // (N_BYTES_PER_PACKET excludes header, so +8 for the header) hputr4(st.buf, "CRCGBPS", 8*(N_BYTES_PER_PACKET+8)/(ns_per_recv+ns_per_proc)); hputr4(st.buf, "CRCWATNS", ns_per_wait); hputr4(st.buf, "CRCRECNS", ns_per_recv); hputr4(st.buf, "CRCPRCNS", ns_per_proc); // TODO Provide some way to recognize request to zero out the // CRCERR and CRCOK fields. hputu8(st.buf, "CRCPKOK", good_count); hputu8(st.buf, "CRCPKERR", error_count); hashpipe_status_unlock_safe(&st); // Start new average elapsed_wait_ns = 0; elapsed_recv_ns = 0; elapsed_proc_ns = 0; packet_count = 0; } /* Will exit if thread has been cancelled */ pthread_testcancel(); } /* Have to close all push's */ pthread_cleanup_pop(1); /* Closes push(hashpipe_udp_close) */ return NULL; } static hashpipe_thread_desc_t crc_thread = { name: "paper_crc_thread", skey: "CRCSTAT", init: NULL, run: run, ibuf_desc: {NULL},
static void *run(void * _args) { // Cast _args struct guppi_thread_args *args = (struct guppi_thread_args *)_args; THREAD_RUN_BEGIN(args); THREAD_RUN_SET_AFFINITY_PRIORITY(args); THREAD_RUN_ATTACH_STATUS(args->instance_id, st); // Attach to paper_ouput_databuf THREAD_RUN_ATTACH_DATABUF(args->instance_id, paper_output_databuf, db, args->input_buffer); // Setup socket and message structures int sockfd; unsigned int xengine_id = 0; struct timespec packet_delay = { .tv_sec = 0, .tv_nsec = PACKET_DELAY_NS }; guppi_status_lock_safe(&st); hgetu4(st.buf, "XID", &xengine_id); // No change if not found hputu4(st.buf, "XID", xengine_id); hputu4(st.buf, "OUTDUMPS", 0); guppi_status_unlock_safe(&st); pkt_t pkt; pkt.hdr.header = HEADER; pkt.hdr.instids = INSTIDS(xengine_id); pkt.hdr.pktinfo = PKTINFO(BYTES_PER_PACKET); pkt.hdr.heaplen = HEAPLEN; // TODO Get catcher hostname and port from somewhere #ifndef CATCHER_PORT #define CATCHER_PORT 7148 #endif #define stringify2(x) #x #define stringify(x) stringify2(x) // Open socket sockfd = open_udp_socket("catcher", stringify(CATCHER_PORT)); if(sockfd == -1) { guppi_error(__FUNCTION__, "error opening socket"); run_threads=0; pthread_exit(NULL); } #ifdef TEST_INDEX_CALCS int i, j; for(i=0; i<32; i++) { for(j=i; j<32; j++) { regtile_index(2*i, 2*j); } } for(i=0; i<32; i++) { for(j=i; j<32; j++) { casper_index(2*i, 2*j); } } run_threads=0; #endif /* Main loop */ int rv; int casper_chan, gpu_chan; int baseline; unsigned int dumps = 0; int block_idx = 0; struct timespec start, stop; signal(SIGINT,cc); signal(SIGTERM,cc); while (run_threads) { guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "waiting"); guppi_status_unlock_safe(&st); // Wait for new block to be filled while ((rv=paper_output_databuf_wait_filled(db, block_idx)) != GUPPI_OK) { if (rv==GUPPI_TIMEOUT) { guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "blocked"); guppi_status_unlock_safe(&st); continue; } else { guppi_error(__FUNCTION__, "error waiting for filled databuf"); run_threads=0; pthread_exit(NULL); break; } } clock_gettime(CLOCK_MONOTONIC, &start); // Note processing status, current input block guppi_status_lock_safe(&st); hputs(st.buf, STATUS_KEY, "processing"); hputi4(st.buf, "OUTBLKIN", block_idx); guppi_status_unlock_safe(&st); // Update header's timestamp for this dump pkt.hdr.timestamp = TIMESTAMP(db->block[block_idx].header.mcnt * N_TIME_PER_PACKET * 2 * N_CHAN_TOTAL / 128); // Init header's offset for this dump uint32_t nbytes = 0; pkt.hdr.offset = OFFSET(nbytes); // Unpack and convert in packet sized chunks float * pf_re = db->block[block_idx].data; float * pf_im = db->block[block_idx].data + xgpu_info.matLength; pktdata_t * p_out = pkt.data; for(casper_chan=0; casper_chan<N_CHAN_PER_X; casper_chan++) { // De-interleave the channels gpu_chan = (casper_chan/Nc) + ((casper_chan%Nc)*Nx); for(baseline=0; baseline<CASPER_CHAN_LENGTH; baseline++) { off_t idx_regtile = idx_map[baseline]; pktdata_t re = CONVERT(pf_re[gpu_chan*REGTILE_CHAN_LENGTH+idx_regtile]); pktdata_t im = CONVERT(pf_im[gpu_chan*REGTILE_CHAN_LENGTH+idx_regtile]); *p_out++ = re; *p_out++ = -im; // Conjugate data to match downstream expectations nbytes += 2*sizeof(pktdata_t); if(nbytes % BYTES_PER_PACKET == 0) { int bytes_sent = send(sockfd, &pkt, sizeof(pkt.hdr)+BYTES_PER_PACKET, 0); if(bytes_sent == -1) { // Send all packets even if cactcher is not listening (i.e. we // we get a connection refused error), but abort sending this // dump if we get any other error. if(errno != ECONNREFUSED) { perror("send"); // Update stats guppi_status_lock_safe(&st); hputu4(st.buf, "OUTDUMPS", ++dumps); hputr4(st.buf, "OUTSECS", 0.0); hputr4(st.buf, "OUTMBPS", 0.0); guppi_status_unlock_safe(&st); // Break out of both for loops goto done_sending; } } else if(bytes_sent != sizeof(pkt.hdr)+BYTES_PER_PACKET) { printf("only sent %d of %lu bytes!!!\n", bytes_sent, sizeof(pkt.hdr)+BYTES_PER_PACKET); } // Delay to prevent overflowing network TX queue nanosleep(&packet_delay, NULL); // Setup for next packet p_out = pkt.data; // Update header's byte_offset for this chunk pkt.hdr.offset = OFFSET(nbytes); } } } clock_gettime(CLOCK_MONOTONIC, &stop); guppi_status_lock_safe(&st); hputu4(st.buf, "OUTDUMPS", ++dumps); hputr4(st.buf, "OUTSECS", (float)ELAPSED_NS(start,stop)/1e9); hputr4(st.buf, "OUTMBPS", (1e3*8*bytes_per_dump)/ELAPSED_NS(start,stop)); guppi_status_unlock_safe(&st); done_sending: // Mark block as free paper_output_databuf_set_free(db, block_idx); // Setup for next block block_idx = (block_idx + 1) % db->header.n_block; /* Will exit if thread has been cancelled */ pthread_testcancel(); } // Have to close all pushes THREAD_RUN_DETACH_DATAUF; THREAD_RUN_DETACH_STATUS; THREAD_RUN_END; // Thread success! return NULL; } static pipeline_thread_module_t module = { name: "paper_gpu_output_thread", type: PIPELINE_OUTPUT_THREAD, init: init, run: run };
// Run method for the thread // It is meant to do the following: // (1) Initialize status buffer // (2) Set up network parameters and socket // (3) Start main loop // (3a) Receive packet on socket // (3b) Error check packet (packet size, etc) // (3c) Call process_packet on received packet // (4) Terminate thread cleanly static void *run(hashpipe_thread_args_t * args) { fprintf(stdout, "N_INPUTS = %d\n", N_INPUTS); fprintf(stdout, "N_CHAN = %d\n", N_CHAN); fprintf(stdout, "N_CHAN_PER_X = %d\n", N_CHAN_PER_X); fprintf(stdout, "N_CHAN_PER_PACKET = %d\n", N_CHAN_PER_PACKET); fprintf(stdout, "N_TIME_PER_PACKET = %d\n", N_TIME_PER_PACKET); fprintf(stdout, "N_TIME_PER_BLOCK = %d\n", N_TIME_PER_BLOCK); fprintf(stdout, "N_BYTES_PER_BLOCK = %d\n", N_BYTES_PER_BLOCK); fprintf(stdout, "N_BYTES_PER_PACKET = %d\n", N_BYTES_PER_PACKET); fprintf(stdout, "N_PACKETS_PER_BLOCK = %d\n", N_PACKETS_PER_BLOCK); fprintf(stdout, "N_COR_MATRIX = %d\n", N_COR_MATRIX); // Local aliases to shorten access to args fields // Our output buffer happens to be a paper_input_databuf flag_input_databuf_t *db = (flag_input_databuf_t *)args->obuf; hashpipe_status_t st = args->st; const char * status_key = args->thread_desc->skey; st_p = &st; // allow global (this source file) access to the status buffer /* Read network params */ fprintf(stdout, "Setting up network parameters\n"); struct hashpipe_udp_params up = { .bindhost = "0.0.0.0", .bindport = 8511, .packet_size = 8008 }; hashpipe_status_lock_safe(&st); // Get info from status buffer if present (no change if not present) hgets(st.buf, "BINDHOST", 80, up.bindhost); hgeti4(st.buf, "BINDPORT", &up.bindport); // Store bind host/port info etc in status buffer hputs(st.buf, "BINDHOST", up.bindhost); hputi4(st.buf, "BINDPORT", up.bindport); hputu4(st.buf, "MISSEDFE", 0); hputu4(st.buf, "MISSEDPK", 0); hputs(st.buf, status_key, "running"); hashpipe_status_unlock_safe(&st); struct hashpipe_udp_packet p; /* Give all the threads a chance to start before opening network socket */ int netready = 0; int corready = 0; int checkready = 0; while (!netready) { sleep(1); // Check the correlator to see if it's ready yet hashpipe_status_lock_safe(&st); hgeti4(st.buf, "CORREADY", &corready); hgeti4(st.buf, "SAVEREADY", &checkready); hashpipe_status_unlock_safe(&st); if (!corready) { continue; } //if (!checkready) { // continue; //} // Check the other threads to see if they're ready yet // TBD // If we get here, then all threads are initialized netready = 1; } sleep(3); /* Set up UDP socket */ fprintf(stderr, "NET: BINDHOST = %s\n", up.bindhost); fprintf(stderr, "NET: BINDPORT = %d\n", up.bindport); int rv = hashpipe_udp_init(&up); if (rv!=HASHPIPE_OK) { hashpipe_error("paper_net_thread", "Error opening UDP socket."); pthread_exit(NULL); } pthread_cleanup_push((void *)hashpipe_udp_close, &up); // Initialize first few blocks in the buffer int i; for (i = 0; i < 2; i++) { // Wait until block semaphore is free if (flag_input_databuf_wait_free(db, i) != HASHPIPE_OK) { if (errno == EINTR) { // Interrupt occurred hashpipe_error(__FUNCTION__, "waiting for free block interrupted\n"); pthread_exit(NULL); } else { hashpipe_error(__FUNCTION__, "error waiting for free block\n"); pthread_exit(NULL); } } initialize_block(db, i*Nm); } // Set correlator to "start" state hashpipe_status_lock_safe(&st); hputs(st.buf, "INTSTAT", "start"); hashpipe_status_unlock_safe(&st); /* Main loop */ uint64_t packet_count = 0; fprintf(stdout, "Net: Starting Thread!\n"); while (run_threads()) { // Get packet do { p.packet_size = recv(up.sock, p.data, HASHPIPE_MAX_PACKET_SIZE, 0); } while (p.packet_size == -1 && (errno == EAGAIN || errno == EWOULDBLOCK) && run_threads()); if(!run_threads()) break; if (up.packet_size != p.packet_size && up.packet_size != p.packet_size-8) { // If an error was returned instead of a valid packet size if (p.packet_size == -1) { fprintf(stderr, "uh oh!\n"); // Log error and exit hashpipe_error("paper_net_thread", "hashpipe_udp_recv returned error"); perror("hashpipe_udp_recv"); pthread_exit(NULL); } else { // Log warning and ignore wrongly sized packet hashpipe_warn("paper_net_thread", "Incorrect pkt size (%d)", p.packet_size); continue; } } packet_count++; process_packet(db, &p); /* Will exit if thread has been cancelled */ pthread_testcancel(); } pthread_cleanup_pop(1); /* Closes push(hashpipe_udp_close) */ hashpipe_status_lock_busywait_safe(&st); hputs(st.buf, status_key, "terminated"); hashpipe_status_unlock_safe(&st); return NULL; } static hashpipe_thread_desc_t net_thread = { name: "flag_net_thread", skey: "NETSTAT", init: NULL, run: run, ibuf_desc: {NULL},
static void *run(void * _args) { // Cast _args struct guppi_thread_args *args = (struct guppi_thread_args *)_args; #ifdef DEBUG_SEMS fprintf(stderr, "s/tid %lu/NET/' <<.\n", pthread_self()); #endif THREAD_RUN_BEGIN(args); THREAD_RUN_SET_AFFINITY_PRIORITY(args); THREAD_RUN_ATTACH_STATUS(args->instance_id, st); st_p = &st; // allow global (this source file) access to the status buffer /* Attach to paper_input_databuf */ THREAD_RUN_ATTACH_DATABUF(args->instance_id, paper_input_databuf, db, args->output_buffer); /* Read in general parameters */ struct guppi_params gp; struct sdfits pf; char status_buf[GUPPI_STATUS_SIZE]; guppi_status_lock_busywait_safe(st_p); memcpy(status_buf, st_p->buf, GUPPI_STATUS_SIZE); guppi_status_unlock_safe(st_p); guppi_read_obs_params(status_buf, &gp, &pf); pthread_cleanup_push((void *)guppi_free_sdfits, &pf); /* Read network params */ struct guppi_udp_params up; //guppi_read_net_params(status_buf, &up); paper_read_net_params(status_buf, &up); // Store bind host/port info etc in status buffer guppi_status_lock_busywait_safe(&st); hputs(st.buf, "BINDHOST", up.bindhost); hputi4(st.buf, "BINDPORT", up.bindport); hputu4(st.buf, "MISSEDFE", 0); hputu4(st.buf, "MISSEDPK", 0); hputs(st.buf, STATUS_KEY, "running"); guppi_status_unlock_safe(&st); struct guppi_udp_packet p; /* Give all the threads a chance to start before opening network socket */ sleep(1); #ifndef TIMING_TEST /* Set up UDP socket */ int rv = guppi_udp_init(&up); if (rv!=GUPPI_OK) { guppi_error("guppi_net_thread", "Error opening UDP socket."); pthread_exit(NULL); } pthread_cleanup_push((void *)guppi_udp_close, &up); #endif /* Main loop */ uint64_t packet_count = 0; uint64_t elapsed_wait_ns = 0; uint64_t elapsed_recv_ns = 0; uint64_t elapsed_proc_ns = 0; float ns_per_wait = 0.0; float ns_per_recv = 0.0; float ns_per_proc = 0.0; struct timespec start, stop; struct timespec recv_start, recv_stop; signal(SIGINT,cc); while (run_threads) { #ifndef TIMING_TEST /* Read packet */ clock_gettime(CLOCK_MONOTONIC, &recv_start); do { clock_gettime(CLOCK_MONOTONIC, &start); p.packet_size = recv(up.sock, p.data, GUPPI_MAX_PACKET_SIZE, 0); clock_gettime(CLOCK_MONOTONIC, &recv_stop); } while (p.packet_size == -1 && (errno == EAGAIN || errno == EWOULDBLOCK) && run_threads); if(!run_threads) break; if (up.packet_size != p.packet_size) { if (p.packet_size != -1) { #ifdef DEBUG_NET guppi_warn("guppi_net_thread", "Incorrect pkt size"); #endif continue; } else { guppi_error("guppi_net_thread", "guppi_udp_recv returned error"); perror("guppi_udp_recv"); pthread_exit(NULL); } } #endif packet_count++; // Copy packet into any blocks where it belongs. const uint64_t mcnt = write_paper_packet_to_blocks((paper_input_databuf_t *)db, &p); clock_gettime(CLOCK_MONOTONIC, &stop); elapsed_wait_ns += ELAPSED_NS(recv_start, start); elapsed_recv_ns += ELAPSED_NS(start, recv_stop); elapsed_proc_ns += ELAPSED_NS(recv_stop, stop); if(mcnt != -1) { // Update status ns_per_wait = (float)elapsed_wait_ns / packet_count; ns_per_recv = (float)elapsed_recv_ns / packet_count; ns_per_proc = (float)elapsed_proc_ns / packet_count; guppi_status_lock_busywait_safe(&st); hputu8(st.buf, "NETMCNT", mcnt); // Gbps = bits_per_packet / ns_per_packet // (N_BYTES_PER_PACKET excludes header, so +8 for the header) hputr4(st.buf, "NETGBPS", 8*(N_BYTES_PER_PACKET+8)/(ns_per_recv+ns_per_proc)); hputr4(st.buf, "NETWATNS", ns_per_wait); hputr4(st.buf, "NETRECNS", ns_per_recv); hputr4(st.buf, "NETPRCNS", ns_per_proc); guppi_status_unlock_safe(&st); // Start new average elapsed_wait_ns = 0; elapsed_recv_ns = 0; elapsed_proc_ns = 0; packet_count = 0; } #if defined TIMING_TEST || defined NET_TIMING_TEST #define END_LOOP_COUNT (1*1000*1000) static int loop_count=0; static struct timespec tt_start, tt_stop; if(loop_count == 0) { clock_gettime(CLOCK_MONOTONIC, &tt_start); } //if(loop_count == 1000000) run_threads = 0; if(loop_count == END_LOOP_COUNT) { clock_gettime(CLOCK_MONOTONIC, &tt_stop); int64_t elapsed = ELAPSED_NS(tt_start, tt_stop); printf("processed %d packets in %.6f ms (%.3f us per packet)\n", END_LOOP_COUNT, elapsed/1e6, elapsed/1e3/END_LOOP_COUNT); exit(0); } loop_count++; #endif /* Will exit if thread has been cancelled */ pthread_testcancel(); } /* Have to close all push's */ #ifndef TIMING_TEST pthread_cleanup_pop(0); /* Closes push(guppi_udp_close) */ #endif pthread_cleanup_pop(0); /* Closes guppi_free_psrfits */ THREAD_RUN_DETACH_DATAUF; THREAD_RUN_DETACH_STATUS; THREAD_RUN_END; return NULL; }