static void ompi_mtl_portals_wait_for_put_end(ptl_seq_t link) { ptl_event_t ev; int ret; /* wait for a PUT_END event that matches the message we're looking for */ while (true) { ret = PtlEQWait(ompi_mtl_portals.ptl_unexpected_recv_eq_h,&ev); if (PTL_OK == ret) { if (PTL_EVENT_PUT_START == ev.type) { ompi_free_list_item_t *item; ompi_mtl_portals_event_t *recv_event; OMPI_FREE_LIST_GET(&ompi_mtl_portals.event_fl, item, ret); recv_event = (ompi_mtl_portals_event_t*) item; recv_event->ev = ev; recv_event->is_complete = false; opal_list_append(&(ompi_mtl_portals.unexpected_messages), (opal_list_item_t*) recv_event); if (PTL_IS_SHORT_MSG(recv_event->ev.match_bits)) { ompi_mtl_portals_recv_short_block_t *block = recv_event->ev.md.user_ptr; OPAL_THREAD_ADD32(&block->pending, 1); } } else if (PTL_EVENT_PUT_END == ev.type) { if (link == ev.link) { /* the one we want */ return; } /* otherwise match it up */ ompi_mtl_portals_match_up_put_end(ev.link); } else { opal_output(fileno(stderr)," Unrecognised event type - %d - ompi_mtl_portals_wait_for_put_end : %d \n",ev.type,ret); abort(); } } else { opal_output(fileno(stderr)," Error returned in ompi_mtl_portals_wait_for_put_end from PtlEQWait : %d \n",ret); abort(); } } }
int server(void *arg) { int ret; int num_interfaces; ptl_handle_ni_t nih; ptl_handle_eq_t eqh; ptl_ni_limits_t ptl_limits; ptl_event_t ev_t; ptl_event_t *ev=&ev_t; printf("IN SERVER\n"); if ((ret=PtlInit(&num_interfaces)) != PTL_OK) { printf("%s: PtlInit failed: %d\n", __FUNCTION__, ret); exit(1); } printf("%s: PtlInit succeeds (%d:%d)\n", __FUNCTION__, ret, num_interfaces); if (((ret=PtlNIInit( IFACE_FROM_BRIDGE_AND_NALID(PTL_BRIDGE_UK, PTL_IFACE_SS), PTL_PID_ANY, NULL, &ptl_limits, &nih)) != PTL_OK) && (ret != PTL_IFACE_DUP)) { printf("%s: PtlNIInit failed: %d\n", __FUNCTION__, ret); exit(1); } printf("%s: PtlNIInit succeeds (%d)\n", __FUNCTION__, ret); if ((ret=PtlEQAlloc(nih, 4096, NULL, &eqh)) != PTL_OK) { printf("%s: PtlEQAlloc failed: %d\n", __FUNCTION__, ret); exit(1); } iv=11; ret=PtlEQWait(nih,ev); printf("%s: PtlEQAlloc succeeds\n", __FUNCTION__); printf("%d\n",iv); iv=13; while(1); }
int main(int argc, char *argv[]) { ptl_handle_ni_t ni_logical; ptl_pt_index_t logical_pt_index; ptl_process_t myself; struct timeval start, stop; int potato = 0; ENTRY_T potato_catcher; HANDLE_T potato_catcher_handle; ptl_md_t potato_launcher; ptl_handle_md_t potato_launcher_handle; int num_procs; ptl_handle_eq_t pt_eq_handle; CHECK_RETURNVAL(PtlInit()); CHECK_RETURNVAL(libtest_init()); num_procs = libtest_get_size(); if (NULL != getenv("MAKELEVEL") && num_procs > 2) { return 77; } CHECK_RETURNVAL(PtlNIInit (PTL_IFACE_DEFAULT, NI_TYPE | PTL_NI_LOGICAL, PTL_PID_ANY, NULL, NULL, &ni_logical)); CHECK_RETURNVAL(PtlSetMap(ni_logical, num_procs, libtest_get_mapping(ni_logical))); CHECK_RETURNVAL(PtlGetId(ni_logical, &myself)); CHECK_RETURNVAL(PtlEQAlloc(ni_logical, 100, &pt_eq_handle)); CHECK_RETURNVAL(PtlPTAlloc (ni_logical, 0, pt_eq_handle, PTL_PT_ANY, &logical_pt_index)); assert(logical_pt_index == 0); /* Now do the initial setup on ni_logical */ potato_catcher.start = &potato; potato_catcher.length = sizeof(potato); potato_catcher.uid = PTL_UID_ANY; potato_catcher.options = OPTIONS; #if INTERFACE == 1 potato_catcher.match_id.rank = PTL_RANK_ANY; potato_catcher.match_bits = 1; potato_catcher.ignore_bits = ~potato_catcher.match_bits; #endif potato_catcher.ct_handle = PTL_CT_NONE; CHECK_RETURNVAL(APPEND (ni_logical, logical_pt_index, &potato_catcher, PTL_PRIORITY_LIST, NULL, &potato_catcher_handle)); { ptl_event_t event; CHECK_RETURNVAL(PtlEQWait(pt_eq_handle, &event)); // wait for link event assert(event.type == PTL_EVENT_LINK); } /* Now do a barrier (on ni_physical) to make sure that everyone has their * logical interface set up */ libtest_barrier(); /* now I can communicate between ranks with ni_logical */ /* set up the potato launcher */ potato_launcher.start = &potato; potato_launcher.length = sizeof(potato); potato_launcher.options = PTL_MD_EVENT_CT_ACK | PTL_MD_EVENT_CT_SEND; potato_launcher.eq_handle = PTL_EQ_NONE; // i.e. don't queue send events CHECK_RETURNVAL(PtlCTAlloc(ni_logical, &potato_launcher.ct_handle)); CHECK_RETURNVAL(PtlMDBind (ni_logical, &potato_launcher, &potato_launcher_handle)); /* rank 0 starts the potato going */ if (myself.rank == 0) { ptl_process_t nextrank; nextrank.rank = myself.rank + 1; nextrank.rank *= (nextrank.rank <= num_procs - 1); gettimeofday(&start, NULL); CHECK_RETURNVAL(PtlPut(potato_launcher_handle, 0, potato_launcher.length, (LOOPS == 1) ? PTL_OC_ACK_REQ : PTL_NO_ACK_REQ, nextrank, logical_pt_index, 1, 0, NULL, 1)); } { /* the potato-passing loop */ size_t waitfor; ptl_ct_event_t ctc; ptl_process_t nextrank; nextrank.rank = myself.rank + 1; nextrank.rank *= (nextrank.rank <= num_procs - 1); for (waitfor = 1; waitfor <= LOOPS; ++waitfor) { do { ptl_event_t event; CHECK_RETURNVAL(PtlEQWait(pt_eq_handle, &event)); // wait for potato if (event.type != PTL_EVENT_PUT) { printf("unexpected event: %i\n", (int)event.type); } else { break; } } while (1); /* I have the potato! */ ++potato; if (potato < LOOPS * (num_procs)) { // otherwise, the recipient may have exited /* Bomb's away! */ if (myself.rank == 0) { CHECK_RETURNVAL(PtlPut(potato_launcher_handle, 0, potato_launcher.length, (waitfor == (LOOPS - 1)) ? PTL_OC_ACK_REQ : PTL_NO_ACK_REQ, nextrank, logical_pt_index, 3, 0, NULL, 2)); } else { CHECK_RETURNVAL(PtlPut(potato_launcher_handle, 0, potato_launcher.length, (waitfor == LOOPS) ? PTL_OC_ACK_REQ : PTL_NO_ACK_REQ, nextrank, logical_pt_index, 3, 0, NULL, 2)); } } } // make sure that last send completed before exiting CHECK_RETURNVAL(PtlCTWait(potato_launcher.ct_handle, LOOPS + 1, &ctc)); assert(ctc.failure == 0); if (myself.rank == 0) { printf("Final value of potato = %i\n", potato); } } if (myself.rank == 0) { double accumulate = 0.0; gettimeofday(&stop, NULL); accumulate = (stop.tv_sec + stop.tv_usec * 1e-6) - (start.tv_sec + start.tv_usec * 1e-6); /* calculate the average time waiting */ printf("Total time: %g secs\n", accumulate); accumulate /= LOOPS; printf("Average time around the loop: %g microseconds\n", accumulate * 1e6); accumulate /= num_procs; printf("Average catch-to-toss latency: %g microseconds\n", accumulate * 1e6); } /* cleanup */ CHECK_RETURNVAL(PtlMDRelease(potato_launcher_handle)); CHECK_RETURNVAL(PtlCTFree(potato_launcher.ct_handle)); CHECK_RETURNVAL(UNLINK(potato_catcher_handle)); /* major cleanup */ CHECK_RETURNVAL(PtlPTFree(ni_logical, logical_pt_index)); CHECK_RETURNVAL(PtlEQFree(pt_eq_handle)); CHECK_RETURNVAL(PtlNIFini(ni_logical)); CHECK_RETURNVAL(libtest_fini()); PtlFini(); return 0; }
int main(int argc, char *argv[]) { ptl_handle_ni_t ni_handle; ptl_process_t *procs; int rank; ptl_pt_index_t pt_index, signal_pt_index; HANDLE_T value_e_handle, signal_e_handle; int num_procs; ptl_handle_eq_t eq_handle; ptl_handle_ct_t ct_handle; ptl_handle_md_t md_handle; CHECK_RETURNVAL(PtlInit()); CHECK_RETURNVAL(libtest_init()); rank = libtest_get_rank(); num_procs = libtest_get_size(); if (num_procs < 2) { fprintf(stderr, "test_flowctl_noeq requires at least two processes\n"); return 77; } CHECK_RETURNVAL(PtlNIInit(PTL_IFACE_DEFAULT, NI_TYPE | PTL_NI_LOGICAL, PTL_PID_ANY, NULL, NULL, &ni_handle)); procs = libtest_get_mapping(ni_handle); CHECK_RETURNVAL(PtlSetMap(ni_handle, num_procs, procs)); if (0 == rank) { ENTRY_T value_e; /* create data ME */ CHECK_RETURNVAL(PtlEQAlloc(ni_handle, (num_procs - 1) * ITERS / 2, &eq_handle)); CHECK_RETURNVAL(PtlPTAlloc(ni_handle, PTL_PT_FLOWCTRL, eq_handle, 5, &pt_index)); value_e.start = NULL; value_e.length = 0; value_e.ct_handle = PTL_CT_NONE; value_e.uid = PTL_UID_ANY; value_e.options = OPTIONS; #if INTERFACE == 1 value_e.match_id.rank = PTL_RANK_ANY; value_e.match_bits = 0; value_e.ignore_bits = 0; #endif CHECK_RETURNVAL(APPEND(ni_handle, 5, &value_e, PTL_PRIORITY_LIST, NULL, &value_e_handle)); /* create signal ME */ CHECK_RETURNVAL(PtlCTAlloc(ni_handle, &ct_handle)); CHECK_RETURNVAL(PtlPTAlloc(ni_handle, 0, PTL_EQ_NONE, 6, &signal_pt_index)); value_e.start = NULL; value_e.length = 0; value_e.ct_handle = ct_handle; value_e.uid = PTL_UID_ANY; value_e.options = OPTIONS | PTL_LE_EVENT_SUCCESS_DISABLE | PTL_LE_EVENT_CT_COMM; #if INTERFACE == 1 value_e.match_id.rank = PTL_RANK_ANY; value_e.match_bits = 0; value_e.ignore_bits = 0; #endif CHECK_RETURNVAL(APPEND(ni_handle, 6, &value_e, PTL_PRIORITY_LIST, NULL, &signal_e_handle)); } else { ptl_md_t md; /* 16 extra just in case... */ CHECK_RETURNVAL(PtlEQAlloc(ni_handle, ITERS * 2 + 16, &eq_handle)); md.start = NULL; md.length = 0; md.options = 0; md.eq_handle = eq_handle; md.ct_handle = PTL_CT_NONE; CHECK_RETURNVAL(PtlMDBind(ni_handle, &md, &md_handle)); } libtest_barrier(); if (0 == rank) { ptl_ct_event_t ct; ptl_event_t ev; int ret, count = 0, saw_dropped = 0, saw_flowctl = 0; /* wait for signal counts */ CHECK_RETURNVAL(PtlCTWait(ct_handle, num_procs - 1, &ct)); if (ct.success != num_procs - 1 || ct.failure != 0) { return 1; } /* wait for event entries */ while (count < ITERS * (num_procs - 1)) { ret = PtlEQWait(eq_handle, &ev); if (PTL_OK == ret) { ; } else if (PTL_EQ_DROPPED == ret) { saw_dropped++; if (ev.type == PTL_EVENT_PT_DISABLED){ saw_flowctl++; CHECK_RETURNVAL(PtlPTEnable(ni_handle, pt_index)); } break; } else { fprintf(stderr, "0: Unexpected return code from EQWait: %d\n", ret); return 1; } if (ev.type == PTL_EVENT_PT_DISABLED) { CHECK_RETURNVAL(PtlPTEnable(ni_handle, pt_index)); saw_flowctl++; } else { count++; } } fprintf(stderr, "0: Saw %d dropped, %d flowctl\n", saw_dropped, saw_flowctl); if (saw_flowctl == 0) { return 1; } } else { ptl_process_t target; ptl_event_t ev; int ret, count = 0, fails = 0; int i; int *fail_seen; fail_seen = malloc(sizeof(int) * ITERS); if (NULL == fail_seen) { fprintf(stderr, "%d: malloc failed\n", rank); return 1; } memset(fail_seen, 0, sizeof(int) * ITERS); target.rank = 0; for (i = 0 ; i < ITERS ; ++i) { CHECK_RETURNVAL(PtlPut(md_handle, 0, 0, PTL_ACK_REQ, target, 5, 0, 0, (void*)(size_t)i, 0)); usleep(100); } while (count < ITERS) { ret = PtlEQGet(eq_handle, &ev); if (PTL_EQ_EMPTY == ret) { continue; } else if (PTL_OK != ret) { fprintf(stderr, "%d: PtlEQGet returned %d\n", rank, ret); return 1; } if (ev.ni_fail_type == PTL_NI_OK) { if (ev.type == PTL_EVENT_SEND) { continue; } else if (ev.type == PTL_EVENT_ACK) { count++; } else { fprintf(stderr, "%d: Unexpected event type %d\n", rank, ev.type); } } else if (ev.ni_fail_type == PTL_NI_PT_DISABLED) { int iter = (size_t) ev.user_ptr; if (fail_seen[iter]++ > 0) { fprintf(stderr, "%d: Double report of PT_DISABLED for " "iteration %d\n", rank, iter); return 1; } count++; fails++; } else { fprintf(stderr, "%d: Unexpected fail type: %d\n", rank, ev.ni_fail_type); return 1; } } fprintf(stderr, "%d: Saw %d of %d events as fails\n", rank, fails, count); CHECK_RETURNVAL(PtlPut(md_handle, 0, 0, PTL_NO_ACK_REQ, target, 6, 0, 0, NULL, 0)); /* wait for the send event on the last put */ CHECK_RETURNVAL(PtlEQWait(eq_handle, &ev)); while (fails > 0) { CHECK_RETURNVAL(PtlPut(md_handle, 0, 0, PTL_ACK_REQ, target, 5, 0, 0, NULL, 0)); while (1) { ret = PtlEQWait(eq_handle, &ev); if (PTL_OK != ret) { fprintf(stderr, "%d: PtlEQWait returned %d\n", rank, ret); return 1; } if (ev.ni_fail_type == PTL_NI_OK) { if (ev.type == PTL_EVENT_SEND) { continue; } else if (ev.type == PTL_EVENT_ACK) { fails--; break; } else { fprintf(stderr, "%d: Unexpected event type %d\n", rank, ev.type); } } else if (ev.ni_fail_type == PTL_NI_PT_DISABLED) { break; } else { fprintf(stderr, "%d: Unexpected fail type: %d\n", rank, ev.ni_fail_type); return 1; } } } } libtest_barrier(); if (0 == rank) { CHECK_RETURNVAL(UNLINK(signal_e_handle)); CHECK_RETURNVAL(PtlPTFree(ni_handle, signal_pt_index)); CHECK_RETURNVAL(PtlCTFree(ct_handle)); CHECK_RETURNVAL(UNLINK(value_e_handle)); CHECK_RETURNVAL(PtlPTFree(ni_handle, pt_index)); CHECK_RETURNVAL(PtlEQFree(eq_handle)); } else { CHECK_RETURNVAL(PtlMDRelease(md_handle)); CHECK_RETURNVAL(PtlEQFree(eq_handle)); } CHECK_RETURNVAL(PtlNIFini(ni_handle)); CHECK_RETURNVAL(libtest_fini()); PtlFini(); return 0; }
void test_prepostME(int cache_size, int *cache_buf, ptl_handle_ni_t ni, int npeers, int nmsgs, int nbytes, int niters) { int i, j, k; double tmp, total = 0; ptl_handle_md_t send_md_handle; ptl_md_t send_md; ptl_process_t dest; ptl_size_t offset; ptl_pt_index_t index; ptl_handle_eq_t recv_eq_handle; ptl_handle_me_t me_handles[npeers * nmsgs]; ptl_event_t event; ptl_assert(PtlEQAlloc(ni, nmsgs * npeers + 1, &send_md.eq_handle), PTL_OK); send_md.start = send_buf; send_md.length = SEND_BUF_SIZE; send_md.options = PTL_MD_UNORDERED; send_md.ct_handle = PTL_CT_NONE; ptl_assert(PtlMDBind(ni, &send_md, &send_md_handle), PTL_OK); ptl_assert(PtlEQAlloc(ni, nmsgs * npeers + 1, &recv_eq_handle), PTL_OK); ptl_assert(PtlPTAlloc(ni, 0, recv_eq_handle, TestSameDirectionIndex, &index), PTL_OK); ptl_assert(TestSameDirectionIndex, index); tmp = timer(); for (j = 0; j < npeers; ++j) { for (k = 0; k < nmsgs; ++k) { ptl_process_t src; src.rank = recv_peers[j]; postME(ni, index, recv_buf + (nbytes * (k + j * nmsgs)), nbytes, src, magic_tag, &me_handles[k + j * nmsgs]); } } total += (timer() - tmp); for (i = 0; i < niters - 1; ++i) { cache_invalidate(cache_size, cache_buf); libtest_Barrier(); tmp = timer(); for (j = 0; j < npeers; ++j) { for (k = 0; k < nmsgs; ++k) { offset = (nbytes * (k + j * nmsgs)); dest.rank = send_peers[npeers - j - 1], ptl_assert(libtest_Put_offset(send_md_handle, offset, nbytes, dest, index, magic_tag, offset), PTL_OK); } } /* wait for sends */ for (j = 0; j < npeers * nmsgs; ++j) { ptl_assert(PtlEQWait(send_md.eq_handle, &event), PTL_OK); ptl_assert(event.type, PTL_EVENT_SEND); } /* wait for receives */ for (j = 0; j < npeers * nmsgs; j++) { PtlEQWait(recv_eq_handle, &event); } for (j = 0; j < npeers; ++j) { for (k = 0; k < nmsgs; ++k) { ptl_process_t src; src.rank = recv_peers[j]; postME(ni, index, recv_buf + (nbytes * (k + j * nmsgs)), nbytes, src, magic_tag, &me_handles[k + j * nmsgs]); } } total += (timer() - tmp); } libtest_Barrier(); tmp = timer(); for (j = 0; j < npeers; ++j) { for (k = 0; k < nmsgs; ++k) { offset = (nbytes * (k + j * nmsgs)); dest.rank = send_peers[npeers - j - 1], ptl_assert(libtest_Put_offset(send_md_handle, offset, nbytes, dest, index, magic_tag, offset), PTL_OK); } } /* wait for sends */ for (j = 0; j < npeers * nmsgs; ++j) { ptl_assert(PtlEQWait(send_md.eq_handle, &event), PTL_OK); ptl_assert(event.type, PTL_EVENT_SEND); } /* wait for receives */ for (j = 0; j < npeers * nmsgs; j++) { PtlEQWait(recv_eq_handle, &event); } total += (timer() - tmp); ptl_assert(PtlEQFree(send_md.eq_handle), PTL_OK); ptl_assert(PtlMDRelease(send_md_handle), PTL_OK); ptl_assert(PtlEQFree(recv_eq_handle), PTL_OK); ptl_assert(PtlPTFree(ni, index), PTL_OK); tmp = libtest_AllreduceDouble(total, PTL_SUM); display_result("pre-post", (niters * npeers * nmsgs * 2) / (tmp / world_size)); }
int main(int argc, char *argv[]) { ptl_handle_ni_t ni_h; ptl_pt_index_t pt_index; uint64_t *buf; ENTRY_T entry; HANDLE_T entry_h; ptl_md_t md; ptl_handle_md_t md_h; int rank; int num_procs; int ret; ptl_process_t *procs; ptl_handle_eq_t eq_h; ptl_event_t ev; ptl_hdr_data_t rcvd = 0; ptl_hdr_data_t goal = 0; ptl_hdr_data_t hdr_data = 1; ptl_size_t offset = sizeof(uint64_t); uint32_t distance; int sends = 0; CHECK_RETURNVAL(PtlInit()); CHECK_RETURNVAL(libtest_init()); rank = libtest_get_rank(); num_procs = libtest_get_size(); /* This test only succeeds if we have more than one rank */ if (num_procs < 2) return 77; CHECK_RETURNVAL(PtlNIInit(PTL_IFACE_DEFAULT, NI_TYPE | PTL_NI_LOGICAL, PTL_PID_ANY, NULL, NULL, &ni_h)); procs = libtest_get_mapping(ni_h); CHECK_RETURNVAL(PtlSetMap(ni_h, num_procs, procs)); CHECK_RETURNVAL(PtlEQAlloc(ni_h, 1024, &eq_h)); CHECK_RETURNVAL(PtlPTAlloc(ni_h, 0, eq_h, 0, &pt_index)); assert(pt_index == 0); buf = malloc(sizeof(uint64_t) * num_procs); assert(NULL != buf); md.start = buf; md.length = sizeof(uint64_t) * num_procs; md.options = PTL_MD_UNORDERED; md.eq_handle = eq_h; md.ct_handle = PTL_CT_NONE; CHECK_RETURNVAL(PtlMDBind(ni_h, &md, &md_h)); entry.start = buf; entry.length = sizeof(uint64_t) * num_procs; entry.ct_handle = PTL_CT_NONE; entry.uid = PTL_UID_ANY; entry.options = OPTIONS; #if MATCHING == 1 entry.match_id.rank = PTL_RANK_ANY; entry.match_bits = 0; entry.ignore_bits = 0; entry.min_free = 0; #endif CHECK_RETURNVAL(APPEND(ni_h, pt_index, &entry, PTL_PRIORITY_LIST, NULL, &entry_h)); /* ensure ME is linked before the barrier */ CHECK_RETURNVAL(PtlEQWait(eq_h, &ev)); assert( ev.type == PTL_EVENT_LINK ); libtest_barrier(); /* Bruck's Concatenation Algorithm */ memcpy(buf, &rank, sizeof(uint64_t)); for (distance = 1; distance < num_procs; distance *= 2) { ptl_size_t to_xfer; int peer; ptl_process_t proc; if (rank >= distance) { peer = rank - distance; } else { peer = rank + (num_procs - distance); } to_xfer = sizeof(uint64_t) * MIN(distance, num_procs - distance); proc.rank = peer; CHECK_RETURNVAL(PtlPut(md_h, 0, to_xfer, PTL_NO_ACK_REQ, proc, 0, 0, offset, NULL, hdr_data)); sends += 1; /* wait for completion of the proper receive, and keep count of uncompleted sends. "rcvd" is an accumulator to deal with out-of-order receives, which are IDed by the hdr_data */ goal |= hdr_data; while ((rcvd & goal) != goal) { ret = PtlEQWait(eq_h, &ev); switch (ret) { case PTL_OK: if (ev.type == PTL_EVENT_SEND) { sends -= 1; } else { rcvd |= ev.hdr_data; assert(ev.type == PTL_EVENT_PUT); assert(ev.rlength == ev.mlength); assert((ev.rlength == to_xfer) || (ev.hdr_data != hdr_data)); } break; default: fprintf(stderr, "PtlEQWait failure: %d\n", ret); abort(); } } hdr_data <<= 1; offset += to_xfer; } /* wait for any SEND_END events not yet seen */ while (sends) { ret = PtlEQWait(eq_h, &ev); switch (ret) { case PTL_OK: assert( ev.type == PTL_EVENT_SEND ); sends -= 1; break; default: fprintf(stderr, "PtlEQWait failure: %d\n", ret); abort(); } } CHECK_RETURNVAL(UNLINK(entry_h)); CHECK_RETURNVAL(PtlMDRelease(md_h)); free(buf); libtest_barrier(); /* cleanup */ CHECK_RETURNVAL(PtlPTFree(ni_h, pt_index)); CHECK_RETURNVAL(PtlEQFree(eq_h)); CHECK_RETURNVAL(PtlNIFini(ni_h)); CHECK_RETURNVAL(libtest_fini()); PtlFini(); return 0; }