int
ibv_close_device(struct ibv_context *context)
{
JNOTE("***** WRAPPER for ibv_close_device");
return _real_ibv_close_device(context);
}
struct ibv_pd *
ibv_alloc_pd(struct ibv_context *context)
{
JNOTE("******* WRAPPER FOR ibv_alloc_pd");
return _real_ibv_alloc_pd(context);
}
int
ibv_destroy_comp_channel(struct ibv_comp_channel *channel)
{
JNOTE("****** WRAPPER for ibv_destroy_comp_channel");
return _real_ibv_destroy_comp_channel(channel);
}
int
ibv_dealloc_pd(struct ibv_pd *pd)
{
JNOTE("****** WRAPPER for ibv_dealloc_pd");
return _real_ibv_dealloc_pd(pd);
}
uint64_t
ibv_get_device_guid(struct ibv_device *device)
{
JNOTE("******* WRAPPER for ibv_get_device_guid");
return _real_ibv_get_device_guid(device);
}
const char *
ibv_get_device_name(struct ibv_device *dev)
{
JNOTE("****** WRAPPER for ibv_get_device_name");
return _real_ibv_get_device_name(dev);
}
struct ibv_context *
ibv_open_device(struct ibv_device *dev)
{
JNOTE("******* WRAPPER for begin of ibv_open_device");
return _real_ibv_open_device(dev);
}
struct ibv_qp *
ibv_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
{
JNOTE("******** WRAPPER for ibv_create_qp");
return _real_ibv_create_qp(pd, qp_init_attr);
}
void
ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
{
JNOTE("******** WRAPPER for ibv_ack_cq_events");
_real_ibv_ack_cq_events(cq, nevents);
}
int
ibv_destroy_qp(struct ibv_qp *qp)
{
JNOTE("****** WRAPPER2 for ibv_destroy qp");
return _real_ibv_destroy_qp(qp);
}
int
ibv_get_async_event(struct ibv_context *context, struct ibv_async_event *event)
{
JNOTE("******** WRAPPER FOR ibv_get_async_event");
return _real_ibv_get_async_event(context, event);
}
int
ibv_dereg_mr(struct ibv_mr *mr)
{
JNOTE("****** WRAPPER for ibv_dereg_mr");
return _real_ibv_dereg_mr(mr);
}
int
ibv_destroy_cq(struct ibv_cq *cq)
{
JNOTE("****** WRAPPER for ibv_destroy_cq");
return _real_ibv_destroy_cq(cq);
}
int
ibv_resize_cq(struct ibv_cq *cq, int cqe)
{
JNOTE("****** WRAPPER for ibv_resize_cq");
return _real_ibv_resize_cq(cq, cqe);
}
void
ibv_ack_async_event(struct ibv_async_event *event)
{
JNOTE("******* WRAPPER FOR ibv_ack_async_event");
_real_ibv_ack_async_event(event);
}
void
ibv_free_device_list(struct ibv_device **list)
{
JNOTE("********* WRAPPER for ibv_free_device_list");
_real_ibv_free_device_list(list);
}
struct ibv_comp_channel *
ibv_create_comp_channel(struct ibv_context
*context)
{
JNOTE("******* WRAPPER for ibv_create_comp_channel");
return _real_ibv_create_comp_channel(context);
}
int
ibv_query_device(struct ibv_context *context,
struct ibv_device_attr *device_attr)
{
JNOTE("******* WRAPPER for begin of ibv_query_device");
return _real_ibv_query_device(context, device_attr);
}
struct ibv_mr *
ibv_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access) // int
// access)
{
JNOTE("******** WRAPPER for ibv_reg_mr");
return _real_ibv_reg_mr(pd, addr, length, access);
}
int
ibv_query_gid(struct ibv_context *context,
uint8_t port_num,
int index,
union ibv_gid *gid)
{
JNOTE("******* WRAPPER for begin of ibv_query_gid");
return _real_ibv_query_gid(context, port_num, index, gid);
}
int
ibv_query_qp(struct ibv_qp *qp,
struct ibv_qp_attr *attr,
int attr_mask,
struct ibv_qp_init_attr *init_attr)
{
JNOTE("******** WRAPPER FOR ibv_query_qp");
return _real_ibv_query_qp(qp, attr, attr_mask, init_attr);
}
int
ibv_query_pkey(struct ibv_context *context,
uint8_t port_num,
int index,
uint16_t *pkey)
{
JNOTE("******* WRAPPER for begin of ibv_query_pkey");
return ibv_query_pkey(context, port_num, index, pkey);
}
DmtcpMessage
sendRecvHandshake(int fd,
DmtcpMessage msg,
string progname,
UniquePid *compId)
{
if (dmtcp_virtual_to_real_pid) {
msg.realPid = dmtcp_virtual_to_real_pid(getpid());
} else {
msg.realPid = getpid();
}
msg.theCheckpointInterval = getCkptInterval();
string hostname = jalib::Filesystem::GetCurrentHostname();
size_t buflen = hostname.length() + progname.length() + 2;
char buf[buflen];
strcpy(buf, hostname.c_str());
strcpy(&buf[hostname.length() + 1], progname.c_str());
sendMsgToCoordinatorRaw(fd, msg, buf, buflen);
recvMsgFromCoordinatorRaw(fd, &msg);
msg.assertValid();
if (msg.type == DMT_KILL_PEER) {
JTRACE("Received KILL message from coordinator, exiting");
_real_exit(0);
}
if (msg.type == DMT_REJECT_NOT_RUNNING) {
JASSERT(false)
.Text("Connection rejected by the coordinator.\n"
"Reason: Current computation not in RUNNING state.\n"
" Is a checkpoint/restart in progress?");
} else if (msg.type == DMT_REJECT_WRONG_COMP) {
JASSERT(compId != NULL);
JASSERT(false) (*compId)
.Text("Connection rejected by the coordinator.\n"
" Reason: This process has a different computation group.");
}
// Coordinator also prints this, but its stderr may go to /dev/null
if (msg.type == DMT_REJECT_NOT_RESTARTING) {
string coordinatorHost = ""; // C++ magic code; "" to be invisibly replaced
int coordinatorPort;
getCoordHostAndPort(COORD_ANY, coordinatorHost, &coordinatorPort);
JNOTE ("\n\n*** Computation not in RESTARTING or CHECKPOINTED state."
"\n***Can't join the existing coordinator, as it is serving a"
"\n***different computation. Consider launching a new coordinator."
"\n***Consider, also, checking with: dmtcp_command --status")
(coordinatorPort);
}
JASSERT(msg.type == DMT_ACCEPT)(msg.type);
return msg;
}
struct ibv_cq *
ibv_create_cq(struct ibv_context *context,
int cqe,
void *cq_context,
struct ibv_comp_channel *channel,
int comp_vector)
{
JNOTE("******** WRAPPER for ibv_create_cq");
return _real_ibv_create_cq(context, cqe, cq_context, channel, comp_vector);
}
void dmtcp::ProcessInfo::restoreHeap()
{
/* If the original start of heap is lower than the current end of heap, we
* want to mmap the area between _savedBrk and current break. This
* happens when the size of checkpointed program is smaller then the size of
* mtcp_restart program.
*/
VA curBrk = (char*) sbrk(0);
if ((uint64_t) curBrk > _savedBrk) {
JNOTE("Area between saved_break and curr_break not mapped, mapping it now")
(_savedBrk) (curBrk);
size_t oldsize = _savedBrk - _savedHeapStart;
size_t newsize = (size_t) (curBrk - _savedHeapStart);
JASSERT(mremap((void*) _savedHeapStart, oldsize, newsize, 0) != NULL)
(_savedBrk) (curBrk)
.Text("mremap failed to map area between saved break and current break");
} else if ((uint64_t) curBrk < _savedBrk) {
if (brk((void*)_savedBrk) != 0) {
JNOTE("Failed to restore area between saved_break and curr_break.")
(_savedBrk) (curBrk) (JASSERT_ERRNO);
}
}
}
int findLibTorque_maps(dmtcp::string &libpath)
{
// /proc/self/maps looks like: "<start addr>-<end addr> <mode> <offset> <device> <inode> <libpath>
// we need to extract libpath
dmtcp::Util::ProcMapsArea area;
int ret = -1;
// we will search for first libpath and first libname
int fd = _real_open ( "/proc/self/maps", O_RDONLY);
if( fd < 0 ){
JTRACE("Cannot open /proc/self/maps file");
return -1;
}
while( dmtcp::Util::readProcMapsLine(fd, &area) ){
libpath = area.name;
JNOTE("Inspect new /proc/seft/maps line")(libpath);
if( libpath.size() == 0 ){
JNOTE("anonymous region, skip");
continue;
}
if( libpath.find("libtorque") != dmtcp::string::npos ){
// this is library path that contains libtorque. This is what we need
JTRACE("Torque PBS libpath")(libpath);
ret = 0;
break;
}else{
JNOTE("Not a libtorque region")(libpath);
}
}
_real_close(fd);
return ret;
}
int
ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask) // int
// attr_mask)
{
JNOTE("********* WRAPPER for ibv_modify_qp");
attr->rq_psn = attr->rq_psn & 0xffffff;
attr->sq_psn = attr->sq_psn & 0xffffff;
if (attr_mask & IBV_QP_RQ_PSN) {
printf("1RQ_PSN is %x\n", attr->rq_psn);
}
if (attr_mask & IBV_QP_SQ_PSN) {
printf("1SQ_PSN is %x\n", attr->sq_psn);
}
if (attr_mask & IBV_QP_STATE) {
if (attr->qp_state == IBV_QPS_RTR) {
printf("RQ_PSN is %x\n", attr->rq_psn);
} else if (attr->qp_state == IBV_QPS_RTS) {
printf("SQ_PSN is %x\n", attr->sq_psn);
}
}
int rslt = _real_ibv_modify_qp(qp, attr, attr_mask);
struct ibv_qp_attr query_attr;
struct ibv_qp_init_attr init_attr;
int foo = _real_ibv_query_qp(qp, &query_attr, attr_mask, &init_attr);
if (attr_mask & IBV_QP_RQ_PSN) {
printf("attr.rq_psn is %x\n", query_attr.rq_psn);
}
if (attr_mask & IBV_QP_SQ_PSN) {
printf("attr.sq_psn is %x\n", query_attr.sq_psn);
}
if (attr_mask & IBV_QP_STATE) {
printf("qp state set attr.sq_psn is %x\n", query_attr.sq_psn);
printf("qp state set attr.rq_psn is %x\n", query_attr.rq_psn);
}
return rslt;
}
