static int psmx_mr_regv(struct fid_domain *domain,
const struct iovec *iov, size_t count,
uint64_t access, uint64_t offset, uint64_t requested_key,
uint64_t flags, struct fid_mr **mr, void *context)
{
struct psmx_fid_domain *fid_domain;
struct psmx_fid_mr *fid_mr;
int i;
uint64_t key;
if (requested_key != PSMX_MR_AUTO_KEY && psmx_mr_hash_get(requested_key))
return -EEXIST;
fid_domain = container_of(domain, struct psmx_fid_domain, domain);
if (count == 0 || iov == NULL)
return -EINVAL;
fid_mr = (struct psmx_fid_mr *)
calloc(1, sizeof(*fid_mr) +
sizeof(struct iovec) * count);
if (!fid_mr)
return -ENOMEM;
fid_mr->mr.fid.fclass = FID_CLASS_MR;
fid_mr->mr.fid.context = context;
fid_mr->mr.fid.ops = &psmx_fi_ops;
fid_mr->mr.mem_desc = fid_mr;
if (requested_key != PSMX_MR_AUTO_KEY) {
key = requested_key;
}
else {
key = (uint64_t)(uintptr_t)fid_mr;
while (psmx_mr_hash_get(key))
key++;
}
fid_mr->mr.key = key;
fid_mr->domain = fid_domain;
fid_mr->access = access;
fid_mr->flags = flags;
fid_mr->iov_count = count;
for (i=0; i<count; i++)
fid_mr->iov[i] = iov[i];
psmx_mr_normalize_iov(fid_mr->iov, &fid_mr->iov_count);
psmx_mr_hash_add(fid_mr);
*mr = &fid_mr->mr;
return 0;
}
static int psmx_mr_reg(struct fid_domain *domain, const void *buf, size_t len,
uint64_t access, uint64_t offset, uint64_t requested_key,
uint64_t flags, struct fid_mr **mr, void *context)
{
struct psmx_fid_domain *fid_domain;
struct psmx_fid_mr *fid_mr;
uint64_t key;
if (requested_key != PSMX_MR_AUTO_KEY && psmx_mr_hash_get(requested_key))
return -EEXIST;
fid_domain = container_of(domain, struct psmx_fid_domain, domain);
fid_mr = (struct psmx_fid_mr *) calloc(1, sizeof(*fid_mr) + sizeof(struct iovec));
if (!fid_mr)
return -ENOMEM;
fid_mr->mr.fid.fclass = FID_CLASS_MR;
fid_mr->mr.fid.context = context;
fid_mr->mr.fid.ops = &psmx_fi_ops;
fid_mr->mr.mem_desc = fid_mr;
if (requested_key != PSMX_MR_AUTO_KEY) {
key = requested_key;
}
else {
key = (uint64_t)(uintptr_t)fid_mr;
while (psmx_mr_hash_get(key))
key++;
}
fid_mr->mr.key = key;
fid_mr->domain = fid_domain;
fid_mr->access = access;
fid_mr->flags = flags;
fid_mr->iov_count = 1;
fid_mr->iov[0].iov_base = (void *)buf;
fid_mr->iov[0].iov_len = len;
psmx_mr_hash_add(fid_mr);
*mr = &fid_mr->mr;
return 0;
}
int psmx_am_rma_handler(psm_am_token_t token, psm_epaddr_t epaddr,
psm_amarg_t *args, int nargs, void *src, uint32_t len)
{
psm_amarg_t rep_args[8];
void *rma_addr;
ssize_t rma_len;
uint64_t key;
int err = 0;
int op_error = 0;
int cmd, eom, has_data;
struct psmx_am_request *req;
struct psmx_cq_event *event;
int chunk_size;
uint64_t offset;
struct psmx_fid_mr *mr;
cmd = args[0].u32w0 & PSMX_AM_OP_MASK;
eom = args[0].u32w0 & PSMX_AM_EOM;
has_data = args[0].u32w0 & PSMX_AM_DATA;
switch (cmd) {
case PSMX_AM_REQ_WRITE:
rma_len = args[0].u32w1;
rma_addr = (void *)(uintptr_t)args[2].u64;
key = args[3].u64;
mr = psmx_mr_hash_get(key);
op_error = mr ?
psmx_mr_validate(mr, (uint64_t)rma_addr, len, FI_REMOTE_WRITE) :
-EINVAL;
if (!op_error) {
rma_addr += mr->offset;
memcpy(rma_addr, src, len);
if (eom) {
if (mr->cq) {
/* TODO: report the addr/len of the whole write */
event = psmx_cq_create_event(
mr->cq,
0, /* context */
rma_addr,
0, /* flags */
rma_len,
has_data ? args[4].u64 : 0,
0, /* tag */
0, /* olen */
0);
if (event)
psmx_cq_enqueue_event(mr->cq, event);
else
err = -ENOMEM;
}
if (mr->cntr)
psmx_cntr_inc(mr->cntr);
if (mr->domain->rma_ep->remote_write_cntr)
psmx_cntr_inc(mr->domain->rma_ep->remote_write_cntr);
}
}
if (eom || op_error) {
rep_args[0].u32w0 = PSMX_AM_REP_WRITE | eom;
rep_args[0].u32w1 = op_error;
rep_args[1].u64 = args[1].u64;
err = psm_am_reply_short(token, PSMX_AM_RMA_HANDLER,
rep_args, 2, NULL, 0, 0,
NULL, NULL );
}
break;
case PSMX_AM_REQ_WRITE_LONG:
rma_len = args[0].u32w1;
rma_addr = (void *)(uintptr_t)args[2].u64;
key = args[3].u64;
mr = psmx_mr_hash_get(key);
op_error = mr ?
psmx_mr_validate(mr, (uint64_t)rma_addr, len, FI_REMOTE_WRITE) :
-EINVAL;
if (op_error) {
rep_args[0].u32w0 = PSMX_AM_REP_WRITE | eom;
rep_args[0].u32w1 = op_error;
rep_args[1].u64 = args[1].u64;
err = psm_am_reply_short(token, PSMX_AM_RMA_HANDLER,
rep_args, 2, NULL, 0, 0,
NULL, NULL );
break;
}
rma_addr += mr->offset;
req = calloc(1, sizeof(*req));
if (!req) {
err = -ENOMEM;
}
else {
req->op = args[0].u32w0;
req->write.addr = (uint64_t)rma_addr;
req->write.len = rma_len;
req->write.key = key;
req->write.context = (void *)args[4].u64;
req->write.data = has_data ? args[5].u64 : 0;
PSMX_CTXT_TYPE(&req->fi_context) = PSMX_REMOTE_WRITE_CONTEXT;
PSMX_CTXT_USER(&req->fi_context) = mr;
psmx_am_enqueue_rma(mr->domain, req);
}
break;
case PSMX_AM_REQ_READ:
rma_len = args[0].u32w1;
rma_addr = (void *)(uintptr_t)args[2].u64;
key = args[3].u64;
offset = args[4].u64;
mr = psmx_mr_hash_get(key);
op_error = mr ?
psmx_mr_validate(mr, (uint64_t)rma_addr, rma_len, FI_REMOTE_READ) :
-EINVAL;
if (!op_error) {
rma_addr += mr->offset;
}
else {
rma_addr = NULL;
rma_len = 0;
}
chunk_size = MIN(PSMX_AM_CHUNK_SIZE, psmx_am_param.max_reply_short);
assert(rma_len <= chunk_size);
rep_args[0].u32w0 = PSMX_AM_REP_READ | eom;
rep_args[0].u32w1 = op_error;
rep_args[1].u64 = args[1].u64;
rep_args[2].u64 = offset;
err = psm_am_reply_short(token, PSMX_AM_RMA_HANDLER,
rep_args, 3, rma_addr, rma_len, 0,
NULL, NULL );
if (eom && !op_error) {
if (mr->domain->rma_ep->remote_read_cntr)
psmx_cntr_inc(mr->domain->rma_ep->remote_read_cntr);
}
break;
case PSMX_AM_REQ_READ_LONG:
rma_len = args[0].u32w1;
rma_addr = (void *)(uintptr_t)args[2].u64;
key = args[3].u64;
mr = psmx_mr_hash_get(key);
op_error = mr ?
psmx_mr_validate(mr, (uint64_t)rma_addr, len, FI_REMOTE_WRITE) :
-EINVAL;
if (op_error) {
rep_args[0].u32w0 = PSMX_AM_REP_READ | eom;
rep_args[0].u32w1 = op_error;
rep_args[1].u64 = args[1].u64;
rep_args[2].u64 = 0;
err = psm_am_reply_short(token, PSMX_AM_RMA_HANDLER,
rep_args, 3, NULL, 0, 0,
NULL, NULL );
break;
}
rma_addr += mr->offset;
req = calloc(1, sizeof(*req));
if (!req) {
err = -ENOMEM;
}
else {
req->op = args[0].u32w0;
req->read.addr = (uint64_t)rma_addr;
req->read.len = rma_len;
req->read.key = key;
req->read.context = (void *)args[4].u64;
req->read.peer_addr = (void *)epaddr;
PSMX_CTXT_TYPE(&req->fi_context) = PSMX_REMOTE_READ_CONTEXT;
PSMX_CTXT_USER(&req->fi_context) = mr;
psmx_am_enqueue_rma(mr->domain, req);
}
break;
case PSMX_AM_REP_WRITE:
req = (struct psmx_am_request *)(uintptr_t)args[1].u64;
assert(req->op == PSMX_AM_REQ_WRITE);
op_error = (int)args[0].u32w1;
if (!req->error)
req->error = op_error;
if (eom) {
if (req->ep->send_cq && !req->no_event) {
event = psmx_cq_create_event(
req->ep->send_cq,
req->write.context,
req->write.buf,
0, /* flags */
req->write.len,
0, /* data */
0, /* tag */
0, /* olen */
req->error);
if (event)
psmx_cq_enqueue_event(req->ep->send_cq, event);
else
err = -ENOMEM;
}
if (req->ep->write_cntr)
psmx_cntr_inc(req->ep->write_cntr);
free(req);
}
break;
case PSMX_AM_REP_READ:
req = (struct psmx_am_request *)(uintptr_t)args[1].u64;
assert(req->op == PSMX_AM_REQ_READ);
op_error = (int)args[0].u32w1;
offset = args[2].u64;
if (!req->error)
req->error = op_error;
if (!op_error) {
memcpy(req->read.buf + offset, src, len);
req->read.len_read += len;
}
if (eom) {
if (req->ep->send_cq && !req->no_event) {
event = psmx_cq_create_event(
req->ep->send_cq,
req->read.context,
req->read.buf,
0, /* flags */
req->read.len_read,
0, /* data */
0, /* tag */
req->read.len - req->read.len_read,
req->error);
if (event)
psmx_cq_enqueue_event(req->ep->send_cq, event);
else
err = -ENOMEM;
}
if (req->ep->read_cntr)
psmx_cntr_inc(req->ep->read_cntr);
free(req);
}
break;
default:
err = -EINVAL;
}
return err;
}
static ssize_t psmx_rma_self(int am_cmd,
struct psmx_fid_ep *ep,
void *buf, size_t len, void *desc,
uint64_t addr, uint64_t key,
void *context, uint64_t flags, uint64_t data)
{
struct psmx_fid_mr *mr;
struct psmx_cq_event *event;
struct psmx_fid_cntr *cntr;
int no_event;
int err = 0;
int op_error = 0;
int access;
void *dst, *src;
switch (am_cmd) {
case PSMX_AM_REQ_WRITE:
access = FI_REMOTE_WRITE;
break;
case PSMX_AM_REQ_READ:
access = FI_REMOTE_READ;
break;
default:
return -EINVAL;
}
mr = psmx_mr_hash_get(key);
op_error = mr ? psmx_mr_validate(mr, addr, len, access) : -EINVAL;
if (!op_error) {
addr += mr->offset;
if (am_cmd == PSMX_AM_REQ_WRITE) {
dst = (void *)addr;
src = buf;
cntr = mr->domain->rma_ep->remote_write_cntr;
}
else {
dst = buf;
src = (void *)addr;
cntr = mr->domain->rma_ep->remote_read_cntr;
}
memcpy(dst, src, len);
if (mr->cq && am_cmd == PSMX_AM_REQ_WRITE) {
event = psmx_cq_create_event(
mr->cq,
0, /* context */
(void *)addr,
0, /* flags */
len,
flags & FI_REMOTE_CQ_DATA ? data : 0,
0, /* tag */
0, /* olen */
0 /* err */);
if (event)
psmx_cq_enqueue_event(mr->cq, event);
else
err = -ENOMEM;
}
if (mr->cntr && am_cmd == PSMX_AM_REQ_WRITE)
psmx_cntr_inc(mr->cntr);
if (cntr)
psmx_cntr_inc(cntr);
}
no_event = (flags & FI_INJECT) ||
(ep->send_cq_event_flag && !(flags & FI_EVENT));
if (ep->send_cq && !no_event) {
event = psmx_cq_create_event(
ep->send_cq,
context,
(void *)buf,
0, /* flags */
len,
0, /* data */
0, /* tag */
0, /* olen */
op_error);
if (event)
psmx_cq_enqueue_event(ep->send_cq, event);
else
err = -ENOMEM;
}
switch (am_cmd) {
case PSMX_AM_REQ_WRITE:
if (ep->write_cntr)
psmx_cntr_inc(ep->write_cntr);
break;
case PSMX_AM_REQ_READ:
if (ep->read_cntr)
psmx_cntr_inc(ep->read_cntr);
break;
}
return err;
}
static int psmx_atomic_self(int am_cmd,
struct psmx_fid_ep *ep,
const void *buf,
size_t count, void *desc,
const void *compare, void *compare_desc,
void *result, void *result_desc,
uint64_t addr, uint64_t key,
enum fi_datatype datatype,
enum fi_op op, void *context,
uint64_t flags)
{
struct psmx_fid_mr *mr;
struct psmx_cq_event *event;
struct psmx_fid_ep *target_ep;
size_t len;
int no_event;
int err = 0;
int op_error;
int access;
if (am_cmd == PSMX_AM_REQ_ATOMIC_WRITE)
access = FI_REMOTE_WRITE;
else
access = FI_REMOTE_READ | FI_REMOTE_WRITE;
len = fi_datatype_size(datatype) * count;
mr = psmx_mr_hash_get(key);
op_error = mr ? psmx_mr_validate(mr, addr, len, access) : -EINVAL;
if (op_error)
goto gen_local_event;
addr += mr->offset;
switch (am_cmd) {
case PSMX_AM_REQ_ATOMIC_WRITE:
err = psmx_atomic_do_write((void *)addr, (void *)buf,
(int)datatype, (int)op, (int)count);
break;
case PSMX_AM_REQ_ATOMIC_READWRITE:
err = psmx_atomic_do_readwrite((void *)addr, (void *)buf,
(void *)result, (int)datatype,
(int)op, (int)count);
break;
case PSMX_AM_REQ_ATOMIC_COMPWRITE:
err = psmx_atomic_do_compwrite((void *)addr, (void *)buf,
(void *)compare, (void *)result,
(int)datatype, (int)op, (int)count);
break;
}
if (op != FI_ATOMIC_READ) {
if (mr->cq) {
event = psmx_cq_create_event(
mr->cq,
0, /* context */
(void *)addr,
0, /* flags */
len,
0, /* data */
0, /* tag */
0, /* olen */
0 /* err */);
if (event)
psmx_cq_enqueue_event(mr->cq, event);
else
err = -ENOMEM;
}
if (mr->cntr)
psmx_cntr_inc(mr->cntr);
}
target_ep = mr->domain->atomics_ep;
if (op == FI_ATOMIC_WRITE) {
if (target_ep->remote_write_cntr)
psmx_cntr_inc(target_ep->remote_write_cntr);
}
else if (op == FI_ATOMIC_READ) {
if (target_ep->remote_read_cntr)
psmx_cntr_inc(target_ep->remote_read_cntr);
}
else {
if (target_ep->remote_write_cntr)
psmx_cntr_inc(target_ep->remote_write_cntr);
if (am_cmd != PSMX_AM_REQ_ATOMIC_WRITE &&
target_ep->remote_read_cntr &&
target_ep->remote_read_cntr != target_ep->remote_write_cntr)
psmx_cntr_inc(target_ep->remote_read_cntr);
}
gen_local_event:
no_event = ((flags & FI_INJECT) ||
(ep->send_cq_event_flag && !(flags & FI_EVENT)));
if (ep->send_cq && !no_event) {
event = psmx_cq_create_event(
ep->send_cq,
context,
(void *)buf,
0, /* flags */
len,
0, /* data */
0, /* tag */
0, /* olen */
op_error);
if (event)
psmx_cq_enqueue_event(ep->send_cq, event);
else
err = -ENOMEM;
}
switch (am_cmd) {
case PSMX_AM_REQ_ATOMIC_WRITE:
if (ep->write_cntr)
psmx_cntr_inc(ep->write_cntr);
break;
case PSMX_AM_REQ_ATOMIC_READWRITE:
case PSMX_AM_REQ_ATOMIC_COMPWRITE:
if (ep->read_cntr)
psmx_cntr_inc(ep->read_cntr);
break;
}
return err;
}
int psmx_am_atomic_handler(psm_am_token_t token, psm_epaddr_t epaddr,
psm_amarg_t *args, int nargs, void *src,
uint32_t len)
{
psm_amarg_t rep_args[8];
int count;
void *addr;
uint64_t key;
int datatype, op;
int err = 0;
int op_error = 0;
struct psmx_am_request *req;
struct psmx_cq_event *event;
struct psmx_fid_mr *mr;
struct psmx_fid_ep *target_ep;
void *tmp_buf;
switch (args[0].u32w0 & PSMX_AM_OP_MASK) {
case PSMX_AM_REQ_ATOMIC_WRITE:
count = args[0].u32w1;
addr = (void *)(uintptr_t)args[2].u64;
key = args[3].u64;
datatype = args[4].u32w0;
op = args[4].u32w1;
assert(len == fi_datatype_size(datatype) * count);
mr = psmx_mr_hash_get(key);
op_error = mr ?
psmx_mr_validate(mr, (uint64_t)addr, len, FI_REMOTE_WRITE) :
-EINVAL;
if (!op_error) {
addr += mr->offset;
psmx_atomic_do_write(addr, src, datatype, op, count);
if (mr->cq) {
event = psmx_cq_create_event(
mr->cq,
0, /* context */
addr,
0, /* flags */
len,
0, /* data */
0, /* tag */
0, /* olen */
0 /* err */);
if (event)
psmx_cq_enqueue_event(mr->cq, event);
else
err = -ENOMEM;
}
if (mr->cntr)
psmx_cntr_inc(mr->cntr);
target_ep = mr->domain->atomics_ep;
if (target_ep->remote_write_cntr)
psmx_cntr_inc(target_ep->remote_write_cntr);
}
rep_args[0].u32w0 = PSMX_AM_REP_ATOMIC_WRITE;
rep_args[0].u32w1 = op_error;
rep_args[1].u64 = args[1].u64;
err = psm_am_reply_short(token, PSMX_AM_ATOMIC_HANDLER,
rep_args, 2, NULL, 0, 0,
NULL, NULL );
break;
case PSMX_AM_REQ_ATOMIC_READWRITE:
count = args[0].u32w1;
addr = (void *)(uintptr_t)args[2].u64;
key = args[3].u64;
datatype = args[4].u32w0;
op = args[4].u32w1;
if (op == FI_ATOMIC_READ)
len = fi_datatype_size(datatype) * count;
assert(len == fi_datatype_size(datatype) * count);
mr = psmx_mr_hash_get(key);
op_error = mr ?
psmx_mr_validate(mr, (uint64_t)addr, len, FI_REMOTE_READ|FI_REMOTE_WRITE) :
-EINVAL;
if (!op_error) {
addr += mr->offset;
tmp_buf = malloc(len);
if (tmp_buf)
psmx_atomic_do_readwrite(addr, src, tmp_buf,
datatype, op, count);
else
err = -ENOMEM;
if (op != FI_ATOMIC_READ) {
if (mr->cq) {
event = psmx_cq_create_event(
mr->cq,
0, /* context */
addr,
0, /* flags */
len,
0, /* data */
0, /* tag */
0, /* olen */
0 /* err */);
if (event)
psmx_cq_enqueue_event(mr->cq, event);
else
err = -ENOMEM;
}
if (mr->cntr)
psmx_cntr_inc(mr->cntr);
}
target_ep = mr->domain->atomics_ep;
if (op == FI_ATOMIC_WRITE) {
if (target_ep->remote_write_cntr)
psmx_cntr_inc(target_ep->remote_write_cntr);
}
else if (op == FI_ATOMIC_READ) {
if (target_ep->remote_read_cntr)
psmx_cntr_inc(target_ep->remote_read_cntr);
}
else {
if (target_ep->remote_write_cntr)
psmx_cntr_inc(target_ep->remote_write_cntr);
if (target_ep->remote_read_cntr &&
target_ep->remote_read_cntr != target_ep->remote_write_cntr)
psmx_cntr_inc(target_ep->remote_read_cntr);
}
}
else {
tmp_buf = NULL;
}
rep_args[0].u32w0 = PSMX_AM_REP_ATOMIC_READWRITE;
rep_args[0].u32w1 = op_error;
rep_args[1].u64 = args[1].u64;
err = psm_am_reply_short(token, PSMX_AM_ATOMIC_HANDLER,
rep_args, 2, tmp_buf, (tmp_buf?len:0), 0,
psmx_am_atomic_completion, tmp_buf );
break;
case PSMX_AM_REQ_ATOMIC_COMPWRITE:
count = args[0].u32w1;
addr = (void *)(uintptr_t)args[2].u64;
key = args[3].u64;
datatype = args[4].u32w0;
op = args[4].u32w1;
len /= 2;
assert(len == fi_datatype_size(datatype) * count);
mr = psmx_mr_hash_get(key);
op_error = mr ?
psmx_mr_validate(mr, (uint64_t)addr, len, FI_REMOTE_READ|FI_REMOTE_WRITE) :
-EINVAL;
if (!op_error) {
addr += mr->offset;
tmp_buf = malloc(len);
if (tmp_buf)
psmx_atomic_do_compwrite(addr, src, src + len,
tmp_buf, datatype, op, count);
else
err = -ENOMEM;
if (mr->cq) {
event = psmx_cq_create_event(
mr->cq,
0, /* context */
addr,
0, /* flags */
len,
0, /* data */
0, /* tag */
0, /* olen */
0 /* err */);
if (event)
psmx_cq_enqueue_event(mr->cq, event);
else
err = -ENOMEM;
}
if (mr->cntr)
psmx_cntr_inc(mr->cntr);
target_ep = mr->domain->atomics_ep;
if (target_ep->remote_write_cntr)
psmx_cntr_inc(target_ep->remote_write_cntr);
if (target_ep->remote_read_cntr &&
target_ep->remote_read_cntr != target_ep->remote_write_cntr)
psmx_cntr_inc(target_ep->remote_read_cntr);
}
else {
tmp_buf = NULL;
}
rep_args[0].u32w0 = PSMX_AM_REP_ATOMIC_READWRITE;
rep_args[0].u32w1 = op_error;
rep_args[1].u64 = args[1].u64;
err = psm_am_reply_short(token, PSMX_AM_ATOMIC_HANDLER,
rep_args, 2, tmp_buf, (tmp_buf?len:0), 0,
psmx_am_atomic_completion, tmp_buf );
break;
case PSMX_AM_REP_ATOMIC_WRITE:
req = (struct psmx_am_request *)(uintptr_t)args[1].u64;
op_error = (int)args[0].u32w1;
assert(req->op == PSMX_AM_REQ_ATOMIC_WRITE);
if (req->ep->send_cq && !req->no_event) {
event = psmx_cq_create_event(
req->ep->send_cq,
req->atomic.context,
req->atomic.buf,
0, /* flags */
req->atomic.len,
0, /* data */
0, /* tag */
0, /* olen */
op_error);
if (event)
psmx_cq_enqueue_event(req->ep->send_cq, event);
else
err = -ENOMEM;
}
if (req->ep->write_cntr)
psmx_cntr_inc(req->ep->write_cntr);
free(req);
break;
case PSMX_AM_REP_ATOMIC_READWRITE:
case PSMX_AM_REP_ATOMIC_COMPWRITE:
req = (struct psmx_am_request *)(uintptr_t)args[1].u64;
op_error = (int)args[0].u32w1;
assert(op_error || req->atomic.len == len);
if (!op_error)
memcpy(req->atomic.result, src, len);
if (req->ep->send_cq && !req->no_event) {
event = psmx_cq_create_event(
req->ep->send_cq,
req->atomic.context,
req->atomic.buf,
0, /* flags */
req->atomic.len,
0, /* data */
0, /* tag */
0, /* olen */
op_error);
if (event)
psmx_cq_enqueue_event(req->ep->send_cq, event);
else
err = -ENOMEM;
}
if (req->ep->read_cntr)
psmx_cntr_inc(req->ep->read_cntr);
free(req);
break;
default:
err = -EINVAL;
}
return err;
}
static int psmx_mr_regattr(struct fid_domain *domain, const struct fi_mr_attr *attr,
uint64_t flags, struct fid_mr **mr)
{
struct psmx_fid_domain *fid_domain;
struct psmx_fid_mr *fid_mr;
int i;
uint64_t key;
if (attr->requested_key != PSMX_MR_AUTO_KEY && psmx_mr_hash_get(attr->requested_key))
return -EEXIST;
fid_domain = container_of(domain, struct psmx_fid_domain, domain);
if (!attr)
return -EINVAL;
if (!(attr->mask & FI_MR_ATTR_IOV))
return -EINVAL;
if (attr->iov_count == 0 || attr->mr_iov == NULL)
return -EINVAL;
fid_mr = (struct psmx_fid_mr *)
calloc(1, sizeof(*fid_mr) +
sizeof(struct iovec) * attr->iov_count);
if (!fid_mr)
return -ENOMEM;
fid_mr->mr.fid.fclass = FID_CLASS_MR;
fid_mr->mr.fid.ops = &psmx_fi_ops;
fid_mr->mr.mem_desc = fid_mr;
if (attr->requested_key != PSMX_MR_AUTO_KEY) {
key = attr->requested_key;
}
else {
key = (uint64_t)(uintptr_t)fid_mr;
while (psmx_mr_hash_get(key))
key++;
}
fid_mr->mr.key = key;
fid_mr->domain = fid_domain;
fid_mr->access = FI_READ | FI_WRITE | FI_REMOTE_READ | FI_REMOTE_WRITE;
fid_mr->flags = flags;
fid_mr->iov_count = attr->iov_count;
for (i=0; i<attr->iov_count; i++)
fid_mr->iov[i] = attr->mr_iov[i];
if (attr->mask & FI_MR_ATTR_CONTEXT)
fid_mr->mr.fid.context = attr->context;
if (attr->mask & FI_MR_ATTR_ACCESS)
fid_mr->access = attr->access;
if (attr->mask & FI_MR_ATTR_KEY)
; /* requested_key is ignored */
psmx_mr_normalize_iov(fid_mr->iov, &fid_mr->iov_count);
psmx_mr_hash_add(fid_mr);
*mr = &fid_mr->mr;
return 0;
}
