/* return a pointer to the request data following an object attributes structure */
const void *get_req_data_after_objattr( const struct object_attributes *attr, data_size_t *len )
{
const void *ptr = (const WCHAR *)((const struct object_attributes *)get_req_data() + 1) +
attr->sd_len / sizeof(WCHAR) + attr->name_len / sizeof(WCHAR);
*len = get_req_data_size() - ((const char *)ptr - (const char *)get_req_data());
return ptr;
}
static
int pscom_openib_rma_read(pscom_req_t *rendezvous_req, pscom_rendezvous_data_t *rd)
{
int err, ret;
pscom_rendezvous_data_openib_t *psopenib_rd = get_req_data(rd);
psoib_rma_req_t *dreq = &psopenib_rd->rma_req;
pscom_con_t *con = get_con(rendezvous_req->pub.connection);
psoib_con_info_t *ci = con->arch.openib.mcon;
perf_add("openib_rma_read");
#ifdef IB_RNDV_USE_PADDING
memcpy(rendezvous_req->pub.data, rd->msg.arch.openib.padding_data, rd->msg.arch.openib.padding_size);
rendezvous_req->pub.data += rd->msg.arch.openib.padding_size;
rendezvous_req->pub.data_len -= rd->msg.arch.openib.padding_size;
#endif
err = psoib_acquire_rma_mreg(&dreq->mreg, rendezvous_req->pub.data, rendezvous_req->pub.data_len, ci);
assert(!err); // ToDo: Catch error
dreq->remote_addr = rd->msg.arch.openib.mr_addr;
dreq->remote_key = rd->msg.arch.openib.mr_key;
dreq->data_len = rendezvous_req->pub.data_len;
dreq->ci = ci;
dreq->io_done = pscom_openib_rma_read_io_done;
dreq->priv = psopenib_rd;
psopenib_rd->rendezvous_req = rendezvous_req;
err = psoib_post_rma_get(dreq);
assert(!err); // ToDo: Catch error
return 0;
}
static
void pscom_openib_rma_mem_deregister(pscom_con_t *con, pscom_rendezvous_data_t *rd)
{
pscom_rendezvous_data_openib_t *openib_rd = get_req_data(rd);
psoib_rma_mreg_t *mreg = &openib_rd->rma_req.mreg;
perf_add("openib_release_rma_mreg");
psoib_release_rma_mreg(mreg);
}
static
void pscom_elan_rma_mem_deregister(pscom_con_t *con, pscom_rendezvous_data_t *rd)
{
pscom_rendezvous_data_elan_t *elan_rd = get_req_data(rd);
if (elan_rd->rma_req.mreg) {
pselan_put_mregion(elan_rd->rma_req.mreg);
elan_rd->rma_req.mreg = NULL;
}
}
static
void pscom_openib_rma_write_io_done(void *priv, int err)
{
pscom_rendezvous_data_t *rd_data = (pscom_rendezvous_data_t *)priv;
pscom_rendezvous_data_openib_t *rd_data_openib = get_req_data(rd_data);
// ToDo: Error propagation
rd_data_openib->io_done(rd_data_openib->priv);
pscom_openib_rma_mem_deregister(rd_data_openib->con, rd_data);
pscom_free(rd_data);
}
void get_req_path(struct unicode_str *str, int skip_root)
{
static const WCHAR root_name[] = { '\\','R','e','g','i','s','t','r','y','\\' };
str->str = get_req_data();
str->len = (get_req_data_size() / sizeof(WCHAR)) * sizeof(WCHAR);
if (skip_root && str->len >= sizeof(root_name) &&
!memicmpW(str->str, root_name, sizeof(root_name) / sizeof(WCHAR))) {
str->str += sizeof(root_name) / sizeof(WCHAR);
str->len -= sizeof(root_name);
}
}
static
unsigned int pscom_openib_rma_mem_register(pscom_con_t *con, pscom_rendezvous_data_t *rd)
{
int err = 0;
pscom_rendezvous_data_openib_t *openib_rd = get_req_data(rd);
psoib_con_info_t *ci = con->arch.openib.mcon;
psoib_rma_mreg_t *mreg = &openib_rd->rma_req.mreg;
if (rd->msg.data_len > IB_MAX_RDMA_MSG_SIZE) goto err_size;
#ifdef IB_RNDV_USE_PADDING
#ifdef IB_RNDV_RDMA_WRITE
#error IB_RNDV_USE_PADDING and IB_RNDV_RDMA_WRITE are mutually exclusive!
#endif
rd->msg.arch.openib.padding_size = (IB_RNDV_PADDING_SIZE - ((long long int)rd->msg.data) % IB_RNDV_PADDING_SIZE) % IB_RNDV_PADDING_SIZE;
memcpy(rd->msg.arch.openib.padding_data, rd->msg.data, rd->msg.arch.openib.padding_size);
/* get mem region */
perf_add("openib_acquire_rma_mreg");
err = psoib_acquire_rma_mreg(mreg, rd->msg.data + rd->msg.arch.openib.padding_size, rd->msg.data_len - rd->msg.arch.openib.padding_size, ci);
assert(!err);
if (err) goto err_get_region;
rd->msg.arch.openib.mr_key = mreg->mem_info.mr->rkey;
rd->msg.arch.openib.mr_addr = (uint64_t)mreg->mem_info.ptr;
return sizeof(rd->msg.arch.openib) - sizeof(rd->msg.arch.openib.padding_data) + rd->msg.arch.openib.padding_size;
#else
/* get mem region */
perf_add("openib_acquire_rma_mreg2");
err = psoib_acquire_rma_mreg(mreg, rd->msg.data, rd->msg.data_len, ci);
assert(!err);
if (err) goto err_get_region;
rd->msg.arch.openib.mr_key = mreg->mem_info.mr->rkey;
rd->msg.arch.openib.mr_addr = (uint64_t)mreg->mem_info.ptr;
return sizeof(rd->msg.arch.openib) - sizeof(rd->msg.arch.openib.padding_data);
#endif
err_get_region:
err_size:
// ToDo: Handle Errors!
return 0;
}
static
int pscom_openib_rma_write(pscom_con_t *con, void *src, pscom_rendezvous_msg_t *des,
void (*io_done)(void *priv), void *priv)
{
pscom_rendezvous_data_t *rd_data = (pscom_rendezvous_data_t *)pscom_malloc(sizeof(*rd_data));
pscom_rendezvous_data_openib_t *rd_data_openib = get_req_data(rd_data);
psoib_con_info_t *mcon = con->arch.openib.mcon;
psoib_rma_req_t *dreq = &rd_data_openib->rma_req;
size_t len;
int err;
rd_data->msg.id = (void*)42;
rd_data->msg.data = src;
rd_data->msg.data_len = des->data_len;
len = pscom_openib_rma_mem_register(con, rd_data);
assert(len); // ToDo: Catch error
/*
dreq->mreg.mem_info.ptr = xxx;
dreq->mreg.size = xxx;
dreq->mreg.mem_ingo.mr->lkey = xxx;
*/
perf_add("openib_rma_write");
dreq->remote_addr = des->arch.openib.mr_addr;
dreq->remote_key = des->arch.openib.mr_key;
dreq->data_len = des->data_len;
dreq->ci = mcon;
dreq->io_done = pscom_openib_rma_write_io_done;
dreq->priv = rd_data;
rd_data_openib->con = con;
rd_data_openib->io_done = io_done;
rd_data_openib->priv = priv;
err = psoib_post_rma_put(dreq);
assert(!err); // ToDo: Catch error
rd_data = NULL; /* Do not use rd_data after psoib_post_rma_put()!
io_done might already be called and freed rd_data. */
return 0;
}
/* return object attributes from the current request */
const struct object_attributes *get_req_object_attributes( const struct security_descriptor **sd,
struct unicode_str *name,
struct object **root )
{
static const struct object_attributes empty_attributes;
const struct object_attributes *attr = get_req_data();
data_size_t size = get_req_data_size();
if (root) *root = NULL;
if (!size)
{
*sd = NULL;
name->len = 0;
return &empty_attributes;
}
if ((size < sizeof(*attr)) || (size - sizeof(*attr) < attr->sd_len) ||
(size - sizeof(*attr) - attr->sd_len < attr->name_len))
{
set_error( STATUS_ACCESS_VIOLATION );
return NULL;
}
if (attr->sd_len && !sd_is_valid( (const struct security_descriptor *)(attr + 1), attr->sd_len ))
{
set_error( STATUS_INVALID_SECURITY_DESCR );
return NULL;
}
if ((attr->name_len & (sizeof(WCHAR) - 1)) || attr->name_len >= 65534)
{
set_error( STATUS_OBJECT_NAME_INVALID );
return NULL;
}
if (root && attr->rootdir && attr->name_len)
{
if (!(*root = get_directory_obj( current->process, attr->rootdir ))) return NULL;
}
*sd = attr->sd_len ? (const struct security_descriptor *)(attr + 1) : NULL;
name->len = attr->name_len;
name->str = (const WCHAR *)(attr + 1) + attr->sd_len / sizeof(WCHAR);
return attr;
}
static obj_handle_t device_file_ioctl( struct fd *fd, ioctl_code_t code, const async_data_t *async_data,
int blocking )
{
struct device_file *file = get_fd_user( fd );
struct irp_call *irp;
obj_handle_t handle;
irp_params_t params;
memset( ¶ms, 0, sizeof(params) );
params.ioctl.major = IRP_MJ_DEVICE_CONTROL;
params.ioctl.code = code;
params.ioctl.file = file->user_ptr;
irp = create_irp( file, ¶ms, get_req_data(), get_req_data_size(),
get_reply_max_size() );
if (!irp) return 0;
handle = queue_irp( file, irp, async_data, blocking );
release_object( irp );
return handle;
}
static obj_handle_t device_file_write( struct fd *fd, const async_data_t *async_data, int blocking,
file_pos_t pos, data_size_t *written )
{
struct device_file *file = get_fd_user( fd );
struct irp_call *irp;
obj_handle_t handle;
irp_params_t params;
memset( ¶ms, 0, sizeof(params) );
params.write.major = IRP_MJ_WRITE;
params.write.key = 0;
params.write.pos = pos;
params.write.file = file->user_ptr;
irp = create_irp( file, ¶ms, get_req_data(), get_req_data_size(), 0 );
if (!irp) return 0;
handle = queue_irp( file, irp, async_data, blocking );
release_object( irp );
return handle;
}
static
int pscom_elan_rma_read(pscom_req_t *rendezvous_req, pscom_rendezvous_data_t *rd)
{
pscom_rendezvous_data_elan_t *elan_rd = get_req_data(rd);
pselan_rdma_req_t *dreq = &elan_rd->rma_req;
pscom_con_t *con = get_con(rendezvous_req->pub.connection);
pselan_con_info_t *ci = con->arch.elan.ci;
dreq->ci = ci;
dreq->rmr_context = rd->msg.arch.elan.rmr_context;
dreq->rmr_vaddr = rd->msg.arch.elan.rmr_vaddr;
dreq->lmr_buf = rendezvous_req->pub.data;
dreq->size = rendezvous_req->pub.data_len;
dreq->io_done = pscom_elan_rma_read_io_done;
dreq->priv = elan_rd;
elan_rd->rendezvous_req = rendezvous_req;
return pselan_post_rdma_get(dreq);
}
static
unsigned int pscom_elan_rma_mem_register(pscom_con_t *con, pscom_rendezvous_data_t *rd)
{
pscom_rendezvous_data_elan_t *elan_rd = get_req_data(rd);
pselan_con_info_t *ci = con->arch.elan.ci;
/* get mem region */
pselan_mregion_cache_t *mreg =
pselan_get_mregion(rd->msg.data, rd->msg.data_len, ci);
if (!mreg)
goto err_get_region;
elan_rd->rma_req.mreg = mreg;
rd->msg.arch.elan.rmr_context = pselan_get_rmr_context(mreg);
rd->msg.arch.elan.rmr_vaddr = pselan_get_rmr_vaddr(rd->msg.data);
return sizeof(rd->msg.arch.elan);
err_get_region:
// ToDo: Count get_mregion errors!
elan_rd->rma_req.mreg = NULL;
return 0;
}