/*---------------------------------------------------------------------------*/
int xio_accept(struct xio_session *session,
const char **portals_array,
size_t portals_array_len,
void *user_context,
size_t user_context_len)
{
int retval = 0;
struct xio_msg *msg;
struct xio_task *task;
msg = xio_session_write_accept_rsp(session,
XIO_ACTION_ACCEPT,
portals_array,
portals_array_len,
user_context,
user_context_len);
if (!msg) {
ERROR_LOG("setup request creation failed\n");
return -1;
}
msg->request = session->setup_req;
msg->type = (enum xio_msg_type)XIO_SESSION_SETUP_RSP;
task = container_of(msg->request,
struct xio_task, imsg);
if (portals_array_len != 0) {
/* server side state is changed to ACCEPT, will be move to
* ONLINE state when first "hello" message arrives
*/
session->state = XIO_SESSION_STATE_ACCEPTED;
/* temporary disable teardown */
session->disable_teardown = 1;
TRACE_LOG("session state is now ACCEPT. session:%p\n",
session);
} else {
/* initialize credits */
task->connection->peer_credits_msgs =
session->peer_rcv_queue_depth_msgs;
task->connection->credits_msgs = 0;
task->connection->peer_credits_bytes =
session->peer_rcv_queue_depth_bytes;
task->connection->credits_bytes = 0;
/* server side state is changed to ONLINE, immediately */
session->state = XIO_SESSION_STATE_ONLINE;
TRACE_LOG("session state changed to ONLINE. session:%p\n",
session);
}
retval = xio_connection_send(task->connection, msg);
if (retval && retval != -EAGAIN) {
ERROR_LOG("failed to send message. errno:%d\n", -retval);
xio_set_error(-retval);
return -1;
}
return 0;
}
/*---------------------------------------------------------------------------*/
int xio_redirect(struct xio_session *session,
const char **portals_array,
size_t portals_array_len)
{
int retval = 0;
struct xio_msg *msg;
struct xio_task *task;
if (portals_array_len == 0 || !portals_array) {
xio_set_error(EINVAL);
ERROR_LOG("portals array for redirect is mandatory\n");
return -1;
}
msg = xio_session_write_accept_rsp(session,
XIO_ACTION_REDIRECT,
portals_array,
portals_array_len,
NULL,
0);
if (unlikely(!msg)) {
ERROR_LOG("setup request creation failed\n");
return -1;
}
if (portals_array_len != 0) {
/* server side state is changed to ACCEPT */
session->state = XIO_SESSION_STATE_REDIRECTED;
TRACE_LOG("session state is now REDIRECTED. session:%p\n",
session);
}
msg->request = session->setup_req;
msg->type = (enum xio_msg_type)XIO_SESSION_SETUP_RSP;
task = container_of(msg->request,
struct xio_task, imsg);
retval = xio_connection_send(task->connection, msg);
if (retval && retval != -EAGAIN) {
ERROR_LOG("failed to send message errno:%d\n", -retval);
xio_set_error(-retval);
return -1;
}
return 0;
}
/*---------------------------------------------------------------------------*/
struct xio_mr *xio_reg_mr(void *addr, size_t length)
{
if (addr == NULL) {
xio_set_error(EINVAL);
return NULL;
}
return &dummy_mr;
}
/*---------------------------------------------------------------------------*/
int xio_get_opt(void *xio_obj, int level, int optname,
void *optval, int *optlen)
{
static struct xio_transport *rdma_transport;
static struct xio_transport *tcp_transport;
switch (level) {
case XIO_OPTLEVEL_ACCELIO:
return xio_general_get_opt(xio_obj, optname, optval, optlen);
case XIO_OPTLEVEL_RDMA:
if (!rdma_transport) {
rdma_transport = xio_get_transport("rdma");
if (!rdma_transport) {
xio_set_error(EFAULT);
return -1;
}
}
if (!rdma_transport->get_opt)
break;
return rdma_transport->get_opt(xio_obj,
optname, optval, optlen);
break;
case XIO_OPTLEVEL_TCP:
if (!tcp_transport) {
tcp_transport = xio_get_transport("tcp");
if (!tcp_transport) {
xio_set_error(EFAULT);
return -1;
}
}
if (!tcp_transport->get_opt)
break;
return tcp_transport->get_opt(xio_obj,
optname, optval, optlen);
break;
default:
break;
}
xio_set_error(XIO_E_NOT_SUPPORTED);
return -1;
}
/*---------------------------------------------------------------------------*/
int xio_ctx_del_work(struct xio_context *ctx,
xio_ctx_work_t *work)
{
int retval;
retval = xio_workqueue_del_work(ctx->workqueue, work);
if (retval) {
xio_set_error(retval);
ERROR_LOG("xio_workqueue_del_work failed. err=%d\n", retval);
}
return retval;
}
/*---------------------------------------------------------------------------*/
int xio_query_context(struct xio_context *ctx,
struct xio_context_attr *attr,
int attr_mask)
{
if (!ctx || !attr) {
xio_set_error(EINVAL);
ERROR_LOG("invalid parameters\n");
return -1;
}
if (attr_mask & XIO_CONTEXT_ATTR_USER_CTX)
attr->user_context = ctx->user_context;
return 0;
}
/*---------------------------------------------------------------------------*/
int xio_ctx_add_work(struct xio_context *ctx,
void *data,
void (*function)(void *data),
xio_ctx_work_t *work)
{
int retval;
retval = xio_workqueue_add_work(ctx->workqueue,
data, function, work);
if (retval) {
xio_set_error(retval);
ERROR_LOG("xio_workqueue_add_work failed. err=%d\n", retval);
}
return retval;
}
/*---------------------------------------------------------------------------*/
int xio_ctx_add_delayed_work(struct xio_context *ctx,
int msec_duration, void *data,
void (*timer_fn)(void *data),
xio_ctx_delayed_work_t *work)
{
int retval;
retval = xio_workqueue_add_delayed_work(ctx->workqueue,
msec_duration, data,
timer_fn, work);
if (retval) {
xio_set_error(retval);
ERROR_LOG("xio_workqueue_add_delayed_work failed. err=%d\n",
retval);
}
return retval;
}
/*---------------------------------------------------------------------------*/
int xio_ctx_del_work(struct xio_context *ctx,
xio_ctx_work_t *work)
{
int retval;
/* test if work is pending */
if (!xio_is_work_pending(work))
return 0;
retval = xio_workqueue_del_work(ctx->workqueue, work);
if (retval) {
xio_set_error(retval);
ERROR_LOG("xio_workqueue_del_work failed. err=%d\n", retval);
}
return retval;
}
/*---------------------------------------------------------------------------*/
struct xio_buf *xio_alloc(size_t length)
{
struct xio_buf *buf;
size_t real_size;
int alloced = 0;
buf = (struct xio_buf *)ucalloc(1, sizeof(*buf));
if (!buf) {
xio_set_error(errno);
ERROR_LOG("calloc failed. (errno=%d %m)\n", errno);
return NULL;
}
real_size = ALIGN(length, page_size);
buf->addr = umemalign(page_size, real_size);
if (!buf->addr) {
ERROR_LOG("xio_memalign failed. sz:%zu\n", real_size);
goto cleanup;
}
memset(buf->addr, 0, real_size);
alloced = 1;
buf->mr = xio_reg_mr(&buf->addr, length);
if (!buf->mr) {
ERROR_LOG("xio_reg_mr failed. addr:%p, length:%d\n",
buf->addr, length, access);
goto cleanup1;
}
buf->length = length;
return buf;
cleanup1:
if (alloced)
ufree(buf->addr);
cleanup:
ufree(buf);
return NULL;
}
/*---------------------------------------------------------------------------*/
int xio_ctx_set_work_destructor(
struct xio_context *ctx, void *data,
void (*destructor)(void *data),
xio_ctx_work_t *work)
{
int retval;
/* test if work is pending */
if (xio_is_work_pending(work))
return 0;
retval = xio_workqueue_set_work_destructor(
ctx->workqueue,
data, destructor, work);
if (retval) {
xio_set_error(retval);
ERROR_LOG("xio_workqueue_set_work_destructor failed. %m\n");
}
return retval;
}
/*---------------------------------------------------------------------------*/
int xio_reject(struct xio_session *session,
enum xio_status reason,
void *user_context,
size_t user_context_len)
{
int retval = 0;
struct xio_msg *msg;
struct xio_task *task;
msg = xio_session_write_reject_rsp(session,
reason,
user_context,
user_context_len);
if (msg == NULL) {
ERROR_LOG("setup request creation failed\n");
return -1;
}
/* server side state is changed to REJECTED */
session->state = XIO_SESSION_STATE_REJECTED;
TRACE_LOG("session state is now REJECT. session:%p\n",
session);
msg->request = session->setup_req;
msg->type = XIO_SESSION_SETUP_RSP;
task = container_of(msg->request,
struct xio_task, imsg);
task->connection->close_reason = XIO_E_SESSION_REJECTED;
retval = xio_connection_send(task->connection, msg);
if (retval && retval != -EAGAIN) {
ERROR_LOG("failed to send message. errno:%d\n", -retval);
xio_set_error(-retval);
return -1;
}
return 0;
}
void xio_context_destroy(struct xio_context *ctx)
{
int found;
found = xio_idr_lookup_uobj(usr_idr, ctx);
if (found) {
xio_idr_remove_uobj(usr_idr, ctx);
} else {
ERROR_LOG("context not found:%p\n", ctx);
xio_set_error(XIO_E_USER_OBJ_NOT_FOUND);
return;
}
ctx->run_private = 0;
xio_observable_notify_all_observers(&ctx->observable,
XIO_CONTEXT_EVENT_CLOSE, NULL);
/* allow internally to run the loop for final cleanup */
if (ctx->run_private)
xio_context_run_loop(ctx);
if (ctx->run_private == 0)
xio_destroy_context_continue(&ctx->destroy_ctx_work.work);
}
/*---------------------------------------------------------------------------*/
int xio_unbind(struct xio_server *server)
{
int retval = 0;
int found;
if (!server)
return -1;
found = xio_idr_lookup_uobj(usr_idr, server);
if (found) {
xio_idr_remove_uobj(usr_idr, server);
} else {
ERROR_LOG("server not found:%p\n", server);
xio_set_error(XIO_E_USER_OBJ_NOT_FOUND);
return -1;
}
/* notify all observers that the server wishes to exit */
xio_observable_notify_all_observers(&server->nexus_observable,
XIO_SERVER_EVENT_CLOSE, NULL);
kref_put(&server->kref, xio_server_destroy);
return retval;
}
/*---------------------------------------------------------------------------*/
int xio_on_setup_req_recv(struct xio_connection *connection,
struct xio_task *task)
{
struct xio_msg *msg = &task->imsg;
struct xio_new_session_req req;
uint8_t *ptr;
uint16_t len;
struct xio_session_hdr hdr;
struct xio_session *session = connection->session;
int retval;
struct xio_session_event_data error_event = {};
error_event.event = XIO_SESSION_ERROR_EVENT;
/* read session header */
xio_session_read_header(task, &hdr);
#ifdef XIO_SESSION_DEBUG
connection->peer_connection = hdr.connection;
connection->peer_session = hdr.session;
#endif
task->imsg.sn = hdr.serial_num;
task->connection = connection;
task->session = session;
connection->session->setup_req = msg;
/* read the header */
ptr = (uint8_t *)msg->in.header.iov_base;
memset(&req, 0, sizeof(req));
/* session id */
len = xio_read_uint32(&session->peer_session_id, 0, ptr);
ptr = ptr + len;
/* queue depth bytes */
len = xio_read_uint64(&session->peer_snd_queue_depth_bytes, 0, ptr);
ptr = ptr + len;
len = xio_read_uint64(&session->peer_rcv_queue_depth_bytes, 0, ptr);
ptr = ptr + len;
/* queue depth msgs */
len = xio_read_uint16((uint16_t *)&session->peer_snd_queue_depth_msgs,
0, ptr);
ptr = ptr + len;
len = xio_read_uint16((uint16_t *)&session->peer_rcv_queue_depth_msgs,
0, ptr);
ptr = ptr + len;
/* uri length */
len = xio_read_uint16(&req.uri_len, 0, ptr);
ptr = ptr + len;
/* private length */
len = xio_read_uint16(&req.private_data_len, 0, ptr);
ptr = ptr + len;
if (req.uri_len) {
req.uri =
(char *)kcalloc(req.uri_len, sizeof(char), GFP_KERNEL);
if (unlikely(!req.uri)) {
xio_set_error(ENOMEM);
ERROR_LOG("uri allocation failed. len:%d\n",
req.uri_len);
goto cleanup1;
}
len = xio_read_array((uint8_t *)req.uri,
req.uri_len, 0, ptr);
ptr = ptr + len;
}
if (req.private_data_len) {
req.private_data = kcalloc(req.private_data_len,
sizeof(uint8_t), GFP_KERNEL);
if (unlikely(!req.private_data)) {
xio_set_error(ENOMEM);
ERROR_LOG("private data allocation failed. len:%d\n",
req.private_data_len);
goto cleanup2;
}
len = xio_read_array((uint8_t *)req.private_data,
req.private_data_len,
0, ptr);
ptr = ptr + len;
}
req.proto = (enum xio_proto)xio_nexus_get_proto(connection->nexus);
xio_nexus_get_peer_addr(connection->nexus,
&req.src_addr, sizeof(req.src_addr));
/* cache the task in io queue*/
xio_connection_queue_io_task(connection, task);
/* notify the upper layer */
if (connection->ses_ops.on_new_session) {
retval = connection->ses_ops.on_new_session(
session, &req,
connection->cb_user_context);
if (retval)
goto cleanup2;
} else {
retval = xio_accept(session, NULL, 0, NULL, 0);
if (retval) {
ERROR_LOG("failed to auto accept session. session:%p\n",
session);
goto cleanup2;
}
}
/* Don't move session state to ONLINE. In case of multiple portals
* the accept moves the state to ACCEPTED until the first "HELLO"
* message arrives. Note that the "upper layer" may call redirect or
* reject.
*/
xio_session_notify_new_connection(session, connection);
kfree(req.private_data);
kfree(req.uri);
return 0;
cleanup2:
kfree(req.private_data);
cleanup1:
kfree(req.uri);
if (session->ses_ops.on_session_event) {
error_event.reason = (enum xio_status)xio_errno();
session->ses_ops.on_session_event(
session, &error_event,
session->cb_user_context);
}
return 0;
}
/*---------------------------------------------------------------------------*/
void *xio_ev_loop_init(unsigned long flags, struct xio_context *ctx,
struct xio_loop_ops *loop_ops)
{
struct xio_ev_loop *loop;
char queue_name[64];
loop = kzalloc(sizeof(struct xio_ev_loop), GFP_KERNEL);
if (loop == NULL) {
xio_set_error(ENOMEM);
ERROR_LOG("kmalloc failed. %m\n");
goto cleanup0;
}
set_bit(XIO_EV_LOOP_STOP, &loop->states);
init_llist_head(&loop->ev_llist);
/* use default implementation */
loop->run = priv_ev_loop_run;
loop->stop = priv_ev_loop_stop;
loop->loop_object = loop;
switch (flags) {
case XIO_LOOP_USER_LOOP:
/* override with user provided routines and object */
loop->run = loop_ops->run;
loop->stop = loop_ops->stop;
loop->add_event = loop_ops->add_event;
loop->loop_object = loop_ops->ev_loop;
break;
case XIO_LOOP_GIVEN_THREAD:
loop->add_event = priv_ev_add_thread;
init_waitqueue_head(&loop->wait);
break;
case XIO_LOOP_TASKLET:
loop->add_event = priv_ev_add_tasklet;
tasklet_init(&loop->tasklet, priv_ev_loop_run_tasklet,
(unsigned long)loop);
break;
case XIO_LOOP_WORKQUEUE:
/* temp (also change to single thread) */
sprintf(queue_name, "xio-%p", loop);
/* check flags and bw comp */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 36)
loop->workqueue = create_workqueue(queue_name);
#else
loop->workqueue = alloc_workqueue(queue_name,
WQ_MEM_RECLAIM | WQ_HIGHPRI,
0);
#endif
if (!loop->workqueue) {
ERROR_LOG("workqueue create failed.\n");
goto cleanup1;
}
loop->add_event = priv_ev_add_workqueue;
break;
default:
ERROR_LOG("wrong type. %lu\n", flags);
goto cleanup1;
}
loop->flags = flags;
loop->ctx = ctx;
return loop;
cleanup1:
clear_bit(XIO_EV_LOOP_STOP, &loop->states);
kfree(loop);
cleanup0:
ERROR_LOG("event loop creation failed.\n");
return NULL;
}
/*---------------------------------------------------------------------------*/
struct xio_msg *xio_session_write_reject_rsp(struct xio_session *session,
enum xio_status reason,
void *user_context,
uint16_t user_context_len)
{
struct xio_msg *msg;
uint8_t *buf;
uint8_t *ptr;
uint16_t len, tot_len;
uint16_t action = XIO_ACTION_REJECT;
/* calclate length */
tot_len = 2*sizeof(uint16_t) + 2*sizeof(uint32_t);
tot_len += user_context_len;
if (tot_len > SETUP_BUFFER_LEN) {
ERROR_LOG("buffer is too small\n");
xio_set_error(EMSGSIZE);
return NULL;
}
/* allocate message */
buf = (uint8_t *)kcalloc(SETUP_BUFFER_LEN + sizeof(struct xio_msg),
sizeof(uint8_t), GFP_KERNEL);
if (!buf) {
ERROR_LOG("message allocation failed\n");
xio_set_error(ENOMEM);
return NULL;
}
/* fill the message */
msg = (struct xio_msg *)buf;
msg->out.header.iov_base = buf + sizeof(struct xio_msg);
msg->out.header.iov_len = 0;
ptr = (uint8_t *)msg->out.header.iov_base;
len = 0;
/* serialize message into the buffer */
/* session_id */
len = xio_write_uint32(session->session_id, 0, ptr);
ptr = ptr + len;
/* action */
len = xio_write_uint16(action, 0, ptr);
ptr = ptr + len;
/* reason */
len = xio_write_uint32(reason, 0, ptr);
ptr = ptr + len;
/* user_context_len */
len = xio_write_uint16(user_context_len, 0, ptr);
ptr = ptr + len;
if (user_context_len) {
len = xio_write_array((const uint8_t *)user_context,
user_context_len,
0, ptr);
ptr = ptr + len;
}
msg->out.header.iov_len = ptr - (uint8_t *)msg->out.header.iov_base;
if (msg->out.header.iov_len != tot_len) {
ERROR_LOG("calculated length %d != actual length %zd\n",
tot_len, msg->out.header.iov_len);
}
return msg;
}
/*---------------------------------------------------------------------------*/
struct xio_msg *xio_session_write_accept_rsp(struct xio_session *session,
uint16_t action,
const char **portals_array,
uint16_t portals_array_len,
void *user_context,
uint16_t user_context_len)
{
struct xio_msg *msg;
uint8_t *buf;
uint8_t *ptr;
uint16_t len, i, str_len, tot_len;
/* calculate length */
tot_len = 5*sizeof(uint16_t) + sizeof(uint32_t) + 2*sizeof(uint64_t);
for (i = 0; i < portals_array_len; i++)
tot_len += strlen(portals_array[i]) + sizeof(uint16_t);
tot_len += user_context_len;
if (tot_len > SETUP_BUFFER_LEN) {
ERROR_LOG("buffer is too small\n");
xio_set_error(EMSGSIZE);
return NULL;
}
/* allocate message */
buf = (uint8_t *)kcalloc(SETUP_BUFFER_LEN + sizeof(struct xio_msg),
sizeof(uint8_t), GFP_KERNEL);
if (unlikely(!buf)) {
ERROR_LOG("message allocation failed\n");
xio_set_error(ENOMEM);
return NULL;
}
/* fill the message */
msg = (struct xio_msg *)buf;
msg->out.header.iov_base = buf + sizeof(struct xio_msg);
msg->out.header.iov_len = 0;
ptr = (uint8_t *)msg->out.header.iov_base;
len = 0;
/* serialize message into the buffer */
/* session_id */
len = xio_write_uint32(session->session_id, 0, ptr);
ptr = ptr + len;
/* action */
len = xio_write_uint16(action, 0, ptr);
ptr = ptr + len;
if (action == XIO_ACTION_ACCEPT) {
/* tx queue depth bytes */
len = xio_write_uint64(session->snd_queue_depth_bytes, 0, ptr);
ptr = ptr + len;
/* rx queue depth bytes */
len = xio_write_uint64(session->rcv_queue_depth_bytes, 0, ptr);
ptr = ptr + len;
/* tx queue depth msgs */
len = xio_write_uint16(session->snd_queue_depth_msgs, 0, ptr);
ptr = ptr + len;
/* rx queue depth msgs */
len = xio_write_uint16(session->rcv_queue_depth_msgs, 0, ptr);
ptr = ptr + len;
}
/* portals_array_len */
len = xio_write_uint16(portals_array_len, 0, ptr);
ptr = ptr + len;
/* user_context_len */
len = xio_write_uint16(user_context_len, 0, ptr);
ptr = ptr + len;
for (i = 0; i < portals_array_len; i++) {
str_len = strlen(portals_array[i]);
len = xio_write_uint16(str_len, 0, ptr);
ptr = ptr + len;
len = xio_write_array((uint8_t *)portals_array[i],
str_len, 0, ptr);
ptr = ptr + len;
}
if (user_context_len) {
len = xio_write_array((const uint8_t *)user_context,
user_context_len,
0, ptr);
ptr = ptr + len;
}
msg->out.header.iov_len = ptr - (uint8_t *)msg->out.header.iov_base;
if (msg->out.header.iov_len != tot_len) {
ERROR_LOG("calculated length %d != actual length %zd\n",
tot_len, msg->out.header.iov_len);
}
return msg;
}
/*---------------------------------------------------------------------------*/
struct xio_context *xio_context_create(struct xio_context_params *ctx_params,
int polling_timeout,
int cpu_hint)
{
struct xio_context *ctx;
struct xio_loop_ops *loop_ops;
struct task_struct *worker;
struct xio_transport *transport;
int flags, cpu;
if (!ctx_params) {
xio_set_error(EINVAL);
ERROR_LOG("ctx_params is NULL\n");
goto cleanup0;
}
loop_ops = ctx_params->loop_ops;
worker = ctx_params->worker;
flags = ctx_params->flags;
if (cpu_hint > 0 && cpu_hint >= num_online_cpus()) {
xio_set_error(EINVAL);
ERROR_LOG("cpu_hint(%d) >= num_online_cpus(%d)\n",
cpu_hint, num_online_cpus());
goto cleanup0;
}
if ((flags == XIO_LOOP_USER_LOOP) &&
(!(loop_ops && loop_ops->add_event && loop_ops->ev_loop))) {
xio_set_error(EINVAL);
ERROR_LOG("loop_ops and ev_loop and ev_loop_add_event are " \
"mandatory with loop_ops\n");
goto cleanup0;
}
xio_read_logging_level();
/* no need to disable preemption */
cpu = raw_smp_processor_id();
if (cpu == -1)
goto cleanup0;
/* allocate new context */
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
if (!ctx) {
xio_set_error(ENOMEM);
ERROR_LOG("kzalloc failed\n");
goto cleanup0;
}
if (cpu_hint < 0)
cpu_hint = cpu;
ctx->run_private = 0;
ctx->user_context = ctx_params->user_context;
ctx->flags = flags;
ctx->cpuid = cpu_hint;
ctx->nodeid = cpu_to_node(cpu_hint);
ctx->polling_timeout = polling_timeout;
ctx->prealloc_xio_inline_bufs =
!!ctx_params->prealloc_xio_inline_bufs;
ctx->rq_depth = ctx_params->rq_depth;
if (!ctx_params->max_conns_per_ctx)
ctx->max_conns_per_ctx = 100;
else
ctx->max_conns_per_ctx =
max(ctx_params->max_conns_per_ctx , 2);
ctx->workqueue = xio_workqueue_create(ctx);
if (!ctx->workqueue) {
xio_set_error(ENOMEM);
ERROR_LOG("xio_workqueue_init failed.\n");
goto cleanup1;
}
ctx->msg_pool = xio_objpool_create(sizeof(struct xio_msg),
MSGPOOL_INIT_NR, MSGPOOL_GROW_NR);
if (!ctx->msg_pool) {
xio_set_error(ENOMEM);
ERROR_LOG("context's msg_pool create failed. %m\n");
goto cleanup2;
}
XIO_OBSERVABLE_INIT(&ctx->observable, ctx);
INIT_LIST_HEAD(&ctx->ctx_list);
switch (flags) {
case XIO_LOOP_USER_LOOP:
break;
case XIO_LOOP_GIVEN_THREAD:
set_cpus_allowed_ptr(worker, cpumask_of(cpu_hint));
ctx->worker = (uint64_t)worker;
break;
case XIO_LOOP_TASKLET:
break;
case XIO_LOOP_WORKQUEUE:
break;
default:
ERROR_LOG("wrong type. %u\n", flags);
goto cleanup3;
}
ctx->ev_loop = xio_ev_loop_init(flags, ctx, loop_ops);
if (!ctx->ev_loop)
goto cleanup3;
ctx->stats.hertz = HZ;
/* Initialize default counters' name */
ctx->stats.name[XIO_STAT_TX_MSG] = kstrdup("TX_MSG", GFP_KERNEL);
ctx->stats.name[XIO_STAT_RX_MSG] = kstrdup("RX_MSG", GFP_KERNEL);
ctx->stats.name[XIO_STAT_TX_BYTES] = kstrdup("TX_BYTES", GFP_KERNEL);
ctx->stats.name[XIO_STAT_RX_BYTES] = kstrdup("RX_BYTES", GFP_KERNEL);
ctx->stats.name[XIO_STAT_DELAY] = kstrdup("DELAY", GFP_KERNEL);
ctx->stats.name[XIO_STAT_APPDELAY] = kstrdup("APPDELAY", GFP_KERNEL);
/* initialize rdma pools only */
transport = xio_get_transport("rdma");
if (transport && ctx->prealloc_xio_inline_bufs) {
int retval = xio_ctx_pool_create(ctx, XIO_PROTO_RDMA,
XIO_CONTEXT_POOL_CLASS_INITIAL);
if (retval) {
ERROR_LOG("Failed to create initial pool. ctx:%p\n", ctx);
goto cleanup2;
}
retval = xio_ctx_pool_create(ctx, XIO_PROTO_RDMA,
XIO_CONTEXT_POOL_CLASS_PRIMARY);
if (retval) {
ERROR_LOG("Failed to create primary pool. ctx:%p\n", ctx);
goto cleanup2;
}
}
spin_lock_init(&ctx->ctx_list_lock);
xio_idr_add_uobj(usr_idr, ctx, "xio_context");
return ctx;
cleanup3:
xio_objpool_destroy(ctx->msg_pool);
cleanup2:
xio_workqueue_destroy(ctx->workqueue);
cleanup1:
kfree(ctx);
cleanup0:
ERROR_LOG("xio_ctx_open failed\n");
return NULL;
}
/*---------------------------------------------------------------------------*/
int xio_ctx_pool_create(struct xio_context *ctx, enum xio_proto proto,
enum xio_context_pool_class pool_cls)
{
struct xio_tasks_pool_ops *pool_ops;
struct xio_tasks_pool **tasks_pool;
struct xio_transport *transport;
struct xio_tasks_pool_params params;
char pool_name[64];
const char *proto_str = xio_proto_str(proto);
/* get the transport's proto */
transport = xio_get_transport(proto_str);
if (!transport) {
ERROR_LOG("failed to load %s transport layer.\n", proto_str);
ERROR_LOG("validate that your system support %s " \
"and the accelio's %s module is loaded\n",
proto_str, proto_str);
xio_set_error(ENOPROTOOPT);
return -1;
}
if (transport->get_pools_setup_ops) {
if (!ctx->primary_pool_ops[proto] ||
!ctx->initial_pool_ops[proto])
transport->get_pools_setup_ops(
NULL,
&ctx->initial_pool_ops[proto],
&ctx->primary_pool_ops[proto]);
} else {
ERROR_LOG("transport does not implement " \
"\"get_pools_setup_ops\"\n");
return -1;
}
switch(pool_cls) {
case XIO_CONTEXT_POOL_CLASS_INITIAL:
tasks_pool = &ctx->initial_tasks_pool[proto];
pool_ops = ctx->initial_pool_ops[proto];
sprintf(pool_name, "ctx:%p - initial_pool_%s", ctx, proto_str);
break;
case XIO_CONTEXT_POOL_CLASS_PRIMARY:
tasks_pool = &ctx->primary_tasks_pool[proto];
pool_ops = ctx->primary_pool_ops[proto];
sprintf(pool_name, "ctx:%p - primary_pool_%s", ctx, proto_str);
break;
default:
xio_set_error(EINVAL);
ERROR_LOG("unknown pool class\n");
return -1;
};
/* if already exist */
if (*tasks_pool)
return 0;
if (!pool_ops)
return -1;
if (!pool_ops->pool_get_params ||
!pool_ops->slab_pre_create ||
!pool_ops->slab_init_task ||
!pool_ops->pool_post_create ||
!pool_ops->slab_destroy)
return -1;
/* get pool properties from the transport */
memset(¶ms, 0, sizeof(params));
pool_ops->pool_get_params(NULL,
(int *)¶ms.start_nr,
(int *)¶ms.max_nr,
(int *)¶ms.alloc_nr,
(int *)¶ms.pool_dd_data_sz,
(int *)¶ms.slab_dd_data_sz,
(int *)¶ms.task_dd_data_sz);
if (ctx->prealloc_xio_inline_bufs) {
params.start_nr = params.max_nr;
params.alloc_nr = 0;
}
params.pool_hooks.slab_pre_create =
(int (*)(void *, int, void *, void *))
pool_ops->slab_pre_create;
params.pool_hooks.slab_post_create = (int (*)(void *, void *, void *))
pool_ops->slab_post_create;
params.pool_hooks.slab_destroy = (int (*)(void *, void *, void *))
pool_ops->slab_destroy;
params.pool_hooks.slab_init_task =
(int (*)(void *, void *, void *, int, struct xio_task *))
pool_ops->slab_init_task;
params.pool_hooks.slab_uninit_task =
(int (*)(void *, void *, void *, struct xio_task *))
pool_ops->slab_uninit_task;
params.pool_hooks.slab_remap_task =
(int (*)(void *, void *, void *, void *, struct xio_task *))
pool_ops->slab_remap_task;
params.pool_hooks.pool_pre_create = (int (*)(void *, void *, void *))
pool_ops->pool_pre_create;
params.pool_hooks.pool_post_create = (int (*)(void *, void *, void *))
pool_ops->pool_post_create;
params.pool_hooks.pool_destroy = (int (*)(void *, void *, void *))
pool_ops->pool_destroy;
params.pool_hooks.task_pre_put = (int (*)(void *, struct xio_task *))
pool_ops->task_pre_put;
params.pool_hooks.task_post_get = (int (*)(void *, struct xio_task *))
pool_ops->task_post_get;
params.pool_name = kstrdup(pool_name, GFP_KERNEL);
/* initialize the tasks pool */
*tasks_pool = xio_tasks_pool_create(¶ms);
if (!*tasks_pool) {
ERROR_LOG("xio_tasks_pool_create failed\n");
return -1;
}
return 0;
}
/*---------------------------------------------------------------------------*/
static int xio_general_get_opt(void *xio_obj, int optname,
void *optval, int *optlen)
{
switch (optname) {
case XIO_OPTNAME_LOG_LEVEL:
*((enum xio_log_level *)optval) = xio_get_log_level();
*optlen = sizeof(enum xio_log_level);
return 0;
case XIO_OPTNAME_MAX_IN_IOVLEN:
*optlen = sizeof(int);
*((int *)optval) = g_options.max_in_iovsz;
return 0;
case XIO_OPTNAME_MAX_OUT_IOVLEN:
*optlen = sizeof(int);
*((int *)optval) = g_options.max_out_iovsz;
return 0;
case XIO_OPTNAME_ENABLE_RECONNECT:
*optlen = sizeof(int);
*((int *)optval) = g_options.reconnect;
return 0;
case XIO_OPTNAME_ENABLE_FLOW_CONTROL:
*optlen = sizeof(int);
*((int *)optval) = g_options.enable_flow_control;
return 0;
case XIO_OPTNAME_SND_QUEUE_DEPTH_MSGS:
*optlen = sizeof(int);
*((int *)optval) = g_options.snd_queue_depth_msgs;
return 0;
case XIO_OPTNAME_RCV_QUEUE_DEPTH_MSGS:
*optlen = sizeof(int);
*((int *)optval) = g_options.rcv_queue_depth_msgs;
return 0;
case XIO_OPTNAME_SND_QUEUE_DEPTH_BYTES:
*optlen = sizeof(uint64_t);
*((uint64_t *)optval) = g_options.snd_queue_depth_bytes;
return 0;
case XIO_OPTNAME_RCV_QUEUE_DEPTH_BYTES:
*optlen = sizeof(uint64_t);
*((uint64_t *)optval) = g_options.rcv_queue_depth_bytes;
return 0;
case XIO_OPTNAME_MAX_INLINE_XIO_HEADER:
*optlen = sizeof(int);
*((int *)optval) = g_options.max_inline_xio_hdr;
return 0;
case XIO_OPTNAME_MAX_INLINE_XIO_DATA:
*optlen = sizeof(int);
*((int *)optval) = g_options.max_inline_xio_data;
return 0;
case XIO_OPTNAME_INLINE_XIO_DATA_ALIGN:
*optlen = sizeof(int);
*((int *)optval) = g_options.inline_xio_data_align;
return 0;
case XIO_OPTNAME_XFER_BUF_ALIGN:
*optlen = sizeof(int);
*((int *)optval) = g_options.xfer_buf_align;
return 0;
case XIO_OPTNAME_ENABLE_KEEPALIVE:
*optlen = sizeof(int);
*((int *)optval) = g_options.enable_keepalive;
return 0;
case XIO_OPTNAME_CONFIG_KEEPALIVE:
if (*optlen == sizeof(struct xio_options_keepalive)) {
memcpy(optval, &g_options.ka, *optlen);
return 0;
} else {
xio_set_error(EINVAL);
return -1;
}
default:
break;
}
xio_set_error(XIO_E_NOT_SUPPORTED);
return -1;
}
/*---------------------------------------------------------------------------*/
static int xio_general_set_opt(void *xio_obj, int optname,
const void *optval, int optlen)
{
int tmp;
switch (optname) {
case XIO_OPTNAME_LOG_FN:
if (optlen == 0 && !optval)
return xio_set_log_fn(NULL);
else if (optlen == sizeof(xio_log_fn))
return xio_set_log_fn((xio_log_fn)optval);
break;
case XIO_OPTNAME_LOG_LEVEL:
if (optlen != sizeof(enum xio_log_level))
return -1;
return xio_set_log_level(*((enum xio_log_level *)optval));
case XIO_OPTNAME_DISABLE_HUGETBL:
xio_disable_huge_pages(*((int *)optval));
return 0;
case XIO_OPTNAME_MEM_ALLOCATOR:
if (optlen == sizeof(struct xio_mem_allocator))
return xio_set_mem_allocator(
(struct xio_mem_allocator *)optval);
break;
case XIO_OPTNAME_CONFIG_MEMPOOL:
if (optlen == sizeof(struct xio_mempool_config)) {
memcpy(&g_mempool_config,
(struct xio_mempool_config *)optval, optlen);
return 0;
}
break;
case XIO_OPTNAME_MAX_IN_IOVLEN:
if (optlen == sizeof(int)) {
struct xio_transport *rdma_transport =
xio_get_transport("rdma");
struct xio_transport *tcp_transport =
xio_get_transport("tcp");
int retval = 0;
if (*((int *)optval) > XIO_IOVLEN &&
*((int *)optval) <= XIO_MAX_IOV) {
g_options.max_in_iovsz = *((int *)optval);
if (rdma_transport &&
rdma_transport->set_opt)
retval |= rdma_transport->set_opt(
xio_obj, optname,
optval, optlen);
if (tcp_transport &&
tcp_transport->set_opt)
retval |= tcp_transport->set_opt(
xio_obj, optname,
optval, optlen);
}
return retval;
}
break;
case XIO_OPTNAME_MAX_OUT_IOVLEN:
if (optlen == sizeof(int)) {
struct xio_transport *rdma_transport =
xio_get_transport("rdma");
struct xio_transport *tcp_transport =
xio_get_transport("tcp");
int retval = 0;
if (*((int *)optval) > XIO_IOVLEN &&
*((int *)optval) <= XIO_MAX_IOV) {
g_options.max_out_iovsz = *((int *)optval);
if (rdma_transport &&
rdma_transport->set_opt)
retval |= rdma_transport->set_opt(
xio_obj, optname,
optval, optlen);
if (tcp_transport &&
tcp_transport->set_opt)
retval |= tcp_transport->set_opt(
xio_obj, optname,
optval, optlen);
}
return retval;
}
break;
case XIO_OPTNAME_ENABLE_DMA_LATENCY:
if (optlen == sizeof(int)) {
struct xio_transport *rdma_transport =
xio_get_transport("rdma");
struct xio_transport *tcp_transport =
xio_get_transport("tcp");
int retval = 0;
if (rdma_transport &&
rdma_transport->set_opt)
retval |= rdma_transport->set_opt(
xio_obj, optname,
optval, optlen);
if (tcp_transport &&
tcp_transport->set_opt)
retval |= tcp_transport->set_opt(
xio_obj, optname,
optval, optlen);
return retval;
}
break;
case XIO_OPTNAME_ENABLE_RECONNECT:
g_options.reconnect = *((int *)optval);
if (g_options.reconnect){
g_options.enable_keepalive = 0;
}
return 0;
case XIO_OPTNAME_ENABLE_FLOW_CONTROL:
g_options.enable_flow_control = *((int *)optval);
return 0;
case XIO_OPTNAME_SND_QUEUE_DEPTH_MSGS:
if (*((int *)optval) < 1)
break;
g_options.snd_queue_depth_msgs = (int)*((uint64_t *)optval);
return 0;
case XIO_OPTNAME_RCV_QUEUE_DEPTH_MSGS:
if (*((int *)optval) < 1)
break;
g_options.rcv_queue_depth_msgs = *((int *)optval);
return 0;
case XIO_OPTNAME_SND_QUEUE_DEPTH_BYTES:
if (*((uint64_t *)optval) < 1)
break;
g_options.snd_queue_depth_bytes = *((uint64_t *)optval);
return 0;
case XIO_OPTNAME_RCV_QUEUE_DEPTH_BYTES:
if (*((uint64_t *)optval) < 1)
break;
g_options.rcv_queue_depth_bytes = *((uint64_t *)optval);
return 0;
case XIO_OPTNAME_MAX_INLINE_XIO_HEADER:
if (optlen != sizeof(int))
break;
if (*((int *)optval) < 0)
break;
g_options.max_inline_xio_hdr = *((int *)optval);
return 0;
case XIO_OPTNAME_MAX_INLINE_XIO_DATA:
if (optlen != sizeof(int))
break;
if (*((int *)optval) < 0)
break;
g_options.max_inline_xio_data = *((int *)optval);
return 0;
case XIO_OPTNAME_XFER_BUF_ALIGN:
if (optlen != sizeof(int))
break;
tmp = *(int *)optval;
if (!is_power_of_2(tmp) || !(tmp % sizeof(void *) == 0)) {
xio_set_error(EINVAL);
return -1;
}
g_options.xfer_buf_align = tmp;
return 0;
case XIO_OPTNAME_INLINE_XIO_DATA_ALIGN:
if (optlen != sizeof(int))
break;
tmp = *(int *)optval;
if (!tmp) {
g_options.inline_xio_data_align = tmp;
return 0;
}
if (!is_power_of_2(tmp) || !(tmp % sizeof(void *) == 0)) {
xio_set_error(EINVAL);
return -1;
}
g_options.inline_xio_data_align = tmp;
return 0;
case XIO_OPTNAME_ENABLE_KEEPALIVE:
g_options.enable_keepalive = *((int *)optval);
return 0;
case XIO_OPTNAME_CONFIG_KEEPALIVE:
if (optlen == sizeof(struct xio_options_keepalive)) {
memcpy(&g_options.ka, optval, optlen);
return 0;
} else {
xio_set_error(EINVAL);
return -1;
}
break;
default:
break;
}
xio_set_error(XIO_E_NOT_SUPPORTED);
return -1;
}
/*---------------------------------------------------------------------------*/
int xio_on_setup_req_recv(struct xio_connection *connection,
struct xio_task *task)
{
struct xio_msg *msg = &task->imsg;
struct xio_new_session_req req;
uint8_t *ptr;
uint16_t len;
struct xio_session_hdr hdr;
struct xio_session *session = connection->session;
int retval;
struct xio_session_event_data error_event = {
.conn = NULL,
.conn_user_context = NULL,
.event = XIO_SESSION_ERROR_EVENT,
.reason = XIO_E_SUCCESS,
.private_data = NULL,
.private_data_len = 0,
};
/* read session header */
xio_session_read_header(task, &hdr);
#ifdef XIO_SESSION_DEBUG
connection->peer_connection = hdr.connection;
connection->peer_session = hdr.session;
#endif
task->imsg.sn = hdr.serial_num;
task->connection = connection;
task->session = session;
connection->session->setup_req = msg;
connection->session->connection_srv_first = connection;
/* read the header */
ptr = (uint8_t *)msg->in.header.iov_base;
memset(&req, 0, sizeof(req));
/* session id */
len = xio_read_uint32(&session->peer_session_id, 0, ptr);
ptr = ptr + len;
/* queue depth bytes */
len = xio_read_uint64(&session->peer_snd_queue_depth_bytes, 0, ptr);
ptr = ptr + len;
len = xio_read_uint64(&session->peer_rcv_queue_depth_bytes, 0, ptr);
ptr = ptr + len;
/* queue depth msgs */
len = xio_read_uint16((uint16_t *)&session->peer_snd_queue_depth_msgs,
0, ptr);
ptr = ptr + len;
len = xio_read_uint16((uint16_t *)&session->peer_rcv_queue_depth_msgs,
0, ptr);
ptr = ptr + len;
/* uri length */
len = xio_read_uint16(&req.uri_len, 0, ptr);
ptr = ptr + len;
/* private length */
len = xio_read_uint16(&req.private_data_len, 0, ptr);
ptr = ptr + len;
if (req.uri_len) {
req.uri =
(char *)kcalloc(req.uri_len, sizeof(char), GFP_KERNEL);
if (unlikely(!req.uri)) {
xio_set_error(ENOMEM);
ERROR_LOG("uri allocation failed. len:%d\n",
req.uri_len);
goto cleanup1;
}
len = xio_read_array((uint8_t *)req.uri,
req.uri_len, 0, ptr);
ptr = ptr + len;
}
if (req.private_data_len) {
req.private_data = kcalloc(req.private_data_len,
sizeof(uint8_t), GFP_KERNEL);
if (unlikely(!req.private_data)) {
xio_set_error(ENOMEM);
ERROR_LOG("private data allocation failed. len:%d\n",
req.private_data_len);
goto cleanup2;
}
len = xio_read_array((uint8_t *)req.private_data,
req.private_data_len,
0, ptr);
ptr = ptr + len;
}
req.proto = (enum xio_proto)xio_nexus_get_proto(connection->nexus);
xio_nexus_get_peer_addr(connection->nexus,
&req.src_addr, sizeof(req.src_addr));
/* cache the task in io queue*/
xio_connection_queue_io_task(connection, task);
/* notify the upper layer */
if (connection->ses_ops.on_new_session) {
#ifdef XIO_THREAD_SAFE_DEBUG
xio_ctx_debug_thread_unlock(connection->ctx);
#endif
retval = connection->ses_ops.on_new_session(
session, &req,
connection->cb_user_context);
#ifdef XIO_THREAD_SAFE_DEBUG
xio_ctx_debug_thread_lock(connection->ctx);
#endif
if (retval)
goto cleanup2;
} else {
retval = xio_accept(session, NULL, 0, NULL, 0);
if (retval) {
ERROR_LOG("failed to auto accept session. session:%p\n",
session);
goto cleanup2;
}
}
/* Don't move session state to ONLINE. In case of multiple portals
* the accept moves the state to ACCEPTED until the first "HELLO"
* message arrives. Note that the "upper layer" may call redirect or
* reject.
*/
xio_session_notify_new_connection(session, connection);
kfree(req.private_data);
kfree(req.uri);
return 0;
cleanup2:
kfree(req.private_data);
cleanup1:
kfree(req.uri);
if (session->ses_ops.on_session_event) {
#ifdef XIO_THREAD_SAFE_DEBUG
xio_ctx_debug_thread_unlock(connection->ctx);
#endif
error_event.reason = (enum xio_status)xio_errno();
session->ses_ops.on_session_event(
session, &error_event,
session->cb_user_context);
#ifdef XIO_THREAD_SAFE_DEBUG
xio_ctx_debug_thread_lock(connection->ctx);
#endif
}
return 0;
}
/*---------------------------------------------------------------------------*/
/* xio_session_write_accept_rsp */
/*---------------------------------------------------------------------------*/
struct xio_msg *xio_session_write_accept_rsp(struct xio_session *session,
uint16_t action,
const char **portals_array,
uint16_t portals_array_len,
void *user_context,
uint16_t user_context_len)
{
struct xio_msg *msg;
uint8_t *buf;
uint8_t *ptr;
uint16_t len, i, str_len, tot_len;
/* calculate length */
tot_len = 5*sizeof(uint16_t) + sizeof(uint32_t) + 2*sizeof(uint64_t);
for (i = 0; i < portals_array_len; i++)
tot_len += strlen(portals_array[i]) + sizeof(uint16_t);
tot_len += user_context_len;
if (tot_len > SETUP_BUFFER_LEN) {
ERROR_LOG("buffer is too small\n");
xio_set_error(EMSGSIZE);
return NULL;
}
/* allocate message */
buf = (uint8_t *)kcalloc(SETUP_BUFFER_LEN + sizeof(struct xio_msg),
sizeof(uint8_t), GFP_KERNEL);
if (unlikely(!buf)) {
ERROR_LOG("message allocation failed\n");
xio_set_error(ENOMEM);
return NULL;
}
/* fill the message */
msg = (struct xio_msg *)buf;
msg->out.header.iov_base = buf + sizeof(struct xio_msg);
msg->out.header.iov_len = 0;
ptr = (uint8_t *)msg->out.header.iov_base;
len = 0;
/* serialize message into the buffer */
/* session_id */
len = xio_write_uint32(session->session_id, 0, ptr);
ptr = ptr + len;
/* action */
len = xio_write_uint16(action, 0, ptr);
ptr = ptr + len;
if (action == XIO_ACTION_ACCEPT) {
/* tx queue depth bytes */
len = xio_write_uint64(session->snd_queue_depth_bytes, 0, ptr);
ptr = ptr + len;
/* rx queue depth bytes */
len = xio_write_uint64(session->rcv_queue_depth_bytes, 0, ptr);
ptr = ptr + len;
/* tx queue depth msgs */
len = xio_write_uint16(session->snd_queue_depth_msgs, 0, ptr);
ptr = ptr + len;
/* rx queue depth msgs */
len = xio_write_uint16(session->rcv_queue_depth_msgs, 0, ptr);
ptr = ptr + len;
}
/* portals_array_len */
len = xio_write_uint16(portals_array_len, 0, ptr);
ptr = ptr + len;
/* user_context_len */
len = xio_write_uint16(user_context_len, 0, ptr);
ptr = ptr + len;
for (i = 0; i < portals_array_len; i++) {
str_len = strlen(portals_array[i]);
len = xio_write_uint16(str_len, 0, ptr);
ptr = ptr + len;
len = xio_write_array((uint8_t *)portals_array[i],
str_len, 0, ptr);
ptr = ptr + len;
}
if (user_context_len) {
len = xio_write_array((const uint8_t *)user_context,
user_context_len,
0, ptr);
ptr = ptr + len;
}
msg->out.header.iov_len = ptr - (uint8_t *)msg->out.header.iov_base;
if (msg->out.header.iov_len != tot_len) {
ERROR_LOG("calculated length %d != actual length %zd\n",
tot_len, msg->out.header.iov_len);
}
return msg;
}
/*---------------------------------------------------------------------------*/
/* xio_session_write_reject_rsp */
/*---------------------------------------------------------------------------*/
struct xio_msg *xio_session_write_reject_rsp(struct xio_session *session,
enum xio_status reason,
void *user_context,
uint16_t user_context_len)
{
struct xio_msg *msg;
uint8_t *buf;
uint8_t *ptr;
uint16_t len, tot_len;
uint16_t action = XIO_ACTION_REJECT;
/* calclate length */
tot_len = 2*sizeof(uint16_t) + 2*sizeof(uint32_t);
tot_len += user_context_len;
if (tot_len > SETUP_BUFFER_LEN) {
ERROR_LOG("buffer is too small\n");
xio_set_error(EMSGSIZE);
return NULL;
}
/* allocate message */
buf = (uint8_t *)kcalloc(SETUP_BUFFER_LEN + sizeof(struct xio_msg),
sizeof(uint8_t), GFP_KERNEL);
if (!buf) {
ERROR_LOG("message allocation failed\n");
xio_set_error(ENOMEM);
return NULL;
}
/* fill the message */
msg = (struct xio_msg *)buf;
msg->out.header.iov_base = buf + sizeof(struct xio_msg);
msg->out.header.iov_len = 0;
ptr = (uint8_t *)msg->out.header.iov_base;
len = 0;
/* serialize message into the buffer */
/* session_id */
len = xio_write_uint32(session->session_id, 0, ptr);
ptr = ptr + len;
/* action */
len = xio_write_uint16(action, 0, ptr);
ptr = ptr + len;
/* reason */
len = xio_write_uint32(reason, 0, ptr);
ptr = ptr + len;
/* user_context_len */
len = xio_write_uint16(user_context_len, 0, ptr);
ptr = ptr + len;
if (user_context_len) {
len = xio_write_array((const uint8_t *)user_context,
user_context_len,
0, ptr);
ptr = ptr + len;
}
msg->out.header.iov_len = ptr - (uint8_t *)msg->out.header.iov_base;
if (msg->out.header.iov_len != tot_len) {
ERROR_LOG("calculated length %d != actual length %zd\n",
tot_len, msg->out.header.iov_len);
}
return msg;
}
/*---------------------------------------------------------------------------*/
/* xio_accept */
/*---------------------------------------------------------------------------*/
int xio_accept(struct xio_session *session,
const char **portals_array,
size_t portals_array_len,
void *user_context,
size_t user_context_len)
{
int retval = 0;
struct xio_msg *msg;
struct xio_task *task;
msg = xio_session_write_accept_rsp(session,
XIO_ACTION_ACCEPT,
portals_array,
portals_array_len,
user_context,
user_context_len);
if (!msg) {
ERROR_LOG("setup request creation failed\n");
return -1;
}
msg->request = session->setup_req;
msg->type = (enum xio_msg_type)XIO_SESSION_SETUP_RSP;
task = container_of(msg->request,
struct xio_task, imsg);
if (portals_array_len != 0) {
/* server side state is changed to ACCEPT, will be move to
* ONLINE state when first "hello" message arrives
*/
session->state = XIO_SESSION_STATE_ACCEPTED;
/* temporary disable teardown */
session->disable_teardown = 1;
TRACE_LOG("session state is now ACCEPT. session:%p\n",
session);
} else {
/* initialize credits */
task->connection->peer_credits_msgs =
session->peer_rcv_queue_depth_msgs;
task->connection->credits_msgs = 0;
task->connection->peer_credits_bytes =
session->peer_rcv_queue_depth_bytes;
task->connection->credits_bytes = 0;
/* server side state is changed to ONLINE, immediately */
session->state = XIO_SESSION_STATE_ONLINE;
TRACE_LOG("session state changed to ONLINE. session:%p\n",
session);
}
retval = xio_connection_send(task->connection, msg);
if (retval && retval != -EAGAIN) {
ERROR_LOG("failed to send message. errno:%d\n", -retval);
xio_set_error(-retval);
return -1;
}
return 0;
}
/*---------------------------------------------------------------------------*/
struct xio_context *xio_context_create(unsigned int flags,
struct xio_loop_ops *loop_ops,
struct task_struct *worker,
int polling_timeout,
int cpu_hint)
{
struct xio_context *ctx;
struct dentry *xio_root;
char name[32];
int cpu;
if (cpu_hint > 0 && cpu_hint >= num_online_cpus()) {
xio_set_error(EINVAL);
ERROR_LOG("cpu_hint(%d) >= num_online_cpus(%d)\n",
cpu_hint, num_online_cpus());
goto cleanup0;
}
if ((flags == XIO_LOOP_USER_LOOP) &&
(!(loop_ops && loop_ops->add_event && loop_ops->ev_loop))) {
xio_set_error(EINVAL);
ERROR_LOG("loop_ops and ev_loop and ev_loop_add_event are mandatory with loop_ops\n");
goto cleanup0;
}
xio_read_logging_level();
/* no need to disable preemption */
cpu = raw_smp_processor_id();
if (cpu == -1)
goto cleanup0;
/* allocate new context */
ctx = kzalloc(sizeof(struct xio_context), GFP_KERNEL);
if (ctx == NULL) {
xio_set_error(ENOMEM);
ERROR_LOG("kzalloc failed\n");
goto cleanup0;
}
if (cpu_hint < 0)
cpu_hint = cpu;
ctx->flags = flags;
ctx->cpuid = cpu_hint;
ctx->nodeid = cpu_to_node(cpu_hint);
ctx->polling_timeout = polling_timeout;
ctx->workqueue = xio_workqueue_create(ctx);
if (!ctx->workqueue) {
xio_set_error(ENOMEM);
ERROR_LOG("xio_workqueue_init failed.\n");
goto cleanup1;
}
XIO_OBSERVABLE_INIT(&ctx->observable, ctx);
INIT_LIST_HEAD(&ctx->ctx_list);
switch (flags) {
case XIO_LOOP_USER_LOOP:
break;
case XIO_LOOP_GIVEN_THREAD:
set_cpus_allowed_ptr(worker, cpumask_of(cpu_hint));
ctx->worker = (uint64_t) worker;
break;
case XIO_LOOP_TASKLET:
break;
case XIO_LOOP_WORKQUEUE:
break;
default:
ERROR_LOG("wrong type. %u\n", flags);
goto cleanup2;
}
xio_root = xio_debugfs_root();
if (xio_root) {
/* More then one contexts can share the core */
sprintf(name, "ctx-%d-%p", cpu_hint, worker);
ctx->ctx_dentry = debugfs_create_dir(name, xio_root);
if (!ctx->ctx_dentry) {
ERROR_LOG("debugfs entry %s create failed\n", name);
goto cleanup2;
}
}
ctx->ev_loop = xio_ev_loop_init(flags, ctx, loop_ops);
if (!ctx->ev_loop)
goto cleanup3;
ctx->stats.hertz = HZ;
/* Initialize default counters' name */
ctx->stats.name[XIO_STAT_TX_MSG] = kstrdup("TX_MSG", GFP_KERNEL);
ctx->stats.name[XIO_STAT_RX_MSG] = kstrdup("RX_MSG", GFP_KERNEL);
ctx->stats.name[XIO_STAT_TX_BYTES] = kstrdup("TX_BYTES", GFP_KERNEL);
ctx->stats.name[XIO_STAT_RX_BYTES] = kstrdup("RX_BYTES", GFP_KERNEL);
ctx->stats.name[XIO_STAT_DELAY] = kstrdup("DELAY", GFP_KERNEL);
ctx->stats.name[XIO_STAT_APPDELAY] = kstrdup("APPDELAY", GFP_KERNEL);
return ctx;
cleanup3:
debugfs_remove_recursive(ctx->ctx_dentry);
ctx->ctx_dentry = NULL;
cleanup2:
xio_workqueue_destroy(ctx->workqueue);
cleanup1:
kfree(ctx);
cleanup0:
ERROR_LOG("xio_ctx_open failed\n");
return NULL;
}
/*---------------------------------------------------------------------------*/
struct xio_server *xio_bind(struct xio_context *ctx,
struct xio_session_ops *ops,
const char *uri,
uint16_t *src_port,
uint32_t session_flags,
void *cb_private_data)
{
struct xio_server *server;
int retval;
int backlog = 4;
if (!ctx || !ops || !uri) {
ERROR_LOG("invalid parameters ctx:%p, ops:%p, uri:%p\n",
ctx, ops, uri);
xio_set_error(EINVAL);
return NULL;
}
TRACE_LOG("bind to %s\n", uri);
/* create the server */
server = (struct xio_server *)
kcalloc(1, sizeof(struct xio_server), GFP_KERNEL);
if (!server) {
xio_set_error(ENOMEM);
return NULL;
}
kref_init(&server->kref);
/* fill server data*/
server->ctx = ctx;
server->cb_private_data = cb_private_data;
server->uri = kstrdup(uri, GFP_KERNEL);
server->session_flags = session_flags;
memcpy(&server->ops, ops, sizeof(*ops));
XIO_OBSERVER_INIT(&server->observer, server, xio_on_nexus_event);
XIO_OBSERVABLE_INIT(&server->nexus_observable, server);
server->listener = xio_nexus_open(ctx, uri, NULL, 0, 0, NULL);
if (!server->listener) {
ERROR_LOG("failed to create connection\n");
goto cleanup;
}
retval = xio_nexus_listen(server->listener,
uri, src_port, backlog);
if (retval != 0) {
ERROR_LOG("connection listen failed\n");
goto cleanup1;
}
xio_nexus_set_server(server->listener, server);
xio_idr_add_uobj(usr_idr, server, "xio_server");
return server;
cleanup1:
xio_nexus_close(server->listener, NULL);
cleanup:
kfree(server->uri);
kfree(server);
return NULL;
}
