/*---------------------------------------------------------------------------*/
int xio_get_opt(void *xio_obj, int level, int optname,
void *optval, int *optlen)
{
static struct xio_transport *rdma_transport;
static struct xio_transport *tcp_transport;
switch (level) {
case XIO_OPTLEVEL_ACCELIO:
return xio_general_get_opt(xio_obj, optname, optval, optlen);
case XIO_OPTLEVEL_RDMA:
if (!rdma_transport) {
rdma_transport = xio_get_transport("rdma");
if (!rdma_transport) {
xio_set_error(EFAULT);
return -1;
}
}
if (!rdma_transport->get_opt)
break;
return rdma_transport->get_opt(xio_obj,
optname, optval, optlen);
break;
case XIO_OPTLEVEL_TCP:
if (!tcp_transport) {
tcp_transport = xio_get_transport("tcp");
if (!tcp_transport) {
xio_set_error(EFAULT);
return -1;
}
}
if (!tcp_transport->get_opt)
break;
return tcp_transport->get_opt(xio_obj,
optname, optval, optlen);
break;
default:
break;
}
xio_set_error(XIO_E_NOT_SUPPORTED);
return -1;
}
/*---------------------------------------------------------------------------*/
struct xio_context *xio_context_create(struct xio_context_params *ctx_params,
int polling_timeout,
int cpu_hint)
{
struct xio_context *ctx;
struct xio_loop_ops *loop_ops;
struct task_struct *worker;
struct xio_transport *transport;
int flags, cpu;
if (!ctx_params) {
xio_set_error(EINVAL);
ERROR_LOG("ctx_params is NULL\n");
goto cleanup0;
}
loop_ops = ctx_params->loop_ops;
worker = ctx_params->worker;
flags = ctx_params->flags;
if (cpu_hint > 0 && cpu_hint >= num_online_cpus()) {
xio_set_error(EINVAL);
ERROR_LOG("cpu_hint(%d) >= num_online_cpus(%d)\n",
cpu_hint, num_online_cpus());
goto cleanup0;
}
if ((flags == XIO_LOOP_USER_LOOP) &&
(!(loop_ops && loop_ops->add_event && loop_ops->ev_loop))) {
xio_set_error(EINVAL);
ERROR_LOG("loop_ops and ev_loop and ev_loop_add_event are " \
"mandatory with loop_ops\n");
goto cleanup0;
}
xio_read_logging_level();
/* no need to disable preemption */
cpu = raw_smp_processor_id();
if (cpu == -1)
goto cleanup0;
/* allocate new context */
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
if (!ctx) {
xio_set_error(ENOMEM);
ERROR_LOG("kzalloc failed\n");
goto cleanup0;
}
if (cpu_hint < 0)
cpu_hint = cpu;
ctx->run_private = 0;
ctx->user_context = ctx_params->user_context;
ctx->flags = flags;
ctx->cpuid = cpu_hint;
ctx->nodeid = cpu_to_node(cpu_hint);
ctx->polling_timeout = polling_timeout;
ctx->prealloc_xio_inline_bufs =
!!ctx_params->prealloc_xio_inline_bufs;
ctx->rq_depth = ctx_params->rq_depth;
if (!ctx_params->max_conns_per_ctx)
ctx->max_conns_per_ctx = 100;
else
ctx->max_conns_per_ctx =
max(ctx_params->max_conns_per_ctx , 2);
ctx->workqueue = xio_workqueue_create(ctx);
if (!ctx->workqueue) {
xio_set_error(ENOMEM);
ERROR_LOG("xio_workqueue_init failed.\n");
goto cleanup1;
}
ctx->msg_pool = xio_objpool_create(sizeof(struct xio_msg),
MSGPOOL_INIT_NR, MSGPOOL_GROW_NR);
if (!ctx->msg_pool) {
xio_set_error(ENOMEM);
ERROR_LOG("context's msg_pool create failed. %m\n");
goto cleanup2;
}
XIO_OBSERVABLE_INIT(&ctx->observable, ctx);
INIT_LIST_HEAD(&ctx->ctx_list);
switch (flags) {
case XIO_LOOP_USER_LOOP:
break;
case XIO_LOOP_GIVEN_THREAD:
set_cpus_allowed_ptr(worker, cpumask_of(cpu_hint));
ctx->worker = (uint64_t)worker;
break;
case XIO_LOOP_TASKLET:
break;
case XIO_LOOP_WORKQUEUE:
break;
default:
ERROR_LOG("wrong type. %u\n", flags);
goto cleanup3;
}
ctx->ev_loop = xio_ev_loop_init(flags, ctx, loop_ops);
if (!ctx->ev_loop)
goto cleanup3;
ctx->stats.hertz = HZ;
/* Initialize default counters' name */
ctx->stats.name[XIO_STAT_TX_MSG] = kstrdup("TX_MSG", GFP_KERNEL);
ctx->stats.name[XIO_STAT_RX_MSG] = kstrdup("RX_MSG", GFP_KERNEL);
ctx->stats.name[XIO_STAT_TX_BYTES] = kstrdup("TX_BYTES", GFP_KERNEL);
ctx->stats.name[XIO_STAT_RX_BYTES] = kstrdup("RX_BYTES", GFP_KERNEL);
ctx->stats.name[XIO_STAT_DELAY] = kstrdup("DELAY", GFP_KERNEL);
ctx->stats.name[XIO_STAT_APPDELAY] = kstrdup("APPDELAY", GFP_KERNEL);
/* initialize rdma pools only */
transport = xio_get_transport("rdma");
if (transport && ctx->prealloc_xio_inline_bufs) {
int retval = xio_ctx_pool_create(ctx, XIO_PROTO_RDMA,
XIO_CONTEXT_POOL_CLASS_INITIAL);
if (retval) {
ERROR_LOG("Failed to create initial pool. ctx:%p\n", ctx);
goto cleanup2;
}
retval = xio_ctx_pool_create(ctx, XIO_PROTO_RDMA,
XIO_CONTEXT_POOL_CLASS_PRIMARY);
if (retval) {
ERROR_LOG("Failed to create primary pool. ctx:%p\n", ctx);
goto cleanup2;
}
}
spin_lock_init(&ctx->ctx_list_lock);
xio_idr_add_uobj(usr_idr, ctx, "xio_context");
return ctx;
cleanup3:
xio_objpool_destroy(ctx->msg_pool);
cleanup2:
xio_workqueue_destroy(ctx->workqueue);
cleanup1:
kfree(ctx);
cleanup0:
ERROR_LOG("xio_ctx_open failed\n");
return NULL;
}
/*---------------------------------------------------------------------------*/
int xio_ctx_pool_create(struct xio_context *ctx, enum xio_proto proto,
enum xio_context_pool_class pool_cls)
{
struct xio_tasks_pool_ops *pool_ops;
struct xio_tasks_pool **tasks_pool;
struct xio_transport *transport;
struct xio_tasks_pool_params params;
char pool_name[64];
const char *proto_str = xio_proto_str(proto);
/* get the transport's proto */
transport = xio_get_transport(proto_str);
if (!transport) {
ERROR_LOG("failed to load %s transport layer.\n", proto_str);
ERROR_LOG("validate that your system support %s " \
"and the accelio's %s module is loaded\n",
proto_str, proto_str);
xio_set_error(ENOPROTOOPT);
return -1;
}
if (transport->get_pools_setup_ops) {
if (!ctx->primary_pool_ops[proto] ||
!ctx->initial_pool_ops[proto])
transport->get_pools_setup_ops(
NULL,
&ctx->initial_pool_ops[proto],
&ctx->primary_pool_ops[proto]);
} else {
ERROR_LOG("transport does not implement " \
"\"get_pools_setup_ops\"\n");
return -1;
}
switch(pool_cls) {
case XIO_CONTEXT_POOL_CLASS_INITIAL:
tasks_pool = &ctx->initial_tasks_pool[proto];
pool_ops = ctx->initial_pool_ops[proto];
sprintf(pool_name, "ctx:%p - initial_pool_%s", ctx, proto_str);
break;
case XIO_CONTEXT_POOL_CLASS_PRIMARY:
tasks_pool = &ctx->primary_tasks_pool[proto];
pool_ops = ctx->primary_pool_ops[proto];
sprintf(pool_name, "ctx:%p - primary_pool_%s", ctx, proto_str);
break;
default:
xio_set_error(EINVAL);
ERROR_LOG("unknown pool class\n");
return -1;
};
/* if already exist */
if (*tasks_pool)
return 0;
if (!pool_ops)
return -1;
if (!pool_ops->pool_get_params ||
!pool_ops->slab_pre_create ||
!pool_ops->slab_init_task ||
!pool_ops->pool_post_create ||
!pool_ops->slab_destroy)
return -1;
/* get pool properties from the transport */
memset(¶ms, 0, sizeof(params));
pool_ops->pool_get_params(NULL,
(int *)¶ms.start_nr,
(int *)¶ms.max_nr,
(int *)¶ms.alloc_nr,
(int *)¶ms.pool_dd_data_sz,
(int *)¶ms.slab_dd_data_sz,
(int *)¶ms.task_dd_data_sz);
if (ctx->prealloc_xio_inline_bufs) {
params.start_nr = params.max_nr;
params.alloc_nr = 0;
}
params.pool_hooks.slab_pre_create =
(int (*)(void *, int, void *, void *))
pool_ops->slab_pre_create;
params.pool_hooks.slab_post_create = (int (*)(void *, void *, void *))
pool_ops->slab_post_create;
params.pool_hooks.slab_destroy = (int (*)(void *, void *, void *))
pool_ops->slab_destroy;
params.pool_hooks.slab_init_task =
(int (*)(void *, void *, void *, int, struct xio_task *))
pool_ops->slab_init_task;
params.pool_hooks.slab_uninit_task =
(int (*)(void *, void *, void *, struct xio_task *))
pool_ops->slab_uninit_task;
params.pool_hooks.slab_remap_task =
(int (*)(void *, void *, void *, void *, struct xio_task *))
pool_ops->slab_remap_task;
params.pool_hooks.pool_pre_create = (int (*)(void *, void *, void *))
pool_ops->pool_pre_create;
params.pool_hooks.pool_post_create = (int (*)(void *, void *, void *))
pool_ops->pool_post_create;
params.pool_hooks.pool_destroy = (int (*)(void *, void *, void *))
pool_ops->pool_destroy;
params.pool_hooks.task_pre_put = (int (*)(void *, struct xio_task *))
pool_ops->task_pre_put;
params.pool_hooks.task_post_get = (int (*)(void *, struct xio_task *))
pool_ops->task_post_get;
params.pool_name = kstrdup(pool_name, GFP_KERNEL);
/* initialize the tasks pool */
*tasks_pool = xio_tasks_pool_create(¶ms);
if (!*tasks_pool) {
ERROR_LOG("xio_tasks_pool_create failed\n");
return -1;
}
return 0;
}
/*---------------------------------------------------------------------------*/
static int xio_general_set_opt(void *xio_obj, int optname,
const void *optval, int optlen)
{
int tmp;
switch (optname) {
case XIO_OPTNAME_LOG_FN:
if (optlen == 0 && !optval)
return xio_set_log_fn(NULL);
else if (optlen == sizeof(xio_log_fn))
return xio_set_log_fn((xio_log_fn)optval);
break;
case XIO_OPTNAME_LOG_LEVEL:
if (optlen != sizeof(enum xio_log_level))
return -1;
return xio_set_log_level(*((enum xio_log_level *)optval));
case XIO_OPTNAME_DISABLE_HUGETBL:
xio_disable_huge_pages(*((int *)optval));
return 0;
case XIO_OPTNAME_MEM_ALLOCATOR:
if (optlen == sizeof(struct xio_mem_allocator))
return xio_set_mem_allocator(
(struct xio_mem_allocator *)optval);
break;
case XIO_OPTNAME_CONFIG_MEMPOOL:
if (optlen == sizeof(struct xio_mempool_config)) {
memcpy(&g_mempool_config,
(struct xio_mempool_config *)optval, optlen);
return 0;
}
break;
case XIO_OPTNAME_MAX_IN_IOVLEN:
if (optlen == sizeof(int)) {
struct xio_transport *rdma_transport =
xio_get_transport("rdma");
struct xio_transport *tcp_transport =
xio_get_transport("tcp");
int retval = 0;
if (*((int *)optval) > XIO_IOVLEN &&
*((int *)optval) <= XIO_MAX_IOV) {
g_options.max_in_iovsz = *((int *)optval);
if (rdma_transport &&
rdma_transport->set_opt)
retval |= rdma_transport->set_opt(
xio_obj, optname,
optval, optlen);
if (tcp_transport &&
tcp_transport->set_opt)
retval |= tcp_transport->set_opt(
xio_obj, optname,
optval, optlen);
}
return retval;
}
break;
case XIO_OPTNAME_MAX_OUT_IOVLEN:
if (optlen == sizeof(int)) {
struct xio_transport *rdma_transport =
xio_get_transport("rdma");
struct xio_transport *tcp_transport =
xio_get_transport("tcp");
int retval = 0;
if (*((int *)optval) > XIO_IOVLEN &&
*((int *)optval) <= XIO_MAX_IOV) {
g_options.max_out_iovsz = *((int *)optval);
if (rdma_transport &&
rdma_transport->set_opt)
retval |= rdma_transport->set_opt(
xio_obj, optname,
optval, optlen);
if (tcp_transport &&
tcp_transport->set_opt)
retval |= tcp_transport->set_opt(
xio_obj, optname,
optval, optlen);
}
return retval;
}
break;
case XIO_OPTNAME_ENABLE_DMA_LATENCY:
if (optlen == sizeof(int)) {
struct xio_transport *rdma_transport =
xio_get_transport("rdma");
struct xio_transport *tcp_transport =
xio_get_transport("tcp");
int retval = 0;
if (rdma_transport &&
rdma_transport->set_opt)
retval |= rdma_transport->set_opt(
xio_obj, optname,
optval, optlen);
if (tcp_transport &&
tcp_transport->set_opt)
retval |= tcp_transport->set_opt(
xio_obj, optname,
optval, optlen);
return retval;
}
break;
case XIO_OPTNAME_ENABLE_RECONNECT:
g_options.reconnect = *((int *)optval);
if (g_options.reconnect){
g_options.enable_keepalive = 0;
}
return 0;
case XIO_OPTNAME_ENABLE_FLOW_CONTROL:
g_options.enable_flow_control = *((int *)optval);
return 0;
case XIO_OPTNAME_SND_QUEUE_DEPTH_MSGS:
if (*((int *)optval) < 1)
break;
g_options.snd_queue_depth_msgs = (int)*((uint64_t *)optval);
return 0;
case XIO_OPTNAME_RCV_QUEUE_DEPTH_MSGS:
if (*((int *)optval) < 1)
break;
g_options.rcv_queue_depth_msgs = *((int *)optval);
return 0;
case XIO_OPTNAME_SND_QUEUE_DEPTH_BYTES:
if (*((uint64_t *)optval) < 1)
break;
g_options.snd_queue_depth_bytes = *((uint64_t *)optval);
return 0;
case XIO_OPTNAME_RCV_QUEUE_DEPTH_BYTES:
if (*((uint64_t *)optval) < 1)
break;
g_options.rcv_queue_depth_bytes = *((uint64_t *)optval);
return 0;
case XIO_OPTNAME_MAX_INLINE_XIO_HEADER:
if (optlen != sizeof(int))
break;
if (*((int *)optval) < 0)
break;
g_options.max_inline_xio_hdr = *((int *)optval);
return 0;
case XIO_OPTNAME_MAX_INLINE_XIO_DATA:
if (optlen != sizeof(int))
break;
if (*((int *)optval) < 0)
break;
g_options.max_inline_xio_data = *((int *)optval);
return 0;
case XIO_OPTNAME_XFER_BUF_ALIGN:
if (optlen != sizeof(int))
break;
tmp = *(int *)optval;
if (!is_power_of_2(tmp) || !(tmp % sizeof(void *) == 0)) {
xio_set_error(EINVAL);
return -1;
}
g_options.xfer_buf_align = tmp;
return 0;
case XIO_OPTNAME_INLINE_XIO_DATA_ALIGN:
if (optlen != sizeof(int))
break;
tmp = *(int *)optval;
if (!tmp) {
g_options.inline_xio_data_align = tmp;
return 0;
}
if (!is_power_of_2(tmp) || !(tmp % sizeof(void *) == 0)) {
xio_set_error(EINVAL);
return -1;
}
g_options.inline_xio_data_align = tmp;
return 0;
case XIO_OPTNAME_ENABLE_KEEPALIVE:
g_options.enable_keepalive = *((int *)optval);
return 0;
case XIO_OPTNAME_CONFIG_KEEPALIVE:
if (optlen == sizeof(struct xio_options_keepalive)) {
memcpy(&g_options.ka, optval, optlen);
return 0;
} else {
xio_set_error(EINVAL);
return -1;
}
break;
default:
break;
}
xio_set_error(XIO_E_NOT_SUPPORTED);
return -1;
}
