/**
* Initialize threads
*
* \return void
*/
void
fsd_workthread_init(void)
{
int i;
for (i = 0; i < EXA_FS_REQUEST_LAST; ++i)
{
char thread_name[32];
char queue_name[EXA_MAXSIZE_REQUEST_QUEUE_NAME+1];
os_snprintf(thread_name, sizeof(thread_name), "exa_fsd_%d", i);
os_snprintf(queue_name, sizeof(queue_name), "queue_%d", i);
/* Initialize queue */
requests[i] = daemon_request_queue_new(queue_name);
EXA_ASSERT(requests[i]);
exathread_create_named (&fs_thread[i],
FSD_THREAD_STACK_SIZE+MIN_THREAD_STACK_SIZE,
& fsd_workthread_run, (void *)(requests[i]),
thread_name);
}
}
/**
* init_plugin initialise internal data
*
* if we are server, try to bind and launch a thread to accept
* launch the thread of receive data
*
* @param net_plugin Info on the new instance that we will fill
*
* @return EXA_SUCCESS or error
*/
int algopr_init_plugin(exa_nodeid_t node_id, int max_buffer_size)
{
struct sockaddr_in serv_addr;
int retval = -NBD_ERR_MALLOC_FAILED;
int i;
EXA_ASSERT(EXA_NODEID_VALID(node_id));
this_node_id = node_id;
init_peers();
eth.max_buffer_size = max_buffer_size + max_buffer_size;
wq_init(ð.wq_send);
nbd_init_root(MAX_BIG_RECV_ELT, eth.max_buffer_size, ð.root_list_big_recv);
nbd_init_root(MAX_SEND_ELT, sizeof(payload_t), ð.root_list_send);
for (i = 0; i < EXA_MAX_NODES_NUMBER; i++)
nbd_init_list(ð.root_list_send, ð.send_list[i]);
algopr_run = true;
if (!exathread_create_named(ð.receive_thread,
NBD_THREAD_STACK_SIZE +
MIN_THREAD_STACK_SIZE_OF_THIS_PLUGIN,
algopr_receive_thread, NULL, "AlgoPrRcv"))
return -NBD_ERR_THREAD_CREATION;
if (!exathread_create_named(ð.send_thread,
NBD_THREAD_STACK_SIZE +
MIN_THREAD_STACK_SIZE_OF_THIS_PLUGIN,
algopr_send_thread, NULL, "AlgoPrSnd"))
return -NBD_ERR_THREAD_CREATION;
eth.accept_sock = os_socket(PF_INET, SOCK_STREAM, 0);
if (eth.accept_sock < 0)
return -EXA_ERR_CREATE_SOCKET;
/* bind a socket to SERVERD_DATA_PORT port and make it listen for incoming
* connections */
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = INADDR_ANY;
serv_addr.sin_port = htons(algopr_network_port);
retval = internal_setsock_opt(eth.accept_sock, SOCK_LISTEN_FLAGS);
if (retval != EXA_SUCCESS)
return retval;
if (os_bind(eth.accept_sock, (struct sockaddr *) &serv_addr,
sizeof(serv_addr)) < 0)
return -EXA_ERR_CREATE_SOCKET;
if (os_listen(eth.accept_sock, EXA_MAX_NODES_NUMBER) < 0)
return -EXA_ERR_CREATE_SOCKET;
if (!exathread_create_named(ð.accept_thread,
NBD_THREAD_STACK_SIZE +
MIN_THREAD_STACK_SIZE_OF_THIS_PLUGIN,
accept_thread, NULL, "servEthAccPlugin"))
return -NBD_ERR_THREAD_CREATION;
return EXA_SUCCESS;
}
/**
* Initialize the RDEV service:
* - start exa_rdev kernel module,
* - allocate and initialize aligned buffers to read/writes superblocks.
*/
static int
rdev_init(int thr_nb)
{
char path[OS_PATH_MAX];
int err = 0;
if (os_kmod_load("exa_rdev") != 0)
{
exalog_error("Failed to load kernel module 'exa_rdev'");
return -ADMIND_ERR_MODULESTART;
}
/* Load the broken disks table */
err = exa_env_make_path(path, sizeof(path), exa_env_cachedir(), "broken_disks");
if (err != 0)
return err;
err = broken_disk_table_load(&broken_disks, path, true /* open_read_write */);
if (err != 0)
{
exalog_error("Failed loading the broken disk table: %s (%d)",
exa_error_msg(err), err);
return err;
}
/* Initialize the rdev module */
err = exa_rdev_static_init(RDEV_STATIC_CREATE);
if (err != 0)
{
exalog_error("Failed initializing rdev statics: %s (%d)", exa_error_msg(err), err);
goto cleanup_broken;
}
exa_rdev_fd = exa_rdev_init();
if (exa_rdev_fd <= 0)
{
err = exa_rdev_fd;
exalog_error("Failed initializing rdev: %s (%d)", exa_error_msg(err), err);
goto cleanup_broken;
}
mh = examsgInit(EXAMSG_RDEV_ID);
if (!mh)
{
exalog_error("Failed initializing messaging for disk checking thread");
err = -ENOMEM;
goto cleanup_rdev_fd;
}
COMPILE_TIME_ASSERT(RDEV_SUPERBLOCK_SIZE <= SECTORS_TO_BYTES(RDEV_RESERVED_AREA_IN_SECTORS));
rdev_check_buffer = os_aligned_malloc(RDEV_SUPERBLOCK_SIZE, 4096, NULL);
if (!rdev_check_buffer)
{
exalog_error("Failed allocating disk checking buffer");
err = -ENOMEM;
goto cleanup_mh;
}
/* make sure the check thread will not quit */
quit = false;
/* launch the rdev checking thread */
if (!exathread_create_named(&rdev_check_id, MIN_THREAD_STACK_SIZE,
disk_checking_thread, mh, "rdev_check"))
{
exalog_error("Failed creating disk checking thread");
err = -EXA_ERR_DEFAULT;
goto cleanup_rdev_check_buffer;
}
rdev_check_id_started = true;
return EXA_SUCCESS;
cleanup_rdev_check_buffer:
os_aligned_free(rdev_check_buffer);
rdev_check_buffer = NULL;
cleanup_mh:
examsgExit(mh);
mh = NULL;
cleanup_rdev_fd:
close(exa_rdev_fd);
exa_rdev_fd = -1;
cleanup_broken:
broken_disk_table_unload(&broken_disks);
return err;
}
static void
check_internal_msg(void)
{
struct timeval timeout = { .tv_sec = 0, .tv_usec = EXAMSG_TIMEOUT };
static Examsg msg;
command_end_t *end;
int i, ret;
ret = examsgWaitTimeout(cli_mh, &timeout);
if (ret < 0 && ret != -ETIME)
{
exalog_error("Message wait failed %s (%d)",
exa_error_msg(ret), ret);
return;
}
if (ret == -ETIME)
return;
ret = examsgRecv(cli_mh, NULL, &msg, sizeof(msg));
if (ret == 0)
return;
EXA_ASSERT_VERBOSE(ret > 0, "Message receive failed: %s (%d)",
exa_error_msg(ret), ret);
if (ret < 0)
exalog_error("Message receive failed: %s (%d)",
exa_error_msg(ret), ret);
/* The CLI server can only receive EXAMSG_ADM_CLUSTER_CMD_END messages for now */
EXA_ASSERT(msg.any.type == EXAMSG_ADM_CLUSTER_CMD_END);
end = (command_end_t *)msg.payload;
for (i = 0; i < MAX_CONNECTION; i++)
if (end->cuid == connectlist[i].uid)
{
cli_command_end_complete(connectlist[i].fd, &end->err_desc);
connectlist[i].uid = CMD_UID_INVALID;
break;
}
EXA_ASSERT(i < MAX_CONNECTION);
}
static void
check_tcp_connection(void)
{
static struct timeval timeout = { .tv_sec = 0, .tv_usec = 0 };
fd_set setSave = setSocks;
int ret, conn_id;
do
ret = os_select(FD_SETSIZE, &setSave, NULL, NULL, &timeout);
while (ret == -EINTR);
if (ret < 0)
{
/* FIXME should assert ? */
exalog_debug("Select failed %m");
return;
}
/* Check working sockets */
for (conn_id = 0; conn_id < MAX_CONNECTION; ++conn_id)
{
int sock_fd = connectlist[conn_id].fd;
if (sock_fd >= 0 && FD_ISSET(sock_fd, &setSave))
handle_inputdata(conn_id, sock_fd);
}
/* Must be done at the end to make sure messages for current
* working threads are processed first */
if (FD_ISSET(listen_fd, &setSave))
accept_new_client();
}
/*-------------------------------------------------------------------------*/
/** \brief Connection thread: wait on xml message and pass the command
* to the work thread.
*
* \param[in] sock_xml: socket xml on which it receives commands.
*
*/
/*-------------------------------------------------------------------------*/
static void
cli_server(void *data)
{
int i;
/* Initialize exalog */
exalog_as(EXAMSG_ADMIND_ID);
exalog_debug("cli_server: started");
/* Initialization */
FD_ZERO(&setSocks);
FD_SET(listen_fd, &setSocks);
for (i = 0; i < MAX_CONNECTION; i++)
{
connectlist[i].fd = -1;
/* A command cannot be CMD_UID_INVALID, so CMD_UID_INVALID means here
* no command running */
connectlist[i].uid = CMD_UID_INVALID;
}
while (!stop)
{
check_tcp_connection();
check_internal_msg();
}
os_closesocket(listen_fd);
os_net_cleanup();
examsgDelMbox(cli_mh, EXAMSG_ADMIND_CLISERVER_ID);
examsgExit(cli_mh);
}
int
cli_server_start(void)
{
listen_fd = listen_socket_port(ADMIND_SOCKET_PORT);
if (listen_fd < 0)
return listen_fd;
cli_mh = examsgInit(EXAMSG_ADMIND_CLISERVER_ID);
if (!cli_mh)
return -EINVAL;
/* The mailbox needs to be able to receive command end messages from the
* event manager; as there can be at most MAX_CONNECTION client connections
* we can receive at the time at most 10 command end messages. */
examsgAddMbox(cli_mh, EXAMSG_ADMIND_CLISERVER_ID, MAX_CONNECTION,
sizeof(command_end_t));
stop = false;
if (!exathread_create_named(&thr_xml_proto,
ADMIND_THREAD_STACK_SIZE+MIN_THREAD_STACK_SIZE,
&cli_server, NULL, "exa_adm_xml"))
return -EXA_ERR_DEFAULT;
return EXA_SUCCESS;
}
/* A new device is handled by the server, do the init operations in order to make the device usable */
int export_device(const exa_uuid_t *uuid, char *device_path)
{
device_t *dev;
int i, err;
/* If device was already exported, do nothing */
if (find_device_from_uuid(uuid) != NULL)
return EXA_SUCCESS;
dev = os_malloc(sizeof(struct device));
if (dev == NULL)
{
err = -NBD_ERR_MALLOC_FAILED;
goto error;
}
dev->handle = NULL;
err = -CMD_EXP_ERR_OPEN_DEVICE;
dev->handle = exa_rdev_handle_alloc(device_path);
if (dev->handle == NULL)
goto error;
err = get_nb_sectors(device_path, &dev->size_in_sectors);
if (err != EXA_SUCCESS)
goto error;
uuid_copy(&dev->uuid, uuid);
strlcpy(dev->path, device_path, sizeof(dev->path));
for (i = 0; i < NBMAX_DISKS_PER_NODE; i++)
if (nbd_server.devices[i] == NULL)
break;
if (i >= NBMAX_DISKS_PER_NODE)
{
exalog_error("maximum number of exportable devices exceeded");
err = -NBD_ERR_NB_RDEVS_CREATED;
}
dev->dev_index = i;
dev->exit_thread = false;
nbd_init_list(&nbd_server.list_root, &dev->disk_queue);
/* resource needed to lock/unlock a zone */
os_sem_init (&dev->lock_sem_disk, 0);
/* launch disk thread (TD) */
if (!exathread_create_named(&nbd_server.td_pid[dev->dev_index],
NBD_THREAD_STACK_SIZE + MIN_THREAD_STACK_SIZE,
exa_td_main, dev, "TD_thread"))
{
os_sem_destroy(&dev->lock_sem_disk);
err = -NBD_ERR_THREAD_CREATION;
goto error;
}
nbd_server.devices[dev->dev_index] = dev;
return EXA_SUCCESS;
error:
if (dev != NULL)
{
if (dev->handle != NULL)
exa_rdev_handle_free(dev->handle);
os_free(dev);
}
return err;
}
