/**
* Cleanup mailboxes & stuff.
*/
static void
cleanup(void)
{
network_exit();
examsgDelMbox(local_mh, EXAMSG_NETMBOX_ID);
examsgDelMbox(local_mh, EXAMSG_CMSGD_ID);
examsgExit(net_mh);
net_mh = NULL;
examsgExit(local_mh);
local_mh = NULL;
os_net_cleanup();
}
/**
* Cleanup the rdev service:
* - free the buffers allocated bu rdev_init(),
* - stop the exa_rdev kernel module.
*/
static int
rdev_shutdown(int thr_nb)
{
broken_disk_table_unload(&broken_disks);
/* close the rdev checking thread */
quit = true;
if (rdev_check_id_started)
os_thread_join(rdev_check_id);
rdev_check_id_started = false;
os_aligned_free(rdev_check_buffer);
if (exa_rdev_fd > 0)
close(exa_rdev_fd);
exa_rdev_static_clean(RDEV_STATIC_DELETE);
examsgExit(mh);
if (os_kmod_unload("exa_rdev") != 0)
return -ADMIND_ERR_MODULESTOP;
return EXA_SUCCESS;
}
/**
* Free examsg mailbox
*/
void
sup_cleanup_messaging(void)
{
examsgDelMbox(sup_mh, EXAMSG_CSUPD_ID);
examsgExit(sup_mh);
sup_mh = NULL;
}
/**
* Set up the messaging.
*
* \param[in] local_id Local node's id
*
* \return true if successfull, false otherwise
*/
bool
sup_setup_messaging(exa_nodeid_t local_id)
{
int err;
sup_mh = examsgInit(EXAMSG_CSUPD_ID);
if (!sup_mh)
return false;
/* Create local mailbox, buffer at most SUP_MAX_PING_MSG ping messages */
err = examsgAddMbox(sup_mh, EXAMSG_CSUPD_ID, SUP_MAX_PING_MSG,
sizeof(sup_ping_msg_t));
if (err)
{
__error("cannot create mailbox : %s (%d)", exa_error_msg(err), err);
if (sup_mh)
examsgExit(sup_mh);
return false;
}
return true;
}
int main(int argc, char *argv[])
{
bool static_init_ok = false;
const char *action;
int err = 0;
self = os_program_name(argv[0]);
if (argc <= 1)
{
usage();
err = 1;
goto done;
}
err = examsg_static_init(EXAMSG_STATIC_GET);
if (err != 0)
{
fprintf(stderr, "examsg_static_init failed: %s (%d)\n", exa_error_msg(-err), err);
goto done;
}
static_init_ok = true;
mh = examsgInit(EXAMSG_TEST_ID);
if (!mh)
{
fprintf(stderr, "examsgInit failed\n");
goto done;
}
err = examsgAddMbox(mh, examsgOwner(mh), 3, EXAMSG_MSG_MAX);
if (err != 0)
{
fprintf(stderr, "examsgAddMbox failed: %s (%d)\n", exa_error_msg(-err), err);
goto done;
}
action = argv[1];
if (strcmp(action, "is_fs_mounted") == 0)
{
int m;
if (argc != 3)
{
usage();
goto done;
}
m = fsd_is_fs_mounted(mh, argv[2]);
if (m < 0)
err = m;
}
else if (strcmp(action, "is_mountpoint_used") == 0)
{
int u;
if (argc != 3)
{
usage();
goto done;
}
u = fsd_is_mountpoint_used(mh, argv[2]);
if (u < 0)
err = u;
}
else if (strcmp(action, "prepare_gfs") == 0)
{
fs_data_t fs;
size_t sz;
if (argc != 3)
{
usage();
goto done;
}
COMPILE_TIME_ASSERT(sizeof("sfs") <= sizeof(fs.fstype));
os_strlcpy(fs.fstype, "sfs", sizeof(fs.fstype));
sz = os_strlcpy(fs.clustered.gfs.lock_protocol, argv[2],
sizeof(fs.clustered.gfs.lock_protocol));
if (sz >= sizeof(fs.clustered.gfs.lock_protocol))
{
fprintf(stderr, "Invalid lock protocol: '%s' (too long)\n", argv[2]);
goto done;
}
err = fsd_prepare(mh, &fs);
}
else if (strcmp(action, "mount") == 0)
{
fs_data_t fs;
size_t sz;
if (argc != 7)
{
usage();
goto done;
}
sz = os_strlcpy(fs.fstype, argv[2], sizeof(fs.fstype));
if (sz >= sizeof(fs.fstype))
{
fprintf(stderr, "Invalid fs type: '%s' (too long)\n", argv[2]);
goto done;
}
sz = os_strlcpy(fs.mountpoint, argv[3], sizeof(fs.mountpoint));
if (sz >= sizeof(fs.mountpoint))
{
fprintf(stderr, "Invalid mountpoint: '%s' (too long)\n", argv[3]);
goto done;
}
sz = os_strlcpy(fs.devpath, argv[4], sizeof(fs.devpath));
if (sz >= sizeof(fs.devpath))
{
fprintf(stderr, "Invalid dev path: '%s' (too long)\n", argv[4]);
goto done;
}
err = fsd_mount(mh, &fs, 1 , 0, argv[5], argv[6]);
}
else if (strcmp(action, "umount") == 0)
{
fs_data_t fs;
size_t sz;
if (argc != 6)
{
usage();
goto done;
}
sz = os_strlcpy(fs.mountpoint, argv[2], sizeof(fs.mountpoint));
if (sz >= sizeof(fs.mountpoint))
{
fprintf(stderr, "Invalid mountpoint: '%s' (too long)\n", argv[2]);
goto done;
}
sz = os_strlcpy(fs.devpath, argv[3], sizeof(fs.devpath));
if (sz >= sizeof(fs.devpath))
{
fprintf(stderr, "Invalid dev path: '%s' (too long)\n", argv[3]);
goto done;
}
err = fsd_umount(mh, &fs, argv[5], argv[6]);
}
else if (strcmp(action, "unload") == 0)
{
fs_data_t fs;
size_t sz;
if (argc != 3)
{
usage();
goto done;
}
sz = os_strlcpy(fs.fstype, argv[2], sizeof(fs.fstype));
if (sz >= sizeof(fs.fstype))
{
fprintf(stderr, "Invalid fs type: '%s' (too long)\n", argv[2]);
goto done;
}
err = fsd_unload(mh, &fs);
}
else if (strcmp(action, "create_local") == 0)
{
if (argc != 4)
{
usage();
goto done;
}
err = fsd_fs_create_local(mh, argv[2], argv[3]);
}
else if (strcmp(action, "create_gfs") == 0)
{
fs_data_t fs;
size_t sz;
if (argc != 7)
{
usage();
goto done;
}
COMPILE_TIME_ASSERT(sizeof("sfs") <= sizeof(fs.fstype))
sz = os_strlcpy(fs.fstype, "sfs", sizeof(fs.fstype));
sz = os_strlcpy(fs.devpath, argv[2], sizeof(fs.devpath));
if (sz >= sizeof(fs.devpath))
{
fprintf(stderr, "Invalid dev path: '%s' (too long)\n", argv[2]);
goto done;
}
sz = os_strlcpy(fs.clustered.gfs.lock_protocol, argv[3],
sizeof(fs.clustered.gfs.lock_protocol));
if (sz >= sizeof(fs.clustered.gfs.lock_protocol))
{
fprintf(stderr, "Invalid lock protocol: '%s' (too long)\n", argv[3]);
goto done;
}
if (to_uint64(argv[4], &fs.sizeKB) != EXA_SUCCESS)
{
fprintf(stderr, "Invalid size: '%s'\n", argv[4]);
goto done;
}
sz = os_strlcpy(fs.clustered.gfs.uuid, argv[5], sizeof(fs.clustered.gfs.uuid));
if (sz >= sizeof(fs.clustered.gfs.uuid))
{
fprintf(stderr, "Invalid GFS uuid: '%s' (too long)\n", argv[5]);
goto done;
}
if (to_uint64(argv[6], &fs.clustered.gfs.nb_logs) != EXA_SUCCESS)
{
fprintf(stderr, "Invalid number of logs: '%s'\n", argv[6]);
goto done;
}
err = fsd_fs_create_gfs(mh, &fs);
}
else if (strcmp(action, "dfinfo") == 0)
{
struct fsd_capa buf;
if (argc != 3)
{
usage();
goto done;
}
err = fsd_df(mh, argv[2], &buf);
if (err == 0)
printf("size=%"PRId64" bytes\n"
"used=%"PRId64" bytes\n"
"free=%"PRId64" bytes\n",
buf.size, buf.used, buf.free);
}
else if (strcmp(action, "resize") == 0)
{
uint64_t size_kb;
if (argc != 6)
{
usage();
goto done;
}
if (to_uint64(argv[5], &size_kb) != EXA_SUCCESS)
{
fprintf(stderr, "Invalid new size: '%s'\n", argv[5]);
goto done;
}
err = fsd_resize(mh, argv[2], argv[3], argv[4], size_kb);
}
else if (strcmp(action, "prepare_resize") == 0)
{
if (argc != 4)
{
usage();
goto done;
}
err = fsd_prepare_resize(mh, argv[2], argv[3]);
}
else if (strcmp(action, "read_shm") == 0)
{
read_shm();
}
else if (strcmp(action, "add_logs") == 0)
{
fs_data_t fs;
int num_logs, actual_num_logs;
size_t sz;
if (argc != 4)
{
usage();
goto done;
}
COMPILE_TIME_ASSERT(sizeof("sfs") <= sizeof(fs.fstype));
os_strlcpy(fs.fstype, "sfs", sizeof(fs.fstype));
sz = os_strlcpy(fs.devpath, argv[2], sizeof(fs.devpath));
if (sz >= sizeof(fs.devpath))
{
fprintf(stderr, "Invalid dev path: '%s' (too long)\n", argv[2]);
goto done;
}
if (to_int(argv[3], &num_logs) != EXA_SUCCESS)
{
fprintf(stderr, "Invalid number of logs: '%s'\n", argv[3]);
goto done;
}
actual_num_logs = fsd_set_gfs_logs(mh, &fs, num_logs);
if (actual_num_logs < 0)
err = actual_num_logs;
else
{
examsgDelMbox(mh, EXAMSG_TEST_ID);
printf("Number of logs after the operation: %d\n", actual_num_logs);
}
}
else
usage();
if (err != 0)
fprintf(stderr, "Action finished with error %d: %s\n", err,
exa_error_msg(-err));
done:
examsgDelMbox(mh, EXAMSG_TEST_ID);
if (mh != NULL)
examsgExit(mh);
if (static_init_ok)
examsg_static_clean(EXAMSG_STATIC_RELEASE);
return err == 0 ? 0 : 1;
}
static void
check_internal_msg(void)
{
struct timeval timeout = { .tv_sec = 0, .tv_usec = EXAMSG_TIMEOUT };
static Examsg msg;
command_end_t *end;
int i, ret;
ret = examsgWaitTimeout(cli_mh, &timeout);
if (ret < 0 && ret != -ETIME)
{
exalog_error("Message wait failed %s (%d)",
exa_error_msg(ret), ret);
return;
}
if (ret == -ETIME)
return;
ret = examsgRecv(cli_mh, NULL, &msg, sizeof(msg));
if (ret == 0)
return;
EXA_ASSERT_VERBOSE(ret > 0, "Message receive failed: %s (%d)",
exa_error_msg(ret), ret);
if (ret < 0)
exalog_error("Message receive failed: %s (%d)",
exa_error_msg(ret), ret);
/* The CLI server can only receive EXAMSG_ADM_CLUSTER_CMD_END messages for now */
EXA_ASSERT(msg.any.type == EXAMSG_ADM_CLUSTER_CMD_END);
end = (command_end_t *)msg.payload;
for (i = 0; i < MAX_CONNECTION; i++)
if (end->cuid == connectlist[i].uid)
{
cli_command_end_complete(connectlist[i].fd, &end->err_desc);
connectlist[i].uid = CMD_UID_INVALID;
break;
}
EXA_ASSERT(i < MAX_CONNECTION);
}
static void
check_tcp_connection(void)
{
static struct timeval timeout = { .tv_sec = 0, .tv_usec = 0 };
fd_set setSave = setSocks;
int ret, conn_id;
do
ret = os_select(FD_SETSIZE, &setSave, NULL, NULL, &timeout);
while (ret == -EINTR);
if (ret < 0)
{
/* FIXME should assert ? */
exalog_debug("Select failed %m");
return;
}
/* Check working sockets */
for (conn_id = 0; conn_id < MAX_CONNECTION; ++conn_id)
{
int sock_fd = connectlist[conn_id].fd;
if (sock_fd >= 0 && FD_ISSET(sock_fd, &setSave))
handle_inputdata(conn_id, sock_fd);
}
/* Must be done at the end to make sure messages for current
* working threads are processed first */
if (FD_ISSET(listen_fd, &setSave))
accept_new_client();
}
/*-------------------------------------------------------------------------*/
/** \brief Connection thread: wait on xml message and pass the command
* to the work thread.
*
* \param[in] sock_xml: socket xml on which it receives commands.
*
*/
/*-------------------------------------------------------------------------*/
static void
cli_server(void *data)
{
int i;
/* Initialize exalog */
exalog_as(EXAMSG_ADMIND_ID);
exalog_debug("cli_server: started");
/* Initialization */
FD_ZERO(&setSocks);
FD_SET(listen_fd, &setSocks);
for (i = 0; i < MAX_CONNECTION; i++)
{
connectlist[i].fd = -1;
/* A command cannot be CMD_UID_INVALID, so CMD_UID_INVALID means here
* no command running */
connectlist[i].uid = CMD_UID_INVALID;
}
while (!stop)
{
check_tcp_connection();
check_internal_msg();
}
os_closesocket(listen_fd);
os_net_cleanup();
examsgDelMbox(cli_mh, EXAMSG_ADMIND_CLISERVER_ID);
examsgExit(cli_mh);
}
int
cli_server_start(void)
{
listen_fd = listen_socket_port(ADMIND_SOCKET_PORT);
if (listen_fd < 0)
return listen_fd;
cli_mh = examsgInit(EXAMSG_ADMIND_CLISERVER_ID);
if (!cli_mh)
return -EINVAL;
/* The mailbox needs to be able to receive command end messages from the
* event manager; as there can be at most MAX_CONNECTION client connections
* we can receive at the time at most 10 command end messages. */
examsgAddMbox(cli_mh, EXAMSG_ADMIND_CLISERVER_ID, MAX_CONNECTION,
sizeof(command_end_t));
stop = false;
if (!exathread_create_named(&thr_xml_proto,
ADMIND_THREAD_STACK_SIZE+MIN_THREAD_STACK_SIZE,
&cli_server, NULL, "exa_adm_xml"))
return -EXA_ERR_DEFAULT;
return EXA_SUCCESS;
}
/**
* Initialize the RDEV service:
* - start exa_rdev kernel module,
* - allocate and initialize aligned buffers to read/writes superblocks.
*/
static int
rdev_init(int thr_nb)
{
char path[OS_PATH_MAX];
int err = 0;
if (os_kmod_load("exa_rdev") != 0)
{
exalog_error("Failed to load kernel module 'exa_rdev'");
return -ADMIND_ERR_MODULESTART;
}
/* Load the broken disks table */
err = exa_env_make_path(path, sizeof(path), exa_env_cachedir(), "broken_disks");
if (err != 0)
return err;
err = broken_disk_table_load(&broken_disks, path, true /* open_read_write */);
if (err != 0)
{
exalog_error("Failed loading the broken disk table: %s (%d)",
exa_error_msg(err), err);
return err;
}
/* Initialize the rdev module */
err = exa_rdev_static_init(RDEV_STATIC_CREATE);
if (err != 0)
{
exalog_error("Failed initializing rdev statics: %s (%d)", exa_error_msg(err), err);
goto cleanup_broken;
}
exa_rdev_fd = exa_rdev_init();
if (exa_rdev_fd <= 0)
{
err = exa_rdev_fd;
exalog_error("Failed initializing rdev: %s (%d)", exa_error_msg(err), err);
goto cleanup_broken;
}
mh = examsgInit(EXAMSG_RDEV_ID);
if (!mh)
{
exalog_error("Failed initializing messaging for disk checking thread");
err = -ENOMEM;
goto cleanup_rdev_fd;
}
COMPILE_TIME_ASSERT(RDEV_SUPERBLOCK_SIZE <= SECTORS_TO_BYTES(RDEV_RESERVED_AREA_IN_SECTORS));
rdev_check_buffer = os_aligned_malloc(RDEV_SUPERBLOCK_SIZE, 4096, NULL);
if (!rdev_check_buffer)
{
exalog_error("Failed allocating disk checking buffer");
err = -ENOMEM;
goto cleanup_mh;
}
/* make sure the check thread will not quit */
quit = false;
/* launch the rdev checking thread */
if (!exathread_create_named(&rdev_check_id, MIN_THREAD_STACK_SIZE,
disk_checking_thread, mh, "rdev_check"))
{
exalog_error("Failed creating disk checking thread");
err = -EXA_ERR_DEFAULT;
goto cleanup_rdev_check_buffer;
}
rdev_check_id_started = true;
return EXA_SUCCESS;
cleanup_rdev_check_buffer:
os_aligned_free(rdev_check_buffer);
rdev_check_buffer = NULL;
cleanup_mh:
examsgExit(mh);
mh = NULL;
cleanup_rdev_fd:
close(exa_rdev_fd);
exa_rdev_fd = -1;
cleanup_broken:
broken_disk_table_unload(&broken_disks);
return err;
}
void rebuild_helper_thread(void *p)
{
ExamsgHandle mh;
int err;
exalog_as(EXAMSG_NBD_SERVER_ID);
/* initialize examsg framework */
mh = examsgInit(EXAMSG_NBD_LOCKING_ID);
EXA_ASSERT(mh != NULL);
err = examsgAddMbox(mh, EXAMSG_NBD_LOCKING_ID, 1, 5 * EXAMSG_MSG_MAX);
EXA_ASSERT(err == 0);
os_sem_post(&nbd_server.mailbox_sem);
while (nbd_server.run)
{
device_t *device;
ExamsgNbdLock nbd_lock_msg;
ExamsgMID from;
struct timeval timeout = { .tv_sec = 0, .tv_usec = 100000 };
exa_nodeset_t dest_nodes;
err = examsgWaitTimeout(mh, &timeout);
/* Just in order to check stopping the thread is required*/
if (err == -ETIME)
continue;
if (err != 0)
{
exalog_error("Locking thread encountered error %s (%d) while "
"waiting in event loop.", exa_error_msg(err), err);
continue;
}
err = examsgRecv(mh, &from, &nbd_lock_msg, sizeof(nbd_lock_msg));
/* No message */
if (err == 0)
continue;
if (err < 0)
{
exalog_error("Locking thread encountered error %s (%d) while "
"receiving a messsage.", exa_error_msg(err), err);
continue;
}
switch(nbd_lock_msg.any.type)
{
case EXAMSG_NBD_LOCK:
/* find device from name */
/* FIXME devices lock is not held... it should */
device = find_device_from_uuid(&nbd_lock_msg.disk_uuid);
if (device == NULL)
{
exalog_error("Unknown device with UUID " UUID_FMT, UUID_VAL(&nbd_lock_msg.disk_uuid));
err = -CMD_EXP_ERR_UNKNOWN_DEVICE;
break;
}
if (nbd_lock_msg.lock)
{
err = exa_disk_lock_zone(device, nbd_lock_msg.locked_zone_start,
nbd_lock_msg.locked_zone_size);
EXA_ASSERT_VERBOSE(err == 0, "Trying to lock too many zone "
"(>%d). Last zone not succesfully locked "
"(start = %" PRId64 ", size = %" PRId64 " ) "
"on device UUID " UUID_FMT, NBMAX_DISK_LOCKED_ZONES,
nbd_lock_msg.locked_zone_start,
nbd_lock_msg.locked_zone_size,
UUID_VAL(&nbd_lock_msg.disk_uuid));
}
else
{
err = exa_disk_unlock_zone(device, nbd_lock_msg.locked_zone_start,
nbd_lock_msg.locked_zone_size);
EXA_ASSERT_VERBOSE(err == 0, "Trying to unlock a never locked "
"zone (unlocked zone start =%" PRId64 ", "
"unlocked zone size = %" PRId64 ") on device"
" UUID " UUID_FMT,
nbd_lock_msg.locked_zone_start,
nbd_lock_msg.locked_zone_size,
UUID_VAL(&nbd_lock_msg.disk_uuid));
}
break;
default:
/* error */
EXA_ASSERT_VERBOSE(false, "Locking thread got unknown message of"
" type %d ", nbd_lock_msg.any.type);
break;
}
exa_nodeset_single(&dest_nodes, from.netid.node);
examsgAckReply(mh, (Examsg *)&nbd_lock_msg, err, from.id, &dest_nodes);
}
examsgDelMbox(mh, EXAMSG_NBD_LOCKING_ID);
examsgExit(mh);
}
/** get the number of sector of the device
* \param device_path the device to get the number of sector
* \param nb_sectors64 the number of sectors of the device
* \return nb_sectors the returned number of sector
*/
static int get_nb_sectors(const char *device_path, uint64_t *nb_sectors)
{
uint64_t device_size; /* in bytes */
int retval;
int fd;
/* We need the read access to get the size. */
if ((fd = os_disk_open_raw(device_path, OS_DISK_READ)) < 0)
{
exalog_error("cannot open device '%s' error=%s ",
device_path, exa_error_msg(-fd));
return -CMD_EXP_ERR_OPEN_DEVICE;
}
retval = os_disk_get_size(fd, &device_size);
if (retval < 0)
{
exalog_error("os_disk_get_size() error=%s", exa_error_msg(retval));
if (close(fd) != 0)
exalog_error("can't EVEN close dev '%s'", device_path);
return -EXA_ERR_IOCTL;
}
retval = close(fd);
if (retval < 0)
{
retval = -errno;
exalog_error("cannot close device '%s' error=%s ",
device_path, exa_error_msg(retval));
return -CMD_EXP_ERR_CLOSE_DEVICE;
}
*nb_sectors = device_size / SECTOR_SIZE;
/* remove the size of the reserved area for storing admind info */
*nb_sectors -= RDEV_RESERVED_AREA_IN_SECTORS;
/* Align the size on 1K
* this is the best we can do to have the same size of devices on 2.4 and 2.6 kernels due to
* the fact that kernel 2.4 rounds the size of devices with 1 K
*/
*nb_sectors -= *nb_sectors % (1024 / SECTOR_SIZE);
return EXA_SUCCESS;
}