int setup_cluster(void)
{
cs_error_t err;
int fd;
err = quorum_initialize(&qh, &quorum_callbacks);
if (err != CS_OK) {
log_error("quorum init error %d", err);
return -1;
}
err = quorum_fd_get(qh, &fd);
if (err != CS_OK) {
log_error("quorum fd_get error %d", err);
goto fail;
}
err = quorum_trackstart(qh, CS_TRACK_CURRENT);
if (err != CS_OK) {
log_error("quorum trackstart error %d", err);
goto fail;
}
old_node_count = 0;
memset(&old_nodes, 0, sizeof(old_nodes));
quorum_node_count = 0;
memset(&quorum_nodes, 0, sizeof(quorum_nodes));
return fd;
fail:
quorum_finalize(qh);
return -1;
}
gboolean
cluster_connect_quorum(gboolean(*dispatch) (unsigned long long, gboolean),
void (*destroy) (gpointer))
{
int rc = -1;
int fd = 0;
int quorate = 0;
uint32_t quorum_type = 0;
struct mainloop_fd_callbacks quorum_fd_callbacks;
quorum_fd_callbacks.dispatch = pcmk_quorum_dispatch;
quorum_fd_callbacks.destroy = destroy;
crm_debug("Configuring Pacemaker to obtain quorum from Corosync");
rc = quorum_initialize(&pcmk_quorum_handle, &quorum_callbacks, &quorum_type);
if (rc != CS_OK) {
crm_err("Could not connect to the Quorum API: %d\n", rc);
goto bail;
} else if (quorum_type != QUORUM_SET) {
crm_err("Corosync quorum is not configured\n");
goto bail;
}
rc = quorum_getquorate(pcmk_quorum_handle, &quorate);
if (rc != CS_OK) {
crm_err("Could not obtain the current Quorum API state: %d\n", rc);
goto bail;
}
crm_notice("Quorum %s", quorate ? "acquired" : "lost");
quorum_app_callback = dispatch;
crm_have_quorum = quorate;
rc = quorum_trackstart(pcmk_quorum_handle, CS_TRACK_CHANGES | CS_TRACK_CURRENT);
if (rc != CS_OK) {
crm_err("Could not setup Quorum API notifications: %d\n", rc);
goto bail;
}
rc = quorum_fd_get(pcmk_quorum_handle, &fd);
if (rc != CS_OK) {
crm_err("Could not obtain the Quorum API connection: %d\n", rc);
goto bail;
}
mainloop_add_fd("quorum", G_PRIORITY_HIGH, fd, dispatch, &quorum_fd_callbacks);
corosync_initialize_nodelist(NULL, FALSE, NULL);
bail:
if (rc != CS_OK) {
quorum_finalize(pcmk_quorum_handle);
return FALSE;
}
return TRUE;
}
static void _cluster_closedown(void)
{
DEBUGLOG("cluster_closedown\n");
destroy_lvhash();
dlm_release_lockspace(LOCKSPACE_NAME, lockspace, 1);
cpg_finalize(cpg_handle);
quorum_finalize(quorum_handle);
}
void
terminate_cs_connection(crm_cluster_t *cluster)
{
cluster_disconnect_cpg(cluster);
if (pcmk_quorum_handle) {
quorum_finalize(pcmk_quorum_handle);
pcmk_quorum_handle = 0;
}
crm_notice("Disconnected from Corosync");
}
static void my_pre_exit(void)
{
qb_log (LOG_INFO, "PRE EXIT");
if (vq_handle) {
votequorum_finalize(vq_handle);
vq_handle = 0;
}
if (q_handle) {
quorum_finalize(q_handle);
q_handle = 0;
}
}
static void close_all(void) {
if (cmap_handle) {
cmap_finalize(cmap_handle);
}
if (q_handle) {
quorum_finalize(q_handle);
}
if (c_handle) {
corosync_cfg_finalize(c_handle);
}
if (v_handle) {
votequorum_finalize(v_handle);
}
}
static int q_dispatch_wrapper_fn (
int fd,
int revents,
void *data)
{
cs_error_t error = quorum_dispatch (q_handle, CS_DISPATCH_ALL);
if (error != CS_OK) {
qb_log (LOG_ERR, "got %s error, disconnecting.",
cs_strerror(error));
quorum_finalize(q_handle);
q_handle = 0;
return -1;
}
return 0;
}
void
terminate_cs_connection(crm_cluster_t *cluster)
{
crm_notice("Disconnecting from Corosync");
cluster_disconnect_cpg(cluster);
if (pcmk_quorum_handle) {
crm_trace("Disconnecting quorum");
quorum_finalize(pcmk_quorum_handle);
pcmk_quorum_handle = 0;
} else {
crm_info("No Quorum connection");
}
}
void close_cluster(void)
{
quorum_trackstop(qh);
quorum_finalize(qh);
}
static int _init_cluster(void)
{
cs_error_t err;
#ifdef QUORUM_SET /* corosync/quorum.h */
uint32_t quorum_type;
#endif
node_hash = dm_hash_create(100);
err = cpg_initialize(&cpg_handle,
&corosync_cpg_callbacks);
if (err != CS_OK) {
syslog(LOG_ERR, "Cannot initialise Corosync CPG service: %d",
err);
DEBUGLOG("Cannot initialise Corosync CPG service: %d", err);
return cs_to_errno(err);
}
#ifdef QUORUM_SET
err = quorum_initialize(&quorum_handle,
&quorum_callbacks,
&quorum_type);
if (quorum_type != QUORUM_SET) {
syslog(LOG_ERR, "Corosync quorum service is not configured");
DEBUGLOG("Corosync quorum service is not configured");
return EINVAL;
}
#else
err = quorum_initialize(&quorum_handle,
&quorum_callbacks);
#endif
if (err != CS_OK) {
syslog(LOG_ERR, "Cannot initialise Corosync quorum service: %d",
err);
DEBUGLOG("Cannot initialise Corosync quorum service: %d", err);
return cs_to_errno(err);
}
/* Create a lockspace for LV & VG locks to live in */
lockspace = dlm_open_lockspace(LOCKSPACE_NAME);
if (!lockspace) {
lockspace = dlm_create_lockspace(LOCKSPACE_NAME, 0600);
if (!lockspace) {
syslog(LOG_ERR, "Unable to create DLM lockspace for CLVM: %m");
return -1;
}
DEBUGLOG("Created DLM lockspace for CLVMD.\n");
} else
DEBUGLOG("Opened existing DLM lockspace for CLVMD.\n");
dlm_ls_pthread_init(lockspace);
DEBUGLOG("DLM initialisation complete\n");
/* Connect to the clvmd group */
strcpy((char *)cpg_group_name.value, "clvmd");
cpg_group_name.length = strlen((char *)cpg_group_name.value);
err = cpg_join(cpg_handle, &cpg_group_name);
if (err != CS_OK) {
cpg_finalize(cpg_handle);
quorum_finalize(quorum_handle);
dlm_release_lockspace(LOCKSPACE_NAME, lockspace, 1);
syslog(LOG_ERR, "Cannot join clvmd process group");
DEBUGLOG("Cannot join clvmd process group: %d\n", err);
return cs_to_errno(err);
}
err = cpg_local_get(cpg_handle,
&our_nodeid);
if (err != CS_OK) {
cpg_finalize(cpg_handle);
quorum_finalize(quorum_handle);
dlm_release_lockspace(LOCKSPACE_NAME, lockspace, 1);
syslog(LOG_ERR, "Cannot get local node id\n");
return cs_to_errno(err);
}
DEBUGLOG("Our local node id is %d\n", our_nodeid);
DEBUGLOG("Connected to Corosync\n");
return 0;
}
static int show_nodes(nodeid_format_t nodeid_format, name_format_t name_format)
{
quorum_handle_t q_handle = 0;
votequorum_handle_t v_handle = 0;
corosync_cfg_handle_t c_handle = 0;
corosync_cfg_callbacks_t c_callbacks;
int i;
int using_vq = 0;
quorum_callbacks_t q_callbacks;
votequorum_callbacks_t v_callbacks;
int err;
int result = EXIT_FAILURE;
q_callbacks.quorum_notify_fn = quorum_notification_fn;
err=quorum_initialize(&q_handle, &q_callbacks);
if (err != CS_OK) {
fprintf(stderr, "Cannot connect to quorum service, is it loaded?\n");
return result;
}
v_callbacks.votequorum_notify_fn = NULL;
v_callbacks.votequorum_expectedvotes_notify_fn = NULL;
using_vq = using_votequorum();
if (using_vq) {
if ( (err=votequorum_initialize(&v_handle, &v_callbacks)) != CS_OK) {
fprintf(stderr, "votequorum_initialize FAILED: %d, this is probably a configuration error\n", err);
v_handle = 0;
goto err_exit;
}
}
err = quorum_trackstart(q_handle, CS_TRACK_CURRENT);
if (err != CS_OK) {
fprintf(stderr, "quorum_trackstart FAILED: %d\n", err);
goto err_exit;
}
g_called = 0;
while (g_called == 0)
quorum_dispatch(q_handle, CS_DISPATCH_ONE);
quorum_finalize(q_handle);
q_handle = 0;
err = corosync_cfg_initialize(&c_handle, &c_callbacks);
if (err != CS_OK) {
fprintf(stderr, "Cannot initialise CFG service\n");
c_handle = 0;
goto err_exit;
}
if (using_vq)
printf("Nodeid Votes Name\n");
else
printf("Nodeid Name\n");
for (i=0; i < g_view_list_entries; i++) {
if (nodeid_format == NODEID_FORMAT_DECIMAL) {
printf("%4u ", g_view_list[i]);
}
else {
printf("0x%04x ", g_view_list[i]);
}
if (using_vq) {
printf("%3d %s\n", get_votes(v_handle, g_view_list[i]), node_name(c_handle, g_view_list[i], name_format));
}
else {
printf("%s\n", node_name(c_handle, g_view_list[i], name_format));
}
}
result = EXIT_SUCCESS;
err_exit:
if (q_handle != 0) {
quorum_finalize (q_handle);
}
if (using_vq && v_handle != 0) {
votequorum_finalize (v_handle);
}
if (c_handle != 0) {
corosync_cfg_finalize (c_handle);
}
return result;
}
static void show_status(void)
{
quorum_handle_t q_handle;
votequorum_handle_t v_handle;
votequorum_callbacks_t v_callbacks;
quorum_callbacks_t callbacks;
struct votequorum_info info;
int is_quorate;
int err;
callbacks.quorum_notify_fn = quorum_notification_fn;
err=quorum_initialize(&q_handle, &callbacks);
if (err != CS_OK) {
fprintf(stderr, "Cannot connect to quorum service, is it loaded?\n");
return;
}
err=quorum_getquorate(q_handle, &is_quorate);
if (err != CS_OK) {
fprintf(stderr, "quorum_getquorate FAILED: %d\n", err);
return;
}
err=quorum_trackstart(q_handle, CS_TRACK_CURRENT);
if (err != CS_OK) {
fprintf(stderr, "quorum_trackstart FAILED: %d\n", err);
return;
}
g_called = 0;
while (g_called == 0)
quorum_dispatch(q_handle, CS_DISPATCH_ONE);
quorum_finalize(q_handle);
printf("Version: %s\n", VERSION);
printf("Nodes: %d\n", g_view_list_entries);
printf("Ring ID: %" PRIu64 "\n", g_ring_id);
printf("Quorum type: %s\n", get_quorum_type());
printf("Quorate: %s\n", is_quorate?"Yes":"No");
if (using_votequorum()) {
v_callbacks.votequorum_notify_fn = NULL;
v_callbacks.votequorum_expectedvotes_notify_fn = NULL;
if ( (err=votequorum_initialize(&v_handle, &v_callbacks)) != CS_OK) {
fprintf(stderr, "votequorum_initialize FAILED: %d, this is probably a configuration error\n", err);
goto err_exit;
}
if ( (err=votequorum_getinfo(v_handle, 0, &info)) != CS_OK)
fprintf(stderr, "votequorum_getinfo FAILED: %d\n", err);
else {
printf("Node votes: %d\n", info.node_votes);
printf("Expected votes: %d\n", info.node_expected_votes);
printf("Highest expected: %d\n", info.highest_expected);
printf("Total votes: %d\n", info.total_votes);
printf("Quorum: %d %s\n", info.quorum, info.flags & VOTEQUORUM_INFO_FLAG_QUORATE?" ":"Activity blocked");
printf("Flags: ");
if (info.flags & VOTEQUORUM_INFO_FLAG_HASSTATE) printf("HasState ");
if (info.flags & VOTEQUORUM_INFO_FLAG_DISALLOWED) printf("DisallowedNodes ");
if (info.flags & VOTEQUORUM_INFO_FLAG_TWONODE) printf("2Node ");
if (info.flags & VOTEQUORUM_INFO_FLAG_QUORATE) printf("Quorate ");
printf("\n");
}
}
err_exit:
return;
}
gboolean
cluster_connect_quorum(gboolean(*dispatch) (unsigned long long, gboolean),
void (*destroy) (gpointer))
{
cs_error_t rc;
int fd = 0;
int quorate = 0;
uint32_t quorum_type = 0;
struct mainloop_fd_callbacks quorum_fd_callbacks;
uid_t found_uid = 0;
gid_t found_gid = 0;
pid_t found_pid = 0;
int rv;
quorum_fd_callbacks.dispatch = pcmk_quorum_dispatch;
quorum_fd_callbacks.destroy = destroy;
crm_debug("Configuring Pacemaker to obtain quorum from Corosync");
rc = quorum_initialize(&pcmk_quorum_handle, &quorum_callbacks, &quorum_type);
if (rc != CS_OK) {
crm_err("Could not connect to the Quorum API: %s (%d)",
cs_strerror(rc), rc);
goto bail;
} else if (quorum_type != QUORUM_SET) {
crm_err("Corosync quorum is not configured");
goto bail;
}
rc = quorum_fd_get(pcmk_quorum_handle, &fd);
if (rc != CS_OK) {
crm_err("Could not obtain the Quorum API connection: %s (%d)",
strerror(rc), rc);
goto bail;
}
/* Quorum provider run as root (in given user namespace, anyway)? */
if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
&found_uid, &found_gid))) {
crm_err("Quorum provider is not authentic:"
" process %lld (uid: %lld, gid: %lld)",
(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
(long long) found_uid, (long long) found_gid);
rc = CS_ERR_ACCESS;
goto bail;
} else if (rv < 0) {
crm_err("Could not verify authenticity of Quorum provider: %s (%d)",
strerror(-rv), -rv);
rc = CS_ERR_ACCESS;
goto bail;
}
rc = quorum_getquorate(pcmk_quorum_handle, &quorate);
if (rc != CS_OK) {
crm_err("Could not obtain the current Quorum API state: %d", rc);
goto bail;
}
if (quorate) {
crm_notice("Quorum acquired");
} else {
crm_warn("Quorum lost");
}
quorum_app_callback = dispatch;
crm_have_quorum = quorate;
rc = quorum_trackstart(pcmk_quorum_handle, CS_TRACK_CHANGES | CS_TRACK_CURRENT);
if (rc != CS_OK) {
crm_err("Could not setup Quorum API notifications: %d", rc);
goto bail;
}
mainloop_add_fd("quorum", G_PRIORITY_HIGH, fd, dispatch, &quorum_fd_callbacks);
corosync_initialize_nodelist(NULL, FALSE, NULL);
bail:
if (rc != CS_OK) {
quorum_finalize(pcmk_quorum_handle);
return FALSE;
}
return TRUE;
}
static void
_cs_quorum_finalize(void)
{
quorum_finalize (quorum_handle);
}
cs_error_t sam_register (
unsigned int *instance_id)
{
cs_error_t error;
pid_t pid;
int pipe_error;
int pipe_fd_out[2], pipe_fd_in[2];
enum sam_parent_action_t action, old_action;
int child_status;
sam_recovery_policy_t recpol;
if (sam_internal_data.internal_status != SAM_INTERNAL_STATUS_INITIALIZED) {
return (CS_ERR_BAD_HANDLE);
}
recpol = sam_internal_data.recovery_policy;
if (recpol & SAM_RECOVERY_POLICY_CONFDB) {
/*
* Register to objdb
*/
if ((error = sam_confdb_register ()) != CS_OK) {
goto error_exit;
}
}
error = CS_OK;
while (1) {
if ((pipe_error = pipe (pipe_fd_out)) != 0) {
error = CS_ERR_LIBRARY;
goto error_exit;
}
if ((pipe_error = pipe (pipe_fd_in)) != 0) {
close (pipe_fd_out[0]);
close (pipe_fd_out[1]);
error = CS_ERR_LIBRARY;
goto error_exit;
}
if (recpol & SAM_RECOVERY_POLICY_CONFDB) {
if ((error = sam_confdb_update_key (SAM_CONFDB_KEY_STATE, SAM_CONFDB_S_REGISTERED)) != CS_OK) {
goto error_exit;
}
}
sam_internal_data.instance_id++;
sam_internal_data.term_send = 0;
pid = fork ();
if (pid == -1) {
/*
* Fork error
*/
sam_internal_data.instance_id--;
error = CS_ERR_LIBRARY;
goto error_exit;
}
if (pid == 0) {
/*
* Child process
*/
close (pipe_fd_out[0]);
close (pipe_fd_in[1]);
sam_internal_data.child_fd_out = pipe_fd_out[1];
sam_internal_data.child_fd_in = pipe_fd_in[0];
if (instance_id)
*instance_id = sam_internal_data.instance_id;
sam_internal_data.am_i_child = 1;
sam_internal_data.internal_status = SAM_INTERNAL_STATUS_REGISTERED;
pthread_mutex_init (&sam_internal_data.lock, NULL);
goto error_exit;
} else {
/*
* Parent process
*/
close (pipe_fd_out[1]);
close (pipe_fd_in[0]);
action = sam_parent_handler (pipe_fd_out[0], pipe_fd_in[1], pid);
close (pipe_fd_out[0]);
close (pipe_fd_in[1]);
if (action == SAM_PARENT_ACTION_ERROR) {
error = CS_ERR_LIBRARY;
goto error_exit;
}
/*
* We really don't like zombies
*/
while (waitpid (pid, &child_status, 0) == -1 && errno == EINTR)
;
old_action = action;
if (action == SAM_PARENT_ACTION_RECOVERY) {
if (SAM_RP_MASK (sam_internal_data.recovery_policy) == SAM_RECOVERY_POLICY_QUIT)
action = SAM_PARENT_ACTION_QUIT;
}
if (action == SAM_PARENT_ACTION_QUIT) {
if (recpol & SAM_RECOVERY_POLICY_QUORUM) {
quorum_finalize (sam_internal_data.quorum_handle);
}
if (recpol & SAM_RECOVERY_POLICY_CONFDB) {
if (old_action == SAM_PARENT_ACTION_RECOVERY) {
/*
* Mark as failed
*/
sam_confdb_update_key (SAM_CONFDB_KEY_STATE, SAM_CONFDB_S_FAILED);
} else {
sam_confdb_destroy_pid_obj ();
}
}
exit (WEXITSTATUS (child_status));
}
}
}
error_exit:
return (error);
}
static int _init_cluster(void)
{
cs_error_t err;
node_hash = dm_hash_create(100);
err = cpg_initialize(&cpg_handle,
&corosync_cpg_callbacks);
if (err != CS_OK) {
syslog(LOG_ERR, "Cannot initialise Corosync CPG service: %d",
err);
DEBUGLOG("Cannot initialise Corosync CPG service: %d", err);
return cs_to_errno(err);
}
err = quorum_initialize(&quorum_handle,
&quorum_callbacks);
if (err != CS_OK) {
syslog(LOG_ERR, "Cannot initialise Corosync quorum service: %d",
err);
DEBUGLOG("Cannot initialise Corosync quorum service: %d", err);
return cs_to_errno(err);
}
/* Create a lockspace for LV & VG locks to live in */
lockspace = dlm_create_lockspace(LOCKSPACE_NAME, 0600);
if (!lockspace) {
if (errno == EEXIST) {
lockspace = dlm_open_lockspace(LOCKSPACE_NAME);
}
if (!lockspace) {
syslog(LOG_ERR, "Unable to create lockspace for CLVM: %m");
quorum_finalize(quorum_handle);
return -1;
}
}
dlm_ls_pthread_init(lockspace);
DEBUGLOG("DLM initialisation complete\n");
/* Connect to the clvmd group */
strcpy((char *)cpg_group_name.value, "clvmd");
cpg_group_name.length = strlen((char *)cpg_group_name.value);
err = cpg_join(cpg_handle, &cpg_group_name);
if (err != CS_OK) {
cpg_finalize(cpg_handle);
quorum_finalize(quorum_handle);
dlm_release_lockspace(LOCKSPACE_NAME, lockspace, 1);
syslog(LOG_ERR, "Cannot join clvmd process group");
DEBUGLOG("Cannot join clvmd process group: %d\n", err);
return cs_to_errno(err);
}
err = cpg_local_get(cpg_handle,
&our_nodeid);
if (err != CS_OK) {
cpg_finalize(cpg_handle);
quorum_finalize(quorum_handle);
dlm_release_lockspace(LOCKSPACE_NAME, lockspace, 1);
syslog(LOG_ERR, "Cannot get local node id\n");
return cs_to_errno(err);
}
DEBUGLOG("Our local node id is %d\n", our_nodeid);
DEBUGLOG("Connected to Corosync\n");
return 0;
}
cs_error_t sam_initialize (
int time_interval,
sam_recovery_policy_t recovery_policy)
{
quorum_callbacks_t quorum_callbacks;
cs_error_t err;
if (sam_internal_data.internal_status != SAM_INTERNAL_STATUS_NOT_INITIALIZED) {
return (CS_ERR_BAD_HANDLE);
}
if (SAM_RP_MASK (recovery_policy) != SAM_RECOVERY_POLICY_QUIT &&
SAM_RP_MASK (recovery_policy) != SAM_RECOVERY_POLICY_RESTART) {
return (CS_ERR_INVALID_PARAM);
}
if (recovery_policy & SAM_RECOVERY_POLICY_QUORUM) {
/*
* Initialize quorum
*/
quorum_callbacks.quorum_notify_fn = quorum_notification_fn;
if ((err = quorum_initialize (&sam_internal_data.quorum_handle, &quorum_callbacks)) != CS_OK) {
goto exit_error;
}
if ((err = quorum_trackstart (sam_internal_data.quorum_handle, CS_TRACK_CHANGES)) != CS_OK) {
goto exit_error_quorum;
}
if ((err = quorum_fd_get (sam_internal_data.quorum_handle, &sam_internal_data.quorum_fd)) != CS_OK) {
goto exit_error_quorum;
}
/*
* Dispatch initial quorate state
*/
if ((err = quorum_dispatch (sam_internal_data.quorum_handle, CS_DISPATCH_ONE)) != CS_OK) {
goto exit_error_quorum;
}
}
sam_internal_data.recovery_policy = recovery_policy;
sam_internal_data.time_interval = time_interval;
sam_internal_data.internal_status = SAM_INTERNAL_STATUS_INITIALIZED;
sam_internal_data.warn_signal = SIGTERM;
sam_internal_data.am_i_child = 0;
sam_internal_data.user_data = NULL;
sam_internal_data.user_data_size = 0;
sam_internal_data.user_data_allocated = 0;
pthread_mutex_init (&sam_internal_data.lock, NULL);
return (CS_OK);
exit_error_quorum:
quorum_finalize (sam_internal_data.quorum_handle);
exit_error:
return (err);
}
static void _cluster_closedown(void)
{
dlm_release_lockspace(LOCKSPACE_NAME, lockspace, 1);
cpg_finalize(cpg_handle);
quorum_finalize(quorum_handle);
}
