gboolean
query_quorum(gpointer data)
{
int quorum;
size_t len;
char* s = NULL;
char buf[MAXMSG];
struct ha_msg* msg = NULL;
struct ha_msg* ret = NULL;
if(session != NULL) {
msg = ha_msg_new(10);
ha_msg_add(msg, "t","quorum");
ha_msg_add_int(msg, "nodenum", nodenum);
ha_msg_add_int(msg, "weight", weight);
s = msg2wirefmt(msg, &len);
gnutls_record_send(session, s, len);
free(s);
len = gnutls_record_recv(session, buf, MAXMSG);
if ((ssize_t)len < 0) {
gnutls_bye (session, GNUTLS_SHUT_WR);
gnutls_deinit (session);
close(sock);
session = NULL;
cur_quorum = -1;
ha_msg_del(msg);
return TRUE;
}
ret = wirefmt2msg(buf, len, FALSE);
ha_msg_value_int(ret, "quorum", &quorum);
ha_msg_del(ret);
ha_msg_del(msg);
if (cur_quorum!=-1 && cur_quorum!=quorum && callback!=NULL){
cur_quorum = quorum;
callback();
}
cur_quorum = quorum;
}
else {
connect_quorum_server(NULL);
}
return TRUE;
}
static gboolean
send_rexmit_request( gpointer data)
{
struct rexmit_info* ri = (struct rexmit_info*) data;
seqno_t seq = (seqno_t) ri->seq;
struct node_info* node = ri->node;
struct ha_msg* hmsg;
if ((hmsg = ha_msg_new(6)) == NULL) {
cl_log(LOG_ERR, "%s: no memory for " T_REXMIT,
__FUNCTION__);
return FALSE;
}
if (ha_msg_add(hmsg, F_TYPE, T_REXMIT) != HA_OK
|| ha_msg_add(hmsg, F_TO, node->nodename) !=HA_OK
|| ha_msg_add_int(hmsg, F_FIRSTSEQ, seq) != HA_OK
|| ha_msg_add_int(hmsg, F_LASTSEQ, seq) != HA_OK) {
cl_log(LOG_ERR, "%s: adding fields to msg failed",
__FUNCTION__);
ha_msg_del(hmsg);
return FALSE;
}
if (send_cluster_msg(hmsg) != HA_OK) {
cl_log(LOG_ERR, "%s: cannot send " T_REXMIT
" request to %s",__FUNCTION__, node->nodename);
ha_msg_del(hmsg);
return FALSE;
}
node->track.last_rexmit_req = time_longclock();
if (!g_hash_table_remove(rexmit_hash_table, ri)){
cl_log(LOG_ERR, "%s: entry not found in rexmit_hash_table"
"for seq/node(%ld %s)",
__FUNCTION__, ri->seq, ri->node->nodename);
return FALSE;
}
schedule_rexmit_request(node, seq, max_rexmit_delay);
return FALSE;
}
void
cib_replace_notify(crm_data_t *update, enum cib_errors result, crm_data_t *diff)
{
const char *origin = NULL;
HA_Message *replace_msg = NULL;
int add_updates = 0;
int add_epoch = 0;
int add_admin_epoch = 0;
int del_updates = 0;
int del_epoch = 0;
int del_admin_epoch = 0;
if(diff == NULL) {
return;
}
cib_diff_version_details(
diff, &add_admin_epoch, &add_epoch, &add_updates,
&del_admin_epoch, &del_epoch, &del_updates);
origin = crm_element_value(update, F_CRM_ORIGIN);
if(add_updates != del_updates) {
crm_info("Replaced: %d.%d.%d -> %d.%d.%d from %s",
del_admin_epoch, del_epoch, del_updates,
add_admin_epoch, add_epoch, add_updates,
crm_str(origin));
} else if(diff != NULL) {
crm_info("Local-only Replace: %d.%d.%d from %s",
add_admin_epoch, add_epoch, add_updates,
crm_str(origin));
}
replace_msg = ha_msg_new(8);
ha_msg_add(replace_msg, F_TYPE, T_CIB_NOTIFY);
ha_msg_add(replace_msg, F_SUBTYPE, T_CIB_REPLACE_NOTIFY);
ha_msg_add(replace_msg, F_CIB_OPERATION, CIB_OP_REPLACE);
ha_msg_add_int(replace_msg, F_CIB_RC, result);
attach_cib_generation(replace_msg, "cib-replace-generation", update);
crm_log_message_adv(LOG_DEBUG_2,"CIB Replaced", replace_msg);
g_hash_table_foreach(client_list, cib_notify_client, replace_msg);
crm_msg_del(replace_msg);
}
static struct ha_msg *
create_basic_reqmsg_fields(const char * apitype)
{
struct ha_msg * msg = NULL;
if ((msg = ha_msg_new(4)) == NULL) {
stdlib_log(LOG_ERR, "create_basic_msg_fields:out of memory.");
return NULL;
}
/* important error check client name length */
if ( (ha_msg_add(msg, F_STONITHD_TYPE, ST_APIREQ ) != HA_OK )
||( ha_msg_add(msg, F_STONITHD_APIREQ, apitype) != HA_OK)
||( ha_msg_add(msg, F_STONITHD_CNAME, CLIENT_NAME) != HA_OK)
||( ha_msg_add_int(msg, F_STONITHD_CPID, CLIENT_PID) != HA_OK)
) {
stdlib_log(LOG_ERR, "create_basic_msg_fields: "
"cannot add field to ha_msg.");
ZAPMSG(msg);
}
return msg;
}
void
do_cib_notify(
int options, const char *op, crm_data_t *update,
enum cib_errors result, crm_data_t *result_data, const char *msg_type)
{
HA_Message *update_msg = NULL;
const char *type = NULL;
const char *id = NULL;
update_msg = ha_msg_new(8);
if(result_data != NULL) {
id = crm_element_value(result_data, XML_ATTR_ID);
}
ha_msg_add(update_msg, F_TYPE, T_CIB_NOTIFY);
ha_msg_add(update_msg, F_SUBTYPE, msg_type);
ha_msg_add(update_msg, F_CIB_OPERATION, op);
ha_msg_add_int(update_msg, F_CIB_RC, result);
if(id != NULL) {
ha_msg_add(update_msg, F_CIB_OBJID, id);
}
if(update != NULL) {
crm_debug_4("Setting type to update->name: %s",
crm_element_name(update));
ha_msg_add(update_msg, F_CIB_OBJTYPE, crm_element_name(update));
type = crm_element_name(update);
} else if(result_data != NULL) {
crm_debug_4("Setting type to new_obj->name: %s",
crm_element_name(result_data));
ha_msg_add(update_msg, F_CIB_OBJTYPE, crm_element_name(result_data));
type = crm_element_name(result_data);
} else {
crm_debug_4("Not Setting type");
}
attach_cib_generation(update_msg, "cib_generation", the_cib);
if(update != NULL) {
add_message_xml(update_msg, F_CIB_UPDATE, update);
}
if(result_data != NULL) {
add_message_xml(update_msg, F_CIB_UPDATE_RESULT, result_data);
}
crm_debug_3("Notifying clients");
g_hash_table_foreach(client_list, cib_notify_client, update_msg);
crm_msg_del(update_msg);
if(update == NULL) {
if(result == cib_ok) {
crm_debug_2("Operation %s (on section=%s) completed",
op, crm_str(type));
} else {
crm_warn("Operation %s (on section=%s) FAILED: (%d) %s",
op, crm_str(type), result,
cib_error2string(result));
}
} else {
if(result == cib_ok) {
crm_debug_2("Completed %s of <%s %s%s>",
op, crm_str(type), id?"id=":"", id?id:"");
} else {
crm_warn("%s of <%s %s%s> FAILED: %s", op,crm_str(type),
id?"id=":"", id?id:"", cib_error2string(result));
}
}
crm_debug_3("Notify complete");
}
gboolean
connect_quorum_server(gpointer data)
{
struct sockaddr_in addr;
struct ha_msg* msg = NULL;
struct ha_msg* ret = NULL;
const char* version = "2_0_8";
struct hostent* hp;
int quorum;
size_t len;
char* s = NULL;
char buf[MAXMSG];
cl_log(LOG_DEBUG, "quorum plugin: quorumd, connect_quorum_server");
/* initialize gnutls */
initialize_tls_global();
/* create socket */
sock=socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
if (sock == -1 ) {
return FALSE;
}
/* connect to server*/
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
hp = gethostbyname(quorum_server);
if (hp == NULL) {
return FALSE;
}
memcpy(&addr.sin_addr, hp->h_addr_list[0], sizeof(struct in_addr));
addr.sin_port = htons(5561);
if (connect(sock, (struct sockaddr *) &addr, sizeof(addr)) == -1) {
return FALSE;
}
session = initialize_tls_session(sock);
if (session == NULL) {
close(sock);
session = NULL;
return FALSE;
}
/* send the version */
gnutls_record_send(session, version, strlen(version)+1);
/* send initialize msg */
msg = ha_msg_new(10);
ha_msg_add(msg, "t","init");
ha_msg_add(msg, "cl_name", cluster);
s = msg2wirefmt(msg, &len);
gnutls_record_send(session, s, len);
free(s);
len = gnutls_record_recv(session, buf, MAXMSG);
if ((ssize_t)len <=0) {
close(sock);
session = NULL;
return FALSE;
}
ret = wirefmt2msg(buf, len, FALSE);
if (STRNCMP_CONST(ha_msg_value(ret, "result"), "ok") != 0) {
close(sock);
session = NULL;
return FALSE;
}
if (ha_msg_value_int(ret, "interval", &interval)!= HA_OK) {
close(sock);
session = NULL;
return FALSE;
}
ha_msg_del(ret);
ha_msg_del(msg);
/* send quorum query msg */
msg = ha_msg_new(10);
ha_msg_add(msg, "t","quorum");
ha_msg_add_int(msg, "nodenum", nodenum);
ha_msg_add_int(msg, "weight", weight);
s = msg2wirefmt(msg, &len);
gnutls_record_send(session, s, len);
free(s);
len = gnutls_record_recv(session, buf, MAXMSG);
ret = wirefmt2msg(buf, len, FALSE);
ha_msg_value_int(ret, "quorum", &quorum);
LOG(LOG_DEBUG,"quorum:%d\n", quorum);
cur_quorum = quorum;
ha_msg_del(ret);
ha_msg_del(msg);
/* set the repeatly query */
repeat_timer = g_timeout_add(interval, query_quorum, NULL);
return FALSE;
}
int
stonithd_virtual_stonithRA_ops( stonithRA_ops_t * op, int * call_id)
{
int rc = ST_FAIL;
struct ha_msg * request, * reply;
const char * tmpstr;
if (op == NULL) {
stdlib_log(LOG_ERR, "stonithd_virtual_stonithRA_ops: op==NULL");
return ST_FAIL;
}
if (call_id == NULL) {
stdlib_log(LOG_ERR, "stonithd_stonithd_stonithRA_ops: "
"call_id==NULL");
return ST_FAIL;
}
if ( !signed_on(chan) ) {
stdlib_log(LOG_ERR, "not signed on");
return ST_FAIL;
}
if ( (request = create_basic_reqmsg_fields(ST_RAOP)) == NULL) {
return ST_FAIL;
}
if ( (ha_msg_add(request, F_STONITHD_RSCID, op->rsc_id) != HA_OK)
||(ha_msg_add(request, F_STONITHD_RAOPTYPE, op->op_type) != HA_OK)
||(ha_msg_add(request, F_STONITHD_RANAME, op->ra_name) != HA_OK)
||(ha_msg_add_int(request, F_STONITHD_TIMEOUT, op->timeout) != HA_OK)
||(ha_msg_addhash(request, F_STONITHD_PARAMS, op->params) != HA_OK)
) {
stdlib_log(LOG_ERR, "stonithd_virtual_stonithRA_ops: "
"cannot add field to ha_msg.");
ZAPMSG(request);
return ST_FAIL;
}
/* Send the request message */
if (msg2ipcchan(request, chan) != HA_OK) {
ZAPMSG(request);
stdlib_log(LOG_ERR, "can't send stonithRA message to IPC");
return ST_FAIL;
}
/* waiting for the output to finish */
chan_waitout_timeout(chan, DEFAULT_TIMEOUT);
ZAPMSG(request);
/* Read the reply... */
stdlib_log(LOG_DEBUG, "waiting for the stonithRA reply msg.");
if ( IPC_OK != chan_waitin_timeout(chan, DEFAULT_TIMEOUT) ) {
stdlib_log(LOG_ERR, "%s:%d: waitin failed."
, __FUNCTION__, __LINE__);
return ST_FAIL;
}
if ( (reply = msgfromIPC_noauth(chan)) == NULL ) {
stdlib_log(LOG_ERR, "stonithd_virtual_stonithRA_ops: "
"failed to fetch reply");
return ST_FAIL;
}
if ( FALSE == is_expected_msg(reply, F_STONITHD_TYPE, ST_APIRPL,
F_STONITHD_APIRPL, ST_RRAOP, TRUE) ) {
ZAPMSG(reply); /* avoid to zap the msg ? */
stdlib_log(LOG_WARNING, "stonithd_virtual_stonithRA_ops: "
"got an unexpected message");
return ST_FAIL;
}
if ( ((tmpstr = cl_get_string(reply, F_STONITHD_APIRET)) != NULL)
&& (STRNCMP_CONST(tmpstr, ST_APIOK) == 0) ) {
int tmpint;
if ( ha_msg_value_int(reply, F_STONITHD_CALLID, &tmpint)
== HA_OK ) {
*call_id = tmpint;
rc = ST_OK;
stdlib_log(LOG_DEBUG, "a stonith RA operation queue " \
"to run, call_id=%d.", *call_id);
} else {
stdlib_log(LOG_ERR, "no return call_id in reply");
rc = ST_FAIL;
}
} else {
stdlib_log(LOG_WARNING, "failed to do the RA op.");
rc = ST_FAIL;
* call_id = -1;
}
ZAPMSG(reply);
return rc;
}
int
stonithd_node_fence(stonith_ops_t * op)
{
int rc = ST_FAIL;
struct ha_msg *request, *reply;
if (op == NULL) {
stdlib_log(LOG_ERR, "stonithd_node_fence: op==NULL");
goto out;
}
if (!signed_on(chan)) {
stdlib_log(LOG_NOTICE, "not signed on");
goto out;
}
if (!(request = create_basic_reqmsg_fields(ST_STONITH))) {
stdlib_log(LOG_ERR, "stonithd_node_fence: "
"message creation failed.");
goto out;
}
if (ha_msg_add_int(request, F_STONITHD_OPTYPE, op->optype) != HA_OK) {
stdlib_log(LOG_ERR, "stonithd_node_fence: "
"cannot add optype field to ha_msg.");
goto out;
}
if (ha_msg_add(request, F_STONITHD_NODE, op->node_name ) != HA_OK) {
stdlib_log(LOG_ERR, "stonithd_node_fence: "
"cannot add node_name field to ha_msg.");
goto out;
}
if (op->node_uuid == NULL || (ha_msg_add(request, F_STONITHD_NODE_UUID,
op->node_uuid) != HA_OK)) {
stdlib_log(LOG_ERR, "stonithd_node_fence: "
"cannot add node_uuid field to ha_msg.");
goto out;
}
if (ha_msg_add_int(request, F_STONITHD_TIMEOUT, op->timeout) != HA_OK) {
stdlib_log(LOG_ERR, "stonithd_node_fence: "
"cannot add timeout field to ha_msg.");
goto out;
}
if (op->private_data == NULL || (ha_msg_add(request, F_STONITHD_PDATA,
op->private_data) != HA_OK)) {
stdlib_log(LOG_ERR, "stonithd_node_fence: "
"cannot add private_data field to ha_msg.");
goto out;
}
/* Send the stonith request message */
if (msg2ipcchan(request, chan) != HA_OK) {
stdlib_log(LOG_ERR
, "failed to send stonith request to stonithd");
goto out;
}
/* waiting for the output to finish */
chan_waitout_timeout(chan, DEFAULT_TIMEOUT);
/* Read the reply... */
stdlib_log(LOG_DEBUG, "waiting for the stonith reply msg.");
if ( IPC_OK != chan_waitin_timeout(chan, DEFAULT_TIMEOUT) ) {
stdlib_log(LOG_ERR, "%s:%d: waitin failed."
, __FUNCTION__, __LINE__);
goto out;
}
if ( (reply = msgfromIPC_noauth(chan)) == NULL ) {
stdlib_log(LOG_ERR, "stonithd_node_fence: fail to fetch reply");
goto out;
}
if ( TRUE == is_expected_msg(reply, F_STONITHD_TYPE, ST_APIRPL,
F_STONITHD_APIRPL, ST_RSTONITH, TRUE) ) {
if( !STRNCMP_CONST(
cl_get_string(reply,F_STONITHD_APIRET), ST_APIOK) ) {
rc = ST_OK;
stdlib_log(LOG_DEBUG, "%s:%d: %s"
, __FUNCTION__, __LINE__
, "stonithd's synchronous answer is ST_APIOK");
} else {
stdlib_log(LOG_ERR, "%s:%d: %s"
, __FUNCTION__, __LINE__
, "stonithd's synchronous answer is ST_APIFAIL");
}
} else {
stdlib_log(LOG_ERR, "stonithd_node_fence: "
"Got an unexpected message.");
/* Need to handle in other way? */
}
out:
ZAPMSG(reply);
ZAPMSG(request);
return rc;
}
int
stonithd_signon(const char * client_name)
{
int rc = ST_FAIL;
char path[] = IPC_PATH_ATTR;
char sock[] = STONITHD_SOCK;
char cbsock[] = STONITHD_CALLBACK_SOCK;
struct ha_msg * request;
struct ha_msg * reply;
GHashTable * wchanattrs;
uid_t my_euid;
gid_t my_egid;
const char * tmpstr;
int rc_tmp;
gboolean connected = TRUE;
cl_uuid_t cookie, *cptr = NULL;
if (chan == NULL || chan->ch_status != IPC_CONNECT) {
connected = FALSE;
} else if (cbchan == NULL || cbchan->ch_status != IPC_CONNECT) {
connected = FALSE;
}
if(!connected) {
/* cleanup */
if (NULL != chan) {
chan->ops->destroy(chan);
chan = NULL;
}
if (NULL != cbchan) {
cbchan->ops->destroy(cbchan);
cbchan = NULL;
}
stdlib_log(LOG_DEBUG, "stonithd_signon: creating connection");
wchanattrs = g_hash_table_new(g_str_hash, g_str_equal);
g_hash_table_insert(wchanattrs, path, sock);
/* Connect to the stonith deamon */
chan = ipc_channel_constructor(IPC_ANYTYPE, wchanattrs);
g_hash_table_destroy(wchanattrs);
if (chan == NULL) {
stdlib_log(LOG_ERR, "stonithd_signon: Can't connect "
" to stonithd");
rc = ST_FAIL;
goto end;
}
if (chan->ops->initiate_connection(chan) != IPC_OK) {
stdlib_log(LOG_ERR, "stonithd_signon: Can't initiate "
"connection to stonithd");
rc = ST_FAIL;
goto end;
}
}
CLIENT_PID = getpid();
snprintf(CLIENT_PID_STR, sizeof(CLIENT_PID_STR), "%d", CLIENT_PID);
if ( client_name != NULL ) {
CLIENT_NAME = client_name;
} else {
CLIENT_NAME = CLIENT_PID_STR;
}
if ( (request = create_basic_reqmsg_fields(ST_SIGNON)) == NULL) {
rc = ST_FAIL;
goto end;
}
/* important error check client name length */
my_euid = geteuid();
my_egid = getegid();
if ( ( ha_msg_add_int(request, F_STONITHD_CEUID, my_euid) != HA_OK )
||( ha_msg_add_int(request, F_STONITHD_CEGID, my_egid) != HA_OK )
||( ha_msg_add(request, F_STONITHD_COOKIE, "") != HA_OK )
) {
stdlib_log(LOG_ERR, "stonithd_signon: "
"cannot add field to ha_msg.");
ZAPMSG(request);
rc = ST_FAIL;
goto end;
}
stdlib_log(LOG_DEBUG, "sending out the signon msg.");
/* Send the registration request message */
if (msg2ipcchan(request, chan) != HA_OK) {
ZAPMSG(request);
stdlib_log(LOG_ERR, "can't send signon message to IPC");
rc = ST_FAIL;
goto end;
}
/* waiting for the output to finish */
do {
rc_tmp= chan_waitout_timeout(chan, DEFAULT_TIMEOUT);
} while (rc_tmp == IPC_INTR);
ZAPMSG(request);
if (IPC_OK != rc_tmp) {
stdlib_log(LOG_ERR, "%s:%d: waitout failed."
, __FUNCTION__, __LINE__);
rc = ST_FAIL;
goto end;
}
/* Read the reply... */
if ( IPC_OK != chan_waitin_timeout(chan, DEFAULT_TIMEOUT) ) {
stdlib_log(LOG_ERR, "%s:%d: waitin failed."
, __FUNCTION__, __LINE__);
rc = ST_FAIL;
goto end;
}
if ( (reply = msgfromIPC_noauth(chan)) == NULL ) {
stdlib_log(LOG_ERR, "stonithd_signon: failed to fetch reply.");
rc = ST_FAIL;
goto end;
}
if ( TRUE == is_expected_msg(reply, F_STONITHD_TYPE, ST_APIRPL,
F_STONITHD_APIRPL, ST_RSIGNON, TRUE) ) {
if ( ((tmpstr=cl_get_string(reply, F_STONITHD_APIRET)) != NULL)
&& (STRNCMP_CONST(tmpstr, ST_APIOK) == 0) ) {
rc = ST_OK;
stdlib_log(LOG_DEBUG, "signed on to stonithd.");
/* get cookie if any */
if( cl_get_uuid(reply, F_STONITHD_COOKIE, &cookie) == HA_OK ) {
cptr = &cookie;
}
} else {
stdlib_log(LOG_WARNING, "failed to signon to the "
"stonithd.");
}
} else {
stdlib_log(LOG_ERR, "stonithd_signon: "
"Got an unexpected message.");
}
ZAPMSG(reply);
if (ST_OK != rc) { /* Something wrong when try to sign on to stonithd */
goto end;
}
/* Connect to the stonith deamon via callback channel */
wchanattrs = g_hash_table_new(g_str_hash, g_str_equal);
g_hash_table_insert(wchanattrs, path, cbsock);
cbchan = ipc_channel_constructor(IPC_ANYTYPE, wchanattrs);
g_hash_table_destroy(wchanattrs);
if (cbchan == NULL) {
stdlib_log(LOG_ERR, "stonithd_signon: Can't construct "
"callback channel to stonithd.");
rc = ST_FAIL;
goto end;
}
if (cbchan->ops->initiate_connection(cbchan) != IPC_OK) {
stdlib_log(LOG_ERR, "stonithd_signon: Can't initiate "
"connection with the callback channel");
rc = ST_FAIL;
goto end;
}
if ( (reply = msgfromIPC_noauth(cbchan)) == NULL ) {
stdlib_log(LOG_ERR, "%s:%d: failed to fetch reply via the "
" callback channel"
, __FUNCTION__, __LINE__);
rc = ST_FAIL;
goto end;
}
if ( TRUE == is_expected_msg(reply, F_STONITHD_TYPE, ST_APIRPL,
F_STONITHD_APIRPL, ST_RSIGNON, TRUE) ) {
tmpstr=cl_get_string(reply, F_STONITHD_APIRET);
if ( !STRNCMP_CONST(tmpstr, ST_APIOK) ) {
/*
* If the server directly authenticates us (probably
* via pid-auth), go ahead.
*/
stdlib_log(LOG_DEBUG, "%s:%d: Got a good signon reply "
"via the callback channel."
, __FUNCTION__, __LINE__);
} else if ( !STRNCMP_CONST(tmpstr, ST_COOKIE) ) {
/*
* If the server asks for a cookie to identify myself,
* initiate cookie authentication.
*/
if (cptr == NULL) {
stdlib_log(LOG_ERR, "server requested cookie auth on "
"the callback channel, but it didn't "
"provide the cookie on the main channel.");
rc = ST_FAIL;
} else {
rc = authenticate_with_cookie(cbchan, cptr);
}
} else {
/* Unknown response. */
rc = ST_FAIL;
stdlib_log(LOG_ERR, "%s:%d: Got a bad signon reply "
"via the callback channel."
, __FUNCTION__, __LINE__);
}
} else {
rc = ST_FAIL;
stdlib_log(LOG_ERR, "stonithd_signon: "
"Got an unexpected message via the callback chan.");
}
ZAPMSG(reply);
end:
if (ST_OK != rc) {
/* Something wrong when confirm via callback channel */
stonithd_signoff();
}
return rc;
}
