uint32_t get_local_nodeid(cpg_handle_t handle)
{
int rc = CS_OK;
int retries = 0;
static uint32_t local_nodeid = 0;
cpg_handle_t local_handle = handle;
cpg_callbacks_t cb = { };
if(local_nodeid != 0) {
return local_nodeid;
}
if(handle == 0) {
crm_trace("Creating connection");
cs_repeat(retries, 5, rc = cpg_initialize(&local_handle, &cb));
}
if (rc == CS_OK) {
retries = 0;
crm_trace("Performing lookup");
cs_repeat(retries, 5, rc = cpg_local_get(local_handle, &local_nodeid));
}
if (rc != CS_OK) {
crm_err("Could not get local node id from the CPG API: %s (%d)", ais_error2text(rc), rc);
}
if(handle == 0) {
crm_trace("Closing connection");
cpg_finalize(local_handle);
}
crm_debug("Local nodeid is %u", local_nodeid);
return local_nodeid;
}
gboolean
cluster_connect_cpg(void)
{
cs_error_t rc;
unsigned int nodeid;
int fd;
int retries = 0;
static struct mainloop_fd_callbacks cpg_fd_callbacks =
{
.dispatch = pcmk_cpg_dispatch,
.destroy = cpg_connection_destroy,
};
strcpy(cpg_group.value, "pcmk");
cpg_group.length = strlen(cpg_group.value) + 1;
retries = 0;
cs_repeat(retries, 30, rc = cpg_initialize(&cpg_handle, &cpg_callbacks));
if (rc != CS_OK) {
crm_err("corosync cpg init error %d", rc);
return FALSE;
}
rc = cpg_fd_get(cpg_handle, &fd);
if (rc != CS_OK) {
crm_err("corosync cpg fd_get error %d", rc);
goto bail;
}
retries = 0;
cs_repeat(retries, 30, rc = cpg_local_get(cpg_handle, &nodeid));
if (rc != CS_OK) {
crm_err("corosync cpg local_get error %d", rc);
goto bail;
}
crm_debug("Our nodeid: %d", nodeid);
retries = 0;
cs_repeat(retries, 30, rc = cpg_join(cpg_handle, &cpg_group));
if (rc != CS_OK) {
crm_err("Could not join the CPG group '%s': %d", crm_system_name, rc);
goto bail;
}
mainloop_add_fd("corosync-cpg", G_PRIORITY_DEFAULT, fd, &cpg_handle, &cpg_fd_callbacks);
return TRUE;
bail:
cpg_finalize(cpg_handle);
return FALSE;
}
uint32_t get_local_nodeid(cpg_handle_t handle)
{
int rc = CS_OK;
int retries = 0;
static uint32_t local_nodeid = 0;
cpg_handle_t local_handle = handle;
cpg_callbacks_t cb = { };
if(local_nodeid != 0) {
return local_nodeid;
}
#if 0
/* Should not be necessary */
if(get_cluster_type() == pcmk_cluster_classic_ais) {
get_ais_details(&local_nodeid, NULL);
goto done;
}
#endif
if(handle == 0) {
crm_trace("Creating connection");
cs_repeat(retries, 5, rc = cpg_initialize(&local_handle, &cb));
}
if (rc == CS_OK) {
retries = 0;
crm_trace("Performing lookup");
cs_repeat(retries, 5, rc = cpg_local_get(local_handle, &local_nodeid));
}
if (rc != CS_OK) {
crm_err("Could not get local node id from the CPG API: %s (%d)", ais_error2text(rc), rc);
}
if(handle == 0) {
crm_trace("Closing connection");
cpg_finalize(local_handle);
}
crm_debug("Local nodeid is %u", local_nodeid);
return local_nodeid;
}
gboolean
cluster_connect_cfg(uint32_t * nodeid)
{
cs_error_t rc;
int fd = 0, retries = 0;
static struct mainloop_fd_callbacks cfg_fd_callbacks =
{
.dispatch = pcmk_cfg_dispatch,
.destroy = cfg_connection_destroy,
};
cs_repeat(retries, 30, rc = corosync_cfg_initialize(&cfg_handle, &cfg_callbacks));
if (rc != CS_OK) {
crm_err("corosync cfg init error %d", rc);
return FALSE;
}
rc = corosync_cfg_fd_get(cfg_handle, &fd);
if (rc != CS_OK) {
crm_err("corosync cfg fd_get error %d", rc);
goto bail;
}
retries = 0;
cs_repeat(retries, 30, rc = corosync_cfg_local_get(cfg_handle, nodeid));
if (rc != CS_OK) {
crm_err("corosync cfg local_get error %d", rc);
goto bail;
}
crm_debug("Our nodeid: %d", *nodeid);
mainloop_add_fd("corosync-cfg", G_PRIORITY_DEFAULT, fd, &cfg_handle, &cfg_fd_callbacks);
return TRUE;
bail:
corosync_cfg_finalize(cfg_handle);
return FALSE;
}
gboolean
cluster_connect_cfg(uint32_t * nodeid)
{
cs_error_t rc;
int fd = 0, retries = 0;
cs_repeat(retries, 30, rc = corosync_cfg_initialize(&cfg_handle, &cfg_callbacks));
if (rc != CS_OK) {
crm_err("corosync cfg init error %d", rc);
return FALSE;
}
rc = corosync_cfg_fd_get(cfg_handle, &fd);
if (rc != CS_OK) {
crm_err("corosync cfg fd_get error %d", rc);
goto bail;
}
retries = 0;
cs_repeat(retries, 30, rc = corosync_cfg_local_get(cfg_handle, nodeid));
if (rc != CS_OK) {
crm_err("corosync cfg local_get error %d", rc);
goto bail;
}
crm_debug("Our nodeid: %d", *nodeid);
crm_debug("Adding fd=%d to mainloop", fd);
G_main_add_fd(G_PRIORITY_HIGH, fd, FALSE, pcmk_cfg_dispatch, &cfg_handle,
cfg_connection_destroy);
return TRUE;
bail:
corosync_cfg_finalize(cfg_handle);
return FALSE;
}
static int
get_config_opt(cmap_handle_t object_handle, const char *key, char **value, const char *fallback)
{
int rc = 0, retries = 0;
cs_repeat(retries, 5, rc = cmap_get_string(object_handle, key, value));
if(rc != CS_OK) {
crm_trace("Search for %s failed %d, defaulting to %s", key, rc, fallback);
if(fallback) {
*value = strdup(fallback);
} else {
*value = NULL;
}
}
crm_trace("%s: %s", key, *value);
return rc;
}
static void shutdown_do(void)
{
cs_error_t result;
corosync_cfg_handle_t handle;
corosync_cfg_callbacks_t callbacks;
callbacks.corosync_cfg_shutdown_callback = NULL;
result = corosync_cfg_initialize (&handle, &callbacks);
if (result != CS_OK) {
printf ("Could not initialize corosync configuration API error %d\n", result);
exit (1);
}
printf ("Shutting down corosync\n");
cs_repeat(result, 30, corosync_cfg_try_shutdown (handle, COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST));
if (result != CS_OK) {
printf ("Could not shutdown (error = %d)\n", result);
}
(void)corosync_cfg_finalize (handle);
}
static void print_localnodeid(cpg_handle_t handle)
{
char addrStr[128];
unsigned int retries;
unsigned int nodeid;
struct sockaddr_storage addr;
struct sockaddr_in *v4addr = (struct sockaddr_in *)&addr;
int result;
retries = 0;
cs_repeat(retries, 30, result = cpg_local_get(handle, &nodeid));
if (result != CS_OK) {
printf ("Could not get local node id\n");
} else {
v4addr->sin_addr.s_addr = nodeid;
if(inet_ntop(AF_INET, (const void *)&v4addr->sin_addr.s_addr,
addrStr, (socklen_t)sizeof(addrStr)) == NULL) {
addrStr[0] = 0;
}
printf ("Local node id is %s/%x result %d\n", addrStr, nodeid, result);
}
}
static void
test_cpg(void)
{
cpg_handle_t cpg_handle;
struct cpg_name group_name;
cs_error_t cs_res;
unsigned int local_nodeid;
int cpg_fd;
struct pollfd pfd;
int poll_res;
int state;
int cont;
ENTER();
state = 0;
strcpy(group_name.value, TEST_GROUP_NAME);
group_name.length = strlen(TEST_GROUP_NAME);
cs_repeat(cs_res = cpg_model_initialize(&cpg_handle, CPG_MODEL_V1,
(cpg_model_data_t *)&model_data, NULL));
assert(cs_res == CS_OK);
cs_repeat(cs_res = cpg_join(cpg_handle, &group_name));
assert(cs_res == CS_OK);
cs_repeat(cs_res = cpg_local_get(cpg_handle, &local_nodeid));
assert(cs_res == CS_OK);
assert(local_nodeid == TEST_NODEID);
cs_repeat(cs_res = cpg_fd_get(cpg_handle, &cpg_fd));
assert(cs_res == CS_OK);
cont = 1;
do {
pfd.fd = cpg_fd;
pfd.events = POLLIN;
pfd.revents = 0;
poll_res = poll(&pfd, 1, INFTIM);
if (poll_res == -1) {
perror("poll_res == -1");
}
assert(poll_res != 0);
assert(pfd.revents & POLLIN);
cs_repeat(cs_res = cpg_dispatch(cpg_handle, CS_DISPATCH_ALL));
assert(cs_res == CS_OK);
switch (state) {
case 0:
/*
* Waiting for cpg_confchg_received and cpg_totem_confchg_received
*/
if (cpg_confchg_received && cpg_totem_confchg_received) {
/*
* Send first message and wait for it in next state
*/
send_msg(cpg_handle);
state = 1;
}
break;
case 1:
if (received_msg_seq_no >= MESSAGES_TO_SENT) {
cont = 0;
} else if (received_msg_seq_no == sent_msg_seq_no) {
/*
* Message delivered so sent new one and wait for it
*/
send_msg(cpg_handle);
}
break;
}
} while (cont);
cs_repeat(cs_res = cpg_leave(cpg_handle, &group_name));
assert(cs_res == CS_OK);
cs_repeat(cs_res = cpg_finalize(cpg_handle));
assert(cs_res == CS_OK);
LEAVE();
}
gboolean
cluster_connect_cpg(crm_cluster_t *cluster)
{
int rc = -1;
int fd = 0;
int retries = 0;
uint32_t id = 0;
crm_node_t *peer = NULL;
cpg_handle_t handle = 0;
struct mainloop_fd_callbacks cpg_fd_callbacks = {
.dispatch = pcmk_cpg_dispatch,
.destroy = cluster->destroy,
};
cpg_callbacks_t cpg_callbacks = {
.cpg_deliver_fn = cluster->cpg.cpg_deliver_fn,
.cpg_confchg_fn = cluster->cpg.cpg_confchg_fn,
/* .cpg_deliver_fn = pcmk_cpg_deliver, */
/* .cpg_confchg_fn = pcmk_cpg_membership, */
};
cpg_evicted = FALSE;
cluster->group.length = 0;
cluster->group.value[0] = 0;
/* group.value is char[128] */
strncpy(cluster->group.value, crm_system_name?crm_system_name:"unknown", 127);
cluster->group.value[127] = 0;
cluster->group.length = 1 + QB_MIN(127, strlen(cluster->group.value));
cs_repeat(retries, 30, rc = cpg_initialize(&handle, &cpg_callbacks));
if (rc != CS_OK) {
crm_err("Could not connect to the Cluster Process Group API: %d", rc);
goto bail;
}
id = get_local_nodeid(handle);
if (id == 0) {
crm_err("Could not get local node id from the CPG API");
goto bail;
}
cluster->nodeid = id;
retries = 0;
cs_repeat(retries, 30, rc = cpg_join(handle, &cluster->group));
if (rc != CS_OK) {
crm_err("Could not join the CPG group '%s': %d", crm_system_name, rc);
goto bail;
}
rc = cpg_fd_get(handle, &fd);
if (rc != CS_OK) {
crm_err("Could not obtain the CPG API connection: %d", rc);
goto bail;
}
pcmk_cpg_handle = handle;
cluster->cpg_handle = handle;
mainloop_add_fd("corosync-cpg", G_PRIORITY_MEDIUM, fd, cluster, &cpg_fd_callbacks);
bail:
if (rc != CS_OK) {
cpg_finalize(handle);
return FALSE;
}
peer = crm_get_peer(id, NULL);
crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, ONLINESTATUS);
return TRUE;
}
gboolean
send_cluster_message_cs(xmlNode * msg, gboolean local, crm_node_t * node, enum crm_ais_msg_types dest)
{
gboolean rc = TRUE;
char *data = NULL;
data = dump_xml_unformatted(msg);
rc = send_cluster_text(crm_class_cluster, data, local, node, dest);
free(data);
return rc;
}
gboolean
send_cluster_text(int class, const char *data,
gboolean local, crm_node_t * node, enum crm_ais_msg_types dest)
{
static int msg_id = 0;
static int local_pid = 0;
static int local_name_len = 0;
static const char *local_name = NULL;
char *target = NULL;
struct iovec *iov;
AIS_Message *msg = NULL;
enum crm_ais_msg_types sender = text2msg_type(crm_system_name);
/* There are only 6 handlers registered to crm_lib_service in plugin.c */
CRM_CHECK(class < 6, crm_err("Invalid message class: %d", class);
return FALSE);
#if !SUPPORT_PLUGIN
CRM_CHECK(dest != crm_msg_ais, return FALSE);
#endif
if(local_name == NULL) {
local_name = get_local_node_name();
}
if(local_name_len == 0 && local_name) {
local_name_len = strlen(local_name);
}
if (data == NULL) {
data = "";
}
if (local_pid == 0) {
local_pid = getpid();
}
if (sender == crm_msg_none) {
sender = local_pid;
}
msg = calloc(1, sizeof(AIS_Message));
msg_id++;
msg->id = msg_id;
msg->header.id = class;
msg->header.error = CS_OK;
msg->host.type = dest;
msg->host.local = local;
if (node) {
if (node->uname) {
target = strdup(node->uname);
msg->host.size = strlen(node->uname);
memset(msg->host.uname, 0, MAX_NAME);
memcpy(msg->host.uname, node->uname, msg->host.size);
} else {
target = crm_strdup_printf("%u", node->id);
}
msg->host.id = node->id;
} else {
target = strdup("all");
}
msg->sender.id = 0;
msg->sender.type = sender;
msg->sender.pid = local_pid;
msg->sender.size = local_name_len;
memset(msg->sender.uname, 0, MAX_NAME);
if(local_name && msg->sender.size) {
memcpy(msg->sender.uname, local_name, msg->sender.size);
}
msg->size = 1 + strlen(data);
msg->header.size = sizeof(AIS_Message) + msg->size;
if (msg->size < CRM_BZ2_THRESHOLD) {
msg = realloc_safe(msg, msg->header.size);
memcpy(msg->data, data, msg->size);
} else {
char *compressed = NULL;
unsigned int new_size = 0;
char *uncompressed = strdup(data);
if (crm_compress_string(uncompressed, msg->size, 0, &compressed, &new_size)) {
msg->header.size = sizeof(AIS_Message) + new_size;
msg = realloc_safe(msg, msg->header.size);
memcpy(msg->data, compressed, new_size);
msg->is_compressed = TRUE;
msg->compressed_size = new_size;
} else {
msg = realloc_safe(msg, msg->header.size);
memcpy(msg->data, data, msg->size);
}
free(uncompressed);
free(compressed);
}
iov = calloc(1, sizeof(struct iovec));
iov->iov_base = msg;
iov->iov_len = msg->header.size;
if (msg->compressed_size) {
crm_trace("Queueing CPG message %u to %s (%llu bytes, %d bytes compressed payload): %.200s",
msg->id, target, (unsigned long long) iov->iov_len,
msg->compressed_size, data);
} else {
crm_trace("Queueing CPG message %u to %s (%llu bytes, %d bytes payload): %.200s",
msg->id, target, (unsigned long long) iov->iov_len,
msg->size, data);
}
free(target);
#if SUPPORT_PLUGIN
/* The plugin is the only time we dont use CPG messaging */
if(get_cluster_type() == pcmk_cluster_classic_ais) {
return send_plugin_text(class, iov);
}
#endif
send_cpg_iov(iov);
return TRUE;
}
enum crm_ais_msg_types
text2msg_type(const char *text)
{
int type = crm_msg_none;
CRM_CHECK(text != NULL, return type);
if (safe_str_eq(text, "ais")) {
type = crm_msg_ais;
} else if (safe_str_eq(text, "crm_plugin")) {
type = crm_msg_ais;
} else if (safe_str_eq(text, CRM_SYSTEM_CIB)) {
type = crm_msg_cib;
} else if (safe_str_eq(text, CRM_SYSTEM_CRMD)) {
type = crm_msg_crmd;
} else if (safe_str_eq(text, CRM_SYSTEM_DC)) {
type = crm_msg_crmd;
} else if (safe_str_eq(text, CRM_SYSTEM_TENGINE)) {
type = crm_msg_te;
} else if (safe_str_eq(text, CRM_SYSTEM_PENGINE)) {
type = crm_msg_pe;
} else if (safe_str_eq(text, CRM_SYSTEM_LRMD)) {
type = crm_msg_lrmd;
} else if (safe_str_eq(text, CRM_SYSTEM_STONITHD)) {
type = crm_msg_stonithd;
} else if (safe_str_eq(text, "stonith-ng")) {
type = crm_msg_stonith_ng;
} else if (safe_str_eq(text, "attrd")) {
type = crm_msg_attrd;
} else {
/* This will normally be a transient client rather than
* a cluster daemon. Set the type to the pid of the client
*/
int scan_rc = sscanf(text, "%d", &type);
if (scan_rc != 1 || type <= crm_msg_stonith_ng) {
/* Ensure it's sane */
type = crm_msg_none;
}
}
return type;
}
static void send_some_more_messages_normal (void)
{
msg_t my_msg;
struct iovec iov[2];
int i;
int send_now;
size_t payload_size;
cs_error_t res;
cpg_flow_control_state_t fc_state;
int retries = 0;
time_t before;
unsigned int sha1_len;
if (cpg_fd < 0)
return;
send_now = my_msgs_to_send;
qb_log (LOG_TRACE, "send_now:%d", send_now);
my_msg.pid = my_pid;
my_msg.nodeid = my_nodeid;
payload_size = (rand() % 10000);
my_msg.size = sizeof (msg_t) + payload_size;
my_msg.seq = my_msgs_sent;
for (i = 0; i < payload_size; i++) {
buffer[i] = i;
}
PK11_DigestBegin(sha1_context);
PK11_DigestOp(sha1_context, buffer, payload_size);
PK11_DigestFinal(sha1_context, my_msg.sha1, &sha1_len, sizeof(my_msg.sha1));
iov[0].iov_len = sizeof (msg_t);
iov[0].iov_base = &my_msg;
iov[1].iov_len = payload_size;
iov[1].iov_base = buffer;
for (i = 0; i < send_now; i++) {
if (in_cnchg && pcmk_test) {
retries = 0;
before = time(NULL);
cs_repeat(retries, 30, res = cpg_mcast_joined(cpg_handle, CPG_TYPE_AGREED, iov, 2));
if (retries > 20) {
qb_log (LOG_ERR, "cs_repeat: blocked for :%lu secs.",
(unsigned long)(time(NULL) - before));
}
if (res != CS_OK) {
qb_log (LOG_ERR, "cpg_mcast_joined error:%d.",
res);
return;
}
} else {
res = cpg_flow_control_state_get (cpg_handle, &fc_state);
if (res == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) {
/* lets do this later */
send_some_more_messages_later ();
qb_log (LOG_INFO, "flow control enabled.");
return;
}
res = cpg_mcast_joined (cpg_handle, CPG_TYPE_AGREED, iov, 2);
if (res == CS_ERR_TRY_AGAIN) {
/* lets do this later */
send_some_more_messages_later ();
if (i > 0) {
qb_log (LOG_INFO, "TRY_AGAIN %d to send.",
my_msgs_to_send);
}
return;
} else if (res != CS_OK) {
qb_log (LOG_ERR, "cpg_mcast_joined error:%d, exiting.",
res);
exit (-2);
}
}
my_msgs_sent++;
my_msg.seq = my_msgs_sent;
my_msgs_to_send--;
}
qb_log (LOG_TRACE, "sent %d; to send %d.",
my_msgs_sent, my_msgs_to_send);
}
/*
* CFG functionality stolen from node_name() in corosync-quorumtool.c
* This resolves the first address assigned to a node and returns the name or IP address.
*/
char *
corosync_node_name(uint64_t /*cmap_handle_t */ cmap_handle, uint32_t nodeid)
{
int lpc = 0;
int rc = CS_OK;
int retries = 0;
char *name = NULL;
cmap_handle_t local_handle = 0;
/* nodeid == 0 == CMAN_NODEID_US */
if (nodeid == 0 && pcmk_nodeid) {
nodeid = pcmk_nodeid;
} else if (nodeid == 0) {
/* Look it up */
int rc = -1;
int retries = 0;
cpg_handle_t handle = 0;
cpg_callbacks_t cb = { };
cs_repeat(retries, 5, rc = cpg_initialize(&handle, &cb));
if (rc == CS_OK) {
retries = 0;
cs_repeat(retries, 5, rc = cpg_local_get(handle, &pcmk_nodeid));
}
if (rc != CS_OK) {
crm_err("Could not get local node id from the CPG API: %d", rc);
}
cpg_finalize(handle);
}
if (cmap_handle == 0 && local_handle == 0) {
retries = 0;
crm_trace("Initializing CMAP connection");
do {
rc = cmap_initialize(&local_handle);
if (rc != CS_OK) {
retries++;
crm_debug("API connection setup failed: %s. Retrying in %ds", cs_strerror(rc),
retries);
sleep(retries);
}
} while (retries < 5 && rc != CS_OK);
if (rc != CS_OK) {
crm_warn("Could not connect to Cluster Configuration Database API, error %s",
cs_strerror(rc));
local_handle = 0;
}
}
if (cmap_handle == 0) {
cmap_handle = local_handle;
}
while (name == NULL && cmap_handle != 0) {
uint32_t id = 0;
char *key = NULL;
key = g_strdup_printf("nodelist.node.%d.nodeid", lpc);
rc = cmap_get_uint32(cmap_handle, key, &id);
crm_trace("Checking %u vs %u from %s", nodeid, id, key);
g_free(key);
if (rc != CS_OK) {
break;
}
if (nodeid == id) {
crm_trace("Searching for node name for %u in nodelist.node.%d %s", nodeid, lpc, name);
if (name == NULL) {
key = g_strdup_printf("nodelist.node.%d.ring0_addr", lpc);
rc = cmap_get_string(cmap_handle, key, &name);
crm_trace("%s = %s", key, name);
if (node_name_is_valid(key, name) == FALSE) {
free(name);
name = NULL;
}
g_free(key);
}
if (name == NULL) {
key = g_strdup_printf("nodelist.node.%d.name", lpc);
rc = cmap_get_string(cmap_handle, key, &name);
crm_trace("%s = %s %d", key, name, rc);
g_free(key);
}
break;
}
lpc++;
}
if(local_handle) {
cmap_finalize(local_handle);
}
if (name == NULL) {
crm_notice("Unable to get node name for nodeid %u", nodeid);
}
return name;
}
int main (int argc, char *argv[]) {
cpg_handle_t handle;
fd_set read_fds;
int select_fd;
int result;
int retries;
const char *options = "i";
int opt;
unsigned int nodeid;
char *fgets_res;
struct cpg_address member_list[64];
int member_list_entries;
int i;
int recnt;
int doexit;
const char *exitStr = "EXIT";
doexit = 0;
#ifdef QBLOG
qb_log_init("testcpg", LOG_USER, LOG_ERR);
qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_ENABLED, QB_FALSE);
qb_log_filter_ctl(QB_LOG_STDERR, QB_LOG_FILTER_ADD,
QB_LOG_FILTER_FILE, "*", LOG_TRACE);
qb_log_ctl(QB_LOG_STDERR, QB_LOG_CONF_ENABLED, QB_TRUE);
qb_log_format_set(QB_LOG_STDERR, "[%p] %f %b");
#endif
while ( (opt = getopt(argc, argv, options)) != -1 ) {
switch (opt) {
case 'i':
show_ip = 1;
break;
}
}
if (argc > optind) {
strcpy(group_name.value, argv[optind]);
group_name.length = strlen(argv[optind]);
}
else {
strcpy(group_name.value, "GROUP");
group_name.length = 6;
}
recnt = 0;
printf ("Type %s to finish\n", exitStr);
restart = 1;
do {
if(restart) {
restart = 0;
retries = 0;
cs_repeat_init(retries, 30, result = cpg_model_initialize (&handle, CPG_MODEL_V1, (cpg_model_data_t *)&model_data, NULL));
if (result != CS_OK) {
printf ("Could not initialize Cluster Process Group API instance error %d\n", result);
retrybackoff(recnt);
}
retries = 0;
cs_repeat(retries, 30, result = cpg_local_get(handle, &nodeid));
if (result != CS_OK) {
printf ("Could not get local node id\n");
retrybackoff(recnt);
}
printf ("Local node id is %x\n", nodeid);
nodeidStart = nodeid;
retries = 0;
cs_repeat(retries, 30, result = cpg_join(handle, &group_name));
if (result != CS_OK) {
printf ("Could not join process group, error %d\n", result);
retrybackoff(recnt);
}
retries = 0;
cs_repeat(retries, 30, result = cpg_membership_get (handle, &group_name,
(struct cpg_address *)&member_list, &member_list_entries));
if (result != CS_OK) {
printf ("Could not get current membership list %d\n", result);
retrybackoff(recnt);
}
recnt = 0;
printf ("membership list\n");
for (i = 0; i < member_list_entries; i++) {
printf ("node id %d pid %d\n", member_list[i].nodeid,
member_list[i].pid);
}
FD_ZERO (&read_fds);
cpg_fd_get(handle, &select_fd);
}
FD_SET (select_fd, &read_fds);
FD_SET (STDIN_FILENO, &read_fds);
result = select (select_fd + 1, &read_fds, 0, 0, 0);
if (result == -1) {
perror ("select\n");
}
if (FD_ISSET (STDIN_FILENO, &read_fds)) {
char inbuf[132];
struct iovec iov;
fgets_res = fgets(inbuf, (int)sizeof(inbuf), stdin);
if (fgets_res == NULL) {
doexit = 1;
cpg_leave(handle, &group_name);
}
if (strncmp(inbuf, exitStr, strlen(exitStr)) == 0) {
doexit = 1;
cpg_leave(handle, &group_name);
}
else {
iov.iov_base = inbuf;
iov.iov_len = strlen(inbuf)+1;
cpg_mcast_joined(handle, CPG_TYPE_AGREED, &iov, 1);
}
}
if (FD_ISSET (select_fd, &read_fds)) {
if (cpg_dispatch (handle, CS_DISPATCH_ALL) != CS_OK) {
if(doexit) {
exit(1);
}
restart = 1;
}
}
if(restart) {
if(!doexit) {
result = cpg_finalize (handle);
printf ("Finalize+restart result is %d (should be 1)\n", result);
continue;
}
}
} while (result && !quit && !doexit);
result = cpg_finalize (handle);
printf ("Finalize result is %d (should be 1)\n", result);
return (0);
}
static void send_some_more_messages_normal (void)
{
msg_t my_msg;
struct iovec iov[2];
int i;
int send_now;
size_t payload_size;
hash_state sha1_hash;
cs_error_t res;
cpg_flow_control_state_t fc_state;
int retries = 0;
time_t before;
if (cpg_fd < 0)
return;
send_now = my_msgs_to_send;
//syslog (LOG_DEBUG,"%s() send_now:%d", __func__, send_now);
my_msg.pid = my_pid;
my_msg.nodeid = my_nodeid;
payload_size = (rand() % 100000);
my_msg.size = sizeof (msg_t) + payload_size;
my_msg.seq = 0;
for (i = 0; i < payload_size; i++) {
buffer[i] = i;
}
sha1_init (&sha1_hash);
sha1_process (&sha1_hash, buffer, payload_size);
sha1_done (&sha1_hash, my_msg.sha1);
iov[0].iov_len = sizeof (msg_t);
iov[0].iov_base = &my_msg;
iov[1].iov_len = payload_size;
iov[1].iov_base = buffer;
for (i = 0; i < send_now; i++) {
if (in_cnchg && pcmk_test) {
retries = 0;
before = time(NULL);
cs_repeat(retries, 30, res = cpg_mcast_joined(cpg_handle, CPG_TYPE_AGREED, iov, 2));
if (retries > 20) {
syslog (LOG_ERR, "%s() -> cs_repeat: blocked for :%lu secs.",
__func__, (unsigned long)(time(NULL) - before));
}
if (res != CS_OK) {
syslog (LOG_ERR, "%s() -> cpg_mcast_joined error:%d.",
__func__, res);
return;
}
} else {
res = cpg_flow_control_state_get (cpg_handle, &fc_state);
if (res == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) {
/* lets do this later */
send_some_more_messages_later ();
syslog (LOG_INFO, "%s() flow control enabled.", __func__);
return;
}
res = cpg_mcast_joined (cpg_handle, CPG_TYPE_AGREED, iov, 2);
if (res == CS_ERR_TRY_AGAIN) {
/* lets do this later */
send_some_more_messages_later ();
if (i > 0) {
syslog (LOG_INFO, "%s() TRY_AGAIN %d to send.",
__func__, my_msgs_to_send);
}
return;
} else if (res != CS_OK) {
syslog (LOG_ERR, "%s() -> cpg_mcast_joined error:%d, exiting.",
__func__, res);
exit (-2);
}
}
my_msgs_to_send--;
}
}
int
main(int argc, char *argv[])
{
#ifdef USE_CONFDB
confdb_handle_t handle;
confdb_callbacks_t callbacks;
hdb_handle_t object_handle;
#endif
#ifdef USE_CMAP
cmap_handle_t handle;
cmap_track_handle_t track_handle;
int retries;
cs_error_t result;
#endif
uint32_t u32;
int status;
int i;
cs_error_t res;
char str_val[255];
int ch;
char *ep;
change_uint32 = 0;
change_str_len = 0;
no_childs = 16;
burst_count = 64;
while ((ch = getopt(argc, argv, "hus:c:n:")) != -1) {
switch (ch) {
case 'u':
change_uint32 = 1;
break;
case 's':
change_str_len = strtol(optarg, &ep, 10);
if (change_str_len <= 0 || *ep != '\0') {
warnx("illegal number, -s argument -- %s", optarg);
usage();
}
break;
case 'c':
no_childs = strtol(optarg, &ep, 10);
if (no_childs <= 0 || *ep != '\0') {
warnx("illegal number, -c argument -- %s", optarg);
usage();
}
break;
case 'n':
burst_count = strtol(optarg, &ep, 10);
if (burst_count <= 0 || *ep != '\0') {
warnx("illegal number, -n argument -- %s", optarg);
usage();
}
break;
case 'h':
case '?':
default:
usage();
/* NOTREACHED */
}
}
signal(SIGPIPE, SIG_IGN);
setlinebuf(stdout);
#ifdef USE_CONFDB
memset(&callbacks, 0, sizeof(callbacks));
assert(confdb_initialize(&handle, &callbacks) == CS_OK);
assert(confdb_object_create(handle, OBJECT_PARENT_HANDLE,
"testconfdb", strlen("testconfdb"), &object_handle) == CS_OK);
assert(confdb_finalize(handle) == CS_OK);
#endif
my_id = create_childs();
u32 = my_id;
#ifdef USE_CONFDB
snprintf(my_key_uint, sizeof(my_key_uint), "testkeyu32id%u", my_id);
snprintf(my_key_str, sizeof(my_key_str), "testkeystrid%u", my_id);
#endif
#ifdef USE_CMAP
snprintf(my_key_uint, sizeof(my_key_uint), "testconfdb.testkeyu32id%u", my_id);
snprintf(my_key_str, sizeof(my_key_str), "testconfdb.testkeystrid%u", my_id);
#endif
for (i = 0; i < change_str_len; i++) {
str_val[i] = ((my_id + i) % ('Z' - 'A' + 1)) + 'A';
}
str_val[i] = '\0';
if (my_id > 0) {
#ifdef USE_CONFDB
memset(&callbacks, 0, sizeof(callbacks));
callbacks.confdb_key_change_notify_fn = confdb_key_change_notify;
assert(confdb_initialize(&handle, &callbacks) == CS_OK);
#endif
#ifdef USE_CMAP
retries = 0;
cs_repeat(retries, 30, result = cmap_initialize(&handle));
assert(result == CS_OK);
#endif
if (change_uint32) {
#ifdef USE_CONFDB
assert(confdb_key_create_typed(handle, object_handle, my_key_uint,
&u32, sizeof(u32), CONFDB_VALUETYPE_UINT32) == CS_OK);
#endif
#ifdef USE_CMAP
assert(cmap_set_uint32(handle, my_key_uint, u32) == CS_OK);
#endif
}
if (change_str_len > 0) {
#ifdef USE_CONFDB
assert(confdb_key_create_typed(handle, object_handle, my_key_str,
str_val, strlen(str_val), CONFDB_VALUETYPE_STRING) == CS_OK);
#endif
#ifdef USE_CMAP
assert(cmap_set_string(handle, my_key_str, str_val) == CS_OK);
#endif
}
} else {
/*
* "Wait" for other processes to initialize
*/
poll(NULL, 0, 1000);
printf("Confdb-track-and-change initialization finished\n");
}
if (my_id > 0) {
signal(SIGINT, sigint_handler_child);
#ifdef USE_CONFDB
assert(confdb_track_changes(handle, object_handle, OBJECT_KEY_REPLACED) == CS_OK);
#endif
#ifdef USE_CMAP
assert(cmap_track_add(handle, "testconfdb.", CMAP_TRACK_MODIFY | CMAP_TRACK_PREFIX,
cmap_notify, NULL, &track_handle) == CS_OK);
#endif
if (change_uint32) {
#ifdef USE_CONFDB
assert(confdb_key_increment(handle, object_handle, my_key_uint,
strlen(my_key_uint), &u32) == CS_OK);
#endif
#ifdef USE_CMAP
assert(cmap_inc(handle, my_key_uint) == CS_OK);
#endif
expected_msgs_uint = 1;
}
if (change_str_len > 0) {
inc_str(str_val, change_str_len);
#ifdef USE_CONFDB
assert(confdb_key_replace(handle, object_handle, my_key_str, strlen(my_key_str), NULL, 0,
str_val, strlen(str_val)) == CS_OK);
#endif
#ifdef USE_CMAP
assert(cmap_set_string(handle, my_key_str, str_val) == CS_OK);
#endif
expected_msgs_str = 1;
}
/*
* Give other processes a little time to initialize
*/
poll(NULL, 0, 250);
do {
#ifdef USE_CONFDB
res = confdb_dispatch(handle, CS_DISPATCH_BLOCKING);
#endif
#ifdef USE_CMAP
res = cmap_dispatch(handle, CS_DISPATCH_BLOCKING);
#endif
} while (res == CS_OK || res == CS_ERR_TRY_AGAIN);
} else {
signal(SIGINT, sigint_handler_parent);
for (i = 0; i < no_childs; i++) {
wait(&status);
}
#ifdef USE_CONFDB
confdb_object_destroy(handle, object_handle);
#endif
printf("Confdb-track-and-change finalization finished\n");
}
return (0);
}
