CorosyncMemberId CorosyncCpg::self() const {
unsigned int nodeid;
if (CS_OK != (cpg_local_get(handle, &nodeid))) {
LOG(ERROR)<< "Cannot get local CPG identity";
}
return CorosyncMemberId(nodeid, getpid());
}
uint32_t get_local_nodeid(cpg_handle_t handle)
{
int rc = CS_OK;
int retries = 0;
static uint32_t local_nodeid = 0;
cpg_handle_t local_handle = handle;
cpg_callbacks_t cb = { };
if(local_nodeid != 0) {
return local_nodeid;
}
if(handle == 0) {
crm_trace("Creating connection");
cs_repeat(retries, 5, rc = cpg_initialize(&local_handle, &cb));
}
if (rc == CS_OK) {
retries = 0;
crm_trace("Performing lookup");
cs_repeat(retries, 5, rc = cpg_local_get(local_handle, &local_nodeid));
}
if (rc != CS_OK) {
crm_err("Could not get local node id from the CPG API: %s (%d)", ais_error2text(rc), rc);
}
if(handle == 0) {
crm_trace("Closing connection");
cpg_finalize(local_handle);
}
crm_debug("Local nodeid is %u", local_nodeid);
return local_nodeid;
}
static int _init_cluster(void)
{
SaAisErrorT err;
SaVersionT ver = { 'B', 1, 1 };
int select_fd;
node_hash = dm_hash_create(100);
lock_hash = dm_hash_create(10);
err = cpg_initialize(&cpg_handle,
&openais_cpg_callbacks);
if (err != SA_AIS_OK) {
syslog(LOG_ERR, "Cannot initialise OpenAIS CPG service: %d",
err);
DEBUGLOG("Cannot initialise OpenAIS CPG service: %d", err);
return ais_to_errno(err);
}
err = saLckInitialize(&lck_handle,
NULL,
&ver);
if (err != SA_AIS_OK) {
cpg_initialize(&cpg_handle, &openais_cpg_callbacks);
syslog(LOG_ERR, "Cannot initialise OpenAIS lock service: %d",
err);
DEBUGLOG("Cannot initialise OpenAIS lock service: %d\n\n", err);
return ais_to_errno(err);
}
/* Connect to the clvmd group */
strcpy((char *)cpg_group_name.value, "clvmd");
cpg_group_name.length = strlen((char *)cpg_group_name.value);
err = cpg_join(cpg_handle, &cpg_group_name);
if (err != SA_AIS_OK) {
cpg_finalize(cpg_handle);
saLckFinalize(lck_handle);
syslog(LOG_ERR, "Cannot join clvmd process group");
DEBUGLOG("Cannot join clvmd process group: %d\n", err);
return ais_to_errno(err);
}
err = cpg_local_get(cpg_handle,
&our_nodeid);
if (err != SA_AIS_OK) {
cpg_finalize(cpg_handle);
saLckFinalize(lck_handle);
syslog(LOG_ERR, "Cannot get local node id\n");
return ais_to_errno(err);
}
DEBUGLOG("Our local node id is %d\n", our_nodeid);
saLckSelectionObjectGet(lck_handle, (SaSelectionObjectT *)&select_fd);
add_internal_client(select_fd, lck_dispatch);
DEBUGLOG("Connected to OpenAIS\n");
return 0;
}
gboolean
cluster_connect_cpg(void)
{
cs_error_t rc;
unsigned int nodeid;
int fd;
int retries = 0;
static struct mainloop_fd_callbacks cpg_fd_callbacks =
{
.dispatch = pcmk_cpg_dispatch,
.destroy = cpg_connection_destroy,
};
strcpy(cpg_group.value, "pcmk");
cpg_group.length = strlen(cpg_group.value) + 1;
retries = 0;
cs_repeat(retries, 30, rc = cpg_initialize(&cpg_handle, &cpg_callbacks));
if (rc != CS_OK) {
crm_err("corosync cpg init error %d", rc);
return FALSE;
}
rc = cpg_fd_get(cpg_handle, &fd);
if (rc != CS_OK) {
crm_err("corosync cpg fd_get error %d", rc);
goto bail;
}
retries = 0;
cs_repeat(retries, 30, rc = cpg_local_get(cpg_handle, &nodeid));
if (rc != CS_OK) {
crm_err("corosync cpg local_get error %d", rc);
goto bail;
}
crm_debug("Our nodeid: %d", nodeid);
retries = 0;
cs_repeat(retries, 30, rc = cpg_join(cpg_handle, &cpg_group));
if (rc != CS_OK) {
crm_err("Could not join the CPG group '%s': %d", crm_system_name, rc);
goto bail;
}
mainloop_add_fd("corosync-cpg", G_PRIORITY_DEFAULT, fd, &cpg_handle, &cpg_fd_callbacks);
return TRUE;
bail:
cpg_finalize(cpg_handle);
return FALSE;
}
static void msg_blaster_zcb (int sock, char* num_to_send_str)
{
my_msgs_to_send = atoi (num_to_send_str);
my_seq = 1;
my_pid = getpid();
use_zcb = QB_TRUE;
total_stored_msgs = 0;
cpg_local_get (cpg_handle, &my_nodeid);
/* control the limits */
if (my_msgs_to_send <= 0)
my_msgs_to_send = 1;
if (my_msgs_to_send > 10000)
my_msgs_to_send = 10000;
send_some_more_messages_zcb ();
}
uint32_t get_local_nodeid(cpg_handle_t handle)
{
int rc = CS_OK;
int retries = 0;
static uint32_t local_nodeid = 0;
cpg_handle_t local_handle = handle;
cpg_callbacks_t cb = { };
if(local_nodeid != 0) {
return local_nodeid;
}
#if 0
/* Should not be necessary */
if(get_cluster_type() == pcmk_cluster_classic_ais) {
get_ais_details(&local_nodeid, NULL);
goto done;
}
#endif
if(handle == 0) {
crm_trace("Creating connection");
cs_repeat(retries, 5, rc = cpg_initialize(&local_handle, &cb));
}
if (rc == CS_OK) {
retries = 0;
crm_trace("Performing lookup");
cs_repeat(retries, 5, rc = cpg_local_get(local_handle, &local_nodeid));
}
if (rc != CS_OK) {
crm_err("Could not get local node id from the CPG API: %s (%d)", ais_error2text(rc), rc);
}
if(handle == 0) {
crm_trace("Closing connection");
cpg_finalize(local_handle);
}
crm_debug("Local nodeid is %u", local_nodeid);
return local_nodeid;
}
static void print_localnodeid(cpg_handle_t handle)
{
char addrStr[128];
unsigned int retries;
unsigned int nodeid;
struct sockaddr_storage addr;
struct sockaddr_in *v4addr = (struct sockaddr_in *)&addr;
int result;
retries = 0;
cs_repeat(retries, 30, result = cpg_local_get(handle, &nodeid));
if (result != CS_OK) {
printf ("Could not get local node id\n");
} else {
v4addr->sin_addr.s_addr = nodeid;
if(inet_ntop(AF_INET, (const void *)&v4addr->sin_addr.s_addr,
addrStr, (socklen_t)sizeof(addrStr)) == NULL) {
addrStr[0] = 0;
}
printf ("Local node id is %s/%x result %d\n", addrStr, nodeid, result);
}
}
static void do_command (int sock, char* func, char*args[], int num_args)
{
int result;
char response[100];
struct cpg_name group_name;
ssize_t rc;
size_t send_len;
qb_log (LOG_TRACE, "RPC:%s() called.", func);
if (strcmp ("cpg_mcast_joined",func) == 0) {
struct iovec iov[5];
int a;
for (a = 0; a < num_args; a++) {
iov[a].iov_base = args[a];
iov[a].iov_len = strlen(args[a])+1;
}
cpg_mcast_joined (cpg_handle, CPG_TYPE_AGREED, iov, num_args);
} else if (strcmp ("cpg_join",func) == 0) {
if (strlen(args[0]) >= CPG_MAX_NAME_LENGTH) {
qb_log (LOG_ERR, "Invalid group name");
exit (1);
}
strcpy (group_name.value, args[0]);
group_name.length = strlen(args[0]);
result = cpg_join (cpg_handle, &group_name);
if (result != CS_OK) {
qb_log (LOG_ERR,
"Could not join process group, error %d", result);
exit (1);
}
qb_log (LOG_INFO, "called cpg_join(%s)!", group_name.value);
} else if (strcmp ("cpg_leave",func) == 0) {
strcpy (group_name.value, args[0]);
group_name.length = strlen(args[0]);
result = cpg_leave (cpg_handle, &group_name);
if (result != CS_OK) {
qb_log (LOG_ERR,
"Could not leave process group, error %d", result);
exit (1);
}
qb_log (LOG_INFO, "called cpg_leave(%s)!", group_name.value);
} else if (strcmp ("cpg_initialize",func) == 0) {
int retry_count = 0;
result = cpg_initialize (&cpg_handle, &callbacks);
while (result != CS_OK) {
qb_log (LOG_ERR,
"cpg_initialize error %d (attempt %d)",
result, retry_count);
if (retry_count >= 3) {
exit (1);
}
sleep(1);
retry_count++;
result = cpg_initialize (&cpg_handle, &callbacks);
}
cpg_fd_get (cpg_handle, &cpg_fd);
qb_loop_poll_add (ta_poll_handle_get(),
QB_LOOP_MED,
cpg_fd,
POLLIN|POLLNVAL,
NULL,
cpg_dispatch_wrapper_fn);
} else if (strcmp ("cpg_local_get", func) == 0) {
unsigned int local_nodeid;
cpg_local_get (cpg_handle, &local_nodeid);
snprintf (response, 100, "%u",local_nodeid);
send_len = strlen (response);
rc = send (sock, response, send_len, 0);
assert(rc == send_len);
} else if (strcmp ("cpg_finalize", func) == 0) {
if (cpg_handle > 0) {
cpg_finalize (cpg_handle);
cpg_handle = 0;
}
} else if (strcmp ("record_config_events", func) == 0) {
record_config_events (sock);
} else if (strcmp ("record_messages", func) == 0) {
record_messages ();
} else if (strcmp ("read_config_event", func) == 0) {
read_config_event (sock);
} else if (strcmp ("read_messages", func) == 0) {
read_messages (sock, args[0]);
} else if (strcmp ("msg_blaster_zcb", func) == 0) {
msg_blaster_zcb (sock, args[0]);
} else if (strcmp ("pcmk_test", func) == 0) {
pcmk_test = 1;
} else if (strcmp ("msg_blaster", func) == 0) {
msg_blaster (sock, args[0]);
} else if (strcmp ("context_test", func) == 0) {
context_test (sock);
} else if (strcmp ("are_you_ok_dude", func) == 0) {
snprintf (response, 100, "%s", OK_STR);
send_len = strlen (response);
rc = send (sock, response, strlen (response), 0);
assert(rc == send_len);
} else if (strcmp ("cfg_shutdown", func) == 0) {
qb_log (LOG_INFO, "calling %s() called!", func);
result = corosync_cfg_try_shutdown (cfg_handle, COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST);
qb_log (LOG_INFO,"%s() returned %d!", func, result);
} else if (strcmp ("cfg_initialize",func) == 0) {
int retry_count = 0;
qb_log (LOG_INFO,"%s() called!", func);
result = corosync_cfg_initialize (&cfg_handle, &cfg_callbacks);
while (result != CS_OK) {
qb_log (LOG_ERR,
"cfg_initialize error %d (attempt %d)",
result, retry_count);
if (retry_count >= 3) {
exit (1);
}
sleep(1);
retry_count++;
result = corosync_cfg_initialize (&cfg_handle, &cfg_callbacks);
}
qb_log (LOG_INFO,"corosync_cfg_initialize() == %d", result);
result = corosync_cfg_fd_get (cfg_handle, &cfg_fd);
qb_log (LOG_INFO,"corosync_cfg_fd_get() == %d", result);
qb_loop_poll_add (ta_poll_handle_get(),
QB_LOOP_MED,
cfg_fd,
POLLIN|POLLNVAL,
NULL,
cfg_dispatch_wrapper_fn);
} else {
qb_log(LOG_ERR, "RPC:%s not supported!", func);
}
}
int main (int argc, char *argv[]) {
cpg_handle_t handle;
fd_set read_fds;
int select_fd;
int result;
const char *options = "i";
int opt;
unsigned int nodeid;
char *fgets_res;
while ( (opt = getopt(argc, argv, options)) != -1 ) {
switch (opt) {
case 'i':
show_ip = 1;
break;
}
}
if (argc > optind) {
strcpy(group_name.value, argv[optind]);
group_name.length = strlen(argv[optind]);
}
else {
strcpy(group_name.value, "GROUP");
group_name.length = 6;
}
result = cpg_initialize (&handle, &callbacks);
if (result != CS_OK) {
printf ("Could not initialize Cluster Process Group API instance error %d\n", result);
exit (1);
}
result = cpg_local_get (handle, &nodeid);
if (result != CS_OK) {
printf ("Could not get local node id\n");
exit (1);
}
printf ("Local node id is %x\n", nodeid);
result = cpg_join(handle, &group_name);
if (result != CS_OK) {
printf ("Could not join process group, error %d\n", result);
exit (1);
}
FD_ZERO (&read_fds);
cpg_fd_get(handle, &select_fd);
printf ("Type EXIT to finish\n");
do {
FD_SET (select_fd, &read_fds);
FD_SET (STDIN_FILENO, &read_fds);
result = select (select_fd + 1, &read_fds, 0, 0, 0);
if (result == -1) {
perror ("select\n");
}
if (FD_ISSET (STDIN_FILENO, &read_fds)) {
char inbuf[132];
struct iovec iov;
fgets_res = fgets(inbuf, sizeof(inbuf), stdin);
if (fgets_res == NULL) {
cpg_leave(handle, &group_name);
}
if (strncmp(inbuf, "EXIT", 4) == 0) {
cpg_leave(handle, &group_name);
}
else {
iov.iov_base = inbuf;
iov.iov_len = strlen(inbuf)+1;
cpg_mcast_joined(handle, CPG_TYPE_AGREED, &iov, 1);
}
}
if (FD_ISSET (select_fd, &read_fds)) {
if (cpg_dispatch (handle, CS_DISPATCH_ALL) != CS_OK)
exit(1);
}
} while (result && !quit);
result = cpg_finalize (handle);
printf ("Finalize result is %d (should be 1)\n", result);
return (0);
}
/*
* CFG functionality stolen from node_name() in corosync-quorumtool.c
* This resolves the first address assigned to a node and returns the name or IP address.
*/
char *
corosync_node_name(uint64_t /*cmap_handle_t */ cmap_handle, uint32_t nodeid)
{
int lpc = 0;
int rc = CS_OK;
int retries = 0;
char *name = NULL;
cmap_handle_t local_handle = 0;
/* nodeid == 0 == CMAN_NODEID_US */
if (nodeid == 0 && pcmk_nodeid) {
nodeid = pcmk_nodeid;
} else if (nodeid == 0) {
/* Look it up */
int rc = -1;
int retries = 0;
cpg_handle_t handle = 0;
cpg_callbacks_t cb = { };
cs_repeat(retries, 5, rc = cpg_initialize(&handle, &cb));
if (rc == CS_OK) {
retries = 0;
cs_repeat(retries, 5, rc = cpg_local_get(handle, &pcmk_nodeid));
}
if (rc != CS_OK) {
crm_err("Could not get local node id from the CPG API: %d", rc);
}
cpg_finalize(handle);
}
if (cmap_handle == 0 && local_handle == 0) {
retries = 0;
crm_trace("Initializing CMAP connection");
do {
rc = cmap_initialize(&local_handle);
if (rc != CS_OK) {
retries++;
crm_debug("API connection setup failed: %s. Retrying in %ds", cs_strerror(rc),
retries);
sleep(retries);
}
} while (retries < 5 && rc != CS_OK);
if (rc != CS_OK) {
crm_warn("Could not connect to Cluster Configuration Database API, error %s",
cs_strerror(rc));
local_handle = 0;
}
}
if (cmap_handle == 0) {
cmap_handle = local_handle;
}
while (name == NULL && cmap_handle != 0) {
uint32_t id = 0;
char *key = NULL;
key = g_strdup_printf("nodelist.node.%d.nodeid", lpc);
rc = cmap_get_uint32(cmap_handle, key, &id);
crm_trace("Checking %u vs %u from %s", nodeid, id, key);
g_free(key);
if (rc != CS_OK) {
break;
}
if (nodeid == id) {
crm_trace("Searching for node name for %u in nodelist.node.%d %s", nodeid, lpc, name);
if (name == NULL) {
key = g_strdup_printf("nodelist.node.%d.ring0_addr", lpc);
rc = cmap_get_string(cmap_handle, key, &name);
crm_trace("%s = %s", key, name);
if (node_name_is_valid(key, name) == FALSE) {
free(name);
name = NULL;
}
g_free(key);
}
if (name == NULL) {
key = g_strdup_printf("nodelist.node.%d.name", lpc);
rc = cmap_get_string(cmap_handle, key, &name);
crm_trace("%s = %s %d", key, name, rc);
g_free(key);
}
break;
}
lpc++;
}
if(local_handle) {
cmap_finalize(local_handle);
}
if (name == NULL) {
crm_notice("Unable to get node name for nodeid %u", nodeid);
}
return name;
}
static void ConfchgCallback (
cpg_handle_t handle,
const struct cpg_name *groupName,
const struct cpg_address *member_list, size_t member_list_entries,
const struct cpg_address *left_list, size_t left_list_entries,
const struct cpg_address *joined_list, size_t joined_list_entries)
{
unsigned int i;
int result;
uint32_t nodeid;
print_time();
printf("ConfchgCallback: group '");
print_cpgname(groupName);
printf("'\n");
print_localnodeid(handle);
for (i=0; i<joined_list_entries; i++) {
printf("joined %s reason: %d\n",
node_pid_format(joined_list[i].nodeid, joined_list[i].pid),
joined_list[i].reason);
}
for (i=0; i<left_list_entries; i++) {
printf("left %s reason: %d\n",
node_pid_format(left_list[i].nodeid, left_list[i].pid),
left_list[i].reason);
}
printf("nodes in group now %lu\n",
(unsigned long int) member_list_entries);
for (i=0; i<member_list_entries; i++) {
printf("%s\n",
node_pid_format(member_list[i].nodeid, member_list[i].pid));
}
result = cpg_local_get(handle, &nodeid);
if(result != CS_OK) {
printf("failed to get local nodeid %d\n", result);
nodeid = 0;
}
/* Is it us??
NOTE: in reality we should also check the nodeid */
if (left_list_entries && (pid_t)left_list[0].pid == getpid()) {
printf("We might have left the building pid %d\n", left_list[0].pid);
/* can only use nodeidStart as a reliable check (version <= 1.4.2) */
if(nodeidStart) {
/* report dynamic nature of nodeid returned from local_get */
/* local get of nodeid might change identity from original! */
if(htonl((uint32_t)nodeid) == INADDR_LOOPBACK) {
printf("We probably left the building switched identity? start nodeid %d nodeid %d current nodeid %d pid %d\n", nodeidStart, left_list[0].nodeid, nodeid, left_list[0].pid);
} else if(htonl((uint32_t)left_list[0].nodeid) == INADDR_LOOPBACK) {
printf("We probably left the building started alone? start nodeid %d nodeid %d current nodeid %d pid %d\n", nodeidStart, left_list[0].nodeid, nodeid, left_list[0].pid);
}
/* a possibly reliable way to check is based on original address */
if(left_list[0].nodeid == nodeidStart) {
printf("We have left the building direct match start nodeid %d nodeid %d local get current nodeid %d pid %d\n", nodeidStart, left_list[0].nodeid, nodeid, left_list[0].pid);
// quit = 1;
restart = 1;
} else {
printf("Probably another node with matching pid start nodeid %d nodeid %d current nodeid %d pid %d\n", nodeidStart, left_list[0].nodeid, nodeid, left_list[0].pid);
}
}
}
}
static int _init_cluster(void)
{
cs_error_t err;
node_hash = dm_hash_create(100);
err = cpg_initialize(&cpg_handle,
&corosync_cpg_callbacks);
if (err != CS_OK) {
syslog(LOG_ERR, "Cannot initialise Corosync CPG service: %d",
err);
DEBUGLOG("Cannot initialise Corosync CPG service: %d", err);
return cs_to_errno(err);
}
err = quorum_initialize(&quorum_handle,
&quorum_callbacks);
if (err != CS_OK) {
syslog(LOG_ERR, "Cannot initialise Corosync quorum service: %d",
err);
DEBUGLOG("Cannot initialise Corosync quorum service: %d", err);
return cs_to_errno(err);
}
/* Create a lockspace for LV & VG locks to live in */
lockspace = dlm_create_lockspace(LOCKSPACE_NAME, 0600);
if (!lockspace) {
if (errno == EEXIST) {
lockspace = dlm_open_lockspace(LOCKSPACE_NAME);
}
if (!lockspace) {
syslog(LOG_ERR, "Unable to create lockspace for CLVM: %m");
quorum_finalize(quorum_handle);
return -1;
}
}
dlm_ls_pthread_init(lockspace);
DEBUGLOG("DLM initialisation complete\n");
/* Connect to the clvmd group */
strcpy((char *)cpg_group_name.value, "clvmd");
cpg_group_name.length = strlen((char *)cpg_group_name.value);
err = cpg_join(cpg_handle, &cpg_group_name);
if (err != CS_OK) {
cpg_finalize(cpg_handle);
quorum_finalize(quorum_handle);
dlm_release_lockspace(LOCKSPACE_NAME, lockspace, 1);
syslog(LOG_ERR, "Cannot join clvmd process group");
DEBUGLOG("Cannot join clvmd process group: %d\n", err);
return cs_to_errno(err);
}
err = cpg_local_get(cpg_handle,
&our_nodeid);
if (err != CS_OK) {
cpg_finalize(cpg_handle);
quorum_finalize(quorum_handle);
dlm_release_lockspace(LOCKSPACE_NAME, lockspace, 1);
syslog(LOG_ERR, "Cannot get local node id\n");
return cs_to_errno(err);
}
DEBUGLOG("Our local node id is %d\n", our_nodeid);
DEBUGLOG("Connected to Corosync\n");
return 0;
}
int main (int argc, char *argv[]) {
int i;
unsigned int res;
uint32_t maxsize;
int opt;
int bs;
int write_size = 4096;
int delay_time = 1000;
int repetitions = 100;
int print_time = 10;
int have_size = 0;
int listen_only = 0;
int flood = 0;
int model = 1;
int option_index = 0;
struct option long_options[] = {
{"flood-start", required_argument, 0, 0 },
{"flood-mult", required_argument, 0, 0 },
{"flood-max", required_argument, 0, 0 },
{"size-kb", required_argument, 0, 'w' },
{"size-bytes", required_argument, 0, 'W' },
{"name", required_argument, 0, 'n' },
{"rtt", no_argument, 0, 't' },
{"flood", no_argument, 0, 'f' },
{"quiet", no_argument, 0, 'q' },
{"listen", no_argument, 0, 'l' },
{"help", no_argument, 0, '?' },
{0, 0, 0, 0 }
};
while ( (opt = getopt_long(argc, argv, "qlstafMEn:d:r:p:m:w:W:D:",
long_options, &option_index)) != -1 ) {
switch (opt) {
case 0: // Long-only options
if (strcmp(long_options[option_index].name, "flood-start") == 0) {
flood_start = parse_bytes(optarg);
if (flood_start == 0) {
fprintf(stderr, "flood-start value invalid\n");
exit(1);
}
}
if (strcmp(long_options[option_index].name, "flood-mult") == 0) {
flood_multiplier = parse_bytes(optarg);
if (flood_multiplier == 0) {
fprintf(stderr, "flood-mult value invalid\n");
exit(1);
}
}
if (strcmp(long_options[option_index].name, "flood-max") == 0) {
flood_max = parse_bytes(optarg);
if (flood_max == 0) {
fprintf(stderr, "flood-max value invalid\n");
exit(1);
}
}
break;
case 'w': // Write size in K
bs = atoi(optarg);
if (bs > 0) {
write_size = bs*1024;
have_size = 1;
}
break;
case 'W': // Write size in bytes (or with a suffix)
bs = parse_bytes(optarg);
if (bs > 0) {
write_size = bs;
have_size = 1;
}
break;
case 'n':
if (strlen(optarg) >= CPG_MAX_NAME_LENGTH) {
fprintf(stderr, "CPG name too long\n");
exit(1);
}
strcpy(group_name.value, optarg);
group_name.length = strlen(group_name.value);
break;
case 't':
report_rtt = 1;
break;
case 'E':
to_stderr = 1;
break;
case 'M':
machine_readable = 1;
break;
case 'f':
flood = 1;
break;
case 'a':
abort_on_error = 1;
break;
case 'd':
delay_time = atoi(optarg);
break;
case 'D':
delimiter = optarg[0];
break;
case 'r':
repetitions = atoi(optarg);
break;
case 'p':
print_time = atoi(optarg);
break;
case 'l':
listen_only = 1;
break;
case 's':
do_syslog = 1;
break;
case 'q':
quiet++;
break;
case 'm':
model = atoi(optarg);
if (model < 0 || model > 1) {
fprintf(stderr, "%s: Model must be 0-1\n", argv[0]);
exit(1);
}
break;
case '?':
usage(basename(argv[0]));
exit(1);
}
}
if (!have_size && flood) {
write_size = flood_start;
}
signal (SIGALRM, sigalrm_handler);
signal (SIGINT, sigint_handler);
switch (model) {
case 0:
res = cpg_initialize (&handle, &callbacks);
break;
case 1:
res = cpg_model_initialize (&handle, CPG_MODEL_V1, (cpg_model_data_t *)&model1_data, NULL);
break;
default:
res=999; // can't get here but it keeps the compiler happy
break;
}
if (res != CS_OK) {
cpgh_log_printf(CPGH_LOG_ERR, "cpg_initialize failed with result %d\n", res);
exit (1);
}
res = cpg_local_get(handle, &g_our_nodeid);
if (res != CS_OK) {
cpgh_log_printf(CPGH_LOG_ERR, "cpg_local_get failed with result %d\n", res);
exit (1);
}
pthread_create (&thread, NULL, dispatch_thread, NULL);
res = cpg_join (handle, &group_name);
if (res != CS_OK) {
cpgh_log_printf(CPGH_LOG_ERR, "cpg_join failed with result %d\n", res);
exit (1);
}
if (listen_only) {
int secs = 0;
while (!stopped) {
sleep(1);
if (++secs > print_time && !quiet) {
int nodes_printed = 0;
if (!machine_readable) {
for (i=1; i<MAX_NODEID; i++) {
if (g_recv_counter[i]) {
cpgh_log_printf(CPGH_LOG_INFO, "%s: %5d message%s of %d bytes received from node %d\n",
group_name.value, g_recv_counter[i] - g_recv_start[i],
g_recv_counter[i]==1?"":"s",
g_recv_size[i], i);
nodes_printed++;
}
}
}
/* Separate list of nodes if more than one */
if (nodes_printed > 1) {
cpgh_log_printf(CPGH_LOG_INFO, "\n");
}
secs = 0;
}
}
}
else {
cpg_max_atomic_msgsize_get (handle, &maxsize);
if (write_size > maxsize) {
fprintf(stderr, "INFO: packet size (%d) is larger than the maximum atomic size (%d), libcpg will fragment\n",
write_size, maxsize);
}
/* The main job starts here */
if (flood) {
for (i = 0; i < 10; i++) { /* number of repetitions - up to 50k */
cpg_flood (handle, write_size);
signal (SIGALRM, sigalrm_handler);
write_size *= flood_multiplier;
if (write_size > flood_max) {
break;
}
}
}
else {
send_counter = -1; /* So we start from zero to allow listeners to sync */
for (i = 0; i < repetitions && !stopped; i++) {
cpg_test (handle, write_size, delay_time, print_time);
signal (SIGALRM, sigalrm_handler);
}
}
}
res = cpg_finalize (handle);
if (res != CS_OK) {
cpgh_log_printf(CPGH_LOG_ERR, "cpg_finalize failed with result %d\n", res);
exit (1);
}
if (quiet < 2) {
/* Don't print LONG_MAX for min_rtt if we don't have a value */
if (min_rtt == LONG_MAX) {
min_rtt = 0L;
}
if (machine_readable) {
cpgh_log_printf(CPGH_LOG_STATS, "%d%c%d%c%d%c%d%c%d%c%d%c%d%c%ld%c%ld%c%ld\n",
packets_sent, delimiter,
send_fails, delimiter,
send_retries, delimiter,
length_errors, delimiter,
packets_recvd, delimiter,
sequence_errors, delimiter,
crc_errors, delimiter,
min_rtt, delimiter,
avg_rtt, delimiter,
max_rtt);
}
else {
cpgh_log_printf(CPGH_LOG_STATS, "\n");
cpgh_log_printf(CPGH_LOG_STATS, "Stats:\n");
if (!listen_only) {
cpgh_log_printf(CPGH_LOG_STATS, " packets sent: %d\n", packets_sent);
cpgh_log_printf(CPGH_LOG_STATS, " send failures: %d\n", send_fails);
cpgh_log_printf(CPGH_LOG_STATS, " send retries: %d\n", send_retries);
}
cpgh_log_printf(CPGH_LOG_STATS, " length errors: %d\n", length_errors);
cpgh_log_printf(CPGH_LOG_STATS, " packets recvd: %d\n", packets_recvd);
cpgh_log_printf(CPGH_LOG_STATS, " sequence errors: %d\n", sequence_errors);
cpgh_log_printf(CPGH_LOG_STATS, " crc errors: %d\n", crc_errors);
if (!listen_only) {
cpgh_log_printf(CPGH_LOG_STATS, " min RTT: %ld\n", min_rtt);
cpgh_log_printf(CPGH_LOG_STATS, " max RTT: %ld\n", max_rtt);
cpgh_log_printf(CPGH_LOG_STATS, " avg RTT: %ld\n", avg_rtt);
}
cpgh_log_printf(CPGH_LOG_STATS, "\n");
}
}
res = 0;
if (send_fails > 0 || (have_size && length_errors > 0) || sequence_errors > 0 || crc_errors > 0) {
res = 2;
}
return (res);
}
static void
test_cpg(void)
{
cpg_handle_t cpg_handle;
struct cpg_name group_name;
cs_error_t cs_res;
unsigned int local_nodeid;
int cpg_fd;
struct pollfd pfd;
int poll_res;
int state;
int cont;
ENTER();
state = 0;
strcpy(group_name.value, TEST_GROUP_NAME);
group_name.length = strlen(TEST_GROUP_NAME);
cs_repeat(cs_res = cpg_model_initialize(&cpg_handle, CPG_MODEL_V1,
(cpg_model_data_t *)&model_data, NULL));
assert(cs_res == CS_OK);
cs_repeat(cs_res = cpg_join(cpg_handle, &group_name));
assert(cs_res == CS_OK);
cs_repeat(cs_res = cpg_local_get(cpg_handle, &local_nodeid));
assert(cs_res == CS_OK);
assert(local_nodeid == TEST_NODEID);
cs_repeat(cs_res = cpg_fd_get(cpg_handle, &cpg_fd));
assert(cs_res == CS_OK);
cont = 1;
do {
pfd.fd = cpg_fd;
pfd.events = POLLIN;
pfd.revents = 0;
poll_res = poll(&pfd, 1, INFTIM);
if (poll_res == -1) {
perror("poll_res == -1");
}
assert(poll_res != 0);
assert(pfd.revents & POLLIN);
cs_repeat(cs_res = cpg_dispatch(cpg_handle, CS_DISPATCH_ALL));
assert(cs_res == CS_OK);
switch (state) {
case 0:
/*
* Waiting for cpg_confchg_received and cpg_totem_confchg_received
*/
if (cpg_confchg_received && cpg_totem_confchg_received) {
/*
* Send first message and wait for it in next state
*/
send_msg(cpg_handle);
state = 1;
}
break;
case 1:
if (received_msg_seq_no >= MESSAGES_TO_SENT) {
cont = 0;
} else if (received_msg_seq_no == sent_msg_seq_no) {
/*
* Message delivered so sent new one and wait for it
*/
send_msg(cpg_handle);
}
break;
}
} while (cont);
cs_repeat(cs_res = cpg_leave(cpg_handle, &group_name));
assert(cs_res == CS_OK);
cs_repeat(cs_res = cpg_finalize(cpg_handle));
assert(cs_res == CS_OK);
LEAVE();
}
int main (int argc, char *argv[]) {
cpg_handle_t handle;
fd_set read_fds;
int select_fd;
int result;
int retries;
const char *options = "i";
int opt;
unsigned int nodeid;
char *fgets_res;
struct cpg_address member_list[64];
int member_list_entries;
int i;
int recnt;
int doexit;
const char *exitStr = "EXIT";
doexit = 0;
#ifdef QBLOG
qb_log_init("testcpg", LOG_USER, LOG_ERR);
qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_ENABLED, QB_FALSE);
qb_log_filter_ctl(QB_LOG_STDERR, QB_LOG_FILTER_ADD,
QB_LOG_FILTER_FILE, "*", LOG_TRACE);
qb_log_ctl(QB_LOG_STDERR, QB_LOG_CONF_ENABLED, QB_TRUE);
qb_log_format_set(QB_LOG_STDERR, "[%p] %f %b");
#endif
while ( (opt = getopt(argc, argv, options)) != -1 ) {
switch (opt) {
case 'i':
show_ip = 1;
break;
}
}
if (argc > optind) {
strcpy(group_name.value, argv[optind]);
group_name.length = strlen(argv[optind]);
}
else {
strcpy(group_name.value, "GROUP");
group_name.length = 6;
}
recnt = 0;
printf ("Type %s to finish\n", exitStr);
restart = 1;
do {
if(restart) {
restart = 0;
retries = 0;
cs_repeat_init(retries, 30, result = cpg_model_initialize (&handle, CPG_MODEL_V1, (cpg_model_data_t *)&model_data, NULL));
if (result != CS_OK) {
printf ("Could not initialize Cluster Process Group API instance error %d\n", result);
retrybackoff(recnt);
}
retries = 0;
cs_repeat(retries, 30, result = cpg_local_get(handle, &nodeid));
if (result != CS_OK) {
printf ("Could not get local node id\n");
retrybackoff(recnt);
}
printf ("Local node id is %x\n", nodeid);
nodeidStart = nodeid;
retries = 0;
cs_repeat(retries, 30, result = cpg_join(handle, &group_name));
if (result != CS_OK) {
printf ("Could not join process group, error %d\n", result);
retrybackoff(recnt);
}
retries = 0;
cs_repeat(retries, 30, result = cpg_membership_get (handle, &group_name,
(struct cpg_address *)&member_list, &member_list_entries));
if (result != CS_OK) {
printf ("Could not get current membership list %d\n", result);
retrybackoff(recnt);
}
recnt = 0;
printf ("membership list\n");
for (i = 0; i < member_list_entries; i++) {
printf ("node id %d pid %d\n", member_list[i].nodeid,
member_list[i].pid);
}
FD_ZERO (&read_fds);
cpg_fd_get(handle, &select_fd);
}
FD_SET (select_fd, &read_fds);
FD_SET (STDIN_FILENO, &read_fds);
result = select (select_fd + 1, &read_fds, 0, 0, 0);
if (result == -1) {
perror ("select\n");
}
if (FD_ISSET (STDIN_FILENO, &read_fds)) {
char inbuf[132];
struct iovec iov;
fgets_res = fgets(inbuf, (int)sizeof(inbuf), stdin);
if (fgets_res == NULL) {
doexit = 1;
cpg_leave(handle, &group_name);
}
if (strncmp(inbuf, exitStr, strlen(exitStr)) == 0) {
doexit = 1;
cpg_leave(handle, &group_name);
}
else {
iov.iov_base = inbuf;
iov.iov_len = strlen(inbuf)+1;
cpg_mcast_joined(handle, CPG_TYPE_AGREED, &iov, 1);
}
}
if (FD_ISSET (select_fd, &read_fds)) {
if (cpg_dispatch (handle, CS_DISPATCH_ALL) != CS_OK) {
if(doexit) {
exit(1);
}
restart = 1;
}
}
if(restart) {
if(!doexit) {
result = cpg_finalize (handle);
printf ("Finalize+restart result is %d (should be 1)\n", result);
continue;
}
}
} while (result && !quit && !doexit);
result = cpg_finalize (handle);
printf ("Finalize result is %d (should be 1)\n", result);
return (0);
}
static int _init_cluster(void)
{
cs_error_t err;
#ifdef QUORUM_SET /* corosync/quorum.h */
uint32_t quorum_type;
#endif
node_hash = dm_hash_create(100);
err = cpg_initialize(&cpg_handle,
&corosync_cpg_callbacks);
if (err != CS_OK) {
syslog(LOG_ERR, "Cannot initialise Corosync CPG service: %d",
err);
DEBUGLOG("Cannot initialise Corosync CPG service: %d", err);
return cs_to_errno(err);
}
#ifdef QUORUM_SET
err = quorum_initialize(&quorum_handle,
&quorum_callbacks,
&quorum_type);
if (quorum_type != QUORUM_SET) {
syslog(LOG_ERR, "Corosync quorum service is not configured");
DEBUGLOG("Corosync quorum service is not configured");
return EINVAL;
}
#else
err = quorum_initialize(&quorum_handle,
&quorum_callbacks);
#endif
if (err != CS_OK) {
syslog(LOG_ERR, "Cannot initialise Corosync quorum service: %d",
err);
DEBUGLOG("Cannot initialise Corosync quorum service: %d", err);
return cs_to_errno(err);
}
/* Create a lockspace for LV & VG locks to live in */
lockspace = dlm_open_lockspace(LOCKSPACE_NAME);
if (!lockspace) {
lockspace = dlm_create_lockspace(LOCKSPACE_NAME, 0600);
if (!lockspace) {
syslog(LOG_ERR, "Unable to create DLM lockspace for CLVM: %m");
return -1;
}
DEBUGLOG("Created DLM lockspace for CLVMD.\n");
} else
DEBUGLOG("Opened existing DLM lockspace for CLVMD.\n");
dlm_ls_pthread_init(lockspace);
DEBUGLOG("DLM initialisation complete\n");
/* Connect to the clvmd group */
strcpy((char *)cpg_group_name.value, "clvmd");
cpg_group_name.length = strlen((char *)cpg_group_name.value);
err = cpg_join(cpg_handle, &cpg_group_name);
if (err != CS_OK) {
cpg_finalize(cpg_handle);
quorum_finalize(quorum_handle);
dlm_release_lockspace(LOCKSPACE_NAME, lockspace, 1);
syslog(LOG_ERR, "Cannot join clvmd process group");
DEBUGLOG("Cannot join clvmd process group: %d\n", err);
return cs_to_errno(err);
}
err = cpg_local_get(cpg_handle,
&our_nodeid);
if (err != CS_OK) {
cpg_finalize(cpg_handle);
quorum_finalize(quorum_handle);
dlm_release_lockspace(LOCKSPACE_NAME, lockspace, 1);
syslog(LOG_ERR, "Cannot get local node id\n");
return cs_to_errno(err);
}
DEBUGLOG("Our local node id is %d\n", our_nodeid);
DEBUGLOG("Connected to Corosync\n");
return 0;
}
static void do_command (int sock, char* func, char*args[], int num_args)
{
int result;
char response[100];
struct cpg_name group_name;
if (parse_debug)
syslog (LOG_DEBUG,"RPC:%s() called.", func);
if (strcmp ("cpg_mcast_joined",func) == 0) {
struct iovec iov[5];
int a;
for (a = 0; a < num_args; a++) {
iov[a].iov_base = args[a];
iov[a].iov_len = strlen(args[a])+1;
}
cpg_mcast_joined (cpg_handle, CPG_TYPE_AGREED, iov, num_args);
} else if (strcmp ("cpg_join",func) == 0) {
strcpy (group_name.value, args[0]);
group_name.length = strlen(args[0]);
result = cpg_join (cpg_handle, &group_name);
if (result != CS_OK) {
syslog (LOG_ERR,
"Could not join process group, error %d\n", result);
exit (1);
}
syslog (LOG_INFO, "called cpg_join()!");
} else if (strcmp ("cpg_leave",func) == 0) {
strcpy (group_name.value, args[0]);
group_name.length = strlen(args[0]);
result = cpg_leave (cpg_handle, &group_name);
if (result != CS_OK) {
syslog (LOG_ERR,
"Could not leave process group, error %d\n", result);
exit (1);
}
syslog (LOG_INFO, "called cpg_leave()!");
} else if (strcmp ("cpg_initialize",func) == 0) {
int retry_count = 0;
result = cpg_initialize (&cpg_handle, &callbacks);
while (result != CS_OK) {
syslog (LOG_ERR,
"cpg_initialize error %d (attempt %d)\n",
result, retry_count);
if (retry_count >= 3) {
exit (1);
}
sleep(1);
retry_count++;
result = cpg_initialize (&cpg_handle, &callbacks);
}
cpg_fd_get (cpg_handle, &cpg_fd);
poll_dispatch_add (ta_poll_handle_get(), cpg_fd, POLLIN|POLLNVAL, NULL, cpg_dispatch_wrapper_fn);
} else if (strcmp ("cpg_local_get", func) == 0) {
unsigned int local_nodeid;
cpg_local_get (cpg_handle, &local_nodeid);
snprintf (response, 100, "%u",local_nodeid);
send (sock, response, strlen (response), 0);
} else if (strcmp ("cpg_finalize", func) == 0) {
cpg_finalize (cpg_handle);
poll_dispatch_delete (ta_poll_handle_get(), cpg_fd);
cpg_fd = -1;
} else if (strcmp ("record_config_events", func) == 0) {
record_config_events (sock);
} else if (strcmp ("record_messages", func) == 0) {
record_messages ();
} else if (strcmp ("read_config_event", func) == 0) {
read_config_event (sock);
} else if (strcmp ("read_messages", func) == 0) {
read_messages (sock, args[0]);
} else if (strcmp ("msg_blaster_zcb", func) == 0) {
msg_blaster_zcb (sock, args[0]);
} else if (strcmp ("pcmk_test", func) == 0) {
pcmk_test = 1;
} else if (strcmp ("msg_blaster", func) == 0) {
msg_blaster (sock, args[0]);
} else if (strcmp ("context_test", func) == 0) {
context_test (sock);
} else if (strcmp ("are_you_ok_dude", func) == 0) {
snprintf (response, 100, "%s", OK_STR);
send (sock, response, strlen (response), 0);
} else if (strcmp ("cfg_shutdown", func) == 0) {
corosync_cfg_try_shutdown (cfg_handle, COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST);
} else if (strcmp ("cfg_initialize",func) == 0) {
int retry_count = 0;
syslog (LOG_INFO,"%s %s() called!", __func__, func);
result = corosync_cfg_initialize (&cfg_handle, &cfg_callbacks);
while (result != CS_OK) {
syslog (LOG_ERR,
"cfg_initialize error %d (attempt %d)\n",
result, retry_count);
if (retry_count >= 3) {
exit (1);
}
sleep(1);
retry_count++;
result = corosync_cfg_initialize (&cfg_handle, &cfg_callbacks);
}
corosync_cfg_fd_get (cfg_handle, &cfg_fd);
corosync_cfg_state_track (cfg_handle, 0, ¬ification_buffer);
poll_dispatch_add (ta_poll_handle_get(), cfg_fd, POLLIN|POLLNVAL, NULL, cfg_dispatch_wrapper_fn);
} else {
syslog (LOG_ERR,"%s RPC:%s not supported!", __func__, func);
}
}