static void cpg_benchmark (
cpg_handle_t handle_in,
int write_size)
{
struct timeval tv1, tv2, tv_elapsed;
struct iovec iov;
unsigned int res;
alarm_notice = 0;
iov.iov_base = data;
iov.iov_len = write_size;
write_count = 0;
alarm (10);
gettimeofday (&tv1, NULL);
do {
res = cpg_mcast_joined (handle_in, CPG_TYPE_AGREED, &iov, 1);
} while (alarm_notice == 0 && (res == CS_OK || res == CS_ERR_TRY_AGAIN));
gettimeofday (&tv2, NULL);
timersub (&tv2, &tv1, &tv_elapsed);
printf ("%5d messages received ", write_count);
printf ("%5d bytes per write ", write_size);
printf ("%7.3f Seconds runtime ",
(tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)));
printf ("%9.3f TP/s ",
((float)write_count) / (tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)));
printf ("%7.3f MB/s.\n",
((float)write_count) * ((float)write_size) / ((tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)) * 1000000.0));
}
static void
send_msg(cpg_handle_t cpg_handle)
{
struct test_msg msg;
unsigned char *data;
uint32_t u32;
struct iovec iov[2];
cs_error_t cs_res;
ENTER();
msg.data_len = rand_r(&sent_rand_seed) % MAX_MSG_LEN;
msg.seq_no = sent_msg_seq_no;
sent_msg_seq_no++;
data = malloc(sizeof(unsigned char) * msg.data_len);
assert(data != NULL);
for (u32 = 0; u32 < msg.data_len; u32++) {
data[u32] = (unsigned char)rand_r(&sent_rand_seed);
}
iov[0].iov_base = (void *)&msg;
iov[0].iov_len = sizeof(msg);
iov[1].iov_base = (void *)data;
iov[1].iov_len = msg.data_len;
cs_res = cpg_mcast_joined(cpg_handle, CPG_TYPE_AGREED, iov, 2);
assert(cs_res == CS_OK);
free(data);
LEAVE();
}
static void cpg_test (
cpg_handle_t handle_in,
int write_size,
int delay_time,
int print_time)
{
struct timeval tv1, tv2, tv_elapsed;
struct iovec iov;
unsigned int res;
int i;
unsigned int *dataint = (unsigned int *)data;
uLong crc;
alarm_notice = 0;
iov.iov_base = data;
iov.iov_len = write_size;
g_recv_count = 0;
alarm (print_time);
gettimeofday (&tv1, NULL);
do {
dataint[0] = send_counter++;
for (i=2; i<(DATASIZE-sizeof(int)*2)/4; i++) {
dataint[i] = rand();
}
crc = crc32(0, NULL, 0);
dataint[1] = crc32(crc, (Bytef*)&dataint[2], write_size-sizeof(int)*2);
resend:
res = cpg_mcast_joined (handle_in, CPG_TYPE_AGREED, &iov, 1);
if (res == CS_ERR_TRY_AGAIN) {
usleep(10000);
send_retries++;
goto resend;
}
if (res != CS_OK) {
fprintf(stderr, "send failed: %d\n", res);
send_fails++;
}
else {
packets_sent++;
}
usleep(delay_time*1000);
} while (alarm_notice == 0 && (res == CS_OK || res == CS_ERR_TRY_AGAIN) && stopped == 0);
gettimeofday (&tv2, NULL);
timersub (&tv2, &tv1, &tv_elapsed);
if (!quiet) {
printf ("%s: %5d message%s received, ", group_name.value, g_recv_count, g_recv_count==1?"":"s");
printf ("%5d bytes per write\n", write_size);
}
}
static void cpg_test (
cpg_handle_t handle_in,
int write_size,
int delay_time,
int print_time)
{
struct timeval tv1, tv2, tv_elapsed;
struct iovec iov;
unsigned int res;
alarm_notice = 0;
iov.iov_base = data;
iov.iov_len = write_size;
g_recv_count = 0;
alarm (print_time);
do {
send_counter++;
resend:
set_packet(write_size, send_counter);
res = cpg_mcast_joined (handle_in, CPG_TYPE_AGREED, &iov, 1);
if (res == CS_ERR_TRY_AGAIN) {
usleep(10000);
send_retries++;
goto resend;
}
if (res != CS_OK) {
cpgh_log_printf(CPGH_LOG_ERR, "send failed: %d\n", res);
send_fails++;
}
else {
packets_sent++;
}
usleep(delay_time*1000);
} while (alarm_notice == 0 && (res == CS_OK || res == CS_ERR_TRY_AGAIN) && stopped == 0);
gettimeofday (&tv2, NULL);
timersub (&tv2, &tv1, &tv_elapsed);
if (!quiet) {
if (machine_readable) {
cpgh_log_printf(CPGH_LOG_RTT, "%d%c%ld%c%ld%c%ld\n", 0, delimiter, min_rtt, delimiter, avg_rtt, delimiter, max_rtt);
}
else {
cpgh_log_printf(CPGH_LOG_PERF, "%s: %5d message%s received, ", group_name.value, g_recv_count, g_recv_count==1?"":"s");
cpgh_log_printf(CPGH_LOG_PERF, "%5d bytes per write. ", write_size);
cpgh_log_printf(CPGH_LOG_RTT, "RTT min/avg/max: %ld/%ld/%ld\n", min_rtt, avg_rtt, max_rtt);
}
}
}
int main (void)
{
cpg_handle_t handle;
cs_error_t result;
unsigned int i = 0;
struct iovec iov;
int res;
unsigned int msg_size;
result = cpg_initialize (&handle, &callbacks);
if (result != CS_OK) {
printf ("Couldn't initialize CPG service %d\n", result);
exit (0);
}
res = cpg_join (handle, &group_name);
if (res != CS_OK) {
printf ("cpg_join failed with result %d\n", res);
exit (1);
}
iov.iov_base = (void *)buffer;
/*
* Demonstrate cpg_mcast_joined
*/
msg_size = 1025000;
for (i = 0; i < 1000000000; i++) {
iov.iov_len = msg_size;
try_again_one:
result = cpg_mcast_joined (handle, CPG_TYPE_AGREED,
&iov, 1);
if (result == CS_ERR_TRY_AGAIN) {
goto try_again_one;
}
if (result == CS_ERR_INVALID_PARAM) {
printf ("found boundary at %d\n", msg_size);
exit (1);
}
msg_size += 1;
printf ("msg size %d\n", msg_size);
result = cpg_dispatch (handle, CS_DISPATCH_ALL);
}
cpg_finalize (handle);
return (0);
}
static void cpg_benchmark (
cpg_handle_t handle,
int write_size)
{
struct timeval tv1, tv2, tv_elapsed;
struct iovec iov;
unsigned int res;
cpg_flow_control_state_t flow_control_state;
alarm_notice = 0;
iov.iov_base = data;
iov.iov_len = write_size;
write_count = 0;
alarm (10);
gettimeofday (&tv1, NULL);
do {
/*
* Test checkpoint write
*/
cpg_flow_control_state_get (handle, &flow_control_state);
if (flow_control_state == CPG_FLOW_CONTROL_DISABLED) {
retry:
res = cpg_mcast_joined (handle, CPG_TYPE_AGREED, &iov, 1);
if (res == CS_ERR_TRY_AGAIN) {
goto retry;
}
}
res = cpg_dispatch (handle, CS_DISPATCH_ALL);
if (res != CS_OK) {
printf ("cpg dispatch returned error %d\n", res);
exit (1);
}
} while (alarm_notice == 0);
gettimeofday (&tv2, NULL);
timersub (&tv2, &tv1, &tv_elapsed);
printf ("%5d messages received ", write_count);
printf ("%5d bytes per write ", write_size);
printf ("%7.3f Seconds runtime ",
(tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)));
printf ("%9.3f TP/s ",
((float)write_count) / (tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)));
printf ("%7.3f MB/s.\n",
((float)write_count) * ((float)write_size) / ((tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)) * 1000000.0));
}
static void publish_event_to_corosync(struct ast_event *event)
{
cs_error_t cs_err;
struct iovec iov;
iov.iov_base = (void *)event;
iov.iov_len = ast_event_get_size(event);
ast_debug(5, "Publishing event %s (%u) to corosync\n",
ast_event_get_type_name(event), ast_event_get_type(event));
/* The stasis subscription will only exist if we are configured to publish
* these events, so just send away. */
if ((cs_err = cpg_mcast_joined(cpg_handle, CPG_TYPE_FIFO, &iov, 1)) != CS_OK) {
ast_log(LOG_WARNING, "CPG mcast failed (%u) for event %s (%u)\n",
cs_err, ast_event_get_type_name(event), ast_event_get_type(event));
}
}
gboolean
send_cpg_message(struct iovec * iov)
{
int rc = CS_OK;
int retries = 0;
errno = 0;
do {
rc = cpg_mcast_joined(cpg_handle, CPG_TYPE_AGREED, iov, 1);
if (rc == CS_ERR_TRY_AGAIN || rc == CS_ERR_QUEUE_FULL) {
cpg_flow_control_state_t fc_state = CPG_FLOW_CONTROL_DISABLED;
int rc2 = cpg_flow_control_state_get(cpg_handle, &fc_state);
if (rc2 == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) {
crm_debug("Attempting to clear cpg dispatch queue");
rc2 = cpg_dispatch(cpg_handle, CS_DISPATCH_ALL);
}
if (rc2 != CS_OK) {
crm_warn("Could not check/clear the cpg connection");
goto bail;
} else {
retries++;
crm_debug("Retrying operation after %ds", retries);
sleep(retries);
}
} else {
break;
}
/* 5 retires is plenty, we'll resend once the membership reforms anyway */
} while (retries < 5);
bail:
if (rc != CS_OK) {
crm_err("Sending message via cpg FAILED: (rc=%d) %s", rc, ais_error2text(rc));
}
return (rc == CS_OK);
}
cs_error_t CorosyncCpg::multicast(const iovec* iov, int iovLen) {
// Check for flow control
cpg_flow_control_state_t flowState;
cs_error_t result;
if (CS_OK != (result = cpg_flow_control_state_get(handle, &flowState))) {
LOG(ERROR)<< "Cannot get CPG flow control status.";
}
if (flowState == CPG_FLOW_CONTROL_ENABLED) {
return result;
}
do {
result = cpg_mcast_joined(handle, CPG_TYPE_AGREED, const_cast<iovec*>(iov), iovLen);
if (result != CS_ERR_TRY_AGAIN) {
if (CS_OK != result) {
LOG(ERROR)<< "Can't multicast messsage to group: " << group.str();
}
}
} while(result == CS_ERR_TRY_AGAIN);
return result;
}
static PyObject *
py_cpg_mcast_joined(PyObject *self, PyObject *args)
{
int ret;
void *data;
size_t length;
cpg_handle_t handle;
cpg_guarantee_t guarantee;
struct iovec iov;
if (!PyArg_ParseTuple(args, "lis#", &handle, &guarantee, &data, &length))
return NULL;
iov.iov_base = data;
iov.iov_len = length;
ret = cpg_mcast_joined(handle, guarantee, &iov, 1);
RETURN_ON_ERROR(ret, "cpg_mcast_joined");
Py_INCREF(Py_None);
return Py_None;
}
static void publish_to_corosync(struct stasis_message *message)
{
cs_error_t cs_err;
struct iovec iov;
struct ast_event *event;
event = stasis_message_to_event(message);
if (!event) {
return;
}
if (ast_eid_cmp(&ast_eid_default, ast_event_get_ie_raw(event, AST_EVENT_IE_EID))) {
/* If the event didn't originate from this server, don't send it back out. */
ast_event_destroy(event);
return;
}
if (ast_event_get_type(event) == AST_EVENT_PING) {
const struct ast_eid *eid;
char buf[128] = "";
eid = ast_event_get_ie_raw(event, AST_EVENT_IE_EID);
ast_eid_to_str(buf, sizeof(buf), (struct ast_eid *) eid);
ast_log(LOG_NOTICE, "Sending event PING from this server with EID: '%s'\n", buf);
}
iov.iov_base = (void *)event;
iov.iov_len = ast_event_get_size(event);
ast_debug(5, "Publishing event %s (%u) to corosync\n",
ast_event_get_type_name(event), ast_event_get_type(event));
/* The stasis subscription will only exist if we are configured to publish
* these events, so just send away. */
if ((cs_err = cpg_mcast_joined(cpg_handle, CPG_TYPE_FIFO, &iov, 1)) != CS_OK) {
ast_log(LOG_WARNING, "CPG mcast failed (%u)\n", cs_err);
}
}
int main (int argc, char *argv[])
{
cpg_handle_t handle;
cs_error_t result;
int i = 0;
int j;
struct my_msg msg;
struct iovec iov[2];
const char *options = "i:";
int iter = 1000;
int opt;
int run_forever = 1;
unsigned int sha1_len;
while ((opt = getopt(argc, argv, options)) != -1) {
switch (opt) {
case 'i':
run_forever = 0;
iter = atoi(optarg);
break;
}
}
result = cpg_initialize (&handle, &callbacks);
if (result != CS_OK) {
printf ("Couldn't initialize CPG service %d\n", result);
exit (0);
}
if (NSS_NoDB_Init(".") != SECSuccess) {
printf ("Couldn't initialize nss\n");
exit (0);
}
if ((sha1_context = PK11_CreateDigestContext(SEC_OID_SHA1)) == NULL) {
printf ("Couldn't initialize nss\n");
exit (0);
}
result = cpg_join (handle, &group_name);
if (result != CS_OK) {
printf ("cpg_join failed with result %d\n", result);
exit (1);
}
iov[0].iov_base = (void *)&msg;
iov[0].iov_len = sizeof (struct my_msg);
iov[1].iov_base = (void *)buffer;
/*
* Demonstrate cpg_mcast_joined
*/
i = 0;
do {
msg.msg_size = 100 + rand() % 100000;
iov[1].iov_len = msg.msg_size;
for (j = 0; j < msg.msg_size; j++) {
buffer[j] = j;
}
sprintf ((char *)buffer,
"cpg_mcast_joined: This is message %12d", i);
PK11_DigestBegin(sha1_context);
PK11_DigestOp(sha1_context, buffer, msg.msg_size);
PK11_DigestFinal(sha1_context, msg.sha1, &sha1_len, sizeof(msg.sha1));
try_again_one:
result = cpg_mcast_joined (handle, CPG_TYPE_AGREED,
iov, 2);
if (result == CS_ERR_TRY_AGAIN) {
goto try_again_one;
}
result = cpg_dispatch (handle, CS_DISPATCH_ALL);
i++;
} while (run_forever || i < iter);
PK11_DestroyContext(sha1_context, PR_TRUE);
cpg_finalize (handle);
return (0);
}
static void send_some_more_messages_normal (void)
{
msg_t my_msg;
struct iovec iov[2];
int i;
int send_now;
size_t payload_size;
hash_state sha1_hash;
cs_error_t res;
cpg_flow_control_state_t fc_state;
int retries = 0;
time_t before;
if (cpg_fd < 0)
return;
send_now = my_msgs_to_send;
//syslog (LOG_DEBUG,"%s() send_now:%d", __func__, send_now);
my_msg.pid = my_pid;
my_msg.nodeid = my_nodeid;
payload_size = (rand() % 100000);
my_msg.size = sizeof (msg_t) + payload_size;
my_msg.seq = 0;
for (i = 0; i < payload_size; i++) {
buffer[i] = i;
}
sha1_init (&sha1_hash);
sha1_process (&sha1_hash, buffer, payload_size);
sha1_done (&sha1_hash, my_msg.sha1);
iov[0].iov_len = sizeof (msg_t);
iov[0].iov_base = &my_msg;
iov[1].iov_len = payload_size;
iov[1].iov_base = buffer;
for (i = 0; i < send_now; i++) {
if (in_cnchg && pcmk_test) {
retries = 0;
before = time(NULL);
cs_repeat(retries, 30, res = cpg_mcast_joined(cpg_handle, CPG_TYPE_AGREED, iov, 2));
if (retries > 20) {
syslog (LOG_ERR, "%s() -> cs_repeat: blocked for :%lu secs.",
__func__, (unsigned long)(time(NULL) - before));
}
if (res != CS_OK) {
syslog (LOG_ERR, "%s() -> cpg_mcast_joined error:%d.",
__func__, res);
return;
}
} else {
res = cpg_flow_control_state_get (cpg_handle, &fc_state);
if (res == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) {
/* lets do this later */
send_some_more_messages_later ();
syslog (LOG_INFO, "%s() flow control enabled.", __func__);
return;
}
res = cpg_mcast_joined (cpg_handle, CPG_TYPE_AGREED, iov, 2);
if (res == CS_ERR_TRY_AGAIN) {
/* lets do this later */
send_some_more_messages_later ();
if (i > 0) {
syslog (LOG_INFO, "%s() TRY_AGAIN %d to send.",
__func__, my_msgs_to_send);
}
return;
} else if (res != CS_OK) {
syslog (LOG_ERR, "%s() -> cpg_mcast_joined error:%d, exiting.",
__func__, res);
exit (-2);
}
}
my_msgs_to_send--;
}
}
/* Basically this is cpgbench.c */
static void cpg_flood (
cpg_handle_t handle_in,
int write_size)
{
struct timeval tv1, tv2, tv_elapsed;
struct iovec iov;
unsigned int res = CS_OK;
alarm_notice = 0;
iov.iov_base = data;
iov.iov_len = write_size;
alarm (10);
packets_recvd1 = 0;
interim_avg_rtt = 0;
interim_max_rtt = 0;
interim_min_rtt = LONG_MAX;
gettimeofday (&tv1, NULL);
do {
if (res == CS_OK) {
set_packet(write_size, send_counter);
}
res = cpg_mcast_joined (handle_in, CPG_TYPE_AGREED, &iov, 1);
if (res == CS_OK) {
/* Only increment the packet counter if it was sucessfully sent */
packets_sent++;
send_counter++;
}
else {
if (res == CS_ERR_TRY_AGAIN) {
send_retries++;
}
else {
send_fails++;
}
}
} while (!stopped && alarm_notice == 0 && (res == CS_OK || res == CS_ERR_TRY_AGAIN));
gettimeofday (&tv2, NULL);
timersub (&tv2, &tv1, &tv_elapsed);
if (!quiet) {
if (machine_readable) {
cpgh_log_printf (CPGH_LOG_PERF, "%d%c%d%c%f%c%f%c%f%c%ld%c%ld%c%ld\n", packets_recvd1, delimiter, write_size, delimiter,
(tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)), delimiter,
((float)packets_recvd1) / (tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)), delimiter,
((float)packets_recvd1) * ((float)write_size) / ((tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)) * 1000000.0), delimiter,
interim_min_rtt, delimiter, interim_avg_rtt, delimiter, interim_max_rtt);
}
else {
cpgh_log_printf (CPGH_LOG_PERF, "%5d messages received ", packets_recvd1);
cpgh_log_printf (CPGH_LOG_PERF, "%5d bytes per write ", write_size);
cpgh_log_printf (CPGH_LOG_PERF, "%7.3f Seconds runtime ",
(tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)));
cpgh_log_printf (CPGH_LOG_PERF, "%9.3f TP/s ",
((float)packets_recvd1) / (tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)));
cpgh_log_printf (CPGH_LOG_PERF, "%7.3f MB/s ",
((float)packets_recvd1) * ((float)write_size) / ((tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)) * 1000000.0));
cpgh_log_printf (CPGH_LOG_PERF, "RTT for this size (min/avg/max) %ld/%ld/%ld\n",
interim_min_rtt, interim_avg_rtt, interim_max_rtt);
}
}
}
static ssize_t
crm_cs_flush(gpointer data)
{
int sent = 0;
ssize_t rc = 0;
int queue_len = 0;
static unsigned int last_sent = 0;
cpg_handle_t *handle = (cpg_handle_t *)data;
if (*handle == 0) {
crm_trace("Connection is dead");
return pcmk_ok;
}
queue_len = g_list_length(cs_message_queue);
if ((queue_len % 1000) == 0 && queue_len > 1) {
crm_err("CPG queue has grown to %d", queue_len);
} else if (queue_len == CS_SEND_MAX) {
crm_warn("CPG queue has grown to %d", queue_len);
}
if (cs_message_timer) {
/* There is already a timer, wait until it goes off */
crm_trace("Timer active %d", cs_message_timer);
return pcmk_ok;
}
while (cs_message_queue && sent < CS_SEND_MAX) {
struct iovec *iov = cs_message_queue->data;
errno = 0;
rc = cpg_mcast_joined(*handle, CPG_TYPE_AGREED, iov, 1);
if (rc != CS_OK) {
break;
}
sent++;
last_sent++;
crm_trace("CPG message sent, size=%llu",
(unsigned long long) iov->iov_len);
cs_message_queue = g_list_remove(cs_message_queue, iov);
free(iov->iov_base);
free(iov);
}
queue_len -= sent;
if (sent > 1 || cs_message_queue) {
crm_info("Sent %d CPG messages (%d remaining, last=%u): %s (%lld)",
sent, queue_len, last_sent, ais_error2text(rc),
(long long) rc);
} else {
crm_trace("Sent %d CPG messages (%d remaining, last=%u): %s (%lld)",
sent, queue_len, last_sent, ais_error2text(rc),
(long long) rc);
}
if (cs_message_queue) {
uint32_t delay_ms = 100;
if(rc != CS_OK) {
/* Proportionally more if sending failed but cap at 1s */
delay_ms = QB_MIN(1000, CS_SEND_MAX + (10 * queue_len));
}
cs_message_timer = g_timeout_add(delay_ms, crm_cs_flush_cb, data);
}
return rc;
}
static void do_command (int sock, char* func, char*args[], int num_args)
{
int result;
char response[100];
struct cpg_name group_name;
ssize_t rc;
size_t send_len;
qb_log (LOG_TRACE, "RPC:%s() called.", func);
if (strcmp ("cpg_mcast_joined",func) == 0) {
struct iovec iov[5];
int a;
for (a = 0; a < num_args; a++) {
iov[a].iov_base = args[a];
iov[a].iov_len = strlen(args[a])+1;
}
cpg_mcast_joined (cpg_handle, CPG_TYPE_AGREED, iov, num_args);
} else if (strcmp ("cpg_join",func) == 0) {
if (strlen(args[0]) >= CPG_MAX_NAME_LENGTH) {
qb_log (LOG_ERR, "Invalid group name");
exit (1);
}
strcpy (group_name.value, args[0]);
group_name.length = strlen(args[0]);
result = cpg_join (cpg_handle, &group_name);
if (result != CS_OK) {
qb_log (LOG_ERR,
"Could not join process group, error %d", result);
exit (1);
}
qb_log (LOG_INFO, "called cpg_join(%s)!", group_name.value);
} else if (strcmp ("cpg_leave",func) == 0) {
strcpy (group_name.value, args[0]);
group_name.length = strlen(args[0]);
result = cpg_leave (cpg_handle, &group_name);
if (result != CS_OK) {
qb_log (LOG_ERR,
"Could not leave process group, error %d", result);
exit (1);
}
qb_log (LOG_INFO, "called cpg_leave(%s)!", group_name.value);
} else if (strcmp ("cpg_initialize",func) == 0) {
int retry_count = 0;
result = cpg_initialize (&cpg_handle, &callbacks);
while (result != CS_OK) {
qb_log (LOG_ERR,
"cpg_initialize error %d (attempt %d)",
result, retry_count);
if (retry_count >= 3) {
exit (1);
}
sleep(1);
retry_count++;
result = cpg_initialize (&cpg_handle, &callbacks);
}
cpg_fd_get (cpg_handle, &cpg_fd);
qb_loop_poll_add (ta_poll_handle_get(),
QB_LOOP_MED,
cpg_fd,
POLLIN|POLLNVAL,
NULL,
cpg_dispatch_wrapper_fn);
} else if (strcmp ("cpg_local_get", func) == 0) {
unsigned int local_nodeid;
cpg_local_get (cpg_handle, &local_nodeid);
snprintf (response, 100, "%u",local_nodeid);
send_len = strlen (response);
rc = send (sock, response, send_len, 0);
assert(rc == send_len);
} else if (strcmp ("cpg_finalize", func) == 0) {
if (cpg_handle > 0) {
cpg_finalize (cpg_handle);
cpg_handle = 0;
}
} else if (strcmp ("record_config_events", func) == 0) {
record_config_events (sock);
} else if (strcmp ("record_messages", func) == 0) {
record_messages ();
} else if (strcmp ("read_config_event", func) == 0) {
read_config_event (sock);
} else if (strcmp ("read_messages", func) == 0) {
read_messages (sock, args[0]);
} else if (strcmp ("msg_blaster_zcb", func) == 0) {
msg_blaster_zcb (sock, args[0]);
} else if (strcmp ("pcmk_test", func) == 0) {
pcmk_test = 1;
} else if (strcmp ("msg_blaster", func) == 0) {
msg_blaster (sock, args[0]);
} else if (strcmp ("context_test", func) == 0) {
context_test (sock);
} else if (strcmp ("are_you_ok_dude", func) == 0) {
snprintf (response, 100, "%s", OK_STR);
send_len = strlen (response);
rc = send (sock, response, strlen (response), 0);
assert(rc == send_len);
} else if (strcmp ("cfg_shutdown", func) == 0) {
qb_log (LOG_INFO, "calling %s() called!", func);
result = corosync_cfg_try_shutdown (cfg_handle, COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST);
qb_log (LOG_INFO,"%s() returned %d!", func, result);
} else if (strcmp ("cfg_initialize",func) == 0) {
int retry_count = 0;
qb_log (LOG_INFO,"%s() called!", func);
result = corosync_cfg_initialize (&cfg_handle, &cfg_callbacks);
while (result != CS_OK) {
qb_log (LOG_ERR,
"cfg_initialize error %d (attempt %d)",
result, retry_count);
if (retry_count >= 3) {
exit (1);
}
sleep(1);
retry_count++;
result = corosync_cfg_initialize (&cfg_handle, &cfg_callbacks);
}
qb_log (LOG_INFO,"corosync_cfg_initialize() == %d", result);
result = corosync_cfg_fd_get (cfg_handle, &cfg_fd);
qb_log (LOG_INFO,"corosync_cfg_fd_get() == %d", result);
qb_loop_poll_add (ta_poll_handle_get(),
QB_LOOP_MED,
cfg_fd,
POLLIN|POLLNVAL,
NULL,
cfg_dispatch_wrapper_fn);
} else {
qb_log(LOG_ERR, "RPC:%s not supported!", func);
}
}
static void send_some_more_messages_normal (void)
{
msg_t my_msg;
struct iovec iov[2];
int i;
int send_now;
size_t payload_size;
cs_error_t res;
cpg_flow_control_state_t fc_state;
int retries = 0;
time_t before;
unsigned int sha1_len;
if (cpg_fd < 0)
return;
send_now = my_msgs_to_send;
qb_log (LOG_TRACE, "send_now:%d", send_now);
my_msg.pid = my_pid;
my_msg.nodeid = my_nodeid;
payload_size = (rand() % 10000);
my_msg.size = sizeof (msg_t) + payload_size;
my_msg.seq = my_msgs_sent;
for (i = 0; i < payload_size; i++) {
buffer[i] = i;
}
PK11_DigestBegin(sha1_context);
PK11_DigestOp(sha1_context, buffer, payload_size);
PK11_DigestFinal(sha1_context, my_msg.sha1, &sha1_len, sizeof(my_msg.sha1));
iov[0].iov_len = sizeof (msg_t);
iov[0].iov_base = &my_msg;
iov[1].iov_len = payload_size;
iov[1].iov_base = buffer;
for (i = 0; i < send_now; i++) {
if (in_cnchg && pcmk_test) {
retries = 0;
before = time(NULL);
cs_repeat(retries, 30, res = cpg_mcast_joined(cpg_handle, CPG_TYPE_AGREED, iov, 2));
if (retries > 20) {
qb_log (LOG_ERR, "cs_repeat: blocked for :%lu secs.",
(unsigned long)(time(NULL) - before));
}
if (res != CS_OK) {
qb_log (LOG_ERR, "cpg_mcast_joined error:%d.",
res);
return;
}
} else {
res = cpg_flow_control_state_get (cpg_handle, &fc_state);
if (res == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) {
/* lets do this later */
send_some_more_messages_later ();
qb_log (LOG_INFO, "flow control enabled.");
return;
}
res = cpg_mcast_joined (cpg_handle, CPG_TYPE_AGREED, iov, 2);
if (res == CS_ERR_TRY_AGAIN) {
/* lets do this later */
send_some_more_messages_later ();
if (i > 0) {
qb_log (LOG_INFO, "TRY_AGAIN %d to send.",
my_msgs_to_send);
}
return;
} else if (res != CS_OK) {
qb_log (LOG_ERR, "cpg_mcast_joined error:%d, exiting.",
res);
exit (-2);
}
}
my_msgs_sent++;
my_msg.seq = my_msgs_sent;
my_msgs_to_send--;
}
qb_log (LOG_TRACE, "sent %d; to send %d.",
my_msgs_sent, my_msgs_to_send);
}
int main (int argc, char *argv[]) {
cpg_handle_t handle;
fd_set read_fds;
int select_fd;
int result;
const char *options = "i";
int opt;
unsigned int nodeid;
char *fgets_res;
while ( (opt = getopt(argc, argv, options)) != -1 ) {
switch (opt) {
case 'i':
show_ip = 1;
break;
}
}
if (argc > optind) {
strcpy(group_name.value, argv[optind]);
group_name.length = strlen(argv[optind]);
}
else {
strcpy(group_name.value, "GROUP");
group_name.length = 6;
}
result = cpg_initialize (&handle, &callbacks);
if (result != CS_OK) {
printf ("Could not initialize Cluster Process Group API instance error %d\n", result);
exit (1);
}
result = cpg_local_get (handle, &nodeid);
if (result != CS_OK) {
printf ("Could not get local node id\n");
exit (1);
}
printf ("Local node id is %x\n", nodeid);
result = cpg_join(handle, &group_name);
if (result != CS_OK) {
printf ("Could not join process group, error %d\n", result);
exit (1);
}
FD_ZERO (&read_fds);
cpg_fd_get(handle, &select_fd);
printf ("Type EXIT to finish\n");
do {
FD_SET (select_fd, &read_fds);
FD_SET (STDIN_FILENO, &read_fds);
result = select (select_fd + 1, &read_fds, 0, 0, 0);
if (result == -1) {
perror ("select\n");
}
if (FD_ISSET (STDIN_FILENO, &read_fds)) {
char inbuf[132];
struct iovec iov;
fgets_res = fgets(inbuf, sizeof(inbuf), stdin);
if (fgets_res == NULL) {
cpg_leave(handle, &group_name);
}
if (strncmp(inbuf, "EXIT", 4) == 0) {
cpg_leave(handle, &group_name);
}
else {
iov.iov_base = inbuf;
iov.iov_len = strlen(inbuf)+1;
cpg_mcast_joined(handle, CPG_TYPE_AGREED, &iov, 1);
}
}
if (FD_ISSET (select_fd, &read_fds)) {
if (cpg_dispatch (handle, CS_DISPATCH_ALL) != CS_OK)
exit(1);
}
} while (result && !quit);
result = cpg_finalize (handle);
printf ("Finalize result is %d (should be 1)\n", result);
return (0);
}
int main (int argc, char *argv[]) {
cpg_handle_t handle;
fd_set read_fds;
int select_fd;
int result;
int retries;
const char *options = "i";
int opt;
unsigned int nodeid;
char *fgets_res;
struct cpg_address member_list[64];
int member_list_entries;
int i;
int recnt;
int doexit;
const char *exitStr = "EXIT";
doexit = 0;
#ifdef QBLOG
qb_log_init("testcpg", LOG_USER, LOG_ERR);
qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_ENABLED, QB_FALSE);
qb_log_filter_ctl(QB_LOG_STDERR, QB_LOG_FILTER_ADD,
QB_LOG_FILTER_FILE, "*", LOG_TRACE);
qb_log_ctl(QB_LOG_STDERR, QB_LOG_CONF_ENABLED, QB_TRUE);
qb_log_format_set(QB_LOG_STDERR, "[%p] %f %b");
#endif
while ( (opt = getopt(argc, argv, options)) != -1 ) {
switch (opt) {
case 'i':
show_ip = 1;
break;
}
}
if (argc > optind) {
strcpy(group_name.value, argv[optind]);
group_name.length = strlen(argv[optind]);
}
else {
strcpy(group_name.value, "GROUP");
group_name.length = 6;
}
recnt = 0;
printf ("Type %s to finish\n", exitStr);
restart = 1;
do {
if(restart) {
restart = 0;
retries = 0;
cs_repeat_init(retries, 30, result = cpg_model_initialize (&handle, CPG_MODEL_V1, (cpg_model_data_t *)&model_data, NULL));
if (result != CS_OK) {
printf ("Could not initialize Cluster Process Group API instance error %d\n", result);
retrybackoff(recnt);
}
retries = 0;
cs_repeat(retries, 30, result = cpg_local_get(handle, &nodeid));
if (result != CS_OK) {
printf ("Could not get local node id\n");
retrybackoff(recnt);
}
printf ("Local node id is %x\n", nodeid);
nodeidStart = nodeid;
retries = 0;
cs_repeat(retries, 30, result = cpg_join(handle, &group_name));
if (result != CS_OK) {
printf ("Could not join process group, error %d\n", result);
retrybackoff(recnt);
}
retries = 0;
cs_repeat(retries, 30, result = cpg_membership_get (handle, &group_name,
(struct cpg_address *)&member_list, &member_list_entries));
if (result != CS_OK) {
printf ("Could not get current membership list %d\n", result);
retrybackoff(recnt);
}
recnt = 0;
printf ("membership list\n");
for (i = 0; i < member_list_entries; i++) {
printf ("node id %d pid %d\n", member_list[i].nodeid,
member_list[i].pid);
}
FD_ZERO (&read_fds);
cpg_fd_get(handle, &select_fd);
}
FD_SET (select_fd, &read_fds);
FD_SET (STDIN_FILENO, &read_fds);
result = select (select_fd + 1, &read_fds, 0, 0, 0);
if (result == -1) {
perror ("select\n");
}
if (FD_ISSET (STDIN_FILENO, &read_fds)) {
char inbuf[132];
struct iovec iov;
fgets_res = fgets(inbuf, (int)sizeof(inbuf), stdin);
if (fgets_res == NULL) {
doexit = 1;
cpg_leave(handle, &group_name);
}
if (strncmp(inbuf, exitStr, strlen(exitStr)) == 0) {
doexit = 1;
cpg_leave(handle, &group_name);
}
else {
iov.iov_base = inbuf;
iov.iov_len = strlen(inbuf)+1;
cpg_mcast_joined(handle, CPG_TYPE_AGREED, &iov, 1);
}
}
if (FD_ISSET (select_fd, &read_fds)) {
if (cpg_dispatch (handle, CS_DISPATCH_ALL) != CS_OK) {
if(doexit) {
exit(1);
}
restart = 1;
}
}
if(restart) {
if(!doexit) {
result = cpg_finalize (handle);
printf ("Finalize+restart result is %d (should be 1)\n", result);
continue;
}
}
} while (result && !quit && !doexit);
result = cpg_finalize (handle);
printf ("Finalize result is %d (should be 1)\n", result);
return (0);
}
static void do_command (int sock, char* func, char*args[], int num_args)
{
int result;
char response[100];
struct cpg_name group_name;
if (parse_debug)
syslog (LOG_DEBUG,"RPC:%s() called.", func);
if (strcmp ("cpg_mcast_joined",func) == 0) {
struct iovec iov[5];
int a;
for (a = 0; a < num_args; a++) {
iov[a].iov_base = args[a];
iov[a].iov_len = strlen(args[a])+1;
}
cpg_mcast_joined (cpg_handle, CPG_TYPE_AGREED, iov, num_args);
} else if (strcmp ("cpg_join",func) == 0) {
strcpy (group_name.value, args[0]);
group_name.length = strlen(args[0]);
result = cpg_join (cpg_handle, &group_name);
if (result != CS_OK) {
syslog (LOG_ERR,
"Could not join process group, error %d\n", result);
exit (1);
}
syslog (LOG_INFO, "called cpg_join()!");
} else if (strcmp ("cpg_leave",func) == 0) {
strcpy (group_name.value, args[0]);
group_name.length = strlen(args[0]);
result = cpg_leave (cpg_handle, &group_name);
if (result != CS_OK) {
syslog (LOG_ERR,
"Could not leave process group, error %d\n", result);
exit (1);
}
syslog (LOG_INFO, "called cpg_leave()!");
} else if (strcmp ("cpg_initialize",func) == 0) {
int retry_count = 0;
result = cpg_initialize (&cpg_handle, &callbacks);
while (result != CS_OK) {
syslog (LOG_ERR,
"cpg_initialize error %d (attempt %d)\n",
result, retry_count);
if (retry_count >= 3) {
exit (1);
}
sleep(1);
retry_count++;
result = cpg_initialize (&cpg_handle, &callbacks);
}
cpg_fd_get (cpg_handle, &cpg_fd);
poll_dispatch_add (ta_poll_handle_get(), cpg_fd, POLLIN|POLLNVAL, NULL, cpg_dispatch_wrapper_fn);
} else if (strcmp ("cpg_local_get", func) == 0) {
unsigned int local_nodeid;
cpg_local_get (cpg_handle, &local_nodeid);
snprintf (response, 100, "%u",local_nodeid);
send (sock, response, strlen (response), 0);
} else if (strcmp ("cpg_finalize", func) == 0) {
cpg_finalize (cpg_handle);
poll_dispatch_delete (ta_poll_handle_get(), cpg_fd);
cpg_fd = -1;
} else if (strcmp ("record_config_events", func) == 0) {
record_config_events (sock);
} else if (strcmp ("record_messages", func) == 0) {
record_messages ();
} else if (strcmp ("read_config_event", func) == 0) {
read_config_event (sock);
} else if (strcmp ("read_messages", func) == 0) {
read_messages (sock, args[0]);
} else if (strcmp ("msg_blaster_zcb", func) == 0) {
msg_blaster_zcb (sock, args[0]);
} else if (strcmp ("pcmk_test", func) == 0) {
pcmk_test = 1;
} else if (strcmp ("msg_blaster", func) == 0) {
msg_blaster (sock, args[0]);
} else if (strcmp ("context_test", func) == 0) {
context_test (sock);
} else if (strcmp ("are_you_ok_dude", func) == 0) {
snprintf (response, 100, "%s", OK_STR);
send (sock, response, strlen (response), 0);
} else if (strcmp ("cfg_shutdown", func) == 0) {
corosync_cfg_try_shutdown (cfg_handle, COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST);
} else if (strcmp ("cfg_initialize",func) == 0) {
int retry_count = 0;
syslog (LOG_INFO,"%s %s() called!", __func__, func);
result = corosync_cfg_initialize (&cfg_handle, &cfg_callbacks);
while (result != CS_OK) {
syslog (LOG_ERR,
"cfg_initialize error %d (attempt %d)\n",
result, retry_count);
if (retry_count >= 3) {
exit (1);
}
sleep(1);
retry_count++;
result = corosync_cfg_initialize (&cfg_handle, &cfg_callbacks);
}
corosync_cfg_fd_get (cfg_handle, &cfg_fd);
corosync_cfg_state_track (cfg_handle, 0, ¬ification_buffer);
poll_dispatch_add (ta_poll_handle_get(), cfg_fd, POLLIN|POLLNVAL, NULL, cfg_dispatch_wrapper_fn);
} else {
syslog (LOG_ERR,"%s RPC:%s not supported!", __func__, func);
}
}
