static void cpg_benchmark (
cpg_handle_t handle,
int write_size)
{
struct timeval tv1, tv2, tv_elapsed;
struct iovec iov;
unsigned int res;
cpg_flow_control_state_t flow_control_state;
alarm_notice = 0;
iov.iov_base = data;
iov.iov_len = write_size;
write_count = 0;
alarm (10);
gettimeofday (&tv1, NULL);
do {
/*
* Test checkpoint write
*/
cpg_flow_control_state_get (handle, &flow_control_state);
if (flow_control_state == CPG_FLOW_CONTROL_DISABLED) {
retry:
res = cpg_mcast_joined (handle, CPG_TYPE_AGREED, &iov, 1);
if (res == CS_ERR_TRY_AGAIN) {
goto retry;
}
}
res = cpg_dispatch (handle, CS_DISPATCH_ALL);
if (res != CS_OK) {
printf ("cpg dispatch returned error %d\n", res);
exit (1);
}
} while (alarm_notice == 0);
gettimeofday (&tv2, NULL);
timersub (&tv2, &tv1, &tv_elapsed);
printf ("%5d messages received ", write_count);
printf ("%5d bytes per write ", write_size);
printf ("%7.3f Seconds runtime ",
(tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)));
printf ("%9.3f TP/s ",
((float)write_count) / (tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)));
printf ("%7.3f MB/s.\n",
((float)write_count) * ((float)write_size) / ((tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)) * 1000000.0));
}
gboolean
send_cpg_message(struct iovec * iov)
{
int rc = CS_OK;
int retries = 0;
errno = 0;
do {
rc = cpg_mcast_joined(cpg_handle, CPG_TYPE_AGREED, iov, 1);
if (rc == CS_ERR_TRY_AGAIN || rc == CS_ERR_QUEUE_FULL) {
cpg_flow_control_state_t fc_state = CPG_FLOW_CONTROL_DISABLED;
int rc2 = cpg_flow_control_state_get(cpg_handle, &fc_state);
if (rc2 == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) {
crm_debug("Attempting to clear cpg dispatch queue");
rc2 = cpg_dispatch(cpg_handle, CS_DISPATCH_ALL);
}
if (rc2 != CS_OK) {
crm_warn("Could not check/clear the cpg connection");
goto bail;
} else {
retries++;
crm_debug("Retrying operation after %ds", retries);
sleep(retries);
}
} else {
break;
}
/* 5 retires is plenty, we'll resend once the membership reforms anyway */
} while (retries < 5);
bail:
if (rc != CS_OK) {
crm_err("Sending message via cpg FAILED: (rc=%d) %s", rc, ais_error2text(rc));
}
return (rc == CS_OK);
}
cs_error_t CorosyncCpg::multicast(const iovec* iov, int iovLen) {
// Check for flow control
cpg_flow_control_state_t flowState;
cs_error_t result;
if (CS_OK != (result = cpg_flow_control_state_get(handle, &flowState))) {
LOG(ERROR)<< "Cannot get CPG flow control status.";
}
if (flowState == CPG_FLOW_CONTROL_ENABLED) {
return result;
}
do {
result = cpg_mcast_joined(handle, CPG_TYPE_AGREED, const_cast<iovec*>(iov), iovLen);
if (result != CS_ERR_TRY_AGAIN) {
if (CS_OK != result) {
LOG(ERROR)<< "Can't multicast messsage to group: " << group.str();
}
}
} while(result == CS_ERR_TRY_AGAIN);
return result;
}
static void send_some_more_messages_normal (void)
{
msg_t my_msg;
struct iovec iov[2];
int i;
int send_now;
size_t payload_size;
cs_error_t res;
cpg_flow_control_state_t fc_state;
int retries = 0;
time_t before;
unsigned int sha1_len;
if (cpg_fd < 0)
return;
send_now = my_msgs_to_send;
qb_log (LOG_TRACE, "send_now:%d", send_now);
my_msg.pid = my_pid;
my_msg.nodeid = my_nodeid;
payload_size = (rand() % 10000);
my_msg.size = sizeof (msg_t) + payload_size;
my_msg.seq = my_msgs_sent;
for (i = 0; i < payload_size; i++) {
buffer[i] = i;
}
PK11_DigestBegin(sha1_context);
PK11_DigestOp(sha1_context, buffer, payload_size);
PK11_DigestFinal(sha1_context, my_msg.sha1, &sha1_len, sizeof(my_msg.sha1));
iov[0].iov_len = sizeof (msg_t);
iov[0].iov_base = &my_msg;
iov[1].iov_len = payload_size;
iov[1].iov_base = buffer;
for (i = 0; i < send_now; i++) {
if (in_cnchg && pcmk_test) {
retries = 0;
before = time(NULL);
cs_repeat(retries, 30, res = cpg_mcast_joined(cpg_handle, CPG_TYPE_AGREED, iov, 2));
if (retries > 20) {
qb_log (LOG_ERR, "cs_repeat: blocked for :%lu secs.",
(unsigned long)(time(NULL) - before));
}
if (res != CS_OK) {
qb_log (LOG_ERR, "cpg_mcast_joined error:%d.",
res);
return;
}
} else {
res = cpg_flow_control_state_get (cpg_handle, &fc_state);
if (res == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) {
/* lets do this later */
send_some_more_messages_later ();
qb_log (LOG_INFO, "flow control enabled.");
return;
}
res = cpg_mcast_joined (cpg_handle, CPG_TYPE_AGREED, iov, 2);
if (res == CS_ERR_TRY_AGAIN) {
/* lets do this later */
send_some_more_messages_later ();
if (i > 0) {
qb_log (LOG_INFO, "TRY_AGAIN %d to send.",
my_msgs_to_send);
}
return;
} else if (res != CS_OK) {
qb_log (LOG_ERR, "cpg_mcast_joined error:%d, exiting.",
res);
exit (-2);
}
}
my_msgs_sent++;
my_msg.seq = my_msgs_sent;
my_msgs_to_send--;
}
qb_log (LOG_TRACE, "sent %d; to send %d.",
my_msgs_sent, my_msgs_to_send);
}
static void send_some_more_messages_zcb (void)
{
msg_t *my_msg;
int i;
int send_now;
size_t payload_size;
size_t total_size;
unsigned int sha1_len;
cs_error_t res;
cpg_flow_control_state_t fc_state;
void *zcb_buffer;
if (cpg_fd < 0)
return;
send_now = my_msgs_to_send;
payload_size = (rand() % 100000);
total_size = payload_size + sizeof (msg_t);
cpg_zcb_alloc (cpg_handle, total_size, &zcb_buffer);
my_msg = (msg_t*)zcb_buffer;
qb_log(LOG_DEBUG, "send_now:%d", send_now);
my_msg->pid = my_pid;
my_msg->nodeid = my_nodeid;
my_msg->size = sizeof (msg_t) + payload_size;
my_msg->seq = my_msgs_sent;
for (i = 0; i < payload_size; i++) {
my_msg->payload[i] = i;
}
PK11_DigestBegin(sha1_context);
PK11_DigestOp(sha1_context, my_msg->payload, payload_size);
PK11_DigestFinal(sha1_context, my_msg->sha1, &sha1_len, sizeof(my_msg->sha1));
for (i = 0; i < send_now; i++) {
res = cpg_flow_control_state_get (cpg_handle, &fc_state);
if (res == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) {
/* lets do this later */
send_some_more_messages_later ();
qb_log (LOG_INFO, "flow control enabled.");
goto free_buffer;
}
res = cpg_zcb_mcast_joined (cpg_handle, CPG_TYPE_AGREED, zcb_buffer, total_size);
if (res == CS_ERR_TRY_AGAIN) {
/* lets do this later */
send_some_more_messages_later ();
goto free_buffer;
} else if (res != CS_OK) {
qb_log (LOG_ERR, "cpg_mcast_joined error:%d, exiting.",
res);
exit (-2);
}
my_msgs_sent++;
my_msgs_to_send--;
}
free_buffer:
cpg_zcb_free (cpg_handle, zcb_buffer);
}
static void send_some_more_messages_normal (void)
{
msg_t my_msg;
struct iovec iov[2];
int i;
int send_now;
size_t payload_size;
hash_state sha1_hash;
cs_error_t res;
cpg_flow_control_state_t fc_state;
int retries = 0;
time_t before;
if (cpg_fd < 0)
return;
send_now = my_msgs_to_send;
//syslog (LOG_DEBUG,"%s() send_now:%d", __func__, send_now);
my_msg.pid = my_pid;
my_msg.nodeid = my_nodeid;
payload_size = (rand() % 100000);
my_msg.size = sizeof (msg_t) + payload_size;
my_msg.seq = 0;
for (i = 0; i < payload_size; i++) {
buffer[i] = i;
}
sha1_init (&sha1_hash);
sha1_process (&sha1_hash, buffer, payload_size);
sha1_done (&sha1_hash, my_msg.sha1);
iov[0].iov_len = sizeof (msg_t);
iov[0].iov_base = &my_msg;
iov[1].iov_len = payload_size;
iov[1].iov_base = buffer;
for (i = 0; i < send_now; i++) {
if (in_cnchg && pcmk_test) {
retries = 0;
before = time(NULL);
cs_repeat(retries, 30, res = cpg_mcast_joined(cpg_handle, CPG_TYPE_AGREED, iov, 2));
if (retries > 20) {
syslog (LOG_ERR, "%s() -> cs_repeat: blocked for :%lu secs.",
__func__, (unsigned long)(time(NULL) - before));
}
if (res != CS_OK) {
syslog (LOG_ERR, "%s() -> cpg_mcast_joined error:%d.",
__func__, res);
return;
}
} else {
res = cpg_flow_control_state_get (cpg_handle, &fc_state);
if (res == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) {
/* lets do this later */
send_some_more_messages_later ();
syslog (LOG_INFO, "%s() flow control enabled.", __func__);
return;
}
res = cpg_mcast_joined (cpg_handle, CPG_TYPE_AGREED, iov, 2);
if (res == CS_ERR_TRY_AGAIN) {
/* lets do this later */
send_some_more_messages_later ();
if (i > 0) {
syslog (LOG_INFO, "%s() TRY_AGAIN %d to send.",
__func__, my_msgs_to_send);
}
return;
} else if (res != CS_OK) {
syslog (LOG_ERR, "%s() -> cpg_mcast_joined error:%d, exiting.",
__func__, res);
exit (-2);
}
}
my_msgs_to_send--;
}
}
static void send_some_more_messages_zcb (void)
{
msg_t *my_msg;
int i;
int send_now;
size_t payload_size;
size_t total_size;
hash_state sha1_hash;
cs_error_t res;
cpg_flow_control_state_t fc_state;
void *zcb_buffer;
if (cpg_fd < 0)
return;
send_now = my_msgs_to_send;
payload_size = (rand() % 100000);
total_size = payload_size + sizeof (msg_t);
cpg_zcb_alloc (cpg_handle, total_size, &zcb_buffer);
my_msg = (msg_t*)zcb_buffer;
//syslog (LOG_DEBUG,"%s() send_now:%d", __func__, send_now);
my_msg->pid = my_pid;
my_msg->nodeid = my_nodeid;
my_msg->size = sizeof (msg_t) + payload_size;
my_msg->seq = 0;
for (i = 0; i < payload_size; i++) {
my_msg->payload[i] = i;
}
sha1_init (&sha1_hash);
sha1_process (&sha1_hash, my_msg->payload, payload_size);
sha1_done (&sha1_hash, my_msg->sha1);
for (i = 0; i < send_now; i++) {
res = cpg_flow_control_state_get (cpg_handle, &fc_state);
if (res == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) {
/* lets do this later */
send_some_more_messages_later ();
syslog (LOG_INFO, "%s() flow control enabled.", __func__);
goto free_buffer;
}
res = cpg_zcb_mcast_joined (cpg_handle, CPG_TYPE_AGREED, zcb_buffer, total_size);
if (res == CS_ERR_TRY_AGAIN) {
/* lets do this later */
send_some_more_messages_later ();
// if (i > 0) {
// syslog (LOG_INFO, "%s() TRY_AGAIN %d to send.",
// __func__, my_msgs_to_send);
// }
goto free_buffer;
} else if (res != CS_OK) {
syslog (LOG_ERR, "%s() -> cpg_mcast_joined error:%d, exiting.",
__func__, res);
exit (-2);
}
my_msgs_to_send--;
}
free_buffer:
cpg_zcb_free (cpg_handle, zcb_buffer);
}
