ssize_t
crm_ipcs_send(qb_ipcs_connection_t *c, xmlNode *message, enum ipcs_send_flags flags)
{
int rc;
int lpc = 0;
struct iovec iov[2];
static uint32_t id = 0;
const char *type = "Response";
struct qb_ipc_response_header header;
char *buffer = dump_xml_unformatted(message);
iov[0].iov_len = sizeof(struct qb_ipc_response_header);
iov[0].iov_base = &header;
iov[1].iov_len = 1 + strlen(buffer);
iov[1].iov_base = buffer;
header.id = id++; /* We don't really use it, but doesn't hurt to set one */
header.error = 0; /* unused */
header.size = iov[0].iov_len + iov[1].iov_len;
do {
if(flags & ipcs_send_event) {
rc = qb_ipcs_event_sendv(c, iov, 2);
type = "Event";
} else {
rc = qb_ipcs_response_sendv(c, iov, 2);
}
if(rc != -EAGAIN) {
break;
} else if(lpc > 3 && (flags & ipcs_send_error)) {
break;
}
crm_debug("Attempting resend %d of %s %d (%d bytes) to %p[%d]: %.120s",
++lpc, type, header.id, header.size, c, crm_ipcs_client_pid(c), buffer);
sleep(1);
/* Only retry for important stuff, and even then only a limited amount for ipcs_send_error
* Unless ipcs_send_info or ipcs_send_error is specified, we block by default
*/
} while((flags & ipcs_send_info) == 0);
if(rc < header.size) {
do_crm_log((flags & ipcs_send_error)?LOG_ERR:LOG_INFO,
"%s %d failed, size=%d, to=%p[%d], rc=%d: %.120s",
type, header.id, header.size, c, crm_ipcs_client_pid(c), rc, buffer);
} else {
crm_trace("%s %d sent, %d bytes to %p: %.120s", type, header.id, rc, c, buffer);
}
free(buffer);
return rc;
}
static void
convert_xml_child(HA_Message * msg, xmlNode * xml)
{
int orig = 0;
int rc = BZ_OK;
unsigned int len = 0;
char *buffer = NULL;
char *compressed = NULL;
const char *name = NULL;
name = (const char *)xml->name;
buffer = dump_xml_unformatted(xml);
orig = strlen(buffer);
if (orig < CRM_BZ2_THRESHOLD) {
ha_msg_add(msg, name, buffer);
goto done;
}
len = (orig * 1.1) + 600; /* recomended size */
compressed = malloc(len);
rc = BZ2_bzBuffToBuffCompress(compressed, &len, buffer, orig, CRM_BZ2_BLOCKS, 0, CRM_BZ2_WORK);
if (rc != BZ_OK) {
crm_err("Compression failed: %d", rc);
free(compressed);
convert_xml_message_struct(msg, xml, name);
goto done;
}
free(buffer);
buffer = compressed;
crm_trace("Compression details: %d -> %d", orig, len);
ha_msg_addbin(msg, name, buffer, len);
done:
free(buffer);
# if 0
{
unsigned int used = orig;
char *uncompressed = NULL;
crm_debug("Trying to decompress %d bytes", len);
uncompressed = calloc(1, orig);
rc = BZ2_bzBuffToBuffDecompress(uncompressed, &used, compressed, len, 1, 0);
CRM_CHECK(rc == BZ_OK,;
);
CRM_CHECK(used == orig,;
);
int
crm_ipc_send(crm_ipc_t *client, xmlNode *message, xmlNode **reply, int32_t ms_timeout)
{
long rc = 0;
struct iovec iov[2];
static uint32_t id = 0;
struct qb_ipc_request_header header;
char *buffer = dump_xml_unformatted(message);
iov[0].iov_len = sizeof(struct qb_ipc_request_header);
iov[0].iov_base = &header;
iov[1].iov_len = 1 + strlen(buffer);
iov[1].iov_base = buffer;
header.id = id++; /* We don't really use it, but doesn't hurt to set one */
header.size = iov[0].iov_len + iov[1].iov_len;
if(ms_timeout == 0) {
ms_timeout = 5000;
}
crm_trace("Waiting for reply to %u bytes: %.200s...", header.size, buffer);
rc = qb_ipcc_sendv_recv(client->ipc, iov, 2, client->buffer, client->buf_size, ms_timeout);
if(rc > 0) {
struct qb_ipc_response_header *hdr = (struct qb_ipc_response_header *)client->buffer;
crm_trace("Recieved response %d, size=%d, rc=%d, text: %.200s", hdr->id, hdr->size, rc, crm_ipc_buffer(client));
if(reply) {
*reply = string2xml(crm_ipc_buffer(client));
}
} else {
crm_trace("Response not recieved: rc=%d, errno=%d", rc, errno);
}
if(crm_ipc_connected(client) == FALSE) {
crm_notice("Connection to %s closed: %d", client->name, rc);
} else if(rc <= 0) {
crm_perror(LOG_ERR, "Request to %s failed: %ld", client->name, rc);
crm_info("Request was %.120s", buffer);
}
free(buffer);
return rc;
}
/* Exit code means? */
static int32_t
pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
{
uint32_t id = 0;
uint32_t flags = 0;
const char *task = NULL;
crm_client_t *c = crm_client_get(qbc);
xmlNode *msg = crm_ipcs_recv(c, data, size, &id, &flags);
crm_ipcs_send_ack(c, id, flags, "ack", __FUNCTION__, __LINE__);
if (msg == NULL) {
return 0;
}
task = crm_element_value(msg, F_CRM_TASK);
if (crm_str_eq(task, CRM_OP_QUIT, TRUE)) {
/* Time to quit */
crm_notice("Shutting down in response to ticket %s (%s)",
crm_element_value(msg, F_CRM_REFERENCE), crm_element_value(msg, F_CRM_ORIGIN));
pcmk_shutdown(15);
} else if (crm_str_eq(task, CRM_OP_RM_NODE_CACHE, TRUE)) {
/* Send to everyone */
struct iovec *iov;
int id = 0;
const char *name = NULL;
crm_element_value_int(msg, XML_ATTR_ID, &id);
name = crm_element_value(msg, XML_ATTR_UNAME);
crm_notice("Instructing peers to remove references to node %s/%u", name, id);
iov = calloc(1, sizeof(struct iovec));
iov->iov_base = dump_xml_unformatted(msg);
iov->iov_len = 1 + strlen(iov->iov_base);
send_cpg_iov(iov);
} else {
update_process_clients(c);
}
free_xml(msg);
return 0;
}
static int
crm_remote_sendv(crm_remote_t * remote, struct iovec * iov, int iovs)
{
int lpc = 0;
int rc = -ESOCKTNOSUPPORT;
for(; lpc < iovs; lpc++) {
#ifdef HAVE_GNUTLS_GNUTLS_H
if (remote->tls_session) {
rc = crm_send_tls(remote->tls_session, iov[lpc].iov_base, iov[lpc].iov_len);
} else if (remote->tcp_socket) {
#else
if (remote->tcp_socket) {
#endif
rc = crm_send_plaintext(remote->tcp_socket, iov[lpc].iov_base, iov[lpc].iov_len);
} else {
crm_err("Unsupported connection type");
}
}
return rc;
}
int
crm_remote_send(crm_remote_t * remote, xmlNode * msg)
{
int rc = -1;
static uint64_t id = 0;
char *xml_text = dump_xml_unformatted(msg);
struct iovec iov[2];
struct crm_remote_header_v0 *header;
if (xml_text == NULL) {
crm_err("Invalid XML, can not send msg");
return -1;
}
header = calloc(1, sizeof(struct crm_remote_header_v0));
iov[0].iov_base = header;
iov[0].iov_len = sizeof(struct crm_remote_header_v0);
iov[1].iov_base = xml_text;
iov[1].iov_len = 1 + strlen(xml_text);
id++;
header->id = id;
header->endian = ENDIAN_LOCAL;
header->version = REMOTE_MSG_VERSION;
header->payload_offset = iov[0].iov_len;
header->payload_uncompressed = iov[1].iov_len;
header->size_total = iov[0].iov_len + iov[1].iov_len;
crm_trace("Sending len[0]=%d, start=%x\n",
(int)iov[0].iov_len, *(int*)(void*)xml_text);
rc = crm_remote_sendv(remote, iov, 2);
if (rc < 0) {
crm_err("Failed to send remote msg, rc = %d", rc);
}
free(iov[0].iov_base);
free(iov[1].iov_base);
return rc;
}
gboolean
cluster_connect_cpg(crm_cluster_t *cluster)
{
int rc = -1;
int fd = 0;
int retries = 0;
uint32_t id = 0;
crm_node_t *peer = NULL;
cpg_handle_t handle = 0;
struct mainloop_fd_callbacks cpg_fd_callbacks = {
.dispatch = pcmk_cpg_dispatch,
.destroy = cluster->destroy,
};
cpg_callbacks_t cpg_callbacks = {
.cpg_deliver_fn = cluster->cpg.cpg_deliver_fn,
.cpg_confchg_fn = cluster->cpg.cpg_confchg_fn,
/* .cpg_deliver_fn = pcmk_cpg_deliver, */
/* .cpg_confchg_fn = pcmk_cpg_membership, */
};
cpg_evicted = FALSE;
cluster->group.length = 0;
cluster->group.value[0] = 0;
/* group.value is char[128] */
strncpy(cluster->group.value, crm_system_name?crm_system_name:"unknown", 127);
cluster->group.value[127] = 0;
cluster->group.length = 1 + QB_MIN(127, strlen(cluster->group.value));
cs_repeat(retries, 30, rc = cpg_initialize(&handle, &cpg_callbacks));
if (rc != CS_OK) {
crm_err("Could not connect to the Cluster Process Group API: %d", rc);
goto bail;
}
id = get_local_nodeid(handle);
if (id == 0) {
crm_err("Could not get local node id from the CPG API");
goto bail;
}
cluster->nodeid = id;
retries = 0;
cs_repeat(retries, 30, rc = cpg_join(handle, &cluster->group));
if (rc != CS_OK) {
crm_err("Could not join the CPG group '%s': %d", crm_system_name, rc);
goto bail;
}
rc = cpg_fd_get(handle, &fd);
if (rc != CS_OK) {
crm_err("Could not obtain the CPG API connection: %d", rc);
goto bail;
}
pcmk_cpg_handle = handle;
cluster->cpg_handle = handle;
mainloop_add_fd("corosync-cpg", G_PRIORITY_MEDIUM, fd, cluster, &cpg_fd_callbacks);
bail:
if (rc != CS_OK) {
cpg_finalize(handle);
return FALSE;
}
peer = crm_get_peer(id, NULL);
crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, ONLINESTATUS);
return TRUE;
}
gboolean
send_cluster_message_cs(xmlNode * msg, gboolean local, crm_node_t * node, enum crm_ais_msg_types dest)
{
gboolean rc = TRUE;
char *data = NULL;
data = dump_xml_unformatted(msg);
rc = send_cluster_text(crm_class_cluster, data, local, node, dest);
free(data);
return rc;
}
gboolean
send_cluster_text(int class, const char *data,
gboolean local, crm_node_t * node, enum crm_ais_msg_types dest)
{
static int msg_id = 0;
static int local_pid = 0;
static int local_name_len = 0;
static const char *local_name = NULL;
char *target = NULL;
struct iovec *iov;
AIS_Message *msg = NULL;
enum crm_ais_msg_types sender = text2msg_type(crm_system_name);
/* There are only 6 handlers registered to crm_lib_service in plugin.c */
CRM_CHECK(class < 6, crm_err("Invalid message class: %d", class);
return FALSE);
#if !SUPPORT_PLUGIN
CRM_CHECK(dest != crm_msg_ais, return FALSE);
#endif
if(local_name == NULL) {
local_name = get_local_node_name();
}
if(local_name_len == 0 && local_name) {
local_name_len = strlen(local_name);
}
if (data == NULL) {
data = "";
}
if (local_pid == 0) {
local_pid = getpid();
}
if (sender == crm_msg_none) {
sender = local_pid;
}
msg = calloc(1, sizeof(AIS_Message));
msg_id++;
msg->id = msg_id;
msg->header.id = class;
msg->header.error = CS_OK;
msg->host.type = dest;
msg->host.local = local;
if (node) {
if (node->uname) {
target = strdup(node->uname);
msg->host.size = strlen(node->uname);
memset(msg->host.uname, 0, MAX_NAME);
memcpy(msg->host.uname, node->uname, msg->host.size);
} else {
target = crm_strdup_printf("%u", node->id);
}
msg->host.id = node->id;
} else {
target = strdup("all");
}
msg->sender.id = 0;
msg->sender.type = sender;
msg->sender.pid = local_pid;
msg->sender.size = local_name_len;
memset(msg->sender.uname, 0, MAX_NAME);
if(local_name && msg->sender.size) {
memcpy(msg->sender.uname, local_name, msg->sender.size);
}
msg->size = 1 + strlen(data);
msg->header.size = sizeof(AIS_Message) + msg->size;
if (msg->size < CRM_BZ2_THRESHOLD) {
msg = realloc_safe(msg, msg->header.size);
memcpy(msg->data, data, msg->size);
} else {
char *compressed = NULL;
unsigned int new_size = 0;
char *uncompressed = strdup(data);
if (crm_compress_string(uncompressed, msg->size, 0, &compressed, &new_size)) {
msg->header.size = sizeof(AIS_Message) + new_size;
msg = realloc_safe(msg, msg->header.size);
memcpy(msg->data, compressed, new_size);
msg->is_compressed = TRUE;
msg->compressed_size = new_size;
} else {
msg = realloc_safe(msg, msg->header.size);
memcpy(msg->data, data, msg->size);
}
free(uncompressed);
free(compressed);
}
iov = calloc(1, sizeof(struct iovec));
iov->iov_base = msg;
iov->iov_len = msg->header.size;
if (msg->compressed_size) {
crm_trace("Queueing CPG message %u to %s (%llu bytes, %d bytes compressed payload): %.200s",
msg->id, target, (unsigned long long) iov->iov_len,
msg->compressed_size, data);
} else {
crm_trace("Queueing CPG message %u to %s (%llu bytes, %d bytes payload): %.200s",
msg->id, target, (unsigned long long) iov->iov_len,
msg->size, data);
}
free(target);
#if SUPPORT_PLUGIN
/* The plugin is the only time we dont use CPG messaging */
if(get_cluster_type() == pcmk_cluster_classic_ais) {
return send_plugin_text(class, iov);
}
#endif
send_cpg_iov(iov);
return TRUE;
}
enum crm_ais_msg_types
text2msg_type(const char *text)
{
int type = crm_msg_none;
CRM_CHECK(text != NULL, return type);
if (safe_str_eq(text, "ais")) {
type = crm_msg_ais;
} else if (safe_str_eq(text, "crm_plugin")) {
type = crm_msg_ais;
} else if (safe_str_eq(text, CRM_SYSTEM_CIB)) {
type = crm_msg_cib;
} else if (safe_str_eq(text, CRM_SYSTEM_CRMD)) {
type = crm_msg_crmd;
} else if (safe_str_eq(text, CRM_SYSTEM_DC)) {
type = crm_msg_crmd;
} else if (safe_str_eq(text, CRM_SYSTEM_TENGINE)) {
type = crm_msg_te;
} else if (safe_str_eq(text, CRM_SYSTEM_PENGINE)) {
type = crm_msg_pe;
} else if (safe_str_eq(text, CRM_SYSTEM_LRMD)) {
type = crm_msg_lrmd;
} else if (safe_str_eq(text, CRM_SYSTEM_STONITHD)) {
type = crm_msg_stonithd;
} else if (safe_str_eq(text, "stonith-ng")) {
type = crm_msg_stonith_ng;
} else if (safe_str_eq(text, "attrd")) {
type = crm_msg_attrd;
} else {
/* This will normally be a transient client rather than
* a cluster daemon. Set the type to the pid of the client
*/
int scan_rc = sscanf(text, "%d", &type);
if (scan_rc != 1 || type <= crm_msg_stonith_ng) {
/* Ensure it's sane */
type = crm_msg_none;
}
}
return type;
}
ssize_t
crm_ipc_prepare(uint32_t request, xmlNode * message, struct iovec ** result, uint32_t max_send_size)
{
static unsigned int biggest = 0;
struct iovec *iov;
unsigned int total = 0;
char *compressed = NULL;
char *buffer = dump_xml_unformatted(message);
struct crm_ipc_response_header *header = calloc(1, sizeof(struct crm_ipc_response_header));
CRM_ASSERT(result != NULL);
crm_ipc_init();
if (max_send_size == 0) {
max_send_size = ipc_buffer_max;
}
CRM_LOG_ASSERT(max_send_size != 0);
*result = NULL;
iov = calloc(2, sizeof(struct iovec));
iov[0].iov_len = hdr_offset;
iov[0].iov_base = header;
header->version = PCMK_IPC_VERSION;
header->size_uncompressed = 1 + strlen(buffer);
total = iov[0].iov_len + header->size_uncompressed;
if (total < max_send_size) {
iov[1].iov_base = buffer;
iov[1].iov_len = header->size_uncompressed;
} else {
unsigned int new_size = 0;
if (crm_compress_string
(buffer, header->size_uncompressed, max_send_size, &compressed, &new_size)) {
header->flags |= crm_ipc_compressed;
header->size_compressed = new_size;
iov[1].iov_len = header->size_compressed;
iov[1].iov_base = compressed;
free(buffer);
biggest = QB_MAX(header->size_compressed, biggest);
} else {
ssize_t rc = -EMSGSIZE;
crm_log_xml_trace(message, "EMSGSIZE");
biggest = QB_MAX(header->size_uncompressed, biggest);
crm_err
("Could not compress the message (%u bytes) into less than the configured ipc limit (%u bytes). "
"Set PCMK_ipc_buffer to a higher value (%u bytes suggested)",
header->size_uncompressed, max_send_size, 4 * biggest);
free(compressed);
free(buffer);
free(header);
free(iov);
return rc;
}
}
header->qb.size = iov[0].iov_len + iov[1].iov_len;
header->qb.id = (int32_t)request; /* Replying to a specific request */
*result = iov;
CRM_ASSERT(header->qb.size > 0);
return header->qb.size;
}
int
crm_ipc_send(crm_ipc_t *client, xmlNode *message, enum crm_ipc_flags flags, int32_t ms_timeout, xmlNode **reply)
{
long rc = 0;
struct iovec iov[2];
static uint32_t id = 0;
struct crm_ipc_request_header header;
char *buffer = NULL;
if(crm_ipc_connected(client) == FALSE) {
/* Don't even bother */
crm_notice("Connection to %s closed", client->name);
return -ENOTCONN;
}
if(client->need_reply) {
crm_trace("Trying again to obtain pending reply from %s", client->name);
rc = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, 300);
if(rc < 0) {
crm_warn("Sending to %s is disabled until pending reply is recieved", client->name);
free(buffer);
return -EREMOTEIO;
} else {
crm_notice("Lost reply from %s finally arrived, sending re-enabled", client->name);
client->need_reply = FALSE;
}
}
buffer = dump_xml_unformatted(message);
iov[0].iov_len = sizeof(struct crm_ipc_request_header);
iov[0].iov_base = &header;
iov[1].iov_len = 1 + strlen(buffer);
iov[1].iov_base = buffer;
header.qb.id = ++id;
header.qb.size = iov[0].iov_len + iov[1].iov_len;
header.flags = flags;
if(ms_timeout == 0) {
ms_timeout = 5000;
}
crm_trace("Sending from client: %s request id: %d bytes: %u timeout:%d msg: %.200s...",
client->name, header.qb.id, header.qb.size, ms_timeout, buffer);
if(ms_timeout > 0) {
rc = internal_ipc_send_request(client, iov, ms_timeout);
if (rc <= 0) {
crm_trace("Failed to send from client %s request %d with %u bytes: %.200s...",
client->name, header.qb.id, header.qb.size, buffer);
goto send_cleanup;
} else if(is_not_set(flags, crm_ipc_client_response)) {
crm_trace("Message sent, not waiting for reply to %d from %s to %u bytes: %.200s...",
header.qb.id, client->name, header.qb.size, buffer);
goto send_cleanup;
}
rc = internal_ipc_get_reply(client, header.qb.id, ms_timeout);
if(rc < 0) {
/* No reply, for now, disable sending
*
* The alternative is to close the connection since we don't know
* how to detect and discard out-of-sequence replies
*
* TODO - implement the above
*/
client->need_reply = TRUE;
}
} else {
rc = internal_ipc_send_recv(client, iov);
}
if(rc > 0) {
struct qb_ipc_response_header *hdr = (struct qb_ipc_response_header *)client->buffer;
crm_trace("Recieved response %d, size=%d, rc=%ld, text: %.200s", hdr->id, hdr->size, rc, crm_ipc_buffer(client));
if(reply) {
*reply = string2xml(crm_ipc_buffer(client));
}
} else {
crm_trace("Response not recieved: rc=%ld, errno=%d", rc, errno);
}
send_cleanup:
if(crm_ipc_connected(client) == FALSE) {
crm_notice("Connection to %s closed: %s (%ld)", client->name, pcmk_strerror(rc), rc);
} else if(rc <= 0) {
crm_warn("Request %d to %s failed: %s (%ld)", header.qb.id, client->name, pcmk_strerror(rc), rc);
crm_info("Request was %.120s", buffer);
}
free(buffer);
return rc;
}
ssize_t
crm_ipcs_send(qb_ipcs_connection_t *c, uint32_t request, xmlNode *message, enum crm_ipc_server_flags flags)
{
int rc;
int lpc = 0;
int retries = 40;
int level = LOG_CRIT;
struct iovec iov[2];
static uint32_t id = 1;
const char *type = "Response";
struct qb_ipc_response_header header;
char *buffer = dump_xml_unformatted(message);
struct timespec delay = { 0, 250000000 }; /* 250ms */
memset(&iov, 0, 2 * sizeof(struct iovec));
iov[0].iov_len = sizeof(struct qb_ipc_response_header);
iov[0].iov_base = &header;
iov[1].iov_len = 1 + strlen(buffer);
iov[1].iov_base = buffer;
if(flags & crm_ipc_server_event) {
header.id = id++; /* We don't really use it, but doesn't hurt to set one */
} else {
CRM_LOG_ASSERT (request != 0);
header.id = request; /* Replying to a specific request */
}
header.error = 0; /* unused */
header.size = iov[0].iov_len + iov[1].iov_len;
if(flags & crm_ipc_server_error) {
retries = 20;
level = LOG_ERR;
} else if(flags & crm_ipc_server_info) {
retries = 10;
level = LOG_INFO;
}
while(lpc < retries) {
if(flags & crm_ipc_server_event) {
type = "Event";
rc = qb_ipcs_event_sendv(c, iov, 2);
if(rc == -EPIPE || rc == -ENOTCONN) {
crm_trace("Client %p disconnected", c);
level = LOG_INFO;
}
} else {
rc = qb_ipcs_response_sendv(c, iov, 2);
}
if(rc != -EAGAIN) {
break;
}
lpc++;
crm_debug("Attempting resend %d of %s %d (%d bytes) to %p[%d]: %.120s",
lpc, type, header.id, header.size, c, crm_ipcs_client_pid(c), buffer);
nanosleep(&delay, NULL);
}
if(rc < header.size) {
struct qb_ipcs_connection_stats_2 *stats = qb_ipcs_connection_stats_get_2(c, 0);
do_crm_log(level,
"%s %d failed, size=%d, to=%p[%d], queue=%d, rc=%d: %.120s",
type, header.id, header.size, c, stats->client_pid, stats->event_q_length, rc, buffer);
free(stats);
} else {
crm_trace("%s %d sent, %d bytes to %p[%d]: %.120s", type, header.id, rc,
c, crm_ipcs_client_pid(c), buffer);
}
free(buffer);
return rc;
}
