int
metadata_write(struct hast_resource *res)
{
struct ebuf *eb;
struct nv *nv;
unsigned char *buf, *ptr;
size_t size;
ssize_t done;
int ret;
buf = calloc(1, METADATA_SIZE);
if (buf == NULL) {
pjdlog_error("Unable to allocate %zu bytes for metadata.",
(size_t)METADATA_SIZE);
return (-1);
}
ret = -1;
nv = nv_alloc();
nv_add_string(nv, res->hr_name, "resource");
nv_add_uint64(nv, (uint64_t)res->hr_datasize, "datasize");
nv_add_uint32(nv, (uint32_t)res->hr_extentsize, "extentsize");
nv_add_uint32(nv, (uint32_t)res->hr_keepdirty, "keepdirty");
nv_add_uint64(nv, (uint64_t)res->hr_localoff, "offset");
nv_add_uint64(nv, res->hr_resuid, "resuid");
if (res->hr_role == HAST_ROLE_PRIMARY ||
res->hr_role == HAST_ROLE_INIT) {
nv_add_uint64(nv, res->hr_primary_localcnt, "localcnt");
nv_add_uint64(nv, res->hr_primary_remotecnt, "remotecnt");
} else /* if (res->hr_role == HAST_ROLE_SECONDARY) */ {
PJDLOG_ASSERT(res->hr_role == HAST_ROLE_SECONDARY);
nv_add_uint64(nv, res->hr_secondary_localcnt, "localcnt");
nv_add_uint64(nv, res->hr_secondary_remotecnt, "remotecnt");
}
nv_add_string(nv, role2str(res->hr_role), "prevrole");
if (nv_error(nv) != 0) {
pjdlog_error("Unable to create metadata.");
goto end;
}
res->hr_previous_role = res->hr_role;
eb = nv_hton(nv);
PJDLOG_ASSERT(eb != NULL);
ptr = ebuf_data(eb, &size);
PJDLOG_ASSERT(ptr != NULL);
PJDLOG_ASSERT(size < METADATA_SIZE);
bcopy(ptr, buf, size);
done = pwrite(res->hr_localfd, buf, METADATA_SIZE, 0);
if (done == -1 || done != METADATA_SIZE) {
pjdlog_errno(LOG_ERR, "Unable to write metadata");
goto end;
}
ret = 0;
end:
free(buf);
nv_free(nv);
return (ret);
}
static void
init_remote(struct hast_resource *res, struct nv *nvin)
{
uint64_t resuid;
struct nv *nvout;
unsigned char *map;
size_t mapsize;
#ifdef notyet
/* Setup direction. */
if (proto_send(res->hr_remoteout, NULL, 0) == -1)
pjdlog_errno(LOG_WARNING, "Unable to set connection direction");
#endif
map = NULL;
mapsize = 0;
nvout = nv_alloc();
nv_add_int64(nvout, (int64_t)res->hr_datasize, "datasize");
nv_add_int32(nvout, (int32_t)res->hr_extentsize, "extentsize");
resuid = nv_get_uint64(nvin, "resuid");
res->hr_primary_localcnt = nv_get_uint64(nvin, "localcnt");
res->hr_primary_remotecnt = nv_get_uint64(nvin, "remotecnt");
nv_add_uint64(nvout, res->hr_secondary_localcnt, "localcnt");
nv_add_uint64(nvout, res->hr_secondary_remotecnt, "remotecnt");
mapsize = activemap_calc_ondisk_size(res->hr_local_mediasize -
METADATA_SIZE, res->hr_extentsize, res->hr_local_sectorsize);
map = malloc(mapsize);
if (map == NULL) {
pjdlog_exitx(EX_TEMPFAIL,
"Unable to allocate memory (%zu bytes) for activemap.",
mapsize);
}
/*
* When we work as primary and secondary is missing we will increase
* localcnt in our metadata. When secondary is connected and synced
* we make localcnt be equal to remotecnt, which means nodes are more
* or less in sync.
* Split-brain condition is when both nodes are not able to communicate
* and are both configured as primary nodes. In turn, they can both
* make incompatible changes to the data and we have to detect that.
* Under split-brain condition we will increase our localcnt on first
* write and remote node will increase its localcnt on first write.
* When we connect we can see that primary's localcnt is greater than
* our remotecnt (primary was modified while we weren't watching) and
* our localcnt is greater than primary's remotecnt (we were modified
* while primary wasn't watching).
* There are many possible combinations which are all gathered below.
* Don't pay too much attention to exact numbers, the more important
* is to compare them. We compare secondary's local with primary's
* remote and secondary's remote with primary's local.
* Note that every case where primary's localcnt is smaller than
* secondary's remotecnt and where secondary's localcnt is smaller than
* primary's remotecnt should be impossible in practise. We will perform
* full synchronization then. Those cases are marked with an asterisk.
* Regular synchronization means that only extents marked as dirty are
* synchronized (regular synchronization).
*
* SECONDARY METADATA PRIMARY METADATA
* local=3 remote=3 local=2 remote=2* ?! Full sync from secondary.
* local=3 remote=3 local=2 remote=3* ?! Full sync from primary.
* local=3 remote=3 local=2 remote=4* ?! Full sync from primary.
* local=3 remote=3 local=3 remote=2 Primary is out-of-date,
* regular sync from secondary.
* local=3 remote=3 local=3 remote=3 Regular sync just in case.
* local=3 remote=3 local=3 remote=4* ?! Full sync from primary.
* local=3 remote=3 local=4 remote=2 Split-brain condition.
* local=3 remote=3 local=4 remote=3 Secondary out-of-date,
* regular sync from primary.
* local=3 remote=3 local=4 remote=4* ?! Full sync from primary.
*/
if (res->hr_resuid == 0) {
/*
* Provider is used for the first time. If primary node done no
* writes yet as well (we will find "virgin" argument) then
* there is no need to synchronize anything. If primary node
* done any writes already we have to synchronize everything.
*/
PJDLOG_ASSERT(res->hr_secondary_localcnt == 0);
res->hr_resuid = resuid;
if (metadata_write(res) < 0)
exit(EX_NOINPUT);
if (nv_exists(nvin, "virgin")) {
free(map);
map = NULL;
mapsize = 0;
} else {
memset(map, 0xff, mapsize);
}
nv_add_int8(nvout, 1, "virgin");
nv_add_uint8(nvout, HAST_SYNCSRC_PRIMARY, "syncsrc");
} else if (res->hr_resuid != resuid) {
char errmsg[256];
free(map);
(void)snprintf(errmsg, sizeof(errmsg),
"Resource unique ID mismatch (primary=%ju, secondary=%ju).",
(uintmax_t)resuid, (uintmax_t)res->hr_resuid);
pjdlog_error("%s", errmsg);
nv_add_string(nvout, errmsg, "errmsg");
if (hast_proto_send(res, res->hr_remotein, nvout, NULL, 0) < 0) {
pjdlog_exit(EX_TEMPFAIL, "Unable to send response to %s",
res->hr_remoteaddr);
}
nv_free(nvout);
exit(EX_CONFIG);
} else if (
/* Is primary out-of-date? */
(res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
res->hr_secondary_remotecnt == res->hr_primary_localcnt) ||
/* Are the nodes more or less in sync? */
(res->hr_secondary_localcnt == res->hr_primary_remotecnt &&
res->hr_secondary_remotecnt == res->hr_primary_localcnt) ||
/* Is secondary out-of-date? */
(res->hr_secondary_localcnt == res->hr_primary_remotecnt &&
res->hr_secondary_remotecnt < res->hr_primary_localcnt)) {
/*
* Nodes are more or less in sync or one of the nodes is
* out-of-date.
* It doesn't matter at this point which one, we just have to
* send out local bitmap to the remote node.
*/
if (pread(res->hr_localfd, map, mapsize, METADATA_SIZE) !=
(ssize_t)mapsize) {
pjdlog_exit(LOG_ERR, "Unable to read activemap");
}
if (res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
res->hr_secondary_remotecnt == res->hr_primary_localcnt) {
/* Primary is out-of-date, sync from secondary. */
nv_add_uint8(nvout, HAST_SYNCSRC_SECONDARY, "syncsrc");
} else {
/*
* Secondary is out-of-date or counts match.
* Sync from primary.
*/
nv_add_uint8(nvout, HAST_SYNCSRC_PRIMARY, "syncsrc");
}
} else if (res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
res->hr_primary_localcnt > res->hr_secondary_remotecnt) {
/*
* Not good, we have split-brain condition.
*/
free(map);
pjdlog_error("Split-brain detected, exiting.");
nv_add_string(nvout, "Split-brain condition!", "errmsg");
if (hast_proto_send(res, res->hr_remotein, nvout, NULL, 0) < 0) {
pjdlog_exit(EX_TEMPFAIL, "Unable to send response to %s",
res->hr_remoteaddr);
}
nv_free(nvout);
/* Exit on split-brain. */
event_send(res, EVENT_SPLITBRAIN);
exit(EX_CONFIG);
} else /* if (res->hr_secondary_localcnt < res->hr_primary_remotecnt ||
res->hr_primary_localcnt < res->hr_secondary_remotecnt) */ {
/*
* This should never happen in practise, but we will perform
* full synchronization.
*/
PJDLOG_ASSERT(res->hr_secondary_localcnt < res->hr_primary_remotecnt ||
res->hr_primary_localcnt < res->hr_secondary_remotecnt);
mapsize = activemap_calc_ondisk_size(res->hr_local_mediasize -
METADATA_SIZE, res->hr_extentsize,
res->hr_local_sectorsize);
memset(map, 0xff, mapsize);
if (res->hr_secondary_localcnt > res->hr_primary_remotecnt) {
/* In this one of five cases sync from secondary. */
nv_add_uint8(nvout, HAST_SYNCSRC_SECONDARY, "syncsrc");
} else {
/* For the rest four cases sync from primary. */
nv_add_uint8(nvout, HAST_SYNCSRC_PRIMARY, "syncsrc");
}
pjdlog_warning("This should never happen, asking for full synchronization (primary(local=%ju, remote=%ju), secondary(local=%ju, remote=%ju)).",
(uintmax_t)res->hr_primary_localcnt,
(uintmax_t)res->hr_primary_remotecnt,
(uintmax_t)res->hr_secondary_localcnt,
(uintmax_t)res->hr_secondary_remotecnt);
}
nv_add_uint32(nvout, (uint32_t)mapsize, "mapsize");
if (hast_proto_send(res, res->hr_remotein, nvout, map, mapsize) < 0) {
pjdlog_exit(EX_TEMPFAIL, "Unable to send activemap to %s",
res->hr_remoteaddr);
}
if (map != NULL)
free(map);
nv_free(nvout);
#ifdef notyet
/* Setup direction. */
if (proto_recv(res->hr_remotein, NULL, 0) == -1)
pjdlog_errno(LOG_WARNING, "Unable to set connection direction");
#endif
}
void json_print_response(uint8_t status)
{
#ifdef __SILENCE_JSON_RESPONSES
return;
#endif
if (js.json_verbosity == JV_SILENT) { // silent means no responses
return;
}
if (js.json_verbosity == JV_EXCEPTIONS) { // cutout for JV_EXCEPTIONS mode
if (status == STAT_OK) {
if (cm.machine_state != MACHINE_INITIALIZING) { // always do full echo during startup
return;
}
}
}
// Body processing
nvObj_t *nv = nv_body;
if (status == STAT_JSON_SYNTAX_ERROR) {
nv_reset_nv_list();
nv_add_string((const char *)"err", escape_string(cs.bufp, cs.saved_buf));
} else if (cm.machine_state != MACHINE_INITIALIZING) { // always do full echo during startup
uint8_t nv_type;
do {
if ((nv_type = nv_get_type(nv)) == NV_TYPE_NULL) break;
if (nv_type == NV_TYPE_GCODE) {
if (js.echo_json_gcode_block == false) { // kill command echo if not enabled
nv->valuetype = TYPE_EMPTY;
}
//++++ } else if (nv_type == NV_TYPE_CONFIG) { // kill config echo if not enabled
//fix me if (js.echo_json_configs == false) {
// nv->valuetype = TYPE_EMPTY;
// }
} else if (nv_type == NV_TYPE_MESSAGE) { // kill message echo if not enabled
if (js.echo_json_messages == false) {
nv->valuetype = TYPE_EMPTY;
}
} else if (nv_type == NV_TYPE_LINENUM) { // kill line number echo if not enabled
if ((js.echo_json_linenum == false) || (fp_ZERO(nv->value))) { // do not report line# 0
nv->valuetype = TYPE_EMPTY;
}
}
} while ((nv = nv->nx) != NULL);
}
// Footer processing
while(nv->valuetype != TYPE_EMPTY) { // find a free nvObj at end of the list...
if ((nv = nv->nx) == NULL) { // oops! No free nvObj!
rpt_exception(STAT_JSON_TOO_LONG, "json_print"); // report this as an exception
return;
}
}
char footer_string[NV_FOOTER_LEN];
// in xio.cpp:xio.readline the CR||LF read from the host is not appended to the string.
// to ensure that the correct number of bytes are reported back to the host we add a +1 to
// cs.linelen so that the number of bytes received matches the number of bytes reported
sprintf((char *)footer_string, "%d,%d,%d", 1, status, cs.linelen + 1);
cs.linelen = 0; // reset linelen so it's only reported once
// if (xio.enable_window_mode) { // 2 footer styles are supported...
// sprintf((char *)footer_string, "%d,%d,%d", 2, status, xio_get_window_slots()); //...windowing
// } else {
// sprintf((char *)footer_string, "%d,%d,%d", 1, status, cs.linelen); //...streaming
// cs.linelen = 0; // reset linelen so it's only reported once
// }
nv_copy_string(nv, footer_string); // link string to nv object
nv->depth = 0; // footer 'f' is a peer to response 'r' (hard wired to 0)
nv->valuetype = TYPE_ARRAY; // declare it as an array
strcpy(nv->token, "f"); // set it to Footer
nv->nx = NULL; // terminate the list
// serialize the JSON response and print it if there were no errors
if (json_serialize(nv_header, cs.out_buf, sizeof(cs.out_buf)) >= 0) {
fprintf(stderr, "%s", cs.out_buf);
}
}
nvObj_t *nv_add_conditional_message(const char_t *string) // conditionally add a message object to the body
{
if ((cfg.comm_mode == JSON_MODE) && (js.echo_json_messages != true)) { return (NULL);}
return(nv_add_string((const char_t *)"msg", string));
}
