static void init_remote(struct hast_resource *res, struct nv *nvin) { uint64_t resuid; struct nv *nvout; unsigned char *map; size_t mapsize; #ifdef notyet /* Setup direction. */ if (proto_send(res->hr_remoteout, NULL, 0) == -1) pjdlog_errno(LOG_WARNING, "Unable to set connection direction"); #endif map = NULL; mapsize = 0; nvout = nv_alloc(); nv_add_int64(nvout, (int64_t)res->hr_datasize, "datasize"); nv_add_int32(nvout, (int32_t)res->hr_extentsize, "extentsize"); resuid = nv_get_uint64(nvin, "resuid"); res->hr_primary_localcnt = nv_get_uint64(nvin, "localcnt"); res->hr_primary_remotecnt = nv_get_uint64(nvin, "remotecnt"); nv_add_uint64(nvout, res->hr_secondary_localcnt, "localcnt"); nv_add_uint64(nvout, res->hr_secondary_remotecnt, "remotecnt"); mapsize = activemap_calc_ondisk_size(res->hr_local_mediasize - METADATA_SIZE, res->hr_extentsize, res->hr_local_sectorsize); map = malloc(mapsize); if (map == NULL) { pjdlog_exitx(EX_TEMPFAIL, "Unable to allocate memory (%zu bytes) for activemap.", mapsize); } /* * When we work as primary and secondary is missing we will increase * localcnt in our metadata. When secondary is connected and synced * we make localcnt be equal to remotecnt, which means nodes are more * or less in sync. * Split-brain condition is when both nodes are not able to communicate * and are both configured as primary nodes. In turn, they can both * make incompatible changes to the data and we have to detect that. * Under split-brain condition we will increase our localcnt on first * write and remote node will increase its localcnt on first write. * When we connect we can see that primary's localcnt is greater than * our remotecnt (primary was modified while we weren't watching) and * our localcnt is greater than primary's remotecnt (we were modified * while primary wasn't watching). * There are many possible combinations which are all gathered below. * Don't pay too much attention to exact numbers, the more important * is to compare them. We compare secondary's local with primary's * remote and secondary's remote with primary's local. * Note that every case where primary's localcnt is smaller than * secondary's remotecnt and where secondary's localcnt is smaller than * primary's remotecnt should be impossible in practise. We will perform * full synchronization then. Those cases are marked with an asterisk. * Regular synchronization means that only extents marked as dirty are * synchronized (regular synchronization). * * SECONDARY METADATA PRIMARY METADATA * local=3 remote=3 local=2 remote=2* ?! Full sync from secondary. * local=3 remote=3 local=2 remote=3* ?! Full sync from primary. * local=3 remote=3 local=2 remote=4* ?! Full sync from primary. * local=3 remote=3 local=3 remote=2 Primary is out-of-date, * regular sync from secondary. * local=3 remote=3 local=3 remote=3 Regular sync just in case. * local=3 remote=3 local=3 remote=4* ?! Full sync from primary. * local=3 remote=3 local=4 remote=2 Split-brain condition. * local=3 remote=3 local=4 remote=3 Secondary out-of-date, * regular sync from primary. * local=3 remote=3 local=4 remote=4* ?! Full sync from primary. */ if (res->hr_resuid == 0) { /* * Provider is used for the first time. If primary node done no * writes yet as well (we will find "virgin" argument) then * there is no need to synchronize anything. If primary node * done any writes already we have to synchronize everything. */ PJDLOG_ASSERT(res->hr_secondary_localcnt == 0); res->hr_resuid = resuid; if (metadata_write(res) < 0) exit(EX_NOINPUT); if (nv_exists(nvin, "virgin")) { free(map); map = NULL; mapsize = 0; } else { memset(map, 0xff, mapsize); } nv_add_int8(nvout, 1, "virgin"); nv_add_uint8(nvout, HAST_SYNCSRC_PRIMARY, "syncsrc"); } else if (res->hr_resuid != resuid) { char errmsg[256]; free(map); (void)snprintf(errmsg, sizeof(errmsg), "Resource unique ID mismatch (primary=%ju, secondary=%ju).", (uintmax_t)resuid, (uintmax_t)res->hr_resuid); pjdlog_error("%s", errmsg); nv_add_string(nvout, errmsg, "errmsg"); if (hast_proto_send(res, res->hr_remotein, nvout, NULL, 0) < 0) { pjdlog_exit(EX_TEMPFAIL, "Unable to send response to %s", res->hr_remoteaddr); } nv_free(nvout); exit(EX_CONFIG); } else if ( /* Is primary out-of-date? */ (res->hr_secondary_localcnt > res->hr_primary_remotecnt && res->hr_secondary_remotecnt == res->hr_primary_localcnt) || /* Are the nodes more or less in sync? */ (res->hr_secondary_localcnt == res->hr_primary_remotecnt && res->hr_secondary_remotecnt == res->hr_primary_localcnt) || /* Is secondary out-of-date? */ (res->hr_secondary_localcnt == res->hr_primary_remotecnt && res->hr_secondary_remotecnt < res->hr_primary_localcnt)) { /* * Nodes are more or less in sync or one of the nodes is * out-of-date. * It doesn't matter at this point which one, we just have to * send out local bitmap to the remote node. */ if (pread(res->hr_localfd, map, mapsize, METADATA_SIZE) != (ssize_t)mapsize) { pjdlog_exit(LOG_ERR, "Unable to read activemap"); } if (res->hr_secondary_localcnt > res->hr_primary_remotecnt && res->hr_secondary_remotecnt == res->hr_primary_localcnt) { /* Primary is out-of-date, sync from secondary. */ nv_add_uint8(nvout, HAST_SYNCSRC_SECONDARY, "syncsrc"); } else { /* * Secondary is out-of-date or counts match. * Sync from primary. */ nv_add_uint8(nvout, HAST_SYNCSRC_PRIMARY, "syncsrc"); } } else if (res->hr_secondary_localcnt > res->hr_primary_remotecnt && res->hr_primary_localcnt > res->hr_secondary_remotecnt) { /* * Not good, we have split-brain condition. */ free(map); pjdlog_error("Split-brain detected, exiting."); nv_add_string(nvout, "Split-brain condition!", "errmsg"); if (hast_proto_send(res, res->hr_remotein, nvout, NULL, 0) < 0) { pjdlog_exit(EX_TEMPFAIL, "Unable to send response to %s", res->hr_remoteaddr); } nv_free(nvout); /* Exit on split-brain. */ event_send(res, EVENT_SPLITBRAIN); exit(EX_CONFIG); } else /* if (res->hr_secondary_localcnt < res->hr_primary_remotecnt || res->hr_primary_localcnt < res->hr_secondary_remotecnt) */ { /* * This should never happen in practise, but we will perform * full synchronization. */ PJDLOG_ASSERT(res->hr_secondary_localcnt < res->hr_primary_remotecnt || res->hr_primary_localcnt < res->hr_secondary_remotecnt); mapsize = activemap_calc_ondisk_size(res->hr_local_mediasize - METADATA_SIZE, res->hr_extentsize, res->hr_local_sectorsize); memset(map, 0xff, mapsize); if (res->hr_secondary_localcnt > res->hr_primary_remotecnt) { /* In this one of five cases sync from secondary. */ nv_add_uint8(nvout, HAST_SYNCSRC_SECONDARY, "syncsrc"); } else { /* For the rest four cases sync from primary. */ nv_add_uint8(nvout, HAST_SYNCSRC_PRIMARY, "syncsrc"); } pjdlog_warning("This should never happen, asking for full synchronization (primary(local=%ju, remote=%ju), secondary(local=%ju, remote=%ju)).", (uintmax_t)res->hr_primary_localcnt, (uintmax_t)res->hr_primary_remotecnt, (uintmax_t)res->hr_secondary_localcnt, (uintmax_t)res->hr_secondary_remotecnt); } nv_add_uint32(nvout, (uint32_t)mapsize, "mapsize"); if (hast_proto_send(res, res->hr_remotein, nvout, map, mapsize) < 0) { pjdlog_exit(EX_TEMPFAIL, "Unable to send activemap to %s", res->hr_remoteaddr); } if (map != NULL) free(map); nv_free(nvout); #ifdef notyet /* Setup direction. */ if (proto_recv(res->hr_remotein, NULL, 0) == -1) pjdlog_errno(LOG_WARNING, "Unable to set connection direction"); #endif }
static int requnpack(struct hast_resource *res, struct hio *hio, struct nv *nv) { hio->hio_cmd = nv_get_uint8(nv, "cmd"); if (hio->hio_cmd == 0) { pjdlog_error("Header contains no 'cmd' field."); hio->hio_error = EINVAL; goto end; } if (hio->hio_cmd != HIO_KEEPALIVE) { hio->hio_seq = nv_get_uint64(nv, "seq"); if (hio->hio_seq == 0) { pjdlog_error("Header contains no 'seq' field."); hio->hio_error = EINVAL; goto end; } } switch (hio->hio_cmd) { case HIO_FLUSH: case HIO_KEEPALIVE: break; case HIO_READ: case HIO_WRITE: case HIO_DELETE: hio->hio_offset = nv_get_uint64(nv, "offset"); if (nv_error(nv) != 0) { pjdlog_error("Header is missing 'offset' field."); hio->hio_error = EINVAL; goto end; } hio->hio_length = nv_get_uint64(nv, "length"); if (nv_error(nv) != 0) { pjdlog_error("Header is missing 'length' field."); hio->hio_error = EINVAL; goto end; } if (hio->hio_length == 0) { pjdlog_error("Data length is zero."); hio->hio_error = EINVAL; goto end; } if (hio->hio_length > MAXPHYS) { pjdlog_error("Data length is too large (%ju > %ju).", (uintmax_t)hio->hio_length, (uintmax_t)MAXPHYS); hio->hio_error = EINVAL; goto end; } if ((hio->hio_offset % res->hr_local_sectorsize) != 0) { pjdlog_error("Offset %ju is not multiple of sector size.", (uintmax_t)hio->hio_offset); hio->hio_error = EINVAL; goto end; } if ((hio->hio_length % res->hr_local_sectorsize) != 0) { pjdlog_error("Length %ju is not multiple of sector size.", (uintmax_t)hio->hio_length); hio->hio_error = EINVAL; goto end; } if (hio->hio_offset + hio->hio_length > (uint64_t)res->hr_datasize) { pjdlog_error("Data offset is too large (%ju > %ju).", (uintmax_t)(hio->hio_offset + hio->hio_length), (uintmax_t)res->hr_datasize); hio->hio_error = EINVAL; goto end; } break; default: pjdlog_error("Header contains invalid 'cmd' (%hhu).", hio->hio_cmd); hio->hio_error = EINVAL; goto end; } hio->hio_error = 0; end: return (hio->hio_error); }
int metadata_read(struct hast_resource *res, bool openrw) { unsigned char *buf; struct ebuf *eb; struct nv *nv; ssize_t done; const char *str; int rerrno; bool opened_here; opened_here = false; rerrno = 0; /* * Is this first metadata_read() call for this resource? */ if (res->hr_localfd == -1) { if (provinfo(res, openrw) == -1) { rerrno = errno; goto fail; } opened_here = true; pjdlog_debug(1, "Obtained info about %s.", res->hr_localpath); if (openrw) { if (flock(res->hr_localfd, LOCK_EX | LOCK_NB) == -1) { rerrno = errno; if (errno == EOPNOTSUPP) { pjdlog_warning("Unable to lock %s (operation not supported), but continuing.", res->hr_localpath); } else { pjdlog_errno(LOG_ERR, "Unable to lock %s", res->hr_localpath); goto fail; } } pjdlog_debug(1, "Locked %s.", res->hr_localpath); } } eb = ebuf_alloc(METADATA_SIZE); if (eb == NULL) { rerrno = errno; pjdlog_errno(LOG_ERR, "Unable to allocate memory to read metadata"); goto fail; } if (ebuf_add_tail(eb, NULL, METADATA_SIZE) == -1) { rerrno = errno; pjdlog_errno(LOG_ERR, "Unable to allocate memory to read metadata"); ebuf_free(eb); goto fail; } buf = ebuf_data(eb, NULL); PJDLOG_ASSERT(buf != NULL); done = pread(res->hr_localfd, buf, METADATA_SIZE, 0); if (done == -1 || done != METADATA_SIZE) { rerrno = errno; pjdlog_errno(LOG_ERR, "Unable to read metadata"); ebuf_free(eb); goto fail; } nv = nv_ntoh(eb); if (nv == NULL) { rerrno = errno; pjdlog_errno(LOG_ERR, "Metadata read from %s is invalid", res->hr_localpath); ebuf_free(eb); goto fail; } str = nv_get_string(nv, "resource"); if (str != NULL && strcmp(str, res->hr_name) != 0) { pjdlog_error("Provider %s is not part of resource %s.", res->hr_localpath, res->hr_name); nv_free(nv); goto fail; } res->hr_datasize = nv_get_uint64(nv, "datasize"); res->hr_extentsize = (int)nv_get_uint32(nv, "extentsize"); res->hr_keepdirty = (int)nv_get_uint32(nv, "keepdirty"); res->hr_localoff = nv_get_uint64(nv, "offset"); res->hr_resuid = nv_get_uint64(nv, "resuid"); if (res->hr_role != HAST_ROLE_PRIMARY) { /* Secondary or init role. */ res->hr_secondary_localcnt = nv_get_uint64(nv, "localcnt"); res->hr_secondary_remotecnt = nv_get_uint64(nv, "remotecnt"); } if (res->hr_role != HAST_ROLE_SECONDARY) { /* Primary or init role. */ res->hr_primary_localcnt = nv_get_uint64(nv, "localcnt"); res->hr_primary_remotecnt = nv_get_uint64(nv, "remotecnt"); } str = nv_get_string(nv, "prevrole"); if (str != NULL) { if (strcmp(str, "primary") == 0) res->hr_previous_role = HAST_ROLE_PRIMARY; else if (strcmp(str, "secondary") == 0) res->hr_previous_role = HAST_ROLE_SECONDARY; } if (nv_error(nv) != 0) { errno = rerrno = nv_error(nv); pjdlog_errno(LOG_ERR, "Unable to read metadata from %s", res->hr_localpath); nv_free(nv); goto fail; } nv_free(nv); return (0); fail: if (opened_here) { close(res->hr_localfd); res->hr_localfd = -1; } errno = rerrno; return (-1); }