Esempio n. 1
0
static void
output_status_aux(struct nv *nvout)
{

	nv_add_uint64(nvout, (uint64_t)hio_free_list_size, "idle_queue_size");
	nv_add_uint64(nvout, (uint64_t)hio_disk_list_size, "local_queue_size");
	nv_add_uint64(nvout, (uint64_t)hio_send_list_size, "send_queue_size");
}
Esempio n. 2
0
int
metadata_write(struct hast_resource *res)
{
	struct ebuf *eb;
	struct nv *nv;
	unsigned char *buf, *ptr;
	size_t size;
	ssize_t done;
	int ret;

	buf = calloc(1, METADATA_SIZE);
	if (buf == NULL) {
		pjdlog_error("Unable to allocate %zu bytes for metadata.",
		    (size_t)METADATA_SIZE);
		return (-1);
	}

	ret = -1;

	nv = nv_alloc();
	nv_add_string(nv, res->hr_name, "resource");
	nv_add_uint64(nv, (uint64_t)res->hr_datasize, "datasize");
	nv_add_uint32(nv, (uint32_t)res->hr_extentsize, "extentsize");
	nv_add_uint32(nv, (uint32_t)res->hr_keepdirty, "keepdirty");
	nv_add_uint64(nv, (uint64_t)res->hr_localoff, "offset");
	nv_add_uint64(nv, res->hr_resuid, "resuid");
	if (res->hr_role == HAST_ROLE_PRIMARY ||
	    res->hr_role == HAST_ROLE_INIT) {
		nv_add_uint64(nv, res->hr_primary_localcnt, "localcnt");
		nv_add_uint64(nv, res->hr_primary_remotecnt, "remotecnt");
	} else /* if (res->hr_role == HAST_ROLE_SECONDARY) */ {
		PJDLOG_ASSERT(res->hr_role == HAST_ROLE_SECONDARY);
		nv_add_uint64(nv, res->hr_secondary_localcnt, "localcnt");
		nv_add_uint64(nv, res->hr_secondary_remotecnt, "remotecnt");
	}
	nv_add_string(nv, role2str(res->hr_role), "prevrole");
	if (nv_error(nv) != 0) {
		pjdlog_error("Unable to create metadata.");
		goto end;
	}
	res->hr_previous_role = res->hr_role;
	eb = nv_hton(nv);
	PJDLOG_ASSERT(eb != NULL);
	ptr = ebuf_data(eb, &size);
	PJDLOG_ASSERT(ptr != NULL);
	PJDLOG_ASSERT(size < METADATA_SIZE);
	bcopy(ptr, buf, size);
	done = pwrite(res->hr_localfd, buf, METADATA_SIZE, 0);
	if (done == -1 || done != METADATA_SIZE) {
		pjdlog_errno(LOG_ERR, "Unable to write metadata");
		goto end;
	}
	ret = 0;
end:
	free(buf);
	nv_free(nv);
	return (ret);
}
Esempio n. 3
0
/*
 * Thread sends requests back to primary node.
 */
static void *
send_thread(void *arg)
{
	struct hast_resource *res = arg;
	struct nv *nvout;
	struct hio *hio;
	void *data;
	size_t length;

	for (;;) {
		pjdlog_debug(2, "send: Taking request.");
		QUEUE_TAKE(send, hio);
		reqlog(LOG_DEBUG, 2, -1, hio, "send: (%p) Got request: ", hio);
		nvout = nv_alloc();
		/* Copy sequence number. */
		nv_add_uint64(nvout, hio->hio_seq, "seq");
		switch (hio->hio_cmd) {
		case HIO_READ:
			if (hio->hio_error == 0) {
				data = hio->hio_data;
				length = hio->hio_length;
				break;
			}
			/*
			 * We send no data in case of an error.
			 */
			/* FALLTHROUGH */
		case HIO_DELETE:
		case HIO_FLUSH:
		case HIO_WRITE:
			data = NULL;
			length = 0;
			break;
		default:
			PJDLOG_ABORT("Unexpected command (cmd=%hhu).",
			    hio->hio_cmd);
		}
		if (hio->hio_error != 0)
			nv_add_int16(nvout, hio->hio_error, "error");
		if (hast_proto_send(res, res->hr_remoteout, nvout, data,
		    length) < 0) {
			secondary_exit(EX_TEMPFAIL, "Unable to send reply.");
		}
		nv_free(nvout);
		pjdlog_debug(2, "send: (%p) Moving request to the free queue.",
		    hio);
		hio_clear(hio);
		QUEUE_INSERT(free, hio);
	}
	/* NOTREACHED */
	return (NULL);
}
Esempio n. 4
0
static void
init_remote(struct hast_resource *res, struct nv *nvin)
{
	uint64_t resuid;
	struct nv *nvout;
	unsigned char *map;
	size_t mapsize;

#ifdef notyet
	/* Setup direction. */
	if (proto_send(res->hr_remoteout, NULL, 0) == -1)
		pjdlog_errno(LOG_WARNING, "Unable to set connection direction");
#endif

	map = NULL;
	mapsize = 0;
	nvout = nv_alloc();
	nv_add_int64(nvout, (int64_t)res->hr_datasize, "datasize");
	nv_add_int32(nvout, (int32_t)res->hr_extentsize, "extentsize");
	resuid = nv_get_uint64(nvin, "resuid");
	res->hr_primary_localcnt = nv_get_uint64(nvin, "localcnt");
	res->hr_primary_remotecnt = nv_get_uint64(nvin, "remotecnt");
	nv_add_uint64(nvout, res->hr_secondary_localcnt, "localcnt");
	nv_add_uint64(nvout, res->hr_secondary_remotecnt, "remotecnt");
	mapsize = activemap_calc_ondisk_size(res->hr_local_mediasize -
	    METADATA_SIZE, res->hr_extentsize, res->hr_local_sectorsize);
	map = malloc(mapsize);
	if (map == NULL) {
		pjdlog_exitx(EX_TEMPFAIL,
		    "Unable to allocate memory (%zu bytes) for activemap.",
		    mapsize);
	}
	/*
	 * When we work as primary and secondary is missing we will increase
	 * localcnt in our metadata. When secondary is connected and synced
	 * we make localcnt be equal to remotecnt, which means nodes are more
	 * or less in sync.
	 * Split-brain condition is when both nodes are not able to communicate
	 * and are both configured as primary nodes. In turn, they can both
	 * make incompatible changes to the data and we have to detect that.
	 * Under split-brain condition we will increase our localcnt on first
	 * write and remote node will increase its localcnt on first write.
	 * When we connect we can see that primary's localcnt is greater than
	 * our remotecnt (primary was modified while we weren't watching) and
	 * our localcnt is greater than primary's remotecnt (we were modified
	 * while primary wasn't watching).
	 * There are many possible combinations which are all gathered below.
	 * Don't pay too much attention to exact numbers, the more important
	 * is to compare them. We compare secondary's local with primary's
	 * remote and secondary's remote with primary's local.
	 * Note that every case where primary's localcnt is smaller than
	 * secondary's remotecnt and where secondary's localcnt is smaller than
	 * primary's remotecnt should be impossible in practise. We will perform
	 * full synchronization then. Those cases are marked with an asterisk.
	 * Regular synchronization means that only extents marked as dirty are
	 * synchronized (regular synchronization).
	 *
	 * SECONDARY METADATA PRIMARY METADATA
	 * local=3 remote=3   local=2 remote=2*  ?! Full sync from secondary.
	 * local=3 remote=3   local=2 remote=3*  ?! Full sync from primary.
	 * local=3 remote=3   local=2 remote=4*  ?! Full sync from primary.
	 * local=3 remote=3   local=3 remote=2   Primary is out-of-date,
	 *                                       regular sync from secondary.
	 * local=3 remote=3   local=3 remote=3   Regular sync just in case.
	 * local=3 remote=3   local=3 remote=4*  ?! Full sync from primary.
	 * local=3 remote=3   local=4 remote=2   Split-brain condition.
	 * local=3 remote=3   local=4 remote=3   Secondary out-of-date,
	 *                                       regular sync from primary.
	 * local=3 remote=3   local=4 remote=4*  ?! Full sync from primary.
	 */
	if (res->hr_resuid == 0) {
		/*
		 * Provider is used for the first time. If primary node done no
		 * writes yet as well (we will find "virgin" argument) then
		 * there is no need to synchronize anything. If primary node
		 * done any writes already we have to synchronize everything.
		 */
		PJDLOG_ASSERT(res->hr_secondary_localcnt == 0);
		res->hr_resuid = resuid;
		if (metadata_write(res) < 0)
			exit(EX_NOINPUT);
		if (nv_exists(nvin, "virgin")) {
			free(map);
			map = NULL;
			mapsize = 0;
		} else {
			memset(map, 0xff, mapsize);
		}
		nv_add_int8(nvout, 1, "virgin");
		nv_add_uint8(nvout, HAST_SYNCSRC_PRIMARY, "syncsrc");
	} else if (res->hr_resuid != resuid) {
		char errmsg[256];

		free(map);
		(void)snprintf(errmsg, sizeof(errmsg),
		    "Resource unique ID mismatch (primary=%ju, secondary=%ju).",
		    (uintmax_t)resuid, (uintmax_t)res->hr_resuid);
		pjdlog_error("%s", errmsg);
		nv_add_string(nvout, errmsg, "errmsg");
		if (hast_proto_send(res, res->hr_remotein, nvout, NULL, 0) < 0) {
			pjdlog_exit(EX_TEMPFAIL, "Unable to send response to %s",
			    res->hr_remoteaddr);
		}
		nv_free(nvout);
		exit(EX_CONFIG);
	} else if (
	    /* Is primary out-of-date? */
	    (res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
	     res->hr_secondary_remotecnt == res->hr_primary_localcnt) ||
	    /* Are the nodes more or less in sync? */
	    (res->hr_secondary_localcnt == res->hr_primary_remotecnt &&
	     res->hr_secondary_remotecnt == res->hr_primary_localcnt) ||
	    /* Is secondary out-of-date? */
	    (res->hr_secondary_localcnt == res->hr_primary_remotecnt &&
	     res->hr_secondary_remotecnt < res->hr_primary_localcnt)) {
		/*
		 * Nodes are more or less in sync or one of the nodes is
		 * out-of-date.
		 * It doesn't matter at this point which one, we just have to
		 * send out local bitmap to the remote node.
		 */
		if (pread(res->hr_localfd, map, mapsize, METADATA_SIZE) !=
		    (ssize_t)mapsize) {
			pjdlog_exit(LOG_ERR, "Unable to read activemap");
		}
		if (res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
		     res->hr_secondary_remotecnt == res->hr_primary_localcnt) {
			/* Primary is out-of-date, sync from secondary. */
			nv_add_uint8(nvout, HAST_SYNCSRC_SECONDARY, "syncsrc");
		} else {
			/*
			 * Secondary is out-of-date or counts match.
			 * Sync from primary.
			 */
			nv_add_uint8(nvout, HAST_SYNCSRC_PRIMARY, "syncsrc");
		}
	} else if (res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
	     res->hr_primary_localcnt > res->hr_secondary_remotecnt) {
		/*
		 * Not good, we have split-brain condition.
		 */
		free(map);
		pjdlog_error("Split-brain detected, exiting.");
		nv_add_string(nvout, "Split-brain condition!", "errmsg");
		if (hast_proto_send(res, res->hr_remotein, nvout, NULL, 0) < 0) {
			pjdlog_exit(EX_TEMPFAIL, "Unable to send response to %s",
			    res->hr_remoteaddr);
		}
		nv_free(nvout);
		/* Exit on split-brain. */
		event_send(res, EVENT_SPLITBRAIN);
		exit(EX_CONFIG);
	} else /* if (res->hr_secondary_localcnt < res->hr_primary_remotecnt ||
	    res->hr_primary_localcnt < res->hr_secondary_remotecnt) */ {
		/*
		 * This should never happen in practise, but we will perform
		 * full synchronization.
		 */
		PJDLOG_ASSERT(res->hr_secondary_localcnt < res->hr_primary_remotecnt ||
		    res->hr_primary_localcnt < res->hr_secondary_remotecnt);
		mapsize = activemap_calc_ondisk_size(res->hr_local_mediasize -
		    METADATA_SIZE, res->hr_extentsize,
		    res->hr_local_sectorsize);
		memset(map, 0xff, mapsize);
		if (res->hr_secondary_localcnt > res->hr_primary_remotecnt) {
			/* In this one of five cases sync from secondary. */
			nv_add_uint8(nvout, HAST_SYNCSRC_SECONDARY, "syncsrc");
		} else {
			/* For the rest four cases sync from primary. */
			nv_add_uint8(nvout, HAST_SYNCSRC_PRIMARY, "syncsrc");
		}
		pjdlog_warning("This should never happen, asking for full synchronization (primary(local=%ju, remote=%ju), secondary(local=%ju, remote=%ju)).",
		    (uintmax_t)res->hr_primary_localcnt,
		    (uintmax_t)res->hr_primary_remotecnt,
		    (uintmax_t)res->hr_secondary_localcnt,
		    (uintmax_t)res->hr_secondary_remotecnt);
	}
	nv_add_uint32(nvout, (uint32_t)mapsize, "mapsize");
	if (hast_proto_send(res, res->hr_remotein, nvout, map, mapsize) < 0) {
		pjdlog_exit(EX_TEMPFAIL, "Unable to send activemap to %s",
		    res->hr_remoteaddr);
	}
	if (map != NULL)
		free(map);
	nv_free(nvout);
#ifdef notyet
	/* Setup direction. */
	if (proto_recv(res->hr_remotein, NULL, 0) == -1)
		pjdlog_errno(LOG_WARNING, "Unable to set connection direction");
#endif
}