static int
ssl_check_error(SSL *ssl, int ret)
{
int error;
error = SSL_get_error(ssl, ret);
switch (error) {
case SSL_ERROR_NONE:
return (0);
case SSL_ERROR_WANT_READ:
pjdlog_debug(2, "SSL_ERROR_WANT_READ");
return (-1);
case SSL_ERROR_WANT_WRITE:
pjdlog_debug(2, "SSL_ERROR_WANT_WRITE");
return (-1);
case SSL_ERROR_ZERO_RETURN:
pjdlog_exitx(EX_OK, "Connection closed.");
case SSL_ERROR_SYSCALL:
ssl_log_errors();
pjdlog_exitx(EX_TEMPFAIL, "SSL I/O error.");
case SSL_ERROR_SSL:
ssl_log_errors();
pjdlog_exitx(EX_TEMPFAIL, "SSL protocol error.");
default:
ssl_log_errors();
pjdlog_exitx(EX_TEMPFAIL, "Unknown SSL error (%d).", error);
}
}
static void
init_environment(void)
{
struct adreq *adreq;
unsigned int ii;
rw_init(&adist_remote_lock);
mtx_init(&adist_remote_mtx);
cv_init(&adist_remote_cond);
TAILQ_INIT(&adist_free_list);
mtx_init(&adist_free_list_lock);
cv_init(&adist_free_list_cond);
TAILQ_INIT(&adist_send_list);
mtx_init(&adist_send_list_lock);
cv_init(&adist_send_list_cond);
TAILQ_INIT(&adist_recv_list);
mtx_init(&adist_recv_list_lock);
cv_init(&adist_recv_list_cond);
for (ii = 0; ii < ADIST_QUEUE_SIZE; ii++) {
adreq = malloc(sizeof(*adreq) + ADIST_BUF_SIZE);
if (adreq == NULL) {
pjdlog_exitx(EX_TEMPFAIL,
"Unable to allocate %zu bytes of memory for adreq object.",
sizeof(*adreq) + ADIST_BUF_SIZE);
}
adreq->adr_byteorder = ADIST_BYTEORDER;
adreq->adr_cmd = ADIST_CMD_UNDEFINED;
adreq->adr_seq = 0;
adreq->adr_datasize = 0;
TAILQ_INSERT_TAIL(&adist_free_list, adreq, adr_next);
}
}
static void
usage(void)
{
pjdlog_exitx(1,
"usage: decryptcore [-Lv] -p privatekeyfile -k keyfile -e encryptedcore -c core\n"
" decryptcore [-Lv] [-d crashdir] -p privatekeyfile -n dumpnr");
}
static void
init_environment(void)
{
struct hio *hio;
unsigned int ii;
/*
* Initialize lists, their locks and theirs condition variables.
*/
TAILQ_INIT(&hio_free_list);
mtx_init(&hio_free_list_lock);
cv_init(&hio_free_list_cond);
TAILQ_INIT(&hio_disk_list);
mtx_init(&hio_disk_list_lock);
cv_init(&hio_disk_list_cond);
TAILQ_INIT(&hio_send_list);
mtx_init(&hio_send_list_lock);
cv_init(&hio_send_list_cond);
/*
* Allocate requests pool and initialize requests.
*/
for (ii = 0; ii < HAST_HIO_MAX; ii++) {
hio = malloc(sizeof(*hio));
if (hio == NULL) {
pjdlog_exitx(EX_TEMPFAIL,
"Unable to allocate memory (%zu bytes) for hio request.",
sizeof(*hio));
}
hio->hio_data = malloc(MAXPHYS);
if (hio->hio_data == NULL) {
pjdlog_exitx(EX_TEMPFAIL,
"Unable to allocate memory (%zu bytes) for gctl_data.",
(size_t)MAXPHYS);
}
hio_clear(hio);
TAILQ_INSERT_HEAD(&hio_free_list, hio, hio_next);
hio_free_list_size++;
}
}
static void
tls_certificate_verify(SSL *ssl, const char *fingerprint)
{
unsigned char md[EVP_MAX_MD_SIZE];
char mdstr[sizeof("SHA256=") - 1 + EVP_MAX_MD_SIZE * 3];
char *mdstrp;
unsigned int i, mdsize;
X509 *cert;
if (fingerprint[0] == '\0') {
pjdlog_debug(1, "No fingerprint verification requested.");
return;
}
cert = SSL_get_peer_certificate(ssl);
if (cert == NULL)
pjdlog_exitx(EX_TEMPFAIL, "No peer certificate received.");
if (X509_digest(cert, EVP_sha256(), md, &mdsize) != 1)
pjdlog_exitx(EX_TEMPFAIL, "X509_digest() failed.");
PJDLOG_ASSERT(mdsize <= EVP_MAX_MD_SIZE);
X509_free(cert);
(void)strlcpy(mdstr, "SHA256=", sizeof(mdstr));
mdstrp = mdstr + strlen(mdstr);
for (i = 0; i < mdsize; i++) {
PJDLOG_VERIFY(mdstrp + 3 <= mdstr + sizeof(mdstr));
(void)sprintf(mdstrp, "%02hhX:", md[i]);
mdstrp += 3;
}
/* Clear last colon. */
mdstrp[-1] = '\0';
if (strcasecmp(mdstr, fingerprint) != 0) {
pjdlog_exitx(EX_NOPERM,
"Finger print doesn't match. Received \"%s\", expected \"%s\"",
mdstr, fingerprint);
}
}
static void
tls_exec_server(const char *user, int startfd, const char *privkey,
const char *cert, int debuglevel)
{
SSL_CTX *sslctx;
SSL *ssl;
int sockfd, tcpfd, ret;
pjdlog_debug_set(debuglevel);
pjdlog_prefix_set("[TLS sandbox] (server) ");
#ifdef HAVE_SETPROCTITLE
setproctitle("[TLS sandbox] (server) ");
#endif
sockfd = startfd;
tcpfd = startfd + 1;
SSL_load_error_strings();
SSL_library_init();
sslctx = SSL_CTX_new(TLSv1_server_method());
if (sslctx == NULL)
pjdlog_exitx(EX_TEMPFAIL, "SSL_CTX_new() failed.");
SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3);
ssl = SSL_new(sslctx);
if (ssl == NULL)
pjdlog_exitx(EX_TEMPFAIL, "SSL_new() failed.");
if (SSL_use_RSAPrivateKey_file(ssl, privkey, SSL_FILETYPE_PEM) != 1) {
ssl_log_errors();
pjdlog_exitx(EX_CONFIG,
"SSL_use_RSAPrivateKey_file(%s) failed.", privkey);
}
if (SSL_use_certificate_file(ssl, cert, SSL_FILETYPE_PEM) != 1) {
ssl_log_errors();
pjdlog_exitx(EX_CONFIG, "SSL_use_certificate_file(%s) failed.",
cert);
}
if (sandbox(user, true, "proto_tls server") != 0)
pjdlog_exitx(EX_CONFIG, "Unable to sandbox TLS server.");
pjdlog_debug(1, "Privileges successfully dropped.");
nonblock(sockfd);
nonblock(tcpfd);
if (SSL_set_fd(ssl, tcpfd) != 1)
pjdlog_exitx(EX_TEMPFAIL, "SSL_set_fd() failed.");
ret = SSL_accept(ssl);
ssl_check_error(ssl, ret);
tls_loop(sockfd, ssl);
}
static void
init_remote(struct hast_resource *res, struct nv *nvin)
{
uint64_t resuid;
struct nv *nvout;
unsigned char *map;
size_t mapsize;
#ifdef notyet
/* Setup direction. */
if (proto_send(res->hr_remoteout, NULL, 0) == -1)
pjdlog_errno(LOG_WARNING, "Unable to set connection direction");
#endif
map = NULL;
mapsize = 0;
nvout = nv_alloc();
nv_add_int64(nvout, (int64_t)res->hr_datasize, "datasize");
nv_add_int32(nvout, (int32_t)res->hr_extentsize, "extentsize");
resuid = nv_get_uint64(nvin, "resuid");
res->hr_primary_localcnt = nv_get_uint64(nvin, "localcnt");
res->hr_primary_remotecnt = nv_get_uint64(nvin, "remotecnt");
nv_add_uint64(nvout, res->hr_secondary_localcnt, "localcnt");
nv_add_uint64(nvout, res->hr_secondary_remotecnt, "remotecnt");
mapsize = activemap_calc_ondisk_size(res->hr_local_mediasize -
METADATA_SIZE, res->hr_extentsize, res->hr_local_sectorsize);
map = malloc(mapsize);
if (map == NULL) {
pjdlog_exitx(EX_TEMPFAIL,
"Unable to allocate memory (%zu bytes) for activemap.",
mapsize);
}
/*
* When we work as primary and secondary is missing we will increase
* localcnt in our metadata. When secondary is connected and synced
* we make localcnt be equal to remotecnt, which means nodes are more
* or less in sync.
* Split-brain condition is when both nodes are not able to communicate
* and are both configured as primary nodes. In turn, they can both
* make incompatible changes to the data and we have to detect that.
* Under split-brain condition we will increase our localcnt on first
* write and remote node will increase its localcnt on first write.
* When we connect we can see that primary's localcnt is greater than
* our remotecnt (primary was modified while we weren't watching) and
* our localcnt is greater than primary's remotecnt (we were modified
* while primary wasn't watching).
* There are many possible combinations which are all gathered below.
* Don't pay too much attention to exact numbers, the more important
* is to compare them. We compare secondary's local with primary's
* remote and secondary's remote with primary's local.
* Note that every case where primary's localcnt is smaller than
* secondary's remotecnt and where secondary's localcnt is smaller than
* primary's remotecnt should be impossible in practise. We will perform
* full synchronization then. Those cases are marked with an asterisk.
* Regular synchronization means that only extents marked as dirty are
* synchronized (regular synchronization).
*
* SECONDARY METADATA PRIMARY METADATA
* local=3 remote=3 local=2 remote=2* ?! Full sync from secondary.
* local=3 remote=3 local=2 remote=3* ?! Full sync from primary.
* local=3 remote=3 local=2 remote=4* ?! Full sync from primary.
* local=3 remote=3 local=3 remote=2 Primary is out-of-date,
* regular sync from secondary.
* local=3 remote=3 local=3 remote=3 Regular sync just in case.
* local=3 remote=3 local=3 remote=4* ?! Full sync from primary.
* local=3 remote=3 local=4 remote=2 Split-brain condition.
* local=3 remote=3 local=4 remote=3 Secondary out-of-date,
* regular sync from primary.
* local=3 remote=3 local=4 remote=4* ?! Full sync from primary.
*/
if (res->hr_resuid == 0) {
/*
* Provider is used for the first time. If primary node done no
* writes yet as well (we will find "virgin" argument) then
* there is no need to synchronize anything. If primary node
* done any writes already we have to synchronize everything.
*/
PJDLOG_ASSERT(res->hr_secondary_localcnt == 0);
res->hr_resuid = resuid;
if (metadata_write(res) < 0)
exit(EX_NOINPUT);
if (nv_exists(nvin, "virgin")) {
free(map);
map = NULL;
mapsize = 0;
} else {
memset(map, 0xff, mapsize);
}
nv_add_int8(nvout, 1, "virgin");
nv_add_uint8(nvout, HAST_SYNCSRC_PRIMARY, "syncsrc");
} else if (res->hr_resuid != resuid) {
char errmsg[256];
free(map);
(void)snprintf(errmsg, sizeof(errmsg),
"Resource unique ID mismatch (primary=%ju, secondary=%ju).",
(uintmax_t)resuid, (uintmax_t)res->hr_resuid);
pjdlog_error("%s", errmsg);
nv_add_string(nvout, errmsg, "errmsg");
if (hast_proto_send(res, res->hr_remotein, nvout, NULL, 0) < 0) {
pjdlog_exit(EX_TEMPFAIL, "Unable to send response to %s",
res->hr_remoteaddr);
}
nv_free(nvout);
exit(EX_CONFIG);
} else if (
/* Is primary out-of-date? */
(res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
res->hr_secondary_remotecnt == res->hr_primary_localcnt) ||
/* Are the nodes more or less in sync? */
(res->hr_secondary_localcnt == res->hr_primary_remotecnt &&
res->hr_secondary_remotecnt == res->hr_primary_localcnt) ||
/* Is secondary out-of-date? */
(res->hr_secondary_localcnt == res->hr_primary_remotecnt &&
res->hr_secondary_remotecnt < res->hr_primary_localcnt)) {
/*
* Nodes are more or less in sync or one of the nodes is
* out-of-date.
* It doesn't matter at this point which one, we just have to
* send out local bitmap to the remote node.
*/
if (pread(res->hr_localfd, map, mapsize, METADATA_SIZE) !=
(ssize_t)mapsize) {
pjdlog_exit(LOG_ERR, "Unable to read activemap");
}
if (res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
res->hr_secondary_remotecnt == res->hr_primary_localcnt) {
/* Primary is out-of-date, sync from secondary. */
nv_add_uint8(nvout, HAST_SYNCSRC_SECONDARY, "syncsrc");
} else {
/*
* Secondary is out-of-date or counts match.
* Sync from primary.
*/
nv_add_uint8(nvout, HAST_SYNCSRC_PRIMARY, "syncsrc");
}
} else if (res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
res->hr_primary_localcnt > res->hr_secondary_remotecnt) {
/*
* Not good, we have split-brain condition.
*/
free(map);
pjdlog_error("Split-brain detected, exiting.");
nv_add_string(nvout, "Split-brain condition!", "errmsg");
if (hast_proto_send(res, res->hr_remotein, nvout, NULL, 0) < 0) {
pjdlog_exit(EX_TEMPFAIL, "Unable to send response to %s",
res->hr_remoteaddr);
}
nv_free(nvout);
/* Exit on split-brain. */
event_send(res, EVENT_SPLITBRAIN);
exit(EX_CONFIG);
} else /* if (res->hr_secondary_localcnt < res->hr_primary_remotecnt ||
res->hr_primary_localcnt < res->hr_secondary_remotecnt) */ {
/*
* This should never happen in practise, but we will perform
* full synchronization.
*/
PJDLOG_ASSERT(res->hr_secondary_localcnt < res->hr_primary_remotecnt ||
res->hr_primary_localcnt < res->hr_secondary_remotecnt);
mapsize = activemap_calc_ondisk_size(res->hr_local_mediasize -
METADATA_SIZE, res->hr_extentsize,
res->hr_local_sectorsize);
memset(map, 0xff, mapsize);
if (res->hr_secondary_localcnt > res->hr_primary_remotecnt) {
/* In this one of five cases sync from secondary. */
nv_add_uint8(nvout, HAST_SYNCSRC_SECONDARY, "syncsrc");
} else {
/* For the rest four cases sync from primary. */
nv_add_uint8(nvout, HAST_SYNCSRC_PRIMARY, "syncsrc");
}
pjdlog_warning("This should never happen, asking for full synchronization (primary(local=%ju, remote=%ju), secondary(local=%ju, remote=%ju)).",
(uintmax_t)res->hr_primary_localcnt,
(uintmax_t)res->hr_primary_remotecnt,
(uintmax_t)res->hr_secondary_localcnt,
(uintmax_t)res->hr_secondary_remotecnt);
}
nv_add_uint32(nvout, (uint32_t)mapsize, "mapsize");
if (hast_proto_send(res, res->hr_remotein, nvout, map, mapsize) < 0) {
pjdlog_exit(EX_TEMPFAIL, "Unable to send activemap to %s",
res->hr_remoteaddr);
}
if (map != NULL)
free(map);
nv_free(nvout);
#ifdef notyet
/* Setup direction. */
if (proto_recv(res->hr_remotein, NULL, 0) == -1)
pjdlog_errno(LOG_WARNING, "Unable to set connection direction");
#endif
}
static void
tls_exec_client(const char *user, int startfd, const char *srcaddr,
const char *dstaddr, const char *fingerprint, const char *defport,
int timeout, int debuglevel)
{
struct proto_conn *tcp;
char *saddr, *daddr;
SSL_CTX *sslctx;
SSL *ssl;
long ret;
int sockfd, tcpfd;
uint8_t connected;
pjdlog_debug_set(debuglevel);
pjdlog_prefix_set("[TLS sandbox] (client) ");
#ifdef HAVE_SETPROCTITLE
setproctitle("[TLS sandbox] (client) ");
#endif
proto_set("tcp:port", defport);
sockfd = startfd;
/* Change tls:// to tcp://. */
if (srcaddr == NULL) {
saddr = NULL;
} else {
saddr = strdup(srcaddr);
if (saddr == NULL)
pjdlog_exitx(EX_TEMPFAIL, "Unable to allocate memory.");
bcopy("tcp://", saddr, 6);
}
daddr = strdup(dstaddr);
if (daddr == NULL)
pjdlog_exitx(EX_TEMPFAIL, "Unable to allocate memory.");
bcopy("tcp://", daddr, 6);
/* Establish TCP connection. */
if (proto_connect(saddr, daddr, timeout, &tcp) == -1)
exit(EX_TEMPFAIL);
SSL_load_error_strings();
SSL_library_init();
/*
* TODO: On FreeBSD we could move this below sandbox() once libc and
* libcrypto use sysctl kern.arandom to obtain random data
* instead of /dev/urandom and friends.
*/
sslctx = SSL_CTX_new(TLSv1_client_method());
if (sslctx == NULL)
pjdlog_exitx(EX_TEMPFAIL, "SSL_CTX_new() failed.");
if (sandbox(user, true, "proto_tls client: %s", dstaddr) != 0)
pjdlog_exitx(EX_CONFIG, "Unable to sandbox TLS client.");
pjdlog_debug(1, "Privileges successfully dropped.");
SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3);
/* Load CA certs. */
/* TODO */
//SSL_CTX_load_verify_locations(sslctx, cacerts_file, NULL);
ssl = SSL_new(sslctx);
if (ssl == NULL)
pjdlog_exitx(EX_TEMPFAIL, "SSL_new() failed.");
tcpfd = proto_descriptor(tcp);
block(tcpfd);
if (SSL_set_fd(ssl, tcpfd) != 1)
pjdlog_exitx(EX_TEMPFAIL, "SSL_set_fd() failed.");
ret = SSL_connect(ssl);
ssl_check_error(ssl, (int)ret);
nonblock(sockfd);
nonblock(tcpfd);
tls_certificate_verify(ssl, fingerprint);
/*
* The following byte is send to make proto_connect_wait() to work.
*/
connected = 1;
for (;;) {
switch (send(sockfd, &connected, sizeof(connected), 0)) {
case -1:
if (errno == EINTR || errno == ENOBUFS)
continue;
if (errno == EAGAIN) {
(void)wait_for_fd(sockfd, -1);
continue;
}
pjdlog_exit(EX_TEMPFAIL, "send() failed");
case 0:
pjdlog_debug(1, "Connection terminated.");
exit(0);
case 1:
break;
}
break;
}
tls_loop(sockfd, ssl);
}