/*
* The incoming or outgoing socket of a node can be read from. We try to
* connect to peer nodes asynchronously, so we can get ECONNREFUSED errors
* here.
*/
static void
proto_read(int fd, short revents, void *arg)
{
char buf[sizeof(struct proto_msg) + 1];
struct proto_msg *msg = (void *)buf;
struct node *node = arg;
ssize_t ret;
buf[sizeof(struct proto_msg)] = 0;
for(;;) {
ret = read(fd, buf, sizeof(struct proto_msg));
if (ret == -1) {
if (errno == EAGAIN || errno == EWOULDBLOCK)
return;
if (errno == ECONNREFUSED)
ret = 0;
}
if (ret == 0) {
proto_close(fd, node);
return;
}
if (ret != sizeof(struct proto_msg))
fail(NULL);
if (verbose) {
printf("< %u %s", node->nodeid,
msg_name(ntohs(msg->msg)));
if (msg->lockspace_name)
printf(" %s", msg->lockspace_name);
printf("\n");
fflush(stdout);
}
switch(ntohs(msg->msg)) {
case MSG_CLOSE:
proto_close(fd, node);
return;
case MSG_LOCKSPACE_STOPPED:
proto_lockspace_stopped(node, msg->lockspace_name);
break;
case MSG_STOP_LOCKSPACE:
proto_stop_lockspace(node, msg->lockspace_name);
break;
case MSG_JOIN_LOCKSPACE:
proto_join_lockspace(node, msg->lockspace_name);
break;
case MSG_LEAVE_LOCKSPACE:
proto_leave_lockspace(node, msg->lockspace_name);
break;
default:
failf("Unknown message %u received", ntohs(msg->msg));
}
}
}
static void
sender_disconnect(void)
{
rw_wlock(&adist_remote_lock);
/*
* Check for a race between dropping rlock and acquiring wlock -
* another thread can close connection in-between.
*/
if (adhost->adh_remote == NULL) {
rw_unlock(&adist_remote_lock);
return;
}
pjdlog_debug(2, "Closing connection to %s.", adhost->adh_remoteaddr);
proto_close(adhost->adh_remote);
mtx_lock(&adist_remote_mtx);
adhost->adh_remote = NULL;
adhost->adh_reset = true;
adhost->adh_trail_name[0] = '\0';
adhost->adh_trail_offset = 0;
mtx_unlock(&adist_remote_mtx);
rw_unlock(&adist_remote_lock);
pjdlog_warning("Disconnected from %s.", adhost->adh_remoteaddr);
/* Move all in-flight requests back onto free list. */
mtx_lock(&adist_free_list_lock);
mtx_lock(&adist_send_list_lock);
TAILQ_CONCAT(&adist_free_list, &adist_send_list, adr_next);
mtx_unlock(&adist_send_list_lock);
mtx_lock(&adist_recv_list_lock);
TAILQ_CONCAT(&adist_free_list, &adist_recv_list, adr_next);
mtx_unlock(&adist_recv_list_lock);
mtx_unlock(&adist_free_list_lock);
}
static void
tls_close(void *ctx)
{
struct tls_ctx *tlsctx = ctx;
PJDLOG_ASSERT(tlsctx != NULL);
PJDLOG_ASSERT(tlsctx->tls_magic == TLS_CTX_MAGIC);
if (tlsctx->tls_sock != NULL) {
proto_close(tlsctx->tls_sock);
tlsctx->tls_sock = NULL;
}
if (tlsctx->tls_tcp != NULL) {
proto_close(tlsctx->tls_tcp);
tlsctx->tls_tcp = NULL;
}
tlsctx->tls_side = 0;
tlsctx->tls_magic = 0;
free(tlsctx);
}
static void
tls_call_exec_client(struct proto_conn *sock, const char *srcaddr,
const char *dstaddr, int timeout)
{
char *timeoutstr, *startfdstr, *debugstr;
int startfd;
/* Declare that we are receiver. */
proto_recv(sock, NULL, 0);
if (pjdlog_mode_get() == PJDLOG_MODE_STD)
startfd = 3;
else /* if (pjdlog_mode_get() == PJDLOG_MODE_SYSLOG) */
startfd = 0;
if (proto_descriptor(sock) != startfd) {
/* Move socketpair descriptor to descriptor number startfd. */
if (dup2(proto_descriptor(sock), startfd) == -1)
pjdlog_exit(EX_OSERR, "dup2() failed");
proto_close(sock);
} else {
/*
* The FD_CLOEXEC is cleared by dup2(2), so when we not
* call it, we have to clear it by hand in case it is set.
*/
if (fcntl(startfd, F_SETFD, 0) == -1)
pjdlog_exit(EX_OSERR, "fcntl() failed");
}
closefrom(startfd + 1);
if (asprintf(&startfdstr, "%d", startfd) == -1)
pjdlog_exit(EX_TEMPFAIL, "asprintf() failed");
if (timeout == -1)
timeout = TLS_DEFAULT_TIMEOUT;
if (asprintf(&timeoutstr, "%d", timeout) == -1)
pjdlog_exit(EX_TEMPFAIL, "asprintf() failed");
if (asprintf(&debugstr, "%d", pjdlog_debug_get()) == -1)
pjdlog_exit(EX_TEMPFAIL, "asprintf() failed");
execl(proto_get("execpath"), proto_get("execpath"), "proto", "tls",
proto_get("user"), "client", startfdstr,
srcaddr == NULL ? "" : srcaddr, dstaddr,
proto_get("tls:fingerprint"), proto_get("tcp:port"), timeoutstr,
debugstr, NULL);
pjdlog_exit(EX_SOFTWARE, "execl() failed");
}
void
hastd_secondary(struct hast_resource *res, struct nv *nvin)
{
sigset_t mask;
pthread_t td;
pid_t pid;
int error, mode, debuglevel;
/*
* Create communication channel between parent and child.
*/
if (proto_client(NULL, "socketpair://", &res->hr_ctrl) < 0) {
KEEP_ERRNO((void)pidfile_remove(pfh));
pjdlog_exit(EX_OSERR,
"Unable to create control sockets between parent and child");
}
/*
* Create communication channel between child and parent.
*/
if (proto_client(NULL, "socketpair://", &res->hr_event) < 0) {
KEEP_ERRNO((void)pidfile_remove(pfh));
pjdlog_exit(EX_OSERR,
"Unable to create event sockets between child and parent");
}
pid = fork();
if (pid < 0) {
KEEP_ERRNO((void)pidfile_remove(pfh));
pjdlog_exit(EX_OSERR, "Unable to fork");
}
if (pid > 0) {
/* This is parent. */
proto_close(res->hr_remotein);
res->hr_remotein = NULL;
proto_close(res->hr_remoteout);
res->hr_remoteout = NULL;
/* Declare that we are receiver. */
proto_recv(res->hr_event, NULL, 0);
/* Declare that we are sender. */
proto_send(res->hr_ctrl, NULL, 0);
res->hr_workerpid = pid;
return;
}
gres = res;
mode = pjdlog_mode_get();
debuglevel = pjdlog_debug_get();
/* Declare that we are sender. */
proto_send(res->hr_event, NULL, 0);
/* Declare that we are receiver. */
proto_recv(res->hr_ctrl, NULL, 0);
descriptors_cleanup(res);
descriptors_assert(res, mode);
pjdlog_init(mode);
pjdlog_debug_set(debuglevel);
pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role));
setproctitle("%s (%s)", res->hr_name, role2str(res->hr_role));
PJDLOG_VERIFY(sigemptyset(&mask) == 0);
PJDLOG_VERIFY(sigprocmask(SIG_SETMASK, &mask, NULL) == 0);
/* Error in setting timeout is not critical, but why should it fail? */
if (proto_timeout(res->hr_remotein, 2 * HAST_KEEPALIVE) < 0)
pjdlog_errno(LOG_WARNING, "Unable to set connection timeout");
if (proto_timeout(res->hr_remoteout, res->hr_timeout) < 0)
pjdlog_errno(LOG_WARNING, "Unable to set connection timeout");
init_local(res);
init_environment();
if (drop_privs(res) != 0)
exit(EX_CONFIG);
pjdlog_info("Privileges successfully dropped.");
/*
* Create the control thread before sending any event to the parent,
* as we can deadlock when parent sends control request to worker,
* but worker has no control thread started yet, so parent waits.
* In the meantime worker sends an event to the parent, but parent
* is unable to handle the event, because it waits for control
* request response.
*/
error = pthread_create(&td, NULL, ctrl_thread, res);
PJDLOG_ASSERT(error == 0);
init_remote(res, nvin);
event_send(res, EVENT_CONNECT);
error = pthread_create(&td, NULL, recv_thread, res);
PJDLOG_ASSERT(error == 0);
error = pthread_create(&td, NULL, disk_thread, res);
PJDLOG_ASSERT(error == 0);
(void)send_thread(res);
}
static int
tls_accept(void *ctx, void **newctxp)
{
struct tls_ctx *tlsctx = ctx;
struct tls_ctx *newtlsctx;
struct proto_conn *sock, *tcp;
pid_t pid;
int error;
PJDLOG_ASSERT(tlsctx != NULL);
PJDLOG_ASSERT(tlsctx->tls_magic == TLS_CTX_MAGIC);
PJDLOG_ASSERT(tlsctx->tls_side == TLS_SIDE_SERVER_LISTEN);
if (proto_connect(NULL, "socketpair://", -1, &sock) == -1)
return (errno);
/* Accept TCP connection. */
if (proto_accept(tlsctx->tls_tcp, &tcp) == -1) {
error = errno;
proto_close(sock);
return (error);
}
pid = fork();
switch (pid) {
case -1:
/* Failure. */
error = errno;
proto_close(sock);
return (error);
case 0:
/* Child. */
pjdlog_prefix_set("[TLS sandbox] (server) ");
#ifdef HAVE_SETPROCTITLE
setproctitle("[TLS sandbox] (server) ");
#endif
/* Close listen socket. */
proto_close(tlsctx->tls_tcp);
tls_call_exec_server(sock, tcp);
/* NOTREACHED */
PJDLOG_ABORT("Unreachable.");
default:
/* Parent. */
newtlsctx = calloc(1, sizeof(*tlsctx));
if (newtlsctx == NULL) {
error = errno;
proto_close(sock);
proto_close(tcp);
(void)kill(pid, SIGKILL);
return (error);
}
proto_local_address(tcp, newtlsctx->tls_laddr,
sizeof(newtlsctx->tls_laddr));
PJDLOG_ASSERT(strncmp(newtlsctx->tls_laddr, "tcp://", 6) == 0);
bcopy("tls://", newtlsctx->tls_laddr, 6);
*strrchr(newtlsctx->tls_laddr, ':') = '\0';
proto_remote_address(tcp, newtlsctx->tls_raddr,
sizeof(newtlsctx->tls_raddr));
PJDLOG_ASSERT(strncmp(newtlsctx->tls_raddr, "tcp://", 6) == 0);
bcopy("tls://", newtlsctx->tls_raddr, 6);
*strrchr(newtlsctx->tls_raddr, ':') = '\0';
proto_close(tcp);
proto_recv(sock, NULL, 0);
newtlsctx->tls_sock = sock;
newtlsctx->tls_tcp = NULL;
newtlsctx->tls_wait_called = true;
newtlsctx->tls_side = TLS_SIDE_SERVER_WORK;
newtlsctx->tls_magic = TLS_CTX_MAGIC;
*newctxp = newtlsctx;
return (0);
}
}
static void
tls_call_exec_server(struct proto_conn *sock, struct proto_conn *tcp)
{
int startfd, sockfd, tcpfd, safefd;
char *startfdstr, *debugstr;
if (pjdlog_mode_get() == PJDLOG_MODE_STD)
startfd = 3;
else /* if (pjdlog_mode_get() == PJDLOG_MODE_SYSLOG) */
startfd = 0;
/* Declare that we are receiver. */
proto_send(sock, NULL, 0);
sockfd = proto_descriptor(sock);
tcpfd = proto_descriptor(tcp);
safefd = MAX(sockfd, tcpfd);
safefd = MAX(safefd, startfd);
safefd++;
/* Move sockfd and tcpfd to safe numbers first. */
if (dup2(sockfd, safefd) == -1)
pjdlog_exit(EX_OSERR, "dup2() failed");
proto_close(sock);
sockfd = safefd;
if (dup2(tcpfd, safefd + 1) == -1)
pjdlog_exit(EX_OSERR, "dup2() failed");
proto_close(tcp);
tcpfd = safefd + 1;
/* Move socketpair descriptor to descriptor number startfd. */
if (dup2(sockfd, startfd) == -1)
pjdlog_exit(EX_OSERR, "dup2() failed");
(void)close(sockfd);
/* Move tcp descriptor to descriptor number startfd + 1. */
if (dup2(tcpfd, startfd + 1) == -1)
pjdlog_exit(EX_OSERR, "dup2() failed");
(void)close(tcpfd);
closefrom(startfd + 2);
/*
* Even if FD_CLOEXEC was set on descriptors before dup2(), it should
* have been cleared on dup2(), but better be safe than sorry.
*/
if (fcntl(startfd, F_SETFD, 0) == -1)
pjdlog_exit(EX_OSERR, "fcntl() failed");
if (fcntl(startfd + 1, F_SETFD, 0) == -1)
pjdlog_exit(EX_OSERR, "fcntl() failed");
if (asprintf(&startfdstr, "%d", startfd) == -1)
pjdlog_exit(EX_TEMPFAIL, "asprintf() failed");
if (asprintf(&debugstr, "%d", pjdlog_debug_get()) == -1)
pjdlog_exit(EX_TEMPFAIL, "asprintf() failed");
execl(proto_get("execpath"), proto_get("execpath"), "proto", "tls",
proto_get("user"), "server", startfdstr, proto_get("tls:keyfile"),
proto_get("tls:certfile"), debugstr, NULL);
pjdlog_exit(EX_SOFTWARE, "execl() failed");
}
static int
tls_connect(const char *srcaddr, const char *dstaddr, int timeout, void **ctxp)
{
struct tls_ctx *tlsctx;
struct proto_conn *sock;
pid_t pid;
int error;
PJDLOG_ASSERT(srcaddr == NULL || srcaddr[0] != '\0');
PJDLOG_ASSERT(dstaddr != NULL);
PJDLOG_ASSERT(timeout >= -1);
PJDLOG_ASSERT(ctxp != NULL);
if (strncmp(dstaddr, "tls://", 6) != 0)
return (-1);
if (srcaddr != NULL && strncmp(srcaddr, "tls://", 6) != 0)
return (-1);
if (proto_connect(NULL, "socketpair://", -1, &sock) == -1)
return (errno);
#if 0
/*
* We use rfork() with the following flags to disable SIGCHLD
* delivery upon the sandbox process exit.
*/
pid = rfork(RFFDG | RFPROC | RFTSIGZMB | RFTSIGFLAGS(0));
#else
/*
* We don't use rfork() to be able to log information about sandbox
* process exiting.
*/
pid = fork();
#endif
switch (pid) {
case -1:
/* Failure. */
error = errno;
proto_close(sock);
return (error);
case 0:
/* Child. */
pjdlog_prefix_set("[TLS sandbox] (client) ");
#ifdef HAVE_SETPROCTITLE
setproctitle("[TLS sandbox] (client) ");
#endif
tls_call_exec_client(sock, srcaddr, dstaddr, timeout);
/* NOTREACHED */
default:
/* Parent. */
tlsctx = calloc(1, sizeof(*tlsctx));
if (tlsctx == NULL) {
error = errno;
proto_close(sock);
(void)kill(pid, SIGKILL);
return (error);
}
proto_send(sock, NULL, 0);
tlsctx->tls_sock = sock;
tlsctx->tls_tcp = NULL;
tlsctx->tls_side = TLS_SIDE_CLIENT;
tlsctx->tls_wait_called = false;
tlsctx->tls_magic = TLS_CTX_MAGIC;
if (timeout >= 0) {
error = tls_connect_wait(tlsctx, timeout);
if (error != 0) {
(void)kill(pid, SIGKILL);
tls_close(tlsctx);
return (error);
}
}
*ctxp = tlsctx;
return (0);
}
}
static int
sender_connect(void)
{
unsigned char rnd[32], hash[32], resp[32];
struct proto_conn *conn;
char welcome[8];
int16_t val;
val = 1;
if (proto_send(adhost->adh_conn, &val, sizeof(val)) < 0) {
pjdlog_exit(EX_TEMPFAIL,
"Unable to send connection request to parent");
}
if (proto_recv(adhost->adh_conn, &val, sizeof(val)) < 0) {
pjdlog_exit(EX_TEMPFAIL,
"Unable to receive reply to connection request from parent");
}
if (val != 0) {
errno = val;
pjdlog_errno(LOG_WARNING, "Unable to connect to %s",
adhost->adh_remoteaddr);
return (-1);
}
if (proto_connection_recv(adhost->adh_conn, true, &conn) < 0) {
pjdlog_exit(EX_TEMPFAIL,
"Unable to receive connection from parent");
}
if (proto_connect_wait(conn, adcfg->adc_timeout) < 0) {
pjdlog_errno(LOG_WARNING, "Unable to connect to %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Connected to %s.", adhost->adh_remoteaddr);
/* Error in setting timeout is not critical, but why should it fail? */
if (proto_timeout(conn, adcfg->adc_timeout) < 0)
pjdlog_errno(LOG_WARNING, "Unable to set connection timeout");
else
pjdlog_debug(1, "Timeout set to %d.", adcfg->adc_timeout);
/* Exchange welcome message, which includes version number. */
(void)snprintf(welcome, sizeof(welcome), "ADIST%02d", ADIST_VERSION);
if (proto_send(conn, welcome, sizeof(welcome)) < 0) {
pjdlog_errno(LOG_WARNING,
"Unable to send welcome message to %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Welcome message sent (%s).", welcome);
bzero(welcome, sizeof(welcome));
if (proto_recv(conn, welcome, sizeof(welcome)) < 0) {
pjdlog_errno(LOG_WARNING,
"Unable to receive welcome message from %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
if (strncmp(welcome, "ADIST", 5) != 0 || !isdigit(welcome[5]) ||
!isdigit(welcome[6]) || welcome[7] != '\0') {
pjdlog_warning("Invalid welcome message from %s.",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Welcome message received (%s).", welcome);
/*
* Receiver can only reply with version number lower or equal to
* the one we sent.
*/
adhost->adh_version = atoi(welcome + 5);
if (adhost->adh_version > ADIST_VERSION) {
pjdlog_warning("Invalid version number from %s (%d received, up to %d supported).",
adhost->adh_remoteaddr, adhost->adh_version, ADIST_VERSION);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Version %d negotiated with %s.", adhost->adh_version,
adhost->adh_remoteaddr);
if (proto_send(conn, adcfg->adc_name, sizeof(adcfg->adc_name)) == -1) {
pjdlog_errno(LOG_WARNING, "Unable to send name to %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Name (%s) sent.", adcfg->adc_name);
if (proto_recv(conn, rnd, sizeof(rnd)) == -1) {
pjdlog_errno(LOG_WARNING, "Unable to receive challenge from %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Challenge received.");
if (HMAC(EVP_sha256(), adhost->adh_password,
(int)strlen(adhost->adh_password), rnd, (int)sizeof(rnd), hash,
NULL) == NULL) {
pjdlog_warning("Unable to generate response.");
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Response generated.");
if (proto_send(conn, hash, sizeof(hash)) == -1) {
pjdlog_errno(LOG_WARNING, "Unable to send response to %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Response sent.");
if (adist_random(rnd, sizeof(rnd)) == -1) {
pjdlog_warning("Unable to generate challenge.");
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Challenge generated.");
if (proto_send(conn, rnd, sizeof(rnd)) == -1) {
pjdlog_errno(LOG_WARNING, "Unable to send challenge to %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Challenge sent.");
if (proto_recv(conn, resp, sizeof(resp)) == -1) {
pjdlog_errno(LOG_WARNING, "Unable to receive response from %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Response received.");
if (HMAC(EVP_sha256(), adhost->adh_password,
(int)strlen(adhost->adh_password), rnd, (int)sizeof(rnd), hash,
NULL) == NULL) {
pjdlog_warning("Unable to generate hash.");
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Hash generated.");
if (memcmp(resp, hash, sizeof(hash)) != 0) {
pjdlog_warning("Invalid response from %s (wrong password?).",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_info("Receiver authenticated.");
if (proto_recv(conn, &adhost->adh_trail_offset,
sizeof(adhost->adh_trail_offset)) == -1) {
pjdlog_errno(LOG_WARNING,
"Unable to receive size of the most recent trail file from %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
adhost->adh_trail_offset = le64toh(adhost->adh_trail_offset);
if (proto_recv(conn, &adhost->adh_trail_name,
sizeof(adhost->adh_trail_name)) == -1) {
pjdlog_errno(LOG_WARNING,
"Unable to receive name of the most recent trail file from %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Trail name (%s) and offset (%ju) received.",
adhost->adh_trail_name, (uintmax_t)adhost->adh_trail_offset);
rw_wlock(&adist_remote_lock);
mtx_lock(&adist_remote_mtx);
PJDLOG_ASSERT(adhost->adh_remote == NULL);
PJDLOG_ASSERT(conn != NULL);
adhost->adh_remote = conn;
mtx_unlock(&adist_remote_mtx);
rw_unlock(&adist_remote_lock);
cv_signal(&adist_remote_cond);
return (0);
}
