static int
ssl_check_error(SSL *ssl, int ret)
{
int error;
error = SSL_get_error(ssl, ret);
switch (error) {
case SSL_ERROR_NONE:
return (0);
case SSL_ERROR_WANT_READ:
pjdlog_debug(2, "SSL_ERROR_WANT_READ");
return (-1);
case SSL_ERROR_WANT_WRITE:
pjdlog_debug(2, "SSL_ERROR_WANT_WRITE");
return (-1);
case SSL_ERROR_ZERO_RETURN:
pjdlog_exitx(EX_OK, "Connection closed.");
case SSL_ERROR_SYSCALL:
ssl_log_errors();
pjdlog_exitx(EX_TEMPFAIL, "SSL I/O error.");
case SSL_ERROR_SSL:
ssl_log_errors();
pjdlog_exitx(EX_TEMPFAIL, "SSL protocol error.");
default:
ssl_log_errors();
pjdlog_exitx(EX_TEMPFAIL, "Unknown SSL error (%d).", error);
}
}
/*
* Thread sends requests back to primary node.
*/
static void *
send_thread(void *arg)
{
struct hast_resource *res = arg;
struct nv *nvout;
struct hio *hio;
void *data;
size_t length;
for (;;) {
pjdlog_debug(2, "send: Taking request.");
QUEUE_TAKE(send, hio);
reqlog(LOG_DEBUG, 2, -1, hio, "send: (%p) Got request: ", hio);
nvout = nv_alloc();
/* Copy sequence number. */
nv_add_uint64(nvout, hio->hio_seq, "seq");
switch (hio->hio_cmd) {
case HIO_READ:
if (hio->hio_error == 0) {
data = hio->hio_data;
length = hio->hio_length;
break;
}
/*
* We send no data in case of an error.
*/
/* FALLTHROUGH */
case HIO_DELETE:
case HIO_FLUSH:
case HIO_WRITE:
data = NULL;
length = 0;
break;
default:
PJDLOG_ABORT("Unexpected command (cmd=%hhu).",
hio->hio_cmd);
}
if (hio->hio_error != 0)
nv_add_int16(nvout, hio->hio_error, "error");
if (hast_proto_send(res, res->hr_remoteout, nvout, data,
length) < 0) {
secondary_exit(EX_TEMPFAIL, "Unable to send reply.");
}
nv_free(nvout);
pjdlog_debug(2, "send: (%p) Moving request to the free queue.",
hio);
hio_clear(hio);
QUEUE_INSERT(free, hio);
}
/* NOTREACHED */
return (NULL);
}
static void
sender_disconnect(void)
{
rw_wlock(&adist_remote_lock);
/*
* Check for a race between dropping rlock and acquiring wlock -
* another thread can close connection in-between.
*/
if (adhost->adh_remote == NULL) {
rw_unlock(&adist_remote_lock);
return;
}
pjdlog_debug(2, "Closing connection to %s.", adhost->adh_remoteaddr);
proto_close(adhost->adh_remote);
mtx_lock(&adist_remote_mtx);
adhost->adh_remote = NULL;
adhost->adh_reset = true;
adhost->adh_trail_name[0] = '\0';
adhost->adh_trail_offset = 0;
mtx_unlock(&adist_remote_mtx);
rw_unlock(&adist_remote_lock);
pjdlog_warning("Disconnected from %s.", adhost->adh_remoteaddr);
/* Move all in-flight requests back onto free list. */
mtx_lock(&adist_free_list_lock);
mtx_lock(&adist_send_list_lock);
TAILQ_CONCAT(&adist_free_list, &adist_send_list, adr_next);
mtx_unlock(&adist_send_list_lock);
mtx_lock(&adist_recv_list_lock);
TAILQ_CONCAT(&adist_free_list, &adist_recv_list, adr_next);
mtx_unlock(&adist_recv_list_lock);
mtx_unlock(&adist_free_list_lock);
}
static void
tls_exec_server(const char *user, int startfd, const char *privkey,
const char *cert, int debuglevel)
{
SSL_CTX *sslctx;
SSL *ssl;
int sockfd, tcpfd, ret;
pjdlog_debug_set(debuglevel);
pjdlog_prefix_set("[TLS sandbox] (server) ");
#ifdef HAVE_SETPROCTITLE
setproctitle("[TLS sandbox] (server) ");
#endif
sockfd = startfd;
tcpfd = startfd + 1;
SSL_load_error_strings();
SSL_library_init();
sslctx = SSL_CTX_new(TLSv1_server_method());
if (sslctx == NULL)
pjdlog_exitx(EX_TEMPFAIL, "SSL_CTX_new() failed.");
SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3);
ssl = SSL_new(sslctx);
if (ssl == NULL)
pjdlog_exitx(EX_TEMPFAIL, "SSL_new() failed.");
if (SSL_use_RSAPrivateKey_file(ssl, privkey, SSL_FILETYPE_PEM) != 1) {
ssl_log_errors();
pjdlog_exitx(EX_CONFIG,
"SSL_use_RSAPrivateKey_file(%s) failed.", privkey);
}
if (SSL_use_certificate_file(ssl, cert, SSL_FILETYPE_PEM) != 1) {
ssl_log_errors();
pjdlog_exitx(EX_CONFIG, "SSL_use_certificate_file(%s) failed.",
cert);
}
if (sandbox(user, true, "proto_tls server") != 0)
pjdlog_exitx(EX_CONFIG, "Unable to sandbox TLS server.");
pjdlog_debug(1, "Privileges successfully dropped.");
nonblock(sockfd);
nonblock(tcpfd);
if (SSL_set_fd(ssl, tcpfd) != 1)
pjdlog_exitx(EX_TEMPFAIL, "SSL_set_fd() failed.");
ret = SSL_accept(ssl);
ssl_check_error(ssl, ret);
tls_loop(sockfd, ssl);
}
static void
ssl_recv_tcp_send(SSL *recvssl, int sendfd)
{
static unsigned char buf[65536];
unsigned char *ptr;
ssize_t tcpdone;
size_t todo;
int recvfd, ssldone;
recvfd = SSL_get_fd(recvssl);
PJDLOG_ASSERT(recvfd >= 0);
pjdlog_debug(2, "%s: start %d -> %d", __func__, recvfd, sendfd);
for (;;) {
ssldone = SSL_read(recvssl, buf, sizeof(buf));
pjdlog_debug(2, "%s: SSL_read() returned %d", __func__,
ssldone);
if (ssl_check_error(recvssl, ssldone) == -1)
break;
todo = (size_t)ssldone;
ptr = buf;
do {
tcpdone = send(sendfd, ptr, todo, MSG_NOSIGNAL);
pjdlog_debug(2, "%s: send() returned %zd", __func__,
tcpdone);
if (tcpdone == 0) {
pjdlog_debug(1, "Connection terminated.");
exit(0);
} else if (tcpdone == -1) {
if (errno == EINTR || errno == ENOBUFS)
continue;
if (errno == EAGAIN) {
(void)wait_for_fd(sendfd, -1);
continue;
}
pjdlog_exit(EX_TEMPFAIL, "send() failed");
}
todo -= tcpdone;
ptr += tcpdone;
} while (todo > 0);
}
pjdlog_debug(2, "%s: done %d -> %d", __func__, recvfd, sendfd);
}
static void
tcp_recv_ssl_send(int recvfd, SSL *sendssl)
{
static unsigned char buf[65536];
ssize_t tcpdone;
int sendfd, ssldone;
sendfd = SSL_get_fd(sendssl);
PJDLOG_ASSERT(sendfd >= 0);
pjdlog_debug(2, "%s: start %d -> %d", __func__, recvfd, sendfd);
for (;;) {
tcpdone = recv(recvfd, buf, sizeof(buf), 0);
pjdlog_debug(2, "%s: recv() returned %zd", __func__, tcpdone);
if (tcpdone == 0) {
pjdlog_debug(1, "Connection terminated.");
exit(0);
} else if (tcpdone == -1) {
if (errno == EINTR)
continue;
else if (errno == EAGAIN)
break;
pjdlog_exit(EX_TEMPFAIL, "recv() failed");
}
for (;;) {
ssldone = SSL_write(sendssl, buf, (int)tcpdone);
pjdlog_debug(2, "%s: send() returned %d", __func__,
ssldone);
if (ssl_check_error(sendssl, ssldone) == -1) {
(void)wait_for_fd(sendfd, -1);
continue;
}
PJDLOG_ASSERT(ssldone == tcpdone);
break;
}
}
pjdlog_debug(2, "%s: done %d -> %d", __func__, recvfd, sendfd);
}
static bool
read_thread_wait(void)
{
bool newfile = false;
mtx_lock(&adist_remote_mtx);
if (adhost->adh_reset) {
reset:
adhost->adh_reset = false;
if (trail_filefd(adist_trail) != -1)
trail_close(adist_trail);
trail_reset(adist_trail);
while (adhost->adh_remote == NULL)
cv_wait(&adist_remote_cond, &adist_remote_mtx);
trail_start(adist_trail, adhost->adh_trail_name,
adhost->adh_trail_offset);
newfile = true;
}
mtx_unlock(&adist_remote_mtx);
while (trail_filefd(adist_trail) == -1) {
newfile = true;
wait_for_dir();
/*
* We may have been disconnected and reconnected in the
* meantime, check if reset is set.
*/
mtx_lock(&adist_remote_mtx);
if (adhost->adh_reset)
goto reset;
mtx_unlock(&adist_remote_mtx);
if (trail_filefd(adist_trail) == -1)
trail_next(adist_trail);
}
if (newfile) {
pjdlog_debug(1, "Trail file \"%s/%s\" opened.",
adhost->adh_directory,
trail_filename(adist_trail));
(void)wait_for_file_init(trail_filefd(adist_trail));
}
return (newfile);
}
static int
tls_connect_wait(void *ctx, int timeout)
{
struct tls_ctx *tlsctx = ctx;
int error, sockfd;
uint8_t connected;
PJDLOG_ASSERT(tlsctx != NULL);
PJDLOG_ASSERT(tlsctx->tls_magic == TLS_CTX_MAGIC);
PJDLOG_ASSERT(tlsctx->tls_side == TLS_SIDE_CLIENT);
PJDLOG_ASSERT(tlsctx->tls_sock != NULL);
PJDLOG_ASSERT(!tlsctx->tls_wait_called);
PJDLOG_ASSERT(timeout >= 0);
sockfd = proto_descriptor(tlsctx->tls_sock);
error = wait_for_fd(sockfd, timeout);
if (error != 0)
return (error);
for (;;) {
switch (recv(sockfd, &connected, sizeof(connected),
MSG_WAITALL)) {
case -1:
if (errno == EINTR || errno == ENOBUFS)
continue;
error = errno;
break;
case 0:
pjdlog_debug(1, "Connection terminated.");
error = ENOTCONN;
break;
case 1:
tlsctx->tls_wait_called = true;
break;
}
break;
}
return (error);
}
static void
tls_certificate_verify(SSL *ssl, const char *fingerprint)
{
unsigned char md[EVP_MAX_MD_SIZE];
char mdstr[sizeof("SHA256=") - 1 + EVP_MAX_MD_SIZE * 3];
char *mdstrp;
unsigned int i, mdsize;
X509 *cert;
if (fingerprint[0] == '\0') {
pjdlog_debug(1, "No fingerprint verification requested.");
return;
}
cert = SSL_get_peer_certificate(ssl);
if (cert == NULL)
pjdlog_exitx(EX_TEMPFAIL, "No peer certificate received.");
if (X509_digest(cert, EVP_sha256(), md, &mdsize) != 1)
pjdlog_exitx(EX_TEMPFAIL, "X509_digest() failed.");
PJDLOG_ASSERT(mdsize <= EVP_MAX_MD_SIZE);
X509_free(cert);
(void)strlcpy(mdstr, "SHA256=", sizeof(mdstr));
mdstrp = mdstr + strlen(mdstr);
for (i = 0; i < mdsize; i++) {
PJDLOG_VERIFY(mdstrp + 3 <= mdstr + sizeof(mdstr));
(void)sprintf(mdstrp, "%02hhX:", md[i]);
mdstrp += 3;
}
/* Clear last colon. */
mdstrp[-1] = '\0';
if (strcasecmp(mdstr, fingerprint) != 0) {
pjdlog_exitx(EX_NOPERM,
"Finger print doesn't match. Received \"%s\", expected \"%s\"",
mdstr, fingerprint);
}
}
int
metadata_read(struct hast_resource *res, bool openrw)
{
unsigned char *buf;
struct ebuf *eb;
struct nv *nv;
ssize_t done;
const char *str;
int rerrno;
bool opened_here;
opened_here = false;
rerrno = 0;
/*
* Is this first metadata_read() call for this resource?
*/
if (res->hr_localfd == -1) {
if (provinfo(res, openrw) == -1) {
rerrno = errno;
goto fail;
}
opened_here = true;
pjdlog_debug(1, "Obtained info about %s.", res->hr_localpath);
if (openrw) {
if (flock(res->hr_localfd, LOCK_EX | LOCK_NB) == -1) {
rerrno = errno;
if (errno == EOPNOTSUPP) {
pjdlog_warning("Unable to lock %s (operation not supported), but continuing.",
res->hr_localpath);
} else {
pjdlog_errno(LOG_ERR,
"Unable to lock %s",
res->hr_localpath);
goto fail;
}
}
pjdlog_debug(1, "Locked %s.", res->hr_localpath);
}
}
eb = ebuf_alloc(METADATA_SIZE);
if (eb == NULL) {
rerrno = errno;
pjdlog_errno(LOG_ERR,
"Unable to allocate memory to read metadata");
goto fail;
}
if (ebuf_add_tail(eb, NULL, METADATA_SIZE) == -1) {
rerrno = errno;
pjdlog_errno(LOG_ERR,
"Unable to allocate memory to read metadata");
ebuf_free(eb);
goto fail;
}
buf = ebuf_data(eb, NULL);
PJDLOG_ASSERT(buf != NULL);
done = pread(res->hr_localfd, buf, METADATA_SIZE, 0);
if (done == -1 || done != METADATA_SIZE) {
rerrno = errno;
pjdlog_errno(LOG_ERR, "Unable to read metadata");
ebuf_free(eb);
goto fail;
}
nv = nv_ntoh(eb);
if (nv == NULL) {
rerrno = errno;
pjdlog_errno(LOG_ERR, "Metadata read from %s is invalid",
res->hr_localpath);
ebuf_free(eb);
goto fail;
}
str = nv_get_string(nv, "resource");
if (str != NULL && strcmp(str, res->hr_name) != 0) {
pjdlog_error("Provider %s is not part of resource %s.",
res->hr_localpath, res->hr_name);
nv_free(nv);
goto fail;
}
res->hr_datasize = nv_get_uint64(nv, "datasize");
res->hr_extentsize = (int)nv_get_uint32(nv, "extentsize");
res->hr_keepdirty = (int)nv_get_uint32(nv, "keepdirty");
res->hr_localoff = nv_get_uint64(nv, "offset");
res->hr_resuid = nv_get_uint64(nv, "resuid");
if (res->hr_role != HAST_ROLE_PRIMARY) {
/* Secondary or init role. */
res->hr_secondary_localcnt = nv_get_uint64(nv, "localcnt");
res->hr_secondary_remotecnt = nv_get_uint64(nv, "remotecnt");
}
if (res->hr_role != HAST_ROLE_SECONDARY) {
/* Primary or init role. */
res->hr_primary_localcnt = nv_get_uint64(nv, "localcnt");
res->hr_primary_remotecnt = nv_get_uint64(nv, "remotecnt");
}
str = nv_get_string(nv, "prevrole");
if (str != NULL) {
if (strcmp(str, "primary") == 0)
res->hr_previous_role = HAST_ROLE_PRIMARY;
else if (strcmp(str, "secondary") == 0)
res->hr_previous_role = HAST_ROLE_SECONDARY;
}
if (nv_error(nv) != 0) {
errno = rerrno = nv_error(nv);
pjdlog_errno(LOG_ERR, "Unable to read metadata from %s",
res->hr_localpath);
nv_free(nv);
goto fail;
}
nv_free(nv);
return (0);
fail:
if (opened_here) {
close(res->hr_localfd);
res->hr_localfd = -1;
}
errno = rerrno;
return (-1);
}
/*
* Thread receives requests from the primary node.
*/
static void *
recv_thread(void *arg)
{
struct hast_resource *res = arg;
struct hio *hio;
struct nv *nv;
for (;;) {
pjdlog_debug(2, "recv: Taking free request.");
QUEUE_TAKE(free, hio);
pjdlog_debug(2, "recv: (%p) Got request.", hio);
if (hast_proto_recv_hdr(res->hr_remotein, &nv) < 0) {
secondary_exit(EX_TEMPFAIL,
"Unable to receive request header");
}
if (requnpack(res, hio, nv) != 0) {
nv_free(nv);
pjdlog_debug(2,
"recv: (%p) Moving request to the send queue.",
hio);
QUEUE_INSERT(send, hio);
continue;
}
switch (hio->hio_cmd) {
case HIO_READ:
res->hr_stat_read++;
break;
case HIO_WRITE:
res->hr_stat_write++;
break;
case HIO_DELETE:
res->hr_stat_delete++;
break;
case HIO_FLUSH:
res->hr_stat_flush++;
break;
case HIO_KEEPALIVE:
break;
default:
PJDLOG_ABORT("Unexpected command (cmd=%hhu).",
hio->hio_cmd);
}
reqlog(LOG_DEBUG, 2, -1, hio,
"recv: (%p) Got request header: ", hio);
if (hio->hio_cmd == HIO_KEEPALIVE) {
nv_free(nv);
pjdlog_debug(2,
"recv: (%p) Moving request to the free queue.",
hio);
hio_clear(hio);
QUEUE_INSERT(free, hio);
continue;
} else if (hio->hio_cmd == HIO_WRITE) {
if (hast_proto_recv_data(res, res->hr_remotein, nv,
hio->hio_data, MAXPHYS) < 0) {
secondary_exit(EX_TEMPFAIL,
"Unable to receive request data");
}
}
nv_free(nv);
pjdlog_debug(2, "recv: (%p) Moving request to the disk queue.",
hio);
QUEUE_INSERT(disk, hio);
}
/* NOTREACHED */
return (NULL);
}
/*
* Thread reads from or writes to local component and also handles DELETE and
* FLUSH requests.
*/
static void *
disk_thread(void *arg)
{
struct hast_resource *res = arg;
struct hio *hio;
ssize_t ret;
bool clear_activemap, logerror;
clear_activemap = true;
for (;;) {
pjdlog_debug(2, "disk: Taking request.");
QUEUE_TAKE(disk, hio);
while (clear_activemap) {
unsigned char *map;
size_t mapsize;
/*
* When first request is received, it means that primary
* already received our activemap, merged it and stored
* locally. We can now safely clear our activemap.
*/
mapsize =
activemap_calc_ondisk_size(res->hr_local_mediasize -
METADATA_SIZE, res->hr_extentsize,
res->hr_local_sectorsize);
map = calloc(1, mapsize);
if (map == NULL) {
pjdlog_warning("Unable to allocate memory to clear local activemap.");
break;
}
if (pwrite(res->hr_localfd, map, mapsize,
METADATA_SIZE) != (ssize_t)mapsize) {
pjdlog_errno(LOG_WARNING,
"Unable to store cleared activemap");
free(map);
break;
}
free(map);
clear_activemap = false;
pjdlog_debug(1, "Local activemap cleared.");
break;
}
reqlog(LOG_DEBUG, 2, -1, hio, "disk: (%p) Got request: ", hio);
logerror = true;
/* Handle the actual request. */
switch (hio->hio_cmd) {
case HIO_READ:
ret = pread(res->hr_localfd, hio->hio_data,
hio->hio_length,
hio->hio_offset + res->hr_localoff);
if (ret < 0)
hio->hio_error = errno;
else if (ret != (int64_t)hio->hio_length)
hio->hio_error = EIO;
else
hio->hio_error = 0;
break;
case HIO_WRITE:
ret = pwrite(res->hr_localfd, hio->hio_data,
hio->hio_length,
hio->hio_offset + res->hr_localoff);
if (ret < 0)
hio->hio_error = errno;
else if (ret != (int64_t)hio->hio_length)
hio->hio_error = EIO;
else
hio->hio_error = 0;
break;
case HIO_DELETE:
ret = g_delete(res->hr_localfd,
hio->hio_offset + res->hr_localoff,
hio->hio_length);
if (ret < 0)
hio->hio_error = errno;
else
hio->hio_error = 0;
break;
case HIO_FLUSH:
if (!res->hr_localflush) {
ret = -1;
hio->hio_error = EOPNOTSUPP;
logerror = false;
break;
}
ret = g_flush(res->hr_localfd);
if (ret < 0) {
if (errno == EOPNOTSUPP)
res->hr_localflush = false;
hio->hio_error = errno;
} else {
hio->hio_error = 0;
}
break;
default:
PJDLOG_ABORT("Unexpected command (cmd=%hhu).",
hio->hio_cmd);
}
if (logerror && hio->hio_error != 0) {
reqlog(LOG_ERR, 0, hio->hio_error, hio,
"Request failed: ");
}
pjdlog_debug(2, "disk: (%p) Moving request to the send queue.",
hio);
QUEUE_INSERT(send, hio);
}
/* NOTREACHED */
return (NULL);
}
int
sandbox(const char *user, bool capsicum, const char *fmt, ...)
{
#ifdef HAVE_JAIL
struct jail jailst;
char *jailhost;
va_list ap;
#endif
struct passwd *pw;
uid_t ruid, euid;
gid_t rgid, egid;
#ifdef HAVE_GETRESUID
uid_t suid;
#endif
#ifdef HAVE_GETRESGID
gid_t sgid;
#endif
gid_t *groups, *ggroups;
bool jailed;
int ngroups, ret;
PJDLOG_ASSERT(user != NULL);
PJDLOG_ASSERT(fmt != NULL);
ret = -1;
groups = NULL;
ggroups = NULL;
/*
* According to getpwnam(3) we have to clear errno before calling the
* function to be able to distinguish between an error and missing
* entry (with is not treated as error by getpwnam(3)).
*/
errno = 0;
pw = getpwnam(user);
if (pw == NULL) {
if (errno != 0) {
pjdlog_errno(LOG_ERR,
"Unable to find info about '%s' user", user);
goto out;
} else {
pjdlog_error("'%s' user doesn't exist.", user);
errno = ENOENT;
goto out;
}
}
ngroups = sysconf(_SC_NGROUPS_MAX);
if (ngroups == -1) {
pjdlog_errno(LOG_WARNING,
"Unable to obtain maximum number of groups");
ngroups = NGROUPS_MAX;
}
ngroups++; /* For base gid. */
groups = malloc(sizeof(groups[0]) * ngroups);
if (groups == NULL) {
pjdlog_error("Unable to allocate memory for %d groups.",
ngroups);
goto out;
}
if (getgrouplist(user, pw->pw_gid, groups, &ngroups) == -1) {
pjdlog_error("Unable to obtain groups of user %s.", user);
goto out;
}
#ifdef HAVE_JAIL
va_start(ap, fmt);
(void)vasprintf(&jailhost, fmt, ap);
va_end(ap);
if (jailhost == NULL) {
pjdlog_error("Unable to allocate memory for jail host name.");
goto out;
}
bzero(&jailst, sizeof(jailst));
jailst.version = JAIL_API_VERSION;
jailst.path = pw->pw_dir;
jailst.hostname = jailhost;
if (jail(&jailst) >= 0) {
jailed = true;
} else {
jailed = false;
pjdlog_errno(LOG_WARNING,
"Unable to jail to directory %s", pw->pw_dir);
}
free(jailhost);
#else /* !HAVE_JAIL */
jailed = false;
#endif /* !HAVE_JAIL */
if (!jailed) {
if (chroot(pw->pw_dir) == -1) {
pjdlog_errno(LOG_ERR,
"Unable to change root directory to %s",
pw->pw_dir);
goto out;
}
}
PJDLOG_VERIFY(chdir("/") == 0);
if (setgroups(ngroups, groups) == -1) {
pjdlog_errno(LOG_ERR, "Unable to set groups");
goto out;
}
if (setgid(pw->pw_gid) == -1) {
pjdlog_errno(LOG_ERR, "Unable to set gid to %u",
(unsigned int)pw->pw_gid);
goto out;
}
if (setuid(pw->pw_uid) == -1) {
pjdlog_errno(LOG_ERR, "Unable to set uid to %u",
(unsigned int)pw->pw_uid);
goto out;
}
#ifdef HAVE_CAP_ENTER
if (capsicum) {
capsicum = (cap_enter() == 0);
if (!capsicum) {
pjdlog_common(LOG_DEBUG, 1, errno,
"Unable to sandbox using capsicum");
}
}
#else /* !HAVE_CAP_ENTER */
capsicum = false;
#endif /* !HAVE_CAP_ENTER */
/*
* Better be sure that everything succeeded.
*/
#ifdef HAVE_GETRESUID
PJDLOG_VERIFY(getresuid(&ruid, &euid, &suid) == 0);
PJDLOG_VERIFY(suid == pw->pw_uid);
#else
ruid = getuid();
euid = geteuid();
#endif
PJDLOG_VERIFY(ruid == pw->pw_uid);
PJDLOG_VERIFY(euid == pw->pw_uid);
#ifdef HAVE_GETRESGID
PJDLOG_VERIFY(getresgid(&rgid, &egid, &sgid) == 0);
PJDLOG_VERIFY(sgid == pw->pw_gid);
#else
rgid = getgid();
egid = getegid();
#endif
PJDLOG_VERIFY(rgid == pw->pw_gid);
PJDLOG_VERIFY(egid == pw->pw_gid);
PJDLOG_VERIFY(getgroups(0, NULL) == ngroups);
ggroups = malloc(sizeof(ggroups[0]) * ngroups);
if (ggroups == NULL) {
pjdlog_error("Unable to allocate memory for %d groups.",
ngroups);
goto out;
}
PJDLOG_VERIFY(getgroups(ngroups, ggroups) == ngroups);
qsort(groups, (size_t)ngroups, sizeof(groups[0]), groups_compare);
qsort(ggroups, (size_t)ngroups, sizeof(ggroups[0]), groups_compare);
PJDLOG_VERIFY(bcmp(groups, ggroups, sizeof(groups[0]) * ngroups) == 0);
pjdlog_debug(1,
"Privileges successfully dropped using %s%s+setgid+setuid.",
capsicum ? "capsicum+" : "", jailed ? "jail" : "chroot");
ret = 0;
out:
if (groups != NULL)
free(groups);
if (ggroups != NULL)
free(ggroups);
return (ret);
}
static int
tcp_addr(const char *addr, int defport, struct sockaddr_storage *sap)
{
char iporhost[MAXHOSTNAMELEN], portstr[6];
struct addrinfo hints;
struct addrinfo *res;
const char *pp;
intmax_t port;
size_t size;
int error;
if (addr == NULL)
return (-1);
bzero(&hints, sizeof(hints));
hints.ai_flags = AI_ADDRCONFIG | AI_NUMERICSERV;
hints.ai_family = PF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
if (strncasecmp(addr, "tcp4://", 7) == 0) {
addr += 7;
hints.ai_family = PF_INET;
} else if (strncasecmp(addr, "tcp6://", 7) == 0) {
addr += 7;
hints.ai_family = PF_INET6;
} else if (strncasecmp(addr, "tcp://", 6) == 0) {
addr += 6;
} else {
/*
* Because TCP is the default assume IP or host is given without
* prefix.
*/
}
/*
* Extract optional port.
* There are three cases to consider.
* 1. hostname with port, eg. freefall.freebsd.org:8457
* 2. IPv4 address with port, eg. 192.168.0.101:8457
* 3. IPv6 address with port, eg. [fe80::1]:8457
* We discover IPv6 address by checking for two colons and if port is
* given, the address has to start with [.
*/
pp = NULL;
if (strchr(addr, ':') != strrchr(addr, ':')) {
if (addr[0] == '[')
pp = strrchr(addr, ':');
} else {
pp = strrchr(addr, ':');
}
if (pp == NULL) {
/* Port not given, use the default. */
port = defport;
} else {
if (numfromstr(pp + 1, 1, 65535, &port) == -1)
return (errno);
}
(void)snprintf(portstr, sizeof(portstr), "%jd", (intmax_t)port);
/* Extract host name or IP address. */
if (pp == NULL) {
size = sizeof(iporhost);
if (strlcpy(iporhost, addr, size) >= size)
return (ENAMETOOLONG);
} else if (addr[0] == '[' && pp[-1] == ']') {
size = (size_t)(pp - addr - 2 + 1);
if (size > sizeof(iporhost))
return (ENAMETOOLONG);
(void)strlcpy(iporhost, addr + 1, size);
} else {
size = (size_t)(pp - addr + 1);
if (size > sizeof(iporhost))
return (ENAMETOOLONG);
(void)strlcpy(iporhost, addr, size);
}
error = getaddrinfo(iporhost, portstr, &hints, &res);
if (error != 0) {
pjdlog_debug(1, "getaddrinfo(%s, %s) failed: %s.", iporhost,
portstr, gai_strerror(error));
return (EINVAL);
}
if (res == NULL)
return (ENOENT);
memcpy(sap, res->ai_addr, res->ai_addrlen);
freeaddrinfo(res);
return (0);
}
int
drop_privs(const struct hast_resource *res)
{
char jailhost[sizeof(res->hr_name) * 2];
struct jail jailst;
struct passwd *pw;
uid_t ruid, euid, suid;
gid_t rgid, egid, sgid;
gid_t gidset[1];
bool capsicum, jailed;
/*
* According to getpwnam(3) we have to clear errno before calling the
* function to be able to distinguish between an error and missing
* entry (with is not treated as error by getpwnam(3)).
*/
errno = 0;
pw = getpwnam(HAST_USER);
if (pw == NULL) {
if (errno != 0) {
pjdlog_errno(LOG_ERR,
"Unable to find info about '%s' user", HAST_USER);
return (-1);
} else {
pjdlog_error("'%s' user doesn't exist.", HAST_USER);
errno = ENOENT;
return (-1);
}
}
bzero(&jailst, sizeof(jailst));
jailst.version = JAIL_API_VERSION;
jailst.path = pw->pw_dir;
if (res == NULL) {
(void)snprintf(jailhost, sizeof(jailhost), "hastctl");
} else {
(void)snprintf(jailhost, sizeof(jailhost), "hastd: %s (%s)",
res->hr_name, role2str(res->hr_role));
}
jailst.hostname = jailhost;
jailst.jailname = NULL;
jailst.ip4s = 0;
jailst.ip4 = NULL;
jailst.ip6s = 0;
jailst.ip6 = NULL;
if (jail(&jailst) >= 0) {
jailed = true;
} else {
jailed = false;
pjdlog_errno(LOG_WARNING,
"Unable to jail to directory to %s", pw->pw_dir);
if (chroot(pw->pw_dir) == -1) {
pjdlog_errno(LOG_ERR,
"Unable to change root directory to %s",
pw->pw_dir);
return (-1);
}
}
PJDLOG_VERIFY(chdir("/") == 0);
gidset[0] = pw->pw_gid;
if (setgroups(1, gidset) == -1) {
pjdlog_errno(LOG_ERR, "Unable to set groups to gid %u",
(unsigned int)pw->pw_gid);
return (-1);
}
if (setgid(pw->pw_gid) == -1) {
pjdlog_errno(LOG_ERR, "Unable to set gid to %u",
(unsigned int)pw->pw_gid);
return (-1);
}
if (setuid(pw->pw_uid) == -1) {
pjdlog_errno(LOG_ERR, "Unable to set uid to %u",
(unsigned int)pw->pw_uid);
return (-1);
}
#ifdef HAVE_CAPSICUM
capsicum = (cap_enter() == 0);
if (!capsicum) {
pjdlog_common(LOG_DEBUG, 1, errno,
"Unable to sandbox using capsicum");
} else if (res != NULL) {
cap_rights_t rights;
static const unsigned long geomcmds[] = {
DIOCGDELETE,
DIOCGFLUSH
};
PJDLOG_ASSERT(res->hr_role == HAST_ROLE_PRIMARY ||
res->hr_role == HAST_ROLE_SECONDARY);
cap_rights_init(&rights, CAP_FLOCK, CAP_IOCTL, CAP_PREAD,
CAP_PWRITE);
if (cap_rights_limit(res->hr_localfd, &rights) == -1) {
pjdlog_errno(LOG_ERR,
"Unable to limit capability rights on local descriptor");
}
if (cap_ioctls_limit(res->hr_localfd, geomcmds,
sizeof(geomcmds) / sizeof(geomcmds[0])) == -1) {
pjdlog_errno(LOG_ERR,
"Unable to limit allowed GEOM ioctls");
}
if (res->hr_role == HAST_ROLE_PRIMARY) {
static const unsigned long ggatecmds[] = {
G_GATE_CMD_MODIFY,
G_GATE_CMD_START,
G_GATE_CMD_DONE,
G_GATE_CMD_DESTROY
};
cap_rights_init(&rights, CAP_IOCTL);
if (cap_rights_limit(res->hr_ggatefd, &rights) == -1) {
pjdlog_errno(LOG_ERR,
"Unable to limit capability rights to CAP_IOCTL on ggate descriptor");
}
if (cap_ioctls_limit(res->hr_ggatefd, ggatecmds,
sizeof(ggatecmds) / sizeof(ggatecmds[0])) == -1) {
pjdlog_errno(LOG_ERR,
"Unable to limit allowed ggate ioctls");
}
}
}
#else
capsicum = false;
#endif
/*
* Better be sure that everything succeeded.
*/
PJDLOG_VERIFY(getresuid(&ruid, &euid, &suid) == 0);
PJDLOG_VERIFY(ruid == pw->pw_uid);
PJDLOG_VERIFY(euid == pw->pw_uid);
PJDLOG_VERIFY(suid == pw->pw_uid);
PJDLOG_VERIFY(getresgid(&rgid, &egid, &sgid) == 0);
PJDLOG_VERIFY(rgid == pw->pw_gid);
PJDLOG_VERIFY(egid == pw->pw_gid);
PJDLOG_VERIFY(sgid == pw->pw_gid);
PJDLOG_VERIFY(getgroups(0, NULL) == 1);
PJDLOG_VERIFY(getgroups(1, gidset) == 1);
PJDLOG_VERIFY(gidset[0] == pw->pw_gid);
pjdlog_debug(1,
"Privileges successfully dropped using %s%s+setgid+setuid.",
capsicum ? "capsicum+" : "", jailed ? "jail" : "chroot");
return (0);
}
void
service_message(struct service *service, struct service_connection *sconn)
{
nvlist_t *nvlin, *nvlout;
const char *cmd;
int error;
nvlin = cap_recv_nvlist(service_connection_get_chan(sconn));
if (nvlin == NULL) {
if (errno == ENOTCONN) {
pjdlog_debug(1, "Connection closed by the client.");
} else {
pjdlog_errno(LOG_ERR,
"Unable to receive message from client");
}
service_connection_remove(service, sconn);
return;
}
error = EDOOFUS;
nvlout = nvlist_create(0);
cmd = nvlist_get_string(nvlin, "cmd");
pjdlog_debug(1, "Command received from client: %s.", cmd);
if (pjdlog_debug_get() >= 2)
nvlist_fdump(nvlin, stderr);
if (strcmp(cmd, "limit_set") == 0) {
nvlist_t *nvllim;
nvllim = nvlist_take_nvlist(nvlin, "limits");
error = service->s_limit(service_connection_get_limits(sconn),
nvllim);
if (error == 0) {
service_connection_set_limits(sconn, nvllim);
/* Function consumes nvllim. */
} else {
nvlist_destroy(nvllim);
}
} else if (strcmp(cmd, "limit_get") == 0) {
const nvlist_t *nvllim;
nvllim = service_connection_get_limits(sconn);
if (nvllim != NULL)
nvlist_add_nvlist(nvlout, "limits", nvllim);
else
nvlist_add_null(nvlout, "limits");
error = 0;
} else if (strcmp(cmd, "clone") == 0) {
int sock;
sock = service_connection_clone(service, sconn);
if (sock == -1) {
error = errno;
} else {
nvlist_add_descriptor(nvlout, "sock", sock);
error = 0;
}
} else {
nvlout = nvlist_create(0);
error = service->s_command(cmd,
service_connection_get_limits(sconn), nvlin, nvlout);
}
nvlist_destroy(nvlin);
nvlist_add_number(nvlout, "error", (uint64_t)error);
pjdlog_debug(1, "Sending reply to client (error=%d).", error);
if (pjdlog_debug_get() >= 2)
nvlist_fdump(nvlout, stderr);
if (cap_send_nvlist(service_connection_get_chan(sconn), nvlout) == -1) {
pjdlog_errno(LOG_ERR, "Unable to send message to client");
service_connection_remove(service, sconn);
return;
}
}
/*
* Thread receives requests from the primary node.
*/
static void *
recv_thread(void *arg)
{
struct hast_resource *res = arg;
struct hio *hio, *mshio;
struct nv *nv;
for (;;) {
pjdlog_debug(2, "recv: Taking free request.");
QUEUE_TAKE(free, hio);
pjdlog_debug(2, "recv: (%p) Got request.", hio);
if (hast_proto_recv_hdr(res->hr_remotein, &nv) == -1) {
secondary_exit(EX_TEMPFAIL,
"Unable to receive request header");
}
if (requnpack(res, hio, nv) != 0) {
nv_free(nv);
pjdlog_debug(2,
"recv: (%p) Moving request to the send queue.",
hio);
QUEUE_INSERT(send, hio);
continue;
}
switch (hio->hio_cmd) {
case HIO_READ:
res->hr_stat_read++;
break;
case HIO_WRITE:
res->hr_stat_write++;
break;
case HIO_DELETE:
res->hr_stat_delete++;
break;
case HIO_FLUSH:
res->hr_stat_flush++;
break;
case HIO_KEEPALIVE:
break;
default:
PJDLOG_ABORT("Unexpected command (cmd=%hhu).",
hio->hio_cmd);
}
reqlog(LOG_DEBUG, 2, -1, hio,
"recv: (%p) Got request header: ", hio);
if (hio->hio_cmd == HIO_KEEPALIVE) {
nv_free(nv);
pjdlog_debug(2,
"recv: (%p) Moving request to the free queue.",
hio);
hio_clear(hio);
QUEUE_INSERT(free, hio);
continue;
} else if (hio->hio_cmd == HIO_WRITE) {
if (hast_proto_recv_data(res, res->hr_remotein, nv,
hio->hio_data, MAXPHYS) == -1) {
secondary_exit(EX_TEMPFAIL,
"Unable to receive request data");
}
if (hio->hio_memsync) {
/*
* For memsync requests we expect two replies.
* Clone the hio so we can handle both of them.
*/
pjdlog_debug(2, "recv: Taking free request.");
QUEUE_TAKE(free, mshio);
pjdlog_debug(2, "recv: (%p) Got request.",
mshio);
hio_copy(hio, mshio);
mshio->hio_error = 0;
/*
* We want to keep 'memsync' tag only on the
* request going onto send queue (mshio).
*/
hio->hio_memsync = false;
pjdlog_debug(2,
"recv: (%p) Moving memsync request to the send queue.",
mshio);
QUEUE_INSERT(send, mshio);
}
}
nv_free(nv);
pjdlog_debug(2, "recv: (%p) Moving request to the disk queue.",
hio);
QUEUE_INSERT(disk, hio);
}
/* NOTREACHED */
return (NULL);
}
static void
tls_exec_client(const char *user, int startfd, const char *srcaddr,
const char *dstaddr, const char *fingerprint, const char *defport,
int timeout, int debuglevel)
{
struct proto_conn *tcp;
char *saddr, *daddr;
SSL_CTX *sslctx;
SSL *ssl;
long ret;
int sockfd, tcpfd;
uint8_t connected;
pjdlog_debug_set(debuglevel);
pjdlog_prefix_set("[TLS sandbox] (client) ");
#ifdef HAVE_SETPROCTITLE
setproctitle("[TLS sandbox] (client) ");
#endif
proto_set("tcp:port", defport);
sockfd = startfd;
/* Change tls:// to tcp://. */
if (srcaddr == NULL) {
saddr = NULL;
} else {
saddr = strdup(srcaddr);
if (saddr == NULL)
pjdlog_exitx(EX_TEMPFAIL, "Unable to allocate memory.");
bcopy("tcp://", saddr, 6);
}
daddr = strdup(dstaddr);
if (daddr == NULL)
pjdlog_exitx(EX_TEMPFAIL, "Unable to allocate memory.");
bcopy("tcp://", daddr, 6);
/* Establish TCP connection. */
if (proto_connect(saddr, daddr, timeout, &tcp) == -1)
exit(EX_TEMPFAIL);
SSL_load_error_strings();
SSL_library_init();
/*
* TODO: On FreeBSD we could move this below sandbox() once libc and
* libcrypto use sysctl kern.arandom to obtain random data
* instead of /dev/urandom and friends.
*/
sslctx = SSL_CTX_new(TLSv1_client_method());
if (sslctx == NULL)
pjdlog_exitx(EX_TEMPFAIL, "SSL_CTX_new() failed.");
if (sandbox(user, true, "proto_tls client: %s", dstaddr) != 0)
pjdlog_exitx(EX_CONFIG, "Unable to sandbox TLS client.");
pjdlog_debug(1, "Privileges successfully dropped.");
SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3);
/* Load CA certs. */
/* TODO */
//SSL_CTX_load_verify_locations(sslctx, cacerts_file, NULL);
ssl = SSL_new(sslctx);
if (ssl == NULL)
pjdlog_exitx(EX_TEMPFAIL, "SSL_new() failed.");
tcpfd = proto_descriptor(tcp);
block(tcpfd);
if (SSL_set_fd(ssl, tcpfd) != 1)
pjdlog_exitx(EX_TEMPFAIL, "SSL_set_fd() failed.");
ret = SSL_connect(ssl);
ssl_check_error(ssl, (int)ret);
nonblock(sockfd);
nonblock(tcpfd);
tls_certificate_verify(ssl, fingerprint);
/*
* The following byte is send to make proto_connect_wait() to work.
*/
connected = 1;
for (;;) {
switch (send(sockfd, &connected, sizeof(connected), 0)) {
case -1:
if (errno == EINTR || errno == ENOBUFS)
continue;
if (errno == EAGAIN) {
(void)wait_for_fd(sockfd, -1);
continue;
}
pjdlog_exit(EX_TEMPFAIL, "send() failed");
case 0:
pjdlog_debug(1, "Connection terminated.");
exit(0);
case 1:
break;
}
break;
}
tls_loop(sockfd, ssl);
}
static int
sender_connect(void)
{
unsigned char rnd[32], hash[32], resp[32];
struct proto_conn *conn;
char welcome[8];
int16_t val;
val = 1;
if (proto_send(adhost->adh_conn, &val, sizeof(val)) < 0) {
pjdlog_exit(EX_TEMPFAIL,
"Unable to send connection request to parent");
}
if (proto_recv(adhost->adh_conn, &val, sizeof(val)) < 0) {
pjdlog_exit(EX_TEMPFAIL,
"Unable to receive reply to connection request from parent");
}
if (val != 0) {
errno = val;
pjdlog_errno(LOG_WARNING, "Unable to connect to %s",
adhost->adh_remoteaddr);
return (-1);
}
if (proto_connection_recv(adhost->adh_conn, true, &conn) < 0) {
pjdlog_exit(EX_TEMPFAIL,
"Unable to receive connection from parent");
}
if (proto_connect_wait(conn, adcfg->adc_timeout) < 0) {
pjdlog_errno(LOG_WARNING, "Unable to connect to %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Connected to %s.", adhost->adh_remoteaddr);
/* Error in setting timeout is not critical, but why should it fail? */
if (proto_timeout(conn, adcfg->adc_timeout) < 0)
pjdlog_errno(LOG_WARNING, "Unable to set connection timeout");
else
pjdlog_debug(1, "Timeout set to %d.", adcfg->adc_timeout);
/* Exchange welcome message, which includes version number. */
(void)snprintf(welcome, sizeof(welcome), "ADIST%02d", ADIST_VERSION);
if (proto_send(conn, welcome, sizeof(welcome)) < 0) {
pjdlog_errno(LOG_WARNING,
"Unable to send welcome message to %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Welcome message sent (%s).", welcome);
bzero(welcome, sizeof(welcome));
if (proto_recv(conn, welcome, sizeof(welcome)) < 0) {
pjdlog_errno(LOG_WARNING,
"Unable to receive welcome message from %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
if (strncmp(welcome, "ADIST", 5) != 0 || !isdigit(welcome[5]) ||
!isdigit(welcome[6]) || welcome[7] != '\0') {
pjdlog_warning("Invalid welcome message from %s.",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Welcome message received (%s).", welcome);
/*
* Receiver can only reply with version number lower or equal to
* the one we sent.
*/
adhost->adh_version = atoi(welcome + 5);
if (adhost->adh_version > ADIST_VERSION) {
pjdlog_warning("Invalid version number from %s (%d received, up to %d supported).",
adhost->adh_remoteaddr, adhost->adh_version, ADIST_VERSION);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Version %d negotiated with %s.", adhost->adh_version,
adhost->adh_remoteaddr);
if (proto_send(conn, adcfg->adc_name, sizeof(adcfg->adc_name)) == -1) {
pjdlog_errno(LOG_WARNING, "Unable to send name to %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Name (%s) sent.", adcfg->adc_name);
if (proto_recv(conn, rnd, sizeof(rnd)) == -1) {
pjdlog_errno(LOG_WARNING, "Unable to receive challenge from %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Challenge received.");
if (HMAC(EVP_sha256(), adhost->adh_password,
(int)strlen(adhost->adh_password), rnd, (int)sizeof(rnd), hash,
NULL) == NULL) {
pjdlog_warning("Unable to generate response.");
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Response generated.");
if (proto_send(conn, hash, sizeof(hash)) == -1) {
pjdlog_errno(LOG_WARNING, "Unable to send response to %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Response sent.");
if (adist_random(rnd, sizeof(rnd)) == -1) {
pjdlog_warning("Unable to generate challenge.");
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Challenge generated.");
if (proto_send(conn, rnd, sizeof(rnd)) == -1) {
pjdlog_errno(LOG_WARNING, "Unable to send challenge to %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Challenge sent.");
if (proto_recv(conn, resp, sizeof(resp)) == -1) {
pjdlog_errno(LOG_WARNING, "Unable to receive response from %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Response received.");
if (HMAC(EVP_sha256(), adhost->adh_password,
(int)strlen(adhost->adh_password), rnd, (int)sizeof(rnd), hash,
NULL) == NULL) {
pjdlog_warning("Unable to generate hash.");
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Hash generated.");
if (memcmp(resp, hash, sizeof(hash)) != 0) {
pjdlog_warning("Invalid response from %s (wrong password?).",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_info("Receiver authenticated.");
if (proto_recv(conn, &adhost->adh_trail_offset,
sizeof(adhost->adh_trail_offset)) == -1) {
pjdlog_errno(LOG_WARNING,
"Unable to receive size of the most recent trail file from %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
adhost->adh_trail_offset = le64toh(adhost->adh_trail_offset);
if (proto_recv(conn, &adhost->adh_trail_name,
sizeof(adhost->adh_trail_name)) == -1) {
pjdlog_errno(LOG_WARNING,
"Unable to receive name of the most recent trail file from %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Trail name (%s) and offset (%ju) received.",
adhost->adh_trail_name, (uintmax_t)adhost->adh_trail_offset);
rw_wlock(&adist_remote_lock);
mtx_lock(&adist_remote_mtx);
PJDLOG_ASSERT(adhost->adh_remote == NULL);
PJDLOG_ASSERT(conn != NULL);
adhost->adh_remote = conn;
mtx_unlock(&adist_remote_mtx);
rw_unlock(&adist_remote_lock);
cv_signal(&adist_remote_cond);
return (0);
}
