int
proto_common_send(int sock, const unsigned char *data, size_t size, int fd)
{
ssize_t done;
size_t sendsize;
int errcount = 0;
PJDLOG_ASSERT(sock >= 0);
if (data == NULL) {
/* The caller is just trying to decide about direction. */
PJDLOG_ASSERT(size == 0);
if (shutdown(sock, SHUT_RD) == -1)
return (errno);
return (0);
}
PJDLOG_ASSERT(data != NULL);
PJDLOG_ASSERT(size > 0);
do {
sendsize = size < MAX_SEND_SIZE ? size : MAX_SEND_SIZE;
done = send(sock, data, sendsize, MSG_NOSIGNAL);
if (done == 0) {
return (ENOTCONN);
} else if (done == -1) {
if (errno == EINTR)
continue;
if (errno == ENOBUFS) {
/*
* If there are no buffers we retry.
* After each try we increase delay before the
* next one and we give up after fifteen times.
* This gives 11s of total wait time.
*/
if (errcount == 15) {
pjdlog_warning("Getting ENOBUFS errors for 11s on send(), giving up.");
} else {
if (errcount == 0)
pjdlog_warning("Got ENOBUFS error on send(), retrying for a bit.");
errcount++;
usleep(100000 * errcount);
continue;
}
}
/*
* If this is blocking socket and we got EAGAIN, this
* means the request timed out. Translate errno to
* ETIMEDOUT, to give administrator a hint to
* eventually increase timeout.
*/
if (errno == EAGAIN && blocking_socket(sock))
errno = ETIMEDOUT;
return (errno);
}
data += done;
size -= done;
} while (size > 0);
if (errcount > 0) {
pjdlog_info("Data sent successfully after %d ENOBUFS error%s.",
errcount, errcount == 1 ? "" : "s");
}
if (fd == -1)
return (0);
return (proto_descriptor_send(sock, fd));
}
void
hastd_secondary(struct hast_resource *res, struct nv *nvin)
{
sigset_t mask;
pthread_t td;
pid_t pid;
int error, mode, debuglevel;
/*
* Create communication channel between parent and child.
*/
if (proto_client(NULL, "socketpair://", &res->hr_ctrl) < 0) {
KEEP_ERRNO((void)pidfile_remove(pfh));
pjdlog_exit(EX_OSERR,
"Unable to create control sockets between parent and child");
}
/*
* Create communication channel between child and parent.
*/
if (proto_client(NULL, "socketpair://", &res->hr_event) < 0) {
KEEP_ERRNO((void)pidfile_remove(pfh));
pjdlog_exit(EX_OSERR,
"Unable to create event sockets between child and parent");
}
pid = fork();
if (pid < 0) {
KEEP_ERRNO((void)pidfile_remove(pfh));
pjdlog_exit(EX_OSERR, "Unable to fork");
}
if (pid > 0) {
/* This is parent. */
proto_close(res->hr_remotein);
res->hr_remotein = NULL;
proto_close(res->hr_remoteout);
res->hr_remoteout = NULL;
/* Declare that we are receiver. */
proto_recv(res->hr_event, NULL, 0);
/* Declare that we are sender. */
proto_send(res->hr_ctrl, NULL, 0);
res->hr_workerpid = pid;
return;
}
gres = res;
mode = pjdlog_mode_get();
debuglevel = pjdlog_debug_get();
/* Declare that we are sender. */
proto_send(res->hr_event, NULL, 0);
/* Declare that we are receiver. */
proto_recv(res->hr_ctrl, NULL, 0);
descriptors_cleanup(res);
descriptors_assert(res, mode);
pjdlog_init(mode);
pjdlog_debug_set(debuglevel);
pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role));
setproctitle("%s (%s)", res->hr_name, role2str(res->hr_role));
PJDLOG_VERIFY(sigemptyset(&mask) == 0);
PJDLOG_VERIFY(sigprocmask(SIG_SETMASK, &mask, NULL) == 0);
/* Error in setting timeout is not critical, but why should it fail? */
if (proto_timeout(res->hr_remotein, 2 * HAST_KEEPALIVE) < 0)
pjdlog_errno(LOG_WARNING, "Unable to set connection timeout");
if (proto_timeout(res->hr_remoteout, res->hr_timeout) < 0)
pjdlog_errno(LOG_WARNING, "Unable to set connection timeout");
init_local(res);
init_environment();
if (drop_privs(res) != 0)
exit(EX_CONFIG);
pjdlog_info("Privileges successfully dropped.");
/*
* Create the control thread before sending any event to the parent,
* as we can deadlock when parent sends control request to worker,
* but worker has no control thread started yet, so parent waits.
* In the meantime worker sends an event to the parent, but parent
* is unable to handle the event, because it waits for control
* request response.
*/
error = pthread_create(&td, NULL, ctrl_thread, res);
PJDLOG_ASSERT(error == 0);
init_remote(res, nvin);
event_send(res, EVENT_CONNECT);
error = pthread_create(&td, NULL, recv_thread, res);
PJDLOG_ASSERT(error == 0);
error = pthread_create(&td, NULL, disk_thread, res);
PJDLOG_ASSERT(error == 0);
(void)send_thread(res);
}
static int
sender_connect(void)
{
unsigned char rnd[32], hash[32], resp[32];
struct proto_conn *conn;
char welcome[8];
int16_t val;
val = 1;
if (proto_send(adhost->adh_conn, &val, sizeof(val)) < 0) {
pjdlog_exit(EX_TEMPFAIL,
"Unable to send connection request to parent");
}
if (proto_recv(adhost->adh_conn, &val, sizeof(val)) < 0) {
pjdlog_exit(EX_TEMPFAIL,
"Unable to receive reply to connection request from parent");
}
if (val != 0) {
errno = val;
pjdlog_errno(LOG_WARNING, "Unable to connect to %s",
adhost->adh_remoteaddr);
return (-1);
}
if (proto_connection_recv(adhost->adh_conn, true, &conn) < 0) {
pjdlog_exit(EX_TEMPFAIL,
"Unable to receive connection from parent");
}
if (proto_connect_wait(conn, adcfg->adc_timeout) < 0) {
pjdlog_errno(LOG_WARNING, "Unable to connect to %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Connected to %s.", adhost->adh_remoteaddr);
/* Error in setting timeout is not critical, but why should it fail? */
if (proto_timeout(conn, adcfg->adc_timeout) < 0)
pjdlog_errno(LOG_WARNING, "Unable to set connection timeout");
else
pjdlog_debug(1, "Timeout set to %d.", adcfg->adc_timeout);
/* Exchange welcome message, which includes version number. */
(void)snprintf(welcome, sizeof(welcome), "ADIST%02d", ADIST_VERSION);
if (proto_send(conn, welcome, sizeof(welcome)) < 0) {
pjdlog_errno(LOG_WARNING,
"Unable to send welcome message to %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Welcome message sent (%s).", welcome);
bzero(welcome, sizeof(welcome));
if (proto_recv(conn, welcome, sizeof(welcome)) < 0) {
pjdlog_errno(LOG_WARNING,
"Unable to receive welcome message from %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
if (strncmp(welcome, "ADIST", 5) != 0 || !isdigit(welcome[5]) ||
!isdigit(welcome[6]) || welcome[7] != '\0') {
pjdlog_warning("Invalid welcome message from %s.",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Welcome message received (%s).", welcome);
/*
* Receiver can only reply with version number lower or equal to
* the one we sent.
*/
adhost->adh_version = atoi(welcome + 5);
if (adhost->adh_version > ADIST_VERSION) {
pjdlog_warning("Invalid version number from %s (%d received, up to %d supported).",
adhost->adh_remoteaddr, adhost->adh_version, ADIST_VERSION);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Version %d negotiated with %s.", adhost->adh_version,
adhost->adh_remoteaddr);
if (proto_send(conn, adcfg->adc_name, sizeof(adcfg->adc_name)) == -1) {
pjdlog_errno(LOG_WARNING, "Unable to send name to %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Name (%s) sent.", adcfg->adc_name);
if (proto_recv(conn, rnd, sizeof(rnd)) == -1) {
pjdlog_errno(LOG_WARNING, "Unable to receive challenge from %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Challenge received.");
if (HMAC(EVP_sha256(), adhost->adh_password,
(int)strlen(adhost->adh_password), rnd, (int)sizeof(rnd), hash,
NULL) == NULL) {
pjdlog_warning("Unable to generate response.");
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Response generated.");
if (proto_send(conn, hash, sizeof(hash)) == -1) {
pjdlog_errno(LOG_WARNING, "Unable to send response to %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Response sent.");
if (adist_random(rnd, sizeof(rnd)) == -1) {
pjdlog_warning("Unable to generate challenge.");
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Challenge generated.");
if (proto_send(conn, rnd, sizeof(rnd)) == -1) {
pjdlog_errno(LOG_WARNING, "Unable to send challenge to %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Challenge sent.");
if (proto_recv(conn, resp, sizeof(resp)) == -1) {
pjdlog_errno(LOG_WARNING, "Unable to receive response from %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Response received.");
if (HMAC(EVP_sha256(), adhost->adh_password,
(int)strlen(adhost->adh_password), rnd, (int)sizeof(rnd), hash,
NULL) == NULL) {
pjdlog_warning("Unable to generate hash.");
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Hash generated.");
if (memcmp(resp, hash, sizeof(hash)) != 0) {
pjdlog_warning("Invalid response from %s (wrong password?).",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_info("Receiver authenticated.");
if (proto_recv(conn, &adhost->adh_trail_offset,
sizeof(adhost->adh_trail_offset)) == -1) {
pjdlog_errno(LOG_WARNING,
"Unable to receive size of the most recent trail file from %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
adhost->adh_trail_offset = le64toh(adhost->adh_trail_offset);
if (proto_recv(conn, &adhost->adh_trail_name,
sizeof(adhost->adh_trail_name)) == -1) {
pjdlog_errno(LOG_WARNING,
"Unable to receive name of the most recent trail file from %s",
adhost->adh_remoteaddr);
proto_close(conn);
return (-1);
}
pjdlog_debug(1, "Trail name (%s) and offset (%ju) received.",
adhost->adh_trail_name, (uintmax_t)adhost->adh_trail_offset);
rw_wlock(&adist_remote_lock);
mtx_lock(&adist_remote_mtx);
PJDLOG_ASSERT(adhost->adh_remote == NULL);
PJDLOG_ASSERT(conn != NULL);
adhost->adh_remote = conn;
mtx_unlock(&adist_remote_mtx);
rw_unlock(&adist_remote_lock);
cv_signal(&adist_remote_cond);
return (0);
}