void
send_read_response(as_transaction* tr, as_msg_op** ops, as_bin** response_bins,
uint16_t n_bins, cf_dyn_buf* db)
{
// Paranoia - shouldn't get here on losing race with timeout.
if (! tr->from.any) {
cf_warning(AS_RW, "transaction origin %u has null 'from'", tr->origin);
return;
}
// Note - if tr was setup from rw, rw->from.any has been set null and
// informs timeout it lost the race.
switch (tr->origin) {
case FROM_CLIENT:
BENCHMARK_NEXT_DATA_POINT(tr, read, local);
if (db && db->used_sz != 0) {
as_msg_send_ops_reply(tr->from.proto_fd_h, db);
}
else {
as_msg_send_reply(tr->from.proto_fd_h, tr->result_code,
tr->generation, tr->void_time, ops, response_bins, n_bins,
tr->rsv.ns, as_transaction_trid(tr));
}
BENCHMARK_NEXT_DATA_POINT(tr, read, response);
HIST_TRACK_ACTIVATE_INSERT_DATA_POINT(tr, read_hist);
client_read_update_stats(tr->rsv.ns, tr->result_code);
break;
case FROM_PROXY:
if (db && db->used_sz != 0) {
as_proxy_send_ops_response(tr->from.proxy_node,
tr->from_data.proxy_tid, db);
}
else {
as_proxy_send_response(tr->from.proxy_node, tr->from_data.proxy_tid,
tr->result_code, tr->generation, tr->void_time, ops,
response_bins, n_bins, tr->rsv.ns, as_transaction_trid(tr));
}
if (as_transaction_is_batch_sub(tr)) {
from_proxy_batch_sub_read_update_stats(tr->rsv.ns, tr->result_code);
}
else {
from_proxy_read_update_stats(tr->rsv.ns, tr->result_code);
}
break;
case FROM_BATCH:
BENCHMARK_NEXT_DATA_POINT(tr, batch_sub, read_local);
as_batch_add_result(tr, n_bins, response_bins, ops);
BENCHMARK_NEXT_DATA_POINT(tr, batch_sub, response);
batch_sub_read_update_stats(tr->rsv.ns, tr->result_code);
break;
default:
cf_crash(AS_RW, "unexpected transaction origin %u", tr->origin);
break;
}
tr->from.any = NULL; // pattern, not needed
}
void
as_transaction_error(as_transaction* tr, uint32_t error_code)
{
if (tr->proto_fd_h) {
if (tr->batch_shared) {
as_batch_add_error(tr->batch_shared, tr->batch_index, error_code);
// Clear this transaction's msgp so calling code does not free it.
tr->msgp = 0;
}
else {
as_msg_send_reply(tr->proto_fd_h, error_code, 0, 0, NULL, NULL, 0, NULL, NULL, as_transaction_trid(tr), NULL);
tr->proto_fd_h = 0;
MICROBENCHMARK_HIST_INSERT_P(error_hist);
cf_atomic_int_incr(&g_config.err_tsvc_requests);
if (error_code == AS_PROTO_RESULT_FAIL_TIMEOUT) {
cf_atomic_int_incr(&g_config.err_tsvc_requests_timeout);
}
}
}
else if (tr->proxy_msg) {
as_proxy_send_response(tr->proxy_node, tr->proxy_msg, error_code, 0, 0, NULL, NULL, 0, NULL, as_transaction_trid(tr), NULL);
tr->proxy_msg = NULL;
}
else if (tr->udata.req_udata) {
if (udf_rw_needcomplete(tr)) {
udf_rw_complete(tr, error_code, __FILE__,__LINE__);
}
}
}
/**
* Send failure notification of general UDF execution, but check for special
* LDT errors and return specific Wire Protocol error codes for these cases:
* (1) Record not found (2)
* (2) LDT Collection item not found (125)
*
* All other errors get the generic 100 (UDF FAIL) code.
*/
static inline int
process_udf_failure(udf_call *call, const as_string *s, cf_dyn_buf *db)
{
char *val = as_string_tostring(s);
size_t vlen = as_string_len((as_string *)s); // TODO - make as_string_len() take const
long error_code = ldt_get_error_code(val, vlen);
if (error_code) {
if (error_code == AS_PROTO_RESULT_FAIL_NOTFOUND ||
error_code == AS_PROTO_RESULT_FAIL_COLLECTION_ITEM_NOT_FOUND) {
call->tr->result_code = (uint8_t)error_code;
// Send an "empty" response, with no failure bin.
as_transaction * tr = call->tr;
if (db) {
size_t msg_sz = 0;
uint8_t *msgp = (uint8_t *)as_msg_make_response_msg(
tr->result_code, 0, 0, NULL, NULL, 0, tr->rsv.ns, NULL,
&msg_sz, as_transaction_trid(tr), NULL);
if (! msgp) {
cf_warning_digest(AS_RW, &tr->keyd, "{%s} LDT UDF failed to make response msg ", tr->rsv.ns->name);
return -1;
}
// Stash the message, to be sent later.
db->buf = msgp;
db->is_stack = false;
db->alloc_sz = msg_sz;
db->used_sz = msg_sz;
}
else {
single_transaction_response(tr, tr->rsv.ns, NULL/*ops*/,
NULL /*bin*/, 0 /*nbins*/, 0, 0, NULL, NULL);
}
return 0;
}
}
cf_debug(AS_UDF, "Non-special LDT or General UDF Error(%s)", (char *) val);
call->tr->result_code = AS_PROTO_RESULT_FAIL_UDF_EXECUTION;
return process_failure(call, as_string_toval(s), db);
}
transaction_status
read_local(as_transaction* tr)
{
as_msg* m = &tr->msgp->msg;
as_namespace* ns = tr->rsv.ns;
as_index_ref r_ref;
if (as_record_get(tr->rsv.tree, &tr->keyd, &r_ref) != 0) {
read_local_done(tr, NULL, NULL, AS_ERR_NOT_FOUND);
return TRANS_DONE_ERROR;
}
as_record* r = r_ref.r;
// Check if it's an expired or truncated record.
if (as_record_is_doomed(r, ns)) {
read_local_done(tr, &r_ref, NULL, AS_ERR_NOT_FOUND);
return TRANS_DONE_ERROR;
}
int result = repl_state_check(r, tr);
if (result != 0) {
if (result == -3) {
read_local_done(tr, &r_ref, NULL, AS_ERR_UNAVAILABLE);
return TRANS_DONE_ERROR;
}
// No response sent to origin.
as_record_done(&r_ref, ns);
return result == 1 ? TRANS_IN_PROGRESS : TRANS_WAITING;
}
// Check if it's a tombstone.
if (! as_record_is_live(r)) {
read_local_done(tr, &r_ref, NULL, AS_ERR_NOT_FOUND);
return TRANS_DONE_ERROR;
}
as_storage_rd rd;
as_storage_record_open(ns, r, &rd);
// If configuration permits, allow reads to use page cache.
rd.read_page_cache = ns->storage_read_page_cache;
// Check the key if required.
// Note - for data-not-in-memory "exists" ops, key check is expensive!
if (as_transaction_has_key(tr) &&
as_storage_record_get_key(&rd) && ! check_msg_key(m, &rd)) {
read_local_done(tr, &r_ref, &rd, AS_ERR_KEY_MISMATCH);
return TRANS_DONE_ERROR;
}
if ((m->info1 & AS_MSG_INFO1_GET_NO_BINS) != 0) {
tr->generation = r->generation;
tr->void_time = r->void_time;
tr->last_update_time = r->last_update_time;
read_local_done(tr, &r_ref, &rd, AS_OK);
return TRANS_DONE_SUCCESS;
}
if ((result = as_storage_rd_load_n_bins(&rd)) < 0) {
cf_warning_digest(AS_RW, &tr->keyd, "{%s} read_local: failed as_storage_rd_load_n_bins() ", ns->name);
read_local_done(tr, &r_ref, &rd, -result);
return TRANS_DONE_ERROR;
}
as_bin stack_bins[ns->storage_data_in_memory ? 0 : rd.n_bins];
if ((result = as_storage_rd_load_bins(&rd, stack_bins)) < 0) {
cf_warning_digest(AS_RW, &tr->keyd, "{%s} read_local: failed as_storage_rd_load_bins() ", ns->name);
read_local_done(tr, &r_ref, &rd, -result);
return TRANS_DONE_ERROR;
}
if (! as_bin_inuse_has(&rd)) {
cf_warning_digest(AS_RW, &tr->keyd, "{%s} read_local: found record with no bins ", ns->name);
read_local_done(tr, &r_ref, &rd, AS_ERR_UNKNOWN);
return TRANS_DONE_ERROR;
}
uint32_t bin_count = (m->info1 & AS_MSG_INFO1_GET_ALL) != 0 ?
rd.n_bins : m->n_ops;
as_msg_op* ops[bin_count];
as_msg_op** p_ops = ops;
as_bin* response_bins[bin_count];
uint16_t n_bins = 0;
as_bin result_bins[bin_count];
uint32_t n_result_bins = 0;
if ((m->info1 & AS_MSG_INFO1_GET_ALL) != 0) {
p_ops = NULL;
n_bins = rd.n_bins;
as_bin_get_all_p(&rd, response_bins);
}
else {
if (m->n_ops == 0) {
cf_warning_digest(AS_RW, &tr->keyd, "{%s} read_local: bin op(s) expected, none present ", ns->name);
read_local_done(tr, &r_ref, &rd, AS_ERR_PARAMETER);
return TRANS_DONE_ERROR;
}
bool respond_all_ops = (m->info2 & AS_MSG_INFO2_RESPOND_ALL_OPS) != 0;
as_msg_op* op = 0;
int n = 0;
while ((op = as_msg_op_iterate(m, op, &n)) != NULL) {
if (op->op == AS_MSG_OP_READ) {
as_bin* b = as_bin_get_from_buf(&rd, op->name, op->name_sz);
if (b || respond_all_ops) {
ops[n_bins] = op;
response_bins[n_bins++] = b;
}
}
else if (op->op == AS_MSG_OP_CDT_READ) {
as_bin* b = as_bin_get_from_buf(&rd, op->name, op->name_sz);
if (b) {
as_bin* rb = &result_bins[n_result_bins];
as_bin_set_empty(rb);
if ((result = as_bin_cdt_read_from_client(b, op, rb)) < 0) {
cf_warning_digest(AS_RW, &tr->keyd, "{%s} read_local: failed as_bin_cdt_read_from_client() ", ns->name);
destroy_stack_bins(result_bins, n_result_bins);
read_local_done(tr, &r_ref, &rd, -result);
return TRANS_DONE_ERROR;
}
if (as_bin_inuse(rb)) {
n_result_bins++;
ops[n_bins] = op;
response_bins[n_bins++] = rb;
}
else if (respond_all_ops) {
ops[n_bins] = op;
response_bins[n_bins++] = NULL;
}
}
else if (respond_all_ops) {
ops[n_bins] = op;
response_bins[n_bins++] = NULL;
}
}
else {
cf_warning_digest(AS_RW, &tr->keyd, "{%s} read_local: unexpected bin op %u ", ns->name, op->op);
destroy_stack_bins(result_bins, n_result_bins);
read_local_done(tr, &r_ref, &rd, AS_ERR_PARAMETER);
return TRANS_DONE_ERROR;
}
}
}
cf_dyn_buf_define_size(db, 16 * 1024);
if (tr->origin != FROM_BATCH) {
db.used_sz = db.alloc_sz;
db.buf = (uint8_t*)as_msg_make_response_msg(tr->result_code,
r->generation, r->void_time, p_ops, response_bins, n_bins, ns,
(cl_msg*)dyn_bufdb, &db.used_sz, as_transaction_trid(tr));
db.is_stack = db.buf == dyn_bufdb;
// Note - not bothering to correct alloc_sz if buf was allocated.
}
else {
tr->generation = r->generation;
tr->void_time = r->void_time;
tr->last_update_time = r->last_update_time;
// Since as_batch_add_result() constructs response directly in shared
// buffer to avoid extra copies, can't use db.
send_read_response(tr, p_ops, response_bins, n_bins, NULL);
}
destroy_stack_bins(result_bins, n_result_bins);
as_storage_record_close(&rd);
as_record_done(&r_ref, ns);
// Now that we're not under the record lock, send the message we just built.
if (db.used_sz != 0) {
send_read_response(tr, NULL, NULL, 0, &db);
cf_dyn_buf_free(&db);
tr->from.proto_fd_h = NULL;
}
return TRANS_DONE_SUCCESS;
}
/* Workhorse function to send response back to the client after UDF execution.
*
* Assumption: The call should be setup properly pointing to the tr.
*
* Special Handling: If it is background udf job do not send any
* response to client
*/
int
process_response(udf_call *call, const char *bin_name, const as_val *val, cf_dyn_buf *db)
{
// NO response if background UDF
if (call->def->type == AS_UDF_OP_BACKGROUND) {
return 0;
}
// Note - this function quietly handles a null val. The response call will
// be given a bin with a name but not 'in use', and it does the right thing.
as_bin stack_bin;
as_bin *bin = &stack_bin;
uint32_t particle_size = as_particle_size_from_asval(val);
static const size_t MAX_STACK_SIZE = 32 * 1024;
uint8_t stack_particle[particle_size > MAX_STACK_SIZE ? 0 : particle_size];
uint8_t *particle_buf = stack_particle;
if (particle_size > MAX_STACK_SIZE) {
particle_buf = (uint8_t *)cf_malloc(particle_size);
if (! particle_buf) {
cf_warning(AS_UDF, "failed alloc for particle size %u", particle_size);
return -1;
}
}
as_transaction *tr = call->tr;
as_namespace *ns = tr->rsv.ns;
as_bin_init(ns, bin, bin_name);
as_bin_particle_stack_from_asval(bin, particle_buf, val);
if (db) {
size_t msg_sz = 0;
uint8_t *msgp = (uint8_t *)as_msg_make_response_msg(tr->result_code,
tr->generation, tr->void_time, NULL, &bin, 1, ns, NULL, &msg_sz,
as_transaction_trid(tr), NULL);
if (! msgp) {
cf_warning_digest(AS_RW, &tr->keyd, "{%s} UDF failed to make response msg ", ns->name);
if (particle_buf != stack_particle) {
cf_free(particle_buf);
}
return -1;
}
// Stash the message, to be sent later.
db->buf = msgp;
db->is_stack = false;
db->alloc_sz = msg_sz;
db->used_sz = msg_sz;
}
else {
single_transaction_response(tr, ns, NULL, &bin, 1, tr->generation, tr->void_time, NULL, NULL);
}
if (particle_buf != stack_particle) {
cf_free(particle_buf);
}
return 0;
}
// Set of threads which talk to client over the connection for doing the needful
// processing. Note that once fd is assigned to a thread all the work on that fd
// is done by that thread. Fair fd usage is expected of the client. First thread
// is special - also does accept [listens for new connections]. It is the only
// thread which does it.
void *
thr_demarshal(void *arg)
{
cf_socket_cfg *s, *ls, *xs;
// Create my epoll fd, register in the global list.
struct epoll_event ev;
int nevents, i, n, epoll_fd;
cf_clock last_fd_print = 0;
#if defined(USE_SYSTEMTAP)
uint64_t nodeid = g_config.self_node;
#endif
// Early stage aborts; these will cause faults in process scope.
cf_assert(arg, AS_DEMARSHAL, CF_CRITICAL, "invalid argument");
s = &g_config.socket;
ls = &g_config.localhost_socket;
xs = &g_config.xdr_socket;
#ifdef USE_JEM
int orig_arena;
if (0 > (orig_arena = jem_get_arena())) {
cf_crash(AS_DEMARSHAL, "Failed to get original arena for thr_demarshal()!");
} else {
cf_info(AS_DEMARSHAL, "Saved original JEMalloc arena #%d for thr_demarshal()", orig_arena);
}
#endif
// Figure out my thread index.
pthread_t self = pthread_self();
int thr_id;
for (thr_id = 0; thr_id < MAX_DEMARSHAL_THREADS; thr_id++) {
if (0 != pthread_equal(g_demarshal_args->dm_th[thr_id], self))
break;
}
if (thr_id == MAX_DEMARSHAL_THREADS) {
cf_debug(AS_FABRIC, "Demarshal thread could not figure own ID, bogus, exit, fu!");
return(0);
}
// First thread accepts new connection at interface socket.
if (thr_id == 0) {
demarshal_file_handle_init();
epoll_fd = epoll_create(EPOLL_SZ);
if (epoll_fd == -1) {
cf_crash(AS_DEMARSHAL, "epoll_create(): %s", cf_strerror(errno));
}
memset(&ev, 0, sizeof (ev));
ev.events = EPOLLIN | EPOLLERR | EPOLLHUP;
ev.data.fd = s->sock;
if (0 > epoll_ctl(epoll_fd, EPOLL_CTL_ADD, s->sock, &ev)) {
cf_crash(AS_DEMARSHAL, "epoll_ctl(): %s", cf_strerror(errno));
}
cf_info(AS_DEMARSHAL, "Service started: socket %s:%d", s->addr, s->port);
if (ls->sock) {
ev.events = EPOLLIN | EPOLLERR | EPOLLHUP;
ev.data.fd = ls->sock;
if (0 > epoll_ctl(epoll_fd, EPOLL_CTL_ADD, ls->sock, &ev)) {
cf_crash(AS_DEMARSHAL, "epoll_ctl(): %s", cf_strerror(errno));
}
cf_info(AS_DEMARSHAL, "Service also listening on localhost socket %s:%d", ls->addr, ls->port);
}
if (xs->sock) {
ev.events = EPOLLIN | EPOLLERR | EPOLLHUP;
ev.data.fd = xs->sock;
if (0 > epoll_ctl(epoll_fd, EPOLL_CTL_ADD, xs->sock, &ev)) {
cf_crash(AS_DEMARSHAL, "epoll_ctl(): %s", cf_strerror(errno));
}
cf_info(AS_DEMARSHAL, "Service also listening on XDR info socket %s:%d", xs->addr, xs->port);
}
}
else {
epoll_fd = epoll_create(EPOLL_SZ);
if (epoll_fd == -1) {
cf_crash(AS_DEMARSHAL, "epoll_create(): %s", cf_strerror(errno));
}
}
g_demarshal_args->epoll_fd[thr_id] = epoll_fd;
cf_detail(AS_DEMARSHAL, "demarshal thread started: id %d", thr_id);
int id_cntr = 0;
// Demarshal transactions from the socket.
for ( ; ; ) {
struct epoll_event events[EPOLL_SZ];
cf_detail(AS_DEMARSHAL, "calling epoll");
nevents = epoll_wait(epoll_fd, events, EPOLL_SZ, -1);
if (0 > nevents) {
cf_debug(AS_DEMARSHAL, "epoll_wait() returned %d ; errno = %d (%s)", nevents, errno, cf_strerror(errno));
}
cf_detail(AS_DEMARSHAL, "epoll event received: nevents %d", nevents);
uint64_t now_ns = cf_getns();
uint64_t now_ms = now_ns / 1000000;
// Iterate over all events.
for (i = 0; i < nevents; i++) {
if ((s->sock == events[i].data.fd) || (ls->sock == events[i].data.fd) || (xs->sock == events[i].data.fd)) {
// Accept new connections on the service socket.
int csocket = -1;
struct sockaddr_in caddr;
socklen_t clen = sizeof(caddr);
char cpaddr[64];
if (-1 == (csocket = accept(events[i].data.fd, (struct sockaddr *)&caddr, &clen))) {
// This means we're out of file descriptors - could be a SYN
// flood attack or misbehaving client. Eventually we'd like
// to make the reaper fairer, but for now we'll just have to
// ignore the accept error and move on.
if ((errno == EMFILE) || (errno == ENFILE)) {
if (last_fd_print != (cf_getms() / 1000L)) {
cf_warning(AS_DEMARSHAL, "Hit OS file descriptor limit (EMFILE on accept). Consider raising limit for uid %d", g_config.uid);
last_fd_print = cf_getms() / 1000L;
}
continue;
}
cf_crash(AS_DEMARSHAL, "accept: %s (errno %d)", cf_strerror(errno), errno);
}
// Get the client IP address in string form.
if (caddr.sin_family == AF_INET) {
if (NULL == inet_ntop(AF_INET, &caddr.sin_addr.s_addr, (char *)cpaddr, sizeof(cpaddr))) {
cf_crash(AS_DEMARSHAL, "inet_ntop(): %s (errno %d)", cf_strerror(errno), errno);
}
}
else if (caddr.sin_family == AF_INET6) {
struct sockaddr_in6* addr_in6 = (struct sockaddr_in6*)&caddr;
if (NULL == inet_ntop(AF_INET6, &addr_in6->sin6_addr, (char *)cpaddr, sizeof(cpaddr))) {
cf_crash(AS_DEMARSHAL, "inet_ntop(): %s (errno %d)", cf_strerror(errno), errno);
}
}
else {
cf_crash(AS_DEMARSHAL, "unknown address family %u", caddr.sin_family);
}
cf_detail(AS_DEMARSHAL, "new connection: %s (fd %d)", cpaddr, csocket);
// Validate the limit of protocol connections we allow.
uint32_t conns_open = g_stats.proto_connections_opened - g_stats.proto_connections_closed;
if (xs->sock != events[i].data.fd && conns_open > g_config.n_proto_fd_max) {
if ((last_fd_print + 5000L) < cf_getms()) { // no more than 5 secs
cf_warning(AS_DEMARSHAL, "dropping incoming client connection: hit limit %d connections", conns_open);
last_fd_print = cf_getms();
}
shutdown(csocket, SHUT_RDWR);
close(csocket);
csocket = -1;
continue;
}
// Set the socket to nonblocking.
if (-1 == cf_socket_set_nonblocking(csocket)) {
cf_info(AS_DEMARSHAL, "unable to set client socket to nonblocking mode");
shutdown(csocket, SHUT_RDWR);
close(csocket);
csocket = -1;
continue;
}
// Create as_file_handle and queue it up in epoll_fd for further
// communication on one of the demarshal threads.
as_file_handle *fd_h = cf_rc_alloc(sizeof(as_file_handle));
if (!fd_h) {
cf_crash(AS_DEMARSHAL, "malloc");
}
sprintf(fd_h->client, "%s:%d", cpaddr, ntohs(caddr.sin_port));
fd_h->fd = csocket;
fd_h->last_used = cf_getms();
fd_h->reap_me = false;
fd_h->trans_active = false;
fd_h->proto = 0;
fd_h->proto_unread = 0;
fd_h->fh_info = 0;
fd_h->security_filter = as_security_filter_create();
// Insert into the global table so the reaper can manage it. Do
// this before queueing it up for demarshal threads - once
// EPOLL_CTL_ADD is done it's difficult to back out (if insert
// into global table fails) because fd state could be anything.
cf_rc_reserve(fd_h);
pthread_mutex_lock(&g_file_handle_a_LOCK);
int j;
bool inserted = true;
if (0 != cf_queue_pop(g_freeslot, &j, CF_QUEUE_NOWAIT)) {
inserted = false;
}
else {
g_file_handle_a[j] = fd_h;
}
pthread_mutex_unlock(&g_file_handle_a_LOCK);
if (!inserted) {
cf_info(AS_DEMARSHAL, "unable to add socket to file handle table");
shutdown(csocket, SHUT_RDWR);
close(csocket);
csocket = -1;
cf_rc_free(fd_h); // will free even with ref-count of 2
}
else {
// Place the client socket in the event queue.
memset(&ev, 0, sizeof(ev));
ev.events = EPOLLIN | EPOLLET | EPOLLRDHUP ;
ev.data.ptr = fd_h;
// Round-robin pick up demarshal thread epoll_fd and add
// this new connection to epoll.
int id = (id_cntr++) % g_demarshal_args->num_threads;
fd_h->epoll_fd = g_demarshal_args->epoll_fd[id];
if (0 > (n = epoll_ctl(fd_h->epoll_fd, EPOLL_CTL_ADD, csocket, &ev))) {
cf_info(AS_DEMARSHAL, "unable to add socket to event queue of demarshal thread %d %d", id, g_demarshal_args->num_threads);
pthread_mutex_lock(&g_file_handle_a_LOCK);
fd_h->reap_me = true;
as_release_file_handle(fd_h);
fd_h = 0;
pthread_mutex_unlock(&g_file_handle_a_LOCK);
}
else {
cf_atomic64_incr(&g_stats.proto_connections_opened);
}
}
}
else {
bool has_extra_ref = false;
as_file_handle *fd_h = events[i].data.ptr;
if (fd_h == 0) {
cf_info(AS_DEMARSHAL, "event with null handle, continuing");
goto NextEvent;
}
cf_detail(AS_DEMARSHAL, "epoll connection event: fd %d, events 0x%x", fd_h->fd, events[i].events);
// Process data on an existing connection: this might be more
// activity on an already existing transaction, so we have some
// state to manage.
as_proto *proto_p = 0;
int fd = fd_h->fd;
if (events[i].events & (EPOLLRDHUP | EPOLLERR | EPOLLHUP)) {
cf_detail(AS_DEMARSHAL, "proto socket: remote close: fd %d event %x", fd, events[i].events);
// no longer in use: out of epoll etc
goto NextEvent_FD_Cleanup;
}
if (fd_h->trans_active) {
goto NextEvent;
}
// If pointer is NULL, then we need to create a transaction and
// store it in the buffer.
if (fd_h->proto == NULL) {
as_proto proto;
int sz;
/* Get the number of available bytes */
if (-1 == ioctl(fd, FIONREAD, &sz)) {
cf_info(AS_DEMARSHAL, "unable to get number of available bytes");
goto NextEvent_FD_Cleanup;
}
// If we don't have enough data to fill the message buffer,
// just wait and we'll come back to this one. However, we'll
// let messages with zero size through, since they are
// likely errors. We don't cleanup the FD in this case since
// we'll get more data on it.
if (sz < sizeof(as_proto) && sz != 0) {
goto NextEvent;
}
// Do a preliminary read of the header into a stack-
// allocated structure, so that later on we can allocate the
// entire message buffer.
if (0 >= (n = cf_socket_recv(fd, &proto, sizeof(as_proto), MSG_WAITALL))) {
cf_detail(AS_DEMARSHAL, "proto socket: read header fail: error: rv %d sz was %d errno %d", n, sz, errno);
goto NextEvent_FD_Cleanup;
}
if (proto.version != PROTO_VERSION &&
// For backward compatibility, allow version 0 with
// security messages.
! (proto.version == 0 && proto.type == PROTO_TYPE_SECURITY)) {
cf_warning(AS_DEMARSHAL, "proto input from %s: unsupported proto version %u",
fd_h->client, proto.version);
goto NextEvent_FD_Cleanup;
}
// Swap the necessary elements of the as_proto.
as_proto_swap(&proto);
if (proto.sz > PROTO_SIZE_MAX) {
cf_warning(AS_DEMARSHAL, "proto input from %s: msg greater than %d, likely request from non-Aerospike client, rejecting: sz %"PRIu64,
fd_h->client, PROTO_SIZE_MAX, (uint64_t)proto.sz);
goto NextEvent_FD_Cleanup;
}
#ifdef USE_JEM
// Attempt to peek the namespace and set the JEMalloc arena accordingly.
size_t peeked_data_sz = 0;
size_t min_field_sz = sizeof(uint32_t) + sizeof(char);
size_t min_as_msg_sz = sizeof(as_msg) + min_field_sz;
size_t peekbuf_sz = 2048; // (Arbitrary "large enough" size for peeking the fields of "most" AS_MSGs.)
uint8_t peekbuf[peekbuf_sz];
if (PROTO_TYPE_AS_MSG == proto.type) {
size_t offset = sizeof(as_msg);
// Number of bytes to peek from the socket.
// size_t peek_sz = peekbuf_sz; // Peak up to the size of the peek buffer.
size_t peek_sz = MIN(proto.sz, peekbuf_sz); // Peek only up to the minimum necessary number of bytes.
if (!(peeked_data_sz = cf_socket_recv(fd, peekbuf, peek_sz, 0))) {
// That's actually legitimate. The as_proto may have gone into one
// packet, the as_msg into the next one, which we haven't yet received.
// This just "never happened" without async.
cf_detail(AS_DEMARSHAL, "could not peek the as_msg header, expected %zu byte(s)", peek_sz);
}
if (peeked_data_sz > min_as_msg_sz) {
// cf_debug(AS_DEMARSHAL, "(Peeked %zu bytes.)", peeked_data_sz);
if (peeked_data_sz > proto.sz) {
cf_warning(AS_DEMARSHAL, "Received unexpected extra data from client %s socket %d when peeking as_proto!", fd_h->client, fd);
log_as_proto_and_peeked_data(&proto, peekbuf, peeked_data_sz);
goto NextEvent_FD_Cleanup;
}
if (((as_msg*)peekbuf)->info1 & AS_MSG_INFO1_BATCH) {
jem_set_arena(orig_arena);
} else {
uint16_t n_fields = ntohs(((as_msg *) peekbuf)->n_fields), field_num = 0;
bool found = false;
// cf_debug(AS_DEMARSHAL, "Found %d AS_MSG fields", n_fields);
while (!found && (field_num < n_fields)) {
as_msg_field *field = (as_msg_field *) (&peekbuf[offset]);
uint32_t value_sz = ntohl(field->field_sz) - 1;
// cf_debug(AS_DEMARSHAL, "Field #%d offset: %lu", field_num, offset);
// cf_debug(AS_DEMARSHAL, "\tvalue_sz %u", value_sz);
// cf_debug(AS_DEMARSHAL, "\ttype %d", field->type);
if (AS_MSG_FIELD_TYPE_NAMESPACE == field->type) {
if (value_sz >= AS_ID_NAMESPACE_SZ) {
cf_warning(AS_DEMARSHAL, "namespace too long (%u) in as_msg", value_sz);
goto NextEvent_FD_Cleanup;
}
char ns[AS_ID_NAMESPACE_SZ];
found = true;
memcpy(ns, field->data, value_sz);
ns[value_sz] = '\0';
// cf_debug(AS_DEMARSHAL, "Found ns \"%s\" in field #%d.", ns, field_num);
jem_set_arena(as_namespace_get_jem_arena(ns));
} else {
// cf_debug(AS_DEMARSHAL, "Message field %d is not namespace (type %d) ~~ Reading next field", field_num, field->type);
field_num++;
offset += sizeof(as_msg_field) + value_sz;
if (offset >= peeked_data_sz) {
break;
}
}
}
if (!found) {
cf_warning(AS_DEMARSHAL, "Can't get namespace from AS_MSG (peeked %zu bytes) ~~ Using default thr_demarshal arena.", peeked_data_sz);
jem_set_arena(orig_arena);
}
}
} else {
jem_set_arena(orig_arena);
}
} else {
jem_set_arena(orig_arena);
}
#endif
// Allocate the complete message buffer.
proto_p = cf_malloc(sizeof(as_proto) + proto.sz);
cf_assert(proto_p, AS_DEMARSHAL, CF_CRITICAL, "allocation: %zu %s", (sizeof(as_proto) + proto.sz), cf_strerror(errno));
memcpy(proto_p, &proto, sizeof(as_proto));
#ifdef USE_JEM
// Jam in the peeked data.
if (peeked_data_sz) {
memcpy(proto_p->data, &peekbuf, peeked_data_sz);
}
fd_h->proto_unread = proto_p->sz - peeked_data_sz;
#else
fd_h->proto_unread = proto_p->sz;
#endif
fd_h->proto = (void *) proto_p;
}
else {
proto_p = fd_h->proto;
}
if (fd_h->proto_unread > 0) {
// Read the data.
n = cf_socket_recv(fd, proto_p->data + (proto_p->sz - fd_h->proto_unread), fd_h->proto_unread, 0);
if (0 >= n) {
if (errno == EAGAIN) {
continue;
}
cf_info(AS_DEMARSHAL, "receive socket: fail? n %d errno %d %s closing connection.", n, errno, cf_strerror(errno));
goto NextEvent_FD_Cleanup;
}
// Decrement bytes-unread counter.
cf_detail(AS_DEMARSHAL, "read fd %d (%d %"PRIu64")", fd, n, fd_h->proto_unread);
fd_h->proto_unread -= n;
}
// Check for a finished read.
if (0 == fd_h->proto_unread) {
// It's only really live if it's injecting a transaction.
fd_h->last_used = now_ms;
thr_demarshal_pause(fd_h); // pause reading while the transaction is in progress
fd_h->proto = 0;
fd_h->proto_unread = 0;
cf_rc_reserve(fd_h);
has_extra_ref = true;
// Info protocol requests.
if (proto_p->type == PROTO_TYPE_INFO) {
as_info_transaction it = { fd_h, proto_p, now_ns };
as_info(&it);
goto NextEvent;
}
// INIT_TR
as_transaction tr;
as_transaction_init_head(&tr, NULL, (cl_msg *)proto_p);
tr.origin = FROM_CLIENT;
tr.from.proto_fd_h = fd_h;
tr.start_time = now_ns;
if (! as_proto_is_valid_type(proto_p)) {
cf_warning(AS_DEMARSHAL, "unsupported proto message type %u", proto_p->type);
// We got a proto message type we don't recognize, so it
// may not do any good to send back an as_msg error, but
// it's the best we can do. At least we can keep the fd.
as_transaction_demarshal_error(&tr, AS_PROTO_RESULT_FAIL_UNKNOWN);
goto NextEvent;
}
// Check if it's compressed.
if (tr.msgp->proto.type == PROTO_TYPE_AS_MSG_COMPRESSED) {
// Decompress it - allocate buffer to hold decompressed
// packet.
uint8_t *decompressed_buf = NULL;
size_t decompressed_buf_size = 0;
int rv = 0;
if ((rv = as_packet_decompression((uint8_t *)proto_p, &decompressed_buf, &decompressed_buf_size))) {
cf_warning(AS_DEMARSHAL, "as_proto decompression failed! (rv %d)", rv);
cf_warning_binary(AS_DEMARSHAL, proto_p, sizeof(as_proto) + proto_p->sz, CF_DISPLAY_HEX_SPACED, "compressed proto_p");
as_transaction_demarshal_error(&tr, AS_PROTO_RESULT_FAIL_UNKNOWN);
goto NextEvent;
}
// Free the compressed packet since we'll be using the
// decompressed packet from now on.
cf_free(proto_p);
proto_p = NULL;
// Get original packet.
tr.msgp = (cl_msg *)decompressed_buf;
as_proto_swap(&(tr.msgp->proto));
if (! as_proto_wrapped_is_valid(&tr.msgp->proto, decompressed_buf_size)) {
cf_warning(AS_DEMARSHAL, "decompressed unusable proto: version %u, type %u, sz %lu [%lu]",
tr.msgp->proto.version, tr.msgp->proto.type, (uint64_t)tr.msgp->proto.sz, decompressed_buf_size);
as_transaction_demarshal_error(&tr, AS_PROTO_RESULT_FAIL_UNKNOWN);
goto NextEvent;
}
}
// If it's an XDR connection and we haven't yet modified the connection settings, ...
if (tr.msgp->proto.type == PROTO_TYPE_AS_MSG &&
as_transaction_is_xdr(&tr) &&
(fd_h->fh_info & FH_INFO_XDR) == 0) {
// ... modify them.
if (thr_demarshal_config_xdr(fd_h->fd) != 0) {
cf_warning(AS_DEMARSHAL, "Failed to configure XDR connection");
goto NextEvent_FD_Cleanup;
}
fd_h->fh_info |= FH_INFO_XDR;
}
// Security protocol transactions.
if (tr.msgp->proto.type == PROTO_TYPE_SECURITY) {
as_security_transact(&tr);
goto NextEvent;
}
// For now only AS_MSG's contribute to this benchmark.
if (g_config.svc_benchmarks_enabled) {
tr.benchmark_time = histogram_insert_data_point(g_stats.svc_demarshal_hist, now_ns);
}
// Fast path for batch requests.
if (tr.msgp->msg.info1 & AS_MSG_INFO1_BATCH) {
as_batch_queue_task(&tr);
goto NextEvent;
}
// Swap as_msg fields and bin-ops to host order, and flag
// which fields are present, to reduce re-parsing.
if (! as_transaction_demarshal_prepare(&tr)) {
as_transaction_demarshal_error(&tr, AS_PROTO_RESULT_FAIL_PARAMETER);
goto NextEvent;
}
ASD_TRANS_DEMARSHAL(nodeid, (uint64_t) tr.msgp, as_transaction_trid(&tr));
// Either process the transaction directly in this thread,
// or queue it for processing by another thread (tsvc/info).
if (0 != thr_tsvc_process_or_enqueue(&tr)) {
cf_warning(AS_DEMARSHAL, "Failed to queue transaction to the service thread");
goto NextEvent_FD_Cleanup;
}
}
// Jump the proto message free & FD cleanup. If we get here, the
// above operations went smoothly. The message free & FD cleanup
// job is handled elsewhere as directed by
// thr_tsvc_process_or_enqueue().
goto NextEvent;
NextEvent_FD_Cleanup:
// If we allocated memory for the incoming message, free it.
if (proto_p) {
cf_free(proto_p);
fd_h->proto = 0;
}
// If fd has extra reference for transaction, release it.
if (has_extra_ref) {
cf_rc_release(fd_h);
}
// Remove the fd from the events list.
if (epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, 0) < 0) {
cf_crash(AS_DEMARSHAL, "unable to remove socket FD %d from epoll instance FD %d: %d (%s)",
fd, epoll_fd, errno, cf_strerror(errno));
}
pthread_mutex_lock(&g_file_handle_a_LOCK);
fd_h->reap_me = true;
as_release_file_handle(fd_h);
fd_h = 0;
pthread_mutex_unlock(&g_file_handle_a_LOCK);
NextEvent:
;
}
// We should never be canceled externally, but just in case...
pthread_testcancel();
}
}
return NULL;
}
