static void eventer_epoll_impl_trigger(eventer_t e, int mask) {
struct epoll_spec *spec;
struct timeval __now;
int fd, newmask;
const char *cbname;
ev_lock_state_t lockstate;
int cross_thread = mask & EVENTER_CROSS_THREAD_TRIGGER;
int added_to_master_fds = 0;
u_int64_t start, duration;
mask = mask & ~(EVENTER_RESERVED);
fd = e->fd;
if(cross_thread) {
if(master_fds[fd].e != NULL) {
mtevL(eventer_deb, "Attempting to trigger already-registered event fd: %d cross thread.\n", fd);
}
/* mtevAssert(master_fds[fd].e == NULL); */
}
if(!pthread_equal(pthread_self(), e->thr_owner)) {
/* If we're triggering across threads, it can't be registered yet */
if(master_fds[fd].e != NULL) {
mtevL(eventer_deb, "Attempting to trigger already-registered event fd: %d cross thread.\n", fd);
}
/* mtevAssert(master_fds[fd].e == NULL); */
eventer_cross_thread_trigger(e,mask);
return;
}
if(master_fds[fd].e == NULL) {
master_fds[fd].e = e;
e->mask = 0;
added_to_master_fds = 1;
}
if(e != master_fds[fd].e) return;
lockstate = acquire_master_fd(fd);
if(lockstate == EV_ALREADY_OWNED) return;
mtevAssert(lockstate == EV_OWNED);
mtev_gettimeofday(&__now, NULL);
cbname = eventer_name_for_callback_e(e->callback, e);
mtevLT(eventer_deb, &__now, "epoll: fire on %d/%x to %s(%p)\n",
fd, mask, cbname?cbname:"???", e->callback);
mtev_memory_begin();
LIBMTEV_EVENTER_CALLBACK_ENTRY((void *)e, (void *)e->callback, (char *)cbname, fd, e->mask, mask);
start = mtev_gethrtime();
newmask = e->callback(e, mask, e->closure, &__now);
duration = mtev_gethrtime() - start;
LIBMTEV_EVENTER_CALLBACK_RETURN((void *)e, (void *)e->callback, (char *)cbname, newmask);
mtev_memory_end();
stats_set_hist_intscale(eventer_callback_latency, duration, -9, 1);
stats_set_hist_intscale(eventer_latency_handle_for_callback(e->callback), duration, -9, 1);
if(newmask) {
struct epoll_event _ev;
memset(&_ev, 0, sizeof(_ev));
_ev.data.fd = fd;
if(newmask & EVENTER_READ) _ev.events |= (EPOLLIN|EPOLLPRI);
if(newmask & EVENTER_WRITE) _ev.events |= (EPOLLOUT);
if(newmask & EVENTER_EXCEPTION) _ev.events |= (EPOLLERR|EPOLLHUP);
if(master_fds[fd].e == NULL) {
mtevL(mtev_debug, "eventer %s(%p) epoll asked to modify descheduled fd: %d\n",
cbname?cbname:"???", e->callback, fd);
} else {
if(!pthread_equal(pthread_self(), e->thr_owner)) {
pthread_t tgt = e->thr_owner;
e->thr_owner = pthread_self();
spec = eventer_get_spec_for_event(e);
if(! added_to_master_fds && epoll_ctl(spec->epoll_fd, EPOLL_CTL_DEL, fd, &_ev) != 0) {
mtevFatal(mtev_error,
"epoll_ctl(spec->epoll_fd, EPOLL_CTL_DEL, fd, &_ev) failed; "
"spec->epoll_fd: %d; fd: %d; errno: %d (%s)\n",
spec->epoll_fd, fd, errno, strerror(errno));
}
e->thr_owner = tgt;
spec = eventer_get_spec_for_event(e);
mtevAssert(epoll_ctl(spec->epoll_fd, EPOLL_CTL_ADD, fd, &_ev) == 0);
mtevL(eventer_deb, "moved event[%p] from t@%d to t@%d\n", e, (int)pthread_self(), (int)tgt);
}
else {
int epoll_cmd = added_to_master_fds ? EPOLL_CTL_ADD : EPOLL_CTL_MOD;
spec = eventer_get_spec_for_event(e);
if(epoll_ctl(spec->epoll_fd, epoll_cmd, fd, &_ev) != 0) {
const char *cb_name = eventer_name_for_callback_e(e->callback, e);
mtevFatal(mtev_error,
"epoll_ctl(spec->epoll_fd, EPOLL_CTL_MOD, fd, &_ev) failed; "
"spec->epoll_fd: %d; fd: %d; errno: %d (%s); callback: %s\n",
spec->epoll_fd, fd, errno, strerror(errno), cb_name ? cb_name : "???");
}
}
}
/* Set our mask */
e->mask = newmask;
}
else {
/* see kqueue implementation for details on the next line */
if(master_fds[fd].e == e) master_fds[fd].e = NULL;
eventer_free(e);
}
release_master_fd(fd, lockstate);
}
static int eventer_epoll_impl_loop() {
struct epoll_event *epev;
struct epoll_spec *spec;
spec = eventer_get_spec_for_event(NULL);
epev = malloc(sizeof(*epev) * maxfds);
#ifdef HAVE_SYS_EVENTFD_H
if(spec->event_fd >= 0) {
eventer_t e = eventer_alloc();
e->callback = eventer_epoll_eventfd_read;
e->fd = spec->event_fd;
e->mask = EVENTER_READ;
eventer_add(e);
}
#endif
while(1) {
struct timeval __now, __sleeptime;
int fd_cnt = 0;
__sleeptime = eventer_max_sleeptime;
mtev_gettimeofday(&__now, NULL);
eventer_dispatch_timed(&__now, &__sleeptime);
/* Handle cross_thread dispatches */
eventer_cross_thread_process();
/* Handle recurrent events */
eventer_dispatch_recurrent(&__now);
/* Now we move on to our fd-based events */
do {
fd_cnt = epoll_wait(spec->epoll_fd, epev, maxfds,
__sleeptime.tv_sec * 1000 + __sleeptime.tv_usec / 1000);
} while(fd_cnt < 0 && errno == EINTR);
mtevLT(eventer_deb, &__now, "debug: epoll_wait(%d, [], %d) => %d\n",
spec->epoll_fd, maxfds, fd_cnt);
if(fd_cnt < 0) {
mtevLT(eventer_err, &__now, "epoll_wait: %s\n", strerror(errno));
}
else {
int idx;
/* loop once to clear */
for(idx = 0; idx < fd_cnt; idx++) {
struct epoll_event *ev;
eventer_t e;
int fd, mask = 0;
ev = &epev[idx];
if(ev->events & (EPOLLIN | EPOLLPRI)) mask |= EVENTER_READ;
if(ev->events & (EPOLLOUT)) mask |= EVENTER_WRITE;
if(ev->events & (EPOLLERR|EPOLLHUP)) mask |= EVENTER_EXCEPTION;
fd = ev->data.fd;
e = master_fds[fd].e;
/* It's possible that someone removed the event and freed it
* before we got here.
*/
if(!e) continue;
eventer_epoll_impl_trigger(e, mask);
}
}
}
/* NOTREACHED */
return 0;
}
static int ping_icmp_handler(eventer_t e, int mask,
void *closure, struct timeval *now,
u_int8_t family) {
noit_module_t *self = (noit_module_t *)closure;
ping_icmp_data_t *ping_data;
struct check_info *data;
char packet[1500];
int packet_len = sizeof(packet);
union {
struct sockaddr_in in4;
struct sockaddr_in6 in6;
} from;
unsigned int from_len;
struct ping_payload *payload;
if(family != AF_INET && family != AF_INET6) return EVENTER_READ;
ping_data = noit_module_get_userdata(self);
while(1) {
struct ping_session_key k;
int inlen;
u_int8_t iphlen = 0;
void *vcheck;
noit_check_t *check;
struct timeval tt, whence;
from_len = sizeof(from);
inlen = recvfrom(e->fd, packet, packet_len, 0,
(struct sockaddr *)&from, &from_len);
mtev_gettimeofday(now, NULL); /* set it, as we care about accuracy */
if(inlen < 0) {
if(errno == EAGAIN || errno == EINTR) break;
mtevLT(nldeb, now, "ping_icmp recvfrom: %s\n", strerror(errno));
break;
}
if(family == AF_INET) {
struct icmp *icp4;
iphlen = ((struct ip *)packet)->ip_hl << 2;
if((inlen-iphlen) != sizeof(struct icmp)+PING_PAYLOAD_LEN) {
mtevLT(nldeb, now,
"ping_icmp bad size: %d+%d\n", iphlen, inlen-iphlen);
continue;
}
icp4 = (struct icmp *)(packet + iphlen);
payload = (struct ping_payload *)(icp4 + 1);
if(icp4->icmp_type != ICMP_ECHOREPLY) {
mtevLT(nldeb, now, "ping_icmp bad type: %d\n", icp4->icmp_type);
continue;
}
if(icp4->icmp_id != (((vpsized_uint)self) & 0xffff)) {
mtevLT(nldeb, now,
"ping_icmp not sent from this instance (%d:%d) vs. %lu\n",
icp4->icmp_id, ntohs(icp4->icmp_seq),
(unsigned long)(((vpsized_uint)self) & 0xffff));
continue;
}
}
else if(family == AF_INET6) {
struct icmp6_hdr *icp6 = (struct icmp6_hdr *)packet;
if((inlen) != sizeof(struct icmp6_hdr)+PING_PAYLOAD_LEN) {
mtevLT(nldeb, now,
"ping_icmp bad size: %d+%d\n", iphlen, inlen-iphlen);
continue;
}
payload = (struct ping_payload *)(icp6+1);
if(icp6->icmp6_type != ICMP6_ECHO_REPLY) {
mtevLT(nldeb, now, "ping_icmp bad type: %d\n", icp6->icmp6_type);
continue;
}
if(icp6->icmp6_id != (((vpsized_uint)self) & 0xffff)) {
mtevLT(nldeb, now,
"ping_icmp not sent from this instance (%d:%d) vs. %lu\n",
icp6->icmp6_id, ntohs(icp6->icmp6_seq),
(unsigned long)(((vpsized_uint)self) & 0xffff));
continue;
}
}
else {
/* This should be unreachable */
continue;
}
check = NULL;
k.addr_of_check = payload->addr_of_check;
uuid_copy(k.checkid, payload->checkid);
if(mtev_hash_retrieve(ping_data->in_flight,
(const char *)&k, sizeof(k),
&vcheck))
check = vcheck;
/* make sure this check is from this generation! */
if(!check) {
char uuid_str[37];
uuid_unparse_lower(payload->checkid, uuid_str);
mtevLT(nldeb, now,
"ping_icmp response for unknown check '%s'\n", uuid_str);
continue;
}
if((check->generation & 0xffff) != payload->generation) {
mtevLT(nldeb, now,
"ping_icmp response in generation gap\n");
continue;
}
data = (struct check_info *)check->closure;
/* If there is no timeout_event, the check must have completed.
* We have nothing to do. */
if(!data->timeout_event) continue;
/* Sanity check the payload */
if(payload->check_no != data->check_no) continue;
if(payload->check_pack_cnt != data->expected_count) continue;
if(payload->check_pack_no >= data->expected_count) continue;
whence.tv_sec = payload->tv_sec;
whence.tv_usec = payload->tv_usec;
sub_timeval(*now, whence, &tt);
data->turnaround[payload->check_pack_no] =
(float)tt.tv_sec + (float)tt.tv_usec / 1000000.0;
if(ping_icmp_is_complete(self, check)) {
ping_icmp_log_results(self, check);
eventer_remove(data->timeout_event);
free(data->timeout_event->closure);
eventer_free(data->timeout_event);
data->timeout_event = NULL;
check->flags &= ~NP_RUNNING;
k.addr_of_check = (vpsized_uint)check ^ random_num;
uuid_copy(k.checkid, check->checkid);
mtev_hash_delete(ping_data->in_flight, (const char *)&k,
sizeof(k), free, NULL);
}
}
return EVENTER_READ;
}
static int eventer_kqueue_impl_loop() {
struct timeval __dyna_sleep = { 0, 0 };
KQUEUE_DECL;
KQUEUE_SETUP(NULL);
if(eventer_kqueue_impl_register_wakeup(kqs) == -1) {
mtevFatal(mtev_error, "error in eventer_kqueue_impl_loop: could not eventer_kqueue_impl_register_wakeup\n");
}
while(1) {
struct timeval __now, __sleeptime;
struct timespec __kqueue_sleeptime;
int fd_cnt = 0;
if(compare_timeval(eventer_max_sleeptime, __dyna_sleep) < 0)
__dyna_sleep = eventer_max_sleeptime;
__sleeptime = __dyna_sleep;
eventer_dispatch_timed(&__now, &__sleeptime);
if(compare_timeval(__sleeptime, __dyna_sleep) > 0)
__sleeptime = __dyna_sleep;
/* Handle cross_thread dispatches */
eventer_cross_thread_process();
/* Handle recurrent events */
eventer_dispatch_recurrent(&__now);
/* Now we move on to our fd-based events */
__kqueue_sleeptime.tv_sec = __sleeptime.tv_sec;
__kqueue_sleeptime.tv_nsec = __sleeptime.tv_usec * 1000;
fd_cnt = kevent(kqs->kqueue_fd, ke_vec, ke_vec_used,
ke_vec, ke_vec_a,
&__kqueue_sleeptime);
kqs->wakeup_notify = 0;
if(fd_cnt > 0 || ke_vec_used)
mtevLT(eventer_deb, &__now, "[t@%llx] kevent(%d, [...], %d) => %d\n", (vpsized_int)pthread_self(), kqs->kqueue_fd, ke_vec_used, fd_cnt);
ke_vec_used = 0;
if(fd_cnt < 0) {
mtevLT(eventer_err, &__now, "kevent(s/%d): %s\n", kqs->kqueue_fd, strerror(errno));
}
else if(fd_cnt == 0 ||
(fd_cnt == 1 && ke_vec[0].filter == EVFILT_USER)) {
/* timeout */
if(fd_cnt) eventer_kqueue_impl_register_wakeup(kqs);
add_timeval(__dyna_sleep, __dyna_increment, &__dyna_sleep);
}
else {
int idx;
__dyna_sleep.tv_sec = __dyna_sleep.tv_usec = 0; /* reset */
/* loop once to clear */
for(idx = 0; idx < fd_cnt; idx++) {
struct kevent *ke;
ke = &ke_vec[idx];
if(ke->flags & EV_ERROR) continue;
if(ke->filter == EVFILT_USER) {
eventer_kqueue_impl_register_wakeup(kqs);
continue;
}
masks[ke->ident] = 0;
}
/* Loop again to aggregate */
for(idx = 0; idx < fd_cnt; idx++) {
struct kevent *ke;
ke = &ke_vec[idx];
if(ke->flags & EV_ERROR) continue;
if(ke->filter == EVFILT_USER) continue;
if(ke->filter == EVFILT_READ) masks[ke->ident] |= EVENTER_READ;
if(ke->filter == EVFILT_WRITE) masks[ke->ident] |= EVENTER_WRITE;
}
/* Loop a last time to process */
for(idx = 0; idx < fd_cnt; idx++) {
struct kevent *ke;
eventer_t e;
int fd;
ke = &ke_vec[idx];
if(ke->filter == EVFILT_USER) continue;
if(ke->flags & EV_ERROR) {
if(ke->data != EBADF && ke->data != ENOENT)
mtevLT(eventer_err, &__now, "error [%d]: %s\n",
(int)ke->ident, strerror(ke->data));
continue;
}
mtevAssert((vpsized_int)ke->udata == (vpsized_int)ke->ident);
fd = ke->ident;
e = master_fds[fd].e;
/* If we've seen this fd, don't callback twice */
if(!masks[fd]) continue;
/* It's possible that someone removed the event and freed it
* before we got here.
*/
if(e) eventer_kqueue_impl_trigger(e, masks[fd]);
masks[fd] = 0; /* indicates we've processed this fd */
}
}
}
/* NOTREACHED */
return 0;
}
static void eventer_kqueue_impl_trigger(eventer_t e, int mask) {
ev_lock_state_t lockstate;
struct timeval __now;
int oldmask, newmask;
const char *cbname;
int fd;
u_int64_t start, duration;
int cross_thread = mask & EVENTER_CROSS_THREAD_TRIGGER;
mask = mask & ~(EVENTER_RESERVED);
fd = e->fd;
if(cross_thread) {
if(master_fds[fd].e != NULL) {
mtevL(eventer_deb, "Attempting to trigger already-registered event fd: %d cross thread.\n", fd);
}
/* mtevAssert(master_fds[fd].e == NULL); */
}
if(!pthread_equal(pthread_self(), e->thr_owner)) {
/* If we're triggering across threads, it can't be registered yet */
if(master_fds[fd].e != NULL) {
mtevL(eventer_deb, "Attempting to trigger already-registered event fd: %d cross thread.\n", fd);
}
/* mtevAssert(master_fds[fd].e == NULL); */
eventer_cross_thread_trigger(e,mask);
return;
}
if(master_fds[fd].e == NULL) {
master_fds[fd].e = e;
e->mask = 0;
}
if(e != master_fds[fd].e) return;
lockstate = acquire_master_fd(fd);
if(lockstate == EV_ALREADY_OWNED) return;
mtevAssert(lockstate == EV_OWNED);
mtev_gettimeofday(&__now, NULL);
/* We're going to lie to ourselves. You'd think this should be:
* oldmask = e->mask; However, we just fired with masks[fd], so
* kqueue is clearly looking for all of the events in masks[fd].
* So, we combine them "just to be safe."
*/
oldmask = e->mask | masks[fd];
cbname = eventer_name_for_callback_e(e->callback, e);
mtevLT(eventer_deb, &__now, "kqueue: fire on %d/%x to %s(%p)\n",
fd, masks[fd], cbname?cbname:"???", e->callback);
mtev_memory_begin();
LIBMTEV_EVENTER_CALLBACK_ENTRY((void *)e, (void *)e->callback, (char *)cbname, fd, e->mask, mask);
start = mtev_gethrtime();
newmask = e->callback(e, mask, e->closure, &__now);
duration = mtev_gethrtime() - start;
LIBMTEV_EVENTER_CALLBACK_RETURN((void *)e, (void *)e->callback, (char *)cbname, newmask);
mtev_memory_end();
stats_set_hist_intscale(eventer_callback_latency, duration, -9, 1);
stats_set_hist_intscale(eventer_latency_handle_for_callback(e->callback), duration, -9, 1);
if(newmask) {
if(!pthread_equal(pthread_self(), e->thr_owner)) {
pthread_t tgt = e->thr_owner;
e->thr_owner = pthread_self();
alter_kqueue_mask(e, oldmask, 0);
e->thr_owner = tgt;
mtevL(eventer_deb, "moved event[%p] from t@%llx to t@%llx\n", e, (vpsized_int)pthread_self(), (vpsized_int)tgt);
if(newmask) eventer_cross_thread_trigger(e, newmask & ~(EVENTER_EXCEPTION));
}
else {
if(master_fds[fd].e != e) {
e = master_fds[fd].e;
mtevL(eventer_deb, "%strigger complete [event switched] %d : %x->%x\n", cross_thread ? "[X]" : "", e->fd, master_fds[fd].e->mask, newmask);
} else {
mtevL(eventer_deb, "%strigger complete %d : %x->%x\n", cross_thread ? "[X]" : "", e->fd, oldmask, newmask);
}
alter_kqueue_mask(e, (e->mask == 0 || cross_thread) ? 0 : oldmask, newmask);
/* Set our mask */
e->mask = newmask;
}
}
else {
/*
* Long story long:
* When integrating with a few external event systems, we find
* it difficult to make their use of remove+add as an update
* as it can be recurrent in a single handler call and you cannot
* remove completely from the event system if you are going to
* just update (otherwise the eventer_t in your call stack could
* be stale). What we do is perform a superficial remove, marking
* the mask as 0, but not eventer_remove_fd. Then on an add, if
* we already have an event, we just update the mask (as we
* have not yet returned to the eventer's loop.
* This leaves us in a tricky situation when a remove is called
* and the add doesn't roll in, we return 0 (mask == 0) and hit
* this spot. We have intended to remove the event, but it still
* resides at master_fds[fd].e -- even after we free it.
* So, in the evnet that we return 0 and the event that
* master_fds[fd].e == the event we're about to free... we NULL
* it out.
*/
if(master_fds[fd].e == e) master_fds[fd].e = NULL;
eventer_free(e);
}
release_master_fd(fd, lockstate);
}
static void eventer_epoll_impl_trigger(eventer_t e, int mask) {
struct epoll_spec *spec;
struct timeval __now;
int fd, newmask, needs_add = 0;
const char *cbname;
ev_lock_state_t lockstate;
int cross_thread = mask & EVENTER_CROSS_THREAD_TRIGGER;
uint64_t start, duration;
mask = mask & ~(EVENTER_RESERVED);
fd = e->fd;
if(cross_thread) {
if(master_fds[fd].e != NULL) {
mtevL(eventer_deb, "Attempting to trigger already-registered event fd: %d cross thread.\n", fd);
}
/* mtevAssert(master_fds[fd].e == NULL); */
}
if(!pthread_equal(pthread_self(), e->thr_owner)) {
/* If we're triggering across threads, it can't be registered yet */
if(master_fds[fd].e != NULL) {
mtevL(eventer_deb, "Attempting to trigger already-registered event fd: %d cross thread.\n", fd);
}
/* mtevAssert(master_fds[fd].e == NULL); */
eventer_cross_thread_trigger(e,mask);
return;
}
if(master_fds[fd].e == NULL) {
lockstate = acquire_master_fd(fd);
if (lockstate == EV_ALREADY_OWNED) {
/* The incoming triggered event is already owned by this thread.
* This means our floated event completed before the current
* event handler even exited. So it retriggered recursively
* from inside the event handler.
*
* Treat this special case the same as a cross thread trigger
* and just queue this event to be picked up on the next loop
*/
eventer_cross_thread_trigger(e, mask);
return;
}
/*
* If we are readding the event to the master list here, also do the needful
* with the epoll_ctl.
*
* This can happen in cases where some event was floated and the float
* completed so fast that we finished the job in the same thread
* that it started in. Since we `eventer_remove_fd` before we float
* the re-add here should replace the fd in the epoll_ctl.
*/
master_fds[fd].e = e;
e->mask = 0;
struct epoll_event _ev;
memset(&_ev, 0, sizeof(_ev));
_ev.data.fd = fd;
spec = eventer_get_spec_for_event(e);
if(mask & EVENTER_READ) _ev.events |= (EPOLLIN|EPOLLPRI);
if(mask & EVENTER_WRITE) _ev.events |= (EPOLLOUT);
if(mask & EVENTER_EXCEPTION) _ev.events |= (EPOLLERR|EPOLLHUP);
mtevL(eventer_deb, "epoll_ctl(%d, add, %d)\n", spec->epoll_fd, fd);
if (epoll_ctl(spec->epoll_fd, EPOLL_CTL_ADD, fd, &_ev) != 0) {
mtevL(mtev_error, "epoll_ctl(%d, add, %d, %d)\n", spec->epoll_fd, fd, errno);
}
release_master_fd(fd, lockstate);
}
if(e != master_fds[fd].e) {
mtevL(mtev_error, "Incoming event: %p, does not match master list: %p\n", e, master_fds[fd].e);
return;
}
lockstate = acquire_master_fd(fd);
if(lockstate == EV_ALREADY_OWNED) {
mtevL(eventer_deb, "Incoming event: %p already owned by this thread\n", e);
return;
}
mtevAssert(lockstate == EV_OWNED);
mtev_gettimeofday(&__now, NULL);
cbname = eventer_name_for_callback_e(e->callback, e);
spec = eventer_get_spec_for_event(e);
mtevLT(eventer_deb, &__now, "epoll(%d): fire on %d/%x to %s(%p)\n",
spec->epoll_fd, fd, mask, cbname?cbname:"???", e->callback);
mtev_memory_begin();
LIBMTEV_EVENTER_CALLBACK_ENTRY((void *)e, (void *)e->callback, (char *)cbname, fd, e->mask, mask);
start = mtev_gethrtime();
newmask = e->callback(e, mask, e->closure, &__now);
duration = mtev_gethrtime() - start;
LIBMTEV_EVENTER_CALLBACK_RETURN((void *)e, (void *)e->callback, (char *)cbname, newmask);
mtev_memory_end();
stats_set_hist_intscale(eventer_callback_latency, duration, -9, 1);
stats_set_hist_intscale(eventer_latency_handle_for_callback(e->callback), duration, -9, 1);
if(newmask) {
struct epoll_event _ev;
memset(&_ev, 0, sizeof(_ev));
_ev.data.fd = fd;
if(newmask & EVENTER_READ) _ev.events |= (EPOLLIN|EPOLLPRI);
if(newmask & EVENTER_WRITE) _ev.events |= (EPOLLOUT);
if(newmask & EVENTER_EXCEPTION) _ev.events |= (EPOLLERR|EPOLLHUP);
if(master_fds[fd].e == NULL) {
mtevL(mtev_debug, "eventer %s(%p) epoll asked to modify descheduled fd: %d\n",
cbname?cbname:"???", e->callback, fd);
} else {
if(!pthread_equal(pthread_self(), e->thr_owner)) {
pthread_t tgt = e->thr_owner;
e->thr_owner = pthread_self();
spec = eventer_get_spec_for_event(e);
if(e->mask != 0 && !needs_add) {
mtevL(eventer_deb, "epoll_ctl(%d, del, %d)\n", spec->epoll_fd, fd);
if(epoll_ctl(spec->epoll_fd, EPOLL_CTL_DEL, fd, &_ev) != 0) {
mtevFatal(mtev_error,
"epoll_ctl(spec->epoll_fd, EPOLL_CTL_DEL, fd, &_ev) failed; "
"spec->epoll_fd: %d; fd: %d; errno: %d (%s)\n",
spec->epoll_fd, fd, errno, strerror(errno));
}
}
e->thr_owner = tgt;
spec = eventer_get_spec_for_event(e);
mtevL(eventer_deb, "epoll_ctl(%d, add, %d)\n", spec->epoll_fd, fd);
mtevAssert(epoll_ctl(spec->epoll_fd, EPOLL_CTL_ADD, fd, &_ev) == 0);
mtevL(eventer_deb, "epoll(%d) moved event[%p] from t@%d to t@%d\n", spec->epoll_fd, e, (int)pthread_self(), (int)tgt);
}
else {
int epoll_rv;
int epoll_cmd = (e->mask == 0 || needs_add) ? EPOLL_CTL_ADD : EPOLL_CTL_MOD;
spec = eventer_get_spec_for_event(e);
mtevL(eventer_deb, "epoll_ctl(%d, %s, %d)\n", spec->epoll_fd, epoll_cmd == EPOLL_CTL_ADD ? "add" : "mod", fd);
epoll_rv = epoll_ctl(spec->epoll_fd, epoll_cmd, fd, &_ev);
if(epoll_rv != 0 &&
((epoll_cmd == EPOLL_CTL_ADD && errno == EEXIST) ||
(epoll_cmd == EPOLL_CTL_MOD && errno == ENOENT))) {
/* try the other way */
epoll_cmd = (epoll_cmd == EPOLL_CTL_ADD) ? EPOLL_CTL_MOD : EPOLL_CTL_ADD;
mtevL(eventer_deb, "retry epoll_ctl(%d, %s, %d)\n", spec->epoll_fd, epoll_cmd == EPOLL_CTL_ADD ? "add" : "mod", fd);
epoll_rv = epoll_ctl(spec->epoll_fd, epoll_cmd, fd, &_ev);
}
if(epoll_rv != 0) {
const char *cb_name = eventer_name_for_callback_e(e->callback, e);
mtevFatal(mtev_error,
"epoll_ctl(spec->epoll_fd, %s, fd, &_ev) failed; "
"spec->epoll_fd: %d; fd: %d; errno: %d (%s); callback: %s\n",
epoll_cmd == EPOLL_CTL_ADD ? "EPOLL_CTL_ADD" : "EPOLL_CTL_MOD",
spec->epoll_fd, fd, errno, strerror(errno), cb_name ? cb_name : "???");
}
}
}
/* Set our mask */
e->mask = newmask;
}
else {
/* see kqueue implementation for details on the next line */
if(master_fds[fd].e == e) {
/* if newmask == 0 the user has floated the connection. If we get here
* and they have not called `eventer_remove_fd` it is a misuse of mtev.
*
* Check if they are compliant with floats here and remove_fd if they
* forgot to and warn in the log
*/
spec = eventer_get_spec_for_event(e);
struct epoll_event _ev;
memset(&_ev, 0, sizeof(_ev));
_ev.data.fd = fd;
if (epoll_ctl(spec->epoll_fd, EPOLL_CTL_DEL, e->fd, &_ev) == 0) {
mtevL(mtev_error, "WARNING: You forgot to 'eventer_remove_fd()' before returning a mask of zero.\n");
}
master_fds[fd].e = NULL;
}
eventer_free(e);
}
release_master_fd(fd, lockstate);
}
static int eventer_ports_impl_loop() {
struct timeval __dyna_sleep = { 0, 0 };
struct ports_spec *spec;
spec = eventer_get_spec_for_event(NULL);
while(1) {
struct timeval __now, __sleeptime;
struct timespec __ports_sleeptime;
unsigned int fd_cnt = 0;
int ret;
port_event_t pevents[MAX_PORT_EVENTS];
mtev_gettimeofday(&__now, NULL);
if(compare_timeval(eventer_max_sleeptime, __dyna_sleep) < 0)
__dyna_sleep = eventer_max_sleeptime;
__sleeptime = __dyna_sleep;
eventer_dispatch_timed(&__now, &__sleeptime);
if(compare_timeval(__sleeptime, __dyna_sleep) > 0)
__sleeptime = __dyna_sleep;
/* Handle cross_thread dispatches */
eventer_cross_thread_process();
/* Handle recurrent events */
eventer_dispatch_recurrent(&__now);
/* Now we move on to our fd-based events */
__ports_sleeptime.tv_sec = __sleeptime.tv_sec;
__ports_sleeptime.tv_nsec = __sleeptime.tv_usec * 1000;
fd_cnt = 1;
pevents[0].portev_source = 65535; /* This is impossible */
ret = port_getn(spec->port_fd, pevents, MAX_PORT_EVENTS, &fd_cnt,
&__ports_sleeptime);
spec->wakeup_notify = 0; /* force unlock */
/* The timeout case is a tad complex with ports. -1/ETIME is clearly
* a timeout. However, it i spossible that we got that and fd_cnt isn't
* 0, which means we both timed out and got events... WTF?
*/
if(fd_cnt == 0 ||
(ret == -1 && errno == ETIME && pevents[0].portev_source == 65535))
add_timeval(__dyna_sleep, __dyna_increment, &__dyna_sleep);
if(ret == -1 && (errno != ETIME && errno != EINTR))
mtevLT(eventer_err, &__now, "port_getn: %s\n", strerror(errno));
if(ret < 0)
mtevLT(eventer_deb, &__now, "port_getn: %s\n", strerror(errno));
mtevLT(eventer_deb, &__now, "debug: port_getn(%d, [], %d) => %d\n",
spec->port_fd, fd_cnt, ret);
if(pevents[0].portev_source == 65535) {
/* the impossible still remains, which means our fd_cnt _must_ be 0 */
fd_cnt = 0;
}
if(fd_cnt > 0) {
int idx;
/* Loop a last time to process */
__dyna_sleep.tv_sec = __dyna_sleep.tv_usec = 0; /* reset */
for(idx = 0; idx < fd_cnt; idx++) {
port_event_t *pe;
eventer_t e;
int fd, mask;
pe = &pevents[idx];
if(pe->portev_source != PORT_SOURCE_FD) continue;
fd = (int)pe->portev_object;
mtevAssert((vpsized_int)pe->portev_user == fd);
e = master_fds[fd].e;
/* It's possible that someone removed the event and freed it
* before we got here.... bail out if we're null.
*/
if (!e) continue;
mask = 0;
if(pe->portev_events & (POLLIN | POLLHUP))
mask |= EVENTER_READ;
if(pe->portev_events & (POLLOUT))
mask |= EVENTER_WRITE;
if(pe->portev_events & (POLLERR | POLLHUP | POLLNVAL))
mask |= EVENTER_EXCEPTION;
eventer_ports_impl_trigger(e, mask);
}
}
}
/* NOTREACHED */
return 0;
}
