/*---------------------------Idle------------------------*/
static void Idle (struct ev_loop *loop, ev_idle *w, int revents) {
if (SHUTDOWN) {
LogDebug("Idle processing (SHUTDOWN=%d)", SHUTDOWN);
ev_idle_stop(loop, w);
ev_unloop (loop, EVUNLOOP_ALL);
} else {
ev_tstamp yieldtime = 0.10;
ev_sleep(yieldtime);
}
}
static PyObject *
pyev_sleep(PyObject *module, PyObject *args)
{
double interval;
if (!PyArg_ParseTuple(args, "d:sleep", &interval)) {
return NULL;
}
Py_BEGIN_ALLOW_THREADS
ev_sleep(interval);
Py_END_ALLOW_THREADS
Py_RETURN_NONE;
}
static void
select_poll (EV_P_ ev_tstamp timeout)
{
struct timeval tv;
int res;
#if EV_SELECT_USE_FD_SET
memcpy (vec_ro, vec_ri, sizeof (fd_set));
memcpy (vec_wo, vec_wi, sizeof (fd_set));
#else
memcpy (vec_ro, vec_ri, vec_max * NFDBYTES);
memcpy (vec_wo, vec_wi, vec_max * NFDBYTES);
#endif
tv.tv_sec = (long)timeout;
tv.tv_usec = (long)((timeout - (ev_tstamp)tv.tv_sec) * 1e6);
res = select (vec_max * NFDBITS, (fd_set *)vec_ro, (fd_set *)vec_wo, 0, &tv);
if (expect_false (res < 0))
{
#if EV_SELECT_IS_WINSOCKET
errno = WSAGetLastError ();
#endif
#ifdef WSABASEERR
/* on windows, select returns incompatible error codes, fix this */
if (errno >= WSABASEERR && errno < WSABASEERR + 1000)
if (errno == WSAENOTSOCK)
errno = EBADF;
else
errno -= WSABASEERR;
#endif
#ifdef _WIN32
/* select on windows errornously returns EINVAL when no fd sets have been
* provided (this is documented). what microsoft doesn't tell you that this bug
* exists even when the fd sets are provided, so we have to check for this bug
* here and emulate by sleeping manually.
* we also get EINVAL when the timeout is invalid, but we ignore this case here
* and assume that EINVAL always means: you have to wait manually.
*/
if (errno == EINVAL)
{
ev_sleep (timeout);
return;
}
#endif
if (errno == EBADF)
fd_ebadf (EV_A);
else if (errno == ENOMEM && !syserr_cb)
fd_enomem (EV_A);
else if (errno != EINTR)
syserr ("(libev) select");
return;
}
#if EV_SELECT_USE_FD_SET
{
int fd;
for (fd = 0; fd < anfdmax; ++fd)
if (anfds [fd].events)
{
int events = 0;
#if EV_SELECT_IS_WINSOCKET
SOCKET handle = anfds [fd].handle;
#else
int handle = fd;
#endif
if (FD_ISSET (handle, (fd_set *)vec_ro)) events |= EV_READ;
if (FD_ISSET (handle, (fd_set *)vec_wo)) events |= EV_WRITE;
if (expect_true (events))
fd_event (EV_A_ fd, events);
}
}
#else
{
int word, bit;
for (word = vec_max; word--; )
{
fd_mask word_r = ((fd_mask *)vec_ro) [word];
fd_mask word_w = ((fd_mask *)vec_wo) [word];
if (word_r || word_w)
for (bit = NFDBITS; bit--; )
{
fd_mask mask = 1UL << bit;
int events = 0;
events |= word_r & mask ? EV_READ : 0;
events |= word_w & mask ? EV_WRITE : 0;
if (expect_true (events))
fd_event (EV_A_ word * NFDBITS + bit, events);
}
}
}
#endif
}
static void
select_poll (EV_P_ ev_tstamp timeout)
{
struct timeval tv;
int res;
int fd_setsize;
EV_RELEASE_CB;
EV_TV_SET (tv, timeout);
#if EV_SELECT_USE_FD_SET
fd_setsize = sizeof (fd_set);
#else
fd_setsize = vec_max * NFDBYTES;
#endif
memcpy (vec_ro, vec_ri, fd_setsize);
memcpy (vec_wo, vec_wi, fd_setsize);
#ifdef _WIN32
/* pass in the write set as except set.
* the idea behind this is to work around a windows bug that causes
* errors to be reported as an exception and not by setting
* the writable bit. this is so uncontrollably lame.
*/
memcpy (vec_eo, vec_wi, fd_setsize);
res = select (vec_max * NFDBITS, (fd_set *)vec_ro, (fd_set *)vec_wo, (fd_set *)vec_eo, &tv);
#elif EV_SELECT_USE_FD_SET
fd_setsize = anfdmax < FD_SETSIZE ? anfdmax : FD_SETSIZE;
res = select (fd_setsize, (fd_set *)vec_ro, (fd_set *)vec_wo, 0, &tv);
#else
res = select (vec_max * NFDBITS, (fd_set *)vec_ro, (fd_set *)vec_wo, 0, &tv);
#endif
EV_ACQUIRE_CB;
if (expect_false (res < 0))
{
#if EV_SELECT_IS_WINSOCKET
errno = WSAGetLastError ();
#endif
#ifdef WSABASEERR
/* on windows, select returns incompatible error codes, fix this */
if (errno >= WSABASEERR && errno < WSABASEERR + 1000)
if (errno == WSAENOTSOCK)
errno = EBADF;
else
errno -= WSABASEERR;
#endif
#ifdef _WIN32
/* select on windows erroneously returns EINVAL when no fd sets have been
* provided (this is documented). what microsoft doesn't tell you that this bug
* exists even when the fd sets _are_ provided, so we have to check for this bug
* here and emulate by sleeping manually.
* we also get EINVAL when the timeout is invalid, but we ignore this case here
* and assume that EINVAL always means: you have to wait manually.
*/
if (errno == EINVAL)
{
ev_sleep (timeout);
return;
}
#endif
if (errno == EBADF)
fd_ebadf (EV_A);
else if (errno == ENOMEM && !syserr_cb)
fd_enomem (EV_A);
else if (errno != EINTR)
ev_syserr ("(libev) select");
return;
}
#if EV_SELECT_USE_FD_SET
{
int fd;
for (fd = 0; fd < anfdmax; ++fd)
if (anfds [fd].events)
{
int events = 0;
#if EV_SELECT_IS_WINSOCKET
SOCKET handle = anfds [fd].handle;
#else
int handle = fd;
#endif
if (FD_ISSET (handle, (fd_set *)vec_ro)) events |= EV_READ;
if (FD_ISSET (handle, (fd_set *)vec_wo)) events |= EV_WRITE;
#ifdef _WIN32
if (FD_ISSET (handle, (fd_set *)vec_eo)) events |= EV_WRITE;
#endif
if (expect_true (events))
fd_event (EV_A_ fd, events);
}
}
#else
{
int word, bit;
for (word = vec_max; word--; )
{
fd_mask word_r = ((fd_mask *)vec_ro) [word];
fd_mask word_w = ((fd_mask *)vec_wo) [word];
#ifdef _WIN32
word_w |= ((fd_mask *)vec_eo) [word];
#endif
if (word_r || word_w)
for (bit = NFDBITS; bit--; )
{
fd_mask mask = 1UL << bit;
int events = 0;
events |= word_r & mask ? EV_READ : 0;
events |= word_w & mask ? EV_WRITE : 0;
if (expect_true (events))
fd_event (EV_A_ word * NFDBITS + bit, events);
}
}
}
#endif
}
int QCrawlerDNS::resolve_host(const std::string &host, struct in_addr *addr, int now)
{
// check cache
if (dns_cache.find(host) != dns_cache.end()) {
QCrawlerDNSEntry &entry = dns_cache[host];
if (!entry.addr_list.empty()) {
if (entry.resolve_time + ok_dns_cache_time >= now) {
entry.last_pos = (entry.last_pos + 1) % entry.addr_list.size();
*addr = entry.addr_list[entry.last_pos];
return 0;
}
} else {
if (entry.resolve_time + fail_dns_cache_time > now) {
return 1;
}
}
}
QCrawlerDNSEntry entry;
entry.last_pos = 0;
entry.resolve_time = now;
struct addrinfo hints, *res, *result;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
int error;
error = getaddrinfo(host.c_str(), "http", &hints, &result);
int cnt = 0;
while (error == EAI_AGAIN && ++cnt < 3) {
ev_sleep(1);
error = getaddrinfo(host.c_str(), NULL, NULL, &result);
}
if (error == EAI_AGAIN && cnt >= 3) {
LOG(WARNING) << "error in get addrinfo EAI AGAIN fatal error " << gai_strerror(error);
return -1;
}
if (error != 0) {
LOG(INFO) << "error in get addrinfo " << gai_strerror(error);
if (error != EAI_AGAIN) {
dns_cache[host] = entry;
}
return 1;
} else {
for (res = result; res != NULL; res = res->ai_next) {
entry.addr_list.push_back(((sockaddr_in*)(res->ai_addr))->sin_addr);
}
freeaddrinfo(result);
*addr = entry.addr_list[0];
dns_cache[host] = entry;
return 0;
}
}
