void fetch_pages()
{
time_t now = time(NULL);
/* the number of elements in ready sites fifo */
int nr_elements = ready_fifo_num();
/* the number of free connections */
int space = settings.max_conns - statis.conns;
int i;
for (i = 0; i < space && nr_elements > 0; ++i, nr_elements--)
{
site_t *si = ready_fifo_get ();
if (!si)
break;
/*TODO: check site's time interval*/
if (now <= si->next_call)
{
ready_fifo_put(si);
continue;
}
url_t *u = site_get_url(si);
// if (si->robots_gotten)
// {
// while(1)
// {
// u = site_get_url(si);
// /* check if the url is forbidden by robots.txt */
// if (site_test_robots(si, u->path))
// {
// break;
// }
// else
// {
// url_destroy(u);
// }
// }
// /* all the urls are forbidden by robots.txt
// so ignore this site and goto next loop
// */
// if (!u)
// {
// continue;
// }
// }
// else
// {
// u = url_new_with_host("http", si->host, "/robots.txt");
// }
conn_t *cn = get_free_conn();
assert(cn);
cn->site = si;
cn->url = u;
create_bufferev(cn);
/* update the next time when we can connect to this server*/
cn->site->next_call = now + settings.interval;
// if (!cn->site->robots_gotten)
// {
// cn->site->next_call = now + settings.interval;
// }
// else
// {
// cn->site->next_call = 0;
// }
if (!site_empty(cn->site))
{
ready_fifo_put(cn->site);
}
}
}
int main(int argc, char *argv[])
{
struct sockaddr_in sockname;
int max = -1, omax; /* the biggest value sd. for select */
int sd; /* our listen socket */
fd_set *readable = NULL , *writable = NULL; /* fd_sets for select */
u_short port;
u_long p;
char *ep;
int i;
/*
* first, figure out what port we will listen on - it should
* be our first parameter.
*/
if (argc != 2)
usage();
errno = 0;
p = strtoul(argv[1], &ep, 10);
if (*argv[1] == '\0' || *ep != '\0') {
/* parameter wasn't a number, or was empty */
fprintf(stderr, "%s - not a number\n", argv[1]);
usage();
}
if ((errno == ERANGE && p == ULONG_MAX) || (p > USHRT_MAX)) {
/* It's a number, but it either can't fit in an unsigned
* long, or is too big for an unsigned short
*/
fprintf(stderr, "%s - value out of range\n", argv[1]);
usage();
}
/* now safe to do this */
port = p;
/* now before we get going, decide if we want to daemonize, that
* is, run in the background like a real system process
*/
#ifndef DEBUG
/* don't daemonize if we compile with -DDEBUG */
if (daemon(1, 0) == -1)
err(1, "daemon() failed");
#endif
/* now off to the races - let's set up our listening socket */
memset(&sockname, 0, sizeof(sockname));
sockname.sin_family = AF_INET;
sockname.sin_port = htons(port);
sockname.sin_addr.s_addr = htonl(INADDR_ANY);
sd=socket(AF_INET,SOCK_STREAM,0);
if ( sd == -1)
err(1, "socket failed");
if (bind(sd, (struct sockaddr *) &sockname, sizeof(sockname)) == -1)
err(1, "bind failed");
if (listen(sd,3) == -1)
err(1, "listen failed");
/*
* We're now bound, and listening for connections on "sd".
* Each call to "accept" will return us a descriptor talking to
* a connected client.
*/
/*
* finally - the main loop. accept connections and deal with 'em
*/
#ifndef DEBUG
/*
* since we'll be running as a daemon if we're not compiled with
* -DDEBUG, we better not be using printf - since stdout will be
* unusable
*/
printf("Server up and listening for connections on port %u\n", port);
#endif
/* initialize all our connection structures */
for (i = 0; i < MAXCONN; i++)
closecon(&connections[i], 1);
for(;;) {
int i;
int maxfd = -1; /* the biggest value sd we are interested in.*/
/*
* first we have to initialize the fd_sets to keep
* track of readable and writable sockets. we have
* to make sure we have fd_sets that are big enough
* to hold our largest valued socket descriptor.
* so first, we find the max value by iterating through
* all the connections, and then we allocate fd sets
* that are big enough, if they aren't already.
*/
omax = max;
max = sd; /* the listen socket */
for (i = 0; i < MAXCONN; i++) {
if (connections[i].sd > max)
max = connections[i].sd;
}
if (max > omax) {
/* we need bigger fd_sets allocated */
/* free the old ones - does nothing if they are NULL */
free(readable);
free(writable);
/*
* this is how to allocate fd_sets for select
*/
readable = (fd_set *)calloc(howmany(max + 1, NFDBITS),
sizeof(fd_mask));
if (readable == NULL)
err(1, "out of memory");
writable = (fd_set *)calloc(howmany(max + 1, NFDBITS),
sizeof(fd_mask));
if (writable == NULL)
err(1, "out of memory");
omax = max;
/*
* note that calloc always returns 0'ed memory,
* (unlike malloc) so these sets are all set to 0
* and ready to go
*/
} else {
/*
* our allocated sets are big enough, just make
* sure they are cleared to 0.
*/
memset(readable, 0, howmany(max+1, NFDBITS) *
sizeof(fd_mask));
memset(writable, 0, howmany(max+1, NFDBITS) *
sizeof(fd_mask));
}
/*
* Now, we decide which sockets we are interested
* in reading and writing, by setting the corresponding
* bit in the readable and writable fd_sets.
*/
/*
* we are always interesting in reading from the
* listening socket. so put it in the read set.
*/
FD_SET(sd, readable);
if (maxfd < sd)
maxfd = sd;
/*
* now go through the list of connections, and if we
* are interested in reading from, or writing to, the
* connection's socket, put it in the readable, or
* writable fd_set - in preparation to call select
* to tell us which ones we can read and write to.
*/
for (i = 0; i<MAXCONN; i++) {
if (connections[i].state == STATE_READING) {
FD_SET(connections[i].sd, readable);
if (maxfd < connections[i].sd)
maxfd = connections[i].sd;
}
if (connections[i].state == STATE_WRITING) {
FD_SET(connections[i].sd, writable);
if (maxfd < connections[i].sd)
maxfd = connections[i].sd;
}
}
/*
* finally, we can call select. we have filled in "readable"
* and "writable" with everything we are interested in, and
* when select returns, it will indicate in each fd_set
* which sockets are readable and writable
*/
i = select(maxfd + 1, readable, writable, NULL,NULL);
if (i == -1 && errno != EINTR)
err(1, "select failed");
if (i > 0) {
/* something is readable or writable... */
/*
* First things first. check the listen socket.
* If it was readable - we have a new connection
* to accept.
*/
if (FD_ISSET(sd, readable)) {
struct con *cp;
int newsd;
socklen_t slen;
struct sockaddr_in sa;
slen = sizeof(sa);
newsd = accept(sd, (struct sockaddr *)&sa,
&slen);
if (newsd == -1)
err(1, "accept failed");
cp = get_free_conn();
if (cp == NULL) {
/*
* we have no connection structures
* so we close connection to our
* client to not leave him hanging
* because we are too busy to
* service his request
*/
close(newsd);
} else {
/*
* ok, if this worked, we now have a
* new connection. set him up to be
* READING so we do something with him
*/
cp->state = STATE_READING;
cp->sd = newsd;
cp->slen = slen;
memcpy(&cp->sa, &sa, sizeof(sa));
}
}
/*
* now, iterate through all of our connections,
* check to see if they are readble or writable,
* and if so, do a read or write accordingly
*/
for (i = 0; i<MAXCONN; i++) {
if ((connections[i].state == STATE_READING) &&
FD_ISSET(connections[i].sd, readable))
handleread(&connections[i]);
if ((connections[i].state == STATE_WRITING) &&
FD_ISSET(connections[i].sd, writable))
handlewrite(&connections[i]);
}
}
}
}
