void
convert_all_links (void)
{
double secs;
int file_count = 0;
struct ptimer *timer = ptimer_new ();
convert_links_in_hashtable (downloaded_html_set, 0, &file_count);
convert_links_in_hashtable (downloaded_css_set, 1, &file_count);
secs = ptimer_measure (timer);
logprintf (LOG_VERBOSE, _("Converted %d files in %s seconds.\n"),
file_count, print_decimal (secs));
ptimer_destroy (timer);
}
static void
wait_ares (ares_channel channel)
{
struct ptimer *timer = NULL;
if (opt.dns_timeout)
timer = ptimer_new ();
for (;;)
{
struct timeval *tvp, tv;
fd_set read_fds, write_fds;
int nfds, rc;
FD_ZERO (&read_fds);
FD_ZERO (&write_fds);
nfds = ares_fds (channel, &read_fds, &write_fds);
if (nfds == 0)
break;
if (timer)
{
double max = opt.dns_timeout - ptimer_measure (timer);
tv.tv_sec = (long) max;
tv.tv_usec = 1000000 * (max - (long) max);
tvp = ares_timeout (channel, &tv, &tv);
}
else
tvp = ares_timeout (channel, NULL, &tv);
rc = select (nfds, &read_fds, &write_fds, NULL, tvp);
if (rc == 0 && timer && ptimer_measure (timer) >= opt.dns_timeout)
ares_cancel (channel);
else
ares_process (channel, &read_fds, &write_fds);
}
if (timer)
ptimer_destroy (timer);
}
int
fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
wgint *qtyread, wgint *qtywritten, double *elapsed, int flags,
FILE *out2)
{
int ret = 0;
#undef max
#define max(a,b) ((a) > (b) ? (a) : (b))
int dlbufsize = max (BUFSIZ, 8 * 1024);
char *dlbuf = xmalloc (dlbufsize);
struct ptimer *timer = NULL;
double last_successful_read_tm = 0;
/* The progress gauge, set according to the user preferences. */
void *progress = NULL;
/* Non-zero if the progress gauge is interactive, i.e. if it can
continually update the display. When true, smaller timeout
values are used so that the gauge can update the display when
data arrives slowly. */
bool progress_interactive = false;
bool exact = !!(flags & rb_read_exactly);
/* Used only by HTTP/HTTPS chunked transfer encoding. */
bool chunked = flags & rb_chunked_transfer_encoding;
wgint skip = 0;
/* How much data we've read/written. */
wgint sum_read = 0;
wgint sum_written = 0;
wgint remaining_chunk_size = 0;
if (flags & rb_skip_startpos)
skip = startpos;
if (opt.verbose)
{
/* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
argument to progress_create because the indicator doesn't
(yet) know about "skipping" data. */
wgint start = skip ? 0 : startpos;
progress = progress_create (start, start + toread);
progress_interactive = progress_interactive_p (progress);
}
if (opt.limit_rate)
limit_bandwidth_reset ();
/* A timer is needed for tracking progress, for throttling, and for
tracking elapsed time. If either of these are requested, start
the timer. */
if (progress || opt.limit_rate || elapsed)
{
timer = ptimer_new ();
last_successful_read_tm = 0;
}
/* Use a smaller buffer for low requested bandwidths. For example,
with --limit-rate=2k, it doesn't make sense to slurp in 16K of
data and then sleep for 8s. With buffer size equal to the limit,
we never have to sleep for more than one second. */
if (opt.limit_rate && opt.limit_rate < dlbufsize)
dlbufsize = opt.limit_rate;
/* Read from FD while there is data to read. Normally toread==0
means that it is unknown how much data is to arrive. However, if
EXACT is set, then toread==0 means what it says: that no data
should be read. */
while (!exact || (sum_read < toread))
{
int rdsize;
double tmout = opt.read_timeout;
if (chunked)
{
if (remaining_chunk_size == 0)
{
char *line = fd_read_line (fd);
char *endl;
if (line == NULL)
{
ret = -1;
break;
}
else if (out2 != NULL)
fwrite (line, 1, strlen (line), out2);
remaining_chunk_size = strtol (line, &endl, 16);
xfree (line);
if (remaining_chunk_size == 0)
{
ret = 0;
line = fd_read_line (fd);
if (line == NULL)
ret = -1;
else
{
if (out2 != NULL)
fwrite (line, 1, strlen (line), out2);
xfree (line);
}
break;
}
}
rdsize = MIN (remaining_chunk_size, dlbufsize);
}
else
rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
if (progress_interactive)
{
/* For interactive progress gauges, always specify a ~1s
timeout, so that the gauge can be updated regularly even
when the data arrives very slowly or stalls. */
tmout = 0.95;
if (opt.read_timeout)
{
double waittm;
waittm = ptimer_read (timer) - last_successful_read_tm;
if (waittm + tmout > opt.read_timeout)
{
/* Don't let total idle time exceed read timeout. */
tmout = opt.read_timeout - waittm;
if (tmout < 0)
{
/* We've already exceeded the timeout. */
ret = -1, errno = ETIMEDOUT;
break;
}
}
}
}
ret = fd_read (fd, dlbuf, rdsize, tmout);
if (progress_interactive && ret < 0 && errno == ETIMEDOUT)
ret = 0; /* interactive timeout, handled above */
else if (ret <= 0)
break; /* EOF or read error */
if (progress || opt.limit_rate || elapsed)
{
ptimer_measure (timer);
if (ret > 0)
last_successful_read_tm = ptimer_read (timer);
}
if (ret > 0)
{
sum_read += ret;
int write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written);
if (write_res < 0)
{
ret = (write_res == -3) ? -3 : -2;
goto out;
}
if (chunked)
{
remaining_chunk_size -= ret;
if (remaining_chunk_size == 0)
{
char *line = fd_read_line (fd);
if (line == NULL)
{
ret = -1;
break;
}
else
{
if (out2 != NULL)
fwrite (line, 1, strlen (line), out2);
xfree (line);
}
}
}
}
if (opt.limit_rate)
limit_bandwidth (ret, timer);
if (progress)
progress_update (progress, ret, ptimer_read (timer));
#ifdef WINDOWS
if (toread > 0 && !opt.quiet)
ws_percenttitle (100.0 *
(startpos + sum_read) / (startpos + toread));
#endif
}
if (ret < -1)
ret = -1;
out:
if (progress)
progress_finish (progress, ptimer_read (timer));
if (elapsed)
*elapsed = ptimer_read (timer);
if (timer)
ptimer_destroy (timer);
if (qtyread)
*qtyread += sum_read;
if (qtywritten)
*qtywritten += sum_written;
free (dlbuf);
return ret;
}
int
fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
wgint *qtyread, wgint *qtywritten, double *elapsed, int flags)
{
double receive_message_time = 0; // VisualWget
double send_message_time = 0; // VisualWget
int ret = 0;
static char dlbuf[16384];
int dlbufsize = sizeof (dlbuf);
struct ptimer *timer = NULL;
double last_successful_read_tm = 0;
/* The progress gauge, set according to the user preferences. */
void *progress = NULL;
/* Non-zero if the progress gauge is interactive, i.e. if it can
continually update the display. When true, smaller timeout
values are used so that the gauge can update the display when
data arrives slowly. */
bool progress_interactive = false;
bool exact = !!(flags & rb_read_exactly);
wgint skip = 0;
/* How much data we've read/written. */
wgint sum_read = 0;
wgint sum_written = 0;
if (flags & rb_skip_startpos)
skip = startpos;
if (opt.visualwget_listening_port != -1) // VisualWget: Send a start position and the number of bytes remain.
{
wgint sp = skip ? 0 : startpos;
char *msg = (char *) xmalloc (2 + numdigit (sp) + 1 + numdigit (toread) + 2);
sprintf (msg, "1 %I64d %I64d\n", sp, toread);
SendMessageToVisualWget (msg);
xfree (msg);
}
if (opt.verbose)
{
/* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
argument to progress_create because the indicator doesn't
(yet) know about "skipping" data. */
progress = progress_create (skip ? 0 : startpos, startpos + toread);
progress_interactive = progress_interactive_p (progress);
}
limit_bandwidth_reset (); // VisualWget: Always reset.
/* A timer is needed for tracking progress, for throttling, and for
tracking elapsed time. If either of these are requested, start
the timer. */
if (progress || opt.limit_rate || elapsed || opt.visualwget_listening_port != -1) // VisualWget
{
timer = ptimer_new ();
last_successful_read_tm = 0;
}
/* Read from FD while there is data to read. Normally toread==0
means that it is unknown how much data is to arrive. However, if
EXACT is set, then toread==0 means what it says: that no data
should be read. */
while (!exact || (sum_read < toread))
{
int rdsize;
double tmout = opt.read_timeout;
if (progress_interactive)
{
/* For interactive progress gauges, always specify a ~1s
timeout, so that the gauge can be updated regularly even
when the data arrives very slowly or stalls. */
tmout = 0.95;
if (opt.read_timeout)
{
double waittm;
waittm = ptimer_read (timer) - last_successful_read_tm;
if (waittm + tmout > opt.read_timeout)
{
/* Don't let total idle time exceed read timeout. */
tmout = opt.read_timeout - waittm;
if (tmout < 0)
{
/* We've already exceeded the timeout. */
ret = -1, errno = ETIMEDOUT;
break;
}
}
}
}
//
// VisualWget: Receive message every 0.1 second
// BUG FIXED: http://groups.google.com/group/visualwget-discuss/browse_thread/thread/5ef2e324c70722aa
//
if (receive_message_time == 0
|| ptimer_read (timer) - receive_message_time > 0.1)
{
receive_message_time = ptimer_read (timer);
ReceiveMessagesFromVisualWget ();
}
/* Use a smaller buffer for low requested bandwidths. For example,
with --limit-rate=2k, it doesn't make sense to slurp in 16K of
data and then sleep for 8s. With buffer size equal to the limit,
we never have to sleep for more than one second. */
if (opt.limit_rate && opt.limit_rate < sizeof (dlbuf)) // VisualWget: Set a correct download buffer size.
dlbufsize = opt.limit_rate;
else
dlbufsize = sizeof (dlbuf);
rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
ret = fd_read (fd, dlbuf, rdsize, tmout);
if (progress_interactive && ret < 0 && errno == ETIMEDOUT)
ret = 0; /* interactive timeout, handled above */
else if (ret <= 0)
break; /* EOF or read error */
if (progress || opt.limit_rate || opt.visualwget_listening_port != -1) // VisualWget
{
ptimer_measure (timer);
if (ret > 0)
last_successful_read_tm = ptimer_read (timer);
}
if (opt.limit_rate) // VisualWget: Do this before writing data in order to prevent an incorrect information getting sent to VisualWget.
limit_bandwidth (ret, timer);
if (ret > 0)
{
sum_read += ret;
if (!write_data (out, dlbuf, ret, &skip, &sum_written))
{
ret = -2;
goto out;
}
// VisualWget: Send the total number of bytes read and elapsed time.
if (send_message_time == 0
|| ptimer_read (timer) - send_message_time > 0.1
|| sum_read == toread)
{
send_message_time = ptimer_read (timer);
if (opt.visualwget_listening_port != -1)
{
wgint elapsed_time = (wgint) (ptimer_read (timer) * 1000);
char *msg = (char *) xmalloc (2 + numdigit (sum_read) + 1 + numdigit (elapsed_time) + 2);
sprintf (msg, "2 %I64d %I64d\n", sum_read, elapsed_time);
SendMessageToVisualWget (msg);
xfree (msg);
}
}
}
if (progress)
progress_update (progress, ret, ptimer_read (timer));
#ifdef WINDOWS
if (toread > 0 && !opt.quiet)
ws_percenttitle (100.0 *
(startpos + sum_read) / (startpos + toread));
#endif
}
if (ret < -1)
ret = -1;
out:
if (progress)
progress_finish (progress, ptimer_read (timer));
if (elapsed)
*elapsed = ptimer_read (timer);
if (timer)
ptimer_destroy (timer);
if (qtyread)
*qtyread += sum_read;
if (qtywritten)
*qtywritten += sum_written;
return ret;
}
