int
fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
wgint *qtyread, wgint *qtywritten, double *elapsed, int flags)
{
double receive_message_time = 0; // VisualWget
double send_message_time = 0; // VisualWget
int ret = 0;
static char dlbuf[16384];
int dlbufsize = sizeof (dlbuf);
struct ptimer *timer = NULL;
double last_successful_read_tm = 0;
/* The progress gauge, set according to the user preferences. */
void *progress = NULL;
/* Non-zero if the progress gauge is interactive, i.e. if it can
continually update the display. When true, smaller timeout
values are used so that the gauge can update the display when
data arrives slowly. */
bool progress_interactive = false;
bool exact = !!(flags & rb_read_exactly);
wgint skip = 0;
/* How much data we've read/written. */
wgint sum_read = 0;
wgint sum_written = 0;
if (flags & rb_skip_startpos)
skip = startpos;
if (opt.visualwget_listening_port != -1) // VisualWget: Send a start position and the number of bytes remain.
{
wgint sp = skip ? 0 : startpos;
char *msg = (char *) xmalloc (2 + numdigit (sp) + 1 + numdigit (toread) + 2);
sprintf (msg, "1 %I64d %I64d\n", sp, toread);
SendMessageToVisualWget (msg);
xfree (msg);
}
if (opt.verbose)
{
/* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
argument to progress_create because the indicator doesn't
(yet) know about "skipping" data. */
progress = progress_create (skip ? 0 : startpos, startpos + toread);
progress_interactive = progress_interactive_p (progress);
}
limit_bandwidth_reset (); // VisualWget: Always reset.
/* A timer is needed for tracking progress, for throttling, and for
tracking elapsed time. If either of these are requested, start
the timer. */
if (progress || opt.limit_rate || elapsed || opt.visualwget_listening_port != -1) // VisualWget
{
timer = ptimer_new ();
last_successful_read_tm = 0;
}
/* Read from FD while there is data to read. Normally toread==0
means that it is unknown how much data is to arrive. However, if
EXACT is set, then toread==0 means what it says: that no data
should be read. */
while (!exact || (sum_read < toread))
{
int rdsize;
double tmout = opt.read_timeout;
if (progress_interactive)
{
/* For interactive progress gauges, always specify a ~1s
timeout, so that the gauge can be updated regularly even
when the data arrives very slowly or stalls. */
tmout = 0.95;
if (opt.read_timeout)
{
double waittm;
waittm = ptimer_read (timer) - last_successful_read_tm;
if (waittm + tmout > opt.read_timeout)
{
/* Don't let total idle time exceed read timeout. */
tmout = opt.read_timeout - waittm;
if (tmout < 0)
{
/* We've already exceeded the timeout. */
ret = -1, errno = ETIMEDOUT;
break;
}
}
}
}
//
// VisualWget: Receive message every 0.1 second
// BUG FIXED: http://groups.google.com/group/visualwget-discuss/browse_thread/thread/5ef2e324c70722aa
//
if (receive_message_time == 0
|| ptimer_read (timer) - receive_message_time > 0.1)
{
receive_message_time = ptimer_read (timer);
ReceiveMessagesFromVisualWget ();
}
/* Use a smaller buffer for low requested bandwidths. For example,
with --limit-rate=2k, it doesn't make sense to slurp in 16K of
data and then sleep for 8s. With buffer size equal to the limit,
we never have to sleep for more than one second. */
if (opt.limit_rate && opt.limit_rate < sizeof (dlbuf)) // VisualWget: Set a correct download buffer size.
dlbufsize = opt.limit_rate;
else
dlbufsize = sizeof (dlbuf);
rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
ret = fd_read (fd, dlbuf, rdsize, tmout);
if (progress_interactive && ret < 0 && errno == ETIMEDOUT)
ret = 0; /* interactive timeout, handled above */
else if (ret <= 0)
break; /* EOF or read error */
if (progress || opt.limit_rate || opt.visualwget_listening_port != -1) // VisualWget
{
ptimer_measure (timer);
if (ret > 0)
last_successful_read_tm = ptimer_read (timer);
}
if (opt.limit_rate) // VisualWget: Do this before writing data in order to prevent an incorrect information getting sent to VisualWget.
limit_bandwidth (ret, timer);
if (ret > 0)
{
sum_read += ret;
if (!write_data (out, dlbuf, ret, &skip, &sum_written))
{
ret = -2;
goto out;
}
// VisualWget: Send the total number of bytes read and elapsed time.
if (send_message_time == 0
|| ptimer_read (timer) - send_message_time > 0.1
|| sum_read == toread)
{
send_message_time = ptimer_read (timer);
if (opt.visualwget_listening_port != -1)
{
wgint elapsed_time = (wgint) (ptimer_read (timer) * 1000);
char *msg = (char *) xmalloc (2 + numdigit (sum_read) + 1 + numdigit (elapsed_time) + 2);
sprintf (msg, "2 %I64d %I64d\n", sum_read, elapsed_time);
SendMessageToVisualWget (msg);
xfree (msg);
}
}
}
if (progress)
progress_update (progress, ret, ptimer_read (timer));
#ifdef WINDOWS
if (toread > 0 && !opt.quiet)
ws_percenttitle (100.0 *
(startpos + sum_read) / (startpos + toread));
#endif
}
if (ret < -1)
ret = -1;
out:
if (progress)
progress_finish (progress, ptimer_read (timer));
if (elapsed)
*elapsed = ptimer_read (timer);
if (timer)
ptimer_destroy (timer);
if (qtyread)
*qtyread += sum_read;
if (qtywritten)
*qtywritten += sum_written;
return ret;
}
int
fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
wgint *qtyread, wgint *qtywritten, double *elapsed, int flags,
FILE *out2)
{
int ret = 0;
#undef max
#define max(a,b) ((a) > (b) ? (a) : (b))
int dlbufsize = max (BUFSIZ, 8 * 1024);
char *dlbuf = xmalloc (dlbufsize);
struct ptimer *timer = NULL;
double last_successful_read_tm = 0;
/* The progress gauge, set according to the user preferences. */
void *progress = NULL;
/* Non-zero if the progress gauge is interactive, i.e. if it can
continually update the display. When true, smaller timeout
values are used so that the gauge can update the display when
data arrives slowly. */
bool progress_interactive = false;
bool exact = !!(flags & rb_read_exactly);
/* Used only by HTTP/HTTPS chunked transfer encoding. */
bool chunked = flags & rb_chunked_transfer_encoding;
wgint skip = 0;
/* How much data we've read/written. */
wgint sum_read = 0;
wgint sum_written = 0;
wgint remaining_chunk_size = 0;
if (flags & rb_skip_startpos)
skip = startpos;
if (opt.verbose)
{
/* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
argument to progress_create because the indicator doesn't
(yet) know about "skipping" data. */
wgint start = skip ? 0 : startpos;
progress = progress_create (start, start + toread);
progress_interactive = progress_interactive_p (progress);
}
if (opt.limit_rate)
limit_bandwidth_reset ();
/* A timer is needed for tracking progress, for throttling, and for
tracking elapsed time. If either of these are requested, start
the timer. */
if (progress || opt.limit_rate || elapsed)
{
timer = ptimer_new ();
last_successful_read_tm = 0;
}
/* Use a smaller buffer for low requested bandwidths. For example,
with --limit-rate=2k, it doesn't make sense to slurp in 16K of
data and then sleep for 8s. With buffer size equal to the limit,
we never have to sleep for more than one second. */
if (opt.limit_rate && opt.limit_rate < dlbufsize)
dlbufsize = opt.limit_rate;
/* Read from FD while there is data to read. Normally toread==0
means that it is unknown how much data is to arrive. However, if
EXACT is set, then toread==0 means what it says: that no data
should be read. */
while (!exact || (sum_read < toread))
{
int rdsize;
double tmout = opt.read_timeout;
if (chunked)
{
if (remaining_chunk_size == 0)
{
char *line = fd_read_line (fd);
char *endl;
if (line == NULL)
{
ret = -1;
break;
}
else if (out2 != NULL)
fwrite (line, 1, strlen (line), out2);
remaining_chunk_size = strtol (line, &endl, 16);
xfree (line);
if (remaining_chunk_size == 0)
{
ret = 0;
line = fd_read_line (fd);
if (line == NULL)
ret = -1;
else
{
if (out2 != NULL)
fwrite (line, 1, strlen (line), out2);
xfree (line);
}
break;
}
}
rdsize = MIN (remaining_chunk_size, dlbufsize);
}
else
rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
if (progress_interactive)
{
/* For interactive progress gauges, always specify a ~1s
timeout, so that the gauge can be updated regularly even
when the data arrives very slowly or stalls. */
tmout = 0.95;
if (opt.read_timeout)
{
double waittm;
waittm = ptimer_read (timer) - last_successful_read_tm;
if (waittm + tmout > opt.read_timeout)
{
/* Don't let total idle time exceed read timeout. */
tmout = opt.read_timeout - waittm;
if (tmout < 0)
{
/* We've already exceeded the timeout. */
ret = -1, errno = ETIMEDOUT;
break;
}
}
}
}
ret = fd_read (fd, dlbuf, rdsize, tmout);
if (progress_interactive && ret < 0 && errno == ETIMEDOUT)
ret = 0; /* interactive timeout, handled above */
else if (ret <= 0)
break; /* EOF or read error */
if (progress || opt.limit_rate || elapsed)
{
ptimer_measure (timer);
if (ret > 0)
last_successful_read_tm = ptimer_read (timer);
}
if (ret > 0)
{
sum_read += ret;
int write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written);
if (write_res < 0)
{
ret = (write_res == -3) ? -3 : -2;
goto out;
}
if (chunked)
{
remaining_chunk_size -= ret;
if (remaining_chunk_size == 0)
{
char *line = fd_read_line (fd);
if (line == NULL)
{
ret = -1;
break;
}
else
{
if (out2 != NULL)
fwrite (line, 1, strlen (line), out2);
xfree (line);
}
}
}
}
if (opt.limit_rate)
limit_bandwidth (ret, timer);
if (progress)
progress_update (progress, ret, ptimer_read (timer));
#ifdef WINDOWS
if (toread > 0 && !opt.quiet)
ws_percenttitle (100.0 *
(startpos + sum_read) / (startpos + toread));
#endif
}
if (ret < -1)
ret = -1;
out:
if (progress)
progress_finish (progress, ptimer_read (timer));
if (elapsed)
*elapsed = ptimer_read (timer);
if (timer)
ptimer_destroy (timer);
if (qtyread)
*qtyread += sum_read;
if (qtywritten)
*qtywritten += sum_written;
free (dlbuf);
return ret;
}
/* Reads the contents of file descriptor FD, until it is closed, or a
read error occurs. The data is read in 8K chunks, and stored to
stream fp, which should have been open for writing. If BUF is
non-NULL and its file descriptor is equal to FD, flush RBUF first.
This function will *not* use the rbuf_* functions!
The EXPECTED argument is passed to show_progress() unchanged, but
otherwise ignored.
If opt.verbose is set, the progress is also shown. RESTVAL
represents a value from which to start downloading (which will be
shown accordingly). If RESTVAL is non-zero, the stream should have
been open for appending.
The function exits and returns codes of 0, -1 and -2 if the
connection was closed, there was a read error, or if it could not
write to the output stream, respectively.
IMPORTANT: The function flushes the contents of the buffer in
rbuf_flush() before actually reading from fd. If you wish to read
from fd immediately, flush or discard the buffer. */
int
get_contents (int fd, FILE *fp, long *len, long restval, long expected,
struct rbuf *rbuf, int use_expected, long *elapsed)
{
int res = 0;
static char c[8192];
void *progress = NULL;
struct wget_timer *timer = wtimer_allocate ();
long dltime = 0, last_dltime = 0;
*len = restval;
if (opt.verbose)
progress = progress_create (restval, expected);
if (rbuf && RBUF_FD (rbuf) == fd)
{
int sz = 0;
while ((res = rbuf_flush (rbuf, c, sizeof (c))) != 0)
{
fwrite (c, sizeof (char), res, fp);
*len += res;
sz += res;
}
if (sz)
fflush (fp);
if (ferror (fp))
{
res = -2;
goto out;
}
if (opt.verbose)
progress_update (progress, sz, 0);
}
if (opt.limit_rate)
limit_bandwidth_reset ();
wtimer_reset (timer);
/* Read from fd while there is available data.
Normally, if expected is 0, it means that it is not known how
much data is expected. However, if use_expected is specified,
then expected being zero means exactly that. */
while (!use_expected || (*len < expected))
{
int amount_to_read = (use_expected
? MIN (expected - *len, sizeof (c))
: sizeof (c));
#ifdef HAVE_SSL
if (rbuf->ssl!=NULL)
res = ssl_iread (rbuf->ssl, c, amount_to_read);
else
#endif /* HAVE_SSL */
res = iread (fd, c, amount_to_read);
if (res > 0)
{
fwrite (c, sizeof (char), res, fp);
/* Always flush the contents of the network packet. This
should not be adverse to performance, as the network
packets typically won't be too tiny anyway. */
fflush (fp);
if (ferror (fp))
{
res = -2;
goto out;
}
/* If bandwidth is not limited, one call to wtimer_elapsed
is sufficient. */
dltime = wtimer_elapsed (timer);
if (opt.limit_rate)
{
limit_bandwidth (res, dltime - last_dltime);
dltime = wtimer_elapsed (timer);
last_dltime = dltime;
}
if (opt.verbose)
progress_update (progress, res, dltime);
*len += res;
}
else
break;
}
if (res < -1)
res = -1;
out:
if (opt.verbose)
progress_finish (progress, dltime);
if (elapsed)
*elapsed = dltime;
wtimer_delete (timer);
return res;
}