Beispiel #1
0
int
fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
              wgint *qtyread, wgint *qtywritten, double *elapsed, int flags)
{
  double receive_message_time = 0; // VisualWget
  double send_message_time = 0; // VisualWget
  int ret = 0;

  static char dlbuf[16384];
  int dlbufsize = sizeof (dlbuf);

  struct ptimer *timer = NULL;
  double last_successful_read_tm = 0;

  /* The progress gauge, set according to the user preferences. */
  void *progress = NULL;

  /* Non-zero if the progress gauge is interactive, i.e. if it can
     continually update the display.  When true, smaller timeout
     values are used so that the gauge can update the display when
     data arrives slowly. */
  bool progress_interactive = false;

  bool exact = !!(flags & rb_read_exactly);
  wgint skip = 0;

  /* How much data we've read/written.  */
  wgint sum_read = 0;
  wgint sum_written = 0;

  if (flags & rb_skip_startpos)
    skip = startpos;

  if (opt.visualwget_listening_port != -1) // VisualWget: Send a start position and the number of bytes remain.
    {
      wgint sp = skip ? 0 : startpos;
      char *msg = (char *) xmalloc (2 + numdigit (sp) + 1 + numdigit (toread) + 2);

      sprintf (msg, "1 %I64d %I64d\n", sp, toread);
      SendMessageToVisualWget (msg);
      xfree (msg);
    }

  if (opt.verbose)
    {
      /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
         argument to progress_create because the indicator doesn't
         (yet) know about "skipping" data.  */
      progress = progress_create (skip ? 0 : startpos, startpos + toread);
      progress_interactive = progress_interactive_p (progress);
    }

  limit_bandwidth_reset (); // VisualWget: Always reset.

  /* A timer is needed for tracking progress, for throttling, and for
     tracking elapsed time.  If either of these are requested, start
     the timer.  */
  if (progress || opt.limit_rate || elapsed || opt.visualwget_listening_port != -1) // VisualWget
    {
      timer = ptimer_new ();
      last_successful_read_tm = 0;
    }

  /* Read from FD while there is data to read.  Normally toread==0
     means that it is unknown how much data is to arrive.  However, if
     EXACT is set, then toread==0 means what it says: that no data
     should be read.  */
  while (!exact || (sum_read < toread))
    {
      int rdsize;
      double tmout = opt.read_timeout;
      if (progress_interactive)
        {
          /* For interactive progress gauges, always specify a ~1s
             timeout, so that the gauge can be updated regularly even
             when the data arrives very slowly or stalls.  */
          tmout = 0.95;
          if (opt.read_timeout)
            {
              double waittm;
              waittm = ptimer_read (timer) - last_successful_read_tm;
              if (waittm + tmout > opt.read_timeout)
                {
                  /* Don't let total idle time exceed read timeout. */
                  tmout = opt.read_timeout - waittm;
                  if (tmout < 0)
                    {
                      /* We've already exceeded the timeout. */
                      ret = -1, errno = ETIMEDOUT;
                      break;
                    }
                }
            }
        }

      //
      // VisualWget: Receive message every 0.1 second
      // BUG FIXED: http://groups.google.com/group/visualwget-discuss/browse_thread/thread/5ef2e324c70722aa
      //
      if (receive_message_time == 0
          || ptimer_read (timer) - receive_message_time > 0.1)
        {
          receive_message_time = ptimer_read (timer);
          ReceiveMessagesFromVisualWget ();
        }

      /* Use a smaller buffer for low requested bandwidths.  For example,
         with --limit-rate=2k, it doesn't make sense to slurp in 16K of
         data and then sleep for 8s.  With buffer size equal to the limit,
         we never have to sleep for more than one second.  */
      if (opt.limit_rate && opt.limit_rate < sizeof (dlbuf)) // VisualWget: Set a correct download buffer size.
        dlbufsize = opt.limit_rate;
      else
        dlbufsize = sizeof (dlbuf);

      rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
      ret = fd_read (fd, dlbuf, rdsize, tmout);

      if (progress_interactive && ret < 0 && errno == ETIMEDOUT)
        ret = 0;                /* interactive timeout, handled above */
      else if (ret <= 0)
        break;                  /* EOF or read error */

      if (progress || opt.limit_rate || opt.visualwget_listening_port != -1) // VisualWget
        {
          ptimer_measure (timer);
          if (ret > 0)
            last_successful_read_tm = ptimer_read (timer);
        }

      if (opt.limit_rate) // VisualWget: Do this before writing data in order to prevent an incorrect information getting sent to VisualWget.
        limit_bandwidth (ret, timer);

      if (ret > 0)
        {
          sum_read += ret;
          if (!write_data (out, dlbuf, ret, &skip, &sum_written))
            {
              ret = -2;
              goto out;
            }

          // VisualWget: Send the total number of bytes read and elapsed time.
          if (send_message_time == 0
              || ptimer_read (timer) - send_message_time > 0.1
              || sum_read == toread)
            {
              send_message_time = ptimer_read (timer);

              if (opt.visualwget_listening_port != -1)
                {
                  wgint elapsed_time = (wgint) (ptimer_read (timer) * 1000);
                  char *msg = (char *) xmalloc (2 + numdigit (sum_read) + 1 + numdigit (elapsed_time) + 2);

                  sprintf (msg, "2 %I64d %I64d\n", sum_read, elapsed_time);
                  SendMessageToVisualWget (msg);
                  xfree (msg);
                }
            }
        }

      if (progress)
        progress_update (progress, ret, ptimer_read (timer));
#ifdef WINDOWS
      if (toread > 0 && !opt.quiet)
        ws_percenttitle (100.0 *
                         (startpos + sum_read) / (startpos + toread));
#endif
    }
  if (ret < -1)
    ret = -1;

 out:
  if (progress)
    progress_finish (progress, ptimer_read (timer));

  if (elapsed)
    *elapsed = ptimer_read (timer);
  if (timer)
    ptimer_destroy (timer);

  if (qtyread)
    *qtyread += sum_read;
  if (qtywritten)
    *qtywritten += sum_written;

  return ret;
}
Beispiel #2
0
int
fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
              wgint *qtyread, wgint *qtywritten, double *elapsed, int flags,
              FILE *out2)
{
  int ret = 0;
#undef max
#define max(a,b) ((a) > (b) ? (a) : (b))
  int dlbufsize = max (BUFSIZ, 8 * 1024);
  char *dlbuf = xmalloc (dlbufsize);

  struct ptimer *timer = NULL;
  double last_successful_read_tm = 0;

  /* The progress gauge, set according to the user preferences. */
  void *progress = NULL;

  /* Non-zero if the progress gauge is interactive, i.e. if it can
     continually update the display.  When true, smaller timeout
     values are used so that the gauge can update the display when
     data arrives slowly. */
  bool progress_interactive = false;

  bool exact = !!(flags & rb_read_exactly);

  /* Used only by HTTP/HTTPS chunked transfer encoding.  */
  bool chunked = flags & rb_chunked_transfer_encoding;
  wgint skip = 0;

  /* How much data we've read/written.  */
  wgint sum_read = 0;
  wgint sum_written = 0;
  wgint remaining_chunk_size = 0;

  if (flags & rb_skip_startpos)
    skip = startpos;

  if (opt.verbose)
    {
      /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
         argument to progress_create because the indicator doesn't
         (yet) know about "skipping" data.  */
      wgint start = skip ? 0 : startpos;
      progress = progress_create (start, start + toread);
      progress_interactive = progress_interactive_p (progress);
    }

  if (opt.limit_rate)
    limit_bandwidth_reset ();

  /* A timer is needed for tracking progress, for throttling, and for
     tracking elapsed time.  If either of these are requested, start
     the timer.  */
  if (progress || opt.limit_rate || elapsed)
    {
      timer = ptimer_new ();
      last_successful_read_tm = 0;
    }

  /* Use a smaller buffer for low requested bandwidths.  For example,
     with --limit-rate=2k, it doesn't make sense to slurp in 16K of
     data and then sleep for 8s.  With buffer size equal to the limit,
     we never have to sleep for more than one second.  */
  if (opt.limit_rate && opt.limit_rate < dlbufsize)
    dlbufsize = opt.limit_rate;

  /* Read from FD while there is data to read.  Normally toread==0
     means that it is unknown how much data is to arrive.  However, if
     EXACT is set, then toread==0 means what it says: that no data
     should be read.  */
  while (!exact || (sum_read < toread))
    {
      int rdsize;
      double tmout = opt.read_timeout;

      if (chunked)
        {
          if (remaining_chunk_size == 0)
            {
              char *line = fd_read_line (fd);
              char *endl;
              if (line == NULL)
                {
                  ret = -1;
                  break;
                }
              else if (out2 != NULL)
                fwrite (line, 1, strlen (line), out2);

              remaining_chunk_size = strtol (line, &endl, 16);
              xfree (line);

              if (remaining_chunk_size == 0)
                {
                  ret = 0;
                  line = fd_read_line (fd);
                  if (line == NULL)
                    ret = -1;
                  else
                    {
                      if (out2 != NULL)
                        fwrite (line, 1, strlen (line), out2);
                      xfree (line);
                    }
                  break;
                }
            }

          rdsize = MIN (remaining_chunk_size, dlbufsize);
        }
      else
        rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;

      if (progress_interactive)
        {
          /* For interactive progress gauges, always specify a ~1s
             timeout, so that the gauge can be updated regularly even
             when the data arrives very slowly or stalls.  */
          tmout = 0.95;
          if (opt.read_timeout)
            {
              double waittm;
              waittm = ptimer_read (timer) - last_successful_read_tm;
              if (waittm + tmout > opt.read_timeout)
                {
                  /* Don't let total idle time exceed read timeout. */
                  tmout = opt.read_timeout - waittm;
                  if (tmout < 0)
                    {
                      /* We've already exceeded the timeout. */
                      ret = -1, errno = ETIMEDOUT;
                      break;
                    }
                }
            }
        }
      ret = fd_read (fd, dlbuf, rdsize, tmout);

      if (progress_interactive && ret < 0 && errno == ETIMEDOUT)
        ret = 0;                /* interactive timeout, handled above */
      else if (ret <= 0)
        break;                  /* EOF or read error */

      if (progress || opt.limit_rate || elapsed)
        {
          ptimer_measure (timer);
          if (ret > 0)
            last_successful_read_tm = ptimer_read (timer);
        }

      if (ret > 0)
        {
          sum_read += ret;
          int write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written);
          if (write_res < 0)
            {
              ret = (write_res == -3) ? -3 : -2;
              goto out;
            }
          if (chunked)
            {
              remaining_chunk_size -= ret;
              if (remaining_chunk_size == 0)
                {
                  char *line = fd_read_line (fd);
                  if (line == NULL)
                    {
                      ret = -1;
                      break;
                    }
                  else
                    {
                      if (out2 != NULL)
                        fwrite (line, 1, strlen (line), out2);
                      xfree (line);
                    }
                }
            }
        }

      if (opt.limit_rate)
        limit_bandwidth (ret, timer);

      if (progress)
        progress_update (progress, ret, ptimer_read (timer));
#ifdef WINDOWS
      if (toread > 0 && !opt.quiet)
        ws_percenttitle (100.0 *
                         (startpos + sum_read) / (startpos + toread));
#endif
    }
  if (ret < -1)
    ret = -1;

 out:
  if (progress)
    progress_finish (progress, ptimer_read (timer));

  if (elapsed)
    *elapsed = ptimer_read (timer);
  if (timer)
    ptimer_destroy (timer);

  if (qtyread)
    *qtyread += sum_read;
  if (qtywritten)
    *qtywritten += sum_written;

  free (dlbuf);

  return ret;
}
Beispiel #3
0
/* Reads the contents of file descriptor FD, until it is closed, or a
   read error occurs.  The data is read in 8K chunks, and stored to
   stream fp, which should have been open for writing.  If BUF is
   non-NULL and its file descriptor is equal to FD, flush RBUF first.
   This function will *not* use the rbuf_* functions!

   The EXPECTED argument is passed to show_progress() unchanged, but
   otherwise ignored.

   If opt.verbose is set, the progress is also shown.  RESTVAL
   represents a value from which to start downloading (which will be
   shown accordingly).  If RESTVAL is non-zero, the stream should have
   been open for appending.

   The function exits and returns codes of 0, -1 and -2 if the
   connection was closed, there was a read error, or if it could not
   write to the output stream, respectively.

   IMPORTANT: The function flushes the contents of the buffer in
   rbuf_flush() before actually reading from fd.  If you wish to read
   from fd immediately, flush or discard the buffer.  */
int
get_contents (int fd, FILE *fp, long *len, long restval, long expected,
	      struct rbuf *rbuf, int use_expected, long *elapsed)
{
  int res = 0;
  static char c[8192];
  void *progress = NULL;
  struct wget_timer *timer = wtimer_allocate ();
  long dltime = 0, last_dltime = 0;

  *len = restval;

  if (opt.verbose)
    progress = progress_create (restval, expected);

  if (rbuf && RBUF_FD (rbuf) == fd)
    {
      int sz = 0;
      while ((res = rbuf_flush (rbuf, c, sizeof (c))) != 0)
	{
	  fwrite (c, sizeof (char), res, fp);
	  *len += res;
	  sz += res;
	}
      if (sz)
	fflush (fp);
      if (ferror (fp))
	{
	  res = -2;
	  goto out;
	}
      if (opt.verbose)
	progress_update (progress, sz, 0);
    }

  if (opt.limit_rate)
    limit_bandwidth_reset ();
  wtimer_reset (timer);

  /* Read from fd while there is available data.

     Normally, if expected is 0, it means that it is not known how
     much data is expected.  However, if use_expected is specified,
     then expected being zero means exactly that.  */
  while (!use_expected || (*len < expected))
    {
      int amount_to_read = (use_expected
			    ? MIN (expected - *len, sizeof (c))
			    : sizeof (c));
#ifdef HAVE_SSL
      if (rbuf->ssl!=NULL)
	res = ssl_iread (rbuf->ssl, c, amount_to_read);
      else
#endif /* HAVE_SSL */
	res = iread (fd, c, amount_to_read);

      if (res > 0)
	{
	  fwrite (c, sizeof (char), res, fp);
	  /* Always flush the contents of the network packet.  This
	     should not be adverse to performance, as the network
	     packets typically won't be too tiny anyway.  */
	  fflush (fp);
	  if (ferror (fp))
	    {
	      res = -2;
	      goto out;
	    }

	  /* If bandwidth is not limited, one call to wtimer_elapsed
	     is sufficient.  */
	  dltime = wtimer_elapsed (timer);
	  if (opt.limit_rate)
	    {
	      limit_bandwidth (res, dltime - last_dltime);
	      dltime = wtimer_elapsed (timer);
	      last_dltime = dltime;
	    }

	  if (opt.verbose)
	    progress_update (progress, res, dltime);
	  *len += res;
	}
      else
	break;
    }
  if (res < -1)
    res = -1;

 out:
  if (opt.verbose)
    progress_finish (progress, dltime);
  if (elapsed)
    *elapsed = dltime;
  wtimer_delete (timer);

  return res;
}