Esempio n. 1
0
File: host.c Progetto: mirror/wget
static void
wait_ares (ares_channel channel)
{
  struct ptimer *timer = NULL;

  if (opt.dns_timeout)
    timer = ptimer_new ();

  for (;;)
    {
      struct timeval *tvp, tv;
      fd_set read_fds, write_fds;
      int nfds, rc;

      FD_ZERO (&read_fds);
      FD_ZERO (&write_fds);
      nfds = ares_fds (channel, &read_fds, &write_fds);
      if (nfds == 0)
        break;

      if (timer)
        {
          double max = opt.dns_timeout - ptimer_measure (timer);

          tv.tv_sec = (long) max;
          tv.tv_usec = 1000000 * (max - (long) max);
          tvp = ares_timeout (channel, &tv, &tv);
        }
      else
        tvp = ares_timeout (channel, NULL, &tv);

      rc = select (nfds, &read_fds, &write_fds, NULL, tvp);
      if (rc == 0 && timer && ptimer_measure (timer) >= opt.dns_timeout)
        ares_cancel (channel);
      else
        ares_process (channel, &read_fds, &write_fds);
    }
  if (timer)
    ptimer_destroy (timer);
}
Esempio n. 2
0
static void
limit_bandwidth (wgint bytes, struct ptimer *timer)
{
  double delta_t = ptimer_read (timer) - limit_data.chunk_start;
  double expected;

  limit_data.chunk_bytes += bytes;

  /* Calculate the amount of time we expect downloading the chunk
     should take.  If in reality it took less time, sleep to
     compensate for the difference.  */
  expected = (double) limit_data.chunk_bytes / opt.limit_rate;

  if (expected > delta_t)
    {
      double slp = expected - delta_t + limit_data.sleep_adjust;
      double t0, t1;
      if (slp < 0.2)
        {
          DEBUGP (("deferring a %.2f ms sleep (%s/%.2f).\n",
                   slp * 1000, number_to_static_string (limit_data.chunk_bytes),
                   delta_t));
          return;
        }

      if (slp > 1) // VisualWget: We do not expect to sleep more than 1 second.
        slp = 1;

      DEBUGP (("\nsleeping %.2f ms for %s bytes, adjust %.2f ms\n",
               slp * 1000, number_to_static_string (limit_data.chunk_bytes),
               limit_data.sleep_adjust));

      t0 = ptimer_read (timer);
      xsleep (slp);
      t1 = ptimer_measure (timer);

      /* Due to scheduling, we probably slept slightly longer (or
         shorter) than desired.  Calculate the difference between the
         desired and the actual sleep, and adjust the next sleep by
         that amount.  */
      limit_data.sleep_adjust = slp - (t1 - t0);
      /* If sleep_adjust is very large, it's likely due to suspension
         and not clock inaccuracy.  Don't enforce those.  */
      if (limit_data.sleep_adjust > 0.5)
        limit_data.sleep_adjust = 0.5;
      else if (limit_data.sleep_adjust < -0.5)
        limit_data.sleep_adjust = -0.5;
    }

  limit_data.chunk_bytes = 0;
  limit_data.chunk_start = ptimer_read (timer);
}
Esempio n. 3
0
void
convert_all_links (void)
{
    double secs;
    int file_count = 0;

    struct ptimer *timer = ptimer_new ();

    convert_links_in_hashtable (downloaded_html_set, 0, &file_count);
    convert_links_in_hashtable (downloaded_css_set, 1, &file_count);

    secs = ptimer_measure (timer);
    logprintf (LOG_VERBOSE, _("Converted %d files in %s seconds.\n"),
               file_count, print_decimal (secs));

    ptimer_destroy (timer);
}
Esempio n. 4
0
int
fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
              wgint *qtyread, wgint *qtywritten, double *elapsed, int flags,
              FILE *out2)
{
  int ret = 0;
#undef max
#define max(a,b) ((a) > (b) ? (a) : (b))
  int dlbufsize = max (BUFSIZ, 8 * 1024);
  char *dlbuf = xmalloc (dlbufsize);

  struct ptimer *timer = NULL;
  double last_successful_read_tm = 0;

  /* The progress gauge, set according to the user preferences. */
  void *progress = NULL;

  /* Non-zero if the progress gauge is interactive, i.e. if it can
     continually update the display.  When true, smaller timeout
     values are used so that the gauge can update the display when
     data arrives slowly. */
  bool progress_interactive = false;

  bool exact = !!(flags & rb_read_exactly);

  /* Used only by HTTP/HTTPS chunked transfer encoding.  */
  bool chunked = flags & rb_chunked_transfer_encoding;
  wgint skip = 0;

  /* How much data we've read/written.  */
  wgint sum_read = 0;
  wgint sum_written = 0;
  wgint remaining_chunk_size = 0;

  if (flags & rb_skip_startpos)
    skip = startpos;

  if (opt.verbose)
    {
      /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
         argument to progress_create because the indicator doesn't
         (yet) know about "skipping" data.  */
      wgint start = skip ? 0 : startpos;
      progress = progress_create (start, start + toread);
      progress_interactive = progress_interactive_p (progress);
    }

  if (opt.limit_rate)
    limit_bandwidth_reset ();

  /* A timer is needed for tracking progress, for throttling, and for
     tracking elapsed time.  If either of these are requested, start
     the timer.  */
  if (progress || opt.limit_rate || elapsed)
    {
      timer = ptimer_new ();
      last_successful_read_tm = 0;
    }

  /* Use a smaller buffer for low requested bandwidths.  For example,
     with --limit-rate=2k, it doesn't make sense to slurp in 16K of
     data and then sleep for 8s.  With buffer size equal to the limit,
     we never have to sleep for more than one second.  */
  if (opt.limit_rate && opt.limit_rate < dlbufsize)
    dlbufsize = opt.limit_rate;

  /* Read from FD while there is data to read.  Normally toread==0
     means that it is unknown how much data is to arrive.  However, if
     EXACT is set, then toread==0 means what it says: that no data
     should be read.  */
  while (!exact || (sum_read < toread))
    {
      int rdsize;
      double tmout = opt.read_timeout;

      if (chunked)
        {
          if (remaining_chunk_size == 0)
            {
              char *line = fd_read_line (fd);
              char *endl;
              if (line == NULL)
                {
                  ret = -1;
                  break;
                }
              else if (out2 != NULL)
                fwrite (line, 1, strlen (line), out2);

              remaining_chunk_size = strtol (line, &endl, 16);
              xfree (line);

              if (remaining_chunk_size == 0)
                {
                  ret = 0;
                  line = fd_read_line (fd);
                  if (line == NULL)
                    ret = -1;
                  else
                    {
                      if (out2 != NULL)
                        fwrite (line, 1, strlen (line), out2);
                      xfree (line);
                    }
                  break;
                }
            }

          rdsize = MIN (remaining_chunk_size, dlbufsize);
        }
      else
        rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;

      if (progress_interactive)
        {
          /* For interactive progress gauges, always specify a ~1s
             timeout, so that the gauge can be updated regularly even
             when the data arrives very slowly or stalls.  */
          tmout = 0.95;
          if (opt.read_timeout)
            {
              double waittm;
              waittm = ptimer_read (timer) - last_successful_read_tm;
              if (waittm + tmout > opt.read_timeout)
                {
                  /* Don't let total idle time exceed read timeout. */
                  tmout = opt.read_timeout - waittm;
                  if (tmout < 0)
                    {
                      /* We've already exceeded the timeout. */
                      ret = -1, errno = ETIMEDOUT;
                      break;
                    }
                }
            }
        }
      ret = fd_read (fd, dlbuf, rdsize, tmout);

      if (progress_interactive && ret < 0 && errno == ETIMEDOUT)
        ret = 0;                /* interactive timeout, handled above */
      else if (ret <= 0)
        break;                  /* EOF or read error */

      if (progress || opt.limit_rate || elapsed)
        {
          ptimer_measure (timer);
          if (ret > 0)
            last_successful_read_tm = ptimer_read (timer);
        }

      if (ret > 0)
        {
          sum_read += ret;
          int write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written);
          if (write_res < 0)
            {
              ret = (write_res == -3) ? -3 : -2;
              goto out;
            }
          if (chunked)
            {
              remaining_chunk_size -= ret;
              if (remaining_chunk_size == 0)
                {
                  char *line = fd_read_line (fd);
                  if (line == NULL)
                    {
                      ret = -1;
                      break;
                    }
                  else
                    {
                      if (out2 != NULL)
                        fwrite (line, 1, strlen (line), out2);
                      xfree (line);
                    }
                }
            }
        }

      if (opt.limit_rate)
        limit_bandwidth (ret, timer);

      if (progress)
        progress_update (progress, ret, ptimer_read (timer));
#ifdef WINDOWS
      if (toread > 0 && !opt.quiet)
        ws_percenttitle (100.0 *
                         (startpos + sum_read) / (startpos + toread));
#endif
    }
  if (ret < -1)
    ret = -1;

 out:
  if (progress)
    progress_finish (progress, ptimer_read (timer));

  if (elapsed)
    *elapsed = ptimer_read (timer);
  if (timer)
    ptimer_destroy (timer);

  if (qtyread)
    *qtyread += sum_read;
  if (qtywritten)
    *qtywritten += sum_written;

  free (dlbuf);

  return ret;
}
Esempio n. 5
0
int
fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
              wgint *qtyread, wgint *qtywritten, double *elapsed, int flags)
{
  double receive_message_time = 0; // VisualWget
  double send_message_time = 0; // VisualWget
  int ret = 0;

  static char dlbuf[16384];
  int dlbufsize = sizeof (dlbuf);

  struct ptimer *timer = NULL;
  double last_successful_read_tm = 0;

  /* The progress gauge, set according to the user preferences. */
  void *progress = NULL;

  /* Non-zero if the progress gauge is interactive, i.e. if it can
     continually update the display.  When true, smaller timeout
     values are used so that the gauge can update the display when
     data arrives slowly. */
  bool progress_interactive = false;

  bool exact = !!(flags & rb_read_exactly);
  wgint skip = 0;

  /* How much data we've read/written.  */
  wgint sum_read = 0;
  wgint sum_written = 0;

  if (flags & rb_skip_startpos)
    skip = startpos;

  if (opt.visualwget_listening_port != -1) // VisualWget: Send a start position and the number of bytes remain.
    {
      wgint sp = skip ? 0 : startpos;
      char *msg = (char *) xmalloc (2 + numdigit (sp) + 1 + numdigit (toread) + 2);

      sprintf (msg, "1 %I64d %I64d\n", sp, toread);
      SendMessageToVisualWget (msg);
      xfree (msg);
    }

  if (opt.verbose)
    {
      /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
         argument to progress_create because the indicator doesn't
         (yet) know about "skipping" data.  */
      progress = progress_create (skip ? 0 : startpos, startpos + toread);
      progress_interactive = progress_interactive_p (progress);
    }

  limit_bandwidth_reset (); // VisualWget: Always reset.

  /* A timer is needed for tracking progress, for throttling, and for
     tracking elapsed time.  If either of these are requested, start
     the timer.  */
  if (progress || opt.limit_rate || elapsed || opt.visualwget_listening_port != -1) // VisualWget
    {
      timer = ptimer_new ();
      last_successful_read_tm = 0;
    }

  /* Read from FD while there is data to read.  Normally toread==0
     means that it is unknown how much data is to arrive.  However, if
     EXACT is set, then toread==0 means what it says: that no data
     should be read.  */
  while (!exact || (sum_read < toread))
    {
      int rdsize;
      double tmout = opt.read_timeout;
      if (progress_interactive)
        {
          /* For interactive progress gauges, always specify a ~1s
             timeout, so that the gauge can be updated regularly even
             when the data arrives very slowly or stalls.  */
          tmout = 0.95;
          if (opt.read_timeout)
            {
              double waittm;
              waittm = ptimer_read (timer) - last_successful_read_tm;
              if (waittm + tmout > opt.read_timeout)
                {
                  /* Don't let total idle time exceed read timeout. */
                  tmout = opt.read_timeout - waittm;
                  if (tmout < 0)
                    {
                      /* We've already exceeded the timeout. */
                      ret = -1, errno = ETIMEDOUT;
                      break;
                    }
                }
            }
        }

      //
      // VisualWget: Receive message every 0.1 second
      // BUG FIXED: http://groups.google.com/group/visualwget-discuss/browse_thread/thread/5ef2e324c70722aa
      //
      if (receive_message_time == 0
          || ptimer_read (timer) - receive_message_time > 0.1)
        {
          receive_message_time = ptimer_read (timer);
          ReceiveMessagesFromVisualWget ();
        }

      /* Use a smaller buffer for low requested bandwidths.  For example,
         with --limit-rate=2k, it doesn't make sense to slurp in 16K of
         data and then sleep for 8s.  With buffer size equal to the limit,
         we never have to sleep for more than one second.  */
      if (opt.limit_rate && opt.limit_rate < sizeof (dlbuf)) // VisualWget: Set a correct download buffer size.
        dlbufsize = opt.limit_rate;
      else
        dlbufsize = sizeof (dlbuf);

      rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
      ret = fd_read (fd, dlbuf, rdsize, tmout);

      if (progress_interactive && ret < 0 && errno == ETIMEDOUT)
        ret = 0;                /* interactive timeout, handled above */
      else if (ret <= 0)
        break;                  /* EOF or read error */

      if (progress || opt.limit_rate || opt.visualwget_listening_port != -1) // VisualWget
        {
          ptimer_measure (timer);
          if (ret > 0)
            last_successful_read_tm = ptimer_read (timer);
        }

      if (opt.limit_rate) // VisualWget: Do this before writing data in order to prevent an incorrect information getting sent to VisualWget.
        limit_bandwidth (ret, timer);

      if (ret > 0)
        {
          sum_read += ret;
          if (!write_data (out, dlbuf, ret, &skip, &sum_written))
            {
              ret = -2;
              goto out;
            }

          // VisualWget: Send the total number of bytes read and elapsed time.
          if (send_message_time == 0
              || ptimer_read (timer) - send_message_time > 0.1
              || sum_read == toread)
            {
              send_message_time = ptimer_read (timer);

              if (opt.visualwget_listening_port != -1)
                {
                  wgint elapsed_time = (wgint) (ptimer_read (timer) * 1000);
                  char *msg = (char *) xmalloc (2 + numdigit (sum_read) + 1 + numdigit (elapsed_time) + 2);

                  sprintf (msg, "2 %I64d %I64d\n", sum_read, elapsed_time);
                  SendMessageToVisualWget (msg);
                  xfree (msg);
                }
            }
        }

      if (progress)
        progress_update (progress, ret, ptimer_read (timer));
#ifdef WINDOWS
      if (toread > 0 && !opt.quiet)
        ws_percenttitle (100.0 *
                         (startpos + sum_read) / (startpos + toread));
#endif
    }
  if (ret < -1)
    ret = -1;

 out:
  if (progress)
    progress_finish (progress, ptimer_read (timer));

  if (elapsed)
    *elapsed = ptimer_read (timer);
  if (timer)
    ptimer_destroy (timer);

  if (qtyread)
    *qtyread += sum_read;
  if (qtywritten)
    *qtywritten += sum_written;

  return ret;
}