Exemplo n.º 1
0
/* process a line from the log and store it accordingly */
static int
process_log (GLog * logger, char *line, int test)
{
  GLogItem *glog;
  char date[DATE_LEN];
  char *qmark = NULL, *req_key = NULL;
  int is404 = 0;

  if (valid_line (line)) {
    count_invalid (logger);
    return 0;
  }

  count_process (logger);
  glog = init_log_item (logger);
  /* parse a line of log, and fill structure with appropriate values */
  if (parse_format (glog, conf.log_format, conf.date_format, line)) {
    count_invalid (logger);
    goto cleanup;
  }

  /* must have the following fields */
  if (glog->host == NULL || glog->date == NULL || glog->req == NULL) {
    count_invalid (logger);
    goto cleanup;
  }
  /* agent will be null in cases where %u is not specified */
  if (glog->agent == NULL)
    glog->agent = alloc_string ("-");

  /* testing log only */
  if (test)
    goto cleanup;

  if (process_date (glog, date)) {
    count_invalid (logger);
    goto cleanup;
  }
  /* ignore host or crawlers */
  if (exclude_ip (logger, glog) == 0)
    goto cleanup;
  if (exclude_crawler (glog) == 0)
    goto cleanup;
  if (ignore_referer (glog->site))
    goto cleanup;

  /* is this a 404? */
  if (glog->status && !memcmp (glog->status, "404", 3))
    is404 = 1;
  /* treat 444 as 404? */
  else if (glog->status && !memcmp (glog->status, "444", 3) &&
           conf.code444_as_404)
    is404 = 1;
  /* check if we need to remove the request's query string */
  else if (conf.ignore_qstr) {
    if ((qmark = strchr (glog->req, '?')) != NULL) {
      if ((qmark - glog->req) > 0)
        *qmark = '\0';
    }
  }

  req_key = xstrdup (glog->req);
  /* include HTTP method/protocol to request */
  if (conf.append_method && glog->method) {
    glog->method = strtoupper (glog->method);
    append_method_to_request (&req_key, glog->method);
  }
  if (conf.append_protocol && glog->protocol) {
    glog->protocol = strtoupper (glog->protocol);
    append_protocol_to_request (&req_key, glog->protocol);
  }
  if ((conf.append_method) || (conf.append_protocol))
    req_key = deblank (req_key);

  unique_data (glog, date);
  /* process agents that are part of a host */
  if (conf.list_agents)
    process_host_agents (glog->host, glog->agent);
  /* process status codes */
  if (glog->status)
    process_generic_data (ht_status_code, glog->status);
  /* process 404s */
  if (is404)
    process_request (ht_not_found_requests, req_key, glog);
  /* process static files */
  else if (verify_static_content (glog->req))
    process_request (ht_requests_static, req_key, glog);
  /* process regular files */
  else
    process_request (ht_requests, req_key, glog);

  /* process referrers */
  process_referrers (glog->ref, glog->site);
  /* process hosts */
  process_generic_data (ht_hosts, glog->host);
  /* process bandwidth  */
  process_request_meta (ht_date_bw, date, glog->resp_size);
  process_request_meta (ht_file_bw, req_key, glog->resp_size);
  process_request_meta (ht_host_bw, glog->host, glog->resp_size);

  /* process time taken to serve the request, in microseconds */
  process_request_meta (ht_file_serve_usecs, req_key, glog->serve_time);
  process_request_meta (ht_host_serve_usecs, glog->host, glog->serve_time);

  logger->resp_size += glog->resp_size;
#ifdef TCB_BTREE
  process_request_meta (ht_general_stats, "bandwidth", glog->resp_size);
#endif

cleanup:
  free_logger (glog);
  if (req_key != NULL)
    free (req_key);

  return 0;
}
Exemplo n.º 2
0
/* process a line from the log and store it accordingly */
static int
process_log (GLog * logger, char *line, int test)
{
  GLogItem *glog;
  char buf[DATE_LEN];
  char *qmark = NULL, *req_key = NULL;
  int not_found = 0;            /* 404s */

  /* make compiler happy */
  memset (buf, 0, sizeof (buf));

  if ((line == NULL) || (*line == '\0')) {
    logger->invalid++;
    return 0;
  }

  /* ignore comments */
  if (*line == '#' || *line == '\n')
    return 0;

  if (parsing_spinner != NULL && parsing_spinner->state == SPN_RUN)
    pthread_mutex_lock (&parsing_spinner->mutex);

  logger->process++;

#ifdef TCB_BTREE
  process_generic_data (ht_general_stats, "total_requests");
#endif

  if (parsing_spinner != NULL && parsing_spinner->state == SPN_RUN)
    pthread_mutex_unlock (&parsing_spinner->mutex);

  glog = init_log_item (logger);
  if (parse_format (glog, conf.log_format, conf.date_format, line) == 1) {
    logger->invalid++;

#ifdef TCB_BTREE
    process_generic_data (ht_general_stats, "failed_requests");
#endif
    goto cleanup;
  }

  /* must have the following fields */
  if (glog->host == NULL || glog->date == NULL || glog->status == NULL ||
      glog->req == NULL) {
    logger->invalid++;
    goto cleanup;
  }

  if (test)
    goto cleanup;

  convert_date (buf, glog->date, conf.date_format, "%Y%m%d", DATE_LEN);
  if (buf == NULL)
    goto cleanup;

  /* ignore host */
  if (conf.ignore_ip_idx && ip_in_range (glog->host)) {
    logger->exclude_ip++;
#ifdef TCB_BTREE
    process_generic_data (ht_general_stats, "exclude_ip");
#endif
    goto cleanup;
  }
  /* ignore crawlers */
  if (conf.ignore_crawlers && is_crawler (glog->agent))
    goto cleanup;

  /* agent will be null in cases where %u is not specified */
  if (glog->agent == NULL)
    glog->agent = alloc_string ("-");
  /* process visitors, browsers, and OS */
  process_unique_data (glog->host, buf, glog->agent);

  /* process agents that are part of a host */
  if (conf.list_agents)
    process_host_agents (glog->host, glog->agent);

  /* is this a 404? */
  if (!memcmp (glog->status, "404", 3)) {
    not_found = 1;
  }
  /* treat 444 as 404? */
  else if (!memcmp (glog->status, "444", 3) && conf.code444_as_404) {
    not_found = 1;
  }
  /* check if we need to remove the request's query string */
  else if (conf.ignore_qstr) {
    if ((qmark = strchr (glog->req, '?')) != NULL) {
      if ((qmark - glog->req) > 0)
        *qmark = '\0';
    }
  }

  req_key = xstrdup (glog->req);
  /* include HTTP method/protocol to request */
  if (conf.append_method && glog->method) {
    str_to_upper (glog->method);
    append_method_to_request (&req_key, glog->method);
  }
  if (conf.append_protocol && glog->protocol) {
    str_to_upper (glog->protocol);
    append_protocol_to_request (&req_key, glog->protocol);
  }
  if ((conf.append_method) || (conf.append_protocol))
    req_key = deblank (req_key);

  /* process 404s */
  if (not_found)
    process_request (ht_not_found_requests, req_key, glog);
  /* process static files */
  else if (verify_static_content (glog->req))
    process_request (ht_requests_static, req_key, glog);
  /* process regular files */
  else
    process_request (ht_requests, req_key, glog);

  /* process referrers */
  process_referrers (glog->ref);
  /* process status codes */
  process_generic_data (ht_status_code, glog->status);
  /* process hosts */
  process_generic_data (ht_hosts, glog->host);
  /* process bandwidth  */
  process_request_meta (ht_date_bw, buf, glog->resp_size);
  process_request_meta (ht_file_bw, req_key, glog->resp_size);
  process_request_meta (ht_host_bw, glog->host, glog->resp_size);
  /* process time taken to serve the request, in microseconds */
  process_request_meta (ht_file_serve_usecs, req_key, glog->serve_time);
  process_request_meta (ht_host_serve_usecs, glog->host, glog->serve_time);
  logger->resp_size += glog->resp_size;
#ifdef TCB_BTREE
  process_request_meta (ht_general_stats, "bandwidth", glog->resp_size);
#endif

cleanup:
  free_logger (glog);
  if (req_key != NULL)
    free (req_key);

  return 0;
}