static void
print_html_summary (FILE * fp, GLog * logger)
{
char *bw, *size;
off_t log_size;
print_html_h2 (fp, T_HEAD, 0);
print_html_begin_table (fp);
print_html_begin_tbody (fp);
print_html_begin_tr (fp, 0);
print_html_summary_field (fp, logger->process, T_REQUESTS);
print_html_summary_field (fp, get_ht_size (ht_unique_visitors), T_UNIQUE_VIS);
print_html_summary_field (fp, get_ht_size (ht_referrers), T_REFERRER);
if (!logger->piping) {
log_size = file_size (conf.ifile);
size = filesize_str (log_size);
} else {
size = alloc_string ("N/A");
}
bw = filesize_str ((float) logger->resp_size);
if (conf.ifile == NULL)
conf.ifile = (char *) "STDIN";
fprintf (fp, "<td>%s</td><td>%s</td>", T_LOG, size);
print_html_end_tr (fp);
print_html_begin_tr (fp, 0);
print_html_summary_field (fp, logger->invalid, T_F_REQUESTS);
print_html_summary_field (fp, get_ht_size (ht_requests), T_UNIQUE_FIL);
print_html_summary_field (fp, get_ht_size (ht_not_found_requests),
T_UNIQUE404);
fprintf (fp, "<td>%s</td><td>%s</td>", T_BW, bw);
print_html_end_tr (fp);
print_html_begin_tr (fp, 0);
fprintf (fp, "<td>%s</td>", T_GEN_TIME);
fprintf (fp, "<td>%llu</td>", ((long long) end_proc - start_proc));
print_html_summary_field (fp, get_ht_size (ht_requests_static), T_STATIC_FIL);
fprintf (fp, "<td colspan=\"4\">%s</td>", conf.ifile);
print_html_end_tr (fp);
print_html_end_tbody (fp);
print_html_end_table (fp);
free (bw);
free (size);
}
/* generate JSON unique visitors stats */
static void
print_json_visitors (FILE * fp, GHolder * h)
{
char *data, buf[DATE_LEN];
float percent;
int hits, bw, i, process = get_ht_size (ht_unique_visitors);
/* make compiler happy */
memset (buf, 0, sizeof (buf));
fprintf (fp, "\t\"%s\": [\n", VISIT_ID);
for (i = 0; i < h->idx; i++) {
hits = h->items[i].hits;
data = h->items[i].data;
percent = get_percentage (process, hits);
percent = percent < 0 ? 0 : percent;
bw = h->items[i].bw;
convert_date (buf, data, "%Y%m%d", "%d/%b/%Y", DATE_LEN);
fprintf (fp, "\t\t{\n\t\t\t\"hits\": \"%d\",\n", hits);
fprintf (fp, "\t\t\t\"percent\": \"%4.2f%%\",\n", percent);
fprintf (fp, "\t\t\t\"date\": \"%s\",\n", buf);
fprintf (fp, "\t\t\t\"bytes\": \"%d\"\n", bw);
fprintf (fp, "\t\t}");
if (i != h->idx - 1)
fprintf (fp, ",\n");
else
fprintf (fp, "\n");
}
fprintf (fp, "\t]");
}
uint32_t
get_ht_size_by_metric (GModule module, GMetric metric)
{
GHashTable *ht = get_storage_metric (module, metric);
return get_ht_size (ht);
}
/* generate CSV unique visitors stats */
static void
print_csv_visitors (FILE * fp, GHolder * h)
{
char *data, buf[DATE_LEN];
float percent;
int hits, bw, i, process = get_ht_size (ht_unique_visitors);
/* make compiler happy */
memset (buf, 0, sizeof (buf));
for (i = 0; i < h->idx; i++) {
hits = h->items[i].hits;
data = h->items[i].data;
percent = get_percentage (process, hits);
percent = percent < 0 ? 0 : percent;
bw = h->items[i].bw;
convert_date (buf, data, "%Y%m%d", "%d/%b/%Y", DATE_LEN);
fprintf (fp, "\"%d\",", i); /* idx */
fprintf (fp, ","); /* parent idx */
fprintf (fp, "\"%s\",", VISIT_ID);
fprintf (fp, "\"%d\",", hits);
fprintf (fp, "\"%4.2f%%\",", percent);
fprintf (fp, "\"%s\",", buf);
fprintf (fp, "\"%d\"\r\n", bw);
}
}
int
ht_insert_uniqmap (TCADB * adb, char *uniq_key)
{
void *value_ptr;
int nkey = 0, size = 0;
if ((adb == NULL) || (uniq_key == NULL))
return (EINVAL);
if ((value_ptr = tcadbget2 (adb, uniq_key)) != NULL) {
free (value_ptr);
return 0;
}
size = get_ht_size (adb);
/* the auto increment value starts at SIZE (hash table) + 1 */
nkey = size > 0 ? size + 1 : 1;
if (!tcadbput (adb, uniq_key, strlen (uniq_key), &nkey, sizeof (int)))
LOG_DEBUG (("Unable to tcadbput\n"));
free (uniq_key);
return nkey;
}
uint32_t
get_ht_size_by_metric (GModule module, GMetric metric)
{
TCADB *adb = get_storage_metric (module, metric);
return get_ht_size (adb);
}
/* generate overview stats */
static void
print_csv_summary (FILE * fp, GLog * logger)
{
int i = 0;
off_t log_size = 0;
char now[DATE_TIME];
generate_time ();
strftime (now, DATE_TIME, "%Y-%m-%d %H:%M:%S", now_tm);
/* general statistics info */
fprintf (fp, "\"%d\",,\"%s\",\"date_time\",\"%s\"\r\n", i++, GENER_ID, now);
fprintf (fp, "\"%d\",,\"%s\",\"total_requests\",\"%d\"\r\n", i++, GENER_ID,
logger->process);
fprintf (fp, "\"%d\",,\"%s\",\"unique_visitors\",\"%d\"\r\n", i++, GENER_ID,
get_ht_size (ht_unique_visitors));
fprintf (fp, "\"%d\",,\"%s\",\"referrers\",\"%d\"\r\n", i++, GENER_ID,
get_ht_size (ht_referrers));
if (!logger->piping)
log_size = file_size (conf.ifile);
fprintf (fp, "\"%d\",,\"%s\",\"log_size\",\"%jd\"\r\n", i++, GENER_ID,
(intmax_t) log_size);
fprintf (fp, "\"%d\",,\"%s\",\"failed_requests\",\"%d\"\r\n", i++, GENER_ID,
logger->invalid);
fprintf (fp, "\"%d\",,\"%s\",\"unique_files\",\"%d\"\r\n", i++, GENER_ID,
get_ht_size (ht_requests));
fprintf (fp, "\"%d\",,\"%s\",\"unique_404\",\"%d\"\r\n", i++, GENER_ID,
get_ht_size (ht_not_found_requests));
fprintf (fp, "\"%d\",,\"%s\",\"bandwidth\",\"%lld\"\r\n", i++, GENER_ID,
logger->resp_size);
fprintf (fp, "\"%d\",,\"%s\",\"generation_time\",\"%llu\"\r\n", i++,
GENER_ID, (long long) end_proc - start_proc);
fprintf (fp, "\"%d\",,\"%s\",\"static_files\",\"%d\"\r\n", i++, GENER_ID,
get_ht_size (ht_requests_static));
if (conf.ifile == NULL)
conf.ifile = (char *) "STDIN";
fprintf (fp, "\"%d\",,\"%s\",\"log_file\",\"%s\"\r\n", i, GENER_ID,
conf.ifile);
}
/* follow the JSON style similar to http://developer.github.com/v3/ */
void
output_json (GLog * logger, GHolder * holder)
{
FILE *fp = stdout;
fprintf (fp, "{\n"); /* open */
print_json_summary (fp, logger);
fprintf (fp, ",\n");
print_json_visitors (fp, holder + VISITORS);
fprintf (fp, ",\n");
print_json_complete (fp, holder, logger->process);
fprintf (fp, ",\n");
print_json_generic (fp, holder + OS, get_ht_size (ht_unique_visitors));
fprintf (fp, ",\n");
print_json_generic (fp, holder + BROWSERS, get_ht_size (ht_unique_visitors));
fprintf (fp, ",\n");
print_json_generic (fp, holder + REFERRERS, logger->process);
fprintf (fp, ",\n");
print_json_generic (fp, holder + REFERRING_SITES, logger->process);
fprintf (fp, ",\n");
print_json_generic (fp, holder + KEYPHRASES, logger->process);
fprintf (fp, ",\n");
#ifdef HAVE_LIBGEOIP
print_json_generic (fp, holder + GEO_LOCATION,
get_ht_size (ht_unique_visitors));
fprintf (fp, ",\n");
#endif
print_json_generic (fp, holder + STATUS_CODES, logger->process);
fprintf (fp, "\n");
fprintf (fp, "\n}\n"); /* close */
fclose (fp);
}
/* entry point to generate a a csv report writing it to the fp */
void
output_csv (GLog * logger, GHolder * holder)
{
FILE *fp = stdout;
print_csv_summary (fp, logger);
print_csv_visitors (fp, holder + VISITORS);
print_csv_complete (fp, holder, logger->process);
print_csv_generic (fp, holder + OS, get_ht_size (ht_unique_visitors));
print_csv_generic (fp, holder + BROWSERS, get_ht_size (ht_unique_visitors));
print_csv_generic (fp, holder + REFERRERS, logger->process);
print_csv_generic (fp, holder + REFERRING_SITES, logger->process);
print_csv_generic (fp, holder + KEYPHRASES, logger->process);
#ifdef HAVE_LIBGEOIP
print_csv_generic (fp, holder + GEO_LOCATION, logger->process);
#endif
print_csv_generic (fp, holder + STATUS_CODES, logger->process);
fclose (fp);
}
/* generate overview stats */
static void
print_json_summary (FILE * fp, GLog * logger)
{
off_t log_size = 0;
char now[DATE_TIME];
generate_time ();
strftime (now, DATE_TIME, "%Y-%m-%d %H:%M:%S", now_tm);
fprintf (fp, "\t\"%s\": {\n", GENER_ID);
/* general statistics info */
fprintf (fp, "\t\t\"date_time\": \"%s\",\n", now);
fprintf (fp, "\t\t\"total_requests\": %d,\n", logger->process);
fprintf (fp, "\t\t\"unique_visitors\": %d,\n",
get_ht_size (ht_unique_visitors));
fprintf (fp, "\t\t\"referrers\": %d,\n", get_ht_size (ht_referrers));
if (!logger->piping)
log_size = file_size (conf.ifile);
fprintf (fp, "\t\t\"log_size\": %jd,\n", (intmax_t) log_size);
fprintf (fp, "\t\t\"failed_requests\": %d,\n", logger->invalid);
fprintf (fp, "\t\t\"unique_files\": %d,\n", get_ht_size (ht_requests));
fprintf (fp, "\t\t\"unique_404\": %d,\n",
get_ht_size (ht_not_found_requests));
fprintf (fp, "\t\t\"bandwidth\": %lld,\n", logger->resp_size);
fprintf (fp, "\t\t\"generation_time\": %llu,\n",
((long long) end_proc - start_proc));
fprintf (fp, "\t\t\"excluded_ip_hits\": %u,\n", logger->exclude_ip);
fprintf (fp, "\t\t\"static_files\": %d,\n", get_ht_size (ht_requests_static));
if (conf.ifile == NULL)
conf.ifile = (char *) "STDIN";
fprintf (fp, "\t\t\"log_file\": \"%s\"\n", conf.ifile);
fprintf (fp, "\t}");
}
static void
set_general_stats (void)
{
logger->process = logger->invalid = logger->exclude_ip = 0;
#ifdef TCB_BTREE
logger->exclude_ip = tc_db_get_int (ht_general_stats, "exclude_ip");
logger->invalid = tc_db_get_int (ht_general_stats, "failed_requests");
logger->process = tc_db_get_int (ht_general_stats, "total_requests");
logger->resp_size = tc_db_get_uint64 (ht_general_stats, "bandwidth");
if (logger->resp_size > 0)
conf.bandwidth = 1;
if (get_ht_size (ht_file_serve_usecs) > 0)
conf.serve_usecs = 1;
#endif
}
/* allocate memory for an instance of holder */
static void
allocate_holder_by_module (GModule module)
{
#if defined(TCB_BTREE) || defined(TCB_MEMHASH)
TCADB *ht = NULL;
#else
GHashTable *ht;
#endif
GRawData *raw_data;
unsigned int ht_size = 0;
/* extract data from the corresponding hash table */
ht = get_storage_metric (module, MTRC_HITS);
ht_size = get_ht_size (ht);
raw_data = parse_raw_data (ht, ht_size, module);
load_holder_data (raw_data, holder + module, module, module_sort[module]);
}
int
ht_insert_uniqmap (GHashTable * ht, char *uniq_key)
{
int nkey = 0, size = 0;
if (ht == NULL)
return (EINVAL);
if ((g_hash_table_lookup (ht, uniq_key)) != NULL)
return 0;
size = get_ht_size (ht);
/* the auto increment value starts at SIZE (hash table) + 1 */
nkey = size > 0 ? size + 1 : 1;
g_hash_table_replace (ht, uniq_key, int2ptr (nkey));
return nkey;
}
int
ht_insert_keymap (GHashTable * ht, const char *value)
{
gpointer value_ptr;
int nkey = 0, size = 0;
if (ht == NULL)
return (EINVAL);
value_ptr = g_hash_table_lookup (ht, value);
if (value_ptr != NULL)
return (*(int *) value_ptr);
size = get_ht_size (ht);
/* the auto increment value starts at SIZE (hash table) + 1 */
nkey = size > 0 ? size + 1 : 1;
g_hash_table_replace (ht, g_strdup (value), int2ptr (nkey));
return nkey;
}
/* allocate memory for an instance of holder */
static void
allocate_holder (void)
{
#if defined(TCB_BTREE) || defined(TCB_MEMHASH)
TCADB *ht = NULL;
#else
GHashTable *ht;
#endif
GModule module;
GRawData *raw_data;
unsigned int ht_size = 0;
holder = new_gholder (TOTAL_MODULES);
for (module = 0; module < TOTAL_MODULES; module++) {
/* extract data from the corresponding hits hash table */
ht = get_storage_metric (module, MTRC_HITS);
ht_size = get_ht_size (ht);
raw_data = parse_raw_data (ht, ht_size, module);
load_holder_data (raw_data, holder + module, module, module_sort[module]);
}
}
int
ht_insert_keymap (TCADB * adb, const char *value)
{
void *value_ptr;
int nkey = 0, size = 0, ret = 0;
if ((adb == NULL) || (value == NULL))
return (EINVAL);
if ((value_ptr = tcadbget2 (adb, value)) != NULL) {
ret = (*(int *) value_ptr);
free (value_ptr);
return ret;
}
size = get_ht_size (adb);
/* the auto increment value starts at SIZE (hash table) + 1 */
nkey = size > 0 ? size + 1 : 1;
tcadbput (adb, value, strlen (value), &nkey, sizeof (int));
return nkey;
}
static void
print_html_browser_os (FILE * fp, GHolder * h)
{
char *data;
const char *desc = OPERA_DESC;
const char *head = OPERA_HEAD;
const char *id = OPERA_ID;
float percent, l;
GSubList *sub_list;
int hits, i, max, process = get_ht_size (ht_unique_visitors);
if (h->idx == 0)
return;
if (h->module == BROWSERS) {
head = BROWS_HEAD;
id = BROWS_ID;
desc = BROWS_DESC;
}
print_html_h2 (fp, head, id);
print_p (fp, desc);
print_html_begin_table (fp);
print_html_begin_thead (fp);
fprintf (fp, "<tr>");
fprintf (fp, "<th>Visitors</th>");
fprintf (fp, "<th>%%</th>");
fprintf (fp, "<th>Name</th>");
fprintf (fp, "<th style=\"width:100%%;text-align:right;\">");
fprintf (fp, "<span class=\"r\" onclick=\"t(this)\">◀</span>");
fprintf (fp, "</th>");
fprintf (fp, "</tr>");
print_html_end_thead (fp);
print_html_begin_tbody (fp);
max = 0;
for (i = 0; i < h->idx; i++) {
if (h->items[i].hits > max)
max = h->items[i].hits;
}
for (i = 0; i < h->idx; i++) {
hits = h->items[i].hits;
data = h->items[i].data;
percent = get_percentage (process, hits);
percent = percent < 0 ? 0 : percent;
l = get_percentage (max, hits);
l = l < 1 ? 1 : l;
print_html_begin_tr (fp, 0);
fprintf (fp, "<td>%d</td>", hits);
fprintf (fp, "<td>%4.2f%%</td>", percent);
/* data */
fprintf (fp, "<td>");
clean_output (fp, data);
fprintf (fp, "</td>");
fprintf (fp, "<td class=\"graph\">");
fprintf (fp, "<div class=\"bar\" style=\"width:%f%%\"></div>", l);
fprintf (fp, "</td>");
print_html_end_tr (fp);
sub_list = h->items[i].sub_list;
print_html_sub_browser_os (fp, sub_list, process);
}
print_html_end_tbody (fp);
print_html_end_table (fp);
}
/* render general statistics */
void
display_general (WINDOW * win, char *ifile, GLog * logger)
{
char *bw, *size, *log_file;
char *failed, *not_found, *process, *ref, *req;
char *static_files, *now, *visitors, *exclude_ip;
int x_field = 2, x_value = 0;
size_t n, i, j, max_field = 0, max_value = 0, mod_val, y;
typedef struct Field_
{
const char *field;
char *value; /* char due to log, bw, log_file */
int color;
} Field;
Field fields[] = {
{T_REQUESTS, NULL, COL_CYAN},
{T_UNIQUE_VIS, NULL, COL_CYAN},
{T_REFERRER, NULL, COL_CYAN},
{T_LOG, NULL, COL_CYAN},
{T_F_REQUESTS, NULL, COL_CYAN},
{T_UNIQUE_FIL, NULL, COL_CYAN},
{T_UNIQUE404, NULL, COL_CYAN},
{T_BW, NULL, COL_CYAN},
{T_GEN_TIME, NULL, COL_CYAN},
{T_EXCLUDE_IP, NULL, COL_CYAN},
{T_STATIC_FIL, NULL, COL_CYAN},
{T_LOG_PATH, NULL, COL_YELLOW}
};
werase (win);
draw_header (win, T_HEAD, " %s", 0, 0, getmaxx (stdscr), 1, 0);
if (!logger->piping && ifile != NULL) {
size = filesize_str (file_size (ifile));
log_file = alloc_string (ifile);
} else {
size = alloc_string ("N/A");
log_file = alloc_string ("STDIN");
}
bw = filesize_str ((float) logger->resp_size);
/* *INDENT-OFF* */
failed = int_to_str (logger->invalid);
not_found = int_to_str (get_ht_size (ht_not_found_requests));
process = int_to_str (logger->process);
ref = int_to_str (get_ht_size (ht_referrers));
req = int_to_str (get_ht_size(ht_requests));
static_files = int_to_str (get_ht_size(ht_requests_static));
now = int_to_str (((long long) end_proc - start_proc));
visitors = int_to_str (get_ht_size(ht_unique_visitors));
exclude_ip = int_to_str (logger->exclude_ip);
fields[0].value = process;
fields[1].value = visitors;
fields[2].value = ref;
fields[3].value = size;
fields[4].value = failed;
fields[5].value = req;
fields[6].value = not_found;
fields[7].value = bw;
fields[8].value = now;
fields[9].value = exclude_ip;
fields[10].value = static_files;
fields[11].value = log_file;
n = ARRAY_SIZE (fields);
/* *INDENT-ON* */
for (i = 0, y = 2; i < n; i++) {
mod_val = i % 4;
if (i > 0 && mod_val == 0) {
max_field = 0;
max_value = 0;
x_field = 2;
x_value = 2;
y++;
}
x_field += max_field;
mvwprintw (win, y, x_field, "%s", fields[i].field);
max_field = 0;
for (j = 0; j < n; j++) {
size_t len = strlen (fields[j].field);
if (j % 4 == mod_val && len > max_field)
max_field = len;
}
max_value = 0;
for (j = 0; j < n; j++) {
size_t len = strlen (fields[j].value);
if (j % 4 == mod_val && len > max_value)
max_value = len;
}
x_value = max_field + x_field + 1;
max_field += max_value + 2;
wattron (win, A_BOLD | COLOR_PAIR (fields[i].color));
mvwprintw (win, y, x_value, "%s", fields[i].value);
wattroff (win, A_BOLD | COLOR_PAIR (fields[i].color));
}
for (i = 0; i < n; i++) {
free (fields[i].value);
}
}
static void
print_html_visitors_report (FILE * fp, GHolder * h)
{
char *data, *bandwidth, buf[DATE_LEN];
float percent, l;
int hits, i, max, process = get_ht_size (ht_unique_visitors);
/* make compiler happy */
memset (buf, 0, sizeof (buf));
print_html_h2 (fp, VISIT_HEAD, VISIT_ID);
print_p (fp, VISIT_DESC);
print_html_begin_table (fp);
print_html_begin_thead (fp);
fprintf (fp, "<tr>");
fprintf (fp, "<th>Visitors</th>");
fprintf (fp, "<th>%%</th>");
fprintf (fp, "<th>Date</th>");
fprintf (fp, "<th>Bandwidth</th>");
fprintf (fp, "<th style=\"width:100%%;text-align:right;\">");
fprintf (fp, "<span class=\"r\" onclick=\"t(this)\">◀</span>");
fprintf (fp, "</th>");
fprintf (fp, "</tr>");
print_html_end_thead (fp);
print_html_begin_tbody (fp);
max = 0;
for (i = 0; i < h->idx; i++) {
if (h->items[i].hits > max)
max = h->items[i].hits;
}
for (i = 0; i < h->idx; i++) {
hits = h->items[i].hits;
data = h->items[i].data;
percent = get_percentage (process, hits);
percent = percent < 0 ? 0 : percent;
bandwidth = filesize_str (h->items[i].bw);
l = get_percentage (max, hits);
l = l < 1 ? 1 : l;
print_html_begin_tr (fp, i > OUTPUT_N ? 1 : 0);
/* hits */
fprintf (fp, "<td>%d</td>", hits);
if (hits == max)
fprintf (fp, "<td class=\"max\">%4.2f%%</td>", percent);
else
fprintf (fp, "<td>%4.2f%%</td>", percent);
/* date */
convert_date (buf, data, "%Y%m%d", "%d/%b/%Y", DATE_LEN);
fprintf (fp, "<td>%s</td>", buf);
/* bandwidth */
fprintf (fp, "<td>");
clean_output (fp, bandwidth);
fprintf (fp, "</td>");
/* bars */
fprintf (fp, "<td class=\"graph\">");
fprintf (fp, "<div class=\"bar\" style=\"width:%f%%\"></div>", l);
fprintf (fp, "</td>\n");
print_html_end_tr (fp);
free (bandwidth);
}
print_html_end_tbody (fp);
print_html_end_table (fp);
}
