void dequeuer_stats_rotate(struct dequeuer_stats* st, FILE *log)
{
double wbps, wpps;
time_t now = time(NULL);
time_t uptime;
if ( st->loss_since_last_rotate )
{
LOG_ER(log, "*** %lld packets lost in this journal ***\n",
st->loss_since_last_rotate);
}
LOG_INF(log, "Events written since last rotate:\n");
LOG_INF(log, " %lld bytes, %lld packets in this journal.\n",
st->bytes_written_since_last_rotate,
st->packets_written_since_last_rotate);
uptime = now - st->last_rotate;
wbps = (8. * (double)st->bytes_written_since_last_rotate) / (double)uptime;
wpps = ((double)st->packets_written_since_last_rotate) / (double)uptime;
log_rates(log, wbps,wpps," written");
LOG_INF(log, "Highest queue utilization since last rotate: %d packets.\n",
st->hiq_since_last_rotate);
LOG_INF(log, "Highest burst since last rotate: %lli packets, %lli bytes.\n",
st->packets_written_in_burst_since_last_rotate,
st->bytes_written_in_burst_since_last_rotate);
if (st->rotation_type == LJ_RT_EVENT) {
LOG_INF(log, "Command::Rotate from IP %s traversed the queue in %ld ms\n",
header_sender_ip_formatted(st->latest_rotate_header),
millis_now()-header_receipt_time(st->latest_rotate_header));
}
LOG_INF(log, "Dequeuer stats summary v2:\t%d\t%lld\t%lld\t%lld\t%lld\t%lld\t%lld\t%lld\t%d\t%d\t%d\t%d\t%lld\t%lld\t%d\n",
now, st->loss_since_last_rotate,
st->bytes_written_total, st->bytes_written_since_last_rotate,
st->packets_written_total, st->packets_written_since_last_rotate,
st->bytes_written_in_burst, st->packets_written_in_burst, st->hiq,
st->hiq_start, st->hiq_last, st->hiq_since_last_rotate,
st->bytes_written_in_burst_since_last_rotate,
st->packets_written_in_burst_since_last_rotate, uptime);
md_dequeuer_stats (st);
st->bytes_written_since_last_rotate = 0LL;
st->packets_written_since_last_rotate = 0LL;
st->hiq_since_last_rotate = 0;
st->packets_written_in_burst_since_last_rotate = 0LL;
st->bytes_written_in_burst_since_last_rotate = 0LL;
st->loss_since_last_rotate = 0LL;
st->last_rotate = now;
st->rotation_type = LJ_RT_NONE;
}
int main(int argc, char **argv)
{
gzFile file;
const char *filename;
const char *ctx_delim;
char header[22];
unsigned char buf[65535];
if (argc == 2)
{
filename = argv[1];
ctx_delim = "-";
}
else if (argc == 3)
{
filename = argv[1];
ctx_delim = argv[2];
}
else
{
fprintf (stderr, "Usage: prog <journal> [<context delimiter> (defaults to '-') ]\n");
exit (1);
}
file = gzopen (filename, "rb");
while (gzread (file, header, 22) == 22) {
unsigned short size = header_payload_length (header);
long long timestamp = (long long)(header_receipt_time (header));
long long seconds = (long long)(timestamp / 1000);
if (gzread (file, buf, size) != size)
{
fprintf (stderr, "error reading\n");
exit (1);
}
fprintf (stdout, "%lld\t%lld\t%u\n", seconds, timestamp, size);
}
gzclose (file);
exit (0);
}
int main(int argc, char **argv)
{
gzFile file;
const char *filename;
char header[22];
unsigned char buf[65535];
const char *args = "n:o:r:tdh";
int number = 0; /* (n) total number to emit */
struct lwes_emitter *emitter = NULL; /* (o) where to send the events */
int rate = 0; /* (r) number per second for emission */
bool use_timings = false; /* (t) using timings from file */
bool repeat = false; /* (d) rerun journals over and over */
int ret = 0;
LWES_INT_64 start = 0LL;
LWES_INT_64 stop = 0LL;
/* turn off error messages, I'll handle them */
opterr = 0;
while (1)
{
char c = getopt (argc, argv, args);
if (c == -1)
{
break;
}
switch (c)
{
case 'n':
number = atoi(optarg);
break;
case 'r':
rate = atoi(optarg);
break;
case 'o':
emitter = handle_transport_arg (optarg);
if (emitter == NULL)
{
fprintf (stderr, "ERROR: problem with emitter\n");
ret = 1;
goto cleanup;
}
break;
case 't':
use_timings = true;
break;
case 'd':
repeat = true;
break;
case 'h':
fprintf (stderr, "%s", help);
ret = 1;
goto cleanup;
default:
fprintf (stderr,
"WARNING: unrecognized command line option -%c\n",
optopt);
}
}
if (optind >= argc)
{
fprintf (stderr, "ERROR: journal file is required\n");
ret = 1;
goto cleanup;
}
/* setup sigint/sigkill/sigpipe handler */
setup_sig_handler();
start = currentTimeMillisLongLong ();
int start_index = optind;
int num_files = argc - optind;
int offset = 0;
int total_count = 0;
int this_second_count = 0;
unsigned long long delay_micros = rate > 0 ? 1000000 / rate : 0;
bool done = false;
while (! done)
{
/* deal with multiple files and repeating */
filename = argv[start_index + offset];
if (! repeat && offset == (num_files - 1))
{
done = true; /* last file and we are not repeating,
so done after it's processed */
}
offset = (offset + 1) % num_files; /* skip to next file if available */
file = gzopen (filename, "rb");
struct timeval start_emit_time;
micro_now (&start_emit_time);
struct timeval current_emit_time = { 0, 0 };
struct timeval previous_emit_time = { 0, 0 };
unsigned long long time_between_emit_events = 0ULL;
unsigned long long start_file_timestamp = 0ULL;
unsigned long long end_file_timestamp = 0ULL;
int file_count = 0;
unsigned long long time_to_sleep = 0ULL;
/* read a header from the file */
while (gzread (file, header, 22) == 22)
{
unsigned short size = header_payload_length (header);
unsigned long long cur_file_timestamp =
(unsigned long long)(header_receipt_time (header));
if (start_file_timestamp == 0ULL)
{
start_file_timestamp = cur_file_timestamp;
}
end_file_timestamp = cur_file_timestamp;
/* read an event from the file */
if (gzread (file, buf, size) != size)
{
fprintf (stderr, "ERROR: failure reading journal\n");
done = true;
ret=1;
break;
}
/* keep track of the current emit time */
micro_now (¤t_emit_time);
/* if we are using timings determine how long to sleep before
* emitting the next event */
if (use_timings)
{
unsigned long long time_between_file_events = 0ULL;
time_between_file_events =
(cur_file_timestamp - start_file_timestamp) * 1000ULL;
time_between_emit_events =
micro_timediff (&start_emit_time, ¤t_emit_time);
time_to_sleep =
time_between_file_events > time_between_emit_events
? time_between_file_events - time_between_emit_events
: 0ULL;
/* sleep some number of microseconds */
if (time_to_sleep > 0)
{
usleep (time_to_sleep);
}
}
/* if we are trying to meet a rate, determine how long to sleep
* before next call
*/
if (rate > 0)
{
if (previous_emit_time.tv_sec != 0)
{
/* the difference in micros alway includes the time slept,
* so to figure out how much time we should sleep we subtract
* the previous slept time from how much time it's actually
* been since the last emit. In other words sleep time plus
* processing time should equal delay_ms
*/
unsigned long long diff =
micro_timediff (&previous_emit_time, ¤t_emit_time)
- time_to_sleep;
time_to_sleep =
delay_micros > diff
? delay_micros - diff
: 0ULL;
if (time_to_sleep > 0)
{
usleep (time_to_sleep);
}
}
}
/* then emit the event */
if (lwes_emitter_emit_bytes (emitter, buf, size) != size)
{
fprintf (stderr, "ERROR: failure emitting\n");
done = true;
ret=1;
break;
}
/* keep track of some counts */
file_count++;
total_count++;
this_second_count++;
previous_emit_time = current_emit_time;
/* if we are emitting a limited number we will be done if we hit
* that number
*/
if (number > 0 && number == total_count)
{
done = true;
ret=2;
break;
}
if (gbl_sig)
{
done = true;
ret=3;
break;
}
}
gzclose (file);
fprintf (stderr,
"emitted %d events from %s representing %lld file time "
"in %llu milliseconds\n",
file_count, filename,
(end_file_timestamp - start_file_timestamp),
micro_timediff (&start_emit_time, ¤t_emit_time) / 1000ULL);
}
/* if we hit the count limit (2) it is not an error */
if (ret == 2)
{
ret = 0;
}
stop = currentTimeMillisLongLong ();
fprintf (stderr, "emitted %d events in %lld milliseconds\n",
total_count, (stop - start));
cleanup:
if (emitter != NULL)
{
lwes_emitter_destroy (emitter);
}
exit (ret);
}
int main(int argc, char **argv)
{
char header[22];
unsigned char buf[65535];
int ret = 0;
const char *args = "n:h";
int number = 10000; /* (n) number per file */
/* turn off error messages, I'll handle them */
opterr = 0;
while (1)
{
char c = getopt (argc, argv, args);
if (c == -1)
{
break;
}
switch (c)
{
case 'n':
number = atoi(optarg);
break;
case 'h':
fprintf (stderr, "%s", help);
ret = 1;
goto cleanup;
default:
fprintf (stderr,
"WARNING: unrecognized command line option -%c\n",
optopt);
}
}
int total_count = 0;
int current_count = 0;
unsigned long long start_file_timestamp = 0ULL;
unsigned long long end_file_timestamp = 0ULL;
const char *filename = argv[optind];
const char *tmpfile = "lwes-journal-split-tmp.gz";
char newfile[PATH_MAX];
int newfile_idx = 0;
char *buffer = strdup(filename);
char *tofree = buffer;
int file_part_count = 0;
const char *file_parts[MAX_FILE_PARTS];
const char *sep = ".";
const char *empty = "";
char *part;
int i;
for (i = 0; i < MAX_FILE_PARTS; i++)
{
file_parts[i] = empty;
}
while ((part = strsep (&buffer, sep)) && file_part_count < MAX_FILE_PARTS)
{
file_parts[file_part_count++] = part;
}
for (i = 0 ; i < file_part_count; i++)
{
printf ("part[%d] = %s\n",i,file_parts[i]);
}
newfile[newfile_idx] = '\0';
if (strcmp (file_parts[file_part_count-1],"gz") == 0)
{
if (check_num_and_length (file_parts[file_part_count-2], 10)
&& check_num_and_length (file_parts[file_part_count-3], 10))
{
for (i = 0; i < (file_part_count-3); i++)
{
strcat (newfile, file_parts[i]);
strcat (newfile, sep);
}
}
else
{
fprintf (stderr,
"WARNING: journal file name format is non-standard\n");
for (i = 0; i < (file_part_count-1); i++)
{
strcat (newfile, file_parts[i]);
strcat (newfile, sep);
}
}
}
else
{
fprintf (stderr,
"ERROR: journal must be a gzip file with .gz extension\n");
free (tofree);
ret = 1;
goto cleanup;
}
free (tofree);
newfile_idx = strlen (newfile);
fprintf (stderr, "newfile prefix is %s of length %d\n",newfile, newfile_idx);
gzFile tmp = gzopen (tmpfile, "wb");
gzFile file = gzopen (filename, "rb");
/* read a header from the file */
while (gzread (file, header, 22) == 22)
{
unsigned short size = header_payload_length (header);
unsigned long long cur_file_timestamp =
(unsigned long long)(header_receipt_time (header));
if (start_file_timestamp == 0ULL)
{
start_file_timestamp = cur_file_timestamp;
}
end_file_timestamp = cur_file_timestamp;
if (gzwrite (tmp, header, 22) != 22)
{
fprintf (stderr, "ERROR: failure writing journal\n");
ret=1;
break;
}
/* read an event from the file */
if (gzread (file, buf, size) != size)
{
fprintf (stderr, "ERROR: failure reading journal\n");
ret=1;
break;
}
if (gzwrite (tmp, buf, size) != size)
{
fprintf (stderr, "ERROR: failure writing journal\n");
ret=1;
break;
}
current_count++;
total_count++;
if (current_count == number) {
gzclose (tmp);
fprintf (stderr, "split from %llu to %llu %d\n",
start_file_timestamp, end_file_timestamp, current_count);
char renamedfile[PATH_MAX];
snprintf (renamedfile, sizeof (renamedfile), "%s%llu.%llu%s",
newfile,start_file_timestamp,end_file_timestamp, ".gz");
if (rename (tmpfile, renamedfile) < 0)
{
fprintf (stderr, "ERROR : failed to rename %s to %s\n",
tmpfile, renamedfile);
ret=1;
break;
}
start_file_timestamp = 0ULL;
current_count = 0;
tmp = gzopen (tmpfile, "wb");
}
}
gzclose (file);
gzclose (tmp);
char renamedfile[PATH_MAX];
snprintf (renamedfile, sizeof (renamedfile), "%s%llu.%llu%s",
newfile,start_file_timestamp,end_file_timestamp, ".gz");
if (rename (tmpfile, renamedfile) < 0)
{
fprintf (stderr, "ERROR : failed to rename %s to %s\n",
tmpfile, renamedfile);
ret=1;
}
fprintf (stderr, "%s has %d events\n", filename, total_count);
cleanup:
exit (ret);
}
