void dequeuer_stats_rotate(struct dequeuer_stats* st, FILE *log) { double wbps, wpps; time_t now = time(NULL); time_t uptime; if ( st->loss_since_last_rotate ) { LOG_ER(log, "*** %lld packets lost in this journal ***\n", st->loss_since_last_rotate); } LOG_INF(log, "Events written since last rotate:\n"); LOG_INF(log, " %lld bytes, %lld packets in this journal.\n", st->bytes_written_since_last_rotate, st->packets_written_since_last_rotate); uptime = now - st->last_rotate; wbps = (8. * (double)st->bytes_written_since_last_rotate) / (double)uptime; wpps = ((double)st->packets_written_since_last_rotate) / (double)uptime; log_rates(log, wbps,wpps," written"); LOG_INF(log, "Highest queue utilization since last rotate: %d packets.\n", st->hiq_since_last_rotate); LOG_INF(log, "Highest burst since last rotate: %lli packets, %lli bytes.\n", st->packets_written_in_burst_since_last_rotate, st->bytes_written_in_burst_since_last_rotate); if (st->rotation_type == LJ_RT_EVENT) { LOG_INF(log, "Command::Rotate from IP %s traversed the queue in %ld ms\n", header_sender_ip_formatted(st->latest_rotate_header), millis_now()-header_receipt_time(st->latest_rotate_header)); } LOG_INF(log, "Dequeuer stats summary v2:\t%d\t%lld\t%lld\t%lld\t%lld\t%lld\t%lld\t%lld\t%d\t%d\t%d\t%d\t%lld\t%lld\t%d\n", now, st->loss_since_last_rotate, st->bytes_written_total, st->bytes_written_since_last_rotate, st->packets_written_total, st->packets_written_since_last_rotate, st->bytes_written_in_burst, st->packets_written_in_burst, st->hiq, st->hiq_start, st->hiq_last, st->hiq_since_last_rotate, st->bytes_written_in_burst_since_last_rotate, st->packets_written_in_burst_since_last_rotate, uptime); md_dequeuer_stats (st); st->bytes_written_since_last_rotate = 0LL; st->packets_written_since_last_rotate = 0LL; st->hiq_since_last_rotate = 0; st->packets_written_in_burst_since_last_rotate = 0LL; st->bytes_written_in_burst_since_last_rotate = 0LL; st->loss_since_last_rotate = 0LL; st->last_rotate = now; st->rotation_type = LJ_RT_NONE; }
int main(int argc, char **argv) { gzFile file; const char *filename; const char *ctx_delim; char header[22]; unsigned char buf[65535]; if (argc == 2) { filename = argv[1]; ctx_delim = "-"; } else if (argc == 3) { filename = argv[1]; ctx_delim = argv[2]; } else { fprintf (stderr, "Usage: prog <journal> [<context delimiter> (defaults to '-') ]\n"); exit (1); } file = gzopen (filename, "rb"); while (gzread (file, header, 22) == 22) { unsigned short size = header_payload_length (header); long long timestamp = (long long)(header_receipt_time (header)); long long seconds = (long long)(timestamp / 1000); if (gzread (file, buf, size) != size) { fprintf (stderr, "error reading\n"); exit (1); } fprintf (stdout, "%lld\t%lld\t%u\n", seconds, timestamp, size); } gzclose (file); exit (0); }
int main(int argc, char **argv) { gzFile file; const char *filename; char header[22]; unsigned char buf[65535]; const char *args = "n:o:r:tdh"; int number = 0; /* (n) total number to emit */ struct lwes_emitter *emitter = NULL; /* (o) where to send the events */ int rate = 0; /* (r) number per second for emission */ bool use_timings = false; /* (t) using timings from file */ bool repeat = false; /* (d) rerun journals over and over */ int ret = 0; LWES_INT_64 start = 0LL; LWES_INT_64 stop = 0LL; /* turn off error messages, I'll handle them */ opterr = 0; while (1) { char c = getopt (argc, argv, args); if (c == -1) { break; } switch (c) { case 'n': number = atoi(optarg); break; case 'r': rate = atoi(optarg); break; case 'o': emitter = handle_transport_arg (optarg); if (emitter == NULL) { fprintf (stderr, "ERROR: problem with emitter\n"); ret = 1; goto cleanup; } break; case 't': use_timings = true; break; case 'd': repeat = true; break; case 'h': fprintf (stderr, "%s", help); ret = 1; goto cleanup; default: fprintf (stderr, "WARNING: unrecognized command line option -%c\n", optopt); } } if (optind >= argc) { fprintf (stderr, "ERROR: journal file is required\n"); ret = 1; goto cleanup; } /* setup sigint/sigkill/sigpipe handler */ setup_sig_handler(); start = currentTimeMillisLongLong (); int start_index = optind; int num_files = argc - optind; int offset = 0; int total_count = 0; int this_second_count = 0; unsigned long long delay_micros = rate > 0 ? 1000000 / rate : 0; bool done = false; while (! done) { /* deal with multiple files and repeating */ filename = argv[start_index + offset]; if (! repeat && offset == (num_files - 1)) { done = true; /* last file and we are not repeating, so done after it's processed */ } offset = (offset + 1) % num_files; /* skip to next file if available */ file = gzopen (filename, "rb"); struct timeval start_emit_time; micro_now (&start_emit_time); struct timeval current_emit_time = { 0, 0 }; struct timeval previous_emit_time = { 0, 0 }; unsigned long long time_between_emit_events = 0ULL; unsigned long long start_file_timestamp = 0ULL; unsigned long long end_file_timestamp = 0ULL; int file_count = 0; unsigned long long time_to_sleep = 0ULL; /* read a header from the file */ while (gzread (file, header, 22) == 22) { unsigned short size = header_payload_length (header); unsigned long long cur_file_timestamp = (unsigned long long)(header_receipt_time (header)); if (start_file_timestamp == 0ULL) { start_file_timestamp = cur_file_timestamp; } end_file_timestamp = cur_file_timestamp; /* read an event from the file */ if (gzread (file, buf, size) != size) { fprintf (stderr, "ERROR: failure reading journal\n"); done = true; ret=1; break; } /* keep track of the current emit time */ micro_now (¤t_emit_time); /* if we are using timings determine how long to sleep before * emitting the next event */ if (use_timings) { unsigned long long time_between_file_events = 0ULL; time_between_file_events = (cur_file_timestamp - start_file_timestamp) * 1000ULL; time_between_emit_events = micro_timediff (&start_emit_time, ¤t_emit_time); time_to_sleep = time_between_file_events > time_between_emit_events ? time_between_file_events - time_between_emit_events : 0ULL; /* sleep some number of microseconds */ if (time_to_sleep > 0) { usleep (time_to_sleep); } } /* if we are trying to meet a rate, determine how long to sleep * before next call */ if (rate > 0) { if (previous_emit_time.tv_sec != 0) { /* the difference in micros alway includes the time slept, * so to figure out how much time we should sleep we subtract * the previous slept time from how much time it's actually * been since the last emit. In other words sleep time plus * processing time should equal delay_ms */ unsigned long long diff = micro_timediff (&previous_emit_time, ¤t_emit_time) - time_to_sleep; time_to_sleep = delay_micros > diff ? delay_micros - diff : 0ULL; if (time_to_sleep > 0) { usleep (time_to_sleep); } } } /* then emit the event */ if (lwes_emitter_emit_bytes (emitter, buf, size) != size) { fprintf (stderr, "ERROR: failure emitting\n"); done = true; ret=1; break; } /* keep track of some counts */ file_count++; total_count++; this_second_count++; previous_emit_time = current_emit_time; /* if we are emitting a limited number we will be done if we hit * that number */ if (number > 0 && number == total_count) { done = true; ret=2; break; } if (gbl_sig) { done = true; ret=3; break; } } gzclose (file); fprintf (stderr, "emitted %d events from %s representing %lld file time " "in %llu milliseconds\n", file_count, filename, (end_file_timestamp - start_file_timestamp), micro_timediff (&start_emit_time, ¤t_emit_time) / 1000ULL); } /* if we hit the count limit (2) it is not an error */ if (ret == 2) { ret = 0; } stop = currentTimeMillisLongLong (); fprintf (stderr, "emitted %d events in %lld milliseconds\n", total_count, (stop - start)); cleanup: if (emitter != NULL) { lwes_emitter_destroy (emitter); } exit (ret); }
int main(int argc, char **argv) { char header[22]; unsigned char buf[65535]; int ret = 0; const char *args = "n:h"; int number = 10000; /* (n) number per file */ /* turn off error messages, I'll handle them */ opterr = 0; while (1) { char c = getopt (argc, argv, args); if (c == -1) { break; } switch (c) { case 'n': number = atoi(optarg); break; case 'h': fprintf (stderr, "%s", help); ret = 1; goto cleanup; default: fprintf (stderr, "WARNING: unrecognized command line option -%c\n", optopt); } } int total_count = 0; int current_count = 0; unsigned long long start_file_timestamp = 0ULL; unsigned long long end_file_timestamp = 0ULL; const char *filename = argv[optind]; const char *tmpfile = "lwes-journal-split-tmp.gz"; char newfile[PATH_MAX]; int newfile_idx = 0; char *buffer = strdup(filename); char *tofree = buffer; int file_part_count = 0; const char *file_parts[MAX_FILE_PARTS]; const char *sep = "."; const char *empty = ""; char *part; int i; for (i = 0; i < MAX_FILE_PARTS; i++) { file_parts[i] = empty; } while ((part = strsep (&buffer, sep)) && file_part_count < MAX_FILE_PARTS) { file_parts[file_part_count++] = part; } for (i = 0 ; i < file_part_count; i++) { printf ("part[%d] = %s\n",i,file_parts[i]); } newfile[newfile_idx] = '\0'; if (strcmp (file_parts[file_part_count-1],"gz") == 0) { if (check_num_and_length (file_parts[file_part_count-2], 10) && check_num_and_length (file_parts[file_part_count-3], 10)) { for (i = 0; i < (file_part_count-3); i++) { strcat (newfile, file_parts[i]); strcat (newfile, sep); } } else { fprintf (stderr, "WARNING: journal file name format is non-standard\n"); for (i = 0; i < (file_part_count-1); i++) { strcat (newfile, file_parts[i]); strcat (newfile, sep); } } } else { fprintf (stderr, "ERROR: journal must be a gzip file with .gz extension\n"); free (tofree); ret = 1; goto cleanup; } free (tofree); newfile_idx = strlen (newfile); fprintf (stderr, "newfile prefix is %s of length %d\n",newfile, newfile_idx); gzFile tmp = gzopen (tmpfile, "wb"); gzFile file = gzopen (filename, "rb"); /* read a header from the file */ while (gzread (file, header, 22) == 22) { unsigned short size = header_payload_length (header); unsigned long long cur_file_timestamp = (unsigned long long)(header_receipt_time (header)); if (start_file_timestamp == 0ULL) { start_file_timestamp = cur_file_timestamp; } end_file_timestamp = cur_file_timestamp; if (gzwrite (tmp, header, 22) != 22) { fprintf (stderr, "ERROR: failure writing journal\n"); ret=1; break; } /* read an event from the file */ if (gzread (file, buf, size) != size) { fprintf (stderr, "ERROR: failure reading journal\n"); ret=1; break; } if (gzwrite (tmp, buf, size) != size) { fprintf (stderr, "ERROR: failure writing journal\n"); ret=1; break; } current_count++; total_count++; if (current_count == number) { gzclose (tmp); fprintf (stderr, "split from %llu to %llu %d\n", start_file_timestamp, end_file_timestamp, current_count); char renamedfile[PATH_MAX]; snprintf (renamedfile, sizeof (renamedfile), "%s%llu.%llu%s", newfile,start_file_timestamp,end_file_timestamp, ".gz"); if (rename (tmpfile, renamedfile) < 0) { fprintf (stderr, "ERROR : failed to rename %s to %s\n", tmpfile, renamedfile); ret=1; break; } start_file_timestamp = 0ULL; current_count = 0; tmp = gzopen (tmpfile, "wb"); } } gzclose (file); gzclose (tmp); char renamedfile[PATH_MAX]; snprintf (renamedfile, sizeof (renamedfile), "%s%llu.%llu%s", newfile,start_file_timestamp,end_file_timestamp, ".gz"); if (rename (tmpfile, renamedfile) < 0) { fprintf (stderr, "ERROR : failed to rename %s to %s\n", tmpfile, renamedfile); ret=1; } fprintf (stderr, "%s has %d events\n", filename, total_count); cleanup: exit (ret); }