// Time: seconds.millis
// Interval: number
// IntervalPercentiles: ( 50% 90% Max )
// TotalPercentiles: ( 50% 90% 99% 99.9% 99.99% max )
void print_histogram_line (FILE *out, struct hdr_histogram *interval, struct hdr_histogram *cumulative)
{
struct timeval now;
unsigned int millis_delta, seconds_delta, millis_remainder;
gettimeofday(&now,NULL);
millis_delta = ((1000000 * (now.tv_sec - start.tv_sec)) + now.tv_usec - start.tv_usec)/1000;
seconds_delta = millis_delta/1000;
millis_remainder = millis_delta-(seconds_delta*1000);
fprintf(out,"%d.%3.3d: I:%ld ", seconds_delta,millis_remainder,interval->total_count);
fprintf(out,"( %ld %ld %ld ) ", hdr_value_at_percentile(interval,50.0),
hdr_value_at_percentile(interval,90.0),
hdr_max(interval));
fprintf(out,"T:%ld ",cumulative->total_count);
fprintf(out,"( %ld %ld %ld %ld %ld %ld)\n",
hdr_value_at_percentile(cumulative,50.0),
hdr_value_at_percentile(cumulative,90.0),
hdr_value_at_percentile(cumulative,99.0),
hdr_value_at_percentile(cumulative,99.9),
hdr_value_at_percentile(cumulative,99.99),
hdr_max(cumulative));
return;
}
static char* test_scaling_equivalence()
{
load_histograms();
mu_assert(
"Averages should be equivalent",
compare_values(
hdr_mean(cor_histogram) * 512,
hdr_mean(scaled_cor_histogram),
0.000001));
mu_assert(
"Total count should be equivalent",
compare_int64(
cor_histogram->total_count,
scaled_cor_histogram->total_count));
int64_t expected_99th = hdr_value_at_percentile(cor_histogram, 99.0) * 512;
int64_t scaled_99th = hdr_value_at_percentile(scaled_cor_histogram, 99.0);
mu_assert(
"99%'iles should be equivalent",
compare_int64(
hdr_lowest_equivalent_value(cor_histogram, expected_99th),
hdr_lowest_equivalent_value(scaled_cor_histogram, scaled_99th)));
mu_assert(
"Max should be equivalent",
compare_int64(hdr_max(cor_histogram) * 512, hdr_max(scaled_cor_histogram)));
return 0;
}
static char* test_get_max_value()
{
load_histograms();
int64_t actual_raw_max = hdr_max(raw_histogram);
mu_assert("hdr_max(raw_histogram) != 100000000L",
hdr_values_are_equivalent(raw_histogram, actual_raw_max, 100000000L));
int64_t actual_cor_max = hdr_max(cor_histogram);
mu_assert("hdr_max(cor_histogram) != 100000000L",
hdr_values_are_equivalent(cor_histogram, actual_cor_max, 100000000L));
return 0;
}
ERL_NIF_TERM _hh_max(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
{
hh_ctx_t* ctx = NULL;
ErlNifResourceType* ctx_type = get_hh_ctx_type(env);
if (argc != 1 ||
ctx_type == NULL ||
!enif_get_resource(env, argv[0], ctx_type, (void **)&ctx))
{
return enif_make_badarg(env);
}
if (ctx != NULL)
{
if (ctx->data->total_count == 0)
{
return enif_make_long(env, 0);
}
else
{
return enif_make_long(env, hdr_max(ctx->data));
}
}
return make_error(env, "bad_hdr_histogram_nif_impl");
}
static enum conclusion conclusion_calc(disk_t *disk)
{
if (disk->num_errors > 0)
return CONCLUSION_FAILED_IO_ERRORS;
if (hdr_max(disk->histogram) > 10000000)
return CONCLUSION_FAILED_MAX_LATENCY;
if (hdr_value_at_percentile(disk->histogram, 99.99) > 8000000)
return CONCLUSION_FAILED_LATENCY_PERCENTILE;
VERBOSE("Disk has passed the test");
return CONCLUSION_PASSED;
}
int hdr_log_write(
struct hdr_log_writer* writer,
FILE* file,
const struct timespec* start_timestamp,
const struct timespec* end_timestamp,
struct hdr_histogram* histogram)
{
uint8_t* compressed_histogram = NULL;
size_t compressed_len = 0;
char* encoded_histogram = NULL;
int rc = 0;
int result = 0;
size_t encoded_len;
(void)writer;
rc = hdr_encode_compressed(histogram, &compressed_histogram, &compressed_len);
if (rc != 0)
{
FAIL_AND_CLEANUP(cleanup, result, rc);
}
encoded_len = hdr_base64_encoded_len(compressed_len);
encoded_histogram = calloc(encoded_len + 1, sizeof(char));
rc = hdr_base64_encode(
compressed_histogram, compressed_len, encoded_histogram, encoded_len);
if (rc != 0)
{
FAIL_AND_CLEANUP(cleanup, result, rc);
}
if (fprintf(
file, "%d.%d,%d.%d,%"PRIu64".0,%s\n",
(int) start_timestamp->tv_sec, (int) (start_timestamp->tv_nsec / 1000000),
(int) end_timestamp->tv_sec, (int) (end_timestamp->tv_nsec / 1000000),
hdr_max(histogram),
encoded_histogram) < 0)
{
result = EIO;
}
cleanup:
free(compressed_histogram);
free(encoded_histogram);
return result;
}
void get_snapshot(Snapshot* snapshot) const {
ScopedMutex l(&mutex_);
hdr_histogram* h = histogram_;
for (size_t i = 0; i < thread_state_->max_threads(); ++i) {
histograms_[i].add(h);
}
snapshot->min = hdr_min(h);
snapshot->max = hdr_max(h);
snapshot->mean = static_cast<int64_t>(hdr_mean(h));
snapshot->stddev = static_cast<int64_t>(hdr_stddev(h));
snapshot->median = hdr_value_at_percentile(h, 50.0);
snapshot->percentile_75th = hdr_value_at_percentile(h, 75.0);
snapshot->percentile_95th = hdr_value_at_percentile(h, 95.0);
snapshot->percentile_98th = hdr_value_at_percentile(h, 98.0);
snapshot->percentile_99th = hdr_value_at_percentile(h, 99.0);
snapshot->percentile_999th = hdr_value_at_percentile(h, 99.9);
}
static char* decode_v0_log()
{
const char* v1_log = "jHiccup-2.0.1.logV0.hlog";
FILE* f = fopen(v1_log, "r");
mu_assert("Can not open v1 log file", f != NULL);
struct hdr_histogram* accum;
hdr_init(1, INT64_C(3600000000000), 3, &accum);
struct hdr_histogram* h = NULL;
struct hdr_log_reader reader;
hdr_timespec timestamp;
hdr_timespec interval;
hdr_log_reader_init(&reader);
int rc = hdr_log_read_header(&reader, f);
mu_assert("Failed to read header", rc == 0);
int histogram_count = 0;
int64_t total_count = 0;
while ((rc = hdr_log_read(&reader, f, &h, ×tamp, &interval)) != EOF)
{
mu_assert("Failed to read histogram", rc == 0);
histogram_count++;
total_count += h->total_count;
int64_t dropped = hdr_add(accum, h);
mu_assert("Dropped events", compare_int64(dropped, 0));
free(h);
h = NULL;
}
mu_assert("Wrong number of histograms", compare_int(histogram_count, 81));
mu_assert("Wrong total count", compare_int64(total_count, 61256));
mu_assert("99.9 percentile wrong", compare_int64(1510998015, hdr_value_at_percentile(accum, 99.9)));
mu_assert("max value wrong", compare_int64(1569718271, hdr_max(accum)));
mu_assert("Seconds wrong", compare_int64(1438869961, reader.start_timestamp.tv_sec));
mu_assert("Nanoseconds wrong", compare_int64(225000000, reader.start_timestamp.tv_nsec));
return 0;
}
static char* decode_v3_log()
{
const char* v3_log = "jHiccup-2.0.7S.logV3.hlog";
FILE* f = fopen(v3_log, "r");
mu_assert("Can not open v3 log file", f != NULL);
struct hdr_histogram* accum;
hdr_init(1, INT64_C(3600000000000), 3, &accum);
struct hdr_histogram* h = NULL;
struct hdr_log_reader reader;
hdr_timespec timestamp;
hdr_timespec interval;
hdr_log_reader_init(&reader);
int rc = hdr_log_read_header(&reader, f);
mu_assert("Failed to read header", validate_return_code(rc));
int histogram_count = 0;
int64_t total_count = 0;
while ((rc = hdr_log_read(&reader, f, &h, ×tamp, &interval)) != EOF)
{
mu_assert("Failed to read histogram", validate_return_code(rc));
histogram_count++;
total_count += h->total_count;
int64_t dropped = hdr_add(accum, h);
mu_assert("Dropped events", compare_int64(dropped, 0));
free(h);
h = NULL;
}
mu_assert("Wrong number of histograms", compare_int(histogram_count, 62));
mu_assert("Wrong total count", compare_int64(total_count, 48761));
mu_assert("99.9 percentile wrong", compare_int64(1745879039, hdr_value_at_percentile(accum, 99.9)));
mu_assert("max value wrong", compare_int64(1796210687, hdr_max(accum)));
mu_assert("Seconds wrong", compare_int64(1441812279, reader.start_timestamp.tv_sec));
mu_assert("Nanoseconds wrong", compare_int64(474000000, reader.start_timestamp.tv_nsec));
return 0;
}
void
report_to_statsd(Statsd *statsd, statsd_feedback *sf) {
static statsd_feedback empty_feedback;
if(!statsd) return;
if(!sf) sf = &empty_feedback;
statsd_resetBatch(statsd);
#define SBATCH(t, str, value) \
do { \
int ret = statsd_addToBatch(statsd, t, str, value, 1); \
if(ret == STATSD_BATCH_FULL) { \
statsd_sendBatch(statsd); \
ret = statsd_addToBatch(statsd, t, str, value, 1); \
} \
assert(ret == STATSD_SUCCESS); \
} while(0)
SBATCH(STATSD_COUNT, "connections.opened", sf->opened);
SBATCH(STATSD_GAUGE, "connections.total", sf->conns_in + sf->conns_out);
SBATCH(STATSD_GAUGE, "connections.total.in", sf->conns_in);
SBATCH(STATSD_GAUGE, "connections.total.out", sf->conns_out);
SBATCH(STATSD_GAUGE, "traffic.bitrate", sf->bps_in + sf->bps_out);
SBATCH(STATSD_GAUGE, "traffic.bitrate.in", sf->bps_in);
SBATCH(STATSD_GAUGE, "traffic.bitrate.out", sf->bps_out);
SBATCH(STATSD_COUNT, "traffic.data",
sf->traffic_delta.bytes_rcvd + sf->traffic_delta.bytes_sent);
SBATCH(STATSD_COUNT, "traffic.data.rcvd", sf->traffic_delta.bytes_rcvd);
SBATCH(STATSD_COUNT, "traffic.data.sent", sf->traffic_delta.bytes_sent);
SBATCH(STATSD_COUNT, "traffic.data.reads", sf->traffic_delta.num_reads);
SBATCH(STATSD_COUNT, "traffic.data.writes", sf->traffic_delta.num_writes);
if((sf->latency && sf->latency->marker_histogram)
|| sf == &empty_feedback) {
struct {
unsigned p50;
unsigned p95;
unsigned p99;
unsigned p99_5;
unsigned mean;
unsigned max;
} lat;
if(sf->latency && sf->latency->marker_histogram) {
struct hdr_histogram *hist = sf->latency->marker_histogram;
lat.p50 = hdr_value_at_percentile(hist, 50.0) / 10.0;
lat.p95 = hdr_value_at_percentile(hist, 95.0) / 10.0;
lat.p99 = hdr_value_at_percentile(hist, 99.0) / 10.0;
lat.p99_5 = hdr_value_at_percentile(hist, 99.5) / 10.0;
lat.mean = hdr_mean(hist) / 10.0;
lat.max = hdr_max(hist) / 10.0;
assert(lat.p95 < 1000000);
assert(lat.mean < 1000000);
assert(lat.max < 1000000);
} else {
memset(&lat, 0, sizeof(lat));
}
SBATCH(STATSD_GAUGE, "latency.mean", lat.mean);
SBATCH(STATSD_GAUGE, "latency.50", lat.p50);
SBATCH(STATSD_GAUGE, "latency.95", lat.p95);
SBATCH(STATSD_GAUGE, "latency.99", lat.p99);
SBATCH(STATSD_GAUGE, "latency.99.5", lat.p99_5);
SBATCH(STATSD_GAUGE, "latency.max", lat.max);
}
statsd_sendBatch(statsd);
}
static bool compare_histogram(struct hdr_histogram* a, struct hdr_histogram* b)
{
if (a->counts_len != b->counts_len)
{
printf(
"a.counts_len = %"PRIu32", b.counts_len = %"PRIu32"\n",
a->counts_len, b->counts_len);
return false;
}
int64_t a_max = hdr_max(a);
int64_t b_max = hdr_max(b);
if (a_max != b_max)
{
printf("a.max = %"PRIu64", b.max = %"PRIu64"\n", a_max, b_max);
// return false;
}
int64_t a_min = hdr_min(a);
int64_t b_min = hdr_min(b);
if (a_min != b_min)
{
printf("a.min = %"PRIu64", b.min = %"PRIu64"\n", a_min, b_min);
// return false;
}
size_t a_size = hdr_get_memory_size(a);
size_t b_size = hdr_get_memory_size(b);
if (a_size != b_size)
{
printf("a.size: %zu, b.size: %zu\n", a_size, b_size);
return false;
}
size_t counts_size = a->counts_len * sizeof(int64_t);
if (memcmp(a->counts, b->counts, counts_size) == 0)
{
return true;
}
printf("%s\n", "Counts incorrect");
struct hdr_iter iter_a;
struct hdr_iter iter_b;
hdr_iter_init(&iter_a, a);
hdr_iter_init(&iter_b, b);
while (hdr_iter_next(&iter_a) && hdr_iter_next(&iter_b))
{
if (iter_a.count != iter_b.count ||
iter_a.value != iter_b.value)
{
printf(
"A - value: %"PRIu64", count: %"PRIu64", B - value: %"PRIu64", count: %"PRIu64"\n",
iter_a.value, iter_a.count,
iter_b.value, iter_b.count);
}
}
return false;
}
