// Time: seconds.millis // Interval: number // IntervalPercentiles: ( 50% 90% Max ) // TotalPercentiles: ( 50% 90% 99% 99.9% 99.99% max ) void print_histogram_line (FILE *out, struct hdr_histogram *interval, struct hdr_histogram *cumulative) { struct timeval now; unsigned int millis_delta, seconds_delta, millis_remainder; gettimeofday(&now,NULL); millis_delta = ((1000000 * (now.tv_sec - start.tv_sec)) + now.tv_usec - start.tv_usec)/1000; seconds_delta = millis_delta/1000; millis_remainder = millis_delta-(seconds_delta*1000); fprintf(out,"%d.%3.3d: I:%ld ", seconds_delta,millis_remainder,interval->total_count); fprintf(out,"( %ld %ld %ld ) ", hdr_value_at_percentile(interval,50.0), hdr_value_at_percentile(interval,90.0), hdr_max(interval)); fprintf(out,"T:%ld ",cumulative->total_count); fprintf(out,"( %ld %ld %ld %ld %ld %ld)\n", hdr_value_at_percentile(cumulative,50.0), hdr_value_at_percentile(cumulative,90.0), hdr_value_at_percentile(cumulative,99.0), hdr_value_at_percentile(cumulative,99.9), hdr_value_at_percentile(cumulative,99.99), hdr_max(cumulative)); return; }
static char* test_scaling_equivalence() { load_histograms(); mu_assert( "Averages should be equivalent", compare_values( hdr_mean(cor_histogram) * 512, hdr_mean(scaled_cor_histogram), 0.000001)); mu_assert( "Total count should be equivalent", compare_int64( cor_histogram->total_count, scaled_cor_histogram->total_count)); int64_t expected_99th = hdr_value_at_percentile(cor_histogram, 99.0) * 512; int64_t scaled_99th = hdr_value_at_percentile(scaled_cor_histogram, 99.0); mu_assert( "99%'iles should be equivalent", compare_int64( hdr_lowest_equivalent_value(cor_histogram, expected_99th), hdr_lowest_equivalent_value(scaled_cor_histogram, scaled_99th))); mu_assert( "Max should be equivalent", compare_int64(hdr_max(cor_histogram) * 512, hdr_max(scaled_cor_histogram))); return 0; }
static char* test_get_max_value() { load_histograms(); int64_t actual_raw_max = hdr_max(raw_histogram); mu_assert("hdr_max(raw_histogram) != 100000000L", hdr_values_are_equivalent(raw_histogram, actual_raw_max, 100000000L)); int64_t actual_cor_max = hdr_max(cor_histogram); mu_assert("hdr_max(cor_histogram) != 100000000L", hdr_values_are_equivalent(cor_histogram, actual_cor_max, 100000000L)); return 0; }
ERL_NIF_TERM _hh_max(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { hh_ctx_t* ctx = NULL; ErlNifResourceType* ctx_type = get_hh_ctx_type(env); if (argc != 1 || ctx_type == NULL || !enif_get_resource(env, argv[0], ctx_type, (void **)&ctx)) { return enif_make_badarg(env); } if (ctx != NULL) { if (ctx->data->total_count == 0) { return enif_make_long(env, 0); } else { return enif_make_long(env, hdr_max(ctx->data)); } } return make_error(env, "bad_hdr_histogram_nif_impl"); }
static enum conclusion conclusion_calc(disk_t *disk) { if (disk->num_errors > 0) return CONCLUSION_FAILED_IO_ERRORS; if (hdr_max(disk->histogram) > 10000000) return CONCLUSION_FAILED_MAX_LATENCY; if (hdr_value_at_percentile(disk->histogram, 99.99) > 8000000) return CONCLUSION_FAILED_LATENCY_PERCENTILE; VERBOSE("Disk has passed the test"); return CONCLUSION_PASSED; }
int hdr_log_write( struct hdr_log_writer* writer, FILE* file, const struct timespec* start_timestamp, const struct timespec* end_timestamp, struct hdr_histogram* histogram) { uint8_t* compressed_histogram = NULL; size_t compressed_len = 0; char* encoded_histogram = NULL; int rc = 0; int result = 0; size_t encoded_len; (void)writer; rc = hdr_encode_compressed(histogram, &compressed_histogram, &compressed_len); if (rc != 0) { FAIL_AND_CLEANUP(cleanup, result, rc); } encoded_len = hdr_base64_encoded_len(compressed_len); encoded_histogram = calloc(encoded_len + 1, sizeof(char)); rc = hdr_base64_encode( compressed_histogram, compressed_len, encoded_histogram, encoded_len); if (rc != 0) { FAIL_AND_CLEANUP(cleanup, result, rc); } if (fprintf( file, "%d.%d,%d.%d,%"PRIu64".0,%s\n", (int) start_timestamp->tv_sec, (int) (start_timestamp->tv_nsec / 1000000), (int) end_timestamp->tv_sec, (int) (end_timestamp->tv_nsec / 1000000), hdr_max(histogram), encoded_histogram) < 0) { result = EIO; } cleanup: free(compressed_histogram); free(encoded_histogram); return result; }
void get_snapshot(Snapshot* snapshot) const { ScopedMutex l(&mutex_); hdr_histogram* h = histogram_; for (size_t i = 0; i < thread_state_->max_threads(); ++i) { histograms_[i].add(h); } snapshot->min = hdr_min(h); snapshot->max = hdr_max(h); snapshot->mean = static_cast<int64_t>(hdr_mean(h)); snapshot->stddev = static_cast<int64_t>(hdr_stddev(h)); snapshot->median = hdr_value_at_percentile(h, 50.0); snapshot->percentile_75th = hdr_value_at_percentile(h, 75.0); snapshot->percentile_95th = hdr_value_at_percentile(h, 95.0); snapshot->percentile_98th = hdr_value_at_percentile(h, 98.0); snapshot->percentile_99th = hdr_value_at_percentile(h, 99.0); snapshot->percentile_999th = hdr_value_at_percentile(h, 99.9); }
static char* decode_v0_log() { const char* v1_log = "jHiccup-2.0.1.logV0.hlog"; FILE* f = fopen(v1_log, "r"); mu_assert("Can not open v1 log file", f != NULL); struct hdr_histogram* accum; hdr_init(1, INT64_C(3600000000000), 3, &accum); struct hdr_histogram* h = NULL; struct hdr_log_reader reader; hdr_timespec timestamp; hdr_timespec interval; hdr_log_reader_init(&reader); int rc = hdr_log_read_header(&reader, f); mu_assert("Failed to read header", rc == 0); int histogram_count = 0; int64_t total_count = 0; while ((rc = hdr_log_read(&reader, f, &h, ×tamp, &interval)) != EOF) { mu_assert("Failed to read histogram", rc == 0); histogram_count++; total_count += h->total_count; int64_t dropped = hdr_add(accum, h); mu_assert("Dropped events", compare_int64(dropped, 0)); free(h); h = NULL; } mu_assert("Wrong number of histograms", compare_int(histogram_count, 81)); mu_assert("Wrong total count", compare_int64(total_count, 61256)); mu_assert("99.9 percentile wrong", compare_int64(1510998015, hdr_value_at_percentile(accum, 99.9))); mu_assert("max value wrong", compare_int64(1569718271, hdr_max(accum))); mu_assert("Seconds wrong", compare_int64(1438869961, reader.start_timestamp.tv_sec)); mu_assert("Nanoseconds wrong", compare_int64(225000000, reader.start_timestamp.tv_nsec)); return 0; }
static char* decode_v3_log() { const char* v3_log = "jHiccup-2.0.7S.logV3.hlog"; FILE* f = fopen(v3_log, "r"); mu_assert("Can not open v3 log file", f != NULL); struct hdr_histogram* accum; hdr_init(1, INT64_C(3600000000000), 3, &accum); struct hdr_histogram* h = NULL; struct hdr_log_reader reader; hdr_timespec timestamp; hdr_timespec interval; hdr_log_reader_init(&reader); int rc = hdr_log_read_header(&reader, f); mu_assert("Failed to read header", validate_return_code(rc)); int histogram_count = 0; int64_t total_count = 0; while ((rc = hdr_log_read(&reader, f, &h, ×tamp, &interval)) != EOF) { mu_assert("Failed to read histogram", validate_return_code(rc)); histogram_count++; total_count += h->total_count; int64_t dropped = hdr_add(accum, h); mu_assert("Dropped events", compare_int64(dropped, 0)); free(h); h = NULL; } mu_assert("Wrong number of histograms", compare_int(histogram_count, 62)); mu_assert("Wrong total count", compare_int64(total_count, 48761)); mu_assert("99.9 percentile wrong", compare_int64(1745879039, hdr_value_at_percentile(accum, 99.9))); mu_assert("max value wrong", compare_int64(1796210687, hdr_max(accum))); mu_assert("Seconds wrong", compare_int64(1441812279, reader.start_timestamp.tv_sec)); mu_assert("Nanoseconds wrong", compare_int64(474000000, reader.start_timestamp.tv_nsec)); return 0; }
void report_to_statsd(Statsd *statsd, statsd_feedback *sf) { static statsd_feedback empty_feedback; if(!statsd) return; if(!sf) sf = &empty_feedback; statsd_resetBatch(statsd); #define SBATCH(t, str, value) \ do { \ int ret = statsd_addToBatch(statsd, t, str, value, 1); \ if(ret == STATSD_BATCH_FULL) { \ statsd_sendBatch(statsd); \ ret = statsd_addToBatch(statsd, t, str, value, 1); \ } \ assert(ret == STATSD_SUCCESS); \ } while(0) SBATCH(STATSD_COUNT, "connections.opened", sf->opened); SBATCH(STATSD_GAUGE, "connections.total", sf->conns_in + sf->conns_out); SBATCH(STATSD_GAUGE, "connections.total.in", sf->conns_in); SBATCH(STATSD_GAUGE, "connections.total.out", sf->conns_out); SBATCH(STATSD_GAUGE, "traffic.bitrate", sf->bps_in + sf->bps_out); SBATCH(STATSD_GAUGE, "traffic.bitrate.in", sf->bps_in); SBATCH(STATSD_GAUGE, "traffic.bitrate.out", sf->bps_out); SBATCH(STATSD_COUNT, "traffic.data", sf->traffic_delta.bytes_rcvd + sf->traffic_delta.bytes_sent); SBATCH(STATSD_COUNT, "traffic.data.rcvd", sf->traffic_delta.bytes_rcvd); SBATCH(STATSD_COUNT, "traffic.data.sent", sf->traffic_delta.bytes_sent); SBATCH(STATSD_COUNT, "traffic.data.reads", sf->traffic_delta.num_reads); SBATCH(STATSD_COUNT, "traffic.data.writes", sf->traffic_delta.num_writes); if((sf->latency && sf->latency->marker_histogram) || sf == &empty_feedback) { struct { unsigned p50; unsigned p95; unsigned p99; unsigned p99_5; unsigned mean; unsigned max; } lat; if(sf->latency && sf->latency->marker_histogram) { struct hdr_histogram *hist = sf->latency->marker_histogram; lat.p50 = hdr_value_at_percentile(hist, 50.0) / 10.0; lat.p95 = hdr_value_at_percentile(hist, 95.0) / 10.0; lat.p99 = hdr_value_at_percentile(hist, 99.0) / 10.0; lat.p99_5 = hdr_value_at_percentile(hist, 99.5) / 10.0; lat.mean = hdr_mean(hist) / 10.0; lat.max = hdr_max(hist) / 10.0; assert(lat.p95 < 1000000); assert(lat.mean < 1000000); assert(lat.max < 1000000); } else { memset(&lat, 0, sizeof(lat)); } SBATCH(STATSD_GAUGE, "latency.mean", lat.mean); SBATCH(STATSD_GAUGE, "latency.50", lat.p50); SBATCH(STATSD_GAUGE, "latency.95", lat.p95); SBATCH(STATSD_GAUGE, "latency.99", lat.p99); SBATCH(STATSD_GAUGE, "latency.99.5", lat.p99_5); SBATCH(STATSD_GAUGE, "latency.max", lat.max); } statsd_sendBatch(statsd); }
static bool compare_histogram(struct hdr_histogram* a, struct hdr_histogram* b) { if (a->counts_len != b->counts_len) { printf( "a.counts_len = %"PRIu32", b.counts_len = %"PRIu32"\n", a->counts_len, b->counts_len); return false; } int64_t a_max = hdr_max(a); int64_t b_max = hdr_max(b); if (a_max != b_max) { printf("a.max = %"PRIu64", b.max = %"PRIu64"\n", a_max, b_max); // return false; } int64_t a_min = hdr_min(a); int64_t b_min = hdr_min(b); if (a_min != b_min) { printf("a.min = %"PRIu64", b.min = %"PRIu64"\n", a_min, b_min); // return false; } size_t a_size = hdr_get_memory_size(a); size_t b_size = hdr_get_memory_size(b); if (a_size != b_size) { printf("a.size: %zu, b.size: %zu\n", a_size, b_size); return false; } size_t counts_size = a->counts_len * sizeof(int64_t); if (memcmp(a->counts, b->counts, counts_size) == 0) { return true; } printf("%s\n", "Counts incorrect"); struct hdr_iter iter_a; struct hdr_iter iter_b; hdr_iter_init(&iter_a, a); hdr_iter_init(&iter_b, b); while (hdr_iter_next(&iter_a) && hdr_iter_next(&iter_b)) { if (iter_a.count != iter_b.count || iter_a.value != iter_b.value) { printf( "A - value: %"PRIu64", count: %"PRIu64", B - value: %"PRIu64", count: %"PRIu64"\n", iter_a.value, iter_a.count, iter_b.value, iter_b.count); } } return false; }