int check_storage_number(calculated_number n, int debug) {
char buffer[100];
uint32_t flags = SN_EXISTS;
storage_number s = pack_storage_number(n, flags);
calculated_number d = unpack_storage_number(s);
if(!does_storage_number_exist(s)) {
fprintf(stderr, "Exists flags missing for number " CALCULATED_NUMBER_FORMAT "!\n", n);
return 5;
}
calculated_number ddiff = d - n;
calculated_number dcdiff = ddiff * 100.0 / n;
if(dcdiff < 0) dcdiff = -dcdiff;
size_t len = print_calculated_number(buffer, d);
calculated_number p = strtold(buffer, NULL);
calculated_number pdiff = n - p;
calculated_number pcdiff = pdiff * 100.0 / n;
if(pcdiff < 0) pcdiff = -pcdiff;
if(debug) {
fprintf(stderr,
CALCULATED_NUMBER_FORMAT " original\n"
CALCULATED_NUMBER_FORMAT " packed and unpacked, (stored as 0x%08X, diff " CALCULATED_NUMBER_FORMAT ", " CALCULATED_NUMBER_FORMAT "%%)\n"
"%s printed after unpacked (%zu bytes)\n"
CALCULATED_NUMBER_FORMAT " re-parsed from printed (diff " CALCULATED_NUMBER_FORMAT ", " CALCULATED_NUMBER_FORMAT "%%)\n\n",
n,
d, s, ddiff, dcdiff,
buffer,
len, p, pdiff, pcdiff
);
if(len != strlen(buffer)) fprintf(stderr, "ERROR: printed number %s is reported to have length %zu but it has %zu\n", buffer, len, strlen(buffer));
if(dcdiff > ACCURACY_LOSS) fprintf(stderr, "WARNING: packing number " CALCULATED_NUMBER_FORMAT " has accuracy loss %0.7Lf %%\n", n, dcdiff);
if(pcdiff > ACCURACY_LOSS) fprintf(stderr, "WARNING: re-parsing the packed, unpacked and printed number " CALCULATED_NUMBER_FORMAT " has accuracy loss %0.7Lf %%\n", n, pcdiff);
}
if(len != strlen(buffer)) return 1;
if(dcdiff > ACCURACY_LOSS) return 3;
if(pcdiff > ACCURACY_LOSS) return 4;
return 0;
}
static inline void do_dimension(
RRDR *r
, long points_wanted
, RRDDIM *rd
, long dim_id_in_rrdr
, long after_slot
, long before_slot
, time_t after_wanted
, time_t before_wanted
){
(void) before_slot;
RRDSET *st = r->st;
time_t
now = after_wanted,
dt = st->update_every,
max_date = 0,
min_date = 0;
long
slot = after_slot,
group_size = r->group,
points_added = 0,
values_in_group = 0,
values_in_group_non_zero = 0,
rrdr_line = -1,
entries = st->entries;
RRDR_VALUE_FLAGS
group_value_flags = RRDR_VALUE_NOTHING;
for( ; points_added < points_wanted ; now += dt, slot++ ) {
if(unlikely(slot >= entries)) slot = 0;
// make sure we return data in the proper time range
if(unlikely(now > before_wanted)) {
#ifdef NETDATA_INTERNAL_CHECKS
r->log = "stopped, because attempted to access the db after 'wanted before'";
#endif
break;
}
if(unlikely(now < after_wanted)) {
#ifdef NETDATA_INTERNAL_CHECKS
r->log = "skipped, because attempted to access the db before 'wanted after'";
#endif
continue;
}
// read the value from the database
storage_number n = rd->values[slot];
calculated_number value = NAN;
if(likely(does_storage_number_exist(n))) {
value = unpack_storage_number(n);
if(likely(value != 0.0))
values_in_group_non_zero++;
if(unlikely(did_storage_number_reset(n)))
group_value_flags |= RRDR_VALUE_RESET;
}
// add this value for grouping
r->grouping_add(r, value);
values_in_group++;
if(unlikely(values_in_group == group_size)) {
rrdr_line = rrdr_line_init(r, now, rrdr_line);
if(unlikely(!min_date)) min_date = now;
max_date = now;
// find the place to store our values
RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_line * r->d + dim_id_in_rrdr];
// update the dimension options
if(likely(values_in_group_non_zero))
r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO;
// store the specific point options
*rrdr_value_options_ptr = group_value_flags;
// store the value
r->v[rrdr_line * r->d + dim_id_in_rrdr] = r->grouping_flush(r, rrdr_value_options_ptr);
points_added++;
values_in_group = 0;
group_value_flags = RRDR_VALUE_NOTHING;
values_in_group_non_zero = 0;
}
}
r->before = max_date;
r->after = min_date;
rrdr_done(r, rrdr_line);
#ifdef NETDATA_INTERNAL_CHECKS
if(unlikely(r->rows != points_added))
error("INTERNAL ERROR: %s.%s added %zu rows, but RRDR says I added %zu.", r->st->name, rd->name, (size_t)points_added, (size_t)r->rows);
#endif
}
inline calculated_number backend_calculate_value_from_stored_data(
RRDSET *st // the chart
, RRDDIM *rd // the dimension
, time_t after // the start timestamp
, time_t before // the end timestamp
, uint32_t options // BACKEND_SOURCE_* bitmap
, time_t *first_timestamp // the first point of the database used in this response
, time_t *last_timestamp // the timestamp that should be reported to backend
) {
RRDHOST *host = st->rrdhost;
// find the edges of the rrd database for this chart
time_t first_t = rrdset_first_entry_t(st);
time_t last_t = rrdset_last_entry_t(st);
time_t update_every = st->update_every;
// step back a little, to make sure we have complete data collection
// for all metrics
after -= update_every * 2;
before -= update_every * 2;
// align the time-frame
after = after - (after % update_every);
before = before - (before % update_every);
// for before, loose another iteration
// the latest point will be reported the next time
before -= update_every;
if(unlikely(after > before))
// this can happen when update_every > before - after
after = before;
if(unlikely(after < first_t))
after = first_t;
if(unlikely(before > last_t))
before = last_t;
if(unlikely(before < first_t || after > last_t)) {
// the chart has not been updated in the wanted timeframe
debug(D_BACKEND, "BACKEND: %s.%s.%s: aligned timeframe %lu to %lu is outside the chart's database range %lu to %lu",
host->hostname, st->id, rd->id,
(unsigned long)after, (unsigned long)before,
(unsigned long)first_t, (unsigned long)last_t
);
return NAN;
}
*first_timestamp = after;
*last_timestamp = before;
size_t counter = 0;
calculated_number sum = 0;
long start_at_slot = rrdset_time2slot(st, before),
stop_at_slot = rrdset_time2slot(st, after),
slot, stop_now = 0;
for(slot = start_at_slot; !stop_now ; slot--) {
if(unlikely(slot < 0)) slot = st->entries - 1;
if(unlikely(slot == stop_at_slot)) stop_now = 1;
storage_number n = rd->values[slot];
if(unlikely(!does_storage_number_exist(n))) {
// not collected
continue;
}
calculated_number value = unpack_storage_number(n);
sum += value;
counter++;
}
if(unlikely(!counter)) {
debug(D_BACKEND, "BACKEND: %s.%s.%s: no values stored in database for range %lu to %lu",
host->hostname, st->id, rd->id,
(unsigned long)after, (unsigned long)before
);
return NAN;
}
if(unlikely((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM))
return sum;
return sum / (calculated_number)counter;
}
void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value) {
rrdset_check_rdlock(r->st);
long rows = rrdr_rows(r);
long c, i;
RRDDIM *rd;
//info("JSONWRAPPER(): %s: BEGIN", r->st->id);
char kq[2] = "", // key quote
sq[2] = ""; // string quote
if( options & RRDR_OPTION_GOOGLE_JSON ) {
kq[0] = '\0';
sq[0] = '\'';
}
else {
kq[0] = '"';
sq[0] = '"';
}
buffer_sprintf(wb, "{\n"
" %sapi%s: 1,\n"
" %sid%s: %s%s%s,\n"
" %sname%s: %s%s%s,\n"
" %sview_update_every%s: %d,\n"
" %supdate_every%s: %d,\n"
" %sfirst_entry%s: %u,\n"
" %slast_entry%s: %u,\n"
" %sbefore%s: %u,\n"
" %safter%s: %u,\n"
" %sdimension_names%s: ["
, kq, kq
, kq, kq, sq, r->st->id, sq
, kq, kq, sq, r->st->name, sq
, kq, kq, r->update_every
, kq, kq, r->st->update_every
, kq, kq, (uint32_t)rrdset_first_entry_t(r->st)
, kq, kq, (uint32_t)rrdset_last_entry_t(r->st)
, kq, kq, (uint32_t)r->before
, kq, kq, (uint32_t)r->after
, kq, kq);
for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
if(i) buffer_strcat(wb, ", ");
buffer_strcat(wb, sq);
buffer_strcat(wb, rd->name);
buffer_strcat(wb, sq);
i++;
}
if(!i) {
#ifdef NETDATA_INTERNAL_CHECKS
error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options);
#endif
rows = 0;
buffer_strcat(wb, sq);
buffer_strcat(wb, "no data");
buffer_strcat(wb, sq);
}
buffer_sprintf(wb, "],\n"
" %sdimension_ids%s: ["
, kq, kq);
for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
if(i) buffer_strcat(wb, ", ");
buffer_strcat(wb, sq);
buffer_strcat(wb, rd->id);
buffer_strcat(wb, sq);
i++;
}
if(!i) {
rows = 0;
buffer_strcat(wb, sq);
buffer_strcat(wb, "no data");
buffer_strcat(wb, sq);
}
buffer_sprintf(wb, "],\n"
" %slatest_values%s: ["
, kq, kq);
for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
if(i) buffer_strcat(wb, ", ");
i++;
storage_number n = rd->values[rrdset_last_slot(r->st)];
if(!does_storage_number_exist(n))
buffer_strcat(wb, "null");
else
buffer_rrd_value(wb, unpack_storage_number(n));
}
if(!i) {
rows = 0;
buffer_strcat(wb, "null");
}
buffer_sprintf(wb, "],\n"
" %sview_latest_values%s: ["
, kq, kq);
i = 0;
if(rows) {
calculated_number total = 1;
if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
total = 0;
for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
calculated_number *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];
calculated_number n = cn[c];
if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
n = -n;
total += n;
}
// prevent a division by zero
if(total == 0) total = 1;
}
for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
if(i) buffer_strcat(wb, ", ");
i++;
calculated_number *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];
RRDR_VALUE_FLAGS *co = &r->o[ (rrdr_rows(r) - 1) * r->d ];
calculated_number n = cn[c];
if(co[c] & RRDR_VALUE_EMPTY) {
if(options & RRDR_OPTION_NULL2ZERO)
buffer_strcat(wb, "0");
else
buffer_strcat(wb, "null");
}
else {
if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
n = -n;
if(unlikely(options & RRDR_OPTION_PERCENTAGE))
n = n * 100 / total;
buffer_rrd_value(wb, n);
}
}
}
if(!i) {
rows = 0;
buffer_strcat(wb, "null");
}
buffer_sprintf(wb, "],\n"
" %sdimensions%s: %ld,\n"
" %spoints%s: %ld,\n"
" %sformat%s: %s"
, kq, kq, i
, kq, kq, rows
, kq, kq, sq
);
rrdr_buffer_print_format(wb, format);
buffer_sprintf(wb, "%s,\n"
" %sresult%s: "
, sq
, kq, kq
);
if(string_value) buffer_strcat(wb, sq);
//info("JSONWRAPPER(): %s: END", r->st->id);
}
