inline calculated_number backend_calculate_value_from_stored_data( RRDSET *st // the chart , RRDDIM *rd // the dimension , time_t after // the start timestamp , time_t before // the end timestamp , uint32_t options // BACKEND_SOURCE_* bitmap , time_t *first_timestamp // the first point of the database used in this response , time_t *last_timestamp // the timestamp that should be reported to backend ) { RRDHOST *host = st->rrdhost; // find the edges of the rrd database for this chart time_t first_t = rrdset_first_entry_t(st); time_t last_t = rrdset_last_entry_t(st); time_t update_every = st->update_every; // step back a little, to make sure we have complete data collection // for all metrics after -= update_every * 2; before -= update_every * 2; // align the time-frame after = after - (after % update_every); before = before - (before % update_every); // for before, loose another iteration // the latest point will be reported the next time before -= update_every; if(unlikely(after > before)) // this can happen when update_every > before - after after = before; if(unlikely(after < first_t)) after = first_t; if(unlikely(before > last_t)) before = last_t; if(unlikely(before < first_t || after > last_t)) { // the chart has not been updated in the wanted timeframe debug(D_BACKEND, "BACKEND: %s.%s.%s: aligned timeframe %lu to %lu is outside the chart's database range %lu to %lu", host->hostname, st->id, rd->id, (unsigned long)after, (unsigned long)before, (unsigned long)first_t, (unsigned long)last_t ); return NAN; } *first_timestamp = after; *last_timestamp = before; size_t counter = 0; calculated_number sum = 0; long start_at_slot = rrdset_time2slot(st, before), stop_at_slot = rrdset_time2slot(st, after), slot, stop_now = 0; for(slot = start_at_slot; !stop_now ; slot--) { if(unlikely(slot < 0)) slot = st->entries - 1; if(unlikely(slot == stop_at_slot)) stop_now = 1; storage_number n = rd->values[slot]; if(unlikely(!does_storage_number_exist(n))) { // not collected continue; } calculated_number value = unpack_storage_number(n); sum += value; counter++; } if(unlikely(!counter)) { debug(D_BACKEND, "BACKEND: %s.%s.%s: no values stored in database for range %lu to %lu", host->hostname, st->id, rd->id, (unsigned long)after, (unsigned long)before ); return NAN; } if(unlikely((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM)) return sum; return sum / (calculated_number)counter; }
RRDR *rrd2rrdr( RRDSET *st , long points_requested , long long after_requested , long long before_requested , RRDR_GROUPING group_method , long group_time_requested , RRDR_OPTIONS options , const char *dimensions ) { int aligned = !(options & RRDR_OPTION_NOT_ALIGNED); int absolute_period_requested = -1; time_t first_entry_t = rrdset_first_entry_t(st); time_t last_entry_t = rrdset_last_entry_t(st); if(before_requested == 0 && after_requested == 0) { // dump the all the data before_requested = last_entry_t; after_requested = first_entry_t; absolute_period_requested = 0; } // allow relative for before (smaller than API_RELATIVE_TIME_MAX) if(((before_requested < 0)?-before_requested:before_requested) <= API_RELATIVE_TIME_MAX) { if(abs(before_requested) % st->update_every) { // make sure it is multiple of st->update_every if(before_requested < 0) before_requested = before_requested - st->update_every - before_requested % st->update_every; else before_requested = before_requested + st->update_every - before_requested % st->update_every; } if(before_requested > 0) before_requested = first_entry_t + before_requested; else before_requested = last_entry_t + before_requested; absolute_period_requested = 0; } // allow relative for after (smaller than API_RELATIVE_TIME_MAX) if(((after_requested < 0)?-after_requested:after_requested) <= API_RELATIVE_TIME_MAX) { if(after_requested == 0) after_requested = -st->update_every; if(abs(after_requested) % st->update_every) { // make sure it is multiple of st->update_every if(after_requested < 0) after_requested = after_requested - st->update_every - after_requested % st->update_every; else after_requested = after_requested + st->update_every - after_requested % st->update_every; } after_requested = before_requested + after_requested; absolute_period_requested = 0; } if(absolute_period_requested == -1) absolute_period_requested = 1; // make sure they are within our timeframe if(before_requested > last_entry_t) before_requested = last_entry_t; if(before_requested < first_entry_t) before_requested = first_entry_t; if(after_requested > last_entry_t) after_requested = last_entry_t; if(after_requested < first_entry_t) after_requested = first_entry_t; // check if they are reversed if(after_requested > before_requested) { time_t tmp = before_requested; before_requested = after_requested; after_requested = tmp; } // the duration of the chart time_t duration = before_requested - after_requested; long available_points = duration / st->update_every; if(duration <= 0 || available_points <= 0) return rrdr_create(st, 1); // check the number of wanted points in the result if(unlikely(points_requested < 0)) points_requested = -points_requested; if(unlikely(points_requested > available_points)) points_requested = available_points; if(unlikely(points_requested == 0)) points_requested = available_points; // calculate the desired grouping of source data points long group = available_points / points_requested; if(unlikely(group <= 0)) group = 1; if(unlikely(available_points % points_requested > points_requested / 2)) group++; // rounding to the closest integer // group_time enforces a certain grouping multiple calculated_number group_sum_divisor = 1.0; long group_points = 1; if(unlikely(group_time_requested > st->update_every)) { if (unlikely(group_time_requested > duration)) { // group_time is above the available duration #ifdef NETDATA_INTERNAL_CHECKS info("INTERNAL CHECK: %s: requested gtime %ld secs, is greater than the desired duration %ld secs", st->id, group_time_requested, duration); #endif group = available_points; // use all the points } else { // the points we should group to satisfy gtime group_points = group_time_requested / st->update_every; if(unlikely(group_time_requested % group_points)) { #ifdef NETDATA_INTERNAL_CHECKS info("INTERNAL CHECK: %s: requested gtime %ld secs, is not a multiple of the chart's data collection frequency %d secs", st->id, group_time_requested, st->update_every); #endif group_points++; } // adapt group according to group_points if(unlikely(group < group_points)) group = group_points; // do not allow grouping below the desired one if(unlikely(group % group_points)) group += group_points - (group % group_points); // make sure group is multiple of group_points //group_sum_divisor = group / group_points; group_sum_divisor = (calculated_number)(group * st->update_every) / (calculated_number)group_time_requested; } } // now that we have group, // align the requested timeframe to fit it. if(aligned) { // alignement has been requested, so align the values before_requested -= (before_requested % group); after_requested -= (after_requested % group); } // we align the request on requested_before time_t before_wanted = before_requested; if(likely(before_wanted > last_entry_t)) { #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL ERROR: rrd2rrdr() on %s, before_wanted is after db max", st->name); #endif before_wanted = last_entry_t - (last_entry_t % ( ((aligned)?group:1) * st->update_every )); } size_t before_slot = rrdset_time2slot(st, before_wanted); // we need to estimate the number of points, for having // an integer number of values per point long points_wanted = (before_wanted - after_requested) / st->update_every / group; time_t after_wanted = before_wanted - (points_wanted * group * st->update_every) + st->update_every; if(unlikely(after_wanted < first_entry_t)) { // hm... we go to the past, calculate again points_wanted using all the db from before_wanted to the beginning points_wanted = (before_wanted - first_entry_t) / group; // recalculate after wanted with the new number of points after_wanted = before_wanted - (points_wanted * group * st->update_every) + st->update_every; if(unlikely(after_wanted < first_entry_t)) { #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL ERROR: rrd2rrdr() on %s, after_wanted is before db min", st->name); #endif after_wanted = first_entry_t - (first_entry_t % ( ((aligned)?group:1) * st->update_every )) + ( ((aligned)?group:1) * st->update_every ); } } size_t after_slot = rrdset_time2slot(st, after_wanted); // check if they are reversed if(unlikely(after_wanted > before_wanted)) { #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL ERROR: rrd2rrdr() on %s, reversed wanted after/before", st->name); #endif time_t tmp = before_wanted; before_wanted = after_wanted; after_wanted = tmp; } // recalculate points_wanted using the final time-frame points_wanted = (before_wanted - after_wanted) / st->update_every / group + 1; if(unlikely(points_wanted < 0)) { #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL ERROR: rrd2rrdr() on %s, points_wanted is %ld", st->name, points_wanted); #endif points_wanted = 0; } #ifdef NETDATA_INTERNAL_CHECKS duration = before_wanted - after_wanted; if(after_wanted < first_entry_t) error("INTERNAL CHECK: after_wanted %u is too small, minimum %u", (uint32_t)after_wanted, (uint32_t)first_entry_t); if(after_wanted > last_entry_t) error("INTERNAL CHECK: after_wanted %u is too big, maximum %u", (uint32_t)after_wanted, (uint32_t)last_entry_t); if(before_wanted < first_entry_t) error("INTERNAL CHECK: before_wanted %u is too small, minimum %u", (uint32_t)before_wanted, (uint32_t)first_entry_t); if(before_wanted > last_entry_t) error("INTERNAL CHECK: before_wanted %u is too big, maximum %u", (uint32_t)before_wanted, (uint32_t)last_entry_t); if(before_slot >= (size_t)st->entries) error("INTERNAL CHECK: before_slot is invalid %zu, expected 0 to %ld", before_slot, st->entries - 1); if(after_slot >= (size_t)st->entries) error("INTERNAL CHECK: after_slot is invalid %zu, expected 0 to %ld", after_slot, st->entries - 1); if(points_wanted > (before_wanted - after_wanted) / group / st->update_every + 1) error("INTERNAL CHECK: points_wanted %ld is more than points %ld", points_wanted, (before_wanted - after_wanted) / group / st->update_every + 1); if(group < group_points) error("INTERNAL CHECK: group %ld is less than the desired group points %ld", group, group_points); if(group > group_points && group % group_points) error("INTERNAL CHECK: group %ld is not a multiple of the desired group points %ld", group, group_points); #endif // ------------------------------------------------------------------------- // initialize our result set // this also locks the chart for us RRDR *r = rrdr_create(st, points_wanted); if(unlikely(!r)) { #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted); #endif return NULL; } if(unlikely(!r->d || !points_wanted)) { #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL CHECK: Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%zu, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted); #endif return r; } if(unlikely(absolute_period_requested == 1)) r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE; else r->result_options |= RRDR_RESULT_OPTION_RELATIVE; // find how many dimensions we have long dimensions_count = r->d; // ------------------------------------------------------------------------- // initialize RRDR r->group = group; r->update_every = (int)group * st->update_every; r->before = before_wanted; r->after = after_wanted; r->group_points = group_points; r->group_sum_divisor = group_sum_divisor; // ------------------------------------------------------------------------- // assign the processor functions { int i, found = 0; for(i = 0; !found && api_v1_data_groups[i].name ;i++) { if(api_v1_data_groups[i].value == group_method) { r->grouping_init = api_v1_data_groups[i].init; r->grouping_reset = api_v1_data_groups[i].reset; r->grouping_free = api_v1_data_groups[i].free; r->grouping_add = api_v1_data_groups[i].add; r->grouping_flush = api_v1_data_groups[i].flush; found = 1; } } if(!found) { errno = 0; #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL ERROR: grouping method %u not found for chart '%s'. Using 'average'", (unsigned int)group_method, r->st->name); #endif r->grouping_init = grouping_init_average; r->grouping_reset = grouping_reset_average; r->grouping_free = grouping_free_average; r->grouping_add = grouping_add_average; r->grouping_flush = grouping_flush_average; } } // allocate any memory required by the grouping method r->grouping_data = r->grouping_init(r); // ------------------------------------------------------------------------- // disable the not-wanted dimensions rrdset_check_rdlock(st); if(dimensions) rrdr_disable_not_selected_dimensions(r, options, dimensions); // ------------------------------------------------------------------------- // do the work for each dimension time_t max_after = 0, min_before = 0; long max_rows = 0; RRDDIM *rd; long c, dimensions_used = 0, dimensions_nonzero = 0; for(rd = st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { // if we need a percentage, we need to calculate all dimensions if(unlikely(!(options & RRDR_OPTION_PERCENTAGE) && (r->od[c] & RRDR_DIMENSION_HIDDEN))) continue; // reset the grouping for the new dimension r->grouping_reset(r); do_dimension( r , points_wanted , rd , c , after_slot , before_slot , after_wanted , before_wanted ); if(r->od[c] & RRDR_DIMENSION_NONZERO) dimensions_nonzero++; // verify all dimensions are aligned if(unlikely(!dimensions_used)) { min_before = r->before; max_after = r->after; max_rows = r->rows; } else { if(r->after != max_after) { #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL ERROR: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", st->name, (size_t)max_after, rd->name, (size_t)r->after); #endif r->after = (r->after > max_after) ? r->after : max_after; } if(r->before != min_before) { #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL ERROR: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", st->name, (size_t)min_before, rd->name, (size_t)r->before); #endif r->before = (r->before < min_before) ? r->before : min_before; } if(r->rows != max_rows) { #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL ERROR: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", st->name, (size_t)max_rows, rd->name, (size_t)r->rows); #endif r->rows = (r->rows > max_rows) ? r->rows : max_rows; } } dimensions_used++; } #ifdef NETDATA_INTERNAL_CHECKS if(r->log) rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, group_time_requested, group_points, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, r->log); if(r->rows != points_wanted) rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, group_time_requested, group_points, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "got 'points' is not wanted 'points'"); if(aligned && (r->before % group) != 0) rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, group_time_requested, group_points, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'before' is not aligned but alignment is required"); // 'after' should not be aligned, since we start inside the first group //if(aligned && (r->after % group) != 0) // rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, group_time_requested, group_points, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required"); if(r->before != before_requested) rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, group_time_requested, group_points, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "chart is not aligned to requested 'before'"); if(r->before != before_wanted) rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, group_time_requested, group_points, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "got 'before' is not wanted 'before'"); // reported 'after' varies, depending on group if((r->after - (group - 1) * r->st->update_every) != after_wanted) rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, group_time_requested, group_points, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "got 'after' is not wanted 'after'"); #endif // free all resources used by the grouping method r->grouping_free(r); // when all the dimensions are zero, we should return all of them if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero)) { // all the dimensions are zero // mark them as NONZERO to send them all for(rd = st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; r->od[c] |= RRDR_DIMENSION_NONZERO; } } return r; }