static int check_storage_number_exists() { uint32_t flags = SN_EXISTS; for(flags = 0; flags < 7 ; flags++) { if(get_storage_number_flags(flags << 24) != flags << 24) { fprintf(stderr, "Flag 0x%08x is not checked correctly. It became 0x%08x\n", flags << 24, get_storage_number_flags(flags << 24)); return 1; } } flags = SN_EXISTS; calculated_number n = 0.0; storage_number s = pack_storage_number(n, flags); calculated_number d = unpack_storage_number(s); if(get_storage_number_flags(s) != flags) { fprintf(stderr, "Wrong flags. Given %08x, Got %08x!\n", flags, get_storage_number_flags(s)); return 1; } if(n != d) { fprintf(stderr, "Wrong number returned. Expected " CALCULATED_NUMBER_FORMAT ", returned " CALCULATED_NUMBER_FORMAT "!\n", n, d); return 1; } return 0; }
int check_storage_number(calculated_number n, int debug) { char buffer[100]; uint32_t flags = SN_EXISTS; storage_number s = pack_storage_number(n, flags); calculated_number d = unpack_storage_number(s); if(!does_storage_number_exist(s)) { fprintf(stderr, "Exists flags missing for number " CALCULATED_NUMBER_FORMAT "!\n", n); return 5; } calculated_number ddiff = d - n; calculated_number dcdiff = ddiff * 100.0 / n; if(dcdiff < 0) dcdiff = -dcdiff; size_t len = print_calculated_number(buffer, d); calculated_number p = strtold(buffer, NULL); calculated_number pdiff = n - p; calculated_number pcdiff = pdiff * 100.0 / n; if(pcdiff < 0) pcdiff = -pcdiff; if(debug) { fprintf(stderr, CALCULATED_NUMBER_FORMAT " original\n" CALCULATED_NUMBER_FORMAT " packed and unpacked, (stored as 0x%08X, diff " CALCULATED_NUMBER_FORMAT ", " CALCULATED_NUMBER_FORMAT "%%)\n" "%s printed after unpacked (%zu bytes)\n" CALCULATED_NUMBER_FORMAT " re-parsed from printed (diff " CALCULATED_NUMBER_FORMAT ", " CALCULATED_NUMBER_FORMAT "%%)\n\n", n, d, s, ddiff, dcdiff, buffer, len, p, pdiff, pcdiff ); if(len != strlen(buffer)) fprintf(stderr, "ERROR: printed number %s is reported to have length %zu but it has %zu\n", buffer, len, strlen(buffer)); if(dcdiff > ACCURACY_LOSS) fprintf(stderr, "WARNING: packing number " CALCULATED_NUMBER_FORMAT " has accuracy loss %0.7Lf %%\n", n, dcdiff); if(pcdiff > ACCURACY_LOSS) fprintf(stderr, "WARNING: re-parsing the packed, unpacked and printed number " CALCULATED_NUMBER_FORMAT " has accuracy loss %0.7Lf %%\n", n, pcdiff); } if(len != strlen(buffer)) return 1; if(dcdiff > ACCURACY_LOSS) return 3; if(pcdiff > ACCURACY_LOSS) return 4; return 0; }
int run_test(struct test *test) { fprintf(stderr, "\nRunning test '%s':\n%s\n", test->name, test->description); rrd_memory_mode = RRD_MEMORY_MODE_RAM; rrd_update_every = test->update_every; char name[101]; snprintfz(name, 100, "unittest-%s", test->name); // create the chart RRDSET *st = rrdset_create("netdata", name, name, "netdata", NULL, "Unit Testing", "a value", 1, test->update_every, RRDSET_TYPE_LINE); RRDDIM *rd = rrddim_add(st, "dim1", NULL, test->multiplier, test->divisor, test->algorithm); RRDDIM *rd2 = NULL; if(test->feed2) rd2 = rrddim_add(st, "dim2", NULL, test->multiplier, test->divisor, test->algorithm); st->debug = 1; // feed it with the test data time_t time_now = 0, time_start = now_realtime_sec(); unsigned long c; collected_number last = 0; for(c = 0; c < test->feed_entries; c++) { if(debug_flags) fprintf(stderr, "\n\n"); if(c) { time_now += test->feed[c].microseconds; fprintf(stderr, " > %s: feeding position %lu, after %0.3f seconds (%0.3f seconds from start), delta " CALCULATED_NUMBER_FORMAT ", rate " CALCULATED_NUMBER_FORMAT "\n", test->name, c+1, (float)test->feed[c].microseconds / 1000000.0, (float)time_now / 1000000.0, ((calculated_number)test->feed[c].value - (calculated_number)last) * (calculated_number)test->multiplier / (calculated_number)test->divisor, (((calculated_number)test->feed[c].value - (calculated_number)last) * (calculated_number)test->multiplier / (calculated_number)test->divisor) / (calculated_number)test->feed[c].microseconds * (calculated_number)1000000); rrdset_next_usec_unfiltered(st, test->feed[c].microseconds); } else { fprintf(stderr, " > %s: feeding position %lu\n", test->name, c+1); } fprintf(stderr, " >> %s with value " COLLECTED_NUMBER_FORMAT "\n", rd->name, test->feed[c].value); rrddim_set(st, "dim1", test->feed[c].value); last = test->feed[c].value; if(rd2) { fprintf(stderr, " >> %s with value " COLLECTED_NUMBER_FORMAT "\n", rd2->name, test->feed2[c]); rrddim_set(st, "dim2", test->feed2[c]); } rrdset_done(st); // align the first entry to second boundary if(!c) { fprintf(stderr, " > %s: fixing first collection time to be %llu microseconds to second boundary\n", test->name, test->feed[c].microseconds); rd->last_collected_time.tv_usec = st->last_collected_time.tv_usec = st->last_updated.tv_usec = test->feed[c].microseconds; // time_start = st->last_collected_time.tv_sec; } } // check the result int errors = 0; if(st->counter != test->result_entries) { fprintf(stderr, " %s stored %lu entries, but we were expecting %lu, ### E R R O R ###\n", test->name, st->counter, test->result_entries); errors++; } unsigned long max = (st->counter < test->result_entries)?st->counter:test->result_entries; for(c = 0 ; c < max ; c++) { calculated_number v = unpack_storage_number(rd->values[c]); calculated_number n = test->results[c]; int same = (roundl(v * 10000000.0) == roundl(n * 10000000.0))?1:0; fprintf(stderr, " %s/%s: checking position %lu (at %lu secs), expecting value " CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", %s\n", test->name, rd->name, c+1, (rrdset_first_entry_t(st) + c * st->update_every) - time_start, n, v, (same)?"OK":"### E R R O R ###"); if(!same) errors++; if(rd2) { v = unpack_storage_number(rd2->values[c]); n = test->results2[c]; same = (roundl(v * 10000000.0) == roundl(n * 10000000.0))?1:0; fprintf(stderr, " %s/%s: checking position %lu (at %lu secs), expecting value " CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", %s\n", test->name, rd2->name, c+1, (rrdset_first_entry_t(st) + c * st->update_every) - time_start, n, v, (same)?"OK":"### E R R O R ###"); if(!same) errors++; } } return errors; }
void benchmark_storage_number(int loop, int multiplier) { int i, j; calculated_number n, d; storage_number s; unsigned long long user, system, total, mine, their; char buffer[100]; struct rusage now, last; fprintf(stderr, "\n\nBenchmarking %d numbers, please wait...\n\n", loop); // ------------------------------------------------------------------------ fprintf(stderr, "SYSTEM LONG DOUBLE SIZE: %zu bytes\n", sizeof(calculated_number)); fprintf(stderr, "NETDATA FLOATING POINT SIZE: %zu bytes\n", sizeof(storage_number)); mine = (calculated_number)sizeof(storage_number) * (calculated_number)loop; their = (calculated_number)sizeof(calculated_number) * (calculated_number)loop; if(mine > their) { fprintf(stderr, "\nNETDATA NEEDS %0.2Lf TIMES MORE MEMORY. Sorry!\n", (long double)(mine / their)); } else { fprintf(stderr, "\nNETDATA INTERNAL FLOATING POINT ARITHMETICS NEEDS %0.2Lf TIMES LESS MEMORY.\n", (long double)(their / mine)); } fprintf(stderr, "\nNETDATA FLOATING POINT\n"); fprintf(stderr, "MIN POSITIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", (calculated_number)STORAGE_NUMBER_POSITIVE_MIN); fprintf(stderr, "MAX POSITIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", (calculated_number)STORAGE_NUMBER_POSITIVE_MAX); fprintf(stderr, "MIN NEGATIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", (calculated_number)STORAGE_NUMBER_NEGATIVE_MIN); fprintf(stderr, "MAX NEGATIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", (calculated_number)STORAGE_NUMBER_NEGATIVE_MAX); fprintf(stderr, "Maximum accuracy loss: " CALCULATED_NUMBER_FORMAT "%%\n\n\n", (calculated_number)ACCURACY_LOSS); // ------------------------------------------------------------------------ fprintf(stderr, "INTERNAL LONG DOUBLE PRINTING: "); getrusage(RUSAGE_SELF, &last); // do the job for(j = 1; j < 11 ;j++) { n = STORAGE_NUMBER_POSITIVE_MIN * j; for(i = 0; i < loop ;i++) { n *= multiplier; if(n > STORAGE_NUMBER_POSITIVE_MAX) n = STORAGE_NUMBER_POSITIVE_MIN; print_calculated_number(buffer, n); } } getrusage(RUSAGE_SELF, &now); user = now.ru_utime.tv_sec * 1000000ULL + now.ru_utime.tv_usec - last.ru_utime.tv_sec * 1000000ULL + last.ru_utime.tv_usec; system = now.ru_stime.tv_sec * 1000000ULL + now.ru_stime.tv_usec - last.ru_stime.tv_sec * 1000000ULL + last.ru_stime.tv_usec; total = user + system; mine = total; fprintf(stderr, "user %0.5Lf, system %0.5Lf, total %0.5Lf\n", (long double)(user / 1000000.0), (long double)(system / 1000000.0), (long double)(total / 1000000.0)); // ------------------------------------------------------------------------ fprintf(stderr, "SYSTEM LONG DOUBLE PRINTING: "); getrusage(RUSAGE_SELF, &last); // do the job for(j = 1; j < 11 ;j++) { n = STORAGE_NUMBER_POSITIVE_MIN * j; for(i = 0; i < loop ;i++) { n *= multiplier; if(n > STORAGE_NUMBER_POSITIVE_MAX) n = STORAGE_NUMBER_POSITIVE_MIN; snprintfz(buffer, 100, CALCULATED_NUMBER_FORMAT, n); } } getrusage(RUSAGE_SELF, &now); user = now.ru_utime.tv_sec * 1000000ULL + now.ru_utime.tv_usec - last.ru_utime.tv_sec * 1000000ULL + last.ru_utime.tv_usec; system = now.ru_stime.tv_sec * 1000000ULL + now.ru_stime.tv_usec - last.ru_stime.tv_sec * 1000000ULL + last.ru_stime.tv_usec; total = user + system; their = total; fprintf(stderr, "user %0.5Lf, system %0.5Lf, total %0.5Lf\n", (long double)(user / 1000000.0), (long double)(system / 1000000.0), (long double)(total / 1000000.0)); if(mine > total) { fprintf(stderr, "NETDATA CODE IS SLOWER %0.2Lf %%\n", (long double)(mine * 100.0 / their - 100.0)); } else { fprintf(stderr, "NETDATA CODE IS F A S T E R %0.2Lf %%\n", (long double)(their * 100.0 / mine - 100.0)); } // ------------------------------------------------------------------------ fprintf(stderr, "\nINTERNAL LONG DOUBLE PRINTING WITH PACK / UNPACK: "); getrusage(RUSAGE_SELF, &last); // do the job for(j = 1; j < 11 ;j++) { n = STORAGE_NUMBER_POSITIVE_MIN * j; for(i = 0; i < loop ;i++) { n *= multiplier; if(n > STORAGE_NUMBER_POSITIVE_MAX) n = STORAGE_NUMBER_POSITIVE_MIN; s = pack_storage_number(n, 1); d = unpack_storage_number(s); print_calculated_number(buffer, d); } } getrusage(RUSAGE_SELF, &now); user = now.ru_utime.tv_sec * 1000000ULL + now.ru_utime.tv_usec - last.ru_utime.tv_sec * 1000000ULL + last.ru_utime.tv_usec; system = now.ru_stime.tv_sec * 1000000ULL + now.ru_stime.tv_usec - last.ru_stime.tv_sec * 1000000ULL + last.ru_stime.tv_usec; total = user + system; mine = total; fprintf(stderr, "user %0.5Lf, system %0.5Lf, total %0.5Lf\n", (long double)(user / 1000000.0), (long double)(system / 1000000.0), (long double)(total / 1000000.0)); if(mine > their) { fprintf(stderr, "WITH PACKING UNPACKING NETDATA CODE IS SLOWER %0.2Lf %%\n", (long double)(mine * 100.0 / their - 100.0)); } else { fprintf(stderr, "EVEN WITH PACKING AND UNPACKING, NETDATA CODE IS F A S T E R %0.2Lf %%\n", (long double)(their * 100.0 / mine - 100.0)); } // ------------------------------------------------------------------------ }
int unit_test(long delay, long shift) { static int repeat = 0; repeat++; char name[101]; snprintfz(name, 100, "unittest-%d-%ld-%ld", repeat, delay, shift); //debug_flags = 0xffffffff; rrd_memory_mode = RRD_MEMORY_MODE_RAM; rrd_update_every = 1; int do_abs = 1; int do_inc = 1; int do_abst = 0; int do_absi = 0; RRDSET *st = rrdset_create("netdata", name, name, "netdata", NULL, "Unit Testing", "a value", 1, 1, RRDSET_TYPE_LINE); st->debug = 1; RRDDIM *rdabs = NULL; RRDDIM *rdinc = NULL; RRDDIM *rdabst = NULL; RRDDIM *rdabsi = NULL; if(do_abs) rdabs = rrddim_add(st, "absolute", "absolute", 1, 1, RRDDIM_ABSOLUTE); if(do_inc) rdinc = rrddim_add(st, "incremental", "incremental", 1, 1, RRDDIM_INCREMENTAL); if(do_abst) rdabst = rrddim_add(st, "percentage-of-absolute-row", "percentage-of-absolute-row", 1, 1, RRDDIM_PCENT_OVER_ROW_TOTAL); if(do_absi) rdabsi = rrddim_add(st, "percentage-of-incremental-row", "percentage-of-incremental-row", 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); long increment = 1000; collected_number i = 0; unsigned long c, dimensions = 0; RRDDIM *rd; for(rd = st->dimensions ; rd ; rd = rd->next) dimensions++; for(c = 0; c < 20 ;c++) { i += increment; fprintf(stderr, "\n\nLOOP = %lu, DELAY = %ld, VALUE = " COLLECTED_NUMBER_FORMAT "\n", c, delay, i); if(c) { rrdset_next_usec_unfiltered(st, delay); } if(do_abs) rrddim_set(st, "absolute", i); if(do_inc) rrddim_set(st, "incremental", i); if(do_abst) rrddim_set(st, "percentage-of-absolute-row", i); if(do_absi) rrddim_set(st, "percentage-of-incremental-row", i); if(!c) { now_realtime_timeval(&st->last_collected_time); st->last_collected_time.tv_usec = shift; } // prevent it from deleting the dimensions for(rd = st->dimensions ; rd ; rd = rd->next) rd->last_collected_time.tv_sec = st->last_collected_time.tv_sec; rrdset_done(st); } unsigned long oincrement = increment; increment = increment * st->update_every * 1000000 / delay; fprintf(stderr, "\n\nORIGINAL INCREMENT: %lu, INCREMENT %ld, DELAY %ld, SHIFT %ld\n", oincrement * 10, increment * 10, delay, shift); int ret = 0; storage_number sn; calculated_number cn, v; for(c = 0 ; c < st->counter ; c++) { fprintf(stderr, "\nPOSITION: c = %lu, EXPECTED VALUE %lu\n", c, (oincrement + c * increment + increment * (1000000 - shift) / 1000000 )* 10); for(rd = st->dimensions ; rd ; rd = rd->next) { sn = rd->values[c]; cn = unpack_storage_number(sn); fprintf(stderr, "\t %s " CALCULATED_NUMBER_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ") -> ", rd->id, cn, sn); if(rd == rdabs) v = ( oincrement // + (increment * (1000000 - shift) / 1000000) + (c + 1) * increment ); else if(rd == rdinc) v = (c?(increment):(increment * (1000000 - shift) / 1000000)); else if(rd == rdabst) v = oincrement / dimensions / 10; else if(rd == rdabsi) v = oincrement / dimensions / 10; else v = 0; if(v == cn) fprintf(stderr, "passed.\n"); else { fprintf(stderr, "ERROR! (expected " CALCULATED_NUMBER_FORMAT ")\n", v); ret = 1; } } } if(ret) fprintf(stderr, "\n\nUNIT TEST(%ld, %ld) FAILED\n\n", delay, shift); return ret; }
unsigned long long rrdset_done(RRDSET *st) { if(unlikely(netdata_exit)) return 0; debug(D_RRD_CALLS, "rrdset_done() for chart %s", st->name); RRDDIM *rd, *last; int oldstate, store_this_entry = 1, first_entry = 0; unsigned long long last_ut, now_ut, next_ut, stored_entries = 0; if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0)) error("Cannot set pthread cancel state to DISABLE."); // a read lock is OK here pthread_rwlock_rdlock(&st->rwlock); // enable the chart, if it was disabled if(unlikely(rrd_delete_unupdated_dimensions) && !st->enabled) st->enabled = 1; // check if the chart has a long time to be updated if(unlikely(st->usec_since_last_update > st->entries * st->update_every * 1000000ULL)) { info("%s: took too long to be updated (%0.3Lf secs). Reseting it.", st->name, (long double)(st->usec_since_last_update / 1000000.0)); rrdset_reset(st); st->usec_since_last_update = st->update_every * 1000000ULL; first_entry = 1; } if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update); // set last_collected_time if(unlikely(!st->last_collected_time.tv_sec)) { // it is the first entry // set the last_collected_time to now gettimeofday(&st->last_collected_time, NULL); // the first entry should not be stored store_this_entry = 0; first_entry = 1; if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: has not set last_collected_time. Setting it now. Will not store the next entry.", st->name); } else { // it is not the first entry // calculate the proper last_collected_time, using usec_since_last_update unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec + st->usec_since_last_update; st->last_collected_time.tv_sec = (time_t) (ut / 1000000ULL); st->last_collected_time.tv_usec = (suseconds_t) (ut % 1000000ULL); } // if this set has not been updated in the past // we fake the last_update time to be = now - usec_since_last_update if(unlikely(!st->last_updated.tv_sec)) { // it has never been updated before // set a fake last_updated, in the past using usec_since_last_update unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update; st->last_updated.tv_sec = (time_t) (ut / 1000000ULL); st->last_updated.tv_usec = (suseconds_t) (ut % 1000000ULL); // the first entry should not be stored store_this_entry = 0; first_entry = 1; if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: initializing last_updated to now - %llu microseconds (%0.3Lf). Will not store the next entry.", st->name, st->usec_since_last_update, (long double)ut/1000000.0); } // check if we will re-write the entire data set if(unlikely(usecdiff(&st->last_collected_time, &st->last_updated) > st->update_every * st->entries * 1000000ULL)) { info("%s: too old data (last updated at %ld.%ld, last collected at %ld.%ld). Reseting it. Will not store the next entry.", st->name, st->last_updated.tv_sec, st->last_updated.tv_usec, st->last_collected_time.tv_sec, st->last_collected_time.tv_usec); rrdset_reset(st); st->usec_since_last_update = st->update_every * 1000000ULL; gettimeofday(&st->last_collected_time, NULL); unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update; st->last_updated.tv_sec = (time_t) (ut / 1000000ULL); st->last_updated.tv_usec = (suseconds_t) (ut % 1000000ULL); // the first entry should not be stored store_this_entry = 0; first_entry = 1; } // these are the 3 variables that will help us in interpolation // last_ut = the last time we added a value to the storage // now_ut = the time the current value is taken at // next_ut = the time of the next interpolation point last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec; now_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec; next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL; if(unlikely(!first_entry && now_ut < next_ut)) { if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: THIS IS IN THE SAME INTERPOLATION POINT", st->name); } if(unlikely(st->debug)) { debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0); debug(D_RRD_STATS, "%s: now ut = %0.3Lf (current update time)", st->name, (long double)now_ut/1000000.0); debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0); } if(unlikely(!st->counter_done)) { store_this_entry = 0; if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: Will not store the next entry.", st->name); } st->counter_done++; // calculate totals and count the dimensions int dimensions; st->collected_total = 0; for( rd = st->dimensions, dimensions = 0 ; likely(rd) ; rd = rd->next, dimensions++ ) if(likely(rd->updated)) st->collected_total += rd->collected_value; uint32_t storage_flags = SN_EXISTS; // process all dimensions to calculate their values // based on the collected figures only // at this stage we do not interpolate anything for( rd = st->dimensions ; likely(rd) ; rd = rd->next ) { if(unlikely(!rd->updated)) { rd->calculated_value = 0; continue; } if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: START " " last_collected_value = " COLLECTED_NUMBER_FORMAT " collected_value = " COLLECTED_NUMBER_FORMAT " last_calculated_value = " CALCULATED_NUMBER_FORMAT " calculated_value = " CALCULATED_NUMBER_FORMAT , st->id, rd->name , rd->last_collected_value , rd->collected_value , rd->last_calculated_value , rd->calculated_value ); switch(rd->algorithm) { case RRDDIM_ABSOLUTE: rd->calculated_value = (calculated_number)rd->collected_value * (calculated_number)rd->multiplier / (calculated_number)rd->divisor; if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: CALC ABS/ABS-NO-IN " CALCULATED_NUMBER_FORMAT " = " COLLECTED_NUMBER_FORMAT " * " CALCULATED_NUMBER_FORMAT " / " CALCULATED_NUMBER_FORMAT , st->id, rd->name , rd->calculated_value , rd->collected_value , (calculated_number)rd->multiplier , (calculated_number)rd->divisor ); break; case RRDDIM_PCENT_OVER_ROW_TOTAL: if(unlikely(!st->collected_total)) rd->calculated_value = 0; else // the percentage of the current value // over the total of all dimensions rd->calculated_value = (calculated_number)100 * (calculated_number)rd->collected_value / (calculated_number)st->collected_total; if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: CALC PCENT-ROW " CALCULATED_NUMBER_FORMAT " = 100" " * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT , st->id, rd->name , rd->calculated_value , rd->collected_value , st->collected_total ); break; case RRDDIM_INCREMENTAL: if(unlikely(rd->counter <= 1)) { rd->calculated_value = 0; continue; } // if the new is smaller than the old (an overflow, or reset), set the old equal to the new // to reset the calculation (it will give zero as the calculation for this second) if(unlikely(rd->last_collected_value > rd->collected_value)) { debug(D_RRD_STATS, "%s.%s: RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT , st->name, rd->name , rd->last_collected_value , rd->collected_value); if(!(rd->flags & RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)) storage_flags = SN_EXISTS_RESET; rd->last_collected_value = rd->collected_value; } rd->calculated_value = (calculated_number)(rd->collected_value - rd->last_collected_value) * (calculated_number)rd->multiplier / (calculated_number)rd->divisor; if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: CALC INC PRE " CALCULATED_NUMBER_FORMAT " = (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" " * " CALCULATED_NUMBER_FORMAT " / " CALCULATED_NUMBER_FORMAT , st->id, rd->name , rd->calculated_value , rd->collected_value, rd->last_collected_value , (calculated_number)rd->multiplier , (calculated_number)rd->divisor ); break; case RRDDIM_PCENT_OVER_DIFF_TOTAL: if(unlikely(rd->counter <= 1)) { rd->calculated_value = 0; continue; } // if the new is smaller than the old (an overflow, or reset), set the old equal to the new // to reset the calculation (it will give zero as the calculation for this second) if(unlikely(rd->last_collected_value > rd->collected_value)) { debug(D_RRD_STATS, "%s.%s: RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT , st->name, rd->name , rd->last_collected_value , rd->collected_value); if(!(rd->flags & RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)) storage_flags = SN_EXISTS_RESET; rd->last_collected_value = rd->collected_value; } // the percentage of the current increment // over the increment of all dimensions together if(unlikely(st->collected_total == st->last_collected_total)) rd->calculated_value = 0; else rd->calculated_value = (calculated_number)100 * (calculated_number)(rd->collected_value - rd->last_collected_value) / (calculated_number)(st->collected_total - st->last_collected_total); if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: CALC PCENT-DIFF " CALCULATED_NUMBER_FORMAT " = 100" " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" , st->id, rd->name , rd->calculated_value , rd->collected_value, rd->last_collected_value , st->collected_total, st->last_collected_total ); break; default: // make the default zero, to make sure // it gets noticed when we add new types rd->calculated_value = 0; if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: CALC " CALCULATED_NUMBER_FORMAT " = 0" , st->id, rd->name , rd->calculated_value ); break; } if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: PHASE2 " " last_collected_value = " COLLECTED_NUMBER_FORMAT " collected_value = " COLLECTED_NUMBER_FORMAT " last_calculated_value = " CALCULATED_NUMBER_FORMAT " calculated_value = " CALCULATED_NUMBER_FORMAT , st->id, rd->name , rd->last_collected_value , rd->collected_value , rd->last_calculated_value , rd->calculated_value ); } // at this point we have all the calculated values ready // it is now time to interpolate values on a second boundary unsigned long long first_ut = last_ut; long long iterations = (now_ut - last_ut) / (st->update_every * 1000000ULL); if((now_ut % (st->update_every * 1000000ULL)) == 0) iterations++; for( ; likely(next_ut <= now_ut) ; next_ut += st->update_every * 1000000ULL, iterations-- ) { #ifdef NETDATA_INTERNAL_CHECKS if(iterations < 0) { error("%s: iterations calculation wrapped! first_ut = %llu, last_ut = %llu, next_ut = %llu, now_ut = %llu", st->name, first_ut, last_ut, next_ut, now_ut); } #endif if(unlikely(st->debug)) { debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0); debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0); } st->last_updated.tv_sec = (time_t) (next_ut / 1000000ULL); st->last_updated.tv_usec = 0; for( rd = st->dimensions ; likely(rd) ; rd = rd->next ) { calculated_number new_value; switch(rd->algorithm) { case RRDDIM_INCREMENTAL: new_value = (calculated_number) ( rd->calculated_value * (calculated_number)(next_ut - last_ut) / (calculated_number)(now_ut - last_ut) ); if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: CALC2 INC " CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT " * %llu" " / %llu" , st->id, rd->name , new_value , rd->calculated_value , (next_ut - last_ut) , (now_ut - last_ut) ); rd->calculated_value -= new_value; new_value += rd->last_calculated_value; rd->last_calculated_value = 0; new_value /= (calculated_number)st->update_every; break; case RRDDIM_ABSOLUTE: case RRDDIM_PCENT_OVER_ROW_TOTAL: case RRDDIM_PCENT_OVER_DIFF_TOTAL: default: if(iterations == 1) { // this is the last iteration // do not interpolate // just show the calculated value new_value = rd->calculated_value; } else { // we have missed an update // interpolate in the middle values new_value = (calculated_number) ( ( (rd->calculated_value - rd->last_calculated_value) * (calculated_number)(next_ut - first_ut) / (calculated_number)(now_ut - first_ut) ) + rd->last_calculated_value ); if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: CALC2 DEF " CALCULATED_NUMBER_FORMAT " = (((" "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")" " * %llu" " / %llu) + " CALCULATED_NUMBER_FORMAT , st->id, rd->name , new_value , rd->calculated_value, rd->last_calculated_value , (next_ut - first_ut) , (now_ut - first_ut), rd->last_calculated_value ); // this is wrong // it fades the value towards the target // while we know the calculated value is different // if(likely(next_ut + st->update_every * 1000000ULL > now_ut)) rd->calculated_value = new_value; } break; } if(unlikely(!store_this_entry)) { store_this_entry = 1; continue; } if(likely(rd->updated && rd->counter > 1 && iterations < st->gap_when_lost_iterations_above)) { rd->values[st->current_entry] = pack_storage_number(new_value, storage_flags ); if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: STORE[%ld] " CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT , st->id, rd->name , st->current_entry , unpack_storage_number(rd->values[st->current_entry]), new_value ); } else { if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: STORE[%ld] = NON EXISTING " , st->id, rd->name , st->current_entry ); rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS); } stored_entries++; if(unlikely(st->debug)) { calculated_number t1 = new_value * (calculated_number)rd->multiplier / (calculated_number)rd->divisor; calculated_number t2 = unpack_storage_number(rd->values[st->current_entry]); calculated_number accuracy = accuracy_loss(t1, t2); debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " FLAGS=0x%08x (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)" , st->id, rd->name , st->current_entry , t2 , get_storage_number_flags(rd->values[st->current_entry]) , t1 , accuracy , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : "" ); rd->collected_volume += t1; rd->stored_volume += t2; accuracy = accuracy_loss(rd->collected_volume, rd->stored_volume); debug(D_RRD_STATS, "%s/%s: VOLUME[%ld] = " CALCULATED_NUMBER_FORMAT ", calculated = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s" , st->id, rd->name , st->current_entry , rd->stored_volume , rd->collected_volume , accuracy , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : "" ); } } // reset the storage flags for the next point, if any; storage_flags = SN_EXISTS; st->counter++; st->current_entry = ((st->current_entry + 1) >= st->entries) ? 0 : st->current_entry + 1; last_ut = next_ut; } // align next interpolation to last collection point if(likely(stored_entries || !store_this_entry)) { st->last_updated.tv_sec = st->last_collected_time.tv_sec; st->last_updated.tv_usec = st->last_collected_time.tv_usec; st->last_collected_total = st->collected_total; } for( rd = st->dimensions; likely(rd) ; rd = rd->next ) { if(unlikely(!rd->updated)) continue; if(likely(stored_entries || !store_this_entry)) { if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", st->id, rd->name, rd->last_collected_value, rd->collected_value); rd->last_collected_value = rd->collected_value; if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", st->id, rd->name, rd->last_calculated_value, rd->calculated_value); rd->last_calculated_value = rd->calculated_value; } rd->calculated_value = 0; rd->collected_value = 0; rd->updated = 0; if(unlikely(st->debug)) debug(D_RRD_STATS, "%s/%s: END " " last_collected_value = " COLLECTED_NUMBER_FORMAT " collected_value = " COLLECTED_NUMBER_FORMAT " last_calculated_value = " CALCULATED_NUMBER_FORMAT " calculated_value = " CALCULATED_NUMBER_FORMAT , st->id, rd->name , rd->last_collected_value , rd->collected_value , rd->last_calculated_value , rd->calculated_value ); } // ALL DONE ABOUT THE DATA UPDATE // -------------------------------------------------------------------- // find if there are any obsolete dimensions (not updated recently) if(unlikely(rrd_delete_unupdated_dimensions)) { for( rd = st->dimensions; likely(rd) ; rd = rd->next ) if((rd->last_collected_time.tv_sec + (rrd_delete_unupdated_dimensions * st->update_every)) < st->last_collected_time.tv_sec) break; if(unlikely(rd)) { // there is dimension to free // upgrade our read lock to a write lock pthread_rwlock_unlock(&st->rwlock); pthread_rwlock_wrlock(&st->rwlock); for( rd = st->dimensions, last = NULL ; likely(rd) ; ) { // remove it only it is not updated in rrd_delete_unupdated_dimensions seconds if(unlikely((rd->last_collected_time.tv_sec + (rrd_delete_unupdated_dimensions * st->update_every)) < st->last_collected_time.tv_sec)) { info("Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id); if(unlikely(!last)) { st->dimensions = rd->next; rd->next = NULL; rrddim_free(st, rd); rd = st->dimensions; continue; } else { last->next = rd->next; rd->next = NULL; rrddim_free(st, rd); rd = last->next; continue; } } last = rd; rd = rd->next; } if(unlikely(!st->dimensions)) { info("Disabling chart %s (%s) since it does not have any dimensions", st->name, st->id); st->enabled = 0; } } } pthread_rwlock_unlock(&st->rwlock); if(unlikely(pthread_setcancelstate(oldstate, NULL) != 0)) error("Cannot set pthread cancel state to RESTORE (%d).", oldstate); return(st->usec_since_last_update); }
inline calculated_number backend_calculate_value_from_stored_data( RRDSET *st // the chart , RRDDIM *rd // the dimension , time_t after // the start timestamp , time_t before // the end timestamp , uint32_t options // BACKEND_SOURCE_* bitmap , time_t *first_timestamp // the first point of the database used in this response , time_t *last_timestamp // the timestamp that should be reported to backend ) { RRDHOST *host = st->rrdhost; // find the edges of the rrd database for this chart time_t first_t = rrdset_first_entry_t(st); time_t last_t = rrdset_last_entry_t(st); time_t update_every = st->update_every; // step back a little, to make sure we have complete data collection // for all metrics after -= update_every * 2; before -= update_every * 2; // align the time-frame after = after - (after % update_every); before = before - (before % update_every); // for before, loose another iteration // the latest point will be reported the next time before -= update_every; if(unlikely(after > before)) // this can happen when update_every > before - after after = before; if(unlikely(after < first_t)) after = first_t; if(unlikely(before > last_t)) before = last_t; if(unlikely(before < first_t || after > last_t)) { // the chart has not been updated in the wanted timeframe debug(D_BACKEND, "BACKEND: %s.%s.%s: aligned timeframe %lu to %lu is outside the chart's database range %lu to %lu", host->hostname, st->id, rd->id, (unsigned long)after, (unsigned long)before, (unsigned long)first_t, (unsigned long)last_t ); return NAN; } *first_timestamp = after; *last_timestamp = before; size_t counter = 0; calculated_number sum = 0; long start_at_slot = rrdset_time2slot(st, before), stop_at_slot = rrdset_time2slot(st, after), slot, stop_now = 0; for(slot = start_at_slot; !stop_now ; slot--) { if(unlikely(slot < 0)) slot = st->entries - 1; if(unlikely(slot == stop_at_slot)) stop_now = 1; storage_number n = rd->values[slot]; if(unlikely(!does_storage_number_exist(n))) { // not collected continue; } calculated_number value = unpack_storage_number(n); sum += value; counter++; } if(unlikely(!counter)) { debug(D_BACKEND, "BACKEND: %s.%s.%s: no values stored in database for range %lu to %lu", host->hostname, st->id, rd->id, (unsigned long)after, (unsigned long)before ); return NAN; } if(unlikely((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM)) return sum; return sum / (calculated_number)counter; }
static inline void do_dimension( RRDR *r , long points_wanted , RRDDIM *rd , long dim_id_in_rrdr , long after_slot , long before_slot , time_t after_wanted , time_t before_wanted ){ (void) before_slot; RRDSET *st = r->st; time_t now = after_wanted, dt = st->update_every, max_date = 0, min_date = 0; long slot = after_slot, group_size = r->group, points_added = 0, values_in_group = 0, values_in_group_non_zero = 0, rrdr_line = -1, entries = st->entries; RRDR_VALUE_FLAGS group_value_flags = RRDR_VALUE_NOTHING; for( ; points_added < points_wanted ; now += dt, slot++ ) { if(unlikely(slot >= entries)) slot = 0; // make sure we return data in the proper time range if(unlikely(now > before_wanted)) { #ifdef NETDATA_INTERNAL_CHECKS r->log = "stopped, because attempted to access the db after 'wanted before'"; #endif break; } if(unlikely(now < after_wanted)) { #ifdef NETDATA_INTERNAL_CHECKS r->log = "skipped, because attempted to access the db before 'wanted after'"; #endif continue; } // read the value from the database storage_number n = rd->values[slot]; calculated_number value = NAN; if(likely(does_storage_number_exist(n))) { value = unpack_storage_number(n); if(likely(value != 0.0)) values_in_group_non_zero++; if(unlikely(did_storage_number_reset(n))) group_value_flags |= RRDR_VALUE_RESET; } // add this value for grouping r->grouping_add(r, value); values_in_group++; if(unlikely(values_in_group == group_size)) { rrdr_line = rrdr_line_init(r, now, rrdr_line); if(unlikely(!min_date)) min_date = now; max_date = now; // find the place to store our values RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_line * r->d + dim_id_in_rrdr]; // update the dimension options if(likely(values_in_group_non_zero)) r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO; // store the specific point options *rrdr_value_options_ptr = group_value_flags; // store the value r->v[rrdr_line * r->d + dim_id_in_rrdr] = r->grouping_flush(r, rrdr_value_options_ptr); points_added++; values_in_group = 0; group_value_flags = RRDR_VALUE_NOTHING; values_in_group_non_zero = 0; } } r->before = max_date; r->after = min_date; rrdr_done(r, rrdr_line); #ifdef NETDATA_INTERNAL_CHECKS if(unlikely(r->rows != points_added)) error("INTERNAL ERROR: %s.%s added %zu rows, but RRDR says I added %zu.", r->st->name, rd->name, (size_t)points_added, (size_t)r->rows); #endif }
void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value) { rrdset_check_rdlock(r->st); long rows = rrdr_rows(r); long c, i; RRDDIM *rd; //info("JSONWRAPPER(): %s: BEGIN", r->st->id); char kq[2] = "", // key quote sq[2] = ""; // string quote if( options & RRDR_OPTION_GOOGLE_JSON ) { kq[0] = '\0'; sq[0] = '\''; } else { kq[0] = '"'; sq[0] = '"'; } buffer_sprintf(wb, "{\n" " %sapi%s: 1,\n" " %sid%s: %s%s%s,\n" " %sname%s: %s%s%s,\n" " %sview_update_every%s: %d,\n" " %supdate_every%s: %d,\n" " %sfirst_entry%s: %u,\n" " %slast_entry%s: %u,\n" " %sbefore%s: %u,\n" " %safter%s: %u,\n" " %sdimension_names%s: [" , kq, kq , kq, kq, sq, r->st->id, sq , kq, kq, sq, r->st->name, sq , kq, kq, r->update_every , kq, kq, r->st->update_every , kq, kq, (uint32_t)rrdset_first_entry_t(r->st) , kq, kq, (uint32_t)rrdset_last_entry_t(r->st) , kq, kq, (uint32_t)r->before , kq, kq, (uint32_t)r->after , kq, kq); for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; if(i) buffer_strcat(wb, ", "); buffer_strcat(wb, sq); buffer_strcat(wb, rd->name); buffer_strcat(wb, sq); i++; } if(!i) { #ifdef NETDATA_INTERNAL_CHECKS error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options); #endif rows = 0; buffer_strcat(wb, sq); buffer_strcat(wb, "no data"); buffer_strcat(wb, sq); } buffer_sprintf(wb, "],\n" " %sdimension_ids%s: [" , kq, kq); for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; if(i) buffer_strcat(wb, ", "); buffer_strcat(wb, sq); buffer_strcat(wb, rd->id); buffer_strcat(wb, sq); i++; } if(!i) { rows = 0; buffer_strcat(wb, sq); buffer_strcat(wb, "no data"); buffer_strcat(wb, sq); } buffer_sprintf(wb, "],\n" " %slatest_values%s: [" , kq, kq); for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; if(i) buffer_strcat(wb, ", "); i++; storage_number n = rd->values[rrdset_last_slot(r->st)]; if(!does_storage_number_exist(n)) buffer_strcat(wb, "null"); else buffer_rrd_value(wb, unpack_storage_number(n)); } if(!i) { rows = 0; buffer_strcat(wb, "null"); } buffer_sprintf(wb, "],\n" " %sview_latest_values%s: [" , kq, kq); i = 0; if(rows) { calculated_number total = 1; if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { total = 0; for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { calculated_number *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; calculated_number n = cn[c]; if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) n = -n; total += n; } // prevent a division by zero if(total == 0) total = 1; } for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; if(i) buffer_strcat(wb, ", "); i++; calculated_number *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; RRDR_VALUE_FLAGS *co = &r->o[ (rrdr_rows(r) - 1) * r->d ]; calculated_number n = cn[c]; if(co[c] & RRDR_VALUE_EMPTY) { if(options & RRDR_OPTION_NULL2ZERO) buffer_strcat(wb, "0"); else buffer_strcat(wb, "null"); } else { if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) n = -n; if(unlikely(options & RRDR_OPTION_PERCENTAGE)) n = n * 100 / total; buffer_rrd_value(wb, n); } } } if(!i) { rows = 0; buffer_strcat(wb, "null"); } buffer_sprintf(wb, "],\n" " %sdimensions%s: %ld,\n" " %spoints%s: %ld,\n" " %sformat%s: %s" , kq, kq, i , kq, kq, rows , kq, kq, sq ); rrdr_buffer_print_format(wb, format); buffer_sprintf(wb, "%s,\n" " %sresult%s: " , sq , kq, kq ); if(string_value) buffer_strcat(wb, sq); //info("JSONWRAPPER(): %s: END", r->st->id); }