void cdb_print(cdb_t *cdb) {
const char *date_format = "%Y-%m-%d %H:%M:%S";
cdb_request_t request;
request.start = 0;
request.end = 0;
request.count = 0;
request.step = 0;
request.cooked = false;
printf("============== Header ================\n");
if (cdb_read_header(cdb) == CDB_SUCCESS) {
cdb_print_header(cdb);
}
if (cdb->header->type == CDB_TYPE_COUNTER) {
printf("============= Raw Counter Records =============\n");
cdb_print_records(cdb, &request, stdout, date_format);
printf("============== End Raw Counter Records ==============\n");
printf("============== Cooked Records ================\n");
} else {
printf("============== Records ================\n");
}
request.cooked = true;
cdb_print_records(cdb, &request, stdout, date_format);
printf("============== End ================\n");
}
static PyObject* cdb_py_read_header(PyObject* self) {
cdb_t *cdb = ((StorageObject*)self)->cdb;
if (cdb_read_header(cdb) != CDB_SUCCESS) {
PyErr_SetFromErrno(PyExc_IOError);
}
_cdb_py_update_header(self, cdb);
return self;
}
int cdb_discard_records_in_time_range(cdb_t *cdb, cdb_request_t *request, uint64_t *num_recs) {
uint64_t i = 0;
int64_t lrec;
off_t offset = RECORD_SIZE;
*num_recs = 0;
if (cdb_read_header(cdb) != CDB_SUCCESS) {
return cdb_error();
}
if (_cdb_is_writable(cdb) == false) {
return CDB_ERDONLY;
}
lrec = _logical_record_for_time(cdb, request->start, 0, 0);
if (lrec >= 1) {
lrec -= 1;
}
for (i = lrec; i < cdb->header->num_records; i++) {
cdb_time_t rtime = _time_for_logical_record(cdb, i);
if (rtime >= request->start && rtime <= request->end) {
cdb_record_t record[RECORD_SIZE];
record->time = rtime;
record->value = CDB_NAN;
if (pwrite(cdb->fd, &record, RECORD_SIZE, offset) != RECORD_SIZE) {
return cdb_error();
}
*num_recs += 1;
}
}
if (*num_recs > 0) {
cdb->synced = false;
}
if (cdb_write_header(cdb) != CDB_SUCCESS) {
return cdb_error();
}
return CDB_SUCCESS;
}
static int Storage_init(PyObject *self, PyObject *args, PyObject *kwdict) {
static char *kwlist[] = {
"filename", "flags", "mode", "name", "desc", "max_records", "type", "units", "min_value", "max_value", NULL,
};
char *filename;
int flags = -1;
int mode = -1;
char *name = NULL;
char *desc = "";
char *type = (char*)_string_from_cdb_type(CDB_DEFAULT_DATA_TYPE);
char *units = CDB_DEFAULT_DATA_UNIT;
uint64_t max_records = 0;
double min_value = 0;
double max_value = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwdict, "s|iiszlzzlli:new", kwlist,
&filename, &flags, &mode, &name, &desc, &max_records, &type, &units, &min_value, &max_value)) {
return -1;
}
PyObject *header = ((StorageObject*)self)->header;
cdb_t *cdb = ((StorageObject*)self)->cdb;
cdb->filename = filename;
cdb->flags = flags;
cdb->mode = mode;
// Try the open
if (cdb_open(cdb) != CDB_SUCCESS) {
PyErr_SetFromErrno(PyExc_IOError);
return -1;
}
// Try and read a header, if it doesn't exist, create one
if (name == NULL) {
if (cdb_read_header(cdb) != CDB_SUCCESS) {
PyErr_SetFromErrno(PyExc_IOError);
return -1;
}
} else {
cdb_generate_header(cdb, name, desc, max_records, _cdb_type_from_string(type), units, min_value, max_value);
if (cdb_write_header(cdb) != CDB_SUCCESS) {
PyErr_SetFromErrno(PyExc_IOError);
return -1;
}
}
PyDict_SetItemString(header, "filename", PyString_FromString(filename));
PyDict_SetItemString(header, "name", PyString_FromString(cdb->header->name));
PyDict_SetItemString(header, "desc", PyString_FromString(cdb->header->desc));
PyDict_SetItemString(header, "type", PyString_FromString(_string_from_cdb_type(cdb->header->type)));
PyDict_SetItemString(header, "units", PyString_FromString(cdb->header->units));
PyDict_SetItemString(header, "num_records", PyInt_FromLong(cdb->header->num_records));
PyDict_SetItemString(header, "max_records", PyInt_FromLong(cdb->header->max_records));
PyDict_SetItemString(header, "min_value", PyInt_FromLong(cdb->header->min_value));
PyDict_SetItemString(header, "max_value", PyInt_FromLong(cdb->header->max_value));
return 0;
}
static int _cdb_read_records(cdb_t *cdb, cdb_request_t *request, uint64_t *num_recs, cdb_record_t **records) {
int64_t first_requested_logical_record;
int64_t last_requested_logical_record;
uint64_t last_requested_physical_record;
int64_t seek_logical_record;
int64_t seek_physical_record;
cdb_record_t *buffer = NULL;
if (cdb_read_header(cdb) != CDB_SUCCESS) {
return cdb_error();
}
if (request->start != 0 && request->end != 0 && request->end < request->start) {
return CDB_ETMRANGE;
}
if (cdb->header == NULL || cdb->synced == false) {
return CDB_ESANITY;
}
/* bail out if there are no records */
if (cdb->header->num_records <= 0) {
return CDB_ENORECS;
}
/*
get the number of requested records:
-ve indicates n records from the beginning
+ve indicates n records off of the end.
0 or undef means the whole thing.
switch the meaning of -ve/+ve to be more array like
*/
if (request->count != 0) {
request->count = -request->count;
}
#ifdef DEBUG
printf("read_records start: [%ld]\n", request->start);
printf("read_records end: [%ld]\n", request->end);
printf("read_records num_requested: [%"PRIu64"]\n", request->count);
#endif
if (request->count != 0 && request->count < 0 && request->start == 0) {
/* if reading only few records from the end, just set -ve offset to seek to */
first_requested_logical_record = request->count;
} else {
/* compute which record to start reading from the beginning, based on start time specified. */
first_requested_logical_record = _logical_record_for_time(cdb, request->start, 0, 0);
}
/* if end is not defined, read all the records or only read uptill the specified record. */
if (request->end == 0) {
last_requested_logical_record = cdb->header->num_records - 1;
} else {
last_requested_logical_record = _logical_record_for_time(cdb, request->end, 0, 0);
/* this can return something > end, check for that */
if (_time_for_logical_record(cdb, last_requested_logical_record) > request->end) {
last_requested_logical_record -= 1;
}
}
last_requested_physical_record = (last_requested_logical_record + cdb->header->start_record) % cdb->header->num_records;
seek_logical_record = first_requested_logical_record;
seek_physical_record = _seek_to_logical_record(cdb, seek_logical_record);
if (last_requested_physical_record >= seek_physical_record) {
uint64_t nrec = (last_requested_physical_record - seek_physical_record + 1);
uint64_t rlen = RECORD_SIZE * nrec;
if ((buffer = calloc(1, rlen)) == NULL) {
free(buffer);
return CDB_ENOMEM;
}
/* one slurp - XXX TODO - mmap */
if (read(cdb->fd, buffer, rlen) != rlen) {
free(buffer);
return cdb_error();
}
*num_recs = nrec;
} else {
/* We've wrapped around the end of the file */
uint64_t nrec1 = (cdb->header->num_records - seek_physical_record);
uint64_t nrec2 = (last_requested_physical_record + 1);
uint64_t rlen1 = RECORD_SIZE * nrec1;
uint64_t rlen2 = RECORD_SIZE * nrec2;
if ((buffer = calloc(1, rlen1 + rlen2)) == NULL) {
free(buffer);
return CDB_ENOMEM;
}
if (read(cdb->fd, buffer, rlen1) != rlen1) {
free(buffer);
return cdb_error();
}
if (pread(cdb->fd, &buffer[nrec1], rlen2, HEADER_SIZE) != rlen2) {
free(buffer);
return cdb_error();
}
*num_recs = nrec1 + nrec2;
}
/* Deal with cooking the output */
if (request->cooked) {
bool check_min_max = true;
long factor = 0;
uint64_t i = 0;
uint64_t cooked_recs = 0;
double prev_value = 0.0;
cdb_time_t prev_date = 0;
cdb_record_t *crecords;
if ((crecords = calloc(*num_recs, RECORD_SIZE)) == NULL) {
free(crecords);
free(buffer);
return CDB_ENOMEM;
}
if (_compute_scale_factor_and_num_records(cdb, &request->count, &factor)) {
free(crecords);
free(buffer);
return cdb_error();
}
if (cdb->header->min_value == 0 && cdb->header->max_value == 0) {
check_min_max = false;
}
for (i = 0; i < *num_recs; i++) {
cdb_time_t date = buffer[i].time;
double value = buffer[i].value;
if (cdb->header->type == CDB_TYPE_COUNTER) {
double new_value = value;
value = CDB_NAN;
if (!isnan(prev_value) && !isnan(new_value)) {
double val_delta = new_value - prev_value;
if (val_delta >= 0) {
value = val_delta;
}
}
prev_value = new_value;
}
if (factor != 0 && cdb->header->type == CDB_TYPE_COUNTER) {
/* Skip the first entry, since it's absolute and is needed
* to calculate the second */
if (prev_date == 0) {
prev_date = date;
continue;
}
cdb_time_t time_delta = date - prev_date;
if (time_delta > 0 && !isnan(value)) {
value = factor * (value / time_delta);
}
prev_date = date;
}
/* Check for min/max boundaries */
/* Should this be done on write instead of read? */
if (check_min_max && !isnan(value)) {
if (value > cdb->header->max_value || value < cdb->header->min_value) {
value = CDB_NAN;
}
}
/* Copy the munged data to our new array, since we might skip
* elements. Also keep in mind mmap for the future */
crecords[cooked_recs].time = date;
crecords[cooked_recs].value = value;
cooked_recs += 1;
}
/* Now swap our cooked records for the buffer, so we can slice it as needed. */
free(buffer);
buffer = crecords;
*num_recs = cooked_recs;
}
/* If we've been requested to average the records & timestamps */
if (request->step > 1) {
cdb_record_t *arecords;
long step = request->step;
uint64_t step_recs = 0;
uint64_t leftover = (*num_recs % step);
uint64_t walkend = (*num_recs - leftover);
uint64_t i = 0;
if ((arecords = calloc((int)((*num_recs / step) + leftover), RECORD_SIZE)) == NULL) {
free(arecords);
free(buffer);
return CDB_ENOMEM;
}
/* Walk our list of cooked records, jumping ahead by the given step.
For each set of records within that step, we want to get the average
for those records and place them into a new array.
*/
for (i = 0; i < walkend; i += step) {
int j = 0;
double xi[step];
double yi[step];
for (j = 0; j < step; j++) {
/* No NaNs on average - they make bogus graphs. Is there a
* better value than 0 to use here? */
if (isnan(buffer[i+j].value)) {
buffer[i+j].value = 0;
}
xi[j] = (double)buffer[i+j].time;
yi[j] = buffer[i+j].value;
}
arecords[step_recs].time = (cdb_time_t)gsl_stats_mean(xi, 1, step);
arecords[step_recs].value = gsl_stats_mean(yi, 1, step);
step_recs += 1;
}
/* Now collect from the last step point to the end and average. */
if (leftover > 0) {
uint64_t leftover_start = *num_recs - leftover;
int j = 0;
double xi[leftover];
double yi[leftover];
for (i = leftover_start; i < *num_recs; i++) {
/* No NaNs on average - they make bogus graphs. Is there a
* better value than 0 to use here? */
if (isnan(buffer[i].value)) {
buffer[i].value = 0;
}
xi[j] = (double)buffer[i].time;
yi[j] = buffer[i].value;
j++;
}
arecords[step_recs].time = (cdb_time_t)gsl_stats_mean(xi, 1, j);
arecords[step_recs].value = gsl_stats_mean(yi, 1, j);
step_recs += 1;
}
free(buffer);
buffer = arecords;
*num_recs = step_recs;
}
/* now pull out the number of requested records if asked */
if (request->count != 0 && *num_recs >= abs(request->count)) {
uint64_t start_index = 0;
if (request->count <= 0) {
start_index = *num_recs - abs(request->count);
}
*num_recs = abs(request->count);
if ((*records = calloc(*num_recs, RECORD_SIZE)) == NULL) {
free(buffer);
return CDB_ENOMEM;
}
memcpy(*records, &buffer[start_index], RECORD_SIZE * *num_recs);
free(buffer);
} else {
*records = buffer;
}
return CDB_SUCCESS;
}
int cdb_update_records(cdb_t *cdb, cdb_record_t *records, uint64_t len, uint64_t *num_recs) {
int ret = CDB_SUCCESS;
*num_recs = 0;
uint64_t i = 0;
if (cdb_read_header(cdb) != CDB_SUCCESS) {
return cdb_error();
}
if (_cdb_is_writable(cdb) == false) {
return CDB_ERDONLY;
}
#ifdef DEBUG
printf("in update_records with [%"PRIu64"] num_recs\n", cdb->header->num_records);
#endif
for (i = 0; i < len; i++) {
cdb_time_t time = records[i].time;
cdb_time_t rtime;
uint64_t lrec;
lrec = _logical_record_for_time(cdb, time, 0, 0);
if (lrec >= 1) {
lrec -= 1;
/* DRK things like this are extremely confusing. some logical
functions are zero based? some are one? or you're trying to back up? */
}
rtime = _time_for_logical_record(cdb, lrec);
while (rtime < time && lrec < cdb->header->num_records - 1) {
/* DRK is this true? or is the condition (lrec - start_lrec) <
num_recs -- seems you can't wrap here (i.e. what if the initial
lrec was num_recs - 1) */
lrec += 1;
rtime = _time_for_logical_record(cdb, lrec);
}
#ifdef DEBUG
printf("update_records: value: lrec [%"PRIu64"] time [%d] rtime [%d]\n", lrec, (int)time, (int)rtime);
#endif
while (time == rtime && lrec < cdb->header->num_records - 1) {
_seek_to_logical_record(cdb, lrec);
if (write(cdb->fd, &records[0], RECORD_SIZE) != RECORD_SIZE) {
ret = cdb_error();
break;
}
lrec += 1;
rtime = _time_for_logical_record(cdb, lrec);
}
}
if (ret == CDB_SUCCESS) {
if (i > 0) {
cdb->synced = false;
*num_recs = i;
}
if (cdb_write_header(cdb) != CDB_SUCCESS) {
ret = cdb_error();
}
}
return ret;
}
int cdb_write_records(cdb_t *cdb, cdb_record_t *records, uint64_t len, uint64_t *num_recs) {
/* read old header if it exists.
write a header out, since the db may not have existed.
*/
uint64_t i = 0;
uint64_t j = 0;
off_t offset = 0;
*num_recs = 0;
if (cdb_read_header(cdb) != CDB_SUCCESS) {
return cdb_error();
}
if (_cdb_is_writable(cdb) == false) {
return CDB_ERDONLY;
}
if (cdb->header->max_records <= 0) {
return CDB_EINVMAX;
}
/* Logic for writes:
cdb is 5 records.
try to write 7 records
len = 7
len + 0 > 5
j = (7 + 0) - 5
j = 2
i = 7 - 2
i = 5
need to write j records at offset: HEADER_SIZE + (cdb->header->start_record * RECORD_SIZE)
index into records is i
need to write i records at offset: HEADER_SIZE + (cdb->header->num_records * RECORD_SIZE)
index into records is 0;
*/
/* Calculate our indicies into the records array. */
if (len + cdb->header->num_records >= cdb->header->max_records) {
j = (len + cdb->header->num_records) - cdb->header->max_records;
i = (len - j);
} else {
i = len;
}
/* If we need to wrap around */
if (j > 0) {
offset = HEADER_SIZE + (cdb->header->start_record * RECORD_SIZE);
cdb->header->start_record += j;
cdb->header->start_record %= cdb->header->max_records;
if (pwrite(cdb->fd, &records[i], (RECORD_SIZE * j), offset) != (RECORD_SIZE * j)) {
return cdb_error();
}
cdb->synced = false;
/* start_record is no longer 0, so update the header */
if (cdb_write_header(cdb) != CDB_SUCCESS) {
return cdb_error();
}
}
/* Normal case */
offset = HEADER_SIZE + (cdb->header->num_records * RECORD_SIZE);
cdb->header->num_records += i;
if (pwrite(cdb->fd, &records[0], (RECORD_SIZE * i), offset) != (RECORD_SIZE * i)) {
return cdb_error();
}
*num_recs += len;
#ifdef DEBUG
printf("write_records: wrote [%"PRIu64"] records\n", *num_recs);
#endif
return CDB_SUCCESS;
}
