// called by clean function /////////////////////////
void WipeData(int interval_hour)
{
// traverse db
// read file and get time
// if time stamp earlier than 'time_line'
int time_now = (int)time(NULL);
int time_line = time_now - interval_hour * 60 * 60;
int temp = 0;
char *kbuf = 0;
const char* cvbuf = 0;
size_t ksize, vsize;
KCCUR * cur = kcdbcursor(g_kcdb);
kccurjump(cur);
while ( ( kbuf = kccurget(cur, &ksize, &cvbuf, &vsize, 0)) != NULL ) {
memcpy(&temp, cvbuf, RECORD_HEADER_LEN);
//printf("%d, k:%d, ks:%d, v:%d, vs:%d\n", temp, kbuf, ksize, cvbuf, vsize);
if (temp < time_line) {kccurremove(cur);}
else {kccurstep(cur);}
}
kccurdel(cur);
return ;
}
Datum kc_expand(PG_FUNCTION_ARGS) {
KC_ENTRY *search;
FuncCallContext *funcctx;
int call_cntr;
char *kbuf;
size_t ksiz, vsiz;
const char *cvbuf;
char *kv_kbuf = NULL;
size_t kv_ksiz;
int done;
/* stuff done only on the first call of the function */
if (SRF_IS_FIRSTCALL()) {
MemoryContext oldcontext;
/* create a function context for cross-call persistence */
funcctx = SRF_FIRSTCALL_INIT();
/* switch to memory context appropriate for multiple function calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
// Make sure that there are enough args.
if (PG_NARGS() < MIN_ARGS) {
ereport(ERROR,
(errcode(ERRCODE_EXTERNAL_ROUTINE_INVOCATION_EXCEPTION),
errmsg("Must run expand with at least %d args!", MIN_ARGS)));
}
/* Make the kcdb here. */
search = (KC_ENTRY *)palloc(sizeof(KC_ENTRY));
search->db = kcdbnew();
if (open_db (search->db, text_to_cstring(PG_GETARG_TEXT_PP(0)), text_to_cstring(PG_GETARG_TEXT_PP(1)))) {
// Set the key to jump into:
// Call with -- map_name, result_id, class, doctype, pop, psource
// Here, map_name describes a db to open.
// Otherwise, result_id:class:doctype:pop:psource
(search->jump_key) = (char *) palloc(MAX_JUMP_KEY_LEN * sizeof(char));
int index_point;
search->jump_key = text_to_cstring(PG_GETARG_TEXT_PP(2));
int size_left = MAX_JUMP_KEY_LEN;
for (index_point = START_VARIABLE_INDEX; index_point < END_VARIABLE_INDEX; index_point++) {
if (PG_NARGS() > index_point) {
char *next_idx = text_to_cstring(PG_GETARG_TEXT_PP(index_point));
if (next_idx != NULL) {
size_left = size_left - (2 + strlen(next_idx));
strncat (search->jump_key, CF_LABEL_SEP, size_left);
strncat (search->jump_key, next_idx, size_left);
}
}
}
#ifdef CF_DUBUG
ereport(NOTICE,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("Setting jump buffer -- [%s]", search->jump_key)));
#endif
// Create a cursor, and set it to the base point looking for entries.
search->cur = kcdbcursor(search->db);
kccurjumpkey(search->cur, search->jump_key, MAX_JUMP_KEY_LEN);
} else {
search->db = NULL;
}
search->next_map = 0;
search->msg = NULL;
// Save the search struct for the subsequent calls.
funcctx->user_fctx = search;
MemoryContextSwitchTo(oldcontext);
}
/* stuff done on every call of the function */
funcctx = SRF_PERCALL_SETUP();
call_cntr = funcctx->call_cntr;
search = (KC_ENTRY *) funcctx->user_fctx;
// If no current msg, try to get the next one.
done = 1;
#ifdef CF_DUBUG
ereport(NOTICE,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("beginning run")));
#endif
if (search->msg) {
#ifdef CF_DUBUG
ereport(NOTICE,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("Incrementing next from map %d -- %zu", search->next_map, search->msg->n_map_entry)));
#endif
// Case if we are using the external cursor running over kv map.
// Ready the next
if (search->msg->kv_map_file) {
if ((kv_kbuf = kccurgetkey(search->kv_cur, &kv_ksiz, 1)) == NULL) {
done = 1;
kccurdel(search->kv_cur);
kcdbendtran (search->kv_db, 1);
if (!kcdbclose(search->kv_db)) {
ereport(ERROR,
(errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
errmsg("Error Closeing db: \"%s\"", kcecodename(kcdbecode(search->kv_db)))));
}
// Also need to free this.
cloudflare__zone_time_bucket__free_unpacked(search->msg, NULL);
search->msg = NULL;
} else {
done = 0;
}
} else {
if (search->next_map >= search->msg->n_map_entry) {
// Done with this msg -- move on to the next one.
cloudflare__zone_time_bucket__free_unpacked(search->msg, NULL);
search->msg = NULL;
} else {
done = 0;
}
}
}
if (search->db && !search->msg) {
#ifdef CF_DUBUG
ereport(NOTICE,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("Getting new buf -- %s", search->jump_key)));
#endif
if ((kbuf = kccurget(search->cur, &ksiz, &cvbuf, &vsiz, 1)) != NULL) {
// Pull up the PB and expand it.
search->msg = cloudflare__zone_time_bucket__unpack(NULL, vsiz, (const uint8_t *)cvbuf);
if (search->msg == NULL) { // Something failed
ereport(ERROR,
(errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
errmsg("error unpacking incoming message")));
done = 1;
} else {
// Does the buffer match the searched for string?
// @TODO -- bound this?
if (strstr(search->msg->db_key, search->jump_key)) {
done = 0;
search->next_map = 0;
// And load the kvkc if needed.
if (search->msg->kv_map_file) {
#ifdef CF_DUBUG
ereport(NOTICE,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("Switching to kvs %s", search->msg->kv_map_file)));
#endif
search->kv_db = kcdbnew();
if (!kcdbopen(search->kv_db, search->msg->kv_map_file, KCOWRITER)) {
#ifdef CF_NO_DB_IS_ERR
ereport(ERROR,
(errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
errmsg("Error opening db: \"%s\", \"%s\". Make sure that the map_name is valid.",
search->msg->kv_map_file, kcecodename(kcdbecode(search->kv_db)))));
#endif
#ifdef CF_DUBUG
ereport(NOTICE,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("Error opening db: \"%s\", \"%s\". Make sure that the map_name is valid.",
search->msg->kv_map_file, kcecodename(kcdbecode(search->kv_db)))));
#endif
done = 1;
} else {
kcdbbegintran (search->kv_db, 0);
search->kv_cur = kcdbcursor(search->kv_db);
kccurjump(search->kv_cur);
if ((kv_kbuf = kccurgetkey(search->kv_cur, &kv_ksiz, 1)) == NULL) {
done = 1;
kccurdel(search->kv_cur);
kcdbendtran (search->kv_db, 1);
if (!kcdbclose(search->kv_db)) {
ereport(ERROR,
(errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
errmsg("Error Closeing db: \"%s\"", kcecodename(kcdbecode(search->kv_db)))));
}
} else {
done = 0;
}
}
}
} else {
done = 1;
}
}
kcfree(kbuf);
} else {
#ifdef CF_DUBUG
ereport(NOTICE,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("no msg to find")));
#endif
done = 1;
}
}
#ifdef CF_DUBUG
ereport(NOTICE,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("Done? %d -- next buf -- %d", done, search->next_map)));
#endif
// Take the next itteration over the cursor. If the next is NULL or else not matching the resultid passed in
// End. Otherwise, parse the value, populating the next row of the returning tuple.
if (!done) {
KC_ROW *out;
Datum result;
size_t size = sizeof(KC_ROW);
out = (KC_ROW *)palloc(size);
memset(out, '0', size);
SET_VARSIZE(out, size);
out->classification = (char *)palloc(MAX_KC_ROW_ENTRY * sizeof(char));
out->doctype = (char *)palloc(MAX_KC_ROW_ENTRY * sizeof(char));
out->pop = (char *)palloc(MAX_KC_ROW_ENTRY * sizeof(char));
out->psource = (char *)palloc(MAX_KC_ROW_ENTRY * sizeof(char));
out->key = (char *)palloc(MAX_KC_ROW_ENTRY * sizeof(char));
strncpy(out->classification, search->msg->classification, MAX_KC_ROW_ENTRY);
strncpy(out->doctype, search->msg->doctype, MAX_KC_ROW_ENTRY);
strncpy(out->pop, search->msg->pop, MAX_KC_ROW_ENTRY);
strncpy(out->psource, search->msg->psource, MAX_KC_ROW_ENTRY);
if (search->msg->kv_map_file) {
#ifdef CF_DUBUG
ereport(NOTICE,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("getting val from -- [%s]", search->msg->kv_map_file)));
#endif
snprintf(out->key, MAX_KC_ROW_ENTRY, "%s", kv_kbuf);
out->value = kcdbincrint (search->kv_db, kv_kbuf, kv_ksiz, 0);
if (out->value == INT64_MIN) {
ereport(NOTICE,
(errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
errmsg("ERROR Getting val from key -- [%s], %s", kv_kbuf, kcecodename(kcdbecode(search->kv_db)))));
}
kcfree(kv_kbuf);
} else {
#ifdef CF_DUBUG
ereport(NOTICE,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("Loading %s %ld", search->msg->map_entry[search->next_map]->key,
search->msg->map_entry[search->next_map]->value)));
#endif
snprintf(out->key, MAX_KC_ROW_ENTRY, "%s", search->msg->map_entry[search->next_map]->key);
out->value = search->msg->map_entry[search->next_map]->value;
}
result = PointerGetDatum(out);
/* clean up (this is not really necessary) */
pfree(out->classification);
pfree(out->doctype);
pfree(out->pop);
pfree(out->psource);
pfree(out->key);
pfree(out);
// Remember that we are going to the next step.
search->next_map++;
SRF_RETURN_NEXT(funcctx, result);
} else { /* do when there is no more left */
if (search->db) {
kccurdel(search->cur);
if (!kcdbclose(search->db)) {
ereport(ERROR,
(errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION),
errmsg("Error Closeing db: \"%s\"", kcecodename(kcdbecode(search->db)))));
}
if (search->msg != NULL) {
cloudflare__zone_time_bucket__free_unpacked(search->msg, NULL);
}
pfree(search->jump_key);
}
pfree(search);
#ifdef CF_DUBUG
ereport(NOTICE,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("Done with run")));
#endif
// Don't delete db, this leads to segfaults.
SRF_RETURN_DONE(funcctx);
}
}
