char * pwd_save(KCDB * db, const char * origin_url) { char * pk, * epk, * url; char hash[41]; size_t vsiz; if(!kcdbopen(db, db_file, KCOWRITER|KCOCREATE)) { printf("Status: 500 INTERNAL ERROR\r\n\r\nkcdb open error: %s\n", kcecodename(kcdbecode(db))); return NULL; } if(strstr(origin_url, "http://") == NULL && strstr(origin_url, "https://") == NULL) { url = malloc(strlen(origin_url) + 8); strcpy(url, "http://"); strcat(url, origin_url); } else { url = malloc(strlen(origin_url) + 1); strcpy(url, origin_url); } // calc url hash and check if exists sha1(hash, url); epk = kcdbget(db, hash, strlen(hash), &vsiz); if(epk) { pk = malloc(strlen(epk) + 1); strcpy(pk, epk); kcfree(epk); free(url); kcdbclose(db); return pk; } pk = pwd_get_pk(db); if(pk == NULL) { free(url); kcdbclose(db); return NULL; } //TODO save pk => url && hash => pk if(!kcdbset(db, pk, strlen(pk), url, strlen(url))) { free(pk); pk = NULL; } if(!kcdbset(db, hash, strlen(hash), pk, strlen(pk))) { free(pk); pk = NULL; } free(url); kcdbclose(db); return pk; }
char * pwd_get_pk(KCDB * db) { char * pk, * result; size_t vsiz; int i; const char * map = ".adgjmptw"; pk = kcdbget(db, "master", 6, &vsiz); if(pk) { result = malloc(vsiz + 2); strcpy(result, pk); kcfree(pk); pk = NULL; } else { result = malloc(3); strcpy(result, "0"); } // increase pk vsiz = strlen(result); for(i = 0; ; i++) { if(i == vsiz) { result[i] = i == 0 ? '1' : (result[i - 1] == '1' ? '2' : '1'); result[i + 1] = '\0'; break; } result[i]++; if(result[i] >= '9') result[i] = i > 0 ? (result[i - 1] == '0' ? '1' : '0') : '1'; else if(i > 0 && result[i - 1] == result[i]) i--; else if(i < vsiz && result[i + 1] == result[i]) i--; else break; } if(!kcdbset(db, "master", 6, result, strlen(result))) { free(result); return NULL; } // mapping result vsiz = strlen(result); for(i = 0 ; i < vsiz; i++) { result[i] = map[result[i]&0x0f]; } return result; }
extern CAMLprim value kc_set(value caml_db, value key, value val) { CAMLparam3(caml_db, key, val); KCDB* db = get_db(caml_db); if (! kcdbset(db, String_val(key), caml_string_length(key), String_val(val), caml_string_length(val) )) { RAISE(kcdbemsg(db)); } CAMLreturn(Val_unit); }
Datum kc_shrink(PG_FUNCTION_ARGS) { char *map_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); char *start_time = text_to_cstring(PG_GETARG_TEXT_PP(1)); // Start time + uid!!! char *new_rid = text_to_cstring(PG_GETARG_TEXT_PP(2)); ArrayType *old_rids = PG_GETARG_ARRAYTYPE_P(3); char *classification = text_to_cstring(PG_GETARG_TEXT_PP(4)); char *doctype = text_to_cstring(PG_GETARG_TEXT_PP(5)); char *pop = text_to_cstring(PG_GETARG_TEXT_PP(6)); char *psource = text_to_cstring(PG_GETARG_TEXT_PP(7)); text *tout; int i,j; Datum *rid_datums; bool *rid_nulls; int rid_count; char *next_rid; KCDB *main_db; char *vbuf; size_t vsiz; // Open our DB. main_db = kcdbnew(); if (!open_db (main_db, map_name, start_time)) { tout = cstring_to_text(new_rid); PG_RETURN_TEXT_P(tout); } kcdbbegintran (main_db, 0); // First fill in what we can from the input. Cloudflare__ZoneTimeBucket msg = CLOUDFLARE__ZONE_TIME_BUCKET__INIT; msg.map_name = (char *)palloc(MAX_KC_ROW_ENTRY * sizeof(char)); msg.doctype = (char *)palloc(MAX_KC_ROW_ENTRY * sizeof(char)); msg.classification = (char *)palloc(MAX_KC_ROW_ENTRY * sizeof(char)); msg.pop = (char *)palloc(MAX_KC_ROW_ENTRY * sizeof(char)); msg.psource = (char *)palloc(MAX_KC_ROW_ENTRY * sizeof(char)); msg.result_id = (char *)palloc(MAX_KC_ROW_ENTRY * sizeof(char)); msg.db_key = (char *)palloc(MAX_KC_ROW_ENTRY * sizeof(char)); msg.db_path = (char *)palloc(MAX_KC_ROW_ENTRY * sizeof(char)); msg.map_entry = palloc(MAX_KEYS_BEFORE_KV_MAP * sizeof(Cloudflare__ZoneTimeBucket__Counter)); msg.n_map_entry = 0; strncpy(msg.map_name, map_name, MAX_KC_ROW_ENTRY); strncpy(msg.classification, classification, MAX_KC_ROW_ENTRY); strncpy(msg.doctype, doctype, MAX_KC_ROW_ENTRY); strncpy(msg.pop, pop, MAX_KC_ROW_ENTRY); strncpy(msg.psource, psource, MAX_KC_ROW_ENTRY); strncpy(msg.result_id, new_rid, KC_MAX_RID); snprintf(msg.db_path, MAX_KC_ROW_ENTRY, "%s%s%s", map_name, "/", start_time); snprintf(msg.db_key, KC_MAX_RID, "%s%s%s%s%s%s%s%s%s%s%s", new_rid, CF_LABEL_SEP, classification, CF_LABEL_SEP, doctype, CF_LABEL_SEP, pop, CF_LABEL_SEP, psource, CF_LABEL_SEP, map_name); // Now run over the array. deconstruct_array(old_rids, TEXTOID, -1, false, 'i', &rid_datums, &rid_nulls, &rid_count); if (ARR_HASNULL(old_rids)) { ereport(ERROR, (errcode(ERRCODE_ARRAY_ELEMENT_ERROR), errmsg("cannot work with arrays containing NULLs"))); } int num_new_keys = 0; int num_entries = 0; char keys_to_use[rid_count][KC_MAX_RID]; Cloudflare__ZoneTimeBucket *msg_new[rid_count]; j=0; for (i = 0; i < rid_count; i++) { next_rid = TextDatumGetCString(rid_datums[i]); snprintf(keys_to_use[i], KC_MAX_RID, "%s%s%s%s%s%s%s%s%s%s%s", next_rid, CF_LABEL_SEP, classification, CF_LABEL_SEP, doctype, CF_LABEL_SEP, pop, CF_LABEL_SEP, psource, CF_LABEL_SEP, map_name); vbuf = kcdbget(main_db, keys_to_use[i], strlen(keys_to_use[i]), &vsiz); if (vbuf) { msg_new[j] = cloudflare__zone_time_bucket__unpack(NULL, vsiz, (const uint8_t *)vbuf); if (msg_new[j] == NULL) { // Something failed ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("error unpacking incoming message"))); } else { if (msg_new[j]->kv_map_file) { num_entries = MAX_KEYS_BEFORE_KV_MAP + 1; } else { num_entries += msg_new[j]->n_map_entry; } j++; } kcfree(vbuf); } else { #ifdef CF_DUBUG ereport(NOTICE, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("get error on %s -- %s", keys_to_use[i], kcecodename(kcdbecode(main_db))))); #endif } } // Now merge the buffers. KCDB* msg_db = NULL; if (num_entries > MAX_KEYS_BEFORE_KV_MAP) { msg_db = kcdbnew(); set_kv_path(&msg, map_name, start_time, msg_db); } for (i = 0; i < j; i++) { if (num_entries > MAX_KEYS_BEFORE_KV_MAP) { num_new_keys += merge_using_kv_map(&msg, msg_new[i], msg_db); } else { num_new_keys += merge_messages_basic(&msg, msg_new[i]); } cloudflare__zone_time_bucket__free_unpacked(msg_new[i], NULL); } if (num_entries > MAX_KEYS_BEFORE_KV_MAP) { // Close the db. kcdbendtran (msg_db, 1); kcdbclose(msg_db); } #ifdef CF_DUBUG ereport(NOTICE, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("saving: num map entries: %zu -- writting with %d keys", msg.n_map_entry, num_new_keys))); #endif // Save the updated buffer. if (num_new_keys > 0) { unsigned int len; void *buf; len = cloudflare__zone_time_bucket__get_packed_size (&msg); buf = palloc (len); cloudflare__zone_time_bucket__pack (&msg, buf); if(!kcdbset(main_db, msg.db_key, strlen(msg.db_key), buf, len)) { ereport(ERROR, (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION), errmsg("set error: %s\n", kcecodename(kcdbecode(main_db))))); } pfree (buf); } // Done! kcdbendtran (main_db, 1); if (!kcdbclose(main_db)) { ereport(ERROR, (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION), errmsg("Error Closeing db: \"%s\"", kcecodename(kcdbecode(main_db))))); } tout = cstring_to_text(new_rid); PG_RETURN_TEXT_P(tout); }
/* perform order command */ static int32_t procorder(const char* path, int64_t rnum, int32_t rnd, int32_t etc, int32_t tran, int32_t oflags) { KCDB* db; KCCUR* cur, *paracur; int32_t err; char kbuf[RECBUFSIZ], *vbuf, wbuf[RECBUFSIZ], *corepath, *copypath, *snappath; size_t ksiz, vsiz, psiz; int32_t wsiz; int64_t i, cnt; double stime, etime; VISARG visarg; oprintf("<In-order Test>\n path=%s rnum=%ld rnd=%d etc=%d tran=%d oflags=%d\n\n", path, (long)rnum, rnd, etc, tran, oflags); err = FALSE; db = kcdbnew(); oprintf("opening the database:\n"); stime = kctime(); if (!kcdbopen(db, path, KCOWRITER | KCOCREATE | KCOTRUNCATE | oflags)) { dberrprint(db, __LINE__, "kcdbopen"); err = TRUE; } etime = kctime(); dbmetaprint(db, FALSE); oprintf("time: %.3f\n", etime - stime); oprintf("setting records:\n"); stime = kctime(); for (i = 1; !err && i <= rnum; i++) { if (tran && !kcdbbegintran(db, FALSE)) { dberrprint(db, __LINE__, "kcdbbegintran"); err = TRUE; } ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); if (!kcdbset(db, kbuf, ksiz, kbuf, ksiz)) { dberrprint(db, __LINE__, "kcdbset"); err = TRUE; } if (tran && !kcdbendtran(db, TRUE)) { dberrprint(db, __LINE__, "kcdbendtran"); err = TRUE; } if (rnum > 250 && i % (rnum / 250) == 0) { oputchar('.'); if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); } } etime = kctime(); dbmetaprint(db, FALSE); oprintf("time: %.3f\n", etime - stime); if (etc) { oprintf("adding records:\n"); stime = kctime(); for (i = 1; !err && i <= rnum; i++) { if (tran && !kcdbbegintran(db, FALSE)) { dberrprint(db, __LINE__, "kcdbbegintran"); err = TRUE; } ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); if (!kcdbadd(db, kbuf, ksiz, kbuf, ksiz) && kcdbecode(db) != KCEDUPREC) { dberrprint(db, __LINE__, "kcdbadd"); err = TRUE; } if (tran && !kcdbendtran(db, TRUE)) { dberrprint(db, __LINE__, "kcdbendtran"); err = TRUE; } if (rnum > 250 && i % (rnum / 250) == 0) { oputchar('.'); if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); } } etime = kctime(); dbmetaprint(db, FALSE); oprintf("time: %.3f\n", etime - stime); } if (etc) { oprintf("appending records:\n"); stime = kctime(); for (i = 1; !err && i <= rnum; i++) { if (tran && !kcdbbegintran(db, FALSE)) { dberrprint(db, __LINE__, "kcdbbegintran"); err = TRUE; } ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); if (!kcdbappend(db, kbuf, ksiz, kbuf, ksiz)) { dberrprint(db, __LINE__, "kcdbadd"); err = TRUE; } if (tran && !kcdbendtran(db, TRUE)) { dberrprint(db, __LINE__, "kcdbendtran"); err = TRUE; } if (rnum > 250 && i % (rnum / 250) == 0) { oputchar('.'); if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); } } etime = kctime(); dbmetaprint(db, FALSE); oprintf("time: %.3f\n", etime - stime); } oprintf("getting records:\n"); stime = kctime(); for (i = 1; !err && i <= rnum; i++) { if (tran && !kcdbbegintran(db, FALSE)) { dberrprint(db, __LINE__, "kcdbbegintran"); err = TRUE; } ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); vbuf = kcdbget(db, kbuf, ksiz, &vsiz); if (vbuf) { if (vsiz < ksiz || memcmp(vbuf, kbuf, ksiz)) { dberrprint(db, __LINE__, "kcdbget"); err = TRUE; } kcfree(vbuf); } else if (!rnd || kcdbecode(db) != KCENOREC) { dberrprint(db, __LINE__, "kcdbget"); err = TRUE; } if (tran && !kcdbendtran(db, TRUE)) { dberrprint(db, __LINE__, "kcdbendtran"); err = TRUE; } if (rnum > 250 && i % (rnum / 250) == 0) { oputchar('.'); if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); } } etime = kctime(); dbmetaprint(db, FALSE); oprintf("time: %.3f\n", etime - stime); if (etc) { oprintf("getting records with a buffer:\n"); stime = kctime(); for (i = 1; !err && i <= rnum; i++) { if (tran && !kcdbbegintran(db, FALSE)) { dberrprint(db, __LINE__, "kcdbbegintran"); err = TRUE; } ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); wsiz = kcdbgetbuf(db, kbuf, ksiz, wbuf, sizeof(wbuf)); if (wsiz >= 0) { if (wsiz < (int32_t)ksiz || memcmp(wbuf, kbuf, ksiz)) { dberrprint(db, __LINE__, "kcdbgetbuf"); err = TRUE; } } else if (!rnd || kcdbecode(db) != KCENOREC) { dberrprint(db, __LINE__, "kcdbgetbuf"); err = TRUE; } if (tran && !kcdbendtran(db, TRUE)) { dberrprint(db, __LINE__, "kcdbendtran"); err = TRUE; } if (rnum > 250 && i % (rnum / 250) == 0) { oputchar('.'); if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); } } etime = kctime(); dbmetaprint(db, FALSE); oprintf("time: %.3f\n", etime - stime); } if (etc) { oprintf("traversing the database by the inner iterator:\n"); stime = kctime(); cnt = kcdbcount(db); visarg.rnum = rnum; visarg.rnd = rnd; visarg.cnt = 0; memset(visarg.rbuf, '+', sizeof(visarg.rbuf)); if (tran && !kcdbbegintran(db, FALSE)) { dberrprint(db, __LINE__, "kcdbbegintran"); err = TRUE; } if (!kcdbiterate(db, visitfull, &visarg, TRUE)) { dberrprint(db, __LINE__, "kcdbiterate"); err = TRUE; } if (rnd) oprintf(" (end)\n"); if (tran && !kcdbendtran(db, TRUE)) { dberrprint(db, __LINE__, "kcdbendtran"); err = TRUE; } if (visarg.cnt != cnt) { dberrprint(db, __LINE__, "kcdbiterate"); err = TRUE; } etime = kctime(); dbmetaprint(db, FALSE); oprintf("time: %.3f\n", etime - stime); } if (etc) { oprintf("traversing the database by the outer cursor:\n"); stime = kctime(); cnt = kcdbcount(db); visarg.rnum = rnum; visarg.rnd = rnd; visarg.cnt = 0; if (tran && !kcdbbegintran(db, FALSE)) { dberrprint(db, __LINE__, "kcdbbegintran"); err = TRUE; } cur = kcdbcursor(db); if (!kccurjump(cur) && kccurecode(cur) != KCENOREC) { dberrprint(db, __LINE__, "kccurjump"); err = TRUE; } paracur = kcdbcursor(db); while (!err && kccuraccept(cur, &visitfull, &visarg, TRUE, !rnd)) { if (rnd) { ksiz = sprintf(kbuf, "%08ld", (long)myrand(rnum)); switch (myrand(3)) { case 0: { if (!kcdbremove(db, kbuf, ksiz) && kcdbecode(db) != KCENOREC) { dberrprint(db, __LINE__, "kcdbremove"); err = TRUE; } break; } case 1: { if (!kccurjumpkey(paracur, kbuf, ksiz) && kccurecode(paracur) != KCENOREC) { dberrprint(db, __LINE__, "kccurjump"); err = TRUE; } break; } default: { if (!kccurstep(cur) && kccurecode(cur) != KCENOREC) { dberrprint(db, __LINE__, "kccurstep"); err = TRUE; } break; } } } } oprintf(" (end)\n"); kccurdel(paracur); kccurdel(cur); if (tran && !kcdbendtran(db, TRUE)) { dberrprint(db, __LINE__, "kcdbendtran"); err = TRUE; } if (!rnd && visarg.cnt != cnt) { dberrprint(db, __LINE__, "kccuraccept"); err = TRUE; } etime = kctime(); dbmetaprint(db, FALSE); oprintf("time: %.3f\n", etime - stime); } if (etc) { oprintf("synchronizing the database:\n"); stime = kctime(); if (!kcdbsync(db, FALSE, NULL, NULL)) { dberrprint(db, __LINE__, "kcdbsync"); err = TRUE; } if (!kcdboccupy(db, FALSE, NULL, NULL)) { dberrprint(db, __LINE__, "kcdboccupy"); err = TRUE; } etime = kctime(); dbmetaprint(db, FALSE); oprintf("time: %.3f\n", etime - stime); } if (etc) { corepath = kcdbpath(db); psiz = strlen(corepath); if (strstr(corepath, ".kch") || strstr(corepath, ".kct")) { copypath = kcmalloc(psiz + 256); sprintf(copypath, "%s.tmp", corepath); snappath = kcmalloc(psiz + 256); sprintf(snappath, "%s.kcss", corepath); } else { copypath = kcmalloc(256); sprintf(copypath, "kclangctest.tmp"); snappath = kcmalloc(256); sprintf(snappath, "kclangctest.kcss"); } oprintf("copying the database file:\n"); stime = kctime(); if (!kcdbcopy(db, copypath)) { dberrprint(db, __LINE__, "kcdbcopy"); err = TRUE; } etime = kctime(); dbmetaprint(db, FALSE); oprintf("time: %.3f\n", etime - stime); remove(copypath); oprintf("dumping records into snapshot:\n"); stime = kctime(); if (!kcdbdumpsnap(db, snappath)) { dberrprint(db, __LINE__, "kcdbdumpsnap"); err = TRUE; } etime = kctime(); dbmetaprint(db, FALSE); oprintf("time: %.3f\n", etime - stime); oprintf("loading records into snapshot:\n"); stime = kctime(); cnt = kcdbcount(db); if (rnd && myrand(2) == 0 && !kcdbclear(db)) { dberrprint(db, __LINE__, "kcdbclear"); err = TRUE; } if (!kcdbloadsnap(db, snappath) || kcdbcount(db) != cnt) { dberrprint(db, __LINE__, "kcdbloadsnap"); err = TRUE; } etime = kctime(); dbmetaprint(db, FALSE); oprintf("time: %.3f\n", etime - stime); remove(snappath); kcfree(copypath); kcfree(snappath); kcfree(corepath); } oprintf("removing records:\n"); stime = kctime(); for (i = 1; !err && i <= rnum; i++) { if (tran && !kcdbbegintran(db, FALSE)) { dberrprint(db, __LINE__, "kcdbbegintran"); err = TRUE; } ksiz = sprintf(kbuf, "%08ld", (long)(rnd ? myrand(rnum) + 1 : i)); if (!kcdbremove(db, kbuf, ksiz) && ((!rnd && !etc) || kcdbecode(db) != KCENOREC)) { dberrprint(db, __LINE__, "kcdbremove"); err = TRUE; } if (tran && !kcdbendtran(db, TRUE)) { dberrprint(db, __LINE__, "kcdbendtran"); err = TRUE; } if (rnum > 250 && i % (rnum / 250) == 0) { oputchar('.'); if (i == rnum || i % (rnum / 10) == 0) oprintf(" (%08ld)\n", (long)i); } } etime = kctime(); dbmetaprint(db, TRUE); oprintf("time: %.3f\n", etime - stime); oprintf("closing the database:\n"); stime = kctime(); if (!kcdbclose(db)) { dberrprint(db, __LINE__, "kcdbclose"); err = TRUE; } etime = kctime(); oprintf("time: %.3f\n", etime - stime); kcdbdel(db); oprintf("%s\n\n", err ? "error" : "ok"); return err ? 1 : 0; }
duc_errno db_put(struct db *db, const void *key, size_t key_len, const void *val, size_t val_len) { int r = kcdbset(db->kdb, key, key_len, val, val_len); return (r==1) ? DUC_OK : DUC_E_UNKNOWN; }
static void rdf_parser_statement_handler(void* user_data, const raptor_statement* st) { raptor_term *g, *s, *p, *o; g = st->graph; s = st->subject; p = st->predicate; o = st->object; rdf_parser_internal *parser_obj = (rdf_parser_internal *) user_data; if (parser_obj->counter == 0) parser_obj->partial_parse_time = g_timer_new(); parser_obj->counter++; /* init index logic */ unsigned char *gc = NULL; fs_rid g_rid; if (parser_obj->trig) { gc = raptor_uri_as_string(g->value.uri); g_rid = fs_hash_uri((const char *)gc); } else { g_rid = parser_obj->g_rid ; gc = parser_obj->model; } unsigned char *sc = NULL; if (s->type == RAPTOR_TERM_TYPE_URI) sc = raptor_uri_as_string(s->value.uri); else { sc = (unsigned char *) g_strdup_printf("bnode:b%s%s",s->value.blank.string+5,parser_obj->bnode_ts); } unsigned char *pc = raptor_uri_as_string(p->value.uri); unsigned char *oc = NULL; unsigned char *o_lang = NULL; unsigned char *o_datatype = NULL; fs_rid s_rid = fs_hash_uri((const char *) sc); fs_rid p_rid = fs_hash_uri((const char *) pc); fs_rid o_rid = 0x0; if (o->type == RAPTOR_TERM_TYPE_URI) { oc = raptor_uri_as_string(o->value.uri); o_rid = fs_hash_uri((const char *) oc); } else if (o->type == RAPTOR_TERM_TYPE_LITERAL) { oc = o->value.literal.string; if (o->value.literal.datatype) { o_datatype = raptor_uri_as_string(o->value.literal.datatype); o_rid = fs_hash_literal((const char *) oc,fs_hash_uri((const char *) o_datatype)); } else if (o->value.literal.language != NULL) { o_lang = o->value.literal.language; o_rid = fs_hash_literal((const char *) oc, fs_hash_uri((const char *) o_lang)); } else { o_rid = fs_hash_literal((const char *) oc, FS_RID_NULL); } oc = raptor_term_to_string(o); } else if (o->type == RAPTOR_TERM_TYPE_BLANK) { oc = (unsigned char *) g_strdup_printf("bnode:b%s%s",o->value.blank.string+5,parser_obj->bnode_ts); o_rid = fs_hash_uri((const char *) oc); } int seg_id = s_rid % SEGMENTS; fs_rid *quad = rdf_parser_new_quad(g_rid,s_rid,p_rid,o_rid); g_ptr_array_add(parser_obj->quads[seg_id],quad); /* saves hashes into disk hash TODO looks for optimistions */ rdf_kb *kb = parser_obj->kb; fs_rid hashes[4] = {g_rid,s_rid,p_rid,o_rid}; //log_debug("%llx %llx %llx %llx", hashes[0], hashes[1], hashes[2], hashes[3]); //char tmp_rid[16+1]; char tmp_rid[17]; unsigned char *strings[4] = {gc,sc,pc,oc}; int assigned_hash=0; for(int i=0;i<4;i++) { memset(tmp_rid,0,16); sprintf(tmp_rid,"%llx",hashes[i]); assigned_hash = hashes[i] % HASHES_NUM; //if (i > 0 && strlen(tmp_rid) < 15 ) //printf("ERRRRO NUL %s %s\n", tmp_rid, strings[i]); kcdbset(kb->hash_stores[assigned_hash],(const char *) tmp_rid, 16 ,(const char *) strings[i], strlen((const char *) strings[i])); } /* end of saving hashes into disk */ if (!(parser_obj->counter % STAT_BATCH)) { double kt = parser_obj->counter/1e3; log_debug("parsing progress %.0lf kT %.2lf kT/s %.2lf kT/s",kt,kt/g_timer_elapsed(parser_obj->global_parse_time,NULL), (STAT_BATCH/1e3)/g_timer_elapsed(parser_obj->partial_parse_time,NULL)); g_timer_start(parser_obj->partial_parse_time); } }