void
csp_id_set_build(struct csp_id_set *set, struct csp_id_set_builder *builder)
{
UNNEEDED Word_t dummy;
csp_id_set_build_and_keep(set, builder);
J1FA(dummy, builder->working_set);
builder->hash = CSP_ID_SET_INITIAL_HASH;
}
void free_p_entries(my_int len_sa)
{
my_int i;
Word_t Bytes;
idx=0;
JLF(pval,judy,idx);
while (1)
{
if (pval==NULL) break;
J1FA(Bytes,*pval);
JLN(pval,judy,idx);
}
JLFA(Bytes,judy);
}
/**
* Frees the Judy arrays that make up the index and gathers some basic information about them.
*/
void freeIndex (Pvoid_t *wikiIndex) {
Word_t returnCode = 0;
Word_t totalIndexes = 0;
Word_t indexes = 0;
Word_t totalSize = 0;
Word_t size = 0;
// frees the judy arrays in the index and adds the total indexes and the size of each judy array
// one Judy array per iteration
for (int currentNgram = 0; currentNgram < lastNgram; currentNgram++) {
J1C (indexes, wikiIndex[currentNgram], 0, -1); //count indices
totalIndexes = totalIndexes + indexes;
J1MU (size, wikiIndex[currentNgram]); // memory usage
totalSize = totalSize + size;
J1FA (returnCode, wikiIndex[currentNgram]); //free
}
optionalPrint("%d %s", (int)(totalSize/1048576), "MB of memory used\n");
optionalPrint("%d", (int)((totalIndexes*4)/1048576));
optionalPrint(" MB if not compressed\n");
optionalPrint("Index freed.\n");
}
void rb_refresh()
{
J1FA(idxrb,judyrb);
}
void
p4_pd_cleanup(void) {
Word_t Rc_word;
J1FA(Rc_word, used_session_handles);
}
int main(int argc, char **argv)
{
char *basename = NULL;
char *tname = NULL;
char *dname = NULL;
char *pname = NULL;
char *layers_str;
FILE *part_f;
uint iblock = 0;
dub_init();
PPARM_INT_D(iblock, IBLOCK);
PPARM_STR(layers_str, LAYERS);
glist *layers = str2glist_len(&layers_str, NUM_FW_LAYERS - 1);
open_section(FW, iblock);
open_section(INVA, iblock);
basename = pparm_common_name("fwlayers");
asprintf(&dname, "%s.%u.sect", basename, iblock);
asprintf(&tname, "%s.%u.toc.sect", basename, iblock);
asprintf(&pname, "%s.%u.nfo", basename, iblock);
free(basename);
if (!(data_f = fopen64(dname, "w+")))
dub_sysdie("Couldn't open section %s", dname);
if (!(toc_f = fopen64(tname, "w+")))
dub_sysdie("Couldn't open section %s", tname);
if (!(part_f = fopen64(pname, "w+")))
dub_sysdie("Couldn't open section %s", tname);
if (setvbuf(data_f, NULL, _IOFBF, IO_BUF_SIZE))
dub_sysdie("Couldn't open input buffer");
if (setvbuf(toc_f, NULL, _IOFBF, IO_BUF_SIZE))
dub_sysdie("Couldn't open input buffer");
u32 num_entries = 0;
fw_layers[0].idx = num_entries;
int p, no, prev_p = 0, prev_offs = 0;
for (p = 0; p < NUM_FW_LAYERS - 1; p++){
dub_msg("Layer %u", p);
Pvoid_t ix = ixemes_freq_range(p, prev_p, layers->lst[p]);
prev_p = layers->lst[p];
num_entries += encode_layer(ix);
dub_msg("LAYER %u NUM %u", p, num_entries);
fw_layers[p + 1].idx = num_entries;
fw_layers[p + 1].max_freq = layers->lst[p];
u32 offs = (u32)ftello64(data_f);
dub_msg("Layer takes %u bytes", offs - prev_offs);
prev_offs = offs;
J1FA(no, ix);
}
Pvoid_t ix = ixemes_freq_range(p, prev_p, 1 << 31);
num_entries += encode_layer(ix);
fw_layers[p + 1].idx = num_entries;
fw_layers[p + 1].max_freq = 1 << 31;
u32 offs = (u32)ftello64(data_f);
dub_msg("Layer takes %u bytes", offs - prev_offs);
fwrite(fw_layers, sizeof(fw_layers), 1, part_f);
return 0;
}
int jtableP_free(jtableP *table) {
int ret;
J1FA(ret, table->t);
return ret;
}
/* index, called from absmi.c
*
* Returns:
* NULL (yap fallback) No usable indexing available
*
* Yap_FAILCODE() (fail) No result found
* Yap_CauseListToClause(cl) 1 solution found
* Yap_ClauseListCode(cl) 2+ solutions found
*/
yamop *
Yap_udi_search(PredEntry *p)
{
int r;
struct ClauseList clauselist;
UdiPArg parg;
UdiInfo info;
/* find our structure*/
HASH_FIND_UdiInfo(UdiControlBlocks,p,info);
if (!info || utarray_len(info->args) == 0)
return NULL;
if (utarray_len(info->args) == 1){ //simple case no intersection needed
struct si_callback_h c;
c.cl = Yap_ClauseListInit(&clauselist);
c.clauselist = info->clauselist;
c.pred = info->p;
if (!c.cl)
return NULL;
parg = (UdiPArg) utarray_eltptr(info->args,0);
r = parg->control->search(parg->idxstr, parg->arg, si_callback, (void *) &c);
Yap_ClauseListClose(c.cl);
if (r == -1) {
Yap_ClauseListDestroy(c.cl);
return NULL;
}
if (Yap_ClauseListCount(c.cl) == 0) {
Yap_ClauseListDestroy(c.cl);
return Yap_FAILCODE();
}
} else {//intersection needed using Judy1
#ifdef USE_JUDY
/*TODO: do more tests to this algorithm*/
int i;
Pvoid_t tmp = (Pvoid_t) NULL;
Pvoid_t result = (Pvoid_t) NULL;
Word_t count = 0L;
Word_t idx_r = 0L;
Word_t idx_tmp = 0L;
int rc = 0;
yamop **x;
/*
* I will start with the simplest approach
* for each index create a set and intersect it with the
* next
*
* In the future it could pay to sort according to index type
* to improve intersection part
*/
for (i = 0; i < utarray_len(info->args) ; i++) {
parg = (UdiPArg) utarray_eltptr(info->args,i);
r = parg->control->search(parg->idxstr, parg->arg, j1_callback, &tmp);
if (r == -1) /*this arg does not prune search*/
continue;
rc ++;
J1C(count, result, 0, -1);
if (r == 0) /* this arg gave 0 results -> FAIL*/
{
if (count > 0) // clear previous result if they exists
J1FA(count, result);
return Yap_FAILCODE();
}
if (count == 0) // first result_set
{
result = tmp;
tmp = (Pvoid_t) NULL;
}
else /*intersection*/
{
idx_tmp = 0L;
idx_r = 0L;
J1F(count, result, idx_r); //succeeds one time at least
assert(count > 0);
J1F(count, tmp, idx_tmp); //succeeds one time at least
assert(count > 0);
while (count)
{
while (idx_r < idx_tmp)
{
J1U(count, result, idx_r); //does not belong
J1N(count, result, idx_r); //next
if (! count) break; //end result set
}
if(idx_r == idx_tmp)
{
J1N(count, result, idx_r); //next
if (! count) break; //end result set
J1N(count, tmp, idx_tmp); //next tmp
//if (! count) break; //end tmp set will break while
}
else // (idx_r > idx_tmp)
{
idx_tmp = idx_r; // fast forward
J1F(count, tmp, idx_tmp); // first starting in idx_r
//if (! count) break; //end tmp set will break while
}
}
J1F(count, result, idx_r); // first starting in idx_r
//clear up the rest
while (idx_r > idx_tmp && count) //result has more setted values
{
J1U(count, result, idx_r); //does not belong
J1N(count, result, idx_r); //next
}
J1FA(count, tmp); //free tmp
}
}
if (rc == 0) /*no search performed*/
return NULL;
J1C(count, result, 0, -1);
if (count == 0) { /*result set empty -> FAIL */
J1FA(count, result);
return Yap_FAILCODE();
}
/*convert Juddy1 to clauselist*/
Yap_ClauseListInit(&clauselist);
idx_r = 0L;
J1F(count, result, idx_r);
while (count)
{
x = (yamop **) utarray_eltptr(info->clauselist, idx_r - 1);
Yap_ClauseListExtend(
&clauselist,
*x,
info->p);
J1N(count, result, idx_r);
}
J1FA(count,result);
fprintf(stderr,"J1 used space %ld bytes for %d clausules\n",
count, Yap_ClauseListCount(&clauselist));
Yap_ClauseListClose(&clauselist);
#else
fprintf(stderr,"Without libJudy only one argument indexed is allowed."
"Falling back to Yap Indexing\n");
return NULL; //NO Judy Available
#endif
}
if (Yap_ClauseListCount(&clauselist) == 1)
return Yap_ClauseListToClause(&clauselist);
return Yap_ClauseListCode(&clauselist);
}
int main(int argc, char **argv)
{
uint r = ROUNDS;
uint only_crea = 0;
uint do_judy = 0;
uint seed = 0;
float scaler = 0;
uint sum = 0;
uint false_hits = 0;
uint nof_false = 0;
Pvoid_t judy = NULL;
dub_init();
PPARM_INT(seed, SEED);
PPARM_FLOAT(scaler, SCALER);
/* allow at most one false hit in 10^7 queries */
init_bloom(7, scaler);
if (getenv("ONLY_CREATE")){
dub_msg("Only encoding %u lists", r);
only_crea = 1;
}else
dub_msg("Encoding %u lists and testing %u items per list",
r, TESTMAX);
if (!only_crea && getenv("FALSE_HITS")){
dub_msg("Checking for false hits");
false_hits = 1;
}else if (getenv("JUDY")){
dub_msg("Using Judy");
do_judy = 1;
}
srand(seed);
while (r--){
uint sze = MAXSZE * (rand() / (RAND_MAX + 1.0));
u32 *p = xmalloc(sze * 4);
bloom_s *b = NULL;
uint j, k;
for (j = 0; j < sze; j++)
p[j] = 1000 * (rand() / (RAND_MAX + 1.0));
if (do_judy || false_hits){
uint tmp;
for (j = 0; j < sze; j++)
J1S(tmp, judy, p[j]);
}
if (!do_judy)
b = new_bloom(p, sze);
sum += sze;
if (only_crea)
goto next;
for (j = 1; j < TESTMAX; j++){
if (false_hits){
J1T(k, judy, j);
if (bloom_test(b, j)){
if (!k) ++nof_false;
}else
if (k){
print(p, sze);
dub_die("False negative! "
"Value %u not in bloom. "
"Bloom broken!", j);
}
}else if (do_judy){
J1T(k, judy, j);
}else
bloom_test(b, j);
}
next:
if (do_judy || false_hits){
J1FA(k, judy);
}
if (!do_judy){
free(b);
free(p);
}
}
dub_msg("In total %u items in lists", sum);
if (false_hits)
dub_msg("In total %u false hits found", nof_false);
return 0;
}
void
csp_id_set_builder_done(struct csp_id_set_builder *builder)
{
UNNEEDED Word_t dummy;
J1FA(dummy, builder->working_set);
}