/* intialize evtimer */
EVTIMER *evtimer_init()
{
EVTIMER *evtimer = NULL;
if((evtimer = (EVTIMER *)xmm_new(sizeof(EVTIMER))))
{
MUTEX_INIT(evtimer->mutex);
}
return evtimer;
}
/* add event timer */
int evtimer_add(EVTIMER *evtimer, off_t timeout, EVTCALLBACK *handler, void *arg)
{
int evid = -1, x = 0, i = 0;
struct timeval tv = {0};
EVTNODE *node = NULL;
if(evtimer && handler && timeout > 0)
{
MUTEX_LOCK(evtimer->mutex);
if((node = evtimer->left))
{
evtimer->left = node->next;
}
else
{
if((x = evtimer->nevlist) < EVTNODE_LINE_MAX
&& (node = (EVTNODE *)xmm_new(sizeof(EVTNODE) * EVTNODE_LINE_NUM)))
{
evtimer->evlist[x] = node;
evtimer->nevlist++;
//fprintf(stdout, "%s::%d lines:%d\n", __FILE__, __LINE__, evtimer->nevlist);
x *= EVTNODE_LINE_NUM;
node[0].id = x;
i = 1;
while(i < EVTNODE_LINE_NUM)
{
node[i].id = i + x;
node[i].next = evtimer->left;
evtimer->left = &(node[i]);
++i;
}
}
}
if(node)
{
evid = node->id;
node->handler = handler;
node->arg = arg;
node->ison = 1;
gettimeofday(&tv, NULL);
node->evusec = (off_t)(tv.tv_sec * 1000000ll + tv.tv_usec * 1ll) + timeout;
evtimer_push(evtimer, node);
}
MUTEX_UNLOCK(evtimer->mutex);
}
return evid;
}
/* add index */
int ibase_index(IBASE *ibase, int docid, IBDATA *block)
{
char *term = NULL, buf[IB_BUF_SIZE], *data = NULL, *p = NULL,
*pp = NULL, *nexts = NULL;
int i = 0, termid = 0, n = 0, k = 0, ndocid = 0, ret = -1, ndata = 0, secid = 0,
*intlist = NULL, *np = NULL, last_docid = 0;
DOCHEADER *docheader = NULL;
int64_t *longlist = NULL;
double *doublelist = NULL;
IHEADER *iheader = NULL;
STERM *termlist = NULL;
MDB *index = NULL, *posting = NULL;
off_t size = 0;
if((docheader = (DOCHEADER *)block->data)
&& (secid = docheader->secid) >= 0 && docheader->secid < IB_SEC_MAX)
{
index = (MDB *)(ibase->mindex[docheader->secid]);
posting = (MDB *)(ibase->mposting[docheader->secid]);
if(docheader->dbid != -1)
{
ibase->state->dtotal++;
ibase->state->ttotal += (int64_t)docheader->terms_total;
}
if(ibase->state->used_for == IB_USED_FOR_INDEXD)
{
/* add to source */
if(((off_t)docid * (off_t)sizeof(IHEADER)) >= ibase->state->headers[secid].end)
{
ibase->state->headers[secid].old = ibase->state->headers[secid].end;
size = (off_t)((docid / IB_HEADERS_BASE) + 1)
* (off_t)IB_HEADERS_BASE * (off_t)sizeof(IHEADER);
ret = ftruncate(ibase->state->headers[secid].fd, size);
ibase->state->headers[secid].end = size;
memset(ibase->state->headers[secid].map + ibase->state->headers[secid].old, 0,
ibase->state->headers[secid].end - ibase->state->headers[secid].old);
}
}
/* index */
//end = block->data + block->ndata;
termlist = (STERM *)(block->data + sizeof(DOCHEADER)
+ docheader->nfields * sizeof(XFIELD));
term = block->data + docheader->textblock_off;
nexts = block->data + docheader->nexts_off;
for(i = 0; i < docheader->nterms; i++)
{
termid = termlist[i].termid;
//find/add termid
if(termid > 0 && termlist[i].term_len > 0 && termlist[i].nexts_size >= 0
&& (ret=mmtrie_add((MMTRIE *)ibase->mmtrie,term,termlist[i].term_len,termid))>0)
{
if(ibase->state->index_status != IB_INDEX_DISABLED)
{
last_docid = 0;
ndocid = -1;
if(mdb_get_tag(index, termid, &last_docid) == 0)
{
ndocid = docid - last_docid;
if(ndocid <= 0) goto term_state_update;
}
else
{
ndocid = docid;
}
p = pp = buf;
if((ndata = ((sizeof(int) * 5))) > IB_BUF_SIZE)
p = pp = data = (char *)xmm_new(ndata);
/* compress index */
if(ibase->state->compression_status != IB_COMPRESSION_DISABLED)
{
n = ndocid; np = &n; ZVBCODE(np, p);
n = termlist[i].term_count; np = &n; ZVBCODE(np, p);
n = i; np = &n; ZVBCODE(np, p);
n = termlist[i].bit_fields; np = &n; ZVBCODE(np, p);
n = termlist[i].nexts_size; np = &n; ZVBCODE(np, p);
}
else
{
memcpy(p, &docid, sizeof(int));p += sizeof(int);
memcpy(p, &(termlist[i].term_count), sizeof(int));p += sizeof(int);
memcpy(p, &i, sizeof(int));p += sizeof(int);
memcpy(p, &(termlist[i].bit_fields), sizeof(int));p += sizeof(int);
memcpy(p, &(termlist[i].nexts_size), sizeof(int));p += sizeof(int);
}
if(termlist[i].nexts_size > 0)
{
if(mdb_add_data(posting, termid, nexts, termlist[i].nexts_size) <= 0)
{
FATAL_LOGGER(ibase->logger, "index posting term[%d] failed, %s",
termid, strerror(errno));
_exit(-1);
}
//WARN_LOGGER(ibase->logger, "docid:%lld termid:%d nexts_size:%d", IBLL(docheader->globalid), termid, termlist[i].nexts_size);
}
if(mdb_add_data(index, termid, pp, (p - pp)) <= 0)
{
FATAL_LOGGER(ibase->logger, "index term[%d] failed, %s",
termid, strerror(errno));
if(data){free(data); data = NULL;}
_exit(-1);
}
mdb_set_tag(index, termid, docid);
if(data){xmm_free(data, ndata); data = NULL;}
}
term_state_update:
MUTEX_LOCK(ibase->mutex_termstate);
if(termid > ibase->state->termid){ADD_TERMSTATE(ibase, termid);}
if(ndocid > 0 )
{
((TERMSTATE *)(ibase->termstateio.map))[termid].len=termlist[i].term_len;
((TERMSTATE *)(ibase->termstateio.map))[termid].total++;
}
MUTEX_UNLOCK(ibase->mutex_termstate);
}
else
{
FATAL_LOGGER(ibase->logger, "Invalid ret:%d term[%d]{%.*s} termid:%d in doc:%d", ret, i, termlist[i].term_len, term, termid, docid);
_exit(-1);
}
term += termlist[i].term_len;
nexts += termlist[i].nexts_size;
}
if(ibase->state->used_for == IB_USED_FOR_INDEXD)
{
/* index int */
if((n = ibase->state->int_index_fields_num) > 0
&& (intlist = (int *)(block->data + docheader->intblock_off)))
{
n += IB_INT_OFF;
k = 0;
for(i = IB_INT_OFF; i < n; i++)
{
IMAP_SET(ibase->state->mfields[secid][i], docid, intlist[k]);
k++;
}
}
/* index long */
if((n = ibase->state->long_index_fields_num) > 0
&& (longlist = (int64_t *)(block->data + docheader->longblock_off)))
{
n += IB_LONG_OFF;
k = 0;
for(i = IB_LONG_OFF; i < n; i++)
{
LMAP_SET(ibase->state->mfields[secid][i], docid, longlist[k]);
k++;
}
}
/* index double */
if((n = ibase->state->double_index_fields_num) > 0
&& (doublelist = (double *)(block->data + docheader->doubleblock_off)))
{
n += IB_DOUBLE_OFF;
k = 0;
for(i = IB_DOUBLE_OFF; i < n; i++)
{
DMAP_SET(ibase->state->mfields[secid][i], docid, doublelist[k]);
k++;
}
}
if((iheader = PIHEADER(ibase, docheader->secid, docid)))
{
iheader->status = docheader->status;
iheader->terms_total = docheader->terms_total;
iheader->crc = docheader->crc;
iheader->category = docheader->category;
iheader->slevel = docheader->slevel;
iheader->rank = docheader->rank;
iheader->globalid = docheader->globalid;
//WARN_LOGGER(ibase->logger, "iheader->category:%p docheader->category:%p", (void *)iheader->category, (void *)docheader->category);
}
}
ret = 0;
}
return ret;
}
/* initialize chunk */
CHUNK *chunk_init()
{
CHUNK *chunk = NULL;
chunk = (CHUNK *)xmm_new(sizeof(CHUNK));
return chunk;
}
