void xqueue_clean(void *xqueue)
{
XQUEUE *q = (XQUEUE *)xqueue;
XQNODE *node = NULL;
int i = 0;
if(q)
{
//fprintf(stdout, "%s::%d q:%p nleft:%d qtotal:%d qleft:%p\n", __FILE__, __LINE__, q, q->nleft, q->qtotal, q->left);
for(i = 0; i < q->nlist; i++);
{
if(q->list[i])
xmm_free(q->list[i], XQ_NODES_MAX * sizeof(XQNODE));
}
/*
while((node = q->left))
{
q->left = node->next;
xmm_free(node, sizeof(XQNODE));
}
*/
MUTEX_DESTROY(q->mutex);
xmm_free(q, sizeof(XQUEUE));
}
return ;
}
/* clean chunk */
void chunk_clean(void *chunk)
{
if(chunk)
{
if(CHK(chunk)->mmap) munmap(CHK(chunk)->mmap, MMAP_CHUNK_SIZE);
xmm_free(CHK(chunk)->data, CHK(chunk)->bsize);
if(CHK(chunk)->fd > 0) close(CHK(chunk)->fd);
xmm_free(chunk, sizeof(CHUNK));
}
return ;
}
/* clean chunk */
void chunk_clean(void *chunk)
{
if(chunk)
{
if(CHK(chunk)->mmap && !CHK(chunk)->flag) munmap(CHK(chunk)->mmap, MMAP_CHUNK_SIZE);
CHK(chunk)->mmap = NULL;
if(CHK(chunk)->data && !CHK(chunk)->flag) xmm_free(CHK(chunk)->data, CHK(chunk)->bsize);
CHK(chunk)->data = NULL;
if(CHK(chunk)->fd > 0 && !CHK(chunk)->flag) close(CHK(chunk)->fd);
CHK(chunk)->fd = 0;
xmm_free(chunk, sizeof(CHUNK));
}
return ;
}
/* chunk reset */
void chunk_reset(void *chunk)
{
if(chunk)
{
if(CHK(chunk)->mmap && !CHK(chunk)->flag) munmap(CHK(chunk)->mmap, MMAP_CHUNK_SIZE);
CHK(chunk)->mmap = NULL;
CHK(chunk)->mmleft = 0;
if(CHK(chunk)->fd > 0 && !CHK(chunk)->flag) close(CHK(chunk)->fd);
CHK(chunk)->fd = 0;
CHK(chunk)->status = 0;
CHK(chunk)->type = 0;
CHK(chunk)->ndata = 0;
CHK(chunk)->offset = 0;
CHK(chunk)->left = 0;
CHK(chunk)->mmoff = 0;
CHK(chunk)->mmleft = 0;
if(CHK(chunk)->flag)
{
CHK(chunk)->data = NULL;
CHK(chunk)->bsize = 0;
}
if(CHK(chunk)->bsize > CHUNK_BLOCK_MAX)
{
xmm_free(CHK(chunk)->data, CHK(chunk)->bsize);
CHK(chunk)->data = NULL;
CHK(chunk)->bsize = 0;
}
//if(CHK(chunk)->data) memset(CHK(chunk)->data, 0, CHK(chunk)->bsize);
CHK(chunk)->end = CHK(chunk)->data;
}
return ;
}
void iqueue_clean(void *queue)
{
QUEUE *q = (QUEUE *)queue;
int i = 0;
if(q)
{
for(i = 0; i < q->nlist; i++);
{
if(q->list[i])xmm_free(q->list[i], QNODE_LINE_NUM * sizeof(QNODE));
}
MUTEX_DESTROY(q->mutex);
xmm_free(q, sizeof(QUEUE));
}
return ;
}
/* clean evtimer */
void evtimer_clean(EVTIMER *evtimer)
{
int i = 0;
if(evtimer)
{
MUTEX_DESTROY(evtimer->mutex);
for(i = 0; i < evtimer->nevlist; i++)
{
xmm_free(evtimer->evlist[i], sizeof(EVTNODE) * EVTNODE_LINE_NUM);
}
xmm_free(evtimer, sizeof(EVTIMER));
}
return ;
}
/* clean */
void mmqueue_clean(MMQUEUE *mmq)
{
if(mmq)
{
MUTEX_DESTROY(mmq->mutex);
if(mmq->map) munmap(mmq->map, mmq->end);
if(mmq->fd) close(mmq->fd);
xmm_free(mmq, sizeof(MMQUEUE));
}
return ;
}
/* clean xmap */
void xmap_clean(XMAP *xmap)
{
if(xmap)
{
//WARN_LOGGER(xmap->logger, "Ready clean tree[%p]", xmap->tree);
mmtree_close(xmap->tree);
//WARN_LOGGER(xmap->logger, "Ready clean tree64[%p]", xmap->tree64);
mmtree64_close(xmap->tree64);
//WARN_LOGGER(xmap->logger, "Ready clean queue[%p]", xmap->queue);
mmqueue_clean(xmap->queue);
//WARN_LOGGER(xmap->logger, "Ready clean kmap[%p]", xmap->kmap);
mmtrie_clean(xmap->kmap);
//WARN_LOGGER(xmap->logger, "Ready reset db[%p]", xmap->db);
cdb_reset(xmap->db);
//WARN_LOGGER(xmap->logger, "Ready clean db[%p]", xmap->db);
cdb_clean(xmap->db);
if(xmap->diskio.map)
{
munmap(xmap->diskio.map, xmap->diskio.size);
xmap->diskio.map = NULL;
}
if(xmap->diskio.fd > 0)
{
close(xmap->diskio.fd);
xmap->diskio.fd = 0;
}
if(xmap->metaio.map)
{
munmap(xmap->metaio.map, xmap->metaio.size);
xmap->metaio.map = NULL;
}
if(xmap->metaio.fd > 0)
{
close(xmap->metaio.fd);
xmap->metaio.fd = 0;
}
if(xmap->stateio.map)
{
munmap(xmap->stateio.map, xmap->stateio.size);
xmap->stateio.map = NULL;
}
if(xmap->stateio.fd > 0)
{
close(xmap->stateio.fd);
xmap->stateio.fd = 0;
}
//WARN_LOGGER(xmap->logger, "Ready clean mutex[%p]", xmap->mutex);
MUTEX_DESTROY(xmap->mutex);
MUTEX_DESTROY(xmap->cmutex);
LOGGER_CLEAN(xmap->logger);
xmm_free(xmap, sizeof(XMAP));
}
return ;
}
/* clean hindex */
void mindex_clean(MINDEX *mindex)
{
if(mindex)
{
mtask_close(&(mindex->mtask));
dbase_close(&(mindex->db));
bjson_clean(&(mindex->request));
xmm_free(mindex, sizeof(MINDEX));
}
return ;
}
/* Clean evbase */
void evselect_clean(EVBASE *evbase)
{
if(evbase)
{
//MUTEX_DESTROY(evbase->mutex);
if(evbase->logger)LOGGER_CLEAN(evbase->logger);
if(evbase->evlist)free(evbase->evlist);
if(evbase->evs)free(evbase->evs);
if(evbase->ev_fds)free(evbase->ev_fds);
if(evbase->ev_read_fds)free(evbase->ev_read_fds);
if(evbase->ev_write_fds)free(evbase->ev_write_fds);
xmm_free(evbase, sizeof(EVBASE));
}
return ;
}
/*
* close/clean wtable
* */
void wtable_close(WTABLE *w)
{
if(w)
{
if(w->statefd > 0) close(w->statefd);
w->statefd = 0;
if(w->state) munmap(w->state, sizeof(WSTATE));
w->state = NULL;
db_clean(w->mdb);
mqueue_clean(MQ(w->queue));
mmtrie_clean(w->map);
mtree_close(w->mtree);
mmtree64_close(w->appmap);
LOGGER_CLEAN(w->logger);
xmm_free(w, sizeof(WTABLE));
}
return ;
}
/* clean procthread */
void procthread_clean(PROCTHREAD *pth)
{
if(pth)
{
if(pth->service->working_mode != WORKING_PROC)
{
if(pth->have_evbase)
{
event_destroy(&(pth->event));
if(pth->evbase) pth->evbase->clean(pth->evbase);
}
qmessage_clean(pth->message_queue);
}
if(pth->service->flag & SB_USE_EVSIG)
evsig_close(&(pth->evsig));
MUTEX_DESTROY(pth->mutex);
xmm_free(pth, sizeof(PROCTHREAD));
}
return ;
}
/* clean */
void qindex_clean(QINDEX *qindex)
{
if(qindex)
{
if(qindex->state) munmap(qindex->state, sizeof(QSTATE));
if(qindex->qstatefd > 0)close(qindex->qstatefd);
if(qindex->db) db_clean(PDB(qindex->db));
if(qindex->update) db_clean(PDB(qindex->update));
MUTEX_DESTROY(qindex->mutex);
if(qindex->xdict){mmtrie_clean(qindex->xdict);}
if(qindex->namemap){mmtrie_clean(qindex->namemap);}
if(qindex->idmap){mmtree64_close(qindex->idmap);}
if(qindex->queue){iqueue_clean(qindex->queue);}
if(qindex->mmqueue){mmqueue_clean(MMQ(qindex->mmqueue));}
if(qindex->xpacketio.map) munmap(qindex->xpacketio.map, qindex->xpacketio.size);
if(qindex->xpacketio.fd) close(qindex->xpacketio.fd);
if(qindex->bstermio.map) munmap(qindex->bstermio.map, qindex->bstermio.size);
if(qindex->bstermio.fd) close(qindex->bstermio.fd);
if(qindex->map) mmtrie_clean(qindex->map);
if(qindex->logger){LOGGER_CLEAN(qindex->logger);}
xmm_free(qindex, sizeof(QINDEX));
}
return ;
}
/* add index */
int ibase_index(IBASE *ibase, int docid, IBDATA *block)
{
char *term = NULL, buf[IB_BUF_SIZE], *data = NULL, *p = NULL,
*pp = NULL, *nexts = NULL;
int i = 0, termid = 0, n = 0, k = 0, ndocid = 0, ret = -1, ndata = 0, secid = 0,
*intlist = NULL, *np = NULL, last_docid = 0;
DOCHEADER *docheader = NULL;
int64_t *longlist = NULL;
double *doublelist = NULL;
IHEADER *iheader = NULL;
STERM *termlist = NULL;
MDB *index = NULL, *posting = NULL;
off_t size = 0;
if((docheader = (DOCHEADER *)block->data)
&& (secid = docheader->secid) >= 0 && docheader->secid < IB_SEC_MAX)
{
index = (MDB *)(ibase->mindex[docheader->secid]);
posting = (MDB *)(ibase->mposting[docheader->secid]);
if(docheader->dbid != -1)
{
ibase->state->dtotal++;
ibase->state->ttotal += (int64_t)docheader->terms_total;
}
if(ibase->state->used_for == IB_USED_FOR_INDEXD)
{
/* add to source */
if(((off_t)docid * (off_t)sizeof(IHEADER)) >= ibase->state->headers[secid].end)
{
ibase->state->headers[secid].old = ibase->state->headers[secid].end;
size = (off_t)((docid / IB_HEADERS_BASE) + 1)
* (off_t)IB_HEADERS_BASE * (off_t)sizeof(IHEADER);
ret = ftruncate(ibase->state->headers[secid].fd, size);
ibase->state->headers[secid].end = size;
memset(ibase->state->headers[secid].map + ibase->state->headers[secid].old, 0,
ibase->state->headers[secid].end - ibase->state->headers[secid].old);
}
}
/* index */
//end = block->data + block->ndata;
termlist = (STERM *)(block->data + sizeof(DOCHEADER)
+ docheader->nfields * sizeof(XFIELD));
term = block->data + docheader->textblock_off;
nexts = block->data + docheader->nexts_off;
for(i = 0; i < docheader->nterms; i++)
{
termid = termlist[i].termid;
//find/add termid
if(termid > 0 && termlist[i].term_len > 0 && termlist[i].nexts_size >= 0
&& (ret=mmtrie_add((MMTRIE *)ibase->mmtrie,term,termlist[i].term_len,termid))>0)
{
if(ibase->state->index_status != IB_INDEX_DISABLED)
{
last_docid = 0;
ndocid = -1;
if(mdb_get_tag(index, termid, &last_docid) == 0)
{
ndocid = docid - last_docid;
if(ndocid <= 0) goto term_state_update;
}
else
{
ndocid = docid;
}
p = pp = buf;
if((ndata = ((sizeof(int) * 5))) > IB_BUF_SIZE)
p = pp = data = (char *)xmm_new(ndata);
/* compress index */
if(ibase->state->compression_status != IB_COMPRESSION_DISABLED)
{
n = ndocid; np = &n; ZVBCODE(np, p);
n = termlist[i].term_count; np = &n; ZVBCODE(np, p);
n = i; np = &n; ZVBCODE(np, p);
n = termlist[i].bit_fields; np = &n; ZVBCODE(np, p);
n = termlist[i].nexts_size; np = &n; ZVBCODE(np, p);
}
else
{
memcpy(p, &docid, sizeof(int));p += sizeof(int);
memcpy(p, &(termlist[i].term_count), sizeof(int));p += sizeof(int);
memcpy(p, &i, sizeof(int));p += sizeof(int);
memcpy(p, &(termlist[i].bit_fields), sizeof(int));p += sizeof(int);
memcpy(p, &(termlist[i].nexts_size), sizeof(int));p += sizeof(int);
}
if(termlist[i].nexts_size > 0)
{
if(mdb_add_data(posting, termid, nexts, termlist[i].nexts_size) <= 0)
{
FATAL_LOGGER(ibase->logger, "index posting term[%d] failed, %s",
termid, strerror(errno));
_exit(-1);
}
//WARN_LOGGER(ibase->logger, "docid:%lld termid:%d nexts_size:%d", IBLL(docheader->globalid), termid, termlist[i].nexts_size);
}
if(mdb_add_data(index, termid, pp, (p - pp)) <= 0)
{
FATAL_LOGGER(ibase->logger, "index term[%d] failed, %s",
termid, strerror(errno));
if(data){free(data); data = NULL;}
_exit(-1);
}
mdb_set_tag(index, termid, docid);
if(data){xmm_free(data, ndata); data = NULL;}
}
term_state_update:
MUTEX_LOCK(ibase->mutex_termstate);
if(termid > ibase->state->termid){ADD_TERMSTATE(ibase, termid);}
if(ndocid > 0 )
{
((TERMSTATE *)(ibase->termstateio.map))[termid].len=termlist[i].term_len;
((TERMSTATE *)(ibase->termstateio.map))[termid].total++;
}
MUTEX_UNLOCK(ibase->mutex_termstate);
}
else
{
FATAL_LOGGER(ibase->logger, "Invalid ret:%d term[%d]{%.*s} termid:%d in doc:%d", ret, i, termlist[i].term_len, term, termid, docid);
_exit(-1);
}
term += termlist[i].term_len;
nexts += termlist[i].nexts_size;
}
if(ibase->state->used_for == IB_USED_FOR_INDEXD)
{
/* index int */
if((n = ibase->state->int_index_fields_num) > 0
&& (intlist = (int *)(block->data + docheader->intblock_off)))
{
n += IB_INT_OFF;
k = 0;
for(i = IB_INT_OFF; i < n; i++)
{
IMAP_SET(ibase->state->mfields[secid][i], docid, intlist[k]);
k++;
}
}
/* index long */
if((n = ibase->state->long_index_fields_num) > 0
&& (longlist = (int64_t *)(block->data + docheader->longblock_off)))
{
n += IB_LONG_OFF;
k = 0;
for(i = IB_LONG_OFF; i < n; i++)
{
LMAP_SET(ibase->state->mfields[secid][i], docid, longlist[k]);
k++;
}
}
/* index double */
if((n = ibase->state->double_index_fields_num) > 0
&& (doublelist = (double *)(block->data + docheader->doubleblock_off)))
{
n += IB_DOUBLE_OFF;
k = 0;
for(i = IB_DOUBLE_OFF; i < n; i++)
{
DMAP_SET(ibase->state->mfields[secid][i], docid, doublelist[k]);
k++;
}
}
if((iheader = PIHEADER(ibase, docheader->secid, docid)))
{
iheader->status = docheader->status;
iheader->terms_total = docheader->terms_total;
iheader->crc = docheader->crc;
iheader->category = docheader->category;
iheader->slevel = docheader->slevel;
iheader->rank = docheader->rank;
iheader->globalid = docheader->globalid;
//WARN_LOGGER(ibase->logger, "iheader->category:%p docheader->category:%p", (void *)iheader->category, (void *)docheader->category);
}
}
ret = 0;
}
return ret;
}
/* Initialize evbase */
EVBASE *evbase_init()
{
EVBASE *evbase = NULL;
if((evbase = (EVBASE *)xmm_mnew(sizeof(EVBASE))))
{
MUTEX_RESET(evbase->mutex);
#ifdef HAVE_EVPORT
evops_default = EOP_PORT;
evops[EOP_PORT].name = "PORT";
evops[EOP_PORT].init = evport_init;
evops[EOP_PORT].add = evport_add;
evops[EOP_PORT].update = evport_update;
evops[EOP_PORT].del = evport_del;
evops[EOP_PORT].loop = evport_loop;
evops[EOP_PORT].reset = evport_reset;
evops[EOP_PORT].clean = evport_clean;
#endif
#ifdef HAVE_EVSELECT
evops_default = EOP_SELECT;
evops[EOP_SELECT].name = "SELECT";
evops[EOP_SELECT].init = evselect_init;
evops[EOP_SELECT].add = evselect_add;
evops[EOP_SELECT].update = evselect_update;
evops[EOP_SELECT].del = evselect_del;
evops[EOP_SELECT].loop = evselect_loop;
evops[EOP_SELECT].reset = evselect_reset;
evops[EOP_SELECT].clean = evselect_clean;
#endif
#ifdef HAVE_EVPOLL
evops_default = EOP_POLL;
evops[EOP_POLL].name = "POLL";
evops[EOP_POLL].init = evpoll_init;
evops[EOP_POLL].add = evpoll_add;
evops[EOP_POLL].update = evpoll_update;
evops[EOP_POLL].del = evpoll_del;
evops[EOP_POLL].loop = evpoll_loop;
evops[EOP_POLL].reset = evpoll_reset;
evops[EOP_POLL].clean = evpoll_clean;
#endif
#ifdef HAVE_EVRTSIG
evops_default = EOP_RTSIG;
evops[EOP_RTSIG].name = "RTSIG" ;
evops[EOP_RTSIG].init = evrtsig_init;
evops[EOP_RTSIG].add = evrtsig_add;
evops[EOP_RTSIG].update = evrtsig_update;
evops[EOP_RTSIG].del = evrtsig_del;
evops[EOP_RTSIG].loop = evrtsig_loop;
evops[EOP_RTSIG].reset = evrtsig_reset;
evops[EOP_RTSIG].clean = evrtsig_clean;
#endif
#ifdef HAVE_EVEPOLL
evops_default = EOP_EPOLL;
evops[EOP_EPOLL].name = "EPOLL";
evops[EOP_EPOLL].init = evepoll_init;
evops[EOP_EPOLL].add = evepoll_add;
evops[EOP_EPOLL].update = evepoll_update;
evops[EOP_EPOLL].del = evepoll_del;
evops[EOP_EPOLL].loop = evepoll_loop;
evops[EOP_EPOLL].reset = evepoll_reset;
evops[EOP_EPOLL].clean = evepoll_clean;
#endif
#ifdef HAVE_EVKQUEUE
evops_default = EOP_KQUEUE;
evops[EOP_KQUEUE].name = "KQUEUE";
evops[EOP_KQUEUE].init = evkqueue_init;
evops[EOP_KQUEUE].add = evkqueue_add;
evops[EOP_KQUEUE].update = evkqueue_update;
evops[EOP_KQUEUE].del = evkqueue_del;
evops[EOP_KQUEUE].loop = evkqueue_loop;
evops[EOP_KQUEUE].reset = evkqueue_reset;
evops[EOP_KQUEUE].clean = evkqueue_clean;
#endif
#ifdef HAVE_EVDEVPOLL
evops_default = EOP_DEVPOLL;
evops[EOP_DEVPOLL].name = "/dev/poll";
evops[EOP_DEVPOLL].init = evdevpoll_init;
evops[EOP_DEVPOLL].add = evdevpoll_add;
evops[EOP_DEVPOLL].update = evdevpoll_update;
evops[EOP_DEVPOLL].del = evdevpoll_del;
evops[EOP_DEVPOLL].loop = evdevpoll_loop;
evops[EOP_DEVPOLL].reset = evdevpoll_reset;
evops[EOP_DEVPOLL].clean = evdevpoll_clean;
#endif
#ifdef WIN32
evops_default = EOP_WIN32;
evops[EOP_WIN32].name = "WIN32";
evops[EOP_WIN32].init = evwin32_init;
evops[EOP_WIN32].add = evwin32_add;
evops[EOP_WIN32].update = evwin32_update;
evops[EOP_WIN32].del = evwin32_del;
evops[EOP_WIN32].loop = evwin32_loop;
evops[EOP_WIN32].reset = evwin32_reset;
evops[EOP_WIN32].clean = evwin32_clean;
#endif
evbase->set_logfile = evbase_set_logfile;
evbase->set_log_level = evbase_set_log_level;
evbase->set_evops = evbase_set_evops;
//evbase->clean = evbase_clean;
if(evops_default == -1 || evbase->set_evops(evbase, evops_default) == -1)
{
xmm_free(evbase, sizeof(EVBASE));
fprintf(stderr, "Initialize evbase to default[%d] failed, %s\n",
evops_default, strerror(errno));
evbase = NULL;
}
}
return evbase;
}
