void xqueue_clean(void *xqueue) { XQUEUE *q = (XQUEUE *)xqueue; XQNODE *node = NULL; int i = 0; if(q) { //fprintf(stdout, "%s::%d q:%p nleft:%d qtotal:%d qleft:%p\n", __FILE__, __LINE__, q, q->nleft, q->qtotal, q->left); for(i = 0; i < q->nlist; i++); { if(q->list[i]) xmm_free(q->list[i], XQ_NODES_MAX * sizeof(XQNODE)); } /* while((node = q->left)) { q->left = node->next; xmm_free(node, sizeof(XQNODE)); } */ MUTEX_DESTROY(q->mutex); xmm_free(q, sizeof(XQUEUE)); } return ; }
/* clean chunk */ void chunk_clean(void *chunk) { if(chunk) { if(CHK(chunk)->mmap) munmap(CHK(chunk)->mmap, MMAP_CHUNK_SIZE); xmm_free(CHK(chunk)->data, CHK(chunk)->bsize); if(CHK(chunk)->fd > 0) close(CHK(chunk)->fd); xmm_free(chunk, sizeof(CHUNK)); } return ; }
/* clean chunk */ void chunk_clean(void *chunk) { if(chunk) { if(CHK(chunk)->mmap && !CHK(chunk)->flag) munmap(CHK(chunk)->mmap, MMAP_CHUNK_SIZE); CHK(chunk)->mmap = NULL; if(CHK(chunk)->data && !CHK(chunk)->flag) xmm_free(CHK(chunk)->data, CHK(chunk)->bsize); CHK(chunk)->data = NULL; if(CHK(chunk)->fd > 0 && !CHK(chunk)->flag) close(CHK(chunk)->fd); CHK(chunk)->fd = 0; xmm_free(chunk, sizeof(CHUNK)); } return ; }
/* chunk reset */ void chunk_reset(void *chunk) { if(chunk) { if(CHK(chunk)->mmap && !CHK(chunk)->flag) munmap(CHK(chunk)->mmap, MMAP_CHUNK_SIZE); CHK(chunk)->mmap = NULL; CHK(chunk)->mmleft = 0; if(CHK(chunk)->fd > 0 && !CHK(chunk)->flag) close(CHK(chunk)->fd); CHK(chunk)->fd = 0; CHK(chunk)->status = 0; CHK(chunk)->type = 0; CHK(chunk)->ndata = 0; CHK(chunk)->offset = 0; CHK(chunk)->left = 0; CHK(chunk)->mmoff = 0; CHK(chunk)->mmleft = 0; if(CHK(chunk)->flag) { CHK(chunk)->data = NULL; CHK(chunk)->bsize = 0; } if(CHK(chunk)->bsize > CHUNK_BLOCK_MAX) { xmm_free(CHK(chunk)->data, CHK(chunk)->bsize); CHK(chunk)->data = NULL; CHK(chunk)->bsize = 0; } //if(CHK(chunk)->data) memset(CHK(chunk)->data, 0, CHK(chunk)->bsize); CHK(chunk)->end = CHK(chunk)->data; } return ; }
void iqueue_clean(void *queue) { QUEUE *q = (QUEUE *)queue; int i = 0; if(q) { for(i = 0; i < q->nlist; i++); { if(q->list[i])xmm_free(q->list[i], QNODE_LINE_NUM * sizeof(QNODE)); } MUTEX_DESTROY(q->mutex); xmm_free(q, sizeof(QUEUE)); } return ; }
/* clean evtimer */ void evtimer_clean(EVTIMER *evtimer) { int i = 0; if(evtimer) { MUTEX_DESTROY(evtimer->mutex); for(i = 0; i < evtimer->nevlist; i++) { xmm_free(evtimer->evlist[i], sizeof(EVTNODE) * EVTNODE_LINE_NUM); } xmm_free(evtimer, sizeof(EVTIMER)); } return ; }
/* clean */ void mmqueue_clean(MMQUEUE *mmq) { if(mmq) { MUTEX_DESTROY(mmq->mutex); if(mmq->map) munmap(mmq->map, mmq->end); if(mmq->fd) close(mmq->fd); xmm_free(mmq, sizeof(MMQUEUE)); } return ; }
/* clean xmap */ void xmap_clean(XMAP *xmap) { if(xmap) { //WARN_LOGGER(xmap->logger, "Ready clean tree[%p]", xmap->tree); mmtree_close(xmap->tree); //WARN_LOGGER(xmap->logger, "Ready clean tree64[%p]", xmap->tree64); mmtree64_close(xmap->tree64); //WARN_LOGGER(xmap->logger, "Ready clean queue[%p]", xmap->queue); mmqueue_clean(xmap->queue); //WARN_LOGGER(xmap->logger, "Ready clean kmap[%p]", xmap->kmap); mmtrie_clean(xmap->kmap); //WARN_LOGGER(xmap->logger, "Ready reset db[%p]", xmap->db); cdb_reset(xmap->db); //WARN_LOGGER(xmap->logger, "Ready clean db[%p]", xmap->db); cdb_clean(xmap->db); if(xmap->diskio.map) { munmap(xmap->diskio.map, xmap->diskio.size); xmap->diskio.map = NULL; } if(xmap->diskio.fd > 0) { close(xmap->diskio.fd); xmap->diskio.fd = 0; } if(xmap->metaio.map) { munmap(xmap->metaio.map, xmap->metaio.size); xmap->metaio.map = NULL; } if(xmap->metaio.fd > 0) { close(xmap->metaio.fd); xmap->metaio.fd = 0; } if(xmap->stateio.map) { munmap(xmap->stateio.map, xmap->stateio.size); xmap->stateio.map = NULL; } if(xmap->stateio.fd > 0) { close(xmap->stateio.fd); xmap->stateio.fd = 0; } //WARN_LOGGER(xmap->logger, "Ready clean mutex[%p]", xmap->mutex); MUTEX_DESTROY(xmap->mutex); MUTEX_DESTROY(xmap->cmutex); LOGGER_CLEAN(xmap->logger); xmm_free(xmap, sizeof(XMAP)); } return ; }
/* clean hindex */ void mindex_clean(MINDEX *mindex) { if(mindex) { mtask_close(&(mindex->mtask)); dbase_close(&(mindex->db)); bjson_clean(&(mindex->request)); xmm_free(mindex, sizeof(MINDEX)); } return ; }
/* Clean evbase */ void evselect_clean(EVBASE *evbase) { if(evbase) { //MUTEX_DESTROY(evbase->mutex); if(evbase->logger)LOGGER_CLEAN(evbase->logger); if(evbase->evlist)free(evbase->evlist); if(evbase->evs)free(evbase->evs); if(evbase->ev_fds)free(evbase->ev_fds); if(evbase->ev_read_fds)free(evbase->ev_read_fds); if(evbase->ev_write_fds)free(evbase->ev_write_fds); xmm_free(evbase, sizeof(EVBASE)); } return ; }
/* * close/clean wtable * */ void wtable_close(WTABLE *w) { if(w) { if(w->statefd > 0) close(w->statefd); w->statefd = 0; if(w->state) munmap(w->state, sizeof(WSTATE)); w->state = NULL; db_clean(w->mdb); mqueue_clean(MQ(w->queue)); mmtrie_clean(w->map); mtree_close(w->mtree); mmtree64_close(w->appmap); LOGGER_CLEAN(w->logger); xmm_free(w, sizeof(WTABLE)); } return ; }
/* clean procthread */ void procthread_clean(PROCTHREAD *pth) { if(pth) { if(pth->service->working_mode != WORKING_PROC) { if(pth->have_evbase) { event_destroy(&(pth->event)); if(pth->evbase) pth->evbase->clean(pth->evbase); } qmessage_clean(pth->message_queue); } if(pth->service->flag & SB_USE_EVSIG) evsig_close(&(pth->evsig)); MUTEX_DESTROY(pth->mutex); xmm_free(pth, sizeof(PROCTHREAD)); } return ; }
/* clean */ void qindex_clean(QINDEX *qindex) { if(qindex) { if(qindex->state) munmap(qindex->state, sizeof(QSTATE)); if(qindex->qstatefd > 0)close(qindex->qstatefd); if(qindex->db) db_clean(PDB(qindex->db)); if(qindex->update) db_clean(PDB(qindex->update)); MUTEX_DESTROY(qindex->mutex); if(qindex->xdict){mmtrie_clean(qindex->xdict);} if(qindex->namemap){mmtrie_clean(qindex->namemap);} if(qindex->idmap){mmtree64_close(qindex->idmap);} if(qindex->queue){iqueue_clean(qindex->queue);} if(qindex->mmqueue){mmqueue_clean(MMQ(qindex->mmqueue));} if(qindex->xpacketio.map) munmap(qindex->xpacketio.map, qindex->xpacketio.size); if(qindex->xpacketio.fd) close(qindex->xpacketio.fd); if(qindex->bstermio.map) munmap(qindex->bstermio.map, qindex->bstermio.size); if(qindex->bstermio.fd) close(qindex->bstermio.fd); if(qindex->map) mmtrie_clean(qindex->map); if(qindex->logger){LOGGER_CLEAN(qindex->logger);} xmm_free(qindex, sizeof(QINDEX)); } return ; }
/* add index */ int ibase_index(IBASE *ibase, int docid, IBDATA *block) { char *term = NULL, buf[IB_BUF_SIZE], *data = NULL, *p = NULL, *pp = NULL, *nexts = NULL; int i = 0, termid = 0, n = 0, k = 0, ndocid = 0, ret = -1, ndata = 0, secid = 0, *intlist = NULL, *np = NULL, last_docid = 0; DOCHEADER *docheader = NULL; int64_t *longlist = NULL; double *doublelist = NULL; IHEADER *iheader = NULL; STERM *termlist = NULL; MDB *index = NULL, *posting = NULL; off_t size = 0; if((docheader = (DOCHEADER *)block->data) && (secid = docheader->secid) >= 0 && docheader->secid < IB_SEC_MAX) { index = (MDB *)(ibase->mindex[docheader->secid]); posting = (MDB *)(ibase->mposting[docheader->secid]); if(docheader->dbid != -1) { ibase->state->dtotal++; ibase->state->ttotal += (int64_t)docheader->terms_total; } if(ibase->state->used_for == IB_USED_FOR_INDEXD) { /* add to source */ if(((off_t)docid * (off_t)sizeof(IHEADER)) >= ibase->state->headers[secid].end) { ibase->state->headers[secid].old = ibase->state->headers[secid].end; size = (off_t)((docid / IB_HEADERS_BASE) + 1) * (off_t)IB_HEADERS_BASE * (off_t)sizeof(IHEADER); ret = ftruncate(ibase->state->headers[secid].fd, size); ibase->state->headers[secid].end = size; memset(ibase->state->headers[secid].map + ibase->state->headers[secid].old, 0, ibase->state->headers[secid].end - ibase->state->headers[secid].old); } } /* index */ //end = block->data + block->ndata; termlist = (STERM *)(block->data + sizeof(DOCHEADER) + docheader->nfields * sizeof(XFIELD)); term = block->data + docheader->textblock_off; nexts = block->data + docheader->nexts_off; for(i = 0; i < docheader->nterms; i++) { termid = termlist[i].termid; //find/add termid if(termid > 0 && termlist[i].term_len > 0 && termlist[i].nexts_size >= 0 && (ret=mmtrie_add((MMTRIE *)ibase->mmtrie,term,termlist[i].term_len,termid))>0) { if(ibase->state->index_status != IB_INDEX_DISABLED) { last_docid = 0; ndocid = -1; if(mdb_get_tag(index, termid, &last_docid) == 0) { ndocid = docid - last_docid; if(ndocid <= 0) goto term_state_update; } else { ndocid = docid; } p = pp = buf; if((ndata = ((sizeof(int) * 5))) > IB_BUF_SIZE) p = pp = data = (char *)xmm_new(ndata); /* compress index */ if(ibase->state->compression_status != IB_COMPRESSION_DISABLED) { n = ndocid; np = &n; ZVBCODE(np, p); n = termlist[i].term_count; np = &n; ZVBCODE(np, p); n = i; np = &n; ZVBCODE(np, p); n = termlist[i].bit_fields; np = &n; ZVBCODE(np, p); n = termlist[i].nexts_size; np = &n; ZVBCODE(np, p); } else { memcpy(p, &docid, sizeof(int));p += sizeof(int); memcpy(p, &(termlist[i].term_count), sizeof(int));p += sizeof(int); memcpy(p, &i, sizeof(int));p += sizeof(int); memcpy(p, &(termlist[i].bit_fields), sizeof(int));p += sizeof(int); memcpy(p, &(termlist[i].nexts_size), sizeof(int));p += sizeof(int); } if(termlist[i].nexts_size > 0) { if(mdb_add_data(posting, termid, nexts, termlist[i].nexts_size) <= 0) { FATAL_LOGGER(ibase->logger, "index posting term[%d] failed, %s", termid, strerror(errno)); _exit(-1); } //WARN_LOGGER(ibase->logger, "docid:%lld termid:%d nexts_size:%d", IBLL(docheader->globalid), termid, termlist[i].nexts_size); } if(mdb_add_data(index, termid, pp, (p - pp)) <= 0) { FATAL_LOGGER(ibase->logger, "index term[%d] failed, %s", termid, strerror(errno)); if(data){free(data); data = NULL;} _exit(-1); } mdb_set_tag(index, termid, docid); if(data){xmm_free(data, ndata); data = NULL;} } term_state_update: MUTEX_LOCK(ibase->mutex_termstate); if(termid > ibase->state->termid){ADD_TERMSTATE(ibase, termid);} if(ndocid > 0 ) { ((TERMSTATE *)(ibase->termstateio.map))[termid].len=termlist[i].term_len; ((TERMSTATE *)(ibase->termstateio.map))[termid].total++; } MUTEX_UNLOCK(ibase->mutex_termstate); } else { FATAL_LOGGER(ibase->logger, "Invalid ret:%d term[%d]{%.*s} termid:%d in doc:%d", ret, i, termlist[i].term_len, term, termid, docid); _exit(-1); } term += termlist[i].term_len; nexts += termlist[i].nexts_size; } if(ibase->state->used_for == IB_USED_FOR_INDEXD) { /* index int */ if((n = ibase->state->int_index_fields_num) > 0 && (intlist = (int *)(block->data + docheader->intblock_off))) { n += IB_INT_OFF; k = 0; for(i = IB_INT_OFF; i < n; i++) { IMAP_SET(ibase->state->mfields[secid][i], docid, intlist[k]); k++; } } /* index long */ if((n = ibase->state->long_index_fields_num) > 0 && (longlist = (int64_t *)(block->data + docheader->longblock_off))) { n += IB_LONG_OFF; k = 0; for(i = IB_LONG_OFF; i < n; i++) { LMAP_SET(ibase->state->mfields[secid][i], docid, longlist[k]); k++; } } /* index double */ if((n = ibase->state->double_index_fields_num) > 0 && (doublelist = (double *)(block->data + docheader->doubleblock_off))) { n += IB_DOUBLE_OFF; k = 0; for(i = IB_DOUBLE_OFF; i < n; i++) { DMAP_SET(ibase->state->mfields[secid][i], docid, doublelist[k]); k++; } } if((iheader = PIHEADER(ibase, docheader->secid, docid))) { iheader->status = docheader->status; iheader->terms_total = docheader->terms_total; iheader->crc = docheader->crc; iheader->category = docheader->category; iheader->slevel = docheader->slevel; iheader->rank = docheader->rank; iheader->globalid = docheader->globalid; //WARN_LOGGER(ibase->logger, "iheader->category:%p docheader->category:%p", (void *)iheader->category, (void *)docheader->category); } } ret = 0; } return ret; }
/* Initialize evbase */ EVBASE *evbase_init() { EVBASE *evbase = NULL; if((evbase = (EVBASE *)xmm_mnew(sizeof(EVBASE)))) { MUTEX_RESET(evbase->mutex); #ifdef HAVE_EVPORT evops_default = EOP_PORT; evops[EOP_PORT].name = "PORT"; evops[EOP_PORT].init = evport_init; evops[EOP_PORT].add = evport_add; evops[EOP_PORT].update = evport_update; evops[EOP_PORT].del = evport_del; evops[EOP_PORT].loop = evport_loop; evops[EOP_PORT].reset = evport_reset; evops[EOP_PORT].clean = evport_clean; #endif #ifdef HAVE_EVSELECT evops_default = EOP_SELECT; evops[EOP_SELECT].name = "SELECT"; evops[EOP_SELECT].init = evselect_init; evops[EOP_SELECT].add = evselect_add; evops[EOP_SELECT].update = evselect_update; evops[EOP_SELECT].del = evselect_del; evops[EOP_SELECT].loop = evselect_loop; evops[EOP_SELECT].reset = evselect_reset; evops[EOP_SELECT].clean = evselect_clean; #endif #ifdef HAVE_EVPOLL evops_default = EOP_POLL; evops[EOP_POLL].name = "POLL"; evops[EOP_POLL].init = evpoll_init; evops[EOP_POLL].add = evpoll_add; evops[EOP_POLL].update = evpoll_update; evops[EOP_POLL].del = evpoll_del; evops[EOP_POLL].loop = evpoll_loop; evops[EOP_POLL].reset = evpoll_reset; evops[EOP_POLL].clean = evpoll_clean; #endif #ifdef HAVE_EVRTSIG evops_default = EOP_RTSIG; evops[EOP_RTSIG].name = "RTSIG" ; evops[EOP_RTSIG].init = evrtsig_init; evops[EOP_RTSIG].add = evrtsig_add; evops[EOP_RTSIG].update = evrtsig_update; evops[EOP_RTSIG].del = evrtsig_del; evops[EOP_RTSIG].loop = evrtsig_loop; evops[EOP_RTSIG].reset = evrtsig_reset; evops[EOP_RTSIG].clean = evrtsig_clean; #endif #ifdef HAVE_EVEPOLL evops_default = EOP_EPOLL; evops[EOP_EPOLL].name = "EPOLL"; evops[EOP_EPOLL].init = evepoll_init; evops[EOP_EPOLL].add = evepoll_add; evops[EOP_EPOLL].update = evepoll_update; evops[EOP_EPOLL].del = evepoll_del; evops[EOP_EPOLL].loop = evepoll_loop; evops[EOP_EPOLL].reset = evepoll_reset; evops[EOP_EPOLL].clean = evepoll_clean; #endif #ifdef HAVE_EVKQUEUE evops_default = EOP_KQUEUE; evops[EOP_KQUEUE].name = "KQUEUE"; evops[EOP_KQUEUE].init = evkqueue_init; evops[EOP_KQUEUE].add = evkqueue_add; evops[EOP_KQUEUE].update = evkqueue_update; evops[EOP_KQUEUE].del = evkqueue_del; evops[EOP_KQUEUE].loop = evkqueue_loop; evops[EOP_KQUEUE].reset = evkqueue_reset; evops[EOP_KQUEUE].clean = evkqueue_clean; #endif #ifdef HAVE_EVDEVPOLL evops_default = EOP_DEVPOLL; evops[EOP_DEVPOLL].name = "/dev/poll"; evops[EOP_DEVPOLL].init = evdevpoll_init; evops[EOP_DEVPOLL].add = evdevpoll_add; evops[EOP_DEVPOLL].update = evdevpoll_update; evops[EOP_DEVPOLL].del = evdevpoll_del; evops[EOP_DEVPOLL].loop = evdevpoll_loop; evops[EOP_DEVPOLL].reset = evdevpoll_reset; evops[EOP_DEVPOLL].clean = evdevpoll_clean; #endif #ifdef WIN32 evops_default = EOP_WIN32; evops[EOP_WIN32].name = "WIN32"; evops[EOP_WIN32].init = evwin32_init; evops[EOP_WIN32].add = evwin32_add; evops[EOP_WIN32].update = evwin32_update; evops[EOP_WIN32].del = evwin32_del; evops[EOP_WIN32].loop = evwin32_loop; evops[EOP_WIN32].reset = evwin32_reset; evops[EOP_WIN32].clean = evwin32_clean; #endif evbase->set_logfile = evbase_set_logfile; evbase->set_log_level = evbase_set_log_level; evbase->set_evops = evbase_set_evops; //evbase->clean = evbase_clean; if(evops_default == -1 || evbase->set_evops(evbase, evops_default) == -1) { xmm_free(evbase, sizeof(EVBASE)); fprintf(stderr, "Initialize evbase to default[%d] failed, %s\n", evops_default, strerror(errno)); evbase = NULL; } } return evbase; }