/* push index */
int qindex_push_index(QINDEX *qindex, int64_t key, IBDATA *block)
{
DOCHEADER *docheader = NULL;
XPACKET *xpackets = NULL;
int mid = 0, old = 0;
if(qindex && block->ndata > 0 && (docheader = (DOCHEADER *)block->data))
{
MUTEX_LOCK(qindex->mutex);
if(docheader->size == block->ndata && (xpackets = (XPACKET *)(qindex->xpacketio.map))
&& (mid = mmtree64_try_insert(qindex->idmap, qindex->state->rootid,
key, (qindex->state->id_max+1), &old)) > 0)
{
if(old > 0) mid = old;
else mid = ++(qindex->state->id_max);
db_set_data(PDB(qindex->db), mid, block->data, block->ndata);
if(mid <= qindex->state->xpackettotal)
{
if(xpackets[mid].crc != docheader->crc)
qindex_update(qindex, mid, 1);
else
qindex_update(qindex, mid, 0);
xpackets[mid].status = docheader->status;
xpackets[mid].crc = docheader->crc;
ACCESS_LOGGER(qindex->logger, "update-index{gloablid:%lld mid:%d total:%d}", LL(docheader->globalid), mid, qindex->state->xpackettotal);
}
else
{
qindex->state->xpackettotal = mid;
CHECK_XPACKETIO(qindex);
if((xpackets = (XPACKET *)(qindex->xpacketio.map)))
{
xpackets[mid].status = docheader->status;
xpackets[mid].crc = docheader->crc;
ACCESS_LOGGER(qindex->logger, "new-index{gloablid:%lld mid:%d}", LL(docheader->globalid), mid);
}
}
}
else
{
FATAL_LOGGER(qindex->logger, "Invalid document size:%d ndata:%d id:%lld crc:%d", docheader->size, block->ndata, LL(docheader->globalid), docheader->crc);
_exit(-1);
}
MUTEX_UNLOCK(qindex->mutex);
}
return mid;
}
/* timeout handler*/
int benchmark_timeout_handler(CONN *conn, CB_DATA *packet, CB_DATA *cache, CB_DATA *chunk)
{
if(conn)
{
if(cache && cache->data)
{
ACCESS_LOGGER(logger, "timeout on conn[%s:%d] uri[%s] via %d status:%d", conn->local_ip, conn->local_port, cache->data, conn->fd, conn->status);
}
else
{
ACCESS_LOGGER(logger, "timeout on conn[%s:%d] via %d status:%d", conn->local_ip, conn->local_port, conn->fd, conn->status);
}
ntimeout++;
return http_check_over(conn);
}
return -1;
}
/* run procthread */
void procthread_run(void *arg)
{
PROCTHREAD *pth = (PROCTHREAD *)arg;
int i = 0, usec = 0, sec = 0;
struct timeval tv = {0,0};
struct timespec ts = {0, 0};
int k = 0, n = 0, policy = 0;
if(pth)
{
struct sched_param param;
pthread_getschedparam(pthread_self(), &policy, ¶m);
ACCESS_LOGGER(pth->logger, "Ready for running thread[%p] policy:%d SCHED_FIFO:%d SCHED_RR:%d SCHED_OTHER:%d", (void*)((long)(pth->threadid)), policy, SCHED_FIFO, SCHED_RR, SCHED_OTHER);
pth->running_status = 1;
if(pth->usec_sleep > 1000000) sec = pth->usec_sleep/1000000;
usec = pth->usec_sleep % 1000000;
tv.tv_sec = sec;
tv.tv_usec = usec;
ts.tv_sec = sec;
ts.tv_nsec = (long)usec * 1000l;
if(pth->have_evbase)
{
if(pth->cond > 0)
{
event_set(&(pth->event), pth->cond, E_READ|E_PERSIST,
(void *)pth, (void *)&procthread_event_handler);
pth->evbase->add(pth->evbase, &(pth->event));
if(pth->service->flag & SB_LOG_THREAD)
{
char line[256];
if(pth == pth->service->outdaemon)
{
sprintf(line, "/tmp/%s_outdaemon.log", pth->service->service_name);
evbase_set_logfile(pth->evbase, line);
}
else
{
sprintf(line, "/tmp/%s_indaemon.log", pth->service->service_name);
evbase_set_logfile(pth->evbase, line);
}
}
}
do
{
i = pth->evbase->loop(pth->evbase, 0, NULL);
if(pth->service->flag & SB_LOG_THREAD)
{
if(pth == pth->service->outdaemon)
{
ACCESS_LOGGER(pth->logger, "outdaemon_loop(%d/%d) q[%p]{total:%d left:%d}", i, k, pth->message_queue, QMTOTAL(pth->message_queue), QNLEFT(pth->message_queue));
}
else
{
ACCESS_LOGGER(pth->logger, "iodaemon_loop(%d/%d) q[%p]{total:%d left:%d}", i, k, pth->message_queue, QMTOTAL(pth->message_queue), QNLEFT(pth->message_queue));
}
}
if(pth->message_queue && (k = QMTOTAL(pth->message_queue)) > 0)
{
qmessage_handler(pth->message_queue, pth->logger);
}
if((pth->service->flag & (SB_IO_NANOSLEEP|SB_IO_USLEEP|SB_IO_SELECT))
&& n++ > pth->service->nworking_tosleep)
{
if(pth->service->flag & SB_IO_NANOSLEEP) nanosleep(&ts, NULL);
else if(pth->service->flag & SB_IO_USLEEP) usleep(pth->usec_sleep);
else if(pth->service->flag & SB_IO_SELECT) select(0, NULL, NULL, NULL, &tv);
n = 0;
}
}while(pth->running_status);
if(pth == pth->service->outdaemon)
{
ACCESS_LOGGER(pth->logger, "Ready for stop outdaemons[%p]", pth);
}
else
{
ACCESS_LOGGER(pth->logger, "Ready for stop iodaemons[%p]", pth);
}
}
else if(pth->listenfd > 0)
{
do
{
service_accept_handler(pth->service);
}while(pth->running_status);
ACCESS_LOGGER(pth->logger, "Ready for stop threads[acceptor]");
}
else
{
if(pth->use_cond_wait)
{
do
{
//DEBUG_LOGGER(pth->logger, "starting cond-wait() threads[%p]->qmessage[%p]_handler(%d)", (void *)pth->threadid,pth->message_queue, QMTOTAL(pth->message_queue));
if(pth->message_queue && (k = QMTOTAL(pth->message_queue)) > 0)
{
qmessage_handler(pth->message_queue, pth->logger);
i = 1;
}
if(QMTOTAL(pth->message_queue) < 1)
{
if(pth->service->flag & SB_USE_EVSIG)
evsig_wait(&(pth->evsig));
else if(pth->service->flag & SB_USE_COND)
{
MUTEX_WAIT(pth->mutex);
}
else
nanosleep(&ts, NULL);
}
}while(pth->running_status);
ACCESS_LOGGER(pth->logger, "ready to exit threads/daemons[%d]", pth->index);
}
else
{
do
{
//DEBUG_LOGGER(pth->logger, "starting threads[%p]->qmessage[%p]_handler(%d)", (void *)(pth->threadid),pth->message_queue, QMTOTAL(pth->message_queue));
if(pth->evtimer){EVTIMER_CHECK(pth->evtimer);}
if(pth->message_queue && QMTOTAL(pth->message_queue) > 0)
{
qmessage_handler(pth->message_queue, pth->logger);
}
nanosleep(&ts, NULL);
//WARN_LOGGER(pth->logger, "over threads[%p]->qmessage[%p]_handler(%d)", (void *)(pth->threadid),pth->message_queue, QMTOTAL(pth->message_queue));
}while(pth->running_status);
ACCESS_LOGGER(pth->logger, "ready to exit threads/daemons[%d]", pth->index);
}
}
if(pth->message_queue && QMTOTAL(pth->message_queue) > 0)
qmessage_handler(pth->message_queue, pth->logger);
ACCESS_LOGGER(pth->logger, "terminate threads[%d][%p] evbase[%p] qmessage[%p] ioqmessage[%p] qtotal:%d", pth->index, (void *)(pth->threadid), pth->evbase, pth->message_queue, pth->inqmessage, QMTOTAL(pth->message_queue));
}
#ifdef HAVE_PTHREAD
pthread_exit(NULL);
#endif
return ;
}
/* add document */
int ibase_add_document(IBASE *ibase, IBDATA *block)
{
int ret = -1, localid = 0, secid = 0, n = 0, newid = 0;
DOCHEADER *docheader = NULL;
MHEADER *mheader = NULL;
IHEADER *iheader = NULL;
char line[IB_LINE_MAX];
off_t size = 0;
if(ibase && block && block->ndata > 0 && (docheader = (DOCHEADER *)(block->data))
&& (secid = docheader->secid) >= 0 && secid < IB_SEC_MAX
&& ibase_check_index_state(ibase, docheader) == 0
&& docheader->globalid && (n = sprintf(line, "%lld", IBLL(docheader->globalid))))
{
if((localid = mmtrie_xadd(ibase->docmap, line, n)) > 0)
{
MUTEX_LOCK(ibase->mutex);
if(localid <= ibase->state->docid)
{
if(ibase->state->used_for == IB_USED_FOR_INDEXD)
{
if((mheader = PMHEADER(ibase, localid))
&& (iheader = PIHEADER(ibase, mheader->secid, mheader->docid)))
{
/* disabled */
if(docheader->status < 0)
{
mheader->status = -1;
iheader->status = -1;
DEBUG_LOGGER(ibase->logger, "Update docid:%d globalid:%lld status:%d", localid, IBLL(docheader->globalid), docheader->status);
ret = 0;
}
else if(docheader->crc != mheader->crc)
{
ibase_del_index(ibase, secid, localid);
newid = ++(ibase->state->ids[secid]);
DEBUG_LOGGER(ibase->logger, "ready_reindex[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
mheader->docid = newid;
mheader->crc = docheader->crc;
ret = ibase_index(ibase, newid, block);
DEBUG_LOGGER(ibase->logger, "over_reindex[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
}
else
{
DEBUG_LOGGER(ibase->logger, "update_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
ret = ibase_update_index(ibase, mheader->docid, block);
DEBUG_LOGGER(ibase->logger, "over_update_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
}
}
}
else
{
docheader->dbid = -1;
ret = ibase_index(ibase, localid, block);
}
}
else
{
if(ibase->state->used_for == IB_USED_FOR_INDEXD)
{
if(((off_t)localid * (off_t)sizeof(MHEADER)) >= ibase->mheadersio.end)
{
ibase->mheadersio.old = ibase->mheadersio.end;
size = (off_t)((localid / IB_HEADERS_BASE) + 1)
* (off_t)IB_HEADERS_BASE * (off_t)sizeof(MHEADER);
ret = ftruncate(ibase->mheadersio.fd, size);
ibase->mheadersio.end = size;
memset(ibase->mheadersio.map + ibase->mheadersio.old, 0,
ibase->mheadersio.end - ibase->mheadersio.old);
}
if((mheader = PMHEADER(ibase, localid)))
{
mheader->status = 0;
mheader->docid = ++(ibase->state->ids[secid]);
mheader->secid = docheader->secid;
mheader->crc = docheader->crc;
DEBUG_LOGGER(ibase->logger, "new_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
ret = ibase_index(ibase, mheader->docid, block);
}
}
else
{
ret = ibase_index(ibase, localid, block);
}
ibase->state->docid = localid;
}
MUTEX_UNLOCK(ibase->mutex);
if(ibase->state->used_for != IB_USED_FOR_QPARSERD
&& ibase->state->mmsource_status != IB_MMSOURCE_NULL && localid > 0)
{
ACCESS_LOGGER(ibase->logger, "docid:%lld/%d c_size:%d c_zsize:%d size:%d", docheader->globalid, localid, docheader->content_size, docheader->content_zsize, docheader->nexts_off);
docheader->size = docheader->nexts_off;
ret = db_set_data(PDB(ibase->source), localid, block->data, docheader->size);
}
ret = 0;
}
else
{
FATAL_LOGGER(ibase->logger, "new_index[%lld]{crc:%d rank:%f slevel:%d category:%lld} failed, %s", IBLL(docheader->globalid), docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category), strerror(errno));
}
}
return ret;
}
/* read upindex */
int qindex_read_upindex(QINDEX *qindex, int taskid, char *data, int *len, int *count)
{
int ret = -1, mid = -1, nodeid = -1, id = 0, k = 0, n = 0,
left = 0, prev = 0, next = 0, *px = NULL;
DOCHEADER *docheader = NULL;
XPACKET *xpackets = NULL;
QTNODE *nodes = NULL;
QTASK *tasks = NULL;
char *p = NULL;
if(qindex && taskid >= 0 && data && taskid < Q_NODE_MAX * Q_TASKS_MAX
&& len && count && *len > 0)
{
MUTEX_LOCK(qindex->mutex);
*count = 0;
k = taskid % Q_TASKS_MAX;
if((nodeid = (taskid / Q_TASKS_MAX)) < Q_NODE_MAX && (nodes = qindex->state->nodes)
&& (tasks = qindex->state->nodes[nodeid].tasks) && tasks[k].status > 0
&& (xpackets = (XPACKET *)(qindex->xpacketio.map)))
{
if(tasks[k].nupdates > 0 && tasks[k].upid != tasks[k].upover)
{
if((*len = db_read_data(PDB(qindex->update), taskid, p)) > 0)
{
*count = tasks[k].upcount;
ret = mid = tasks[k].upid;
}
else
{
tasks[k].upid = tasks[k].upover = 0;
}
}
else if(tasks[k].nqueue > 0 && tasks[k].mmqid > 0)
{
tasks[k].upcount = 0;
left = *len;
p = data;
id = mmqueue_head(MMQ(qindex->mmqueue), tasks[k].mmqid, &mid);
do
{
if(id > 0 && mid > 0)
{
if(nodes[nodeid].type == Q_NODE_INDEXD && xpackets[mid].nodeid != nodeid)
{
prev = xpackets[mid].prev;
next = xpackets[mid].next;
FATAL_LOGGER(qindex->logger, "Invalid rootid:%d id:%d prev:%d[nodeid:%d] next:%d[nodeid:%d] xpacket[%d].nodeid[%d] to task[%s:%d].nodeid:%d nqueue:%d", tasks[k].mmqid, id, prev, xpackets[prev].nodeid, next, xpackets[next].nodeid, mid, xpackets[mid].nodeid, tasks[k].ip, tasks[k].port, nodeid, tasks[k].nqueue);
tasks[k].mmqid = 0;tasks[k].nqueue = 0;
break;
}
if(db_get_data_len(PDB(qindex->db), mid) > (left-Q_LEFT_LEN)) break;
tasks[k].upid = mid;
px = (int *)p;
p += sizeof(int);
docheader = (DOCHEADER *)p;
if((n = db_read_data(PDB(qindex->db), mid, p)) > sizeof(DOCHEADER))
{
ACCESS_LOGGER(qindex->logger, "update_index(%d) globalid:%lld mid:%d status:%d nodeid:%d task[%s:%d]", id, LL(docheader->globalid), mid, xpackets[mid].status, xpackets[mid].nodeid, tasks[k].ip, tasks[k].port);
left -= n + sizeof(int);
p += n;
*px = n;
tasks[k].upcount++;
}
else
{
p -= sizeof(int);
FATAL_LOGGER(qindex->logger, "Invalid data id:%d to document:%lld", id, LL(docheader->globalid));
mid = -1;
//_exit(-1);
break;
}
mmqueue_pop(MMQ(qindex->mmqueue),tasks[k].mmqid, &mid);
tasks[k].nqueue--;
id = mmqueue_head(MMQ(qindex->mmqueue), tasks[k].mmqid, &mid);
}
else
{
FATAL_LOGGER(qindex->logger, "Invalid qid:%d mid:%d", id, mid);
tasks[k].mmqid = 0;tasks[k].nqueue = 0;
mid = -1;
break;
}
}while(tasks[k].nqueue > 0 && left > Q_LEFT_LEN && id > 0 && mid > 0);
if((*count = tasks[k].upcount) > 0 && (*len -= left) > 0
&& db_set_data(PDB(qindex->update), taskid, data, *len) >= 0)
{
ret = tasks[k].upid;
}
}
else ret = -1;
}
else ret = -2;
MUTEX_UNLOCK(qindex->mutex);
}
return ret;
}
/* working for get_data() and push_index() */
int qindex_work(QINDEX *qindex, MINDEX *mindex)
{
int ret = -1, i = 0, x = 0 , n = 0, mid = 0;
int64_t *xid = NULL, key = 0, vlong = 0;
BELEMENT *root = NULL, *sub = NULL;
DOCHEADER *docheader = NULL;
STERM *termlist = NULL;
IBDATA block = {0};
char *term = NULL;
BRES *res = NULL;
if(qindex && mindex && mindex_check(mindex) == 0)
{
if(mtask_pop(&(mindex->mtask)) > 0 && (xid = (int64_t *)mindex->mtask.packet)
&& mindex->mtask.length > sizeof(int64_t))
{
n = mindex->mtask.length/sizeof(int64_t);
brequest_reset(&(mindex->request));
brequest_append_keys(&(mindex->request), xid, n);
brequest_finish(&(mindex->request));
if((res = dbase_get_records(&(mindex->db), &(mindex->request))))
{
x = 0;
while((root = dbase_next_record(res, root, &key)))
{
ACCESS_LOGGER(qindex->logger, "index_record[%d/%d] packetid:%d key:%lld", x, n, mindex->mtask.packetid, LLI(key));
vlong = 0;
if((sub = belement_find(root, qindex->db_key_name)))
{
belement_v_long(sub, &vlong);
}
if(key != vlong)
{
ACCESS_LOGGER(qindex->logger, "Invalid record[%d/%d] packetid:%d key:%lld/%lld", x, n, mindex->mtask.packetid, LLI(vlong), LLI(key));
return ret;
}
if((sub = belement_find(root, qindex->db_index_block_name))
&& (block.ndata = belement_v_blob(sub, &(block.data))) > 0)
{
docheader = (DOCHEADER *)block.data;
termlist = (STERM *)(block.data + sizeof(DOCHEADER)
+ docheader->nfields * sizeof(XFIELD));
term = block.data + docheader->textblock_off;
for(i = 0; i < docheader->nterms; i++)
{
termlist[i].termid = mmtrie_xadd(qindex->xdict, term, termlist[i].term_len);
term += termlist[i].term_len;
}
mid = qindex_push_index(qindex, key, &block);
ACCESS_LOGGER(qindex->logger, "over_record[%d/%d] packetid:%d key:%lld", x, n, mindex->mtask.packetid, LLI(key));
}
else
{
FATAL_LOGGER(qindex->logger, "Invalid data feilds[%s]", qindex->db_index_block_name);
return ret;
}
x++;
}
ret = 0;
mtask_finish(&(mindex->mtask), 0);
}
else
{
FATAL_LOGGER(qindex->logger, "get_records(%d) failed, %s", mindex->mtask.packetid, strerror(errno));
return ret;
}
}
}
return ret;
}
