/* kid */
int xmap_kid(XMAP *xmap, int64_t key)
{
int id = -1, k = 0, x = 0, old = 0, root = 0;
if(xmap && key)
{
MUTEX_LOCK(xmap->mutex);
k = DBKMASK(key);
if((root = xmap->state->masks[k].root) > 0)
{
x = xmap->state->id_max + 1;
if(mmtree64_try_insert(xmap->tree64, root, key, x, &old) > 0)
{
if(old > 0)
{
id = old;
}
else
{
id = ++(xmap->state->id_max);
xmap->state->masks[k].total++;
CHECK_XMMETAIO(xmap, id);
}
}
else
{
FATAL_LOGGER(xmap->logger, "Try_insert(%lld,%d) failed", LL64(key), x);
}
}
MUTEX_UNLOCK(xmap->mutex);
}
return id;
}
/* set connection */
int conn_set(CONN *conn)
{
short flag = 0;
if(conn && conn->fd > 0 )
{
fcntl(conn->fd, F_SETFL, O_NONBLOCK);
if(conn->evbase && conn->event)
{
flag = E_READ|E_PERSIST;
if(conn->status == CONN_STATUS_READY) flag |= E_WRITE;
conn->event->set(conn->event, conn->fd, flag, (void *)conn, &conn_event_handler);
conn->evbase->add(conn->evbase, conn->event);
DEBUG_LOGGER(conn->logger, "setting connection[%s:%d] via %d",
conn->ip, conn->port, conn->fd);
return 0;
}
else
{
FATAL_LOGGER(conn->logger, "Connection[%08x] fd[%d] evbase or"
"initialize event failed, %s", conn, conn->fd, strerror(errno));
/* Terminate connection */
CONN_TERMINATE(conn);
}
}
return -1;
}
/* write handler */
int conn_write_handler(CONN *conn)
{
int ret = -1, n = 0;
CONN_CHECK_RET(conn, ret);
CHUNK *cp = NULL;
if(conn && conn->send_queue && QTOTAL(conn->send_queue) > 0)
{
DEBUG_LOGGER(conn->logger, "Ready for send data to %s:%d via %d "
"qtotal:%d qhead:%d qcount:%d",
conn->ip, conn->port, conn->fd, QTOTAL(conn->send_queue),
QHEAD(conn->send_queue), QCOUNT(conn->send_queue));
if(QUEUE_HEAD(conn->send_queue, PCHUNK, &cp) == 0)
{
DEBUG_LOGGER(conn->logger, "Ready for send data to %s:%d via %d qtotal:%d pcp:%08x",
conn->ip, conn->port, conn->fd, QTOTAL(conn->send_queue), cp);
if((n = CHUNK_WRITE(cp, conn->fd)) > 0)
{
conn->sent_data_total += n;
DEBUG_LOGGER(conn->logger, "Sent %d byte(s) (total sent %lld) "
"to %s:%d via %d leave %lld", n, conn->sent_data_total,
conn->ip, conn->port, conn->fd, CK_LEFT(cp));
/* CONN TIMER sample */
TIMER_SAMPLE(conn->timer);
if(CHUNK_STATUS(cp) == CHUNK_STATUS_OVER )
{
if(QUEUE_POP(conn->send_queue, PCHUNK, &cp) == 0)
{
DEBUG_LOGGER(conn->logger, "Completed chunk[%08x] and clean it leave %d",
cp, QTOTAL(conn->send_queue));
CK_CLEAN(cp);
}
}
ret = 0;
}
else
{
FATAL_LOGGER(conn->logger, "Sending data to %s:%d via %d failed, %s",
conn->ip, conn->port, conn->fd, strerror(errno));
/* Terminate connection */
CONN_TERMINATE(conn);
}
}
if(QTOTAL(conn->send_queue) <= 0)
{
conn->event->del(conn->event, E_WRITE);
}
}
return ret;
}
/* push index */
int qindex_push_index(QINDEX *qindex, int64_t key, IBDATA *block)
{
DOCHEADER *docheader = NULL;
XPACKET *xpackets = NULL;
int mid = 0, old = 0;
if(qindex && block->ndata > 0 && (docheader = (DOCHEADER *)block->data))
{
MUTEX_LOCK(qindex->mutex);
if(docheader->size == block->ndata && (xpackets = (XPACKET *)(qindex->xpacketio.map))
&& (mid = mmtree64_try_insert(qindex->idmap, qindex->state->rootid,
key, (qindex->state->id_max+1), &old)) > 0)
{
if(old > 0) mid = old;
else mid = ++(qindex->state->id_max);
db_set_data(PDB(qindex->db), mid, block->data, block->ndata);
if(mid <= qindex->state->xpackettotal)
{
if(xpackets[mid].crc != docheader->crc)
qindex_update(qindex, mid, 1);
else
qindex_update(qindex, mid, 0);
xpackets[mid].status = docheader->status;
xpackets[mid].crc = docheader->crc;
ACCESS_LOGGER(qindex->logger, "update-index{gloablid:%lld mid:%d total:%d}", LL(docheader->globalid), mid, qindex->state->xpackettotal);
}
else
{
qindex->state->xpackettotal = mid;
CHECK_XPACKETIO(qindex);
if((xpackets = (XPACKET *)(qindex->xpacketio.map)))
{
xpackets[mid].status = docheader->status;
xpackets[mid].crc = docheader->crc;
ACCESS_LOGGER(qindex->logger, "new-index{gloablid:%lld mid:%d}", LL(docheader->globalid), mid);
}
}
}
else
{
FATAL_LOGGER(qindex->logger, "Invalid document size:%d ndata:%d id:%lld crc:%d", docheader->size, block->ndata, LL(docheader->globalid), docheader->crc);
_exit(-1);
}
MUTEX_UNLOCK(qindex->mutex);
}
return mid;
}
CONN *http_newconn(int id, char *ip, int port, int is_ssl)
{
CONN *conn = NULL;
if(running_status && ip && port > 0)
{
if(is_ssl) service->session.flags |= SB_USE_SSL;
if((conn = service->newconn(service, -1, -1, ip, port, NULL)))
{
conn->c_id = id;
}
else
{
FATAL_LOGGER(logger, "new_conn(%s:%d) falied, %s", ip, port, strerror(errno));
}
}
return conn;
}
/* new mindex */
MINDEX *mindex_new(QINDEX *qindex)
{
MINDEX *mindex = NULL;
if(strlen(qindex->qtask_server_host) > 0 && qindex->qtask_server_port > 0
&& (qindex->qtask_commitid > 0 || qindex->qtask_queueid > 0)
&& (mindex = xmm_mnew(sizeof(MINDEX))))
{
/* set mtask */
mtask_set(&(mindex->mtask), qindex->qtask_server_host, qindex->qtask_server_port,
qindex->qtask_commitid, qindex->qtask_queueid);
/* set db connection*/
if(dbase_set(&(mindex->db), qindex->db_host, qindex->db_port) != 0
|| dbase_connect(&(mindex->db)) != 0)
{
FATAL_LOGGER(qindex->logger, "connect_source_db(%s:%d) failed, %s", qindex->db_host, qindex->db_port, strerror(errno));
}
}
return mindex;
}
CONN *http_newconn(int id, char *ip, int port, int is_ssl)
{
CONN *conn = NULL;
if(running_status && ip && port > 0)
{
if(is_ssl) service->session.is_use_SSL = 1;
if((conn = service->newconn(service, -1, -1, ip, port, NULL)))
{
conn->c_id = id;
conn->start_cstate(conn);
//service->newtransaction(service, conn, id);
//usleep(10);
}
else
{
FATAL_LOGGER(logger, "new_conn(%s:%d) falied, %s", ip, port, strerror(errno));
}
}
return conn;
}
/* read handler */
int conn_read_handler(CONN *conn)
{
int ret = -1, n = -1;
CONN_CHECK_RET(conn, ret);
if(conn)
{
/* Receive OOB */
if((n = MB_RECV(conn->oob, conn->fd, MSG_OOB)) > 0)
{
conn->recv_oob_total += n;
DEBUG_LOGGER(conn->logger, "Received %d bytes OOB total %lld from %s:%d via %d",
n, conn->recv_oob_total, conn->ip, conn->port, conn->fd);
conn->oob_handler(conn);
/* CONN TIMER sample */
TIMER_SAMPLE(conn->timer);
return (ret = 0);
}
/* Receive to chunk with chunk_read_state before reading to buffer */
CONN_CHUNK_READ(conn, n);
/* Receive normal data */
if((n = MB_READ(conn->buffer, conn->fd)) <= 0)
{
FATAL_LOGGER(conn->logger, "Reading %d bytes left:%d data from %s:%d via %d failed, %s",
n, MB_LEFT(conn->buffer), conn->ip, conn->port, conn->fd, strerror(errno));
/* Terminate connection */
CONN_TERMINATE(conn);
return (ret = 0);
}
/* CONN TIMER sample */
TIMER_SAMPLE(conn->timer);
conn->recv_data_total += n;
DEBUG_LOGGER(conn->logger, "Received %d bytes data total %lld from %s:%d via %d",
n, conn->recv_data_total, conn->ip, conn->port, conn->fd);
conn->packet_reader(conn);
ret = 0;
}
return ret;
}
/* packet reader */
int conn_packet_reader(CONN *conn)
{
int len = -1, i = 0;
CB_DATA *data = NULL;
char *p = NULL, *e = NULL;
int packet_type = 0;
CONN_CHECK_RET(conn, -1);
if(conn)
{
data = PCB(conn->buffer);
packet_type = conn->session.packet_type;
DEBUG_LOGGER(conn->logger, "Reading packet type[%d]", packet_type);
/* Remove invalid packet type */
if(!(packet_type & PACKET_ALL))
{
FATAL_LOGGER(conn->logger, "Unkown packet_type[%d] on %s:%d via %d",
packet_type, conn->ip, conn->port, conn->fd);
/* Terminate connection */
CONN_TERMINATE(conn);
}
/* Read packet with customized function from user */
if(packet_type == PACKET_CUSTOMIZED && conn->session.packet_reader)
{
len = conn->session.packet_reader(conn, data);
DEBUG_LOGGER(conn->logger,
"Reading packet with customized function[%08x] length[%d] on %s:%d via %d",
conn->session.packet_reader, len, conn->ip, conn->port, conn->fd);
goto end;
}
/* Read packet with certain length */
else if(packet_type == PACKET_CERTAIN_LENGTH
&& MB_NDATA(conn->buffer) >= conn->session.packet_length)
{
len = conn->session.packet_length;
DEBUG_LOGGER(conn->logger, "Reading packet with certain length[%d] on %s:%d via %d",
len, conn->ip, conn->port, conn->fd);
goto end;
}
/* Read packet with delimiter */
else if(packet_type == PACKET_DELIMITER && conn->session.packet_delimiter
&& conn->session.packet_delimiter_length > 0)
{
p = MB_DATA(conn->buffer);
e = MB_END(conn->buffer);
i = 0;
while(p < e && i < conn->session.packet_delimiter_length )
{
if(((char *)conn->session.packet_delimiter)[i++] != *p++) i = 0;
if(i == conn->session.packet_delimiter_length)
{
len = p - MB_DATA(conn->buffer);
break;
}
}
DEBUG_LOGGER(conn->logger, "Reading packet with delimiter[%d] "
"length[%d] on %s:%d via %d", conn->session.packet_delimiter_length,
len, conn->ip, conn->port, conn->fd);
goto end;
}
return len;
end:
/* Copy data to packet from buffer */
if(len > 0)
{
MB_RESET(conn->packet);
MB_PUSH(conn->packet, MB_DATA(conn->buffer), len);
MB_DEL(conn->buffer, len);
/* For packet handling */
conn->s_state = S_STATE_PACKET_HANDLING;
conn->push_message(conn, MESSAGE_PACKET);
}
}
return len;
}
/*
* initialize wtable
* */
WTABLE *wtable_init(char *dir)
{
char path[WT_PATH_MAX], *p = NULL;
struct stat st = {0};
WTABLE *w = NULL;
int n = 0;
if(dir && (w = (WTABLE *)xmm_mnew(sizeof(WTABLE))))
{
n = sprintf(path, "%s/%s", dir, WT_LOG_NAME);
force_mkdir(path);
p = path;
LOGGER_INIT(w->logger, p);
/* state */
n = sprintf(path, "%s/%s", dir, WT_STATE_NAME);
if((w->statefd = open(path, O_CREAT|O_RDWR, 0644)) <= 0
|| fstat(w->statefd, &st) != 0)
{
if(w->statefd > 0) close(w->statefd);
FATAL_LOGGER(w->logger, "open state file[%s] failed, %s",
path, strerror(errno));
_exit(-1);
}
else
{
if(st.st_size < sizeof(WSTATE)
&& ftruncate(w->statefd, sizeof(WSTATE)) != 0)
{
_exit(-1);
}
if((w->state = (WSTATE *)mmap(NULL, sizeof(WSTATE),
PROT_READ|PROT_WRITE, MAP_SHARED, w->statefd, 0)) == NULL
|| w->state == (void *)-1)
{
FATAL_LOGGER(w->logger, "mmap state failed, %s", strerror(errno));
_exit(-1);
}
if(st.st_size < sizeof(WSTATE))
memset(((char *)w->state + st.st_size), 0, sizeof(WSTATE) - st.st_size);
}
w->workers = w->state->workers;
memset(w->workers, 0, sizeof(WORKER) * W_WORKER_MAX);
w->state->nworkers = 0;
w->state->conn_total = 0;
n = sprintf(path, "%s/%s", dir, WT_MDB_DIR);
w->mdb = db_init(path, DB_USE_MMAP);
/* mmtrie */
n = sprintf(path, "%s/%s", dir, WT_MAP_NAME);
if((w->map = mmtrie_init(path)) == NULL) _exit(-1);
/* appmap */
n = sprintf(path, "%s/%s", dir, WT_APPS_NAME);
if((w->appmap = mmtree64_init(path)) == NULL) _exit(-1);
mmtree64_use_all(w->appmap);
/* logger & mutex & mtree & mqueue */
if((w->queue = mqueue_init()) == NULL) _exit(-1);
if((w->mtree = mtree_init()) == NULL) _exit(-1);
w->whitelist = mtree_new_tree(w->mtree);
//MQ(w->queue)->logger = w->logger;
MUTEX_INIT(w->mutex);
}
return w;
}
/* initialize XMAP */
XMAP *xmap_init(char *basedir)
{
char path[XM_PATH_MAX];
struct stat st = {0};
XMAP *xmap = NULL;
int i = 0, k = 0, v = 0;
unsigned int id = 0;
if(basedir && (xmap = (XMAP *)xmm_mnew(sizeof(XMAP))))
{
//xmap->mtrie = mtrie_init();
MUTEX_INIT(xmap->mutex);
/* logger */
sprintf(path, "%s/%s", basedir, "xmap.log");
xmap_mkdir(path);
LOGGER_INIT(xmap->logger, path);
/* tree */
sprintf(path, "%s/%s", basedir, "xmap.tree");
xmap->tree = mmtree_init(path);
sprintf(path, "%s/%s", basedir, "xmap.tree64");
xmap->tree64 = mmtree64_init(path);
/* kmap */
sprintf(path, "%s/%s", basedir, "xmap.kmap");
xmap->kmap = mmtrie_init(path);
/* queue */
sprintf(path, "%s/%s", basedir, "xmap.queue");
xmap->queue = mmqueue_init(path);
/* db */
sprintf(path, "%s/%s", basedir, "cache/");
xmap->db = cdb_init(path, CDB_USE_MMAP);
/* state */
sprintf(path, "%s/%s", basedir, "xmap.state");
if((xmap->stateio.fd = open(path, O_CREAT|O_RDWR, 0644)) > 0
&& fstat(xmap->stateio.fd, &st) == 0)
{
if((xmap->stateio.map = mmap(NULL, sizeof(XMSTATE), PROT_READ|PROT_WRITE,
MAP_SHARED, xmap->stateio.fd, 0)) == NULL
|| xmap->stateio.map == (void *)-1)
{
FATAL_LOGGER(xmap->logger, "mmap state:%s failed, %s", path, strerror(errno));
_exit(-1);
}
xmap->state = (XMSTATE *)(xmap->stateio.map);
xmap->stateio.end = st.st_size;
if(st.st_size == 0)
{
xmap->stateio.end = xmap->stateio.size = sizeof(XMSTATE);
if(ftruncate(xmap->stateio.fd, xmap->stateio.end) != 0)
{
FATAL_LOGGER(xmap->logger, "ftruncate state %s failed, %s", path, strerror(errno));
_exit(-1);
}
memset(xmap->state, 0, sizeof(XMSTATE));
}
for(i = 0; i < DBASE_MASK_MAX; i++)
{
if(xmap->state->masks[i].root == 0)
xmap->state->masks[i].root = mmtree64_new_tree(xmap->tree64);
}
}
/* disk */
sprintf(path, "%s/%s", basedir, "xmap.disk");
if((xmap->diskio.fd = open(path, O_CREAT|O_RDWR, 0644)) > 0
&& fstat(xmap->diskio.fd, &st) == 0)
{
if((xmap->diskio.map = mmap(NULL, sizeof(MDISK) * XM_DISK_MAX, PROT_READ|PROT_WRITE,
MAP_SHARED, xmap->diskio.fd, 0)) == NULL
|| xmap->diskio.map == (void *)-1)
{
FATAL_LOGGER(xmap->logger, "mmap disk:%s failed, %s", path, strerror(errno));
_exit(-1);
}
xmap->disks = (MDISK *)(xmap->diskio.map);
xmap->diskio.end = st.st_size;
}
/* meta */
sprintf(path, "%s/%s", basedir, "xmap.meta");
if((xmap->metaio.fd = open(path, O_CREAT|O_RDWR, 0644)) > 0
&& fstat(xmap->metaio.fd, &st) == 0)
{
if((xmap->metaio.map = mmap(NULL, sizeof(XMMETA) * XM_META_MAX, PROT_READ|PROT_WRITE,
MAP_SHARED, xmap->metaio.fd, 0)) == NULL
|| xmap->metaio.map == (void *)-1)
{
FATAL_LOGGER(xmap->logger, "mmap meta:%s failed, %s", path, strerror(errno));
_exit(-1);
}
xmap->metas = (XMMETA *)(xmap->metaio.map);
xmap->metaio.end = st.st_size;
xmap->start_time = time(NULL);
}
if(!xmap->state->qwait) xmap->state->qwait = mmtree_new_tree(xmap->tree);
if(!xmap->state->qleft) xmap->state->qleft = mmqueue_new(xmap->queue);
xmap->state->id_wait = 0;
while((id = mmtree_min(xmap->tree, xmap->state->qwait, &k, &v)))
{
mmtree_remove(xmap->tree, xmap->state->qwait, id, NULL, NULL);
}
while(mmqueue_pop(xmap->queue, xmap->state->qleft, (int *)&id) > 0);
}
return xmap;
}
/* add index */
int ibase_index(IBASE *ibase, int docid, IBDATA *block)
{
char *term = NULL, buf[IB_BUF_SIZE], *data = NULL, *p = NULL,
*pp = NULL, *nexts = NULL;
int i = 0, termid = 0, n = 0, k = 0, ndocid = 0, ret = -1, ndata = 0, secid = 0,
*intlist = NULL, *np = NULL, last_docid = 0;
DOCHEADER *docheader = NULL;
int64_t *longlist = NULL;
double *doublelist = NULL;
IHEADER *iheader = NULL;
STERM *termlist = NULL;
MDB *index = NULL, *posting = NULL;
off_t size = 0;
if((docheader = (DOCHEADER *)block->data)
&& (secid = docheader->secid) >= 0 && docheader->secid < IB_SEC_MAX)
{
index = (MDB *)(ibase->mindex[docheader->secid]);
posting = (MDB *)(ibase->mposting[docheader->secid]);
if(docheader->dbid != -1)
{
ibase->state->dtotal++;
ibase->state->ttotal += (int64_t)docheader->terms_total;
}
if(ibase->state->used_for == IB_USED_FOR_INDEXD)
{
/* add to source */
if(((off_t)docid * (off_t)sizeof(IHEADER)) >= ibase->state->headers[secid].end)
{
ibase->state->headers[secid].old = ibase->state->headers[secid].end;
size = (off_t)((docid / IB_HEADERS_BASE) + 1)
* (off_t)IB_HEADERS_BASE * (off_t)sizeof(IHEADER);
ret = ftruncate(ibase->state->headers[secid].fd, size);
ibase->state->headers[secid].end = size;
memset(ibase->state->headers[secid].map + ibase->state->headers[secid].old, 0,
ibase->state->headers[secid].end - ibase->state->headers[secid].old);
}
}
/* index */
//end = block->data + block->ndata;
termlist = (STERM *)(block->data + sizeof(DOCHEADER)
+ docheader->nfields * sizeof(XFIELD));
term = block->data + docheader->textblock_off;
nexts = block->data + docheader->nexts_off;
for(i = 0; i < docheader->nterms; i++)
{
termid = termlist[i].termid;
//find/add termid
if(termid > 0 && termlist[i].term_len > 0 && termlist[i].nexts_size >= 0
&& (ret=mmtrie_add((MMTRIE *)ibase->mmtrie,term,termlist[i].term_len,termid))>0)
{
if(ibase->state->index_status != IB_INDEX_DISABLED)
{
last_docid = 0;
ndocid = -1;
if(mdb_get_tag(index, termid, &last_docid) == 0)
{
ndocid = docid - last_docid;
if(ndocid <= 0) goto term_state_update;
}
else
{
ndocid = docid;
}
p = pp = buf;
if((ndata = ((sizeof(int) * 5))) > IB_BUF_SIZE)
p = pp = data = (char *)xmm_new(ndata);
/* compress index */
if(ibase->state->compression_status != IB_COMPRESSION_DISABLED)
{
n = ndocid; np = &n; ZVBCODE(np, p);
n = termlist[i].term_count; np = &n; ZVBCODE(np, p);
n = i; np = &n; ZVBCODE(np, p);
n = termlist[i].bit_fields; np = &n; ZVBCODE(np, p);
n = termlist[i].nexts_size; np = &n; ZVBCODE(np, p);
}
else
{
memcpy(p, &docid, sizeof(int));p += sizeof(int);
memcpy(p, &(termlist[i].term_count), sizeof(int));p += sizeof(int);
memcpy(p, &i, sizeof(int));p += sizeof(int);
memcpy(p, &(termlist[i].bit_fields), sizeof(int));p += sizeof(int);
memcpy(p, &(termlist[i].nexts_size), sizeof(int));p += sizeof(int);
}
if(termlist[i].nexts_size > 0)
{
if(mdb_add_data(posting, termid, nexts, termlist[i].nexts_size) <= 0)
{
FATAL_LOGGER(ibase->logger, "index posting term[%d] failed, %s",
termid, strerror(errno));
_exit(-1);
}
//WARN_LOGGER(ibase->logger, "docid:%lld termid:%d nexts_size:%d", IBLL(docheader->globalid), termid, termlist[i].nexts_size);
}
if(mdb_add_data(index, termid, pp, (p - pp)) <= 0)
{
FATAL_LOGGER(ibase->logger, "index term[%d] failed, %s",
termid, strerror(errno));
if(data){free(data); data = NULL;}
_exit(-1);
}
mdb_set_tag(index, termid, docid);
if(data){xmm_free(data, ndata); data = NULL;}
}
term_state_update:
MUTEX_LOCK(ibase->mutex_termstate);
if(termid > ibase->state->termid){ADD_TERMSTATE(ibase, termid);}
if(ndocid > 0 )
{
((TERMSTATE *)(ibase->termstateio.map))[termid].len=termlist[i].term_len;
((TERMSTATE *)(ibase->termstateio.map))[termid].total++;
}
MUTEX_UNLOCK(ibase->mutex_termstate);
}
else
{
FATAL_LOGGER(ibase->logger, "Invalid ret:%d term[%d]{%.*s} termid:%d in doc:%d", ret, i, termlist[i].term_len, term, termid, docid);
_exit(-1);
}
term += termlist[i].term_len;
nexts += termlist[i].nexts_size;
}
if(ibase->state->used_for == IB_USED_FOR_INDEXD)
{
/* index int */
if((n = ibase->state->int_index_fields_num) > 0
&& (intlist = (int *)(block->data + docheader->intblock_off)))
{
n += IB_INT_OFF;
k = 0;
for(i = IB_INT_OFF; i < n; i++)
{
IMAP_SET(ibase->state->mfields[secid][i], docid, intlist[k]);
k++;
}
}
/* index long */
if((n = ibase->state->long_index_fields_num) > 0
&& (longlist = (int64_t *)(block->data + docheader->longblock_off)))
{
n += IB_LONG_OFF;
k = 0;
for(i = IB_LONG_OFF; i < n; i++)
{
LMAP_SET(ibase->state->mfields[secid][i], docid, longlist[k]);
k++;
}
}
/* index double */
if((n = ibase->state->double_index_fields_num) > 0
&& (doublelist = (double *)(block->data + docheader->doubleblock_off)))
{
n += IB_DOUBLE_OFF;
k = 0;
for(i = IB_DOUBLE_OFF; i < n; i++)
{
DMAP_SET(ibase->state->mfields[secid][i], docid, doublelist[k]);
k++;
}
}
if((iheader = PIHEADER(ibase, docheader->secid, docid)))
{
iheader->status = docheader->status;
iheader->terms_total = docheader->terms_total;
iheader->crc = docheader->crc;
iheader->category = docheader->category;
iheader->slevel = docheader->slevel;
iheader->rank = docheader->rank;
iheader->globalid = docheader->globalid;
//WARN_LOGGER(ibase->logger, "iheader->category:%p docheader->category:%p", (void *)iheader->category, (void *)docheader->category);
}
}
ret = 0;
}
return ret;
}
/* add document */
int ibase_add_document(IBASE *ibase, IBDATA *block)
{
int ret = -1, localid = 0, secid = 0, n = 0, newid = 0;
DOCHEADER *docheader = NULL;
MHEADER *mheader = NULL;
IHEADER *iheader = NULL;
char line[IB_LINE_MAX];
off_t size = 0;
if(ibase && block && block->ndata > 0 && (docheader = (DOCHEADER *)(block->data))
&& (secid = docheader->secid) >= 0 && secid < IB_SEC_MAX
&& ibase_check_index_state(ibase, docheader) == 0
&& docheader->globalid && (n = sprintf(line, "%lld", IBLL(docheader->globalid))))
{
if((localid = mmtrie_xadd(ibase->docmap, line, n)) > 0)
{
MUTEX_LOCK(ibase->mutex);
if(localid <= ibase->state->docid)
{
if(ibase->state->used_for == IB_USED_FOR_INDEXD)
{
if((mheader = PMHEADER(ibase, localid))
&& (iheader = PIHEADER(ibase, mheader->secid, mheader->docid)))
{
/* disabled */
if(docheader->status < 0)
{
mheader->status = -1;
iheader->status = -1;
DEBUG_LOGGER(ibase->logger, "Update docid:%d globalid:%lld status:%d", localid, IBLL(docheader->globalid), docheader->status);
ret = 0;
}
else if(docheader->crc != mheader->crc)
{
ibase_del_index(ibase, secid, localid);
newid = ++(ibase->state->ids[secid]);
DEBUG_LOGGER(ibase->logger, "ready_reindex[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
mheader->docid = newid;
mheader->crc = docheader->crc;
ret = ibase_index(ibase, newid, block);
DEBUG_LOGGER(ibase->logger, "over_reindex[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
}
else
{
DEBUG_LOGGER(ibase->logger, "update_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
ret = ibase_update_index(ibase, mheader->docid, block);
DEBUG_LOGGER(ibase->logger, "over_update_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
}
}
}
else
{
docheader->dbid = -1;
ret = ibase_index(ibase, localid, block);
}
}
else
{
if(ibase->state->used_for == IB_USED_FOR_INDEXD)
{
if(((off_t)localid * (off_t)sizeof(MHEADER)) >= ibase->mheadersio.end)
{
ibase->mheadersio.old = ibase->mheadersio.end;
size = (off_t)((localid / IB_HEADERS_BASE) + 1)
* (off_t)IB_HEADERS_BASE * (off_t)sizeof(MHEADER);
ret = ftruncate(ibase->mheadersio.fd, size);
ibase->mheadersio.end = size;
memset(ibase->mheadersio.map + ibase->mheadersio.old, 0,
ibase->mheadersio.end - ibase->mheadersio.old);
}
if((mheader = PMHEADER(ibase, localid)))
{
mheader->status = 0;
mheader->docid = ++(ibase->state->ids[secid]);
mheader->secid = docheader->secid;
mheader->crc = docheader->crc;
DEBUG_LOGGER(ibase->logger, "new_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
ret = ibase_index(ibase, mheader->docid, block);
}
}
else
{
ret = ibase_index(ibase, localid, block);
}
ibase->state->docid = localid;
}
MUTEX_UNLOCK(ibase->mutex);
if(ibase->state->used_for != IB_USED_FOR_QPARSERD
&& ibase->state->mmsource_status != IB_MMSOURCE_NULL && localid > 0)
{
ACCESS_LOGGER(ibase->logger, "docid:%lld/%d c_size:%d c_zsize:%d size:%d", docheader->globalid, localid, docheader->content_size, docheader->content_zsize, docheader->nexts_off);
docheader->size = docheader->nexts_off;
ret = db_set_data(PDB(ibase->source), localid, block->data, docheader->size);
}
ret = 0;
}
else
{
FATAL_LOGGER(ibase->logger, "new_index[%lld]{crc:%d rank:%f slevel:%d category:%lld} failed, %s", IBLL(docheader->globalid), docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category), strerror(errno));
}
}
return ret;
}
/* read upindex */
int qindex_read_upindex(QINDEX *qindex, int taskid, char *data, int *len, int *count)
{
int ret = -1, mid = -1, nodeid = -1, id = 0, k = 0, n = 0,
left = 0, prev = 0, next = 0, *px = NULL;
DOCHEADER *docheader = NULL;
XPACKET *xpackets = NULL;
QTNODE *nodes = NULL;
QTASK *tasks = NULL;
char *p = NULL;
if(qindex && taskid >= 0 && data && taskid < Q_NODE_MAX * Q_TASKS_MAX
&& len && count && *len > 0)
{
MUTEX_LOCK(qindex->mutex);
*count = 0;
k = taskid % Q_TASKS_MAX;
if((nodeid = (taskid / Q_TASKS_MAX)) < Q_NODE_MAX && (nodes = qindex->state->nodes)
&& (tasks = qindex->state->nodes[nodeid].tasks) && tasks[k].status > 0
&& (xpackets = (XPACKET *)(qindex->xpacketio.map)))
{
if(tasks[k].nupdates > 0 && tasks[k].upid != tasks[k].upover)
{
if((*len = db_read_data(PDB(qindex->update), taskid, p)) > 0)
{
*count = tasks[k].upcount;
ret = mid = tasks[k].upid;
}
else
{
tasks[k].upid = tasks[k].upover = 0;
}
}
else if(tasks[k].nqueue > 0 && tasks[k].mmqid > 0)
{
tasks[k].upcount = 0;
left = *len;
p = data;
id = mmqueue_head(MMQ(qindex->mmqueue), tasks[k].mmqid, &mid);
do
{
if(id > 0 && mid > 0)
{
if(nodes[nodeid].type == Q_NODE_INDEXD && xpackets[mid].nodeid != nodeid)
{
prev = xpackets[mid].prev;
next = xpackets[mid].next;
FATAL_LOGGER(qindex->logger, "Invalid rootid:%d id:%d prev:%d[nodeid:%d] next:%d[nodeid:%d] xpacket[%d].nodeid[%d] to task[%s:%d].nodeid:%d nqueue:%d", tasks[k].mmqid, id, prev, xpackets[prev].nodeid, next, xpackets[next].nodeid, mid, xpackets[mid].nodeid, tasks[k].ip, tasks[k].port, nodeid, tasks[k].nqueue);
tasks[k].mmqid = 0;tasks[k].nqueue = 0;
break;
}
if(db_get_data_len(PDB(qindex->db), mid) > (left-Q_LEFT_LEN)) break;
tasks[k].upid = mid;
px = (int *)p;
p += sizeof(int);
docheader = (DOCHEADER *)p;
if((n = db_read_data(PDB(qindex->db), mid, p)) > sizeof(DOCHEADER))
{
ACCESS_LOGGER(qindex->logger, "update_index(%d) globalid:%lld mid:%d status:%d nodeid:%d task[%s:%d]", id, LL(docheader->globalid), mid, xpackets[mid].status, xpackets[mid].nodeid, tasks[k].ip, tasks[k].port);
left -= n + sizeof(int);
p += n;
*px = n;
tasks[k].upcount++;
}
else
{
p -= sizeof(int);
FATAL_LOGGER(qindex->logger, "Invalid data id:%d to document:%lld", id, LL(docheader->globalid));
mid = -1;
//_exit(-1);
break;
}
mmqueue_pop(MMQ(qindex->mmqueue),tasks[k].mmqid, &mid);
tasks[k].nqueue--;
id = mmqueue_head(MMQ(qindex->mmqueue), tasks[k].mmqid, &mid);
}
else
{
FATAL_LOGGER(qindex->logger, "Invalid qid:%d mid:%d", id, mid);
tasks[k].mmqid = 0;tasks[k].nqueue = 0;
mid = -1;
break;
}
}while(tasks[k].nqueue > 0 && left > Q_LEFT_LEN && id > 0 && mid > 0);
if((*count = tasks[k].upcount) > 0 && (*len -= left) > 0
&& db_set_data(PDB(qindex->update), taskid, data, *len) >= 0)
{
ret = tasks[k].upid;
}
}
else ret = -1;
}
else ret = -2;
MUTEX_UNLOCK(qindex->mutex);
}
return ret;
}
/* read index */
int qindex_read_index(QINDEX *qindex, int taskid, char *data, int *len, int *count)
{
int id = -1, nodeid = 0, x = 0, k = 0, *px = NULL, left = 0, last = 0, n = 0;
DOCHEADER *docheader = NULL;
XPACKET *xpackets = NULL;
QTNODE *nodes = NULL;
QTASK *tasks = NULL;
char *p = NULL;
if(qindex && qindex->state && taskid >= 0 && taskid < (Q_NODE_MAX * Q_TASKS_MAX)
&& data && len && *len > 0 && count)
{
MUTEX_LOCK(qindex->mutex);
*count = 0;
k = taskid % Q_TASKS_MAX;
if((nodeid = (taskid / Q_TASKS_MAX)) < Q_NODE_MAX
&& (xpackets = (XPACKET *)(qindex->xpacketio.map))
&& (nodes = qindex->state->nodes) && (tasks = nodes[nodeid].tasks)
&& tasks[k].status > 0)
{
tasks[k].count = 0;
//check limit
if(nodes[nodeid].limit > 0 && tasks[k].nxpackets >= nodes[nodeid].limit) goto end;
//check last over
if(tasks[k].popid != tasks[k].over)
{
tasks[k].popid = tasks[k].last;
}
else
tasks[k].last = tasks[k].popid;
p = data;
left = *len;
while(left > Q_LEFT_LEN)
{
last = tasks[k].popid;
if(nodes[nodeid].type == Q_NODE_DOCD || nodes[nodeid].type == Q_NODE_PARSERD)
{
//if(tasks[k].popid == 0) tasks[k].popid = 1;
if((tasks[k].popid+1) < qindex->state->xpackettotal)
{
if(tasks[k].popid > 0) id = ++(tasks[k].popid);
else id = tasks[k].popid = 1;
if(id > qindex->state->docpopid)
qindex->state->docpopid = id;
if(id > nodes[nodeid].total)
nodes[nodeid].total = id;
}
else
{
id = -1;
break;
}
}
else if(nodes[nodeid].type == Q_NODE_INDEXD)
{
if((x = tasks[k].popid) == nodes[nodeid].last)
{
if(qindex->state->popid == 0) qindex->state->popid = 1;
if((id = qindex->state->popid) > 0 && id <= qindex->state->docpopid)
{
if(nodes[nodeid].total == 0)
{
nodes[nodeid].last = nodes[nodeid].first = id;
}
else
{
xpackets[x].next = id;
xpackets[id].prev = x;
}
xpackets[id].nodeid = nodeid;
//xpackets[id].status = 1;
tasks[k].popid = id;
nodes[nodeid].last = id;
nodes[nodeid].total++;
qindex->state->popid++;
}
else id = -1;
}
else
{
if(nodes[nodeid].total > 0 && tasks[k].popid < nodes[nodeid].last)
{
if(tasks[k].nxpackets == 0 && tasks[k].count == 0)
{
tasks[k].popid = id = nodes[nodeid].first;
}
else
{
x = tasks[k].popid;
id = xpackets[x].next;
tasks[k].popid = id;
}
}
else id = -1;
}
}
else id = -1;
if(id > 0)
{
if(nodes[nodeid].type == Q_NODE_INDEXD && xpackets[id].nodeid != nodeid)
{
FATAL_LOGGER(qindex->logger, "Invalid xpacket[%d].nodeid[%d] to task[%s:%d].nodeid:%d", id, xpackets[id].nodeid, tasks[k].ip, tasks[k].port, nodeid);
_exit(-1);
}
if((n = db_get_data_len(PDB(qindex->db), id)) <= (left - Q_LEFT_LEN))
{
px = (int *)p;
p += sizeof(int);
docheader = (DOCHEADER *)p;
if((n = db_read_data(PDB(qindex->db), id, p)) > sizeof(DOCHEADER))
{
if(docheader->size < 0 || docheader->size != n)
{
FATAL_LOGGER(qindex->logger, "Invalid data id:%d size:%d n:%d", id, docheader->size, n);
break;
}
//docheader->globalid = id;
*px = n;
left -= n + sizeof(int);
p += n;
tasks[k].count++;
}
else
{
p -= sizeof(int);
FATAL_LOGGER(qindex->logger, "Invalid data id:%d to document:%lld", id, LL(docheader->globalid));
//_exit(-1);
break;
}
last = tasks[k].popid;
}
else
{
tasks[k].popid = last;
break;
}
}
else
{
break;
}
}
end:
*len -= left;
if((*count = tasks[k].count) > 0)
{
id = tasks[k].popid = last ;
}
else id = -1;
}
else id = -2;
MUTEX_UNLOCK(qindex->mutex);
}
return id;
}
/* working for get_data() and push_index() */
int qindex_work(QINDEX *qindex, MINDEX *mindex)
{
int ret = -1, i = 0, x = 0 , n = 0, mid = 0;
int64_t *xid = NULL, key = 0, vlong = 0;
BELEMENT *root = NULL, *sub = NULL;
DOCHEADER *docheader = NULL;
STERM *termlist = NULL;
IBDATA block = {0};
char *term = NULL;
BRES *res = NULL;
if(qindex && mindex && mindex_check(mindex) == 0)
{
if(mtask_pop(&(mindex->mtask)) > 0 && (xid = (int64_t *)mindex->mtask.packet)
&& mindex->mtask.length > sizeof(int64_t))
{
n = mindex->mtask.length/sizeof(int64_t);
brequest_reset(&(mindex->request));
brequest_append_keys(&(mindex->request), xid, n);
brequest_finish(&(mindex->request));
if((res = dbase_get_records(&(mindex->db), &(mindex->request))))
{
x = 0;
while((root = dbase_next_record(res, root, &key)))
{
ACCESS_LOGGER(qindex->logger, "index_record[%d/%d] packetid:%d key:%lld", x, n, mindex->mtask.packetid, LLI(key));
vlong = 0;
if((sub = belement_find(root, qindex->db_key_name)))
{
belement_v_long(sub, &vlong);
}
if(key != vlong)
{
ACCESS_LOGGER(qindex->logger, "Invalid record[%d/%d] packetid:%d key:%lld/%lld", x, n, mindex->mtask.packetid, LLI(vlong), LLI(key));
return ret;
}
if((sub = belement_find(root, qindex->db_index_block_name))
&& (block.ndata = belement_v_blob(sub, &(block.data))) > 0)
{
docheader = (DOCHEADER *)block.data;
termlist = (STERM *)(block.data + sizeof(DOCHEADER)
+ docheader->nfields * sizeof(XFIELD));
term = block.data + docheader->textblock_off;
for(i = 0; i < docheader->nterms; i++)
{
termlist[i].termid = mmtrie_xadd(qindex->xdict, term, termlist[i].term_len);
term += termlist[i].term_len;
}
mid = qindex_push_index(qindex, key, &block);
ACCESS_LOGGER(qindex->logger, "over_record[%d/%d] packetid:%d key:%lld", x, n, mindex->mtask.packetid, LLI(key));
}
else
{
FATAL_LOGGER(qindex->logger, "Invalid data feilds[%s]", qindex->db_index_block_name);
return ret;
}
x++;
}
ret = 0;
mtask_finish(&(mindex->mtask), 0);
}
else
{
FATAL_LOGGER(qindex->logger, "get_records(%d) failed, %s", mindex->mtask.packetid, strerror(errno));
return ret;
}
}
}
return ret;
}
