void
mmumapcpu0(void)
{
ulong *pdb, *pte, va, pa, pdbx;
if(strstr(xenstart->magic, "x86_32p"))
paemode = 1;
hypervisor_virt_start = paemode ? 0xF5800000 : 0xFC000000;
patomfn = (ulong*)xenstart->mfn_list;
matopfn = (ulong*)hypervisor_virt_start;
/* Xen bug ? can't touch top entry in PDPT */
if(paemode)
hypervisor_virt_start = 0xC0000000;
/*
* map CPU0MACH at MACHADDR.
* When called the pagedir and page table exist, we just
* need to fill in a page table entry.
*/
pdb = (ulong*)xenstart->pt_base;
va = MACHADDR;
pa = PADDR(CPU0MACH) | PTEVALID|PTEWRITE;
pdbx = PDX(va);
pdb = PDB(pdb, va);
pte = KADDR(MAPPN(pdb[pdbx]));
xenupdate(&pte[PTX(va)], pa);
}
/* del index */
int ibase_del_index(IBASE *ibase, int secid, int localid)
{
MHEADER *mheader = NULL;
IHEADER *iheader = NULL;
int ret = -1, i = 0, k = 0, n = 0;
if((mheader = PMHEADER(ibase, localid)))
{
ibase->state->dtotal--;
if(mheader->docid > 0 && (iheader = PIHEADER(ibase, mheader->secid, mheader->docid)))
{
ibase->state->ttotal -= iheader->terms_total;
iheader->status = -1;
if(ibase->state->used_for == IB_USED_FOR_INDEXD)
{
/* del int index */
if((n = ibase->state->int_index_fields_num) > 0)
{
n += IB_INT_OFF;
k = 0;
for(i = IB_INT_OFF; i < n; i++)
{
IMAP_DEL(ibase->state->mfields[secid][i], localid);
k++;
}
}
/* del long index */
if((n = ibase->state->long_index_fields_num) > 0)
{
n += IB_LONG_OFF;
k = 0;
for(i = IB_LONG_OFF; i < n; i++)
{
LMAP_DEL(ibase->state->mfields[secid][i], localid);
k++;
}
}
/* del double index */
if((n = ibase->state->double_index_fields_num) > 0)
{
n += IB_DOUBLE_OFF;
k = 0;
for(i = IB_DOUBLE_OFF; i < n; i++)
{
DMAP_DEL(ibase->state->mfields[secid][i], localid);
k++;
}
}
}
}
if(ibase->state->used_for == IB_USED_FOR_INDEXD
&& ibase->state->mmsource_status != IB_MMSOURCE_NULL)
{
ret = db_del_data(PDB(ibase->source), localid);
}
ret = 0;
}
return ret;
}
/* clean */
void qindex_clean(QINDEX *qindex)
{
if(qindex)
{
if(qindex->state) munmap(qindex->state, sizeof(QSTATE));
if(qindex->qstatefd > 0)close(qindex->qstatefd);
if(qindex->db) db_clean(PDB(qindex->db));
if(qindex->update) db_clean(PDB(qindex->update));
MUTEX_DESTROY(qindex->mutex);
if(qindex->xdict){mmtrie_clean(qindex->xdict);}
if(qindex->namemap){mmtrie_clean(qindex->namemap);}
if(qindex->idmap){mmtree64_close(qindex->idmap);}
if(qindex->queue){iqueue_clean(qindex->queue);}
if(qindex->mmqueue){mmqueue_clean(MMQ(qindex->mmqueue));}
if(qindex->xpacketio.map) munmap(qindex->xpacketio.map, qindex->xpacketio.size);
if(qindex->xpacketio.fd) close(qindex->xpacketio.fd);
if(qindex->bstermio.map) munmap(qindex->bstermio.map, qindex->bstermio.size);
if(qindex->bstermio.fd) close(qindex->bstermio.fd);
if(qindex->map) mmtrie_clean(qindex->map);
if(qindex->logger){LOGGER_CLEAN(qindex->logger);}
xmm_free(qindex, sizeof(QINDEX));
}
return ;
}
/* push index */
int qindex_push_index(QINDEX *qindex, int64_t key, IBDATA *block)
{
DOCHEADER *docheader = NULL;
XPACKET *xpackets = NULL;
int mid = 0, old = 0;
if(qindex && block->ndata > 0 && (docheader = (DOCHEADER *)block->data))
{
MUTEX_LOCK(qindex->mutex);
if(docheader->size == block->ndata && (xpackets = (XPACKET *)(qindex->xpacketio.map))
&& (mid = mmtree64_try_insert(qindex->idmap, qindex->state->rootid,
key, (qindex->state->id_max+1), &old)) > 0)
{
if(old > 0) mid = old;
else mid = ++(qindex->state->id_max);
db_set_data(PDB(qindex->db), mid, block->data, block->ndata);
if(mid <= qindex->state->xpackettotal)
{
if(xpackets[mid].crc != docheader->crc)
qindex_update(qindex, mid, 1);
else
qindex_update(qindex, mid, 0);
xpackets[mid].status = docheader->status;
xpackets[mid].crc = docheader->crc;
ACCESS_LOGGER(qindex->logger, "update-index{gloablid:%lld mid:%d total:%d}", LL(docheader->globalid), mid, qindex->state->xpackettotal);
}
else
{
qindex->state->xpackettotal = mid;
CHECK_XPACKETIO(qindex);
if((xpackets = (XPACKET *)(qindex->xpacketio.map)))
{
xpackets[mid].status = docheader->status;
xpackets[mid].crc = docheader->crc;
ACCESS_LOGGER(qindex->logger, "new-index{gloablid:%lld mid:%d}", LL(docheader->globalid), mid);
}
}
}
else
{
FATAL_LOGGER(qindex->logger, "Invalid document size:%d ndata:%d id:%lld crc:%d", docheader->size, block->ndata, LL(docheader->globalid), docheader->crc);
_exit(-1);
}
MUTEX_UNLOCK(qindex->mutex);
}
return mid;
}
ulong
mmumapframe(ulong va, ulong mfn)
{
ulong *pte, pdbx;
uvlong ma;
/*
* map machine frame number to a virtual address.
* When called the pagedir and page table exist, we just
* need to fill in a page table entry.
*/
ma = ((uvlong)mfn<<PGSHIFT) | PTEVALID|PTEWRITE;
pdbx = PDX(va);
pte = KADDR(MAPPN(PDB(m->pdb,va)[pdbx]));
xenupdatema(&pte[PTX(va)], ma);
return va;
}
/* add document */
int ibase_add_document(IBASE *ibase, IBDATA *block)
{
int ret = -1, localid = 0, secid = 0, n = 0, newid = 0;
DOCHEADER *docheader = NULL;
MHEADER *mheader = NULL;
IHEADER *iheader = NULL;
char line[IB_LINE_MAX];
off_t size = 0;
if(ibase && block && block->ndata > 0 && (docheader = (DOCHEADER *)(block->data))
&& (secid = docheader->secid) >= 0 && secid < IB_SEC_MAX
&& ibase_check_index_state(ibase, docheader) == 0
&& docheader->globalid && (n = sprintf(line, "%lld", IBLL(docheader->globalid))))
{
if((localid = mmtrie_xadd(ibase->docmap, line, n)) > 0)
{
MUTEX_LOCK(ibase->mutex);
if(localid <= ibase->state->docid)
{
if(ibase->state->used_for == IB_USED_FOR_INDEXD)
{
if((mheader = PMHEADER(ibase, localid))
&& (iheader = PIHEADER(ibase, mheader->secid, mheader->docid)))
{
/* disabled */
if(docheader->status < 0)
{
mheader->status = -1;
iheader->status = -1;
DEBUG_LOGGER(ibase->logger, "Update docid:%d globalid:%lld status:%d", localid, IBLL(docheader->globalid), docheader->status);
ret = 0;
}
else if(docheader->crc != mheader->crc)
{
ibase_del_index(ibase, secid, localid);
newid = ++(ibase->state->ids[secid]);
DEBUG_LOGGER(ibase->logger, "ready_reindex[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
mheader->docid = newid;
mheader->crc = docheader->crc;
ret = ibase_index(ibase, newid, block);
DEBUG_LOGGER(ibase->logger, "over_reindex[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
}
else
{
DEBUG_LOGGER(ibase->logger, "update_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
ret = ibase_update_index(ibase, mheader->docid, block);
DEBUG_LOGGER(ibase->logger, "over_update_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
}
}
}
else
{
docheader->dbid = -1;
ret = ibase_index(ibase, localid, block);
}
}
else
{
if(ibase->state->used_for == IB_USED_FOR_INDEXD)
{
if(((off_t)localid * (off_t)sizeof(MHEADER)) >= ibase->mheadersio.end)
{
ibase->mheadersio.old = ibase->mheadersio.end;
size = (off_t)((localid / IB_HEADERS_BASE) + 1)
* (off_t)IB_HEADERS_BASE * (off_t)sizeof(MHEADER);
ret = ftruncate(ibase->mheadersio.fd, size);
ibase->mheadersio.end = size;
memset(ibase->mheadersio.map + ibase->mheadersio.old, 0,
ibase->mheadersio.end - ibase->mheadersio.old);
}
if((mheader = PMHEADER(ibase, localid)))
{
mheader->status = 0;
mheader->docid = ++(ibase->state->ids[secid]);
mheader->secid = docheader->secid;
mheader->crc = docheader->crc;
DEBUG_LOGGER(ibase->logger, "new_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
ret = ibase_index(ibase, mheader->docid, block);
}
}
else
{
ret = ibase_index(ibase, localid, block);
}
ibase->state->docid = localid;
}
MUTEX_UNLOCK(ibase->mutex);
if(ibase->state->used_for != IB_USED_FOR_QPARSERD
&& ibase->state->mmsource_status != IB_MMSOURCE_NULL && localid > 0)
{
ACCESS_LOGGER(ibase->logger, "docid:%lld/%d c_size:%d c_zsize:%d size:%d", docheader->globalid, localid, docheader->content_size, docheader->content_zsize, docheader->nexts_off);
docheader->size = docheader->nexts_off;
ret = db_set_data(PDB(ibase->source), localid, block->data, docheader->size);
}
ret = 0;
}
else
{
FATAL_LOGGER(ibase->logger, "new_index[%lld]{crc:%d rank:%f slevel:%d category:%lld} failed, %s", IBLL(docheader->globalid), docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category), strerror(errno));
}
}
return ret;
}
/* read upindex */
int qindex_read_upindex(QINDEX *qindex, int taskid, char *data, int *len, int *count)
{
int ret = -1, mid = -1, nodeid = -1, id = 0, k = 0, n = 0,
left = 0, prev = 0, next = 0, *px = NULL;
DOCHEADER *docheader = NULL;
XPACKET *xpackets = NULL;
QTNODE *nodes = NULL;
QTASK *tasks = NULL;
char *p = NULL;
if(qindex && taskid >= 0 && data && taskid < Q_NODE_MAX * Q_TASKS_MAX
&& len && count && *len > 0)
{
MUTEX_LOCK(qindex->mutex);
*count = 0;
k = taskid % Q_TASKS_MAX;
if((nodeid = (taskid / Q_TASKS_MAX)) < Q_NODE_MAX && (nodes = qindex->state->nodes)
&& (tasks = qindex->state->nodes[nodeid].tasks) && tasks[k].status > 0
&& (xpackets = (XPACKET *)(qindex->xpacketio.map)))
{
if(tasks[k].nupdates > 0 && tasks[k].upid != tasks[k].upover)
{
if((*len = db_read_data(PDB(qindex->update), taskid, p)) > 0)
{
*count = tasks[k].upcount;
ret = mid = tasks[k].upid;
}
else
{
tasks[k].upid = tasks[k].upover = 0;
}
}
else if(tasks[k].nqueue > 0 && tasks[k].mmqid > 0)
{
tasks[k].upcount = 0;
left = *len;
p = data;
id = mmqueue_head(MMQ(qindex->mmqueue), tasks[k].mmqid, &mid);
do
{
if(id > 0 && mid > 0)
{
if(nodes[nodeid].type == Q_NODE_INDEXD && xpackets[mid].nodeid != nodeid)
{
prev = xpackets[mid].prev;
next = xpackets[mid].next;
FATAL_LOGGER(qindex->logger, "Invalid rootid:%d id:%d prev:%d[nodeid:%d] next:%d[nodeid:%d] xpacket[%d].nodeid[%d] to task[%s:%d].nodeid:%d nqueue:%d", tasks[k].mmqid, id, prev, xpackets[prev].nodeid, next, xpackets[next].nodeid, mid, xpackets[mid].nodeid, tasks[k].ip, tasks[k].port, nodeid, tasks[k].nqueue);
tasks[k].mmqid = 0;tasks[k].nqueue = 0;
break;
}
if(db_get_data_len(PDB(qindex->db), mid) > (left-Q_LEFT_LEN)) break;
tasks[k].upid = mid;
px = (int *)p;
p += sizeof(int);
docheader = (DOCHEADER *)p;
if((n = db_read_data(PDB(qindex->db), mid, p)) > sizeof(DOCHEADER))
{
ACCESS_LOGGER(qindex->logger, "update_index(%d) globalid:%lld mid:%d status:%d nodeid:%d task[%s:%d]", id, LL(docheader->globalid), mid, xpackets[mid].status, xpackets[mid].nodeid, tasks[k].ip, tasks[k].port);
left -= n + sizeof(int);
p += n;
*px = n;
tasks[k].upcount++;
}
else
{
p -= sizeof(int);
FATAL_LOGGER(qindex->logger, "Invalid data id:%d to document:%lld", id, LL(docheader->globalid));
mid = -1;
//_exit(-1);
break;
}
mmqueue_pop(MMQ(qindex->mmqueue),tasks[k].mmqid, &mid);
tasks[k].nqueue--;
id = mmqueue_head(MMQ(qindex->mmqueue), tasks[k].mmqid, &mid);
}
else
{
FATAL_LOGGER(qindex->logger, "Invalid qid:%d mid:%d", id, mid);
tasks[k].mmqid = 0;tasks[k].nqueue = 0;
mid = -1;
break;
}
}while(tasks[k].nqueue > 0 && left > Q_LEFT_LEN && id > 0 && mid > 0);
if((*count = tasks[k].upcount) > 0 && (*len -= left) > 0
&& db_set_data(PDB(qindex->update), taskid, data, *len) >= 0)
{
ret = tasks[k].upid;
}
}
else ret = -1;
}
else ret = -2;
MUTEX_UNLOCK(qindex->mutex);
}
return ret;
}
/* read index */
int qindex_read_index(QINDEX *qindex, int taskid, char *data, int *len, int *count)
{
int id = -1, nodeid = 0, x = 0, k = 0, *px = NULL, left = 0, last = 0, n = 0;
DOCHEADER *docheader = NULL;
XPACKET *xpackets = NULL;
QTNODE *nodes = NULL;
QTASK *tasks = NULL;
char *p = NULL;
if(qindex && qindex->state && taskid >= 0 && taskid < (Q_NODE_MAX * Q_TASKS_MAX)
&& data && len && *len > 0 && count)
{
MUTEX_LOCK(qindex->mutex);
*count = 0;
k = taskid % Q_TASKS_MAX;
if((nodeid = (taskid / Q_TASKS_MAX)) < Q_NODE_MAX
&& (xpackets = (XPACKET *)(qindex->xpacketio.map))
&& (nodes = qindex->state->nodes) && (tasks = nodes[nodeid].tasks)
&& tasks[k].status > 0)
{
tasks[k].count = 0;
//check limit
if(nodes[nodeid].limit > 0 && tasks[k].nxpackets >= nodes[nodeid].limit) goto end;
//check last over
if(tasks[k].popid != tasks[k].over)
{
tasks[k].popid = tasks[k].last;
}
else
tasks[k].last = tasks[k].popid;
p = data;
left = *len;
while(left > Q_LEFT_LEN)
{
last = tasks[k].popid;
if(nodes[nodeid].type == Q_NODE_DOCD || nodes[nodeid].type == Q_NODE_PARSERD)
{
//if(tasks[k].popid == 0) tasks[k].popid = 1;
if((tasks[k].popid+1) < qindex->state->xpackettotal)
{
if(tasks[k].popid > 0) id = ++(tasks[k].popid);
else id = tasks[k].popid = 1;
if(id > qindex->state->docpopid)
qindex->state->docpopid = id;
if(id > nodes[nodeid].total)
nodes[nodeid].total = id;
}
else
{
id = -1;
break;
}
}
else if(nodes[nodeid].type == Q_NODE_INDEXD)
{
if((x = tasks[k].popid) == nodes[nodeid].last)
{
if(qindex->state->popid == 0) qindex->state->popid = 1;
if((id = qindex->state->popid) > 0 && id <= qindex->state->docpopid)
{
if(nodes[nodeid].total == 0)
{
nodes[nodeid].last = nodes[nodeid].first = id;
}
else
{
xpackets[x].next = id;
xpackets[id].prev = x;
}
xpackets[id].nodeid = nodeid;
//xpackets[id].status = 1;
tasks[k].popid = id;
nodes[nodeid].last = id;
nodes[nodeid].total++;
qindex->state->popid++;
}
else id = -1;
}
else
{
if(nodes[nodeid].total > 0 && tasks[k].popid < nodes[nodeid].last)
{
if(tasks[k].nxpackets == 0 && tasks[k].count == 0)
{
tasks[k].popid = id = nodes[nodeid].first;
}
else
{
x = tasks[k].popid;
id = xpackets[x].next;
tasks[k].popid = id;
}
}
else id = -1;
}
}
else id = -1;
if(id > 0)
{
if(nodes[nodeid].type == Q_NODE_INDEXD && xpackets[id].nodeid != nodeid)
{
FATAL_LOGGER(qindex->logger, "Invalid xpacket[%d].nodeid[%d] to task[%s:%d].nodeid:%d", id, xpackets[id].nodeid, tasks[k].ip, tasks[k].port, nodeid);
_exit(-1);
}
if((n = db_get_data_len(PDB(qindex->db), id)) <= (left - Q_LEFT_LEN))
{
px = (int *)p;
p += sizeof(int);
docheader = (DOCHEADER *)p;
if((n = db_read_data(PDB(qindex->db), id, p)) > sizeof(DOCHEADER))
{
if(docheader->size < 0 || docheader->size != n)
{
FATAL_LOGGER(qindex->logger, "Invalid data id:%d size:%d n:%d", id, docheader->size, n);
break;
}
//docheader->globalid = id;
*px = n;
left -= n + sizeof(int);
p += n;
tasks[k].count++;
}
else
{
p -= sizeof(int);
FATAL_LOGGER(qindex->logger, "Invalid data id:%d to document:%lld", id, LL(docheader->globalid));
//_exit(-1);
break;
}
last = tasks[k].popid;
}
else
{
tasks[k].popid = last;
break;
}
}
else
{
break;
}
}
end:
*len -= left;
if((*count = tasks[k].count) > 0)
{
id = tasks[k].popid = last ;
}
else id = -1;
}
else id = -2;
MUTEX_UNLOCK(qindex->mutex);
}
return id;
}