/* add document */
int ibase_add_document(IBASE *ibase, IBDATA *block)
{
int ret = -1, localid = 0, secid = 0, n = 0, newid = 0;
DOCHEADER *docheader = NULL;
MHEADER *mheader = NULL;
IHEADER *iheader = NULL;
char line[IB_LINE_MAX];
off_t size = 0;
if(ibase && block && block->ndata > 0 && (docheader = (DOCHEADER *)(block->data))
&& (secid = docheader->secid) >= 0 && secid < IB_SEC_MAX
&& ibase_check_index_state(ibase, docheader) == 0
&& docheader->globalid && (n = sprintf(line, "%lld", IBLL(docheader->globalid))))
{
if((localid = mmtrie_xadd(ibase->docmap, line, n)) > 0)
{
MUTEX_LOCK(ibase->mutex);
if(localid <= ibase->state->docid)
{
if(ibase->state->used_for == IB_USED_FOR_INDEXD)
{
if((mheader = PMHEADER(ibase, localid))
&& (iheader = PIHEADER(ibase, mheader->secid, mheader->docid)))
{
/* disabled */
if(docheader->status < 0)
{
mheader->status = -1;
iheader->status = -1;
DEBUG_LOGGER(ibase->logger, "Update docid:%d globalid:%lld status:%d", localid, IBLL(docheader->globalid), docheader->status);
ret = 0;
}
else if(docheader->crc != mheader->crc)
{
ibase_del_index(ibase, secid, localid);
newid = ++(ibase->state->ids[secid]);
DEBUG_LOGGER(ibase->logger, "ready_reindex[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
mheader->docid = newid;
mheader->crc = docheader->crc;
ret = ibase_index(ibase, newid, block);
DEBUG_LOGGER(ibase->logger, "over_reindex[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
}
else
{
DEBUG_LOGGER(ibase->logger, "update_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
ret = ibase_update_index(ibase, mheader->docid, block);
DEBUG_LOGGER(ibase->logger, "over_update_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
}
}
}
else
{
docheader->dbid = -1;
ret = ibase_index(ibase, localid, block);
}
}
else
{
if(ibase->state->used_for == IB_USED_FOR_INDEXD)
{
if(((off_t)localid * (off_t)sizeof(MHEADER)) >= ibase->mheadersio.end)
{
ibase->mheadersio.old = ibase->mheadersio.end;
size = (off_t)((localid / IB_HEADERS_BASE) + 1)
* (off_t)IB_HEADERS_BASE * (off_t)sizeof(MHEADER);
ret = ftruncate(ibase->mheadersio.fd, size);
ibase->mheadersio.end = size;
memset(ibase->mheadersio.map + ibase->mheadersio.old, 0,
ibase->mheadersio.end - ibase->mheadersio.old);
}
if((mheader = PMHEADER(ibase, localid)))
{
mheader->status = 0;
mheader->docid = ++(ibase->state->ids[secid]);
mheader->secid = docheader->secid;
mheader->crc = docheader->crc;
DEBUG_LOGGER(ibase->logger, "new_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category));
ret = ibase_index(ibase, mheader->docid, block);
}
}
else
{
ret = ibase_index(ibase, localid, block);
}
ibase->state->docid = localid;
}
MUTEX_UNLOCK(ibase->mutex);
if(ibase->state->used_for != IB_USED_FOR_QPARSERD
&& ibase->state->mmsource_status != IB_MMSOURCE_NULL && localid > 0)
{
ACCESS_LOGGER(ibase->logger, "docid:%lld/%d c_size:%d c_zsize:%d size:%d", docheader->globalid, localid, docheader->content_size, docheader->content_zsize, docheader->nexts_off);
docheader->size = docheader->nexts_off;
ret = db_set_data(PDB(ibase->source), localid, block->data, docheader->size);
}
ret = 0;
}
else
{
FATAL_LOGGER(ibase->logger, "new_index[%lld]{crc:%d rank:%f slevel:%d category:%lld} failed, %s", IBLL(docheader->globalid), docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category), strerror(errno));
}
}
return ret;
}
/**
* @brief This hook will receive all process stats at configurable interval
*
* @param rc "return value"
* @param stats process stats structure for the process that we are running in
*/
void opra_periodic_stats(FH_STATUS *rc, fh_adm_stats_resp_t *stats)
{
uint32_t i;
for (i = 0; i < stats->stats_line_cnt; i++) {
fh_adm_line_stats_t *line = &stats->stats_lines[i];
FH_LOG(LH, XSTATS, ("Periodic stats on line %s", line->line_name));
FH_LOG(LH, XSTATS, ("- Packets : %lld", LLI(line->line_pkt_rx)));
FH_LOG(LH, XSTATS, ("- Packet errors : %lld", LLI(line->line_pkt_errs)));
FH_LOG(LH, XSTATS, ("- Packet dups : %lld", LLI(line->line_pkt_dups)));
FH_LOG(LH, XSTATS, ("- Packet late : %lld", LLI(line->line_pkt_late)));
FH_LOG(LH, XSTATS, ("- Packet bad time : %lld", LLI(line->line_pkt_bad_times)));
FH_LOG(LH, XSTATS, ("- Packet seq jumps : %lld", LLI(line->line_pkt_seq_jump)));
FH_LOG(LH, XSTATS, ("- Packet wrap noreset : %lld", LLI(line->line_pkt_wrap_noreset)));
FH_LOG(LH, XSTATS, ("- Lost messages : %lld", LLI(line->line_msg_loss)));
FH_LOG(LH, XSTATS, ("- Recovered messages : %lld", LLI(line->line_msg_recovered)));
FH_LOG(LH, XSTATS, ("- Late messages : %lld", LLI(line->line_msg_late)));
FH_LOG(LH, XSTATS, ("- Received messages : %lld", LLI(line->line_msg_rx)));
FH_LOG(LH, XSTATS, ("- Bytes : %lld", LLI(line->line_bytes)));
}
*rc = FH_OK;
}
/* working for get_data() and push_index() */
int qindex_work(QINDEX *qindex, MINDEX *mindex)
{
int ret = -1, i = 0, x = 0 , n = 0, mid = 0;
int64_t *xid = NULL, key = 0, vlong = 0;
BELEMENT *root = NULL, *sub = NULL;
DOCHEADER *docheader = NULL;
STERM *termlist = NULL;
IBDATA block = {0};
char *term = NULL;
BRES *res = NULL;
if(qindex && mindex && mindex_check(mindex) == 0)
{
if(mtask_pop(&(mindex->mtask)) > 0 && (xid = (int64_t *)mindex->mtask.packet)
&& mindex->mtask.length > sizeof(int64_t))
{
n = mindex->mtask.length/sizeof(int64_t);
brequest_reset(&(mindex->request));
brequest_append_keys(&(mindex->request), xid, n);
brequest_finish(&(mindex->request));
if((res = dbase_get_records(&(mindex->db), &(mindex->request))))
{
x = 0;
while((root = dbase_next_record(res, root, &key)))
{
ACCESS_LOGGER(qindex->logger, "index_record[%d/%d] packetid:%d key:%lld", x, n, mindex->mtask.packetid, LLI(key));
vlong = 0;
if((sub = belement_find(root, qindex->db_key_name)))
{
belement_v_long(sub, &vlong);
}
if(key != vlong)
{
ACCESS_LOGGER(qindex->logger, "Invalid record[%d/%d] packetid:%d key:%lld/%lld", x, n, mindex->mtask.packetid, LLI(vlong), LLI(key));
return ret;
}
if((sub = belement_find(root, qindex->db_index_block_name))
&& (block.ndata = belement_v_blob(sub, &(block.data))) > 0)
{
docheader = (DOCHEADER *)block.data;
termlist = (STERM *)(block.data + sizeof(DOCHEADER)
+ docheader->nfields * sizeof(XFIELD));
term = block.data + docheader->textblock_off;
for(i = 0; i < docheader->nterms; i++)
{
termlist[i].termid = mmtrie_xadd(qindex->xdict, term, termlist[i].term_len);
term += termlist[i].term_len;
}
mid = qindex_push_index(qindex, key, &block);
ACCESS_LOGGER(qindex->logger, "over_record[%d/%d] packetid:%d key:%lld", x, n, mindex->mtask.packetid, LLI(key));
}
else
{
FATAL_LOGGER(qindex->logger, "Invalid data feilds[%s]", qindex->db_index_block_name);
return ret;
}
x++;
}
ret = 0;
mtask_finish(&(mindex->mtask), 0);
}
else
{
FATAL_LOGGER(qindex->logger, "get_records(%d) failed, %s", mindex->mtask.packetid, strerror(errno));
return ret;
}
}
}
return ret;
}
