/* add document */ int ibase_add_document(IBASE *ibase, IBDATA *block) { int ret = -1, localid = 0, secid = 0, n = 0, newid = 0; DOCHEADER *docheader = NULL; MHEADER *mheader = NULL; IHEADER *iheader = NULL; char line[IB_LINE_MAX]; off_t size = 0; if(ibase && block && block->ndata > 0 && (docheader = (DOCHEADER *)(block->data)) && (secid = docheader->secid) >= 0 && secid < IB_SEC_MAX && ibase_check_index_state(ibase, docheader) == 0 && docheader->globalid && (n = sprintf(line, "%lld", IBLL(docheader->globalid)))) { if((localid = mmtrie_xadd(ibase->docmap, line, n)) > 0) { MUTEX_LOCK(ibase->mutex); if(localid <= ibase->state->docid) { if(ibase->state->used_for == IB_USED_FOR_INDEXD) { if((mheader = PMHEADER(ibase, localid)) && (iheader = PIHEADER(ibase, mheader->secid, mheader->docid))) { /* disabled */ if(docheader->status < 0) { mheader->status = -1; iheader->status = -1; DEBUG_LOGGER(ibase->logger, "Update docid:%d globalid:%lld status:%d", localid, IBLL(docheader->globalid), docheader->status); ret = 0; } else if(docheader->crc != mheader->crc) { ibase_del_index(ibase, secid, localid); newid = ++(ibase->state->ids[secid]); DEBUG_LOGGER(ibase->logger, "ready_reindex[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category)); mheader->docid = newid; mheader->crc = docheader->crc; ret = ibase_index(ibase, newid, block); DEBUG_LOGGER(ibase->logger, "over_reindex[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category)); } else { DEBUG_LOGGER(ibase->logger, "update_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category)); ret = ibase_update_index(ibase, mheader->docid, block); DEBUG_LOGGER(ibase->logger, "over_update_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category)); } } } else { docheader->dbid = -1; ret = ibase_index(ibase, localid, block); } } else { if(ibase->state->used_for == IB_USED_FOR_INDEXD) { if(((off_t)localid * (off_t)sizeof(MHEADER)) >= ibase->mheadersio.end) { ibase->mheadersio.old = ibase->mheadersio.end; size = (off_t)((localid / IB_HEADERS_BASE) + 1) * (off_t)IB_HEADERS_BASE * (off_t)sizeof(MHEADER); ret = ftruncate(ibase->mheadersio.fd, size); ibase->mheadersio.end = size; memset(ibase->mheadersio.map + ibase->mheadersio.old, 0, ibase->mheadersio.end - ibase->mheadersio.old); } if((mheader = PMHEADER(ibase, localid))) { mheader->status = 0; mheader->docid = ++(ibase->state->ids[secid]); mheader->secid = docheader->secid; mheader->crc = docheader->crc; DEBUG_LOGGER(ibase->logger, "new_index[%lld/%d]{crc:%d rank:%f slevel:%d category:%lld}", IBLL(docheader->globalid), localid, docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category)); ret = ibase_index(ibase, mheader->docid, block); } } else { ret = ibase_index(ibase, localid, block); } ibase->state->docid = localid; } MUTEX_UNLOCK(ibase->mutex); if(ibase->state->used_for != IB_USED_FOR_QPARSERD && ibase->state->mmsource_status != IB_MMSOURCE_NULL && localid > 0) { ACCESS_LOGGER(ibase->logger, "docid:%lld/%d c_size:%d c_zsize:%d size:%d", docheader->globalid, localid, docheader->content_size, docheader->content_zsize, docheader->nexts_off); docheader->size = docheader->nexts_off; ret = db_set_data(PDB(ibase->source), localid, block->data, docheader->size); } ret = 0; } else { FATAL_LOGGER(ibase->logger, "new_index[%lld]{crc:%d rank:%f slevel:%d category:%lld} failed, %s", IBLL(docheader->globalid), docheader->crc, docheader->rank, docheader->slevel, LLI(docheader->category), strerror(errno)); } } return ret; }
/** * @brief This hook will receive all process stats at configurable interval * * @param rc "return value" * @param stats process stats structure for the process that we are running in */ void opra_periodic_stats(FH_STATUS *rc, fh_adm_stats_resp_t *stats) { uint32_t i; for (i = 0; i < stats->stats_line_cnt; i++) { fh_adm_line_stats_t *line = &stats->stats_lines[i]; FH_LOG(LH, XSTATS, ("Periodic stats on line %s", line->line_name)); FH_LOG(LH, XSTATS, ("- Packets : %lld", LLI(line->line_pkt_rx))); FH_LOG(LH, XSTATS, ("- Packet errors : %lld", LLI(line->line_pkt_errs))); FH_LOG(LH, XSTATS, ("- Packet dups : %lld", LLI(line->line_pkt_dups))); FH_LOG(LH, XSTATS, ("- Packet late : %lld", LLI(line->line_pkt_late))); FH_LOG(LH, XSTATS, ("- Packet bad time : %lld", LLI(line->line_pkt_bad_times))); FH_LOG(LH, XSTATS, ("- Packet seq jumps : %lld", LLI(line->line_pkt_seq_jump))); FH_LOG(LH, XSTATS, ("- Packet wrap noreset : %lld", LLI(line->line_pkt_wrap_noreset))); FH_LOG(LH, XSTATS, ("- Lost messages : %lld", LLI(line->line_msg_loss))); FH_LOG(LH, XSTATS, ("- Recovered messages : %lld", LLI(line->line_msg_recovered))); FH_LOG(LH, XSTATS, ("- Late messages : %lld", LLI(line->line_msg_late))); FH_LOG(LH, XSTATS, ("- Received messages : %lld", LLI(line->line_msg_rx))); FH_LOG(LH, XSTATS, ("- Bytes : %lld", LLI(line->line_bytes))); } *rc = FH_OK; }
/* working for get_data() and push_index() */ int qindex_work(QINDEX *qindex, MINDEX *mindex) { int ret = -1, i = 0, x = 0 , n = 0, mid = 0; int64_t *xid = NULL, key = 0, vlong = 0; BELEMENT *root = NULL, *sub = NULL; DOCHEADER *docheader = NULL; STERM *termlist = NULL; IBDATA block = {0}; char *term = NULL; BRES *res = NULL; if(qindex && mindex && mindex_check(mindex) == 0) { if(mtask_pop(&(mindex->mtask)) > 0 && (xid = (int64_t *)mindex->mtask.packet) && mindex->mtask.length > sizeof(int64_t)) { n = mindex->mtask.length/sizeof(int64_t); brequest_reset(&(mindex->request)); brequest_append_keys(&(mindex->request), xid, n); brequest_finish(&(mindex->request)); if((res = dbase_get_records(&(mindex->db), &(mindex->request)))) { x = 0; while((root = dbase_next_record(res, root, &key))) { ACCESS_LOGGER(qindex->logger, "index_record[%d/%d] packetid:%d key:%lld", x, n, mindex->mtask.packetid, LLI(key)); vlong = 0; if((sub = belement_find(root, qindex->db_key_name))) { belement_v_long(sub, &vlong); } if(key != vlong) { ACCESS_LOGGER(qindex->logger, "Invalid record[%d/%d] packetid:%d key:%lld/%lld", x, n, mindex->mtask.packetid, LLI(vlong), LLI(key)); return ret; } if((sub = belement_find(root, qindex->db_index_block_name)) && (block.ndata = belement_v_blob(sub, &(block.data))) > 0) { docheader = (DOCHEADER *)block.data; termlist = (STERM *)(block.data + sizeof(DOCHEADER) + docheader->nfields * sizeof(XFIELD)); term = block.data + docheader->textblock_off; for(i = 0; i < docheader->nterms; i++) { termlist[i].termid = mmtrie_xadd(qindex->xdict, term, termlist[i].term_len); term += termlist[i].term_len; } mid = qindex_push_index(qindex, key, &block); ACCESS_LOGGER(qindex->logger, "over_record[%d/%d] packetid:%d key:%lld", x, n, mindex->mtask.packetid, LLI(key)); } else { FATAL_LOGGER(qindex->logger, "Invalid data feilds[%s]", qindex->db_index_block_name); return ret; } x++; } ret = 0; mtask_finish(&(mindex->mtask), 0); } else { FATAL_LOGGER(qindex->logger, "get_records(%d) failed, %s", mindex->mtask.packetid, strerror(errno)); return ret; } } } return ret; }