bool AsyncThriftForwarder::reloadConfig()
{
TDEBUG("Load configuration...");
if (!config_dir.exists(LOG4CXX_CONFIG_FILE)) {
TWARN("Cannot find '%s'", LOG4CXX_CONFIG_FILE);
return false;
}
if (!config_dir.exists(ASYNCTHRIFT_CONFIG_FILE)) {
TWARN("Cannot find '%s'", ASYNCTHRIFT_CONFIG_FILE);
return false;
}
TLogger::configure(qPrintable(config_dir.absoluteFilePath(LOG4CXX_CONFIG_FILE)));
QSettings settings(config_dir.absoluteFilePath(ASYNCTHRIFT_CONFIG_FILE), QSettings::IniFormat);
QString socket(settings.value("ForwarderSocket", QString(PKGSTATEDIR "/logger")).toString());
// TODO: Check reconfiguration. We could reconfigure delay and flush_interval
settings.beginGroup("LogForwarder");
int nentries = settings.beginReadArray("Forwarders");
for (int i = 0; i < nentries; i++) {
settings.setArrayIndex(i);
forwarders.append(new ForwarderManager(settings.value("Name").toString(), settings.value("ZQuorum").toString(), settings.value("Delay").toUInt(), settings.value("FlushInterval").toUInt(), socket));
}
settings.endArray();
return true;
}
void
mtd_thread_t::did_reply ()
{
time_t tm = sfs_get_timenow() - start;
if (tm > LONG_REPLY_TIME) {
if (cell->sbp) {
TWARN ("long service time (" << tm << " secs) for PROC="
<< cell->sbp->proc ());
} else {
TWARN ("long service time (" << tm << " secs); no procno given");
}
}
}
sth_t
amysql_thread_t::prepare (const str &q, u_int o)
{
if (readied) {
TWARN ("security precaution: cannot prepare queries "
"after servicing requests\n");
return NULL;
}
sth_t r = mysql.prepare (q, o);
if (!r)
TWARN (mysql.error ());
return r;
}
void ConnectionManager::connectionReadyRead()
{
struct msghdr msg;
struct iovec iov;
TTRACE("ConnectionManager::connectionReadyRead");
/* Response data */
iov.iov_base = buffer.data();
iov.iov_len = buffer.length();
/* compose the message */
memset(&msg, 0, sizeof(msg));
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
while(connectionFd >= 0) {
int nread = recvmsg(connectionFd, &msg, MSG_DONTWAIT);
if (nread == 0) {
connectionClose();
return;
} else if (nread < 0) {
if (errno != EAGAIN) {
TWARN("recvmsg error %s", strerror(errno));
//close(connFd);
connectionClose();
}
return;
}
receiveDatagram(buffer.data(), nread);
}
}
void
mtd_thread_t::run ()
{
mtd_status_t rc;
GIANT_LOCK();
bool ok = init_phase0() && init();
GIANT_UNLOCK();
if (!ok) {
TWARN ("thread could not initialize");
msg_send (MTD_SHUTDOWN);
delete this;
return;
}
become_ready ();
do {
GIANT_LOCK();
take_svccb ();
GIANT_UNLOCK();
rc = msg_recv ();
} while (rc == MTD_CONTINUE);
cell->status = MTD_SHUTDOWN;
msg_send (MTD_SHUTDOWN);
return;
}
void ConnectionManager::sendDatagram(const char* buf, size_t len, int controlFd)
{
char control[sizeof(struct cmsghdr)+10];
struct msghdr msg;
struct cmsghdr *cmsg;
struct iovec iov;
/* Response data */
iov.iov_base = const_cast<char*>(buf);
iov.iov_len = len;
/* compose the message */
memset(&msg, 0, sizeof(msg));
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
/* send controlFd */
if (controlFd > 0) {
msg.msg_control = control;
msg.msg_controllen = sizeof(control);
cmsg = CMSG_FIRSTHDR(&msg);
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_RIGHTS;
cmsg->cmsg_len = CMSG_LEN(sizeof(controlFd));
memcpy(reinterpret_cast<void *>(CMSG_DATA(cmsg)), &controlFd, sizeof(controlFd));
msg.msg_controllen = cmsg->cmsg_len;
}
if (sendmsg(connectionFd, &msg, MSG_DONTWAIT) < 0) {
TWARN("sendmsg error %s", strerror(errno));
connectionClose();
//close(connFd);
return;
}
}
void
tst2_srv_t::get (ptr<amt::req_t> b)
{
rpc::tst2_prog_1::tst2_get_srv_t<amt::req_t> srv (b);
const tst2_get_arg_t *arg = srv.getarg ();
adb_status_t rc;
ptr<tst2_get_res_t> res = srv.alloc_res (ADB_OK);
if (!_q_get->execute (*arg)) {
rc = ADB_EXECUTE_ERROR;
} else {
rc = _q_get->fetch (&res->dat->pk, &res->dat->d, &res->dat->i,
&res->dat->d2);
}
if (rc != ADB_OK) {
res->set_status (rc);
str es;
if ((es = _q_get->error ()))
TWARN(es);
}
srv.reply (res);
}
void
mtd_thread_t::become_ready ()
{
readied = true;
TWARN ("called become_ready ()"); // debug
cell->status = MTD_READY;
msg_send (MTD_READY);
}
void ConnectionManager::receiveDatagram(const char* buf, size_t len)
{
if (memcmp(buf, "watch ", 6) == 0) {
receiveWatch(buf, len);
} else if (memcmp(buf, "CAPABILITIES ", 6) == 0) {
receiveHandshake(buf, len);
} else {
TWARN("Unknown command %.*s", (int)len, buf);
}
}
mtd_status_t
mtd_thread_t::msg_recv ()
{
mtd_msg_t msg;
int rc = ::msg_recv (fdin, &msg);
if (rc < 0) {
TWARN ("mtd_thread_t::msg_recv: bad read; (errno=%d" << errno << ")");
return MTD_ERROR;
}
return msg.status;
}
void showSortResult(sort_framework::SortResult *pSortResult)
{
if (pSortResult == NULL)
return ;
FILE *pOutput = NULL;
int32_t num = 0;
int32_t DocsReturn = 0;
int64_t *pNID = NULL;
int64_t *szFirstRanks = NULL;
int64_t *szSecondRanks = NULL;
pOutput = fopen(OUTPUTFILE, "a+");
if (pOutput == NULL) {
fprintf(stderr, "open file %s error\n", OUTPUTFILE);
return ;
}
//Get Nid List
DocsReturn =
pSortResult->getNidList(pNID);
//Get FirstRank List
num = pSortResult->getFirstRankList(szFirstRanks);
if (num < 0) {
TWARN("Get FirstRank List error.");
return ;
}
//Get SecondRank List
num = pSortResult->getSecondRankList(szSecondRanks);
if (num < 0) {
TWARN("Get SecondRank List error.");
return ;
}
fprintf(pOutput, "\nSort Result Begin:\nDocsFound = %d\n", pSortResult->getDocsFound());
for (uint32_t i=0; i<DocsReturn; i++)
{
fprintf(pOutput, "%d\tnid=%lu\tfirst=%lld\tsecond=%lld\n",
i, pNID[i], szFirstRanks[i], szSecondRanks[i]);
}
fprintf(pOutput, "Sort Result End");
fflush(pOutput);
fclose(pOutput);
return ;
}
void
tst2_srv_t::mget (ptr<amt::req_t> b)
{
rpc::tst2_prog_1::tst2_mget_srv_t<amt::req_t> srv (b);
const tst2_mget_arg_t *arg = srv.getarg ();
ptr<tst2_mget_res_t> res = srv.alloc_res (ADB_OK);
if (!_q_mget->execute (arg->sleep_msec, arg->lim)) {
res->set_status (ADB_EXECUTE_ERROR);
TWARN("mget error: " << _q_mget->error ());
} else {
adb_status_t s;
tst2_data_t dat;
while ((s = _q_mget->fetch (&dat.d, &dat.i, &dat.pk, &dat.d2)) == ADB_OK) {
res->rows->push_back (dat);
}
if (s != ADB_NOT_FOUND) {
TWARN("mget error: " << _q_mget->error ());
res->set_status (s);
}
}
srv.reply (res);
}
void
mtd_thread_t::reply (ptr<void> d)
{
if (ok_kthread_safe) {
TWARN ("XX WARNING!! Do not call mth_thread_t::reply "
"in thread-safe mode!\n");
}
did_reply ();
cell->rsp_u.p = d;
d = NULL;
cell->status = MTD_REPLY;
cell->rstat = MTD_RPC_DATA;
msg_send (MTD_REPLY);
}
void
tst2_srv_t::put (ptr<amt::req_t> b)
{
rpc::tst2_prog_1::tst2_put_srv_t<amt::req_t> srv (b);
const tst2_put_arg_t *arg = srv.getarg ();
ptr<adb_status_t> res = srv.alloc_res (ADB_OK);
if (!_q_put->execute (arg->key, arg->data.d, arg->data.i, arg->data.d2)) {
*res = ADB_EXECUTE_ERROR;
str es;
if ((es = _q_put->error ()))
TWARN("put error: " << es);
}
b->reply (res);
}
int32_t SearcherWorker::getDetailInfo(const char *q_reslut, uint32_t q_size, const char *pureString, char** res, int* res_size)
{
//for detail
char** args = NULL;
int fieldCount = 0;
char* pLightKey = 0;
char* pDL = 0;
if(!q_reslut || 0 == q_size){
return KS_EFAILED;
}
pLightKey = FRAMEWORK::Query::getParam(pureString, q_size, LIGHTKEY);
if(pLightKey && pLightKey[0]!=0){
int len = strlen(pLightKey);
int ret = decode_url(pLightKey, len, pLightKey, len+1);
if(ret<0){
TWARN("decode_url pLightKey error, return!");
_session.setStatus(FRAMEWORK::ss_error);
_session.response();
return KS_EFAILED;
}
pLightKey[ret] = 0;
}
pDL = FRAMEWORK::Query::getParam(pureString, q_size, DL);
fieldCount = get_di_field_count();
if(di_detail_arg_alloc(&args)<(fieldCount+1)){
TERR("di_detail_arg_alloc error, return!");
_session.setStatus(FRAMEWORK::ss_error);
_session.response();
return KS_EFAILED;
}
di_detail_arg_set(args, fieldCount+1, "title", 5, pLightKey, pDL);
//Deal with detail
if(di_detail_get_v3(q_reslut, pDL, args, fieldCount, res, res_size)){
*res = (char*)malloc(2);
(*res)[0] = '0';
(*res)[1] = 0;
*res_size = 1;
}
if(pLightKey)std::free(pLightKey);
if(pDL)std::free(pDL);
if(args)di_detail_arg_free(args);
return KS_SUCCESS;
}
bool
tst2_srv_t::init ()
{
bool rc = true;
if (!mysql.connect ("okws_db_tst2", "okws", "localhost", "abc123")) {
TWARN (mysql.error ());
rc = false;
} else if (!(_q_get = PREP("SELECT id,d,i,d2 FROM tst2 WHERE s = ?")) ||
!(_q_put = PREP("INSERT INTO tst2(s,d,i,d2) VALUES(?,?,?,?)")) ||
!(_q_mget = PREP("SELECT SLEEP(?/1000),id,d,i,d2 "
"FROM tst2 ORDER BY RAND() "
"LIMIT ?"))) {
rc = false;
}
return rc;
}
int32_t SearcherWorker::run() {
const char *pureString = NULL;
uint32_t pureSize = 0;
char *resData = NULL;
uint32_t resSize = 0;
char *resDetailData = NULL;
int32_t resDetailSize = 0;
int32_t ret = 0;
MemPool *memPool = NULL;
int64_t *pNid = NULL;
commdef::ClusterResult *pClusterResult = NULL;
queryparser::QueryRewriterResult *pQRWResult = NULL;
queryparser::QPResult *pQueryResult = NULL;
SearchResult *pSearchResult = NULL;
statistic::StatisticResult *pStatisticResult = NULL;
sort_framework::SortResult *pSortResult = NULL;
ResultSerializer resultSerial;
FRAMEWORK::Context context;
FILE *pOutput = NULL;
//check session status
FRAMEWORK::session_status_t status = _session.getStatus();
if (status == FRAMEWORK::ss_timeout) {
handleTimeout();
return KS_SUCCESS;
}
//get query infomation
FRAMEWORK::Query &query = _session.getQuery();
pureSize = query.getPureQuerySize();
pureString = query.getPureQueryData();
if (!pureString || pureSize == 0) {
_session.setStatus(FRAMEWORK::ss_error);
_session.response();
return KS_EFAILED;
}
//set LogInfo level
_session._logInfo._eRole = FRAMEWORK::sr_simple;
//get MemPool from factory
memPool = _memFactory.make((uint64_t)(getOwner()));
if (memPool == NULL) {
TWARN("Make mem pool failed!");
return KS_EFAILED;
}
//create memory pool monitor
MemMonitor memMonitor(memPool, _memLimit);
memPool->setMonitor(&memMonitor);
memMonitor.enableException();
//initialize context class
context.setMemPool(memPool);
//initialize format processor
_formatProcessor.init(memPool);
//Deal with search proccess
do{
if(_session.isHttp()){
pQRWResult = NEW(memPool, queryparser::QueryRewriterResult)();
if (pQRWResult == NULL) {
TWARN("SEARCHER: new Result no mem");
_session.setStatus(FRAMEWORK::ss_error);
break;
}
}
pQueryResult = NEW(memPool, queryparser::QPResult)(memPool);
pSearchResult = NEW(memPool, SearchResult);
pStatisticResult = NEW(memPool, statistic::StatisticResult);
pSortResult = NEW(memPool, sort_framework::SortResult)(memPool);
if(unlikely(!pQueryResult || !pSearchResult || !pStatisticResult
|| !pSortResult)) {
TWARN("SEARCHER: new Result no mem");
_session.setStatus(FRAMEWORK::ss_error);
break;
}
//add queryrewrite process
if(_session.isHttp()){
ret = _qrewriter.doRewrite(&context, pureString, pureSize, pQRWResult);
if (timeoutCheck && (_timeout > 0) && (_session.getLatencyTime() > _timeout)) {
_session.setStatus(FRAMEWORK::ss_timeout);
TWARN("SEARCHER: qrewriter.doRewrite function over time. query is %s", pureString);
break;
}
if (unlikely(ret != KS_SUCCESS)) {
_session.setStatus(FRAMEWORK::ss_error);
TWARN("qrewriter.doRewrite function error. query is %s", pureString);
break;
}
pureString = pQRWResult->getRewriteQuery();
}
//end add
ret = _qp.doParse(&context, pQueryResult, pureString);
if (timeoutCheck && (_timeout > 0) &&
(_session.getLatencyTime() > _timeout))
{
_session.setStatus(FRAMEWORK::ss_timeout);
TWARN("SEARCHER: qp.doParse function over time. query is %s", pureString);
break;
}
if (unlikely(ret != KS_SUCCESS)){
TWARN("SEARCHER: queryparser doParse function error. query is %s", pureString);
_session.setStatus(FRAMEWORK::ss_error);
break;
}
// cache命中情况下, mempool reset时调用CacheResult析构函数,释放命中的节点，
// 否则sortResult对节点内存的引用失效，变为野指针(bug#118631)
{
ret = _is.doQuery(&context, pQueryResult, &_sort, pSearchResult);
if (timeoutCheck && (_timeout > 0) &&
(_session.getLatencyTime() > _timeout))
{
_session.setStatus(FRAMEWORK::ss_timeout);
TWARN("SEARCHER: is.doQuery function over time. query is %s", pureString);
break;
}
if (unlikely(ret != KS_SUCCESS)){
TWARN("SEARCHER: search doQuery function error. query is %s", pureString);
_session.setStatus(FRAMEWORK::ss_error);
break;
}
ret = _stat.doStatisticOnSearcher(&context,
*(pQueryResult->getParam()),
pSearchResult, pStatisticResult);
if (timeoutCheck && (_timeout > 0) &&
(_session.getLatencyTime() > _timeout))
{
_session.setStatus(FRAMEWORK::ss_timeout);
TWARN("SEARCHER: doStatisticOnSearcher function over time. query is %s", pureString);
break;
}
if (unlikely(ret != KS_SUCCESS)){
TWARN("SEARCHER: statistic doStatisticOnSearcher function error. query is %s", pureString);
_session.setStatus(FRAMEWORK::ss_error);
break;
}
ret = _sort.doProcess(context, *(pQueryResult->getParam()),
*pSearchResult, *pSortResult);
if (timeoutCheck && (_timeout > 0) &&
(_session.getLatencyTime() > _timeout))
{
_session.setStatus(FRAMEWORK::ss_timeout);
TWARN("SEARCHER: sort.doProcess function over time. query is %s", pureString);
break;
}
if (unlikely(ret)) {
TWARN("SEARCHER: sort doProcess function error. query is %s", pureString);
_session.setStatus(FRAMEWORK::ss_error);
break;
}
}
pNid = NULL;
//get docs return number
int32_t num = pSortResult->getNidList(pNid);
pClusterResult = NEW(memPool, commdef::ClusterResult)();
if ( NULL == pClusterResult ){
TWARN("SEARCHER: malloc from memPool failed ");
_session.setStatus(FRAMEWORK::ss_error);
break;
}
memset(pClusterResult, 0x0, sizeof(commdef::ClusterResult));
pClusterResult->_nDocsFound = pSortResult->getDocsFound();//pSearchResult->nDocFound;
pClusterResult->_nEstimateDocsFound = pSortResult->getEstimateDocsFound();//pSearchResult->nEstimateDocFound;
pClusterResult->_nDocsSearch = pSearchResult->nDocSearch;
pClusterResult->_nDocsReturn = num;
pClusterResult->_nDocsRestrict = pSearchResult->nEstimateDocFound;//pSearchResult->nDocFound;
pClusterResult->_pStatisticResult = pStatisticResult;
pClusterResult->_pSortResult = pSortResult;
pClusterResult->_pQPResult = pQueryResult;
pClusterResult->_ppDocuments = NULL;
if(_session.isHttp() && _detail){
if(!pClusterResult->_nDocsReturn){
break;
}
char* pid = NULL;
uint32_t pid_size = 0;
ret = get_nid_str(pNid, num, memPool, &pid, &pid_size);
if(KS_SUCCESS != ret){
_session.setStatus(FRAMEWORK::ss_error);
break;
}
ret = getDetailInfo(pid, pid_size, pureString, &resDetailData, &resDetailSize);
if(KS_SUCCESS != ret){
_session.setStatus(FRAMEWORK::ss_error);
break;
}
if(timeoutCheck && _timeout>0 && _session.getLatencyTime()>_timeout){
_session.setStatus(FRAMEWORK::ss_timeout);
break;
}
if (!translateV3(resDetailData, pClusterResult->_ppDocuments, (uint32_t &)pClusterResult->_nDocsReturn, memPool)) {
_session.setStatus(FRAMEWORK::ss_error);
break;
}
}
} while (0);
if(resDetailData)std::free(resDetailData);
if(_session.isHttp() && _detail){
//fromat result
get_outfmt_type(pureString, pureSize);
result_container_t container;
container.pClusterResult = pClusterResult;
container.cost = _session.getLatencyTime();
if(_formatProcessor.frmt(_ofmt_type, container, &resData , &resSize) < 0){
TERR("FORMATRESULT: format_processor process error!");
_session.setStatus(FRAMEWORK::ss_error);
}
}
else{
//serialize search result
ret = resultSerial.serialClusterResult(pClusterResult,
resData, resSize, "Z");
if (ret != KS_SUCCESS) {
_session.setStatus(FRAMEWORK::ss_error);
}
}
//recycle mem pool
context.getMemPool()->reset();
_session.setResponseData(resData, resSize);
_session.response();
return KS_SUCCESS;
}
int32_t SearcherWorkerFactory::initilize(const char *path)
{
ini_context_t cfg;
const ini_section_t *grp = NULL;
const char *val = NULL;
const char *seconf = NULL;
uint32_t nval = 0;
int32_t ret = 0;
if (_ready) {
return KS_SUCCESS;
}
if (!_server)
{
TERR("initialize _server error.");
return KS_EFAILED;
}
_server->_type = FRAMEWORK::srv_type_searcher;
ret = ini_load_from_file(path, &cfg);
if (unlikely(ret != 0)) {
TERR("initialize SearcherWorkerFactory by `%s' error.",
SAFE_STRING(path));
return KS_EFAILED;
}
grp = &cfg.global;
if (unlikely(!grp)) {
TERR("invalid config file `%s' for SearcherWorkerFactory.",
SAFE_STRING(path));
ini_destroy(&cfg);
return KS_EFAILED;
}
//获得搜索xml文件句柄
val = ini_get_str_value1(grp, "module_config_path");
if (val && (val[0] != '\0')) {
FILE *fp = NULL;
if ((fp = fopen(val, "r")) == NULL) {
TERR("模块配置文件 %s 打开出错, 文件可能不存在.\n", val);
ini_destroy(&cfg);
return KS_EFAILED;
}
_pXMLTree = mxmlLoadFile(NULL, fp, MXML_NO_CALLBACK);
if (_pXMLTree == NULL) {
TERR("模块配置文件 %s 格式有错, 请修正您的配置文件.\n", val);
fclose(fp);
ini_destroy(&cfg);
return KS_EFAILED;
}
fclose(fp);
}
else {
TERR("search module config path is null");
ini_destroy(&cfg);
return KS_EFAILED;
}
//各个处理模块的初始化工作
if (index_lib::init(_pXMLTree) != KS_SUCCESS) {
TERR("init index lib failed!");
ini_destroy(&cfg);
return KS_EFAILED;
}
if (_qrewiter.init(_pXMLTree) != KS_SUCCESS){
TERR("init query rewriter failed!");
ini_destroy(&cfg);
return KS_EFAILED;
}
if (_qp.init(_pXMLTree) != KS_SUCCESS){
TERR("init query parser failed!");
ini_destroy(&cfg);
return KS_EFAILED;
}
if (_is.init(_pXMLTree) != KS_SUCCESS){
TERR("init index searcher failed!");
ini_destroy(&cfg);
return KS_EFAILED;
}
if (_stat.init(_pXMLTree) != KS_SUCCESS){
TWARN("init statistic failed!");
}
//get default value of format type
_ofmt_type = get_outfmt_type(_pXMLTree);
//get sort config
val = ini_get_str_value1(grp, "sort_config_path");
if (val && (val[0] != '\0')) {
if (_sort.init(val) != KS_SUCCESS) {
TERR("init sort failed! path = %s\n", val);
ini_destroy(&cfg);
return KS_EFAILED;
}
}
else {
TERR("sort config file is error!");
ini_destroy(&cfg);
return KS_EFAILED;
}
val = ini_get_str_value1(grp, "update_config_path");
if (val && (val[0] != '\0')) {
_pIndexUpdater = new UPDATE::IndexUpdater;
int32_t nRes = -1;
if ((nRes = _pIndexUpdater->init(val)) != 0) {
TERR("init IndexUpdater failed! errno=%d", nRes);
}
else if (pthread_create(&_updateTid, NULL, UPDATE::Updater::start, _pIndexUpdater) != 0) {
TERR("start updater thread failed!");
ini_destroy(&cfg);
return KS_EFAILED;
}
}
else {
TERR("update config path is null");
}
//获得detail_module.xml文件句柄
val = ini_get_str_value1(grp, "detail_module_config_path");
if (val && (val[0] != '\0')) {
FILE *fp = NULL;
if ((fp = fopen(val, "r")) == NULL) {
TERR("模块配置文件 %s 打开出错, 文件可能不存在.\n", val);
ini_destroy(&cfg);
return KS_EFAILED;
}
_pXMLTreeDetail= mxmlLoadFile(NULL, fp, MXML_NO_CALLBACK);
if (_pXMLTreeDetail == NULL) {
TERR("模块配置文件 %s 格式有错, 请修正您的配置文件.\n", val);
fclose(fp);
ini_destroy(&cfg);
return KS_EFAILED;
}
fclose(fp);
_detail = true;
}
else {
_detail = false;
}
if(_detail)
{
//初始化detail各模块
if(di_detail_init(_pXMLTreeDetail)!=KS_SUCCESS){
TERR("di_detail_init failed!");
ini_destroy(&cfg);
return KS_EFAILED;
}
}
ini_destroy(&cfg);
_ready = true;
return KS_SUCCESS;
}
int IndexFieldInc::addTerm(uint64_t termSign, uint32_t docId, uint8_t firstOcc)
{
// 检查并删除idx_dict中超时的buffer空间
idx_dict_free_oldblock(_bitmapDict);
idx_dict_free_oldblock(_invertedDict);
if(unlikely((_fullDocNum + 8*_maxIncBitmapSize) <= docId))
{
TWARN("inc reach max doc count limit!");
return KS_SUCCESS;
}
if(_bitmapDict)
{
idict_node_t* pNodeBitmap = idx_dict_find(_bitmapDict, termSign);
if(pNodeBitmap)
{
bitmap_idx1_unit_t* pIncBitmap = (bitmap_idx1_unit_t*)pNodeBitmap;
uint64_t * pBitmap = (uint64_t*)(_pIncMemManager->offset2Addr(pIncBitmap->pos));
pBitmap -= _fullDocNum / 64;
pBitmap[docId / 64] |= bit_mask_tab[docId % 64];
pIncBitmap->doc_count++;
pIncBitmap->max_docId = docId;
if (_bitmapDict->syncMgr) {
IndexFieldSyncManager * pSyncMgr = (IndexFieldSyncManager *) _bitmapDict->syncMgr;
pSyncMgr->putIdxDictNode(pNodeBitmap);
pSyncMgr->syncToDisk();
}
}
}
inc_idx2_header_t* head = NULL;
// 确定新的2级索引单元的写入位置
idict_node_t* pNode = idx_dict_find(_invertedDict, termSign);
if(pNode == NULL) { // 没有这个token, 申请一个新的block
int32_t pageSize = getPageSize(0, 0);
uint32_t off = 0;
if(_pIncMemManager->makeNewPage(pageSize, off) < 0) {
return KS_ENOMEM;
}
char* buf = _pIncMemManager->offset2Addr(off);
head = (inc_idx2_header_t*)buf;
head->pageSize = _pIncMemManager->size(off);
head->endOff = sizeof(inc_idx2_header_t);
head->zipFlag = 0;
idict_node_t node = {0L, 0, 0, 0, 0};
node.sign = termSign;
((inc_idx1_unit_t*)&node)->pos = off;
if (0 != idx_dict_add(_invertedDict, &node)) {
TWARN("add token %lu to idx1 dict failed", node.sign);
return KS_EFAILED;
}
pNode = idx_dict_find(_invertedDict, termSign);
}
if(NULL == pNode) {
TWARN("add token %lu to idx1 dict failed", termSign);
return KS_EFAILED;
}
// 获取头信息，定位到空白内存处
inc_idx1_unit_t* pind1 = (inc_idx1_unit_t*)pNode;
char* begin = _pIncMemManager->offset2Addr(pind1->pos);
head = (inc_idx2_header_t*)begin;
char* end = begin + head->pageSize;
// begin = _pIncMemManager->offset2Addr(head->endOff);
begin += head->endOff;
uint32_t docCount = getDocNumByEndOff(head->endOff, head->zipFlag);
if (unlikely(docCount != pind1->doc_count)) {
TWARN("doc_count[%u] in idx1 mismatch idx2_header[%u]!", pind1->doc_count, docCount);
pind1->doc_count = docCount;
}
/*
if (head->zipFlag == 1) {
idx2_zip_header_t *zipHeader = (idx2_zip_header_t*)(head + 1);
idx2_zip_skip_unit_t *skip = (idx2_zip_skip_unit_t*)(zipHeader + 1);
if (skip[zipHeader->block_num - 1].off > (docCount - 1)) {
TWARN("reduce block_num!");
zipHeader->block_num--;
skip[zipHeader->block_num].off = docCount;
skip[zipHeader->block_num].doc_id_base = IDX_GET_BASE(INVALID_DOCID);
}
}
*/
// 扩容
uint32_t len = 0;
if(head->zipFlag == 1)
{
len = (_maxOccNum > 0) ? sizeof(idx2_zip_list_unit_t) : sizeof(uint16_t);
if(begin + (len<<1) > end)
{
int32_t ret = expend(head, pind1, begin, end);
if(ret != KS_SUCCESS)
return ret;
}
}
else
{
len = (_maxOccNum > 0) ? sizeof(idx2_nzip_unit_t) : sizeof(uint32_t);
if(begin + (len<<1) > end)
{
int32_t ret = expend(head, pind1, begin, end);
if(ret != KS_SUCCESS)
return ret;
}
}
// 增加一篇文档
if(head->zipFlag == 1)
{
uint16_t base = IDX_GET_BASE(docId);
uint16_t diff = IDX_GET_DIFF(docId);
idx2_zip_header_t *zipHeader = (idx2_zip_header_t*)(head + 1);
idx2_zip_skip_unit_t *skip = (idx2_zip_skip_unit_t*)(zipHeader + 1);
if(_maxOccNum == 0)
{
*(uint16_t*)(begin+sizeof(uint16_t)) = IDX_GET_DIFF(INVALID_DOCID);;
*(uint16_t*)begin = diff;
begin += sizeof(uint16_t);
zipHeader->doc_list_len += sizeof(uint16_t);
}
else
{
((idx2_zip_list_unit_t*)(begin+sizeof(idx2_zip_list_unit_t)))->doc_id_diff = IDX_GET_DIFF(INVALID_DOCID);
((idx2_zip_list_unit_t*)begin)->doc_id_diff = diff;
((idx2_zip_list_unit_t*)begin)->occ = firstOcc;
begin += sizeof(idx2_zip_list_unit_t);
zipHeader->doc_list_len += sizeof(idx2_zip_list_unit_t);
}
if(zipHeader->block_num == 0 || base > skip[zipHeader->block_num-1].doc_id_base)
{
if(unlikely(zipHeader->block_num == _maxIncSkipCnt))
{
TWARN("inc reach max doc count limit!");
return KS_SUCCESS;
}
skip[zipHeader->block_num+1].doc_id_base = IDX_GET_BASE(INVALID_DOCID);
skip[zipHeader->block_num+1].off = pind1->doc_count + 1;
skip[zipHeader->block_num].doc_id_base = base;
skip[zipHeader->block_num].off = pind1->doc_count;
zipHeader->block_num++;
}
else
{
skip[zipHeader->block_num].off = pind1->doc_count + 1;
}
pind1->doc_count++;
}
else
{
if(_maxOccNum == 0)
{
*(uint32_t*)(begin+sizeof(uint32_t)) = INVALID_DOCID;
*(uint32_t*)begin = docId;
pind1->doc_count++;
begin += sizeof(uint32_t);
}
else
{
((idx2_nzip_unit_t*)(begin+sizeof(idx2_nzip_unit_t)))->doc_id = INVALID_DOCID;
((idx2_nzip_unit_t*)begin)->doc_id = docId;
((idx2_nzip_unit_t*)begin)->occ = firstOcc;
pind1->doc_count++;
begin += sizeof(idx2_nzip_unit_t);
}
}
char* p1 = _pIncMemManager->offset2Addr(pind1->pos);
head->endOff = begin - p1;
if (_invertedDict->syncMgr)
{
IndexFieldSyncManager * pSyncMgr = (IndexFieldSyncManager *) _invertedDict->syncMgr;
pSyncMgr->putIdxDictNode(pNode);
pSyncMgr->syncToDisk();
}
return 1;
}
int32_t IndexFieldInc::nzip2zip(inc_idx2_header_t *&head, inc_idx1_unit_t *&pind, char *&begin, char *&end, uint32_t baseNum, uint32_t validCount)
{
int32_t pageSize = getPageSize(validCount, 1);
uint32_t off = 0;
if(_pIncMemManager->makeNewPage(pageSize, off) < 0)
{
TWARN("inc makeNewPage error");
return KS_ENOMEM;
}
char* p1 = _pIncMemManager->offset2Addr(off);
inc_idx2_header_t* tHead = (inc_idx2_header_t*)p1;
char* p2 = _pIncMemManager->offset2Addr(pind->pos);
memcpy(p1, p2, sizeof(inc_idx2_header_t));
char *zip = p1+sizeof(inc_idx2_header_t);
char *nzip = p2+sizeof(inc_idx2_header_t);
idx2_zip_header_t *pheader = (idx2_zip_header_t*)zip;
idx2_zip_skip_unit_t *pskip = (idx2_zip_skip_unit_t*)(pheader + 1);
pheader->skip_list_len = (_maxIncSkipCnt + 1) * sizeof(idx2_zip_skip_unit_t);
pheader->block_num = baseNum;
uint16_t pre_base = IDX_GET_BASE(INVALID_DOCID);
uint16_t cur_base = 0;
uint32_t doc_count = pind->doc_count;
uint32_t doc_pos = 0;
if(_maxOccNum > 0)
{ // has occ info
idx2_nzip_unit_t * notzip = (idx2_nzip_unit_t*)nzip;
pheader->doc_list_len = (validCount +1) * sizeof(idx2_zip_list_unit_t);
idx2_zip_list_unit_t * plist = (idx2_zip_list_unit_t*)(pskip + _maxIncSkipCnt + 1);
for (unsigned int i=0, j=0; i<doc_count; i++)
{
// build doc_list unit
if ( _docIdMgr->isDocIdDelUseLess( notzip[i].doc_id )) continue;
plist[doc_pos].doc_id_diff = IDX_GET_DIFF(notzip[i].doc_id);
plist[doc_pos].occ = notzip[i].occ;
// build skip_list unit
cur_base = IDX_GET_BASE(notzip[i].doc_id);
if (cur_base != pre_base)
{
pskip[j].doc_id_base = cur_base;
pskip[j].off = doc_pos;
pre_base = cur_base;
j++;
}
++doc_pos;
}
if (unlikely(doc_pos != validCount)) {
TERR("inc expand error, count miss[%u:%u]", validCount, doc_pos);
return KS_EINVAL;
}
// build end doc list
plist[validCount].doc_id_diff = IDX_GET_DIFF(INVALID_DOCID);
plist[validCount].occ = 0;
begin = (char*)(plist + validCount);
}
else
{ // null occ info
uint32_t * notzip = (uint32_t*)nzip;
pheader->doc_list_len = (validCount + 1) * sizeof(uint16_t);
uint16_t* plist = (uint16_t*)(pskip + _maxIncSkipCnt + 1);
for (unsigned int i=0, j=0; i<doc_count; i++)
{
if ( _docIdMgr->isDocIdDelUseLess( notzip[i] )) continue;
// build doc_list unit
plist[doc_pos] = IDX_GET_DIFF((notzip[i]));
// build skip_list unit
cur_base = IDX_GET_BASE(notzip[i]);
if (cur_base != pre_base)
{
pskip[j].doc_id_base = cur_base;
pskip[j].off = doc_pos;
pre_base = cur_base;
j++;
}
++doc_pos;
}
if (unlikely(doc_pos != validCount)) {
TERR("inc expand error, count miss[%u:%u]", validCount, doc_pos);
return KS_EINVAL;
}
// build end doc list
plist[validCount] = IDX_GET_DIFF(INVALID_DOCID);
begin = (char*)(plist + validCount);
}
// build auxillary block at the end of skiplist
pskip[baseNum].doc_id_base = IDX_GET_BASE(INVALID_DOCID);
pskip[baseNum].off = validCount;
tHead->pageSize = _pIncMemManager->size(off);
tHead->endOff = begin - p1;
tHead->zipFlag = 1;
end = p1 + tHead->pageSize;
_pIncMemManager->freeOldPage(head->pageSize, pind->pos);
head = tHead;
pind->pos = off;
pind->doc_count = validCount;
return KS_SUCCESS;
}
int32_t IndexFieldInc::nzip2nzip(inc_idx2_header_t *&head, inc_idx1_unit_t *&pind, char *&begin, char *&end, uint32_t validCount)
{
int32_t pageSize = getPageSize(validCount, 0);
uint32_t off = 0;
if(_pIncMemManager->makeNewPage(pageSize, off) < 0) {
TWARN("inc makeNewPage error");
return KS_ENOMEM;
}
char* p1 = _pIncMemManager->offset2Addr(off);
inc_idx2_header_t* tHead = (inc_idx2_header_t*)p1;
char* p2 = _pIncMemManager->offset2Addr(pind->pos);
memcpy(p1, p2, sizeof(inc_idx2_header_t));
char *newNzip = p1 + sizeof(inc_idx2_header_t);
char *oldNzip = p2 + sizeof(inc_idx2_header_t);
uint32_t doc_count = pind->doc_count;
uint32_t doc_pos = 0;
if (_maxOccNum > 0) {
idx2_nzip_unit_t * newNotzip = (idx2_nzip_unit_t*)newNzip;
idx2_nzip_unit_t * oldNotzip = (idx2_nzip_unit_t*)oldNzip;
for (unsigned int i=0; i<doc_count; i++)
{
// build doc_list unit
if ( _docIdMgr->isDocIdDelUseLess( oldNotzip[i].doc_id )) continue;
newNotzip[doc_pos++] = oldNotzip[i];
}
if (unlikely(doc_pos != validCount)) {
TERR("inc expand error, count miss[%u:%u]", validCount, doc_pos);
return KS_EINVAL;
}
// build end doc list
newNotzip[validCount].doc_id = INVALID_DOCID;
newNotzip[validCount].occ = 0;
begin = (char*)(newNotzip + validCount);
}
else {
uint32_t * newNotzip = (uint32_t *)newNzip;
uint32_t * oldNotzip = (uint32_t *)oldNzip;
for (unsigned int i=0; i<doc_count; i++)
{
// build doc_list unit
if ( _docIdMgr->isDocIdDelUseLess( oldNotzip[i] )) continue;
newNotzip[doc_pos++] = oldNotzip[i];
}
if (unlikely(doc_pos != validCount)) {
TERR("inc expand error, count miss[%u:%u]", validCount, doc_pos);
return KS_EINVAL;
}
// build end doc list
newNotzip[validCount] = INVALID_DOCID;
begin = (char*)(newNotzip + validCount);
}
// build auxillary block at the end of skiplist
tHead->pageSize = _pIncMemManager->size(off);
tHead->endOff = begin - p1;
tHead->zipFlag = 0;
end = p1 + tHead->pageSize;
_pIncMemManager->freeOldPage(head->pageSize, pind->pos);
head = tHead;
pind->pos = off;
pind->doc_count = validCount;
return KS_SUCCESS;
}
int32_t IndexFieldInc::zip2zip(inc_idx2_header_t *&head, inc_idx1_unit_t *&pind, char *&begin, char *&end, uint32_t validCount)
{
int32_t pageSize = getPageSize(validCount, 1);
uint32_t off = 0;
if(_pIncMemManager->makeNewPage(pageSize, off) < 0) {
TWARN("inc makeNewPage error");
return KS_ENOMEM;
}
char* p1 = _pIncMemManager->offset2Addr(off);
char* p2 = _pIncMemManager->offset2Addr(pind->pos);
memcpy(p1, p2, sizeof(inc_idx2_header_t));
char *newZip = p1 + sizeof(inc_idx2_header_t);
char *oldZip = p2 + sizeof(inc_idx2_header_t);
inc_idx2_header_t * tHead = (inc_idx2_header_t *)p1;
idx2_zip_header_t *pNewHeader = (idx2_zip_header_t*)newZip;
pNewHeader->skip_list_len = (_maxIncSkipCnt + 1) * sizeof(idx2_zip_skip_unit_t);
idx2_zip_header_t *pOldHeader = (idx2_zip_header_t*)oldZip;
idx2_zip_skip_unit_t *pOldSkip = (idx2_zip_skip_unit_t*)(pOldHeader + 1);
idx2_zip_skip_unit_t *pNewSkip = (idx2_zip_skip_unit_t*)(pNewHeader + 1);
uint32_t baseNum = pOldHeader->block_num;
uint32_t baseId = 0;
uint32_t docId = 0;
uint32_t baseDocCnt = 0;
uint32_t listPos = 0; // 扩展后的doclist个数
uint32_t basePos = 0; // 扩展后的base block个数
uint32_t beginPos = INVALID_DOCID;
if (_maxOccNum > 0) {
idx2_zip_list_unit_t * pOldList = (idx2_zip_list_unit_t*)(pOldSkip + _maxIncSkipCnt + 1);
idx2_zip_list_unit_t * pNewList = (idx2_zip_list_unit_t*)(pNewSkip + _maxIncSkipCnt + 1);
for(unsigned int i = 0; i < baseNum; i++)
{
baseDocCnt = 0;
beginPos = INVALID_DOCID;
baseId = (pOldSkip[i].doc_id_base << 16);
for(unsigned int j = pOldSkip[i].off; j < pOldSkip[i+1].off; j++) {
docId = baseId + pOldList[j].doc_id_diff;
if ( _docIdMgr->isDocIdDelUseLess( docId )) continue;
if (beginPos == INVALID_DOCID) {
beginPos = listPos;
}
pNewList[listPos++] = pOldList[j];
baseDocCnt++;
}
if (baseDocCnt == 0) {
continue;
}
pNewSkip[basePos].doc_id_base = pOldSkip[i].doc_id_base;
pNewSkip[basePos].off = beginPos;
basePos++;
}
pNewList[listPos].doc_id_diff = IDX_GET_DIFF(INVALID_DOCID);
pNewList[listPos].occ = 0;
begin = (char *)(pNewList + listPos);
pNewHeader->doc_list_len = (listPos + 1) * sizeof(idx2_zip_list_unit_t);
}
else {
uint16_t * pOldList = (uint16_t *)(pOldSkip + _maxIncSkipCnt + 1);
uint16_t * pNewList = (uint16_t *)(pNewSkip + _maxIncSkipCnt + 1);
for(unsigned int i = 0; i < baseNum; i++)
{
baseDocCnt = 0;
beginPos = INVALID_DOCID;
baseId = (pOldSkip[i].doc_id_base << 16);
for(unsigned int j = pOldSkip[i].off; j < pOldSkip[i+1].off; j++) {
docId = baseId + pOldList[j];
if ( _docIdMgr->isDocIdDelUseLess( docId )) continue;
if (beginPos == INVALID_DOCID) {
beginPos = listPos;
}
pNewList[listPos++] = pOldList[j];
baseDocCnt++;
}
if (baseDocCnt == 0) {
continue;
}
pNewSkip[basePos].doc_id_base = pOldSkip[i].doc_id_base;
pNewSkip[basePos].off = beginPos;
basePos++;
}
pNewList[listPos] = IDX_GET_DIFF(INVALID_DOCID);
begin = (char *)(pNewList + listPos);
pNewHeader->doc_list_len = (listPos + 1) * sizeof(uint16_t);
}
pNewSkip[basePos].doc_id_base = IDX_GET_BASE(INVALID_DOCID);
pNewSkip[basePos].off = listPos;
pNewHeader->block_num = basePos;
tHead->pageSize = _pIncMemManager->size(off);
tHead->endOff = begin - p1;
tHead->zipFlag = 1;
end = p1 + tHead->pageSize;
_pIncMemManager->freeOldPage(head->pageSize, pind->pos);
head = tHead;
pind->pos = off;
pind->doc_count = listPos;
return KS_SUCCESS;
}
/**
* 将IncManger中的，的字段处理策略初始化
*
* @param node 在配置文件中的 xml 节点 update_add2Modify
*
* @return 0: success ; -1: 程序处理失败
*/
int initIncManager( mxml_node_t * node )
{
if ( NULL == node ) return 0;
mxml_node_t * field = NULL;
IndexReader * reader = IndexReader::getInstance();
IncManager * incMgr = IncManager::getInstance();
ProfileDocAccessor * pflAcr = ProfileManager::getDocAccessor();
field = mxmlFindElement( node, node, "index_field", NULL, NULL, MXML_DESCEND_FIRST );
while ( field != NULL )
{
const char * name = mxmlElementGetAttr( field, "name" );
const char * proc = mxmlElementGetAttr( field, "proc" );
if ( ( NULL == name ) || (0 == strlen(name)) )
{
TERR("update_add2Modify: must have field name");
return -1;
}
if ( ( NULL == proc ) || (0 == strlen(proc)) )
{
TERR("update_add2Modify: must have proc: depend or ignore");
return -1;
}
if ( false == reader->hasField( name ) )
{
TWARN("update_add2Modify: can't find field in index:%s, are you sure", name);
}
if ( 0 == strcmp( proc , "ignore" ) )
{
if ( false == incMgr->addIgnIdxField( name ) )
{
TERR("update_add2Modify: add ignore field to incManager failed:%s", name);
return -1;
}
}
else if ( 0 == strcmp( proc , "depend" ) )
{
const char * pflFieldName = mxmlElementGetAttr( field, "profile_field" );
if ( ( NULL == pflFieldName ) || (0 == strlen(pflFieldName)) )
{
TERR("update_add2Modify: field:%s must have profile_field", name);
return -1;
}
if ( NULL == pflAcr->getProfileField( pflFieldName ) )
{
TERR("update_add2Modify: can't find field in profile:%s", pflFieldName);
return -1;
}
if ( false == incMgr->addIgnIdxField( name ) )
{
TERR("update_add2Modify: add ignore field to incManager failed:%s", name);
return -1;
}
if ( false == incMgr->addDepPflField( pflFieldName ) )
{
TERR("update_add2Modify: add depend pfl field to incManager failed:%s", pflFieldName);
return -1;
}
}
else
{
TERR("update_add2Modify: field:%s proc must be depend or ignore", name);
return -1;
}
field = mxmlFindElement( field, node, "index_field", NULL, NULL, MXML_NO_DESCEND );
}
return 0;
}
/**
* 解析配置文件， 初始化 index_lib内部的各个模块
*
* @param config xml的配置节点。 内部的子节点包括了 索引的各个配置项
* 样例:
* <indexLib>
* <profile path="/dir" />
* <index path="/dir" />
* <idDict path="/dir" />
* <provcity path="/dir" />
* </indexLib>
*
* @return 0: success ; -1: 程序处理失败
*/
int init( mxml_node_t * config )
{
mxml_node_t * modulesNode = NULL;
mxml_node_t * indexLibNode = NULL; // index_lib的根节点
mxml_node_t * profileNode = NULL; // profile配置节点
mxml_node_t * indexNode = NULL; // index配置节点
mxml_node_t * idDictNode = NULL; // nid->docId 字典及deletemap节点
mxml_node_t * provcityNode = NULL; // 行政区划表配置节点
mxml_node_t * add2modNode = NULL; // add转modify配置节点
bool syncFlag = true;
bool exportFlag = false;
modulesNode = mxmlFindElement(config, config, "modules", NULL, NULL, MXML_DESCEND);
if ( modulesNode == NULL || modulesNode->parent != config )
{
TERR("can't find modules's node");
return -1;
}
indexLibNode = mxmlFindElement(modulesNode, modulesNode, "indexLib", NULL, NULL, MXML_DESCEND);
// 检查节点有效性
if ( indexLibNode == NULL || indexLibNode->parent != modulesNode )
{
TERR("can't find indexLib's node");
return -1;
}
// 检查增量持久化配置
const char * syncStr = mxmlElementGetAttr(indexLibNode, "sync" );
if (syncStr && strcasecmp(syncStr, "true") == 0) {
syncFlag = true;
}
else {
syncFlag = false;
}
// 检查mmap_lock配置
const char * mLockStr = mxmlElementGetAttr(indexLibNode, "mmap_lock" );
if (mLockStr && strcasecmp(mLockStr, "true") == 0) {
struct rlimit sysLimit;
if (getrlimit(RLIMIT_MEMLOCK, &sysLimit) == 0) {
if ((sysLimit.rlim_cur==RLIM_INFINITY) &&
(sysLimit.rlim_max==RLIM_INFINITY))
{
IndexConfigParams::getInstance()->setMemLock();
}
else {
TWARN("please unlimited max locked memory to enable mmap_lock");
}
}
else {
TWARN("get RLIMIT_MEMLOCK size failed, disable mmap_lock");
}
}
// 关闭时是否导出倒排索引
const char * et = mxmlElementGetAttr(indexLibNode, "export" );
if (et && strcasecmp(et, "true") == 0) {
exportFlag = true;
}
// 获取配置节点
profileNode = mxmlFindElement(indexLibNode, indexLibNode, "profile", NULL, NULL, MXML_DESCEND);
indexNode = mxmlFindElement(indexLibNode, indexLibNode, "index", NULL, NULL, MXML_DESCEND);
idDictNode = mxmlFindElement(indexLibNode, indexLibNode, "idDict", NULL, NULL, MXML_DESCEND);
provcityNode = mxmlFindElement(indexLibNode, indexLibNode, "provcity", NULL, NULL, MXML_DESCEND);
add2modNode = mxmlFindElement(indexLibNode, indexLibNode, "update_add2Modify", NULL, NULL, MXML_DESCEND);
// 检查节点有效性
if ( profileNode == NULL || profileNode->parent != indexLibNode )
{
TERR("can't find indexLib's child node profile");
return -1;
}
if ( indexNode == NULL || indexNode->parent != indexLibNode )
{
TERR("can't find indexLib's child node index");
return -1;
}
if ( idDictNode == NULL || idDictNode->parent != indexLibNode )
{
TERR("can't find indexLib's child node idDict");
return -1;
}
if ( provcityNode == NULL || provcityNode->parent != indexLibNode )
{
TERR("can't find indexLib's child node provcity");
return -1;
}
int32_t inc_max_num = 0;
const char * inc_max_num_str = mxmlElementGetAttr(indexNode, "inc_max_num" );
if ( inc_max_num_str ) {
inc_max_num = atol(inc_max_num_str);
}
// 开始 初始化 各个模块
if ( initProfile( mxmlElementGetAttr(profileNode, "path" ), syncFlag, false ) ) return -1;
if ( initIdDict ( mxmlElementGetAttr(idDictNode, "path" ), syncFlag ) ) return -1;
if ( initFullIndex( mxmlElementGetAttr(indexNode, "path" ) ) ) return -1;
if ( initIncIndex ( mxmlElementGetAttr(indexNode, "path" ), syncFlag, exportFlag, inc_max_num) ) return -1;
if ( initProvCity ( mxmlElementGetAttr(provcityNode, "path" ) ) ) return -1;
if ( initIncManager( add2modNode ) ) return -1;
return 0;
}
