size_t ProfileMMapFile::makeNewPage(size_t nPageSize) { if (!_mmap) { return 0; } _mmap = false; size_t nOldLength = _nLength; if (ftruncate(_fd, nOldLength + nPageSize) == -1) { TERR("ProfileMMapFile: ftruncate failed:%s", strerror(errno)); return 0; } char* pNew = (char*)mremap((void*)_pBase, nOldLength, (nOldLength + nPageSize), MREMAP_MAYMOVE); if (pNew == MAP_FAILED) { TERR("ProfileMMapFile: mremap failed:%s", strerror(errno)); ::close(_fd); _fd = -1; _nLength = 0; return 0; } _mmap = true; _pBase = pNew; _nLength = nOldLength+nPageSize; return _nLength; }
bool CMMapFile::close() { bool bRes = true; #ifdef _WIN32 if (m_pBase!=NULL) { UnmapViewOfFile(m_pBase); CloseHandle(m_pHander.m_hMapFile); CloseHandle(m_pHander.m_hFile); m_pBase = NULL; m_nLength = 0; m_pHander.m_hFile = m_pHander.m_hMapFile = NULL; } #else if (m_pBase!=NULL) { assert(m_pBase!=NULL&&m_nLength!=0); if(munmap(m_pBase, m_nLength) != 0){ TERR("CMMapFile: munmap fail: %s", strerror(errno)); bRes = false; } if(m_pHander.m_fd != -1 && ::close(m_pHander.m_fd) != 0){ TERR("CMMapFile: close file fail: %s", strerror(errno)); bRes = false; } m_pBase = NULL; m_nLength = 0; m_pHander.m_fd = 0; } #endif if (m_szFileName != NULL) { free(m_szFileName); m_szFileName = NULL; } return bRes; }
/** * 初始化 行政区划转换编码表 * * @param path 数据存放路径 完整路径 * * @return 0: success ; -1: 程序处理失败 */ int initProvCity(const char * path) { ProvCityManager * mgr = ProvCityManager::getInstance(); if ( NULL == mgr ) { TERR("ProvCityManager instance is null"); return -1; } if ( (NULL == path) || strlen( path ) <= 0 ) { TERR("provcity node's path attribute is null"); return -1; } TLOG("begin to load provcity! path:%s", path); if ( false == mgr->load( path ) ) { TERR("load inc index failed! path:%s", path); return -1; } TLOG("load provcity success!"); return 0; }
/** * 初始化 nid->docId的字典 和 deletemap * * @param path 数据存放路径 * * @return 0: success ; -1: 程序处理失败 */ int initIdDict(const char * path, bool sync = true) { DocIdManager * mgr = DocIdManager::getInstance(); if ( NULL == mgr ) { TERR("DocIdManager instance is null"); return -1; } if ( (NULL == path) || strlen( path ) <= 0 ) { TERR("idDict node's path attribute is null"); return -1; } TLOG("begin to load nid->docId dict and deleteMap! path:%s", path); mgr->setDataSync( sync ); if ( false == mgr->load( path ) ) { TERR("load nid->docId dict and deleteMap failed! path:%s", path); return -1; } TLOG("load nid->docId dict and deleteMap success!"); return 0; }
/** * 初始化 增量 index * * @param path 数据存放路径 * * @return 0: success ; -1: 程序处理失败 */ int initIncIndex(const char * path, bool sync, bool et, int32_t inc_max_num ) { IndexIncManager * mgr = IndexIncManager::getInstance(); if ( NULL == mgr ) { TERR("IndexIncManager instance is null"); return -1; } if ( (NULL == path) || strlen( path ) <= 0 ) { TERR("index node's path attribute is null"); return -1; } TLOG("begin to load inc index! path:%s", path); mgr->setDataSync( sync ); mgr->setExportIdx( et ); mgr->setMaxIncNum( inc_max_num ); if ( mgr->open( path ) < 0 ) { TERR("load inc index failed! path:%s", path); return -1; } TLOG("load inc index success!"); return 0; }
/** * 初始化 全量index 的 builder, 重新打开, 为了修改 * * @param path 数据存放路径 * * @return 0: success ; -1: 程序处理失败 */ int initIdxBuilder(const char * path) { IndexBuilder * builder = IndexBuilder::getInstance(); if ( NULL == builder ) { TERR("IndexBuilder instance is null"); return -1; } if ( (NULL == path) || strlen( path ) <= 0 ) { TERR("index node's path attribute is null"); return -1; } TLOG("begin to load full index! path:%s", path); if ( builder->reopen( path ) < 0) { TERR("load full index failed! path:%s", path); return -1; } TLOG("load full index success!"); return 0; }
int IndexFieldInc::dump(bool flag) { char filename[PATH_MAX]; snprintf(filename, PATH_MAX, "%s.%s", _fieldName, INC_IND1_INFIX); int ret = idx_dict_save(_invertedDict, _idxPath, filename); if (ret<0) { TERR("idx_dict_save to %s/%s failed.", _idxPath, filename); return -1; } if(!flag) { // 不需要导出文本索引 return 0; } idx_dict_t* pdict = idx_dict_load(_idxPath, filename); if(NULL == pdict) { TERR("idx_dict_load to %s/%s failed.", _idxPath, filename); return -1; } ret = idx2Txt(pdict); idx_dict_free(pdict); if(ret < 0) { TERR("idx2Txt to %s/%s failed.", _idxPath, filename); return -1; } return 0; }
/** * 重置映射文件大小 * @param size 重置文件大小 * @return false, 重置失败;true, 重置成功 */ bool ProfileMMapFile::resize(size_t size) { if (!_mmap) { return false; } _mmap = false; uint32_t nOldLength = _nLength; if (ftruncate(_fd, size) == -1) { TERR("ProfileMMapFile: resize ftruncate failed:%s", strerror(errno)); return false; } char* pNew = (char*)mremap((void*)_pBase, nOldLength, size, MREMAP_MAYMOVE); if (pNew == MAP_FAILED) { TERR("ProfileMMapFile: resize mremap failed:%s", strerror(errno)); ::close(_fd); _fd = -1; _nLength = 0; return false; } _mmap = true; _pBase = pNew; _nLength = size; return true; }
/** * 为重新进行索引内部数据的整理, 解析配置文件, 初始化 index_lib 内部的builder * * @param config xml的配置节点。 内部的子节点包括了 索引的各个配置项 * 样例: * <indexLib> * <index path="/dir" /> * </indexLib> * * @return 0: success ; -1: 程序处理失败 */ int init_rebuild( mxml_node_t * config ) { mxml_node_t * modulesNode = NULL; mxml_node_t * indexLibNode = NULL; // index_lib的根节点 mxml_node_t * profileNode = NULL; // profile配置节点 mxml_node_t * indexNode = NULL; // index配置节点 mxml_node_t * idDictNode = NULL; // nid->docId 字典及deletemap节点 modulesNode = mxmlFindElement(config, config, "modules", NULL, NULL, MXML_DESCEND); if ( modulesNode == NULL || modulesNode->parent != config ) { TERR("can't find modules's node"); return -1; } indexLibNode = mxmlFindElement(modulesNode, modulesNode, "indexLib", NULL, NULL, MXML_DESCEND); // 检查节点有效性 if ( indexLibNode == NULL || indexLibNode->parent != modulesNode ) { TERR("can't find indexLib's node"); return -1; } // 获取配置节点 profileNode = mxmlFindElement(indexLibNode, indexLibNode, "profile", NULL, NULL, MXML_DESCEND); indexNode = mxmlFindElement(indexLibNode, indexLibNode, "index", NULL, NULL, MXML_DESCEND); idDictNode = mxmlFindElement(indexLibNode, indexLibNode, "idDict", NULL, NULL, MXML_DESCEND); // 检查节点有效性 if ( profileNode == NULL || profileNode->parent != indexLibNode ) { TERR("can't find indexLib's child node profile"); return -1; } if ( indexNode == NULL || indexNode->parent != indexLibNode ) { TERR("can't find indexLib's child node index"); return -1; } if ( idDictNode == NULL || idDictNode->parent != indexLibNode ) { TERR("can't find indexLib's child node idDict"); return -1; } // 开始 初始化 各个模块 if ( initProfile( mxmlElementGetAttr(profileNode, "path" ), false, true ) ) return -1; if ( initIdDict ( mxmlElementGetAttr(idDictNode, "path" ), false) ) return -1; if ( initFullIndex( mxmlElementGetAttr(indexNode, "path" ) ) ) return -1; if ( initIdxBuilder( mxmlElementGetAttr(indexNode, "path" ) ) ) return -1; return 0; }
bool CMMapFile::openEx(const char *szFileName, int flag, size_t length) { if (!(flag & FILE_OPEN_READ)) return false; m_szFileName = strdup(szFileName); assert(m_pBase==NULL); if (m_pHander.m_fd > 0) { if(::close(m_pHander.m_fd) != 0){ TERR("CMMapFile: close file fail: %s", strerror(errno)); }else{ m_pHander.m_fd = -1; } } m_pHander.m_fd = ::open(szFileName, O_RDONLY, 0644); if (m_pHander.m_fd == -1) { TERR("打开文件出错 %s 文件可能不存在, errno: %d, %s", szFileName, errno, strerror(errno)); return false; } struct stat statInfo; if (fstat(m_pHander.m_fd, &statInfo ) < 0) { TERR("文件 %s fstat失败!", szFileName); ::close(m_pHander.m_fd); return false; } m_nLength = statInfo.st_size; //文件长度 if(m_nLength == 0) return true; int mmap_flags = MAP_SHARED; if (index_lib::IndexConfigParams::getInstance()->isMemLock()) { TLOG("open %s with MAP_LOCKED flag!", szFileName); m_pBase = (char*)mmap(0, m_nLength, PROT_READ, mmap_flags | MAP_LOCKED, m_pHander.m_fd, 0); if (m_pBase == MAP_FAILED && errno == EAGAIN) { TERR("open %s with MAP_LOCKED flag failed: %s!", szFileName, strerror(errno)); m_pBase = (char*)mmap(0, m_nLength, PROT_READ, mmap_flags, m_pHander.m_fd, 0); } } else { m_pBase = (char*)mmap(0, m_nLength, PROT_READ, mmap_flags, m_pHander.m_fd, 0); } //允许m_pBase 为NULL(文件size 为0的情况),后面程序逻辑上保证不去读取文件内容 if(m_pBase == MAP_FAILED) { TERR("文件 %s 映射内存文件出错! error code:%d", szFileName,errno); ::close(m_pHander.m_fd); m_pBase = NULL; return false; } madvise(m_pBase, m_nLength, MADV_SEQUENTIAL); //Expect page references in sequential order. return true; }
bool CMMapFile::reopen(off_t nOffset, size_t nLength) { if(m_pHander.m_fd == -1) m_pHander.m_fd = ::open(m_szFileName, O_RDONLY, 0644); if (m_pHander.m_fd == -1) { TERR("打开文件出错 %s 文件可能不存在, errno: %d, %s", m_szFileName, errno, strerror(errno)); return false; } if (m_pBase!=NULL) { assert(m_pBase!=NULL&&m_nLength!=0); if(munmap(m_pBase, m_nLength) != 0) { TERR("CMMapFile::reopen munmap fail: %s", strerror(errno)); return false; } m_pBase = NULL; m_nLength = 0; } int mmap_flags = MAP_SHARED; if (index_lib::IndexConfigParams::getInstance()->isMemLock()) { TLOG("open %s with MAP_LOCKED flag!", m_szFileName); m_pBase = (char*)mmap(0, nLength, PROT_READ, mmap_flags | MAP_LOCKED, m_pHander.m_fd, nOffset); if ( m_pBase == MAP_FAILED && errno == EAGAIN) { TERR("open %s with MAP_LOCKED flag failed: %s!", m_szFileName, strerror(errno)); m_pBase = (char*)mmap(0, nLength, PROT_READ, mmap_flags, m_pHander.m_fd, nOffset); } } else { m_pBase = (char*)mmap(0, nLength, PROT_READ, mmap_flags, m_pHander.m_fd, nOffset); } if (m_pBase == MAP_FAILED) { TERR("CMMapFile::reopen 文件 %s 映射内存文件出错! error code:%d error info:%s", m_szFileName, errno, strerror(errno)); m_pBase = NULL; return false; } m_nLength = nLength; m_nPos = 0; madvise(m_pBase, m_nLength, MADV_SEQUENTIAL); //Expect page references in sequential order. if(::close(m_pHander.m_fd) != 0){ TERR("CMMapFile: close file fail: %s", strerror(errno)); }else{ m_pHander.m_fd = -1; } return true; }
int CfgManager::addPackageElement(char *package_name, char *sub_field, char *p_occ, char *p_weight) { int32_t occ_min = 0; int32_t occ_max = 0; int32_t weight = 0; char *p = strchr(p_occ, '-'); if(p) { *p = '\0'; p += 1; occ_min = atoi(p_occ); occ_max = atoi(p); } else { occ_min = atoi(p_occ); occ_max = occ_min; } weight = atoi(p_weight); PackageElement *element = (PackageElement *) malloc (sizeof(PackageElement)); if(!element) { TERR("malloc mem for element failed"); return -1; } strncpy(element->field_name, sub_field, MAX_FIELD_VALUE_LEN); element->occ_min = occ_min; element->occ_max = occ_max; element->weight = weight; Package *p_package = NULL; HashMap<const char*, Package*>::iterator it = _packages.find(package_name); if(it != _packages.end()) { p_package = it->value; p_package->elements[p_package->size++] = element; } else { p_package = (Package *) malloc (sizeof(Package)); if(!p_package) { free(element); TERR("malloc mem for package failed"); return -1; } p_package->name = package_name; p_package->elements[0] = element; p_package->size = 1; _packages.insert(package_name, p_package); } return 0; }
anet::Connection* BMSCommCenter::makeConnection (clustermap::CMISNode* node_) { char key[64]; if(node_->m_protocol == clustermap::http) { snprintf(key, 64, "tcp:%s:%u", node_->m_ip, node_->m_port); return _conn_pool->makeHttpConnection(key); } else if(node_->m_protocol == clustermap::tcp) { snprintf(key, 64, "tcp:%s:%u", node_->m_ip, node_->m_port); return _conn_pool->makeTcpConnection(key); } else if(node_->m_protocol == clustermap::udp) { TERR("upd protocol is not supnode_->m_portted now."); return NULL; } TERR("unknown protocol."); return NULL; }
/* * func : dump index1 hash tables and close all index2 fds * * args : pbuilder, pointer of index_builder_t struct * * ret : <0, error * : 0, success */ int IndexFieldBuilder::dump() { index_builder_t* pbuilder = (index_builder_t*)_pIndexBuilder; if (!pbuilder) { TERR("parameter error: pbuilder is NULL"); return -1; } int ret = 0; char filename[PATH_MAX]; snprintf(filename, sizeof(filename), "%s.%s", pbuilder->field_name, INVERTED_IDX1_INFIX); ret = idx_dict_save(pbuilder->inverted_dict, pbuilder->path, filename); if (ret<0) { TERR("idx_dict_save to %s/%s failed.", pbuilder->path, filename); return -1; } snprintf(filename, sizeof(filename), "%s.%s", pbuilder->field_name, BITMAP_IDX1_INFIX); ret = idx_dict_save(pbuilder->bitmap_dict, pbuilder->path, filename); if (ret<0) { TERR("idx_dict_save to %s/%s failed.", pbuilder->path, filename); return -1; } for(unsigned int i=0; i < pbuilder->inverted_fd_num; i++) { if (pbuilder->inverted_fd[i] >= 0) { ::close(pbuilder->inverted_fd[i]); pbuilder->inverted_fd[i] = -1; } } pbuilder->inverted_fd_num = 0; if (pbuilder->bitmap_fd >= 0) { off_t off = lseek(pbuilder->bitmap_fd, 0, SEEK_CUR); if(off > 0) {// 防止bitmap访问越界,在最后增加8个byte char buf[8] = {0}; write(pbuilder->bitmap_fd, buf, 8); } ::close(pbuilder->bitmap_fd); pbuilder->bitmap_fd = -1; } return 0; }
bool ProfileMMapFile::release() { if (_unload && _fd > 0) { if (posix_fadvise(_fd, 0, 0, POSIX_FADV_DONTNEED) != 0) { TERR("ProfileMMapFile: posix_fadvise fail:%s", strerror(errno)); } } return true; }
int IndexBuilder::reopen( const char * path ) { if ( NULL == path || strlen(path) >= PATH_MAX) return -1; IndexInfoManager info; if ( info.load( path ) < 0 ) { TERR( "open index info file failed:%s", path ); return -1; } if ( (_maxDocNum = info.getDocNum()) < 0 ) return -1; strcpy(_idxPath, path); uint32_t pos = 0; uint32_t maxOccNum = 0; char * fieldName = NULL; while ( info.getNextField( pos, fieldName, maxOccNum ) >= 0 ) { if ( addField( fieldName, maxOccNum ) < 0) { TERR( "open index failed, path:%s, field:%s", path, fieldName ); return KS_ENOENT; } } void* value = NULL; uint64_t key = 0; IndexFieldBuilder* pBuilder = NULL; _openFlag = true; _fieldMap.itReset(); for(uint32_t i = 0; i < _fieldMap.size(); i++) { if(false == _fieldMap.itNext(key, value)) { return -1; } pBuilder = (IndexFieldBuilder*)value; if(pBuilder->reopen(path) < 0) { return -1; } } return 0; }
bool Node::UpdateVariableImportance(std::vector<float>& vecVarImp) { if(!IsLeaf()) { if(!mpCut->UpdateVariableImportance(vecVarImp)) { TERR("failed to update variable importance: cut."); return false; } if(!mpLeftChild->UpdateVariableImportance(vecVarImp) || !mpRightChild->UpdateVariableImportance(vecVarImp)) { TERR("failed to update variable importance: child."); return false; } } return true; }
Node::~Node() { if(mpSampleSet) { TERR("there is still a sample set in node %d!", mnNodeId); } if(mpCut) { delete mpCut; } }
/** * 向磁盘中写入实际数据 * @param addr 映射地址 * @param len 数据长度 */ int ProfileMMapFile::syncToDisk(char * addr, size_t len) { if (!_mmap || len == 0 || addr == NULL) { return -1; } size_t pos = (addr - _pBase); //printf("fd pos:%lu, %s,addr: %lu, base: %lu\n", pos, _szFileName, addr, _pBase); if (lseek(_fd, pos, SEEK_SET) == -1) { TERR("ProfileMMapFile: syncToDisk error. seek len=%lu. %s", pos, strerror(errno)); return -1; } if (::write(_fd, addr, len) != (ssize_t)len) { TERR("ProfileMMapFile: syncToDisk fail. write error:%s", strerror(errno)); return -1; } return 0; }
int IndexReader::open(const char * path, bool bPreLoad) { if ( NULL == path || strlen(path) >= PATH_MAX) return KS_EINVAL; IndexInfoManager info; if ( info.load( path ) < 0 ) { TERR( "open index info file failed:%s", path ); return KS_ENOENT; } if ( (_maxDocNum = info.getDocNum()) < 0) return KS_EFAILED; strcpy( _path, path ); uint32_t pos = 0; uint32_t maxOccNum = 0; char * fieldName = NULL; IndexFieldReader * pReader = NULL; uint64_t fieldSign = 0; while ( info.getNextField( pos, fieldName, maxOccNum ) >= 0 ) { fieldSign = idx_sign64( fieldName, strlen(fieldName) ); pReader = new IndexFieldReader( maxOccNum, _maxDocNum ); if ( NULL == pReader ) return KS_ENOMEM; if ( 0 != pReader->open( path, fieldName, bPreLoad ) ) { TERR( "open index failed, path:%s, field:%s", path, fieldName ); return KS_EFAILED; } if ( false == _fieldMap.insert( fieldSign, (void *)pReader) ) return KS_EFAILED; } _nidSign = idx_sign64( NID_FIELD_NAME, strlen(NID_FIELD_NAME) ); DocIdManager::getInstance(); // 需要保证docidManager必须保证先起来 return 0; }
int32_t SearcherWorker::get_nid_str(int64_t* pNid, int32_t num, MemPool* memPool, char** output, uint32_t* outlen) { if(!pNid || 0 == num){ return KS_EFAILED; } if(*output != NULL || *outlen != 0){ TERR("memory leak error possiblely "); *outlen = 0; } int len = nidStrMaxLen*num; int print_len = 0; *output = NEW_VEC(memPool, char, len); if(!*output){ return KS_ENOMEM; } print_len = snprintf(*output+*outlen, len, "%lld", pNid[0]); if(print_len < 0){ return KS_EFAILED; } *outlen += print_len; len -= print_len; for(int32_t i = 1; i < num; i++) { print_len = snprintf(*output+*outlen, len, ",%lld", pNid[i]); if(print_len < 0){ return KS_EFAILED; } *outlen += print_len; len -= print_len; if(len <= 0){ TERR("nid num more huge, get mem not enough!!"); return KS_EFAILED; } } return KS_SUCCESS; }
bool Predictor::Store(std::ostream& os) { ModelManagerFactory mmf(ModelManager::PARAMFILE); ModelManager* pModelManager = mmf.Make(); if(!pModelManager) { TERR("unable to make ModelManager."); return false; } bool bRet = pModelManager->Store(mpTreeBoost, os); if(pModelManager) delete pModelManager; return bRet; }
/** * 初始化 profile 部分 * * @param path 数据存放路径 * * @return 0: success ; -1: 程序处理失败 */ int initProfile(const char * path, bool sync = true, bool buildHF = false) { ProfileManager * mgr = ProfileManager::getInstance(); if ( NULL == mgr ) { TERR("ProfileManager instance is null"); return -1; } if ( (NULL == path) || strlen( path ) <= 0 ) { TERR("profile node's path attribute is null"); return -1; } if ( mgr->getProfileDocNum() > 0 ) { TLOG("profile has been loaded!"); return 0; } mgr->setProfilePath( path ); mgr->setDataSync( sync ); TLOG("begin to load profile! path:%s", path); if ( 0 != mgr->load(!buildHF) ) { TERR("load profile failed! path:%s", path); return -1; } TLOG("load profile success!"); return 0; }
bool ProfileMMapFile::close() { bool bRes = true; if (!_mmap) { return false; } _mmap = false; if (munmap(_pBase, _nLength)!=0) { TERR("ProfileMMapFile: munmap fail:%s", strerror(errno)); bRes = false; } if (_fd != -1 && ::close(_fd) != 0) { TERR("ProfileMMapFile: close file fail:%s", strerror(errno)); bRes = false; } _pBase = NULL; _nLength = 0 ; _fd = -1 ; return bRes; }
int DetailWorker::init(Config* cfg, ClusterMessage* info) { int ret = Worker::init(cfg, info); if(ret < 0) { return ret; } if(_que.open(cfg->_enter_data_path)) { TERR("open queue %s error", cfg->_enter_data_path); return -1; } _que.reset(_info.seq); return 0; }
bool CMMapFile::flush() { #ifdef _WIN32 #else if (m_pBase!=NULL) { assert(m_pBase!=NULL&&m_nLength!=0); int nRet = msync(m_pBase, m_nLength, MS_ASYNC); //MS_ASYNC表示同步可以不立即执行 if (nRet == -1){ TERR("CMMapFile: msync fail: %s", strerror(errno)); return false; } } #endif return true; }
int32_t SearcherWorker::getDetailInfo(const char *q_reslut, uint32_t q_size, const char *pureString, char** res, int* res_size) { //for detail char** args = NULL; int fieldCount = 0; char* pLightKey = 0; char* pDL = 0; if(!q_reslut || 0 == q_size){ return KS_EFAILED; } pLightKey = FRAMEWORK::Query::getParam(pureString, q_size, LIGHTKEY); if(pLightKey && pLightKey[0]!=0){ int len = strlen(pLightKey); int ret = decode_url(pLightKey, len, pLightKey, len+1); if(ret<0){ TWARN("decode_url pLightKey error, return!"); _session.setStatus(FRAMEWORK::ss_error); _session.response(); return KS_EFAILED; } pLightKey[ret] = 0; } pDL = FRAMEWORK::Query::getParam(pureString, q_size, DL); fieldCount = get_di_field_count(); if(di_detail_arg_alloc(&args)<(fieldCount+1)){ TERR("di_detail_arg_alloc error, return!"); _session.setStatus(FRAMEWORK::ss_error); _session.response(); return KS_EFAILED; } di_detail_arg_set(args, fieldCount+1, "title", 5, pLightKey, pDL); //Deal with detail if(di_detail_get_v3(q_reslut, pDL, args, fieldCount, res, res_size)){ *res = (char*)malloc(2); (*res)[0] = '0'; (*res)[1] = 0; *res_size = 1; } if(pLightKey)std::free(pLightKey); if(pDL)std::free(pDL); if(args)di_detail_arg_free(args); return KS_SUCCESS; }
DWORD WINAPI MainWorkingThread(LPVOID lParam) { MSG msg; int bret; HANDLE EventHandle = *(HANDLE*) lParam; TINFO("Thread gestartet!"); TINFO("Erstelle Message-Queue..."); PeekMessage(&msg, NULL, WM_USER, WM_USER, PM_NOREMOVE); TINFO("Messasge Queue erstellt!"); SetEvent(EventHandle); while(bret = GetMessage(&msg,NULL,0,0)> 0) { if(handleMessage(msg.message,msg.wParam,msg.lParam) == 0) { TERR("Nachricht konnte nicht bearbeitet werden!"); } } TINFO("Worker Thread wird beendet..."); SetEvent(EventHandle); ExitThread(0); }
bool ProfileMMapFile::flush(bool sync) { if (_bReadOnly == true || !_mmap) { return false; } int nRet; if (!sync) { nRet = msync(_pBase, _nLength, MS_ASYNC); } else { nRet = msync(_pBase, _nLength, MS_SYNC); } if (nRet == -1) { TERR("ProfileMMapFile: msync fail:%s", strerror(errno)); return false; } return true; }
// Parse for parameters & validate them. bool Configuration::ParseConfiguration() { if(!mpConfigParser->Parse()) return false; // get all parameters. // Tree count. if(!mpConfigParser->getItem(std::string("treelink"), std::string("tree_count"), mnTreeCount) || mnTreeCount <= 0) { TERR("invalid TreeLink configuration parameter: tree_count. it should be greater than 0."); return false; } // Maximum leaf count if(!mpConfigParser->getItem(std::string("treelink"), std::string("max_leaf_count"), mnMaxLeafCount) || mnMaxLeafCount <= 0) { TERR("invalid TreeLink configuration parameter: max_leaf_count. it should be greater than 0."); return false; } if(!mpConfigParser->getItem(std::string("treelink"), std::string("max_tree_depth"), mnMaxTreeDepth) || mnMaxTreeDepth <= 0) { TERR("invalid TreeLink configuration parameter: max_tree_depth. it should be greater than 0."); return false; } std::string strParam; // get distribution. if(!mpConfigParser->getItem(std::string("treelink"), std::string("loss_type"), strParam)) { // get distribution. TERR("failed to get treelink parameter: loss_type."); return false; } if(strParam == "gaussian") mLossType = Loss::GAUSSIAN; else if(strParam == "laplacian") mLossType = Loss::LAPLACIAN; else if(strParam == "huber") mLossType = Loss::HUBER; else if(strParam == "adaboost") mLossType = Loss::ADABOOST; else if(strParam == "logistic") mLossType = Loss::LOGISTIC; else if(strParam == "pairwise_ls") mLossType = Loss::PAIRWISE_LS; else { TERR("invalid TreeLink configuration parameter: distribution. it should be gaussian, " "laplacian, huber, adaboost or logistic."); return false; } // get Shrinkage. if(!mpConfigParser->getItem(std::string("treelink"), std::string("shrinkage"), mfShrinkage) || mfShrinkage <= 0.0 || mfShrinkage > 1.0) { TERR("invalid TreeLink configuration parameter: shrinkage." " it should be greater than 0 but less than or equal to 1."); return false; } // get sample rate. if(!mpConfigParser->getItem(std::string("treelink"), std::string("sample_rate"), mfSampleRate) || mfSampleRate <= 0.0 || mfSampleRate > 1.0) { TERR("invalid TreeLink configuration parameter: sample_rate." " it should be greater than 0 but less than or equal to 1."); return false; } // get variable sample rate. if(!mpConfigParser->getItem(std::string("treelink"), std::string("variable_sample_rate"), mfVariableSampleRate) || mfVariableSampleRate <= 0.0 || mfVariableSampleRate > 1.0) { TERR("invalid TreeLink configuration parameter: variable_sample_rate." " it should be greater than 0 but less than or equal to 1."); return false; } if(!mpConfigParser->getItem(std::string("treelink"), std::string("split_balance"), mfSplitBalance) || mfSplitBalance < 0.0 || mfSplitBalance > 0.5) { TERR("invalid treelink configuration parameter: split_balance." " it should be greater than 0 but less or equal to 0.5."); return false; } if(!mpConfigParser->getItem(std::string("treelink"), std::string("min_leaf_sample_count"), mnMinLeafSampleCount) || mnMinLeafSampleCount <= 0) { TERR("invalid treelink configuration parameter: min_leaf_sample_count." " it should be greater than 0"); return false; } if(!mpConfigParser->getItem(std::string("treelink"), std::string("fast_train"), mbFastTrain)) { TERR("invalid TreeLink configuration parameter: fast_train."); return false; } // get discrete cut type if(!mpConfigParser->getItem(std::string("treelink"), std::string("discrete_separator_type"), strParam)) { // get distribution. TERR("failed to get treelink parameter: discrete_separator_type." " it should be leave_one_out or arbitrary."); return false; } if(strParam == "leave_one_out") msepType = Cut::LEAVE_ONE_OUT; else if(strParam == "arbitrary") msepType = Cut::ARBITRARY; else { TERR("invalid treelink configuration parameter: discrete_separator_type." " it should be leave_one_out or arbitrary."); return false; } // get special parameter. if(mLossType == Loss::HUBER) { if(!mpConfigParser->getItem(std::string("treelink"), std::string("quantile_rate"), mfQuantileRate) || mfQuantileRate <= 0.0 || mfQuantileRate > 1.0) { TERR("invalid TreeLink configration parameter: quantile_rate (for huber loss)." " it should be greater than or 0 but less than or equal to 1."); return false; } } if(mLossType == Loss::PAIRWISE_LS) { if(!mpConfigParser->getItem(std::string("treelink"), std::string("margin"), mfMargin) || mfMargin <= 0 || mfMargin > 1) { TERR("invalid treelink configuration parameter: margin (for pairwise_ls loss)." "it should be in range (0, 1]"); return false; } if(!mpConfigParser->getItem(std::string("treelink"), std::string("single_list"), mbSingleList)) { TERR("invalid treelink configuration parameter: single_list (for pairwise_ls loss)."); return false; } } TLOG("succeeded to load TreeLink configuration parameters."); // log parameters TLOG("====treelink training parameters:"); TLOG("loss type: %s", getTypeName().c_str()); TLOG("tree count: %d", mnTreeCount); TLOG("max leaf count: %d", mnMaxLeafCount); TLOG("shrinkage: %f", mfShrinkage); TLOG("subsample rate: %f", mfSampleRate); TLOG("variable subsample rate: %f", mfVariableSampleRate); TLOG("split balance: %f", mfSplitBalance); TLOG("max tree depth: %d", mnMaxTreeDepth); TLOG("min leaf sample count: %d", mnMinLeafSampleCount); TLOG("fast_train: %s", mbFastTrain ? "true" : "false"); if(mLossType == Loss::HUBER) { TLOG("huber quantile: %f", mfQuantileRate); } else if(mLossType == Loss::PAIRWISE_LS) { TLOG("margin: %f", mfMargin); TLOG("single_list: %s", (mbSingleList ? "true" : "false")); } TLOG("====end===="); return true; }