size_t ProfileMMapFile::makeNewPage(size_t nPageSize)
{
if (!_mmap) {
return 0;
}
_mmap = false;
size_t nOldLength = _nLength;
if (ftruncate(_fd, nOldLength + nPageSize) == -1) {
TERR("ProfileMMapFile: ftruncate failed:%s", strerror(errno));
return 0;
}
char* pNew = (char*)mremap((void*)_pBase, nOldLength, (nOldLength + nPageSize), MREMAP_MAYMOVE);
if (pNew == MAP_FAILED) {
TERR("ProfileMMapFile: mremap failed:%s", strerror(errno));
::close(_fd);
_fd = -1;
_nLength = 0;
return 0;
}
_mmap = true;
_pBase = pNew;
_nLength = nOldLength+nPageSize;
return _nLength;
}
bool CMMapFile::close()
{
bool bRes = true;
#ifdef _WIN32
if (m_pBase!=NULL) {
UnmapViewOfFile(m_pBase);
CloseHandle(m_pHander.m_hMapFile);
CloseHandle(m_pHander.m_hFile);
m_pBase = NULL;
m_nLength = 0;
m_pHander.m_hFile = m_pHander.m_hMapFile = NULL;
}
#else
if (m_pBase!=NULL) {
assert(m_pBase!=NULL&&m_nLength!=0);
if(munmap(m_pBase, m_nLength) != 0){
TERR("CMMapFile: munmap fail: %s", strerror(errno));
bRes = false;
}
if(m_pHander.m_fd != -1 && ::close(m_pHander.m_fd) != 0){
TERR("CMMapFile: close file fail: %s", strerror(errno));
bRes = false;
}
m_pBase = NULL;
m_nLength = 0;
m_pHander.m_fd = 0;
}
#endif
if (m_szFileName != NULL) {
free(m_szFileName);
m_szFileName = NULL;
}
return bRes;
}
/**
* 初始化 行政区划转换编码表
*
* @param path 数据存放路径 完整路径
*
* @return 0: success ; -1: 程序处理失败
*/
int initProvCity(const char * path)
{
ProvCityManager * mgr = ProvCityManager::getInstance();
if ( NULL == mgr )
{
TERR("ProvCityManager instance is null");
return -1;
}
if ( (NULL == path) || strlen( path ) <= 0 )
{
TERR("provcity node's path attribute is null");
return -1;
}
TLOG("begin to load provcity! path:%s", path);
if ( false == mgr->load( path ) )
{
TERR("load inc index failed! path:%s", path);
return -1;
}
TLOG("load provcity success!");
return 0;
}
/**
* 初始化 nid->docId的字典 和 deletemap
*
* @param path 数据存放路径
*
* @return 0: success ; -1: 程序处理失败
*/
int initIdDict(const char * path, bool sync = true)
{
DocIdManager * mgr = DocIdManager::getInstance();
if ( NULL == mgr )
{
TERR("DocIdManager instance is null");
return -1;
}
if ( (NULL == path) || strlen( path ) <= 0 )
{
TERR("idDict node's path attribute is null");
return -1;
}
TLOG("begin to load nid->docId dict and deleteMap! path:%s", path);
mgr->setDataSync( sync );
if ( false == mgr->load( path ) )
{
TERR("load nid->docId dict and deleteMap failed! path:%s", path);
return -1;
}
TLOG("load nid->docId dict and deleteMap success!");
return 0;
}
/**
* 初始化 增量 index
*
* @param path 数据存放路径
*
* @return 0: success ; -1: 程序处理失败
*/
int initIncIndex(const char * path, bool sync, bool et, int32_t inc_max_num )
{
IndexIncManager * mgr = IndexIncManager::getInstance();
if ( NULL == mgr )
{
TERR("IndexIncManager instance is null");
return -1;
}
if ( (NULL == path) || strlen( path ) <= 0 )
{
TERR("index node's path attribute is null");
return -1;
}
TLOG("begin to load inc index! path:%s", path);
mgr->setDataSync( sync );
mgr->setExportIdx( et );
mgr->setMaxIncNum( inc_max_num );
if ( mgr->open( path ) < 0 )
{
TERR("load inc index failed! path:%s", path);
return -1;
}
TLOG("load inc index success!");
return 0;
}
/**
* 初始化 全量index 的 builder， 重新打开， 为了修改
*
* @param path 数据存放路径
*
* @return 0: success ; -1: 程序处理失败
*/
int initIdxBuilder(const char * path)
{
IndexBuilder * builder = IndexBuilder::getInstance();
if ( NULL == builder )
{
TERR("IndexBuilder instance is null");
return -1;
}
if ( (NULL == path) || strlen( path ) <= 0 )
{
TERR("index node's path attribute is null");
return -1;
}
TLOG("begin to load full index! path:%s", path);
if ( builder->reopen( path ) < 0)
{
TERR("load full index failed! path:%s", path);
return -1;
}
TLOG("load full index success!");
return 0;
}
int IndexFieldInc::dump(bool flag)
{
char filename[PATH_MAX];
snprintf(filename, PATH_MAX, "%s.%s", _fieldName, INC_IND1_INFIX);
int ret = idx_dict_save(_invertedDict, _idxPath, filename);
if (ret<0) {
TERR("idx_dict_save to %s/%s failed.", _idxPath, filename);
return -1;
}
if(!flag) { // 不需要导出文本索引
return 0;
}
idx_dict_t* pdict = idx_dict_load(_idxPath, filename);
if(NULL == pdict) {
TERR("idx_dict_load to %s/%s failed.", _idxPath, filename);
return -1;
}
ret = idx2Txt(pdict);
idx_dict_free(pdict);
if(ret < 0) {
TERR("idx2Txt to %s/%s failed.", _idxPath, filename);
return -1;
}
return 0;
}
/**
* 重置映射文件大小
* @param size 重置文件大小
* @return false, 重置失败;true, 重置成功
*/
bool ProfileMMapFile::resize(size_t size)
{
if (!_mmap) {
return false;
}
_mmap = false;
uint32_t nOldLength = _nLength;
if (ftruncate(_fd, size) == -1) {
TERR("ProfileMMapFile: resize ftruncate failed:%s", strerror(errno));
return false;
}
char* pNew = (char*)mremap((void*)_pBase, nOldLength, size, MREMAP_MAYMOVE);
if (pNew == MAP_FAILED) {
TERR("ProfileMMapFile: resize mremap failed:%s", strerror(errno));
::close(_fd);
_fd = -1;
_nLength = 0;
return false;
}
_mmap = true;
_pBase = pNew;
_nLength = size;
return true;
}
/**
* 为重新进行索引内部数据的整理， 解析配置文件， 初始化 index_lib 内部的builder
*
* @param config xml的配置节点。 内部的子节点包括了 索引的各个配置项
* 样例:
* <indexLib>
* <index path="/dir" />
* </indexLib>
*
* @return 0: success ; -1: 程序处理失败
*/
int init_rebuild( mxml_node_t * config )
{
mxml_node_t * modulesNode = NULL;
mxml_node_t * indexLibNode = NULL; // index_lib的根节点
mxml_node_t * profileNode = NULL; // profile配置节点
mxml_node_t * indexNode = NULL; // index配置节点
mxml_node_t * idDictNode = NULL; // nid->docId 字典及deletemap节点
modulesNode = mxmlFindElement(config, config, "modules", NULL, NULL, MXML_DESCEND);
if ( modulesNode == NULL || modulesNode->parent != config )
{
TERR("can't find modules's node");
return -1;
}
indexLibNode = mxmlFindElement(modulesNode, modulesNode, "indexLib", NULL, NULL, MXML_DESCEND);
// 检查节点有效性
if ( indexLibNode == NULL || indexLibNode->parent != modulesNode )
{
TERR("can't find indexLib's node");
return -1;
}
// 获取配置节点
profileNode = mxmlFindElement(indexLibNode, indexLibNode, "profile", NULL, NULL, MXML_DESCEND);
indexNode = mxmlFindElement(indexLibNode, indexLibNode, "index", NULL, NULL, MXML_DESCEND);
idDictNode = mxmlFindElement(indexLibNode, indexLibNode, "idDict", NULL, NULL, MXML_DESCEND);
// 检查节点有效性
if ( profileNode == NULL || profileNode->parent != indexLibNode )
{
TERR("can't find indexLib's child node profile");
return -1;
}
if ( indexNode == NULL || indexNode->parent != indexLibNode )
{
TERR("can't find indexLib's child node index");
return -1;
}
if ( idDictNode == NULL || idDictNode->parent != indexLibNode )
{
TERR("can't find indexLib's child node idDict");
return -1;
}
// 开始 初始化 各个模块
if ( initProfile( mxmlElementGetAttr(profileNode, "path" ), false, true ) ) return -1;
if ( initIdDict ( mxmlElementGetAttr(idDictNode, "path" ), false) ) return -1;
if ( initFullIndex( mxmlElementGetAttr(indexNode, "path" ) ) ) return -1;
if ( initIdxBuilder( mxmlElementGetAttr(indexNode, "path" ) ) ) return -1;
return 0;
}
bool CMMapFile::openEx(const char *szFileName, int flag, size_t length)
{
if (!(flag & FILE_OPEN_READ)) return false;
m_szFileName = strdup(szFileName);
assert(m_pBase==NULL);
if (m_pHander.m_fd > 0)
{
if(::close(m_pHander.m_fd) != 0){
TERR("CMMapFile: close file fail: %s", strerror(errno));
}else{
m_pHander.m_fd = -1;
}
}
m_pHander.m_fd = ::open(szFileName, O_RDONLY, 0644);
if (m_pHander.m_fd == -1) {
TERR("打开文件出错 %s 文件可能不存在, errno: %d, %s", szFileName, errno, strerror(errno));
return false;
}
struct stat statInfo;
if (fstat(m_pHander.m_fd, &statInfo ) < 0) {
TERR("文件 %s fstat失败!", szFileName);
::close(m_pHander.m_fd);
return false;
}
m_nLength = statInfo.st_size; //文件长度
if(m_nLength == 0) return true;
int mmap_flags = MAP_SHARED;
if (index_lib::IndexConfigParams::getInstance()->isMemLock()) {
TLOG("open %s with MAP_LOCKED flag!", szFileName);
m_pBase = (char*)mmap(0, m_nLength, PROT_READ, mmap_flags | MAP_LOCKED, m_pHander.m_fd, 0);
if (m_pBase == MAP_FAILED && errno == EAGAIN) {
TERR("open %s with MAP_LOCKED flag failed: %s!", szFileName, strerror(errno));
m_pBase = (char*)mmap(0, m_nLength, PROT_READ, mmap_flags, m_pHander.m_fd, 0);
}
}
else {
m_pBase = (char*)mmap(0, m_nLength, PROT_READ, mmap_flags, m_pHander.m_fd, 0);
}
//允许m_pBase 为NULL(文件size 为0的情况),后面程序逻辑上保证不去读取文件内容
if(m_pBase == MAP_FAILED) {
TERR("文件 %s 映射内存文件出错! error code:%d", szFileName,errno);
::close(m_pHander.m_fd);
m_pBase = NULL;
return false;
}
madvise(m_pBase, m_nLength, MADV_SEQUENTIAL); //Expect page references in sequential order.
return true;
}
bool CMMapFile::reopen(off_t nOffset, size_t nLength)
{
if(m_pHander.m_fd == -1)
m_pHander.m_fd = ::open(m_szFileName, O_RDONLY, 0644);
if (m_pHander.m_fd == -1) {
TERR("打开文件出错 %s 文件可能不存在, errno: %d, %s", m_szFileName, errno, strerror(errno));
return false;
}
if (m_pBase!=NULL)
{
assert(m_pBase!=NULL&&m_nLength!=0);
if(munmap(m_pBase, m_nLength) != 0)
{
TERR("CMMapFile::reopen munmap fail: %s", strerror(errno));
return false;
}
m_pBase = NULL;
m_nLength = 0;
}
int mmap_flags = MAP_SHARED;
if (index_lib::IndexConfigParams::getInstance()->isMemLock()) {
TLOG("open %s with MAP_LOCKED flag!", m_szFileName);
m_pBase = (char*)mmap(0, nLength, PROT_READ, mmap_flags | MAP_LOCKED, m_pHander.m_fd, nOffset);
if ( m_pBase == MAP_FAILED && errno == EAGAIN) {
TERR("open %s with MAP_LOCKED flag failed: %s!", m_szFileName, strerror(errno));
m_pBase = (char*)mmap(0, nLength, PROT_READ, mmap_flags, m_pHander.m_fd, nOffset);
}
}
else {
m_pBase = (char*)mmap(0, nLength, PROT_READ, mmap_flags, m_pHander.m_fd, nOffset);
}
if (m_pBase == MAP_FAILED)
{
TERR("CMMapFile::reopen 文件 %s 映射内存文件出错! error code:%d error info:%s", m_szFileName, errno, strerror(errno));
m_pBase = NULL;
return false;
}
m_nLength = nLength;
m_nPos = 0;
madvise(m_pBase, m_nLength, MADV_SEQUENTIAL); //Expect page references in sequential order.
if(::close(m_pHander.m_fd) != 0){
TERR("CMMapFile: close file fail: %s", strerror(errno));
}else{
m_pHander.m_fd = -1;
}
return true;
}
int CfgManager::addPackageElement(char *package_name,
char *sub_field,
char *p_occ,
char *p_weight)
{
int32_t occ_min = 0;
int32_t occ_max = 0;
int32_t weight = 0;
char *p = strchr(p_occ, '-');
if(p) {
*p = '\0';
p += 1;
occ_min = atoi(p_occ);
occ_max = atoi(p);
} else {
occ_min = atoi(p_occ);
occ_max = occ_min;
}
weight = atoi(p_weight);
PackageElement *element = (PackageElement *) malloc (sizeof(PackageElement));
if(!element) {
TERR("malloc mem for element failed");
return -1;
}
strncpy(element->field_name, sub_field, MAX_FIELD_VALUE_LEN);
element->occ_min = occ_min;
element->occ_max = occ_max;
element->weight = weight;
Package *p_package = NULL;
HashMap<const char*, Package*>::iterator it = _packages.find(package_name);
if(it != _packages.end()) {
p_package = it->value;
p_package->elements[p_package->size++] = element;
} else {
p_package = (Package *) malloc (sizeof(Package));
if(!p_package) {
free(element);
TERR("malloc mem for package failed");
return -1;
}
p_package->name = package_name;
p_package->elements[0] = element;
p_package->size = 1;
_packages.insert(package_name, p_package);
}
return 0;
}
anet::Connection* BMSCommCenter::makeConnection (clustermap::CMISNode* node_) {
char key[64];
if(node_->m_protocol == clustermap::http) {
snprintf(key, 64, "tcp:%s:%u", node_->m_ip, node_->m_port);
return _conn_pool->makeHttpConnection(key);
} else if(node_->m_protocol == clustermap::tcp) {
snprintf(key, 64, "tcp:%s:%u", node_->m_ip, node_->m_port);
return _conn_pool->makeTcpConnection(key);
} else if(node_->m_protocol == clustermap::udp) {
TERR("upd protocol is not supnode_->m_portted now.");
return NULL;
}
TERR("unknown protocol.");
return NULL;
}
/*
* func : dump index1 hash tables and close all index2 fds
*
* args : pbuilder, pointer of index_builder_t struct
*
* ret : <0, error
* : 0, success
*/
int IndexFieldBuilder::dump()
{
index_builder_t* pbuilder = (index_builder_t*)_pIndexBuilder;
if (!pbuilder) {
TERR("parameter error: pbuilder is NULL");
return -1;
}
int ret = 0;
char filename[PATH_MAX];
snprintf(filename, sizeof(filename), "%s.%s",
pbuilder->field_name, INVERTED_IDX1_INFIX);
ret = idx_dict_save(pbuilder->inverted_dict, pbuilder->path, filename);
if (ret<0) {
TERR("idx_dict_save to %s/%s failed.", pbuilder->path, filename);
return -1;
}
snprintf(filename, sizeof(filename), "%s.%s",
pbuilder->field_name, BITMAP_IDX1_INFIX);
ret = idx_dict_save(pbuilder->bitmap_dict, pbuilder->path, filename);
if (ret<0) {
TERR("idx_dict_save to %s/%s failed.", pbuilder->path, filename);
return -1;
}
for(unsigned int i=0; i < pbuilder->inverted_fd_num; i++) {
if (pbuilder->inverted_fd[i] >= 0) {
::close(pbuilder->inverted_fd[i]);
pbuilder->inverted_fd[i] = -1;
}
}
pbuilder->inverted_fd_num = 0;
if (pbuilder->bitmap_fd >= 0) {
off_t off = lseek(pbuilder->bitmap_fd, 0, SEEK_CUR);
if(off > 0) {// 防止bitmap访问越界，在最后增加8个byte
char buf[8] = {0};
write(pbuilder->bitmap_fd, buf, 8);
}
::close(pbuilder->bitmap_fd);
pbuilder->bitmap_fd = -1;
}
return 0;
}
bool ProfileMMapFile::release()
{
if (_unload && _fd > 0) {
if (posix_fadvise(_fd, 0, 0, POSIX_FADV_DONTNEED) != 0) {
TERR("ProfileMMapFile: posix_fadvise fail:%s", strerror(errno));
}
}
return true;
}
int IndexBuilder::reopen( const char * path )
{
if ( NULL == path || strlen(path) >= PATH_MAX) return -1;
IndexInfoManager info;
if ( info.load( path ) < 0 )
{
TERR( "open index info file failed:%s", path );
return -1;
}
if ( (_maxDocNum = info.getDocNum()) < 0 ) return -1;
strcpy(_idxPath, path);
uint32_t pos = 0;
uint32_t maxOccNum = 0;
char * fieldName = NULL;
while ( info.getNextField( pos, fieldName, maxOccNum ) >= 0 )
{
if ( addField( fieldName, maxOccNum ) < 0)
{
TERR( "open index failed, path:%s, field:%s", path, fieldName );
return KS_ENOENT;
}
}
void* value = NULL;
uint64_t key = 0;
IndexFieldBuilder* pBuilder = NULL;
_openFlag = true;
_fieldMap.itReset();
for(uint32_t i = 0; i < _fieldMap.size(); i++) {
if(false == _fieldMap.itNext(key, value)) {
return -1;
}
pBuilder = (IndexFieldBuilder*)value;
if(pBuilder->reopen(path) < 0) {
return -1;
}
}
return 0;
}
bool Node::UpdateVariableImportance(std::vector<float>& vecVarImp)
{
if(!IsLeaf())
{
if(!mpCut->UpdateVariableImportance(vecVarImp))
{
TERR("failed to update variable importance: cut.");
return false;
}
if(!mpLeftChild->UpdateVariableImportance(vecVarImp) ||
!mpRightChild->UpdateVariableImportance(vecVarImp))
{
TERR("failed to update variable importance: child.");
return false;
}
}
return true;
}
Node::~Node()
{
if(mpSampleSet)
{
TERR("there is still a sample set in node %d!", mnNodeId);
}
if(mpCut)
{
delete mpCut;
}
}
/**
* 向磁盘中写入实际数据
* @param addr 映射地址
* @param len 数据长度
*/
int ProfileMMapFile::syncToDisk(char * addr, size_t len)
{
if (!_mmap || len == 0 || addr == NULL) {
return -1;
}
size_t pos = (addr - _pBase);
//printf("fd pos:%lu, %s,addr: %lu, base: %lu\n", pos, _szFileName, addr, _pBase);
if (lseek(_fd, pos, SEEK_SET) == -1) {
TERR("ProfileMMapFile: syncToDisk error. seek len=%lu. %s", pos, strerror(errno));
return -1;
}
if (::write(_fd, addr, len) != (ssize_t)len) {
TERR("ProfileMMapFile: syncToDisk fail. write error:%s", strerror(errno));
return -1;
}
return 0;
}
int IndexReader::open(const char * path, bool bPreLoad)
{
if ( NULL == path || strlen(path) >= PATH_MAX) return KS_EINVAL;
IndexInfoManager info;
if ( info.load( path ) < 0 )
{
TERR( "open index info file failed:%s", path );
return KS_ENOENT;
}
if ( (_maxDocNum = info.getDocNum()) < 0) return KS_EFAILED;
strcpy( _path, path );
uint32_t pos = 0;
uint32_t maxOccNum = 0;
char * fieldName = NULL;
IndexFieldReader * pReader = NULL;
uint64_t fieldSign = 0;
while ( info.getNextField( pos, fieldName, maxOccNum ) >= 0 )
{
fieldSign = idx_sign64( fieldName, strlen(fieldName) );
pReader = new IndexFieldReader( maxOccNum, _maxDocNum );
if ( NULL == pReader ) return KS_ENOMEM;
if ( 0 != pReader->open( path, fieldName, bPreLoad ) )
{
TERR( "open index failed, path:%s, field:%s", path, fieldName );
return KS_EFAILED;
}
if ( false == _fieldMap.insert( fieldSign, (void *)pReader) )
return KS_EFAILED;
}
_nidSign = idx_sign64( NID_FIELD_NAME, strlen(NID_FIELD_NAME) );
DocIdManager::getInstance(); // 需要保证docidManager必须保证先起来
return 0;
}
int32_t SearcherWorker::get_nid_str(int64_t* pNid, int32_t num, MemPool* memPool, char** output, uint32_t* outlen)
{
if(!pNid || 0 == num){
return KS_EFAILED;
}
if(*output != NULL || *outlen != 0){
TERR("memory leak error possiblely ");
*outlen = 0;
}
int len = nidStrMaxLen*num;
int print_len = 0;
*output = NEW_VEC(memPool, char, len);
if(!*output){
return KS_ENOMEM;
}
print_len = snprintf(*output+*outlen, len, "%lld", pNid[0]);
if(print_len < 0){
return KS_EFAILED;
}
*outlen += print_len;
len -= print_len;
for(int32_t i = 1; i < num; i++)
{
print_len = snprintf(*output+*outlen, len, ",%lld", pNid[i]);
if(print_len < 0){
return KS_EFAILED;
}
*outlen += print_len;
len -= print_len;
if(len <= 0){
TERR("nid num more huge, get mem not enough!!");
return KS_EFAILED;
}
}
return KS_SUCCESS;
}
bool Predictor::Store(std::ostream& os)
{
ModelManagerFactory mmf(ModelManager::PARAMFILE);
ModelManager* pModelManager = mmf.Make();
if(!pModelManager)
{
TERR("unable to make ModelManager.");
return false;
}
bool bRet = pModelManager->Store(mpTreeBoost, os);
if(pModelManager)
delete pModelManager;
return bRet;
}
/**
* 初始化 profile 部分
*
* @param path 数据存放路径
*
* @return 0: success ; -1: 程序处理失败
*/
int initProfile(const char * path, bool sync = true, bool buildHF = false)
{
ProfileManager * mgr = ProfileManager::getInstance();
if ( NULL == mgr )
{
TERR("ProfileManager instance is null");
return -1;
}
if ( (NULL == path) || strlen( path ) <= 0 )
{
TERR("profile node's path attribute is null");
return -1;
}
if ( mgr->getProfileDocNum() > 0 )
{
TLOG("profile has been loaded!");
return 0;
}
mgr->setProfilePath( path );
mgr->setDataSync( sync );
TLOG("begin to load profile! path:%s", path);
if ( 0 != mgr->load(!buildHF) )
{
TERR("load profile failed! path:%s", path);
return -1;
}
TLOG("load profile success!");
return 0;
}
bool ProfileMMapFile::close()
{
bool bRes = true;
if (!_mmap) {
return false;
}
_mmap = false;
if (munmap(_pBase, _nLength)!=0) {
TERR("ProfileMMapFile: munmap fail:%s", strerror(errno));
bRes = false;
}
if (_fd != -1 && ::close(_fd) != 0) {
TERR("ProfileMMapFile: close file fail:%s", strerror(errno));
bRes = false;
}
_pBase = NULL;
_nLength = 0 ;
_fd = -1 ;
return bRes;
}
int DetailWorker::init(Config* cfg, ClusterMessage* info)
{
int ret = Worker::init(cfg, info);
if(ret < 0) {
return ret;
}
if(_que.open(cfg->_enter_data_path)) {
TERR("open queue %s error", cfg->_enter_data_path);
return -1;
}
_que.reset(_info.seq);
return 0;
}
bool CMMapFile::flush()
{
#ifdef _WIN32
#else
if (m_pBase!=NULL) {
assert(m_pBase!=NULL&&m_nLength!=0);
int nRet = msync(m_pBase, m_nLength, MS_ASYNC); //MS_ASYNC表示同步可以不立即执行
if (nRet == -1){
TERR("CMMapFile: msync fail: %s", strerror(errno));
return false;
}
}
#endif
return true;
}
int32_t SearcherWorker::getDetailInfo(const char *q_reslut, uint32_t q_size, const char *pureString, char** res, int* res_size)
{
//for detail
char** args = NULL;
int fieldCount = 0;
char* pLightKey = 0;
char* pDL = 0;
if(!q_reslut || 0 == q_size){
return KS_EFAILED;
}
pLightKey = FRAMEWORK::Query::getParam(pureString, q_size, LIGHTKEY);
if(pLightKey && pLightKey[0]!=0){
int len = strlen(pLightKey);
int ret = decode_url(pLightKey, len, pLightKey, len+1);
if(ret<0){
TWARN("decode_url pLightKey error, return!");
_session.setStatus(FRAMEWORK::ss_error);
_session.response();
return KS_EFAILED;
}
pLightKey[ret] = 0;
}
pDL = FRAMEWORK::Query::getParam(pureString, q_size, DL);
fieldCount = get_di_field_count();
if(di_detail_arg_alloc(&args)<(fieldCount+1)){
TERR("di_detail_arg_alloc error, return!");
_session.setStatus(FRAMEWORK::ss_error);
_session.response();
return KS_EFAILED;
}
di_detail_arg_set(args, fieldCount+1, "title", 5, pLightKey, pDL);
//Deal with detail
if(di_detail_get_v3(q_reslut, pDL, args, fieldCount, res, res_size)){
*res = (char*)malloc(2);
(*res)[0] = '0';
(*res)[1] = 0;
*res_size = 1;
}
if(pLightKey)std::free(pLightKey);
if(pDL)std::free(pDL);
if(args)di_detail_arg_free(args);
return KS_SUCCESS;
}
DWORD WINAPI MainWorkingThread(LPVOID lParam) {
MSG msg; int bret;
HANDLE EventHandle = *(HANDLE*) lParam;
TINFO("Thread gestartet!");
TINFO("Erstelle Message-Queue...");
PeekMessage(&msg, NULL, WM_USER, WM_USER, PM_NOREMOVE);
TINFO("Messasge Queue erstellt!");
SetEvent(EventHandle);
while(bret = GetMessage(&msg,NULL,0,0)> 0) {
if(handleMessage(msg.message,msg.wParam,msg.lParam) == 0) {
TERR("Nachricht konnte nicht bearbeitet werden!");
}
}
TINFO("Worker Thread wird beendet...");
SetEvent(EventHandle);
ExitThread(0);
}
bool ProfileMMapFile::flush(bool sync)
{
if (_bReadOnly == true || !_mmap) {
return false;
}
int nRet;
if (!sync) {
nRet = msync(_pBase, _nLength, MS_ASYNC);
}
else {
nRet = msync(_pBase, _nLength, MS_SYNC);
}
if (nRet == -1) {
TERR("ProfileMMapFile: msync fail:%s", strerror(errno));
return false;
}
return true;
}
// Parse for parameters & validate them.
bool Configuration::ParseConfiguration()
{
if(!mpConfigParser->Parse())
return false;
// get all parameters.
// Tree count.
if(!mpConfigParser->getItem(std::string("treelink"), std::string("tree_count"), mnTreeCount)
|| mnTreeCount <= 0)
{
TERR("invalid TreeLink configuration parameter: tree_count. it should be greater than 0.");
return false;
}
// Maximum leaf count
if(!mpConfigParser->getItem(std::string("treelink"), std::string("max_leaf_count"), mnMaxLeafCount)
|| mnMaxLeafCount <= 0)
{
TERR("invalid TreeLink configuration parameter: max_leaf_count. it should be greater than 0.");
return false;
}
if(!mpConfigParser->getItem(std::string("treelink"), std::string("max_tree_depth"), mnMaxTreeDepth)
|| mnMaxTreeDepth <= 0)
{
TERR("invalid TreeLink configuration parameter: max_tree_depth. it should be greater than 0.");
return false;
}
std::string strParam;
// get distribution.
if(!mpConfigParser->getItem(std::string("treelink"), std::string("loss_type"), strParam))
{
// get distribution.
TERR("failed to get treelink parameter: loss_type.");
return false;
}
if(strParam == "gaussian")
mLossType = Loss::GAUSSIAN;
else if(strParam == "laplacian")
mLossType = Loss::LAPLACIAN;
else if(strParam == "huber")
mLossType = Loss::HUBER;
else if(strParam == "adaboost")
mLossType = Loss::ADABOOST;
else if(strParam == "logistic")
mLossType = Loss::LOGISTIC;
else if(strParam == "pairwise_ls")
mLossType = Loss::PAIRWISE_LS;
else
{
TERR("invalid TreeLink configuration parameter: distribution. it should be gaussian, "
"laplacian, huber, adaboost or logistic.");
return false;
}
// get Shrinkage.
if(!mpConfigParser->getItem(std::string("treelink"), std::string("shrinkage"), mfShrinkage)
|| mfShrinkage <= 0.0 || mfShrinkage > 1.0)
{
TERR("invalid TreeLink configuration parameter: shrinkage."
" it should be greater than 0 but less than or equal to 1.");
return false;
}
// get sample rate.
if(!mpConfigParser->getItem(std::string("treelink"), std::string("sample_rate"), mfSampleRate)
|| mfSampleRate <= 0.0 || mfSampleRate > 1.0)
{
TERR("invalid TreeLink configuration parameter: sample_rate."
" it should be greater than 0 but less than or equal to 1.");
return false;
}
// get variable sample rate.
if(!mpConfigParser->getItem(std::string("treelink"),
std::string("variable_sample_rate"), mfVariableSampleRate)
|| mfVariableSampleRate <= 0.0 || mfVariableSampleRate > 1.0)
{
TERR("invalid TreeLink configuration parameter: variable_sample_rate."
" it should be greater than 0 but less than or equal to 1.");
return false;
}
if(!mpConfigParser->getItem(std::string("treelink"), std::string("split_balance"),
mfSplitBalance) || mfSplitBalance < 0.0 || mfSplitBalance > 0.5)
{
TERR("invalid treelink configuration parameter: split_balance."
" it should be greater than 0 but less or equal to 0.5.");
return false;
}
if(!mpConfigParser->getItem(std::string("treelink"), std::string("min_leaf_sample_count"),
mnMinLeafSampleCount) || mnMinLeafSampleCount <= 0)
{
TERR("invalid treelink configuration parameter: min_leaf_sample_count."
" it should be greater than 0");
return false;
}
if(!mpConfigParser->getItem(std::string("treelink"), std::string("fast_train"), mbFastTrain))
{
TERR("invalid TreeLink configuration parameter: fast_train.");
return false;
}
// get discrete cut type
if(!mpConfigParser->getItem(std::string("treelink"),
std::string("discrete_separator_type"), strParam))
{
// get distribution.
TERR("failed to get treelink parameter: discrete_separator_type."
" it should be leave_one_out or arbitrary.");
return false;
}
if(strParam == "leave_one_out")
msepType = Cut::LEAVE_ONE_OUT;
else if(strParam == "arbitrary")
msepType = Cut::ARBITRARY;
else
{
TERR("invalid treelink configuration parameter: discrete_separator_type."
" it should be leave_one_out or arbitrary.");
return false;
}
// get special parameter.
if(mLossType == Loss::HUBER)
{
if(!mpConfigParser->getItem(std::string("treelink"), std::string("quantile_rate"), mfQuantileRate)
|| mfQuantileRate <= 0.0 || mfQuantileRate > 1.0)
{
TERR("invalid TreeLink configration parameter: quantile_rate (for huber loss)."
" it should be greater than or 0 but less than or equal to 1.");
return false;
}
}
if(mLossType == Loss::PAIRWISE_LS)
{
if(!mpConfigParser->getItem(std::string("treelink"), std::string("margin"), mfMargin)
|| mfMargin <= 0 || mfMargin > 1)
{
TERR("invalid treelink configuration parameter: margin (for pairwise_ls loss)."
"it should be in range (0, 1]");
return false;
}
if(!mpConfigParser->getItem(std::string("treelink"), std::string("single_list"), mbSingleList))
{
TERR("invalid treelink configuration parameter: single_list (for pairwise_ls loss).");
return false;
}
}
TLOG("succeeded to load TreeLink configuration parameters.");
// log parameters
TLOG("====treelink training parameters:");
TLOG("loss type: %s", getTypeName().c_str());
TLOG("tree count: %d", mnTreeCount);
TLOG("max leaf count: %d", mnMaxLeafCount);
TLOG("shrinkage: %f", mfShrinkage);
TLOG("subsample rate: %f", mfSampleRate);
TLOG("variable subsample rate: %f", mfVariableSampleRate);
TLOG("split balance: %f", mfSplitBalance);
TLOG("max tree depth: %d", mnMaxTreeDepth);
TLOG("min leaf sample count: %d", mnMinLeafSampleCount);
TLOG("fast_train: %s", mbFastTrain ? "true" : "false");
if(mLossType == Loss::HUBER)
{
TLOG("huber quantile: %f", mfQuantileRate);
}
else if(mLossType == Loss::PAIRWISE_LS)
{
TLOG("margin: %f", mfMargin);
TLOG("single_list: %s", (mbSingleList ? "true" : "false"));
}
TLOG("====end====");
return true;
}
