/****************************************************************************** **函数名称: frwd_load_conf **功 能: 加载配置信息 **输入参数: ** path: 配置路径 ** log: 日志对象 **输出参数: ** conf: 配置信息 **返 回: 0:成功 !0:失败 **实现描述: 载入配置文件,再依次解析各标签内容 **注意事项: **作 者: # Qifeng.zou # 2015.06.10 # ******************************************************************************/ int frwd_load_conf(const char *path, frwd_conf_t *conf, log_cycle_t *log) { xml_opt_t opt; xml_tree_t *xml; memset(&opt, 0, sizeof(opt)); memset(conf, 0, sizeof(frwd_conf_t)); /* > 加载配置 */ opt.log = log; opt.pool = (void *)NULL; opt.alloc = (mem_alloc_cb_t)mem_alloc; opt.dealloc = (mem_dealloc_cb_t)mem_dealloc; xml = xml_creat(path, &opt); if (NULL == xml) { return -1; } /* > 提取通用配置 */ if (frwd_conf_load_comm(xml, conf)) { xml_destroy(xml); return -1; } /* > 提取发送配置 */ if (frwd_conf_load_frwder(xml, ".FRWDER.CONN-INVTD", &conf->conn_invtd)) { xml_destroy(xml); return -1; } xml_destroy(xml); return 0; }
/****************************************************************************** **函数名称: invtd_search_parse **功 能: 解析搜索请求 **输入参数: ** ctx: 全局对象 ** buff: 搜索请求(报头+报体) ** len: 数据长度 **输出参数: ** req: 搜索请求 **返 回: 0:成功 !0:失败 **实现描述: **注意事项: **作 者: # Qifeng.zou # 2015.12.27 03:39:00 # ******************************************************************************/ static int invtd_search_parse(invtd_cntx_t *ctx, char *buf, size_t len, mesg_search_req_t *req) { xml_opt_t opt; xml_tree_t *xml; xml_node_t *node; mesg_header_t *head = (mesg_header_t *)buf; const char *xml_str = (const char *)(head + 1); /* > 字节序转换 */ MESG_HEAD_NTOH(head, head); /* > 校验合法性 */ if (!MESG_CHKSUM_ISVALID(head) || (len != MESG_TOTAL_LEN(head->length))) { log_error(ctx->log, "sid:%lu serial:%lu type:%u flag:%u chksum:0x%X len:%d body:%s", head->sid, head->serial, head->type, head->flag, head->chksum, head->length, xml_str); return -1; } log_trace(ctx->log, "sid:%lu serial:%lu type:%u flag:%u chksum:0x%X len:%d body:%s", head->sid, head->serial, head->type, head->flag, head->chksum, head->length, xml_str); /* > 构建XML树 */ memset(&opt, 0, sizeof(opt)); opt.log = ctx->log; opt.pool = NULL; opt.alloc = mem_alloc; opt.dealloc = mem_dealloc; xml = xml_screat(xml_str, head->length, &opt); if (NULL == xml) { log_error(ctx->log, "Parse xml failed!"); return -1; } do { /* > 提取搜索关键字 */ node = xml_query(xml, ".SEARCH.WORDS"); if (NULL == node) { log_error(ctx->log, "Get search words failed!"); break; } snprintf(req->words, sizeof(req->words), "%s", node->value.str); log_trace(ctx->log, "words:%s", req->words); /* > 释放内存空间 */ xml_destroy(xml); return 0; } while (0); xml_destroy(xml); return -1; }
/****************************************************************************** **函数名称: mon_conf_load **功 能: 加载配置信息 **输入参数: ** path: 配置路径 **输出参数: NONE **返 回: 0:成功 !0:失败 **实现描述: **注意事项: **作 者: # Qifeng.zou # 2015.03.15 # ******************************************************************************/ mon_conf_t *mon_conf_load(const char *path) { xml_opt_t opt; xml_tree_t *xml; mon_conf_t *conf; mem_pool_t *pool; /* > 创建配置对象 */ conf = (mon_conf_t *)calloc(1, sizeof(mon_conf_t)); if (NULL == conf) { return NULL; } /* > 构建XML树 */ pool = mem_pool_creat(4 * KB); if (NULL == pool) { free(conf); return NULL; } do { /* > 加载XML树 */ memset(&opt, 0, sizeof(opt)); opt.pool = pool; opt.alloc = (mem_alloc_cb_t)mem_pool_alloc; opt.dealloc = (mem_dealloc_cb_t)mem_pool_dealloc; xml = xml_creat(path, &opt); if (NULL == xml) { break; } /* > 提取配置信息 */ if (mon_conf_load_menu(xml, conf)) { break; } MON_LOAD_CONF(xml, ".MONITOR.CRAWLER", &conf->crwl); MON_LOAD_CONF(xml, ".MONITOR.FILTER", &conf->filter); MON_LOAD_CONF(xml, ".MONITOR.SEARCH", &conf->search); /* > 释放XML树 */ xml_destroy(xml); mem_pool_destroy(pool); return conf; } while(0); /* > 异常处理 */ free(conf); xml_destroy(xml); mem_pool_destroy(pool); return NULL; }
/****************************************************************************** **函数名称: lwsd_load_conf **功 能: 加载配置信息 **输入参数: ** path: 配置文件路径 ** log: 日志对象 **输出参数: ** conf: 配置信息 **返 回: 0:成功 !0:失败 **实现描述: 载入配置文件, 并提取其中的数据 **注意事项: **作 者: # Qifeng.zou # 2015-06-25 22:43:12 # ******************************************************************************/ int lwsd_load_conf(const char *path, lwsd_conf_t *conf, log_cycle_t *log) { xml_opt_t opt; xml_tree_t *xml = NULL; memset(conf, 0, sizeof(lwsd_conf_t)); do { /* > 构建XML树 */ memset(&opt, 0, sizeof(opt)); opt.log = log; opt.pool = (void *)NULL; opt.alloc = (mem_alloc_cb_t)mem_alloc; opt.dealloc = (mem_dealloc_cb_t)mem_dealloc; xml = xml_creat(path, &opt); if (NULL == xml) { log_error(log, "Create xml failed! path:%s", path); break; } /* > 加载通用配置 */ if (lwsd_conf_load_comm(xml, conf, log)) { log_error(log, "Load common configuration failed!"); break; } /* > 加载LWS配置 */ if (lwsd_conf_load_lws(xml, conf, log)) { log_error(log, "Load lws conf failed! path:%s", path); break; } /* > 加载转发配置 */ if (lwsd_conf_load_frwder(xml, conf, log)) { log_error(log, "Load frwder conf failed! path:%s", path); break; } /* > 释放XML树 */ xml_destroy(xml); return 0; } while(0); /* 异常处理 */ if (NULL != xml) { xml_destroy(xml); } return -1; }
int fmll_svm_net_save(const fmll_svm_net * svm_net, const char * fname_prefix) { int ret = 0; const unsigned num = svm_net->num; unsigned u; char node_name[4096]; const fmll_svm ** svm = (const fmll_svm **) svm_net->svm; mxml_node_t * sub_node, * node, * content_node, * main_node = NULL; fmll_try; fmll_throw_if(xml_create(TYPE_SVM_NET, & main_node, & content_node)); fmll_throw_if(xml_set_int(content_node, "num", num)); fmll_throw_null(node = mxmlNewElement(content_node, "SVM")); for(u = 0; u < num; u++) { sprintf(node_name, "svm_%u", u); fmll_throw_null(sub_node = mxmlNewElement(node, node_name)); fmll_throw_if(fmll_svm_save_main(svm[u], sub_node)); } fmll_throw_if(xml_save(fname_prefix, main_node)); fmll_catch; ret = -1; fmll_finally; xml_destroy(main_node); return ret; }
fmll_som * fmll_som_load(const char * fname_prefix, double (* distance_w)(const double *, const double *, const unsigned), double (* distance)(const double *, const double *, const unsigned)) { char node_name[4096]; int map_dim, dim; unsigned u, v, num, * N = NULL; double ** w; fmll_som * som = NULL; mxml_node_t * sub_sub_node, * sub_node, * node, * content_node, * main_node = NULL; fmll_try; fmll_throw_if(xml_load(fname_prefix, TYPE_SOM, & main_node, & content_node)); fmll_throw_if(xml_get_int(content_node, "map_dim", & map_dim)); fmll_throw_if(xml_get_int(content_node, "dim", & dim)); fmll_throw_null(N = fmll_alloc(sizeof(unsigned), 1, map_dim)); fmll_throw_null(node = mxmlFindElement(content_node, content_node, "N", NULL, NULL, MXML_DESCEND_FIRST)); for(u = 0, sub_node = mxmlFindElement(node, node, NULL, NULL, NULL, MXML_DESCEND_FIRST); u < map_dim; u++) { fmll_throw_null(sub_node); N[u] = sub_node->child->value.integer; sub_node = mxmlFindElement(sub_node, node, NULL, NULL, NULL, MXML_DESCEND); } fmll_throw_null(som = fmll_som_init(N, map_dim, dim, distance_w, distance)); fmll_throw_null(node = mxmlFindElement(content_node, content_node, "W", NULL, NULL, MXML_DESCEND_FIRST)); w = som->w; num = som->num; for(u = 0; u < num; u++) { sprintf(node_name, "w_%u", u); fmll_throw_null(sub_node = mxmlFindElement(node, node, node_name, NULL, NULL, MXML_DESCEND_FIRST)); for(v = 0, sub_sub_node = mxmlFindElement(sub_node, sub_node, NULL, NULL, NULL, MXML_DESCEND_FIRST); v < dim; v++) { w[u][v] = sub_sub_node->child->value.real; sub_sub_node = mxmlFindElement(sub_sub_node, sub_node, NULL, NULL, NULL, MXML_DESCEND); } } fmll_catch; fmll_som_destroy(som); som = NULL; fmll_finally; fmll_free(N); xml_destroy(main_node); return som; }
/****************************************************************************** **函数名称: invtd_search_query **功 能: 从倒排表中搜索关键字 **输入参数: ** ctx: 上下文 ** req: 搜索请求信息 **输出参数: NONE **返 回: 搜索结果(以XML树组织) **实现描述: 从倒排表中查询结果,并将结果以XML树组织. **注意事项: 完成发送后, 必须记得释放XML树的所有内存 **作 者: # Qifeng.zou # 2016.01.04 17:35:35 # ******************************************************************************/ static xml_tree_t *invtd_search_query(invtd_cntx_t *ctx, mesg_search_req_t *req) { xml_opt_t opt; xml_tree_t *xml; xml_node_t *root; invt_dic_word_t *word; memset(&opt, 0, sizeof(opt)); /* > 构建XML树 */ opt.pool = NULL; opt.alloc = mem_alloc; opt.dealloc = mem_dealloc; opt.log = ctx->log; xml = xml_empty(&opt); if (NULL == xml) { log_error(ctx->log, "Create xml failed!"); return NULL; } root = xml_set_root(xml, "SEARCH-RSP"); do { pthread_rwlock_rdlock(&ctx->invtab_lock); /* > 搜索倒排表 */ word = (invt_dic_word_t *)invtab_query(ctx->invtab, req->words); if (NULL == word || NULL == word->doc_list) { pthread_rwlock_unlock(&ctx->invtab_lock); log_warn(ctx->log, "Didn't search anything! words:%s", req->words); if (invtd_search_no_data_hdl(xml)) { break; } return xml; } /* > 构建搜索结果 */ if (list_trav(word->doc_list, (trav_cb_t)invtd_search_list_trav, (void *)xml)) { pthread_rwlock_unlock(&ctx->invtab_lock); log_error(ctx->log, "Contribute respone list failed! words:%s", req->words); break; } pthread_rwlock_unlock(&ctx->invtab_lock); xml_add_attr(xml, root, "CODE", SRCH_CODE_OK); /* 设置返回码 */ return xml; } while(0); xml_destroy(xml); return NULL; }
/****************************************************************************** **函数名称: mon_srch_set_body **功 能: 设置搜索报体 **输入参数: ** word: 搜索信息 ** size: 搜索请求的最大报体 **输出参数: NONE ** body: 搜索请求的报体 **返 回: 报体实际长度 **实现描述: **注意事项: **作 者: # Qifeng.zou # 2015.12.27 03:01:48 # ******************************************************************************/ static int mon_srch_set_body(const char *words, char *body, int size) { int len; xml_opt_t opt; xml_tree_t *xml; xml_node_t *node; memset(&opt, 0, sizeof(opt)); /* > 创建XML树 */ opt.log = NULL; opt.pool = NULL; opt.alloc = mem_alloc; opt.dealloc = mem_dealloc; xml = xml_creat_empty(&opt); if (NULL == xml) { fprintf(stderr, "Create xml failed!"); return -1; } node = xml_add_child(xml, xml->root, "SEARCH", NULL); xml_add_attr(xml, node, "WORDS", words); /* > 计算XML长度 */ len = xml_pack_len(xml); if (len >= size) { xml_destroy(xml); return -1; } /* > 输出XML至缓存 */ len = xml_spack(xml, body); xml_destroy(xml); return len; }
int fmll_som_save(const fmll_som * som, const char * fname_prefix) { int ret = 0; const unsigned num = som->num, map_dim = som->map_dim, dim = som->dim, * N = som->N; const double ** w = (const double **) som->w; char node_name[4096]; unsigned u, v; mxml_node_t * sub_node, * node, * content_node, * main_node = NULL; fmll_try; fmll_throw_if(xml_create(TYPE_SOM, & main_node, & content_node)); fmll_throw_if(xml_set_int(content_node, "map_dim", map_dim)); fmll_throw_if(xml_set_int(content_node, "dim", dim)); fmll_throw_null(node = mxmlNewElement(content_node, "N")); for(u = 0; u < map_dim; u++) { sprintf(node_name, "N_%u", u); fmll_throw_if(xml_set_int(node, node_name, N[u])); } fmll_throw_null(node = mxmlNewElement(content_node, "W")); for(u = 0; u < num; u++) { sprintf(node_name, "w_%u", u); fmll_throw_null(sub_node = mxmlNewElement(node, node_name)); for(v = 0; v < dim; v++) { sprintf(node_name, "%u", v); fmll_throw_if(xml_set_double(sub_node, node_name, w[u][v])); } } fmll_throw_if(xml_save(fname_prefix, main_node)); fmll_catch; ret = -1; fmll_finally; xml_destroy(main_node); return ret; }
/****************************************************************************** **函数名称: invtd_search_word_parse **功 能: 解析搜索请求 **输入参数: ** ctx: 全局对象 ** buff: 搜索请求(报头+报体) ** len: 数据长度 **输出参数: ** req: 搜索请求 **返 回: 0:成功 !0:失败 **实现描述: **注意事项: **作 者: # Qifeng.zou # 2015.12.27 03:39:00 # ******************************************************************************/ static int invtd_search_word_parse(invtd_cntx_t *ctx, char *buf, size_t len, mesg_search_word_req_t *req) { xml_opt_t opt; xml_tree_t *xml; xml_node_t *node; agent_header_t *head = (agent_header_t *)buf; const char *xml_str = (const char *)(head + 1); memset(&opt, 0, sizeof(opt)); /* > 字节序转换 */ head->type = ntohl(head->type); head->flag = ntohl(head->flag); head->length = ntohl(head->length); head->mark = ntohl(head->mark); head->serial = ntoh64(head->serial); req->serial = head->serial; log_trace(ctx->log, "serial:%lu type:%u flag:%u mark:0X%x len:%d body:%s", head->serial, head->type, head->flag, head->mark, head->length, xml_str); /* > 构建XML树 */ opt.pool = NULL; opt.alloc = mem_alloc; opt.dealloc = mem_dealloc; xml = xml_screat(xml_str, head->length, &opt); if (NULL == xml) { log_error(ctx->log, "Parse xml failed!"); return -1; } /* > 解析XML树 */ node = xml_query(xml, ".SEARCH.WORDS"); if (NULL == node) { log_error(ctx->log, "Get search words failed!"); } else { snprintf(req->words, sizeof(req->words), "%s", node->value.str); } xml_destroy(xml); log_trace(ctx->log, "words:%s", req->words); return 0; }
/****************************************************************************** **函数名称: xml_creat_empty **功 能: 创建空XML树 **输入参数: ** opt: 选项信息 **输出参数: NONE **返 回: XML树 **实现描述: **注意事项: **作 者: # Qifeng.zou # 2013.06.12 # ******************************************************************************/ xml_tree_t *xml_creat_empty(xml_opt_t *opt) { xml_tree_t *xml; /* 1. 新建对象 */ xml = (xml_tree_t*)opt->alloc(opt->pool, sizeof(xml_tree_t)); if (NULL == xml) { log_error(opt->log, "Calloc failed!"); return NULL; } xml->log = opt->log; xml->pool = opt->pool; xml->alloc = opt->alloc; xml->dealloc = opt->dealloc; /* 2. 添加根节点 */ xml->root = xml_node_creat(xml, XML_NODE_ROOT); if (NULL == xml->root) { log_error(xml->log, "Create node failed!"); xml_destroy(xml); return NULL; } /* 3. 设置根节点名 */ xml->root->name.str = (char *)xml->alloc(xml->pool, XML_ROOT_NAME_SIZE); if (NULL == xml->root->name.str) { log_error(xml->log, "Calloc failed!"); xml_destroy(xml); return NULL; } xml->root->name.len = snprintf(xml->root->name.str, XML_ROOT_NAME_SIZE, "%s", XML_ROOT_NAME); return xml; }
/****************************************************************************** **函数名称: invtd_search_no_data_hdl **功 能: 无数据的应答处理 **输入参数: ** xml: 应答结果树 **输出参数: NONE **返 回: 0:Succ !0:Fail **实现描述: **注意事项: **作 者: # Qifeng.zou # 2016.05.01 01:49:01 # ******************************************************************************/ static int invtd_search_no_data_hdl(xml_tree_t *xml) { xml_node_t *root, *item; char freq[SRCH_SEG_FREQ_LEN]; xml_add_attr(xml, xml->root, "CODE", SRCH_CODE_NO_DATA); /* 无数据 */ snprintf(freq, sizeof(freq), "%d", 0); root = xml->root->child; item = xml_add_child(xml, root, "ITEM", NULL); if (NULL == item) { xml_destroy(xml); return -1; } xml_add_attr(xml, item, "URL", "Sorry, Didn't search anything!"); xml_add_attr(xml, item, "FREQ", freq); return 0; }
fmll_pca * fmll_pca_load(const char * fname_prefix) { int dim, num; char node_name[4096]; unsigned u, v; double ** w; fmll_pca * pca = NULL; mxml_node_t * sub_sub_node, * sub_node, * node, * content_node, * main_node = NULL; fmll_try; fmll_throw_if(xml_load(fname_prefix, TYPE_PCA, & main_node, & content_node)); fmll_throw_if(xml_get_int(content_node, "dim", & dim)); fmll_throw_if(xml_get_int(content_node, "num", & num)); fmll_throw_null(pca = fmll_pca_init(dim, num)); fmll_throw_null(node = mxmlFindElement(content_node, content_node, "W", NULL, NULL, MXML_DESCEND_FIRST)); for(u = 0, w = pca->w; u < num; u++) { sprintf(node_name, "w_%u", u); fmll_throw_null(sub_node = mxmlFindElement(node, node, node_name, NULL, NULL, MXML_DESCEND_FIRST)); for(v = 0, sub_sub_node = mxmlFindElement(sub_node, sub_node, NULL, NULL, NULL, MXML_DESCEND_FIRST); v < dim; v++) { w[u][v] = sub_sub_node->child->value.real; sub_sub_node = mxmlFindElement(sub_sub_node, sub_node, NULL, NULL, NULL, MXML_DESCEND); } } fmll_catch; fmll_pca_destroy(pca); pca = NULL; fmll_finally; xml_destroy(main_node); return pca; }
fmll_svm_net * fmll_svm_net_load(const char * fname_prefix, double (** K)(const double *, const double *, const unsigned)) { int num; unsigned u; char node_name[4096]; fmll_svm ** svm = NULL; fmll_svm_net * svm_net = NULL; mxml_node_t * sub_node, * node, * content_node, * main_node = NULL; fmll_try; fmll_throw_null(svm_net = fmll_alloc(sizeof(fmll_svm_net), 1, 1)); fmll_throw_if(xml_load(fname_prefix, TYPE_SVM_NET, & main_node, & content_node)); fmll_throw_if(xml_get_int(content_node, "num", & num)); svm_net->num = num; fmll_throw_null(svm = svm_net->svm = (fmll_svm **) fmll_alloc(sizeof(fmll_svm *), 1, num)); fmll_throw_null(node = mxmlFindElement(content_node, content_node, "SVM", NULL, NULL, MXML_DESCEND_FIRST)); for(u = 0; u < num; u++) svm[u] = NULL; for(u = 0; u < num; u++) { sprintf(node_name, "svm_%u", u); fmll_throw_null(sub_node = mxmlFindElement(node, node, node_name, NULL, NULL, MXML_DESCEND_FIRST)); fmll_throw_null(svm[u] = fmll_svm_load_main(sub_node, K[u])); } fmll_catch; fmll_svm_net_destroy(svm_net); svm_net = NULL; fmll_finally; xml_destroy(main_node); return svm_net; }
/****************************************************************************** **函数名称: invtd_search_send_and_free **功 能: 从发送搜索结果并释放内存 **输入参数: ** ctx: 上下文 ** xml: 搜索结果信息 ** req: 搜索请求信息 **输出参数: NONE **返 回: 0:成功 !0:失败 **实现描述: 从倒排表中查询结果,并将结果以XML树组织. **注意事项: **作 者: # Qifeng.zou # 2016.01.04 17:35:35 # ******************************************************************************/ static int invtd_search_send_and_free(invtd_cntx_t *ctx, xml_tree_t *xml, mesg_header_t *head, mesg_search_req_t *req, int orig) { void *addr = NULL; mesg_header_t *rsp; /* 应答 */ int body_len, total_len; if (NULL == xml) { return 0; } /* > 发送搜索应答 */ body_len = XML_PACK_LEN(xml); total_len = MESG_TOTAL_LEN(body_len); do { /* 申请内存空间(注: 多1个字符是为字串结束符'\0'预留空间) */ addr = (char *)calloc(1, total_len+1); if (NULL == addr) { break; } /* 设置发送内容 */ rsp = (mesg_header_t *)addr; MESG_HEAD_SET(rsp, MSG_SEARCH_RSP, head->sid, head->nid, head->serial, body_len); MESG_HEAD_HTON(rsp, rsp); xml_spack(xml, rsp->body); /* 放入发送队列 */ if (rtmq_proxy_async_send(ctx->frwder, MSG_SEARCH_RSP, addr, total_len)) { log_error(ctx->log, "Send response failed! serial:%ld words:%s", head->serial, req->words); } free(addr); } while(0); xml_destroy(xml); return INVT_OK; }
int fmll_pca_save(const fmll_pca * pca, const char * fname_prefix) { int ret = 0; const unsigned dim = pca->dim, num = pca->num; const double ** w = (const double **) pca->w; unsigned u, v; char node_name[4096]; mxml_node_t * sub_node, * node, * main_node = NULL, * content_node; fmll_try; fmll_throw_if(xml_create(TYPE_PCA, & main_node, & content_node)); fmll_throw_if(xml_set_int(content_node, "dim", dim)); fmll_throw_if(xml_set_int(content_node, "num", num)); fmll_throw_null(node = mxmlNewElement(content_node, "W")); for(u = 0; u < num; u++) { sprintf(node_name, "w_%u", u); fmll_throw_null(sub_node = mxmlNewElement(node, node_name)); for(v = 0; v < dim; v++) { sprintf(node_name, "%u", v); fmll_throw_if(xml_set_double(sub_node, node_name, w[u][v])); } } fmll_throw_if(xml_save(fname_prefix, main_node)); fmll_catch; ret = -1; fmll_finally; xml_destroy(main_node); return ret; }
/****************************************************************************** **函数名称: xml_screat **功 能: 将XML字串转为XML树 **输入参数: ** str: XML字串 ** len: 字串长度限制(-1:表示遇到\0结束) ** opt: 选项信息 **输出参数: **返 回: XML树 **实现描述: ** 1. 初始化栈 ** 2. 初始化xml树 ** 3. 在内存中将文件解析为XML树 **注意事项: **作 者: # Qifeng.zou # 2013.02.05 # ******************************************************************************/ xml_tree_t *xml_screat(const char *str, size_t len, xml_opt_t *opt) { Stack_t stack; xml_tree_t *xml; if ((NULL == str) || ('\0' == str[0])) { return xml_creat_empty(opt); } do { /* 1. 初始化栈 */ if (stack_init(&stack, XML_MAX_DEPTH)) { log_error(opt->log, "Init xml stack failed!"); break; } /* 2. 初始化XML树 */ xml = xml_init(opt); if (NULL == xml) { log_error(opt->log, "Init xml tree failed!"); break; } /* 3. 解析XML文件缓存 */ if (xml_parse(xml, &stack, str, len)) { log_error(xml->log, "Parse xml failed!"); xml_destroy(xml); break; } stack_destroy(&stack); return xml; } while(0); /* 4. 释放内存空间 */ stack_destroy(&stack); return NULL; }
/****************************************************************************** **函数名称: intvd_search_send_and_free **功 能: 从发送搜索结果并释放内存 **输入参数: ** ctx: 上下文 ** xml: 搜索结果信息 ** req: 搜索请求信息 **输出参数: NONE **返 回: 0:成功 !0:失败 **实现描述: 从倒排表中查询结果,并将结果以XML树组织. **注意事项: **作 者: # Qifeng.zou # 2016.01.04 17:35:35 # ******************************************************************************/ static int intvd_search_send_and_free(invtd_cntx_t *ctx, xml_tree_t *xml, mesg_header_t *head, mesg_search_word_req_t *req, int orig) { void *addr = NULL; mesg_header_t *rsp; /* 应答 */ int body_len, total_len; if (NULL == xml) { return 0; } /* > 发送搜索应答 */ body_len = xml_pack_len(xml); total_len = mesg_total_len(body_len); do { addr = (char *)calloc(1, total_len); if (NULL == addr) { break; } rsp = (mesg_header_t *)addr; mesg_head_set(rsp, MSG_SEARCH_WORD_RSP, head->serial, sizeof(mesg_search_word_req_t)); mesg_head_hton(rsp, rsp); xml_spack(xml, rsp->body); if (rtsd_cli_send(ctx->frwder, MSG_SEARCH_WORD_RSP, addr, total_len)) { log_error(ctx->log, "Send response failed! serial:%ld words:%s", head->serial, req->words); } free(addr); } while(0); xml_destroy(xml); return INVT_OK; }
/****************************************************************************** **函数名称: intvd_search_send_and_free **功 能: 从发送搜索结果并释放内存 **输入参数: ** ctx: 上下文 ** xml: 搜索结果信息 ** req: 搜索请求信息 **输出参数: NONE **返 回: 0:成功 !0:失败 **实现描述: 从倒排表中查询结果,并将结果以XML树组织. **注意事项: **作 者: # Qifeng.zou # 2016.01.04 17:35:35 # ******************************************************************************/ static int intvd_search_send_and_free(invtd_cntx_t *ctx, xml_tree_t *xml, mesg_search_word_req_t *req, int dev_orig) { void *addr = NULL; mesg_data_t *rsp; /* 应答 */ int body_len, total_len; if (NULL == xml) { return 0; } /* > 发送搜索应答 */ body_len = xml_pack_len(xml); total_len = mesg_data_total(body_len); do { addr = (char *)calloc(1, total_len); if (NULL == addr) { break; } rsp = (mesg_data_t *)addr; xml_spack(xml, rsp->body); rsp->serial = hton64(req->serial); if (rtrd_send(ctx->rtrd, MSG_SEARCH_WORD_RSP, dev_orig, addr, total_len)) { log_error(ctx->log, "Send response failed! serial:%ld words:%s", req->serial, req->words); } free(addr); } while(0); xml_destroy(xml); return INVT_OK; }
/* =============================== =============================== */ void destroy_soar_agent (agent * delete_agent) { //print(delete_agent, "\nDestroying agent %s.\n", delete_agent->name); /* AGR 532 */ ///////////////////////////////////////////////////////// // Soar Modules - could potentially rely on hash tables ///////////////////////////////////////////////////////// // cleanup exploration for ( int i=0; i<EXPLORATION_PARAMS; i++ ) delete delete_agent->exploration_params[ i ]; // cleanup Soar-RL delete_agent->rl_params->apoptosis->set_value( rl_param_container::apoptosis_none ); delete delete_agent->rl_prods; delete delete_agent->rl_params; delete delete_agent->rl_stats; delete_agent->rl_params = NULL; // apoptosis needs to know this for excise_all_productions below // cleanup select select_init( delete_agent ); delete delete_agent->select; // cleanup predict delete delete_agent->prediction; // cleanup wma delete_agent->wma_params->activation->set_value( off ); delete delete_agent->wma_forget_pq; delete delete_agent->wma_touched_elements; delete delete_agent->wma_touched_sets; delete delete_agent->wma_params; delete delete_agent->wma_stats; delete delete_agent->wma_timers; // cleanup epmem epmem_close( delete_agent ); delete delete_agent->epmem_params; delete delete_agent->epmem_stats; delete delete_agent->epmem_timers; delete delete_agent->epmem_node_removals; delete delete_agent->epmem_node_mins; delete delete_agent->epmem_node_maxes; delete delete_agent->epmem_edge_removals; delete delete_agent->epmem_edge_mins; delete delete_agent->epmem_edge_maxes; delete delete_agent->epmem_id_repository; delete delete_agent->epmem_id_replacement; delete delete_agent->epmem_id_ref_counts; delete delete_agent->epmem_id_removes; delete delete_agent->epmem_wme_adds; delete delete_agent->epmem_promotions; delete delete_agent->epmem_db; // cleanup smem smem_close( delete_agent ); delete delete_agent->smem_changed_ids; delete delete_agent->smem_params; delete delete_agent->smem_stats; delete delete_agent->smem_timers; delete delete_agent->smem_db; // cleanup statistics db stats_close( delete_agent ); delete delete_agent->stats_db; delete_agent->stats_db = 0; delete delete_agent->debug_params; ///////////////////////////////////////////////////////// ///////////////////////////////////////////////////////// remove_built_in_rhs_functions(delete_agent); getSoarInstance()->Delete_Agent(delete_agent->name); /* Free structures stored in agent structure */ free(delete_agent->name); free(delete_agent->top_dir_stack->directory); free(delete_agent->top_dir_stack); /* Freeing the agent's multi attributes structure */ multi_attribute* lastmattr = 0; for ( multi_attribute* curmattr = delete_agent->multi_attributes; curmattr != 0; curmattr = curmattr->next ) { symbol_remove_ref(delete_agent, curmattr->symbol); free_memory(delete_agent, lastmattr, MISCELLANEOUS_MEM_USAGE); lastmattr = curmattr; } free_memory(delete_agent, lastmattr, MISCELLANEOUS_MEM_USAGE); /* Freeing all the productions owned by this agent */ excise_all_productions(delete_agent, false); /* Releasing all the predefined symbols */ release_predefined_symbols(delete_agent); /* Releasing rete stuff RPM 11/06 */ free_with_pool(&delete_agent->rete_node_pool, delete_agent->dummy_top_node); free_with_pool(&delete_agent->token_pool, delete_agent->dummy_top_token); /* Cleaning up the various callbacks TODO: Not clear why callbacks need to take the agent pointer essentially twice. */ soar_remove_all_monitorable_callbacks(delete_agent); /* RPM 9/06 begin */ free_memory(delete_agent, delete_agent->left_ht, HASH_TABLE_MEM_USAGE); free_memory(delete_agent, delete_agent->right_ht, HASH_TABLE_MEM_USAGE); free_memory(delete_agent, delete_agent->rhs_variable_bindings, MISCELLANEOUS_MEM_USAGE); /* Releasing memory allocated in inital call to start_lex_from_file from init_lexer */ free_memory_block_for_string(delete_agent, delete_agent->current_file->filename); free_memory (delete_agent, delete_agent->current_file, MISCELLANEOUS_MEM_USAGE); /* Releasing trace formats (needs to happen before tracing hashtables are released) */ remove_trace_format (delete_agent, FALSE, FOR_ANYTHING_TF, NIL); remove_trace_format (delete_agent, FALSE, FOR_STATES_TF, NIL); Symbol *evaluate_object_sym = find_sym_constant (delete_agent, "evaluate-object"); remove_trace_format (delete_agent, FALSE, FOR_OPERATORS_TF, evaluate_object_sym); remove_trace_format (delete_agent, TRUE, FOR_STATES_TF, NIL); remove_trace_format (delete_agent, TRUE, FOR_OPERATORS_TF, NIL); /* Releasing hashtables allocated in init_tracing */ for (int i=0; i<3; i++) { free_hash_table(delete_agent, delete_agent->object_tr_ht[i]); free_hash_table(delete_agent, delete_agent->stack_tr_ht[i]); } /* Releasing memory allocated in init_rete */ for (int i=0; i<16; i++) { free_hash_table(delete_agent, delete_agent->alpha_hash_tables[i]); } /* Releasing other hashtables */ free_hash_table(delete_agent, delete_agent->variable_hash_table); free_hash_table(delete_agent, delete_agent->identifier_hash_table); free_hash_table(delete_agent, delete_agent->sym_constant_hash_table); free_hash_table(delete_agent, delete_agent->int_constant_hash_table); free_hash_table(delete_agent, delete_agent->float_constant_hash_table); /* Releasing memory pools */ memory_pool* cur_pool = delete_agent->memory_pools_in_use; memory_pool* next_pool; while(cur_pool != NIL) { next_pool = cur_pool->next; free_memory_pool(delete_agent, cur_pool); cur_pool = next_pool; } /* RPM 9/06 end */ // dynamic memory pools (cleared in the last step) for ( std::map< size_t, memory_pool* >::iterator it=delete_agent->dyn_memory_pools->begin(); it!=delete_agent->dyn_memory_pools->end(); it++ ) { delete it->second; } delete delete_agent->dyn_memory_pools; delete delete_agent->dyn_counters; // JRV: Frees data used by XML generation xml_destroy( delete_agent ); /* Free soar agent structure */ delete delete_agent; }
/****************************************************************************** **函数名称: invtd_search_word_query **功 能: 从倒排表中搜索关键字 **输入参数: ** ctx: 上下文 ** req: 搜索请求信息 **输出参数: NONE **返 回: 搜索结果(以XML树组织) **实现描述: 从倒排表中查询结果,并将结果以XML树组织. **注意事项: 完成发送后, 必须记得释放XML树的所有内存 **作 者: # Qifeng.zou # 2016.01.04 17:35:35 # ******************************************************************************/ static xml_tree_t *invtd_search_word_query(invtd_cntx_t *ctx, mesg_search_word_req_t *req) { int idx; xml_opt_t opt; xml_tree_t *xml; xml_node_t *root, *item; list_node_t *node; invt_word_doc_t *doc; invt_dic_word_t *word; char freq[SRCH_SEG_FREQ_LEN]; memset(&opt, 0, sizeof(opt)); /* > 构建XML树 */ opt.pool = NULL; opt.alloc = mem_alloc; opt.dealloc = mem_dealloc; opt.log = ctx->log; xml = xml_creat_empty(&opt); if (NULL == xml) { log_error(ctx->log, "Create xml failed!"); return NULL; } root = xml_set_root(xml, "SEARCH-RSP"); if (NULL == root) { log_error(ctx->log, "Set xml root failed!"); goto GOTO_SEARCH_ERR; } /* > 搜索倒排表 */ pthread_rwlock_rdlock(&ctx->invtab_lock); word = invtab_query(ctx->invtab, req->words); if (NULL == word || NULL == word->doc_list) { pthread_rwlock_unlock(&ctx->invtab_lock); log_warn(ctx->log, "Didn't search anything! words:%s", req->words); goto GOTO_SEARCH_NO_DATA; } /* > 构建搜索结果 */ idx = 0; node = word->doc_list->head; for (; NULL!=node; node=node->next, ++idx) { doc = (invt_word_doc_t *)node->data; snprintf(freq, sizeof(freq), "%d", doc->freq); item = xml_add_child(xml, root, "ITEM", NULL); if (NULL == item) { goto GOTO_SEARCH_ERR; } xml_add_attr(xml, item, "URL", doc->url.str); xml_add_attr(xml, item, "FREQ", freq); } pthread_rwlock_unlock(&ctx->invtab_lock); xml_add_attr(xml, root, "CODE", SRCH_CODE_OK); /* 设置返回码 */ return xml; GOTO_SEARCH_ERR: /* 异常错误处理 */ xml_destroy(xml); return NULL; GOTO_SEARCH_NO_DATA: /* 搜索结果为空的处理 */ xml_add_attr(xml, root, "CODE", SRCH_CODE_NO_DATA); /* 无数据 */ snprintf(freq, sizeof(freq), "%d", 0); item = xml_add_child(xml, root, "ITEM", NULL); if (NULL == item) { goto GOTO_SEARCH_ERR; } xml_add_attr(xml, item, "URL", "Sorry, Didn't search anything!"); xml_add_attr(xml, item, "FREQ", freq); return xml; }
/****************************************************************************** **函数名称: flt_worker_get_webpage_info **功 能: 获取网页信息 **输入参数: ** path: 网页信息文件 **输出参数: ** info: 网页信息 **返 回: 0:成功 !0:失败 **实现描述: **注意事项: **作 者: # Qifeng.zou # 2015.03.11 # ******************************************************************************/ static int flt_worker_get_webpage_info( const char *path, flt_webpage_info_t *info, log_cycle_t *log) { xml_opt_t opt; xml_tree_t *xml; mem_pool_t *pool; xml_node_t *node, *fix; memset(&opt, 0, sizeof(opt)); /* 1. 新建内存池 */ pool = mem_pool_creat(4 * KB); if (NULL == pool) { log_error(log, "errmsg:[%d] %s!", errno, strerror(errno)); return FLT_ERR; } /* 2. 新建XML树 */ opt.log = log; opt.pool = pool; opt.alloc = (mem_alloc_cb_t)mem_pool_alloc; opt.dealloc = (mem_dealloc_cb_t)mem_pool_dealloc; xml = xml_creat(path, &opt); if (NULL == xml) { mem_pool_destroy(pool); log_error(log, "Create XML failed! path:%s", path); return FLT_ERR; } /* 2. 提取网页信息 */ do { fix = xml_query(xml, ".WPI"); if (NULL == fix) { log_error(log, "Get WPI mark failed!"); break; } /* 获取URI字段 */ node = xml_search(xml, fix, "URI"); if (NULL == node) { log_error(log, "Get URI mark failed!"); break; } snprintf(info->uri, sizeof(info->uri), "%s", node->value.str); /* 获取DEPTH字段 */ node = xml_search(xml, fix, "URI.DEPTH"); if (NULL == node) { log_error(log, "Get DEPTH mark failed!"); break; } info->depth = atoi(node->value.str); /* 获取IP字段 */ node = xml_search(xml, fix, "URI.IP"); if (NULL == node) { log_error(log, "Get IP mark failed!"); break; } snprintf(info->ip, sizeof(info->ip), "%s", node->value.str); /* 获取PORT字段 */ node = xml_search(xml, fix, "URI.PORT"); if (NULL == node) { log_error(log, "Get PORT mark failed!"); break; } info->port = atoi(node->value.str); /* 获取HTML字段 */ node = xml_search(xml, fix, "HTML"); if (NULL == node) { log_error(log, "Get HTML mark failed!"); break; } snprintf(info->html, sizeof(info->html), "%s", node->value.str); /* 获取HTML.SIZE字段 */ node = xml_search(xml, fix, "HTML.SIZE"); if (NULL == node) { log_error(log, "Get HTML.SIZE mark failed!"); break; } info->size = atoi(node->value.str); if (info->size <= 0) { log_info(log, "Html size is zero!"); break; } snprintf(info->fname, sizeof(info->fname), "%s", path); xml_destroy(xml); mem_pool_destroy(pool); return FLT_OK; } while(0); /* 3. 释放XML树 */ xml_destroy(xml); mem_pool_destroy(pool); return FLT_ERR; }
/****************************************************************************** **函数名称: mon_srch_recv_rsp **功 能: 接收搜索应答信息 **输入参数: ** fd: 文件描述符 **输出参数: NONE **返 回: 0:成功 !0:失败 **实现描述: **注意事项: **作 者: # Qifeng.zou # 2015.06.05 17:01:04 # ******************************************************************************/ static int mon_srch_recv_rsp(mon_cntx_t *ctx, mon_srch_conn_t *conn) { int idx; ssize_t n; char addr[8192]; serial_t serial; xml_opt_t opt; xml_tree_t *xml; xml_node_t *node, *attr; struct timeval ctm; int sec, msec, usec; agent_header_t head; mesg_data_t *rsp; memset(addr, 0, sizeof(addr)); /* > 接收应答数据 */ n = read(conn->fd, (void *)&head, sizeof(head)); gettimeofday(&ctm, NULL); if (n <= 0) { fprintf(stderr, " errmsg:[%d] %s!\n", errno, strerror(errno)); return -1; } else if (0 == conn->wrtm.tv_sec) { fprintf(stderr, " Didn't send search request but received response!\n"); } /* > 字节序转换 */ head.type = ntohl(head.type); head.flag = ntohl(head.flag); head.length = ntohl(head.length); head.mark = ntohl(head.mark); head.serial = ntoh64(head.serial); n = read(conn->fd, addr, head.length); /* > 显示查询结果 */ fprintf(stderr, " ============================================\n"); rsp = (mesg_data_t *)addr; rsp->serial = ntoh64(rsp->serial); serial.serial = head.serial; fprintf(stderr, " >Serial: %lu [nid(%u) sid(%u) seq(%u)]\n", head.serial, serial.nid, serial.sid, serial.seq); fprintf(stderr, " >Serial: %lu:%lu\n", rsp->serial, head.serial); memset(&opt, 0, sizeof(opt)); opt.pool = NULL; opt.alloc = mem_alloc; opt.dealloc = mem_dealloc; xml = xml_screat(rsp->body, head.length, &opt); if (NULL == xml) { fprintf(stderr, " Format isn't right! body:%s\n", rsp->body); return -1; } node = xml_query(xml, ".SEARCH-RSP.ITEM"); for (idx=1; NULL != node; node = xml_brother(node), ++idx) { attr = xml_search(xml, node, "URL"); fprintf(stderr, " [%02d] URL:%s", idx, attr->value.str); attr = xml_search(xml, node, "FREQ"); fprintf(stderr, " FREQ:%s\n", attr->value.str); } xml_destroy(xml); /* > 打印统计信息 */ sec = ctm.tv_sec - conn->wrtm.tv_sec; msec = (ctm.tv_usec - conn->wrtm.tv_usec)/1000; if (msec < 0) { sec -= 1; msec += 1000; } usec = (ctm.tv_usec - conn->wrtm.tv_usec)%1000; if (usec < 0) { usec += 1000; msec -= 1; if (msec < 0) { sec -= 1; msec += 1000; } } if (msec < 0) { msec += 1000; sec -= 1; } fprintf(stderr, " >Spend: %d(s).%03d(ms).%03d(us)\n", sec, msec, usec); fprintf(stderr, " ============================================\n"); return 0; }