/******************************************************************************
**函数名称: frwd_load_conf
**功 能: 加载配置信息
**输入参数:
** path: 配置路径
** log: 日志对象
**输出参数:
** conf: 配置信息
**返 回: 0:成功 !0:失败
**实现描述: 载入配置文件，再依次解析各标签内容
**注意事项:
**作 者: # Qifeng.zou # 2015.06.10 #
******************************************************************************/
int frwd_load_conf(const char *path, frwd_conf_t *conf, log_cycle_t *log)
{
xml_opt_t opt;
xml_tree_t *xml;
memset(&opt, 0, sizeof(opt));
memset(conf, 0, sizeof(frwd_conf_t));
/* > 加载配置 */
opt.log = log;
opt.pool = (void *)NULL;
opt.alloc = (mem_alloc_cb_t)mem_alloc;
opt.dealloc = (mem_dealloc_cb_t)mem_dealloc;
xml = xml_creat(path, &opt);
if (NULL == xml) {
return -1;
}
/* > 提取通用配置 */
if (frwd_conf_load_comm(xml, conf)) {
xml_destroy(xml);
return -1;
}
/* > 提取发送配置 */
if (frwd_conf_load_frwder(xml, ".FRWDER.CONN-INVTD", &conf->conn_invtd)) {
xml_destroy(xml);
return -1;
}
xml_destroy(xml);
return 0;
}
/******************************************************************************
**函数名称: invtd_search_parse
**功 能: 解析搜索请求
**输入参数:
** ctx: 全局对象
** buff: 搜索请求(报头+报体)
** len: 数据长度
**输出参数:
** req: 搜索请求
**返 回: 0:成功 !0:失败
**实现描述:
**注意事项:
**作 者: # Qifeng.zou # 2015.12.27 03:39:00 #
******************************************************************************/
static int invtd_search_parse(invtd_cntx_t *ctx,
char *buf, size_t len, mesg_search_req_t *req)
{
xml_opt_t opt;
xml_tree_t *xml;
xml_node_t *node;
mesg_header_t *head = (mesg_header_t *)buf;
const char *xml_str = (const char *)(head + 1);
/* > 字节序转换 */
MESG_HEAD_NTOH(head, head);
/* > 校验合法性 */
if (!MESG_CHKSUM_ISVALID(head)
|| (len != MESG_TOTAL_LEN(head->length)))
{
log_error(ctx->log, "sid:%lu serial:%lu type:%u flag:%u chksum:0x%X len:%d body:%s",
head->sid, head->serial, head->type,
head->flag, head->chksum, head->length, xml_str);
return -1;
}
log_trace(ctx->log, "sid:%lu serial:%lu type:%u flag:%u chksum:0x%X len:%d body:%s",
head->sid, head->serial, head->type,
head->flag, head->chksum, head->length, xml_str);
/* > 构建XML树 */
memset(&opt, 0, sizeof(opt));
opt.log = ctx->log;
opt.pool = NULL;
opt.alloc = mem_alloc;
opt.dealloc = mem_dealloc;
xml = xml_screat(xml_str, head->length, &opt);
if (NULL == xml) {
log_error(ctx->log, "Parse xml failed!");
return -1;
}
do {
/* > 提取搜索关键字 */
node = xml_query(xml, ".SEARCH.WORDS");
if (NULL == node) {
log_error(ctx->log, "Get search words failed!");
break;
}
snprintf(req->words, sizeof(req->words), "%s", node->value.str);
log_trace(ctx->log, "words:%s", req->words);
/* > 释放内存空间 */
xml_destroy(xml);
return 0;
} while (0);
xml_destroy(xml);
return -1;
}
/******************************************************************************
**函数名称: mon_conf_load
**功 能: 加载配置信息
**输入参数:
** path: 配置路径
**输出参数: NONE
**返 回: 0:成功 !0:失败
**实现描述:
**注意事项:
**作 者: # Qifeng.zou # 2015.03.15 #
******************************************************************************/
mon_conf_t *mon_conf_load(const char *path)
{
xml_opt_t opt;
xml_tree_t *xml;
mon_conf_t *conf;
mem_pool_t *pool;
/* > 创建配置对象 */
conf = (mon_conf_t *)calloc(1, sizeof(mon_conf_t));
if (NULL == conf) {
return NULL;
}
/* > 构建XML树 */
pool = mem_pool_creat(4 * KB);
if (NULL == pool) {
free(conf);
return NULL;
}
do
{
/* > 加载XML树 */
memset(&opt, 0, sizeof(opt));
opt.pool = pool;
opt.alloc = (mem_alloc_cb_t)mem_pool_alloc;
opt.dealloc = (mem_dealloc_cb_t)mem_pool_dealloc;
xml = xml_creat(path, &opt);
if (NULL == xml) {
break;
}
/* > 提取配置信息 */
if (mon_conf_load_menu(xml, conf)) {
break;
}
MON_LOAD_CONF(xml, ".MONITOR.CRAWLER", &conf->crwl);
MON_LOAD_CONF(xml, ".MONITOR.FILTER", &conf->filter);
MON_LOAD_CONF(xml, ".MONITOR.SEARCH", &conf->search);
/* > 释放XML树 */
xml_destroy(xml);
mem_pool_destroy(pool);
return conf;
} while(0);
/* > 异常处理 */
free(conf);
xml_destroy(xml);
mem_pool_destroy(pool);
return NULL;
}
/******************************************************************************
**函数名称: lwsd_load_conf
**功 能: 加载配置信息
**输入参数:
** path: 配置文件路径
** log: 日志对象
**输出参数:
** conf: 配置信息
**返 回: 0:成功 !0:失败
**实现描述: 载入配置文件, 并提取其中的数据
**注意事项:
**作 者: # Qifeng.zou # 2015-06-25 22:43:12 #
******************************************************************************/
int lwsd_load_conf(const char *path, lwsd_conf_t *conf, log_cycle_t *log)
{
xml_opt_t opt;
xml_tree_t *xml = NULL;
memset(conf, 0, sizeof(lwsd_conf_t));
do {
/* > 构建XML树 */
memset(&opt, 0, sizeof(opt));
opt.log = log;
opt.pool = (void *)NULL;
opt.alloc = (mem_alloc_cb_t)mem_alloc;
opt.dealloc = (mem_dealloc_cb_t)mem_dealloc;
xml = xml_creat(path, &opt);
if (NULL == xml) {
log_error(log, "Create xml failed! path:%s", path);
break;
}
/* > 加载通用配置 */
if (lwsd_conf_load_comm(xml, conf, log)) {
log_error(log, "Load common configuration failed!");
break;
}
/* > 加载LWS配置 */
if (lwsd_conf_load_lws(xml, conf, log)) {
log_error(log, "Load lws conf failed! path:%s", path);
break;
}
/* > 加载转发配置 */
if (lwsd_conf_load_frwder(xml, conf, log)) {
log_error(log, "Load frwder conf failed! path:%s", path);
break;
}
/* > 释放XML树 */
xml_destroy(xml);
return 0;
} while(0);
/* 异常处理 */
if (NULL != xml) { xml_destroy(xml); }
return -1;
}
int fmll_svm_net_save(const fmll_svm_net * svm_net, const char * fname_prefix)
{
int ret = 0;
const unsigned num = svm_net->num;
unsigned u;
char node_name[4096];
const fmll_svm ** svm = (const fmll_svm **) svm_net->svm;
mxml_node_t * sub_node, * node, * content_node, * main_node = NULL;
fmll_try;
fmll_throw_if(xml_create(TYPE_SVM_NET, & main_node, & content_node));
fmll_throw_if(xml_set_int(content_node, "num", num));
fmll_throw_null(node = mxmlNewElement(content_node, "SVM"));
for(u = 0; u < num; u++)
{
sprintf(node_name, "svm_%u", u);
fmll_throw_null(sub_node = mxmlNewElement(node, node_name));
fmll_throw_if(fmll_svm_save_main(svm[u], sub_node));
}
fmll_throw_if(xml_save(fname_prefix, main_node));
fmll_catch;
ret = -1;
fmll_finally;
xml_destroy(main_node);
return ret;
}
fmll_som * fmll_som_load(const char * fname_prefix,
double (* distance_w)(const double *, const double *, const unsigned), double (* distance)(const double *, const double *, const unsigned))
{
char node_name[4096];
int map_dim, dim;
unsigned u, v, num, * N = NULL;
double ** w;
fmll_som * som = NULL;
mxml_node_t * sub_sub_node, * sub_node, * node, * content_node, * main_node = NULL;
fmll_try;
fmll_throw_if(xml_load(fname_prefix, TYPE_SOM, & main_node, & content_node));
fmll_throw_if(xml_get_int(content_node, "map_dim", & map_dim));
fmll_throw_if(xml_get_int(content_node, "dim", & dim));
fmll_throw_null(N = fmll_alloc(sizeof(unsigned), 1, map_dim));
fmll_throw_null(node = mxmlFindElement(content_node, content_node, "N", NULL, NULL, MXML_DESCEND_FIRST));
for(u = 0, sub_node = mxmlFindElement(node, node, NULL, NULL, NULL, MXML_DESCEND_FIRST); u < map_dim; u++)
{
fmll_throw_null(sub_node);
N[u] = sub_node->child->value.integer;
sub_node = mxmlFindElement(sub_node, node, NULL, NULL, NULL, MXML_DESCEND);
}
fmll_throw_null(som = fmll_som_init(N, map_dim, dim, distance_w, distance));
fmll_throw_null(node = mxmlFindElement(content_node, content_node, "W", NULL, NULL, MXML_DESCEND_FIRST));
w = som->w;
num = som->num;
for(u = 0; u < num; u++)
{
sprintf(node_name, "w_%u", u);
fmll_throw_null(sub_node = mxmlFindElement(node, node, node_name, NULL, NULL, MXML_DESCEND_FIRST));
for(v = 0, sub_sub_node = mxmlFindElement(sub_node, sub_node, NULL, NULL, NULL, MXML_DESCEND_FIRST); v < dim; v++)
{
w[u][v] = sub_sub_node->child->value.real;
sub_sub_node = mxmlFindElement(sub_sub_node, sub_node, NULL, NULL, NULL, MXML_DESCEND);
}
}
fmll_catch;
fmll_som_destroy(som);
som = NULL;
fmll_finally;
fmll_free(N);
xml_destroy(main_node);
return som;
}
/******************************************************************************
**函数名称: invtd_search_query
**功 能: 从倒排表中搜索关键字
**输入参数:
** ctx: 上下文
** req: 搜索请求信息
**输出参数: NONE
**返 回: 搜索结果(以XML树组织)
**实现描述: 从倒排表中查询结果，并将结果以XML树组织.
**注意事项: 完成发送后, 必须记得释放XML树的所有内存
**作 者: # Qifeng.zou # 2016.01.04 17:35:35 #
******************************************************************************/
static xml_tree_t *invtd_search_query(invtd_cntx_t *ctx, mesg_search_req_t *req)
{
xml_opt_t opt;
xml_tree_t *xml;
xml_node_t *root;
invt_dic_word_t *word;
memset(&opt, 0, sizeof(opt));
/* > 构建XML树 */
opt.pool = NULL;
opt.alloc = mem_alloc;
opt.dealloc = mem_dealloc;
opt.log = ctx->log;
xml = xml_empty(&opt);
if (NULL == xml) {
log_error(ctx->log, "Create xml failed!");
return NULL;
}
root = xml_set_root(xml, "SEARCH-RSP");
do {
pthread_rwlock_rdlock(&ctx->invtab_lock);
/* > 搜索倒排表 */
word = (invt_dic_word_t *)invtab_query(ctx->invtab, req->words);
if (NULL == word
|| NULL == word->doc_list)
{
pthread_rwlock_unlock(&ctx->invtab_lock);
log_warn(ctx->log, "Didn't search anything! words:%s", req->words);
if (invtd_search_no_data_hdl(xml)) {
break;
}
return xml;
}
/* > 构建搜索结果 */
if (list_trav(word->doc_list, (trav_cb_t)invtd_search_list_trav, (void *)xml)) {
pthread_rwlock_unlock(&ctx->invtab_lock);
log_error(ctx->log, "Contribute respone list failed! words:%s", req->words);
break;
}
pthread_rwlock_unlock(&ctx->invtab_lock);
xml_add_attr(xml, root, "CODE", SRCH_CODE_OK); /* 设置返回码 */
return xml;
} while(0);
xml_destroy(xml);
return NULL;
}
/******************************************************************************
**函数名称: mon_srch_set_body
**功 能: 设置搜索报体
**输入参数:
** word: 搜索信息
** size: 搜索请求的最大报体
**输出参数: NONE
** body: 搜索请求的报体
**返 回: 报体实际长度
**实现描述:
**注意事项:
**作 者: # Qifeng.zou # 2015.12.27 03:01:48 #
******************************************************************************/
static int mon_srch_set_body(const char *words, char *body, int size)
{
int len;
xml_opt_t opt;
xml_tree_t *xml;
xml_node_t *node;
memset(&opt, 0, sizeof(opt));
/* > 创建XML树 */
opt.log = NULL;
opt.pool = NULL;
opt.alloc = mem_alloc;
opt.dealloc = mem_dealloc;
xml = xml_creat_empty(&opt);
if (NULL == xml) {
fprintf(stderr, "Create xml failed!");
return -1;
}
node = xml_add_child(xml, xml->root, "SEARCH", NULL);
xml_add_attr(xml, node, "WORDS", words);
/* > 计算XML长度 */
len = xml_pack_len(xml);
if (len >= size) {
xml_destroy(xml);
return -1;
}
/* > 输出XML至缓存 */
len = xml_spack(xml, body);
xml_destroy(xml);
return len;
}
int fmll_som_save(const fmll_som * som, const char * fname_prefix)
{
int ret = 0;
const unsigned num = som->num, map_dim = som->map_dim, dim = som->dim, * N = som->N;
const double ** w = (const double **) som->w;
char node_name[4096];
unsigned u, v;
mxml_node_t * sub_node, * node, * content_node, * main_node = NULL;
fmll_try;
fmll_throw_if(xml_create(TYPE_SOM, & main_node, & content_node));
fmll_throw_if(xml_set_int(content_node, "map_dim", map_dim));
fmll_throw_if(xml_set_int(content_node, "dim", dim));
fmll_throw_null(node = mxmlNewElement(content_node, "N"));
for(u = 0; u < map_dim; u++)
{
sprintf(node_name, "N_%u", u);
fmll_throw_if(xml_set_int(node, node_name, N[u]));
}
fmll_throw_null(node = mxmlNewElement(content_node, "W"));
for(u = 0; u < num; u++)
{
sprintf(node_name, "w_%u", u);
fmll_throw_null(sub_node = mxmlNewElement(node, node_name));
for(v = 0; v < dim; v++)
{
sprintf(node_name, "%u", v);
fmll_throw_if(xml_set_double(sub_node, node_name, w[u][v]));
}
}
fmll_throw_if(xml_save(fname_prefix, main_node));
fmll_catch;
ret = -1;
fmll_finally;
xml_destroy(main_node);
return ret;
}
/******************************************************************************
**函数名称: invtd_search_word_parse
**功 能: 解析搜索请求
**输入参数:
** ctx: 全局对象
** buff: 搜索请求(报头+报体)
** len: 数据长度
**输出参数:
** req: 搜索请求
**返 回: 0:成功 !0:失败
**实现描述:
**注意事项:
**作 者: # Qifeng.zou # 2015.12.27 03:39:00 #
******************************************************************************/
static int invtd_search_word_parse(invtd_cntx_t *ctx,
char *buf, size_t len, mesg_search_word_req_t *req)
{
xml_opt_t opt;
xml_tree_t *xml;
xml_node_t *node;
agent_header_t *head = (agent_header_t *)buf;
const char *xml_str = (const char *)(head + 1);
memset(&opt, 0, sizeof(opt));
/* > 字节序转换 */
head->type = ntohl(head->type);
head->flag = ntohl(head->flag);
head->length = ntohl(head->length);
head->mark = ntohl(head->mark);
head->serial = ntoh64(head->serial);
req->serial = head->serial;
log_trace(ctx->log, "serial:%lu type:%u flag:%u mark:0X%x len:%d body:%s",
head->serial, head->type, head->flag, head->mark, head->length, xml_str);
/* > 构建XML树 */
opt.pool = NULL;
opt.alloc = mem_alloc;
opt.dealloc = mem_dealloc;
xml = xml_screat(xml_str, head->length, &opt);
if (NULL == xml) {
log_error(ctx->log, "Parse xml failed!");
return -1;
}
/* > 解析XML树 */
node = xml_query(xml, ".SEARCH.WORDS");
if (NULL == node) {
log_error(ctx->log, "Get search words failed!");
}
else {
snprintf(req->words, sizeof(req->words), "%s", node->value.str);
}
xml_destroy(xml);
log_trace(ctx->log, "words:%s", req->words);
return 0;
}
/******************************************************************************
**函数名称: xml_creat_empty
**功 能: 创建空XML树
**输入参数:
** opt: 选项信息
**输出参数: NONE
**返 回: XML树
**实现描述:
**注意事项:
**作 者: # Qifeng.zou # 2013.06.12 #
******************************************************************************/
xml_tree_t *xml_creat_empty(xml_opt_t *opt)
{
xml_tree_t *xml;
/* 1. 新建对象 */
xml = (xml_tree_t*)opt->alloc(opt->pool, sizeof(xml_tree_t));
if (NULL == xml) {
log_error(opt->log, "Calloc failed!");
return NULL;
}
xml->log = opt->log;
xml->pool = opt->pool;
xml->alloc = opt->alloc;
xml->dealloc = opt->dealloc;
/* 2. 添加根节点 */
xml->root = xml_node_creat(xml, XML_NODE_ROOT);
if (NULL == xml->root) {
log_error(xml->log, "Create node failed!");
xml_destroy(xml);
return NULL;
}
/* 3. 设置根节点名 */
xml->root->name.str = (char *)xml->alloc(xml->pool, XML_ROOT_NAME_SIZE);
if (NULL == xml->root->name.str) {
log_error(xml->log, "Calloc failed!");
xml_destroy(xml);
return NULL;
}
xml->root->name.len = snprintf(xml->root->name.str, XML_ROOT_NAME_SIZE, "%s", XML_ROOT_NAME);
return xml;
}
/******************************************************************************
**函数名称: invtd_search_no_data_hdl
**功 能: 无数据的应答处理
**输入参数:
** xml: 应答结果树
**输出参数: NONE
**返 回: 0:Succ !0:Fail
**实现描述:
**注意事项:
**作 者: # Qifeng.zou # 2016.05.01 01:49:01 #
******************************************************************************/
static int invtd_search_no_data_hdl(xml_tree_t *xml)
{
xml_node_t *root, *item;
char freq[SRCH_SEG_FREQ_LEN];
xml_add_attr(xml, xml->root, "CODE", SRCH_CODE_NO_DATA); /* 无数据 */
snprintf(freq, sizeof(freq), "%d", 0);
root = xml->root->child;
item = xml_add_child(xml, root, "ITEM", NULL);
if (NULL == item) {
xml_destroy(xml);
return -1;
}
xml_add_attr(xml, item, "URL", "Sorry, Didn't search anything!");
xml_add_attr(xml, item, "FREQ", freq);
return 0;
}
fmll_pca * fmll_pca_load(const char * fname_prefix)
{
int dim, num;
char node_name[4096];
unsigned u, v;
double ** w;
fmll_pca * pca = NULL;
mxml_node_t * sub_sub_node, * sub_node, * node, * content_node, * main_node = NULL;
fmll_try;
fmll_throw_if(xml_load(fname_prefix, TYPE_PCA, & main_node, & content_node));
fmll_throw_if(xml_get_int(content_node, "dim", & dim));
fmll_throw_if(xml_get_int(content_node, "num", & num));
fmll_throw_null(pca = fmll_pca_init(dim, num));
fmll_throw_null(node = mxmlFindElement(content_node, content_node, "W", NULL, NULL, MXML_DESCEND_FIRST));
for(u = 0, w = pca->w; u < num; u++)
{
sprintf(node_name, "w_%u", u);
fmll_throw_null(sub_node = mxmlFindElement(node, node, node_name, NULL, NULL, MXML_DESCEND_FIRST));
for(v = 0, sub_sub_node = mxmlFindElement(sub_node, sub_node, NULL, NULL, NULL, MXML_DESCEND_FIRST); v < dim; v++)
{
w[u][v] = sub_sub_node->child->value.real;
sub_sub_node = mxmlFindElement(sub_sub_node, sub_node, NULL, NULL, NULL, MXML_DESCEND);
}
}
fmll_catch;
fmll_pca_destroy(pca);
pca = NULL;
fmll_finally;
xml_destroy(main_node);
return pca;
}
fmll_svm_net * fmll_svm_net_load(const char * fname_prefix, double (** K)(const double *, const double *, const unsigned))
{
int num;
unsigned u;
char node_name[4096];
fmll_svm ** svm = NULL;
fmll_svm_net * svm_net = NULL;
mxml_node_t * sub_node, * node, * content_node, * main_node = NULL;
fmll_try;
fmll_throw_null(svm_net = fmll_alloc(sizeof(fmll_svm_net), 1, 1));
fmll_throw_if(xml_load(fname_prefix, TYPE_SVM_NET, & main_node, & content_node));
fmll_throw_if(xml_get_int(content_node, "num", & num));
svm_net->num = num;
fmll_throw_null(svm = svm_net->svm = (fmll_svm **) fmll_alloc(sizeof(fmll_svm *), 1, num));
fmll_throw_null(node = mxmlFindElement(content_node, content_node, "SVM", NULL, NULL, MXML_DESCEND_FIRST));
for(u = 0; u < num; u++)
svm[u] = NULL;
for(u = 0; u < num; u++)
{
sprintf(node_name, "svm_%u", u);
fmll_throw_null(sub_node = mxmlFindElement(node, node, node_name, NULL, NULL, MXML_DESCEND_FIRST));
fmll_throw_null(svm[u] = fmll_svm_load_main(sub_node, K[u]));
}
fmll_catch;
fmll_svm_net_destroy(svm_net);
svm_net = NULL;
fmll_finally;
xml_destroy(main_node);
return svm_net;
}
/******************************************************************************
**函数名称: invtd_search_send_and_free
**功 能: 从发送搜索结果并释放内存
**输入参数:
** ctx: 上下文
** xml: 搜索结果信息
** req: 搜索请求信息
**输出参数: NONE
**返 回: 0:成功 !0:失败
**实现描述: 从倒排表中查询结果，并将结果以XML树组织.
**注意事项:
**作 者: # Qifeng.zou # 2016.01.04 17:35:35 #
******************************************************************************/
static int invtd_search_send_and_free(invtd_cntx_t *ctx, xml_tree_t *xml,
mesg_header_t *head, mesg_search_req_t *req, int orig)
{
void *addr = NULL;
mesg_header_t *rsp; /* 应答 */
int body_len, total_len;
if (NULL == xml) { return 0; }
/* > 发送搜索应答 */
body_len = XML_PACK_LEN(xml);
total_len = MESG_TOTAL_LEN(body_len);
do {
/* 申请内存空间(注: 多1个字符是为字串结束符'\0'预留空间) */
addr = (char *)calloc(1, total_len+1);
if (NULL == addr) {
break;
}
/* 设置发送内容 */
rsp = (mesg_header_t *)addr;
MESG_HEAD_SET(rsp, MSG_SEARCH_RSP,
head->sid, head->nid, head->serial, body_len);
MESG_HEAD_HTON(rsp, rsp);
xml_spack(xml, rsp->body);
/* 放入发送队列 */
if (rtmq_proxy_async_send(ctx->frwder, MSG_SEARCH_RSP, addr, total_len)) {
log_error(ctx->log, "Send response failed! serial:%ld words:%s",
head->serial, req->words);
}
free(addr);
} while(0);
xml_destroy(xml);
return INVT_OK;
}
int fmll_pca_save(const fmll_pca * pca, const char * fname_prefix)
{
int ret = 0;
const unsigned dim = pca->dim, num = pca->num;
const double ** w = (const double **) pca->w;
unsigned u, v;
char node_name[4096];
mxml_node_t * sub_node, * node, * main_node = NULL, * content_node;
fmll_try;
fmll_throw_if(xml_create(TYPE_PCA, & main_node, & content_node));
fmll_throw_if(xml_set_int(content_node, "dim", dim));
fmll_throw_if(xml_set_int(content_node, "num", num));
fmll_throw_null(node = mxmlNewElement(content_node, "W"));
for(u = 0; u < num; u++)
{
sprintf(node_name, "w_%u", u);
fmll_throw_null(sub_node = mxmlNewElement(node, node_name));
for(v = 0; v < dim; v++)
{
sprintf(node_name, "%u", v);
fmll_throw_if(xml_set_double(sub_node, node_name, w[u][v]));
}
}
fmll_throw_if(xml_save(fname_prefix, main_node));
fmll_catch;
ret = -1;
fmll_finally;
xml_destroy(main_node);
return ret;
}
/******************************************************************************
**函数名称: xml_screat
**功 能: 将XML字串转为XML树
**输入参数:
** str: XML字串
** len: 字串长度限制(-1:表示遇到\0结束)
** opt: 选项信息
**输出参数:
**返 回: XML树
**实现描述:
** 1. 初始化栈
** 2. 初始化xml树
** 3. 在内存中将文件解析为XML树
**注意事项:
**作 者: # Qifeng.zou # 2013.02.05 #
******************************************************************************/
xml_tree_t *xml_screat(const char *str, size_t len, xml_opt_t *opt)
{
Stack_t stack;
xml_tree_t *xml;
if ((NULL == str) || ('\0' == str[0])) {
return xml_creat_empty(opt);
}
do {
/* 1. 初始化栈 */
if (stack_init(&stack, XML_MAX_DEPTH)) {
log_error(opt->log, "Init xml stack failed!");
break;
}
/* 2. 初始化XML树 */
xml = xml_init(opt);
if (NULL == xml) {
log_error(opt->log, "Init xml tree failed!");
break;
}
/* 3. 解析XML文件缓存 */
if (xml_parse(xml, &stack, str, len)) {
log_error(xml->log, "Parse xml failed!");
xml_destroy(xml);
break;
}
stack_destroy(&stack);
return xml;
} while(0);
/* 4. 释放内存空间 */
stack_destroy(&stack);
return NULL;
}
/******************************************************************************
**函数名称: intvd_search_send_and_free
**功 能: 从发送搜索结果并释放内存
**输入参数:
** ctx: 上下文
** xml: 搜索结果信息
** req: 搜索请求信息
**输出参数: NONE
**返 回: 0:成功 !0:失败
**实现描述: 从倒排表中查询结果，并将结果以XML树组织.
**注意事项:
**作 者: # Qifeng.zou # 2016.01.04 17:35:35 #
******************************************************************************/
static int intvd_search_send_and_free(invtd_cntx_t *ctx, xml_tree_t *xml,
mesg_header_t *head, mesg_search_word_req_t *req, int orig)
{
void *addr = NULL;
mesg_header_t *rsp; /* 应答 */
int body_len, total_len;
if (NULL == xml) { return 0; }
/* > 发送搜索应答 */
body_len = xml_pack_len(xml);
total_len = mesg_total_len(body_len);
do {
addr = (char *)calloc(1, total_len);
if (NULL == addr) {
break;
}
rsp = (mesg_header_t *)addr;
mesg_head_set(rsp, MSG_SEARCH_WORD_RSP, head->serial, sizeof(mesg_search_word_req_t));
mesg_head_hton(rsp, rsp);
xml_spack(xml, rsp->body);
if (rtsd_cli_send(ctx->frwder, MSG_SEARCH_WORD_RSP, addr, total_len)) {
log_error(ctx->log, "Send response failed! serial:%ld words:%s",
head->serial, req->words);
}
free(addr);
} while(0);
xml_destroy(xml);
return INVT_OK;
}
/******************************************************************************
**函数名称: intvd_search_send_and_free
**功 能: 从发送搜索结果并释放内存
**输入参数:
** ctx: 上下文
** xml: 搜索结果信息
** req: 搜索请求信息
**输出参数: NONE
**返 回: 0:成功 !0:失败
**实现描述: 从倒排表中查询结果，并将结果以XML树组织.
**注意事项:
**作 者: # Qifeng.zou # 2016.01.04 17:35:35 #
******************************************************************************/
static int intvd_search_send_and_free(invtd_cntx_t *ctx,
xml_tree_t *xml, mesg_search_word_req_t *req, int dev_orig)
{
void *addr = NULL;
mesg_data_t *rsp; /* 应答 */
int body_len, total_len;
if (NULL == xml) { return 0; }
/* > 发送搜索应答 */
body_len = xml_pack_len(xml);
total_len = mesg_data_total(body_len);
do
{
addr = (char *)calloc(1, total_len);
if (NULL == addr) {
break;
}
rsp = (mesg_data_t *)addr;
xml_spack(xml, rsp->body);
rsp->serial = hton64(req->serial);
if (rtrd_send(ctx->rtrd, MSG_SEARCH_WORD_RSP, dev_orig, addr, total_len)) {
log_error(ctx->log, "Send response failed! serial:%ld words:%s",
req->serial, req->words);
}
free(addr);
} while(0);
xml_destroy(xml);
return INVT_OK;
}
/*
===============================
===============================
*/
void destroy_soar_agent (agent * delete_agent)
{
//print(delete_agent, "\nDestroying agent %s.\n", delete_agent->name); /* AGR 532 */
/////////////////////////////////////////////////////////
// Soar Modules - could potentially rely on hash tables
/////////////////////////////////////////////////////////
// cleanup exploration
for ( int i=0; i<EXPLORATION_PARAMS; i++ )
delete delete_agent->exploration_params[ i ];
// cleanup Soar-RL
delete_agent->rl_params->apoptosis->set_value( rl_param_container::apoptosis_none );
delete delete_agent->rl_prods;
delete delete_agent->rl_params;
delete delete_agent->rl_stats;
delete_agent->rl_params = NULL; // apoptosis needs to know this for excise_all_productions below
// cleanup select
select_init( delete_agent );
delete delete_agent->select;
// cleanup predict
delete delete_agent->prediction;
// cleanup wma
delete_agent->wma_params->activation->set_value( off );
delete delete_agent->wma_forget_pq;
delete delete_agent->wma_touched_elements;
delete delete_agent->wma_touched_sets;
delete delete_agent->wma_params;
delete delete_agent->wma_stats;
delete delete_agent->wma_timers;
// cleanup epmem
epmem_close( delete_agent );
delete delete_agent->epmem_params;
delete delete_agent->epmem_stats;
delete delete_agent->epmem_timers;
delete delete_agent->epmem_node_removals;
delete delete_agent->epmem_node_mins;
delete delete_agent->epmem_node_maxes;
delete delete_agent->epmem_edge_removals;
delete delete_agent->epmem_edge_mins;
delete delete_agent->epmem_edge_maxes;
delete delete_agent->epmem_id_repository;
delete delete_agent->epmem_id_replacement;
delete delete_agent->epmem_id_ref_counts;
delete delete_agent->epmem_id_removes;
delete delete_agent->epmem_wme_adds;
delete delete_agent->epmem_promotions;
delete delete_agent->epmem_db;
// cleanup smem
smem_close( delete_agent );
delete delete_agent->smem_changed_ids;
delete delete_agent->smem_params;
delete delete_agent->smem_stats;
delete delete_agent->smem_timers;
delete delete_agent->smem_db;
// cleanup statistics db
stats_close( delete_agent );
delete delete_agent->stats_db;
delete_agent->stats_db = 0;
delete delete_agent->debug_params;
/////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////
remove_built_in_rhs_functions(delete_agent);
getSoarInstance()->Delete_Agent(delete_agent->name);
/* Free structures stored in agent structure */
free(delete_agent->name);
free(delete_agent->top_dir_stack->directory);
free(delete_agent->top_dir_stack);
/* Freeing the agent's multi attributes structure */
multi_attribute* lastmattr = 0;
for ( multi_attribute* curmattr = delete_agent->multi_attributes;
curmattr != 0;
curmattr = curmattr->next ) {
symbol_remove_ref(delete_agent, curmattr->symbol);
free_memory(delete_agent, lastmattr, MISCELLANEOUS_MEM_USAGE);
lastmattr = curmattr;
}
free_memory(delete_agent, lastmattr, MISCELLANEOUS_MEM_USAGE);
/* Freeing all the productions owned by this agent */
excise_all_productions(delete_agent, false);
/* Releasing all the predefined symbols */
release_predefined_symbols(delete_agent);
/* Releasing rete stuff RPM 11/06 */
free_with_pool(&delete_agent->rete_node_pool, delete_agent->dummy_top_node);
free_with_pool(&delete_agent->token_pool, delete_agent->dummy_top_token);
/* Cleaning up the various callbacks
TODO: Not clear why callbacks need to take the agent pointer essentially twice.
*/
soar_remove_all_monitorable_callbacks(delete_agent);
/* RPM 9/06 begin */
free_memory(delete_agent, delete_agent->left_ht, HASH_TABLE_MEM_USAGE);
free_memory(delete_agent, delete_agent->right_ht, HASH_TABLE_MEM_USAGE);
free_memory(delete_agent, delete_agent->rhs_variable_bindings, MISCELLANEOUS_MEM_USAGE);
/* Releasing memory allocated in inital call to start_lex_from_file from init_lexer */
free_memory_block_for_string(delete_agent, delete_agent->current_file->filename);
free_memory (delete_agent, delete_agent->current_file, MISCELLANEOUS_MEM_USAGE);
/* Releasing trace formats (needs to happen before tracing hashtables are released) */
remove_trace_format (delete_agent, FALSE, FOR_ANYTHING_TF, NIL);
remove_trace_format (delete_agent, FALSE, FOR_STATES_TF, NIL);
Symbol *evaluate_object_sym = find_sym_constant (delete_agent, "evaluate-object");
remove_trace_format (delete_agent, FALSE, FOR_OPERATORS_TF, evaluate_object_sym);
remove_trace_format (delete_agent, TRUE, FOR_STATES_TF, NIL);
remove_trace_format (delete_agent, TRUE, FOR_OPERATORS_TF, NIL);
/* Releasing hashtables allocated in init_tracing */
for (int i=0; i<3; i++) {
free_hash_table(delete_agent, delete_agent->object_tr_ht[i]);
free_hash_table(delete_agent, delete_agent->stack_tr_ht[i]);
}
/* Releasing memory allocated in init_rete */
for (int i=0; i<16; i++) {
free_hash_table(delete_agent, delete_agent->alpha_hash_tables[i]);
}
/* Releasing other hashtables */
free_hash_table(delete_agent, delete_agent->variable_hash_table);
free_hash_table(delete_agent, delete_agent->identifier_hash_table);
free_hash_table(delete_agent, delete_agent->sym_constant_hash_table);
free_hash_table(delete_agent, delete_agent->int_constant_hash_table);
free_hash_table(delete_agent, delete_agent->float_constant_hash_table);
/* Releasing memory pools */
memory_pool* cur_pool = delete_agent->memory_pools_in_use;
memory_pool* next_pool;
while(cur_pool != NIL) {
next_pool = cur_pool->next;
free_memory_pool(delete_agent, cur_pool);
cur_pool = next_pool;
}
/* RPM 9/06 end */
// dynamic memory pools (cleared in the last step)
for ( std::map< size_t, memory_pool* >::iterator it=delete_agent->dyn_memory_pools->begin(); it!=delete_agent->dyn_memory_pools->end(); it++ )
{
delete it->second;
}
delete delete_agent->dyn_memory_pools;
delete delete_agent->dyn_counters;
// JRV: Frees data used by XML generation
xml_destroy( delete_agent );
/* Free soar agent structure */
delete delete_agent;
}
/******************************************************************************
**函数名称: invtd_search_word_query
**功 能: 从倒排表中搜索关键字
**输入参数:
** ctx: 上下文
** req: 搜索请求信息
**输出参数: NONE
**返 回: 搜索结果(以XML树组织)
**实现描述: 从倒排表中查询结果，并将结果以XML树组织.
**注意事项: 完成发送后, 必须记得释放XML树的所有内存
**作 者: # Qifeng.zou # 2016.01.04 17:35:35 #
******************************************************************************/
static xml_tree_t *invtd_search_word_query(invtd_cntx_t *ctx, mesg_search_word_req_t *req)
{
int idx;
xml_opt_t opt;
xml_tree_t *xml;
xml_node_t *root, *item;
list_node_t *node;
invt_word_doc_t *doc;
invt_dic_word_t *word;
char freq[SRCH_SEG_FREQ_LEN];
memset(&opt, 0, sizeof(opt));
/* > 构建XML树 */
opt.pool = NULL;
opt.alloc = mem_alloc;
opt.dealloc = mem_dealloc;
opt.log = ctx->log;
xml = xml_creat_empty(&opt);
if (NULL == xml) {
log_error(ctx->log, "Create xml failed!");
return NULL;
}
root = xml_set_root(xml, "SEARCH-RSP");
if (NULL == root) {
log_error(ctx->log, "Set xml root failed!");
goto GOTO_SEARCH_ERR;
}
/* > 搜索倒排表 */
pthread_rwlock_rdlock(&ctx->invtab_lock);
word = invtab_query(ctx->invtab, req->words);
if (NULL == word
|| NULL == word->doc_list)
{
pthread_rwlock_unlock(&ctx->invtab_lock);
log_warn(ctx->log, "Didn't search anything! words:%s", req->words);
goto GOTO_SEARCH_NO_DATA;
}
/* > 构建搜索结果 */
idx = 0;
node = word->doc_list->head;
for (; NULL!=node; node=node->next, ++idx) {
doc = (invt_word_doc_t *)node->data;
snprintf(freq, sizeof(freq), "%d", doc->freq);
item = xml_add_child(xml, root, "ITEM", NULL);
if (NULL == item) {
goto GOTO_SEARCH_ERR;
}
xml_add_attr(xml, item, "URL", doc->url.str);
xml_add_attr(xml, item, "FREQ", freq);
}
pthread_rwlock_unlock(&ctx->invtab_lock);
xml_add_attr(xml, root, "CODE", SRCH_CODE_OK); /* 设置返回码 */
return xml;
GOTO_SEARCH_ERR: /* 异常错误处理 */
xml_destroy(xml);
return NULL;
GOTO_SEARCH_NO_DATA: /* 搜索结果为空的处理 */
xml_add_attr(xml, root, "CODE", SRCH_CODE_NO_DATA); /* 无数据 */
snprintf(freq, sizeof(freq), "%d", 0);
item = xml_add_child(xml, root, "ITEM", NULL);
if (NULL == item) {
goto GOTO_SEARCH_ERR;
}
xml_add_attr(xml, item, "URL", "Sorry, Didn't search anything!");
xml_add_attr(xml, item, "FREQ", freq);
return xml;
}
/******************************************************************************
**函数名称: flt_worker_get_webpage_info
**功 能: 获取网页信息
**输入参数:
** path: 网页信息文件
**输出参数:
** info: 网页信息
**返 回: 0:成功 !0:失败
**实现描述:
**注意事项:
**作 者: # Qifeng.zou # 2015.03.11 #
******************************************************************************/
static int flt_worker_get_webpage_info(
const char *path, flt_webpage_info_t *info, log_cycle_t *log)
{
xml_opt_t opt;
xml_tree_t *xml;
mem_pool_t *pool;
xml_node_t *node, *fix;
memset(&opt, 0, sizeof(opt));
/* 1. 新建内存池 */
pool = mem_pool_creat(4 * KB);
if (NULL == pool) {
log_error(log, "errmsg:[%d] %s!", errno, strerror(errno));
return FLT_ERR;
}
/* 2. 新建XML树 */
opt.log = log;
opt.pool = pool;
opt.alloc = (mem_alloc_cb_t)mem_pool_alloc;
opt.dealloc = (mem_dealloc_cb_t)mem_pool_dealloc;
xml = xml_creat(path, &opt);
if (NULL == xml) {
mem_pool_destroy(pool);
log_error(log, "Create XML failed! path:%s", path);
return FLT_ERR;
}
/* 2. 提取网页信息 */
do {
fix = xml_query(xml, ".WPI");
if (NULL == fix) {
log_error(log, "Get WPI mark failed!");
break;
}
/* 获取URI字段 */
node = xml_search(xml, fix, "URI");
if (NULL == node) {
log_error(log, "Get URI mark failed!");
break;
}
snprintf(info->uri, sizeof(info->uri), "%s", node->value.str);
/* 获取DEPTH字段 */
node = xml_search(xml, fix, "URI.DEPTH");
if (NULL == node) {
log_error(log, "Get DEPTH mark failed!");
break;
}
info->depth = atoi(node->value.str);
/* 获取IP字段 */
node = xml_search(xml, fix, "URI.IP");
if (NULL == node) {
log_error(log, "Get IP mark failed!");
break;
}
snprintf(info->ip, sizeof(info->ip), "%s", node->value.str);
/* 获取PORT字段 */
node = xml_search(xml, fix, "URI.PORT");
if (NULL == node) {
log_error(log, "Get PORT mark failed!");
break;
}
info->port = atoi(node->value.str);
/* 获取HTML字段 */
node = xml_search(xml, fix, "HTML");
if (NULL == node) {
log_error(log, "Get HTML mark failed!");
break;
}
snprintf(info->html, sizeof(info->html), "%s", node->value.str);
/* 获取HTML.SIZE字段 */
node = xml_search(xml, fix, "HTML.SIZE");
if (NULL == node) {
log_error(log, "Get HTML.SIZE mark failed!");
break;
}
info->size = atoi(node->value.str);
if (info->size <= 0) {
log_info(log, "Html size is zero!");
break;
}
snprintf(info->fname, sizeof(info->fname), "%s", path);
xml_destroy(xml);
mem_pool_destroy(pool);
return FLT_OK;
} while(0);
/* 3. 释放XML树 */
xml_destroy(xml);
mem_pool_destroy(pool);
return FLT_ERR;
}
/******************************************************************************
**函数名称: mon_srch_recv_rsp
**功 能: 接收搜索应答信息
**输入参数:
** fd: 文件描述符
**输出参数: NONE
**返 回: 0:成功 !0:失败
**实现描述:
**注意事项:
**作 者: # Qifeng.zou # 2015.06.05 17:01:04 #
******************************************************************************/
static int mon_srch_recv_rsp(mon_cntx_t *ctx, mon_srch_conn_t *conn)
{
int idx;
ssize_t n;
char addr[8192];
serial_t serial;
xml_opt_t opt;
xml_tree_t *xml;
xml_node_t *node, *attr;
struct timeval ctm;
int sec, msec, usec;
agent_header_t head;
mesg_data_t *rsp;
memset(addr, 0, sizeof(addr));
/* > 接收应答数据 */
n = read(conn->fd, (void *)&head, sizeof(head));
gettimeofday(&ctm, NULL);
if (n <= 0) {
fprintf(stderr, " errmsg:[%d] %s!\n", errno, strerror(errno));
return -1;
}
else if (0 == conn->wrtm.tv_sec) {
fprintf(stderr, " Didn't send search request but received response!\n");
}
/* > 字节序转换 */
head.type = ntohl(head.type);
head.flag = ntohl(head.flag);
head.length = ntohl(head.length);
head.mark = ntohl(head.mark);
head.serial = ntoh64(head.serial);
n = read(conn->fd, addr, head.length);
/* > 显示查询结果 */
fprintf(stderr, " ============================================\n");
rsp = (mesg_data_t *)addr;
rsp->serial = ntoh64(rsp->serial);
serial.serial = head.serial;
fprintf(stderr, " >Serial: %lu [nid(%u) sid(%u) seq(%u)]\n",
head.serial, serial.nid, serial.sid, serial.seq);
fprintf(stderr, " >Serial: %lu:%lu\n",
rsp->serial, head.serial);
memset(&opt, 0, sizeof(opt));
opt.pool = NULL;
opt.alloc = mem_alloc;
opt.dealloc = mem_dealloc;
xml = xml_screat(rsp->body, head.length, &opt);
if (NULL == xml) {
fprintf(stderr, " Format isn't right! body:%s\n", rsp->body);
return -1;
}
node = xml_query(xml, ".SEARCH-RSP.ITEM");
for (idx=1; NULL != node; node = xml_brother(node), ++idx) {
attr = xml_search(xml, node, "URL");
fprintf(stderr, " [%02d] URL:%s", idx, attr->value.str);
attr = xml_search(xml, node, "FREQ");
fprintf(stderr, " FREQ:%s\n", attr->value.str);
}
xml_destroy(xml);
/* > 打印统计信息 */
sec = ctm.tv_sec - conn->wrtm.tv_sec;
msec = (ctm.tv_usec - conn->wrtm.tv_usec)/1000;
if (msec < 0) {
sec -= 1;
msec += 1000;
}
usec = (ctm.tv_usec - conn->wrtm.tv_usec)%1000;
if (usec < 0) {
usec += 1000;
msec -= 1;
if (msec < 0) {
sec -= 1;
msec += 1000;
}
}
if (msec < 0) {
msec += 1000;
sec -= 1;
}
fprintf(stderr, " >Spend: %d(s).%03d(ms).%03d(us)\n", sec, msec, usec);
fprintf(stderr, " ============================================\n");
return 0;
}
