int main(int argc, char* argv[])
{
char* cmd = NULL;
MEM_POOL* mem_pool = mem_pool_init(MB_SIZE);
int socket_fd;
signal(SIGPIPE, SIG_IGN);
signal(SIGHUP, SIG_IGN);
signal(SIGINT, sign_handler);
signal(SIGTERM, sign_handler);
if((socket_fd = parse_cmd_line(argc, argv, mem_pool)) <= 0)
{
mem_pool_destroy(mem_pool);
return 0;
}
while (global_start) {
if ((cmd = readline("MILE>")) == NULL) {
break;
}
parse_cmd(cmd,socket_fd,mem_pool);
mem_pool_reset(mem_pool);
if(cmd != NULL)
{
free(cmd);
cmd = NULL;
}
}
mem_pool_destroy(mem_pool);
return 0;
}
/******************************************************************************
**函数名称: mon_conf_load
**功 能: 加载配置信息
**输入参数:
** path: 配置路径
**输出参数: NONE
**返 回: 0:成功 !0:失败
**实现描述:
**注意事项:
**作 者: # Qifeng.zou # 2015.03.15 #
******************************************************************************/
mon_conf_t *mon_conf_load(const char *path)
{
xml_opt_t opt;
xml_tree_t *xml;
mon_conf_t *conf;
mem_pool_t *pool;
/* > 创建配置对象 */
conf = (mon_conf_t *)calloc(1, sizeof(mon_conf_t));
if (NULL == conf) {
return NULL;
}
/* > 构建XML树 */
pool = mem_pool_creat(4 * KB);
if (NULL == pool) {
free(conf);
return NULL;
}
do
{
/* > 加载XML树 */
memset(&opt, 0, sizeof(opt));
opt.pool = pool;
opt.alloc = (mem_alloc_cb_t)mem_pool_alloc;
opt.dealloc = (mem_dealloc_cb_t)mem_pool_dealloc;
xml = xml_creat(path, &opt);
if (NULL == xml) {
break;
}
/* > 提取配置信息 */
if (mon_conf_load_menu(xml, conf)) {
break;
}
MON_LOAD_CONF(xml, ".MONITOR.CRAWLER", &conf->crwl);
MON_LOAD_CONF(xml, ".MONITOR.FILTER", &conf->filter);
MON_LOAD_CONF(xml, ".MONITOR.SEARCH", &conf->search);
/* > 释放XML树 */
xml_destroy(xml);
mem_pool_destroy(pool);
return conf;
} while(0);
/* > 异常处理 */
free(conf);
xml_destroy(xml);
mem_pool_destroy(pool);
return NULL;
}
int32_t main(int32_t argc, char **argv)
{
struct mem_pool *test_mem_pool;
struct mem_pool *test_mem_pool_2;
struct mem_pool *test_mem_pool_3;
gf_mem_init_mempool_list();
gf_mem_acct_enable_set ();
signals_setup();
mem_acct_init(gf_common_mt_end+1);
test_mem_t ** mem_array = CALLOC(size,sizeof(test_mem_t * ));
test_mem_2_t ** mem_array2 = CALLOC(size,sizeof(test_mem_2_t * ));
test_mem_3_t ** mem_array3 = CALLOC(size,sizeof(test_mem_3_t * ));
int j;
int i;
CPU_TIME_START;
test_mem_pool = mem_pool_new (test_mem_t, size);
test_mem_pool_2 = mem_pool_new (test_mem_2_t, size);
test_mem_pool_3 = mem_pool_new (test_mem_3_t, size);
CPU_TIME_END_PRINT("mem pool");
CPU_TIME_START;
for(j=0; j<10000; j++){
if (!test_mem_pool)
{
DBG_PRINT("create mem pool error");
return -1;
}
for(i=0; i<size; i++)
mem_array[i] = (test_mem_t *)mem_get(test_mem_pool);
mem_array2[i] = (test_mem_2_t *)mem_get(test_mem_pool_2);
mem_array3[i] = (test_mem_3_t *)mem_get(test_mem_pool_3);
for(i=0; i<size; i++)
mem_put(mem_array[i]) ;
mem_put(mem_array2[i]) ;
mem_put(mem_array3[i]) ;
}
CPU_TIME_END_PRINT("mem pool");
mem_pool_destroy(test_mem_pool);
mem_pool_destroy(test_mem_pool_2);
mem_pool_destroy(test_mem_pool_3);
CPU_TIME_START;
for(j=0; j<100; j++){
for(i=0; i<size; i++)
mem_array[i] = (test_mem_t *)MALLOC(sizeof(test_mem_t));
mem_array2[i] = (test_mem_2_t *)MALLOC(sizeof(test_mem_2_t));
mem_array3[i] = (test_mem_3_t *)MALLOC(sizeof(test_mem_3_t));
for(i=0; i<size; i++)
free (mem_array[i]);
free (mem_array2[i]);
free (mem_array3[i]);
}
CPU_TIME_END_PRINT("not mem pool");
//getchar();
return 0;
}
int DocEngine::SpecialSql(struct mile_message_header* msg_head, struct data_buffer* rbuf, struct data_buffer* sbuf){
MEM_POOL_PTR mem_pool;
void* packet;
void* result;
int32_t result_code = MILE_RETURN_SUCCESS;
uint16_t msg_type = msg_head->message_type;
switch (msg_type) {
case MT_MD_EXE_GET_KVS:
mem_pool = mem_pool_init(MB_SIZE);
packet = parse_get_kvs_packet(mem_pool, rbuf);
result = GetKvs((struct get_kvs_packet*) packet, mem_pool);
if(NULL == result){
result_code = ERROR_QUERY_BY_ROWID;
}else{
gen_query_result_packet((ResultSet*) result, msg_head,
sbuf);
}
mem_pool_destroy(mem_pool);
break;
default:
log_error("unsupported command, packet type %d", msg_type);
result_code = ERROR_UNSUPPORTED_SQL_TYPE;
break;
}
return result_code;
}
static void tcp_stream_close(tcp_server_stream_t *stream)
{
tcp_server_t *server = NULL;
if (NULL == stream) {
return;
}
server = stream->server;
if (stream->pool) {
mem_pool_destroy(&stream->pool);
}
if (stream->sock_fd >= 0) {
close(stream->sock_fd);
}
/* 从busy链表中删除 */
delete_stream_from_busy_list(server, stream);
/* 放入到idle链表尾部 */
add_stream_into_idle_list(server, stream);
/* reset stream */
stream->sock_fd = -1;
stream->pool = NULL;
stream->close = 0;
stream->error = 0;
stream->data = NULL;
stream->server = NULL;
}
int32_t process_test_message(struct mile_message_header* msg_head,
struct data_buffer* rbuf, struct data_buffer* sbuf) {
int32_t result_code = MILE_RETURN_SUCCESS;
MEM_POOL_PTR mem_pool;
switch (msg_head->message_type) {
case MT_TEST_REQ_ECHO: // echo message for testing.
{
mem_pool = mem_pool_init(MB_SIZE);
const char *str = read_cstring(rbuf, mem_pool);
log_debug("get echo message [%s]", str);
write_int32(0, sbuf);
write_int8(msg_head->version_major, sbuf);
write_int8(msg_head->version_minor, sbuf);
write_int16(MT_TEST_RES_ECHO, sbuf);
write_int32(msg_head->message_id, sbuf);
write_int32(strlen(str), sbuf);
write_bytes((uint8_t *) str, strlen(str), sbuf);
fill_int32(sbuf->data_len, sbuf, 0);
result_code = MILE_RETURN_SUCCESS;
mem_pool_destroy(mem_pool);
}
break;
default:
result_code = -1;
break;
}
return result_code;
}
int32_t process_master_message(struct mile_message_header* msg_head,
struct data_buffer* rbuf, struct data_buffer* sbuf, void **bl_reader,
MEM_POOL_PTR session_mem) {
int32_t result_code = MILE_RETURN_SUCCESS;
MEM_POOL_PTR mem_pool;
void* packet;
struct slave_sync_res *binlog_res;
switch (msg_head->message_type) {
case MT_SM_GET_BINLOG:
mem_pool = mem_pool_init(MB_SIZE);
// STATIC_ASSERT( MB_SIZE > sizeof(struct slave_sync_res) + MAX_BINLOG_DATA_PER_PACKET, MAX_BINLOG_PACKAGE_SHOULD_LESS_THAN_MEM_POOL_SIZE);
packet = parse_slave_sync_req(mem_pool, rbuf);
binlog_res = execute_slave_sync(mem_pool,
(struct slave_sync_req *) packet, bl_reader, session_mem);
assert(NULL != binlog_res);
gen_slave_sync_res_packet(binlog_res, msg_head, sbuf);
result_code = 0;
mem_pool_destroy(mem_pool);
break;
default:
result_code = -1;
break;
}
return result_code;
}
int32_t process_client_message(struct mile_message_header* msg_head,
struct data_buffer* rbuf, struct data_buffer* sbuf) {
int32_t result_code = MILE_RETURN_SUCCESS;
MEM_POOL_PTR mem_pool;
void* packet;
struct slave_sync_res *binlog_res;
switch (msg_head->message_type) {
case MT_CD_EXE_GET_LOAD_THRESHOLD: {
log_debug("get get_load_threshold command");
double load = g_load_threshold;
gen_dc_get_load_threshold_packet(load, msg_head, sbuf);
}
break;
case MT_CD_EXE_SET_LOAD_THRESHOLD: {
log_debug("get set_load_threshold command");
double old_load = g_load_threshold;
mem_pool = mem_pool_init(KB_SIZE);
struct set_load_threshold_packet *p = parse_set_load_threshold_packet(
mem_pool, rbuf);
g_load_threshold = p->value;
log_info(
"change os overloading threshold from %g to %g", old_load, g_load_threshold);
gen_dc_set_load_threshold_packet(old_load, msg_head, sbuf);
mem_pool_destroy(mem_pool);
}
break;
default:
result_code = -1;
break;
}
return result_code;
}
void test_key_value()
{
MemPool *pool = NULL;
key_value_t *kv = NULL;
key_value_node_t *node = NULL;
char *keys[5], *vals[5];
pool = mem_pool_create(160); //160M
kv = key_value_init(pool, 20);
keys[0] = "abcde";
vals[0] = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
key_value_set(pool, kv, "abcde", strlen("abcde"),
vals[0], strlen(vals[0]), 0);
keys[1] = "qwerty";
vals[1] = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
key_value_set(pool, kv, keys[1], strlen(keys[1]),
vals[1], strlen(vals[1]), 0);
keys[2] = "asdf";
vals[2] = "cccccccccccc";
key_value_set(pool, kv, keys[2], strlen(keys[2]),
vals[2], strlen(vals[2]), 0);
node = key_value_get(kv, keys[2], strlen(keys[2]));
printf("get %s node data: %s\n", keys[2], (char *)node->data);
keys[3] = "asdf";
vals[3] = "dddddddddddddddddddddddddddddddddddddddddddddddddddd";
key_value_set(pool, kv, keys[3], strlen(keys[3]),
vals[3], strlen(vals[3]), 0);
node = key_value_get(kv, keys[3], strlen(keys[3]));
printf("get %s node data: %s\n", keys[3], (char *)node->data);
keys[4] = "lkjhgfdsa";
vals[4] = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee";
key_value_set(pool, kv, keys[4], strlen(keys[4]),
vals[4], strlen(vals[4]), 0);
node = key_value_get(kv, "aaaa", strlen("aaaa"));
if(node)
printf("get aaaa node data: %s\n", (char *)node->data);
else
printf("get aaaa node data: Not Found\n");
printf("Key Value: used memory: %u, mem pool used memory: %lu \n",
kv->used_memory, pool->used_size);
printf("Key-Value Pairs number: %u\n", kv->count);
key_value_destroy(pool, kv);
mem_pool_destroy(pool);
}
int main()
{
MemPool *pool = mem_pool_create(640); //640M
configure_handler_t *config_handler = load_configure_file(pool, "test.conf");
test_log(pool, config_handler);
configure_handler_destroy(pool, config_handler);
mem_pool_destroy(pool);
return 0;
}
void rs_releasemgr()
{
struct linked_list* unload_item = g_rs.unload_items;
while (unload_item != NULL) {
struct linked_list* nitem = unload_item->next;
FREE(unload_item->data);
unload_item = nitem;
}
hashtable_chained_destroy(&g_rs.dict);
mem_pool_destroy(&g_rs.dict_itempool);
mem_pool_destroy(&g_rs.freeslot_pool);
arr_destroy(&g_rs.ress);
mem_pool_destroy(&g_rs.load_data_pool);
if (g_rs.job_params.load_items != NULL)
FREE(g_rs.job_params.load_items);
if (g_rs.job_result.ptrs != NULL)
FREE(g_rs.job_result.ptrs);
log_print(LOG_TEXT, "res-mgr released.");
}
void
rbthash_table_destroy (rbthash_table_t *tbl)
{
if (!tbl)
return;
rbthash_table_destroy_buckets (tbl);
if (tbl->pool_alloced)
mem_pool_destroy (tbl->entrypool);
GF_FREE (tbl->buckets);
GF_FREE (tbl);
}
mem_pool_t *mem_pool_create()
{
void *addr = NULL;
mem_pool_t *mem_pool = NULL;
int page_size = 0;
mem_page_t *page = NULL;
/* 获取内存page的大小 */
page_size = getpagesize();
/* 分配物理内存page */
if(posix_memalign(&addr, page_size, page_size) != 0){
printf("posix_memalign in mem_pool_create fails.\n");
goto error;
}
/* 内存池信息和第一个page的信息都存储在第一个page的头部 */
mem_pool = (mem_pool_t *)addr;
mem_pool->page_size = page_size;
mem_pool->page_count = 1;
page = (mem_page_t *)((char *)addr + sizeof(mem_pool_t));
mem_pool->current = page;
mem_pool->head = page;
mem_pool->tail = page;
#ifdef MEM_POOL_LOCK
pthread_mutex_init(&mem_pool->lock, NULL);
#endif
/* 设置第一个page的信息 */
page->start = (char *)addr;
page->last = page->start +
sizeof(mem_pool_t) +
sizeof(mem_page_t);
page->end = (char *)addr + page_size;
page->next = NULL;
page->failed = 0;
return mem_pool;
error:
if(mem_pool){
mem_pool_destroy(&mem_pool);
}
return NULL;
}
void test_configure()
{
configure_handler_t *config_handler;
char *filename = "test.conf";
MemPool *pool = mem_pool_create(640); //640M
char section_search[] = "[filter:dlo]";
char key_search[] = "use";
config_handler = load_configure_file(pool, filename);
//print_configure(config_handler);
printf("%s %s: %s\n", section_search, key_search,
get_configure_item(config_handler,section_search ,key_search ));
mem_pool_destroy(pool);
}
void* MainStep::Execute(TableManager* table, void* input, int32_t &result_code,
int64_t timeout, MEM_POOL_PTR mem_pool) {
MileIterator* iter = (MileIterator*) input;
MileIterator* step_iter = substeps->CreateIterator();
uint32_t i = 0;
uint32_t cut_threshold=table->getCutThreshold();
ExecuteSubstep* substep;
MEM_POOL_PTR inner_mem_pool = mem_pool_init(MB_SIZE);
for (iter->First(); !iter->IsDone(); iter->Next()) {
for (step_iter->First(); !step_iter->IsDone(); step_iter->Next()) {
substep = (ExecuteSubstep*) step_iter->CurrentItem();
result_code = substep->Execute(table,
(MileHandler*) iter->CurrentItem(), result, inner_mem_pool);
if (result_code != 0) {
break;
}
}
mem_pool_reset(inner_mem_pool);
if (result_code < 0) {
break;
}
if((++i) % 100 == 0){
if(get_time_msec() > timeout){
result_code = ERROR_TIMEOUT;
break;
}
}
if( cut_threshold > 0 && i >= cut_threshold )
{
break;
}
}
mem_pool_destroy(inner_mem_pool);
if (result_code == WARN_EXCEED_QUERY_LIMIT || result_code >= 0) {
result_code = MILE_RETURN_SUCCESS;
}
return result;
}
xml_doc_t* xml_doc_create(){
xml_doc_t* doc;
mem_pool_t* pool;
pool = mem_pool_create();
if(!pool){
return 0;
}
doc = (xml_doc_t*)mem_pool_alloc(pool,sizeof(xml_doc_t));
if(!doc){
mem_pool_destroy(pool);
return 0;
}
doc->pool = pool;
doc->root = 0;
doc->_raw_item_head = xml_raw_item_create(doc->pool,item_type_none,0,0);
doc->_raw_item_cur = doc->_raw_item_head;
return doc;
}
TEST(DATA_BUFFER_TEST, READ_AND_WRITE){
MEM_POOL_PTR pMemPool = mem_pool_init(MB_SIZE);
struct data_buffer* buffer = init_data_buffer();
buffer->data_len = 256;
buffer->data = (uint8*) mem_pool_malloc(pMemPool, 256);
uint8 tmp8;
uint16 tmp16;
uint32 tmp32;
uint64 tmp64;
char* str = (char*) mem_pool_malloc(pMemPool, 100);
char* result = (char*) mem_pool_malloc(pMemPool, 100);
strcpy(str, "hello world!");
write_int8(122, buffer);
write_int16(1111, buffer);
write_int32(324, buffer);
write_int64(2321, buffer);
write_bytes((uint8*)str, strlen(str)+1, buffer);
tmp8 = read_int8(buffer);
ASSERT_EQ(tmp8, 122);
tmp16 = read_int16(buffer);
ASSERT_EQ(tmp16, 1111);
tmp32 = read_int32(buffer);
ASSERT_EQ(tmp32, 324);
tmp64 = read_int64(buffer);
ASSERT_EQ(tmp64, 2321);
read_bytes(buffer, (uint8*)result, strlen(str)+1);
ASSERT_EQ(0, strcmp(str, result));
destroy_data_buffer(buffer);
mem_pool_destroy(pMemPool);
}
static void tcp_client_stream_reset(tcp_client_stream_t *stream)
{
tcp_client_t *client = NULL;
if (NULL == stream) {
return;
}
client = stream->client;
if (stream->pool) {
mem_pool_destroy(&stream->pool);
}
if (stream->sock_fd >= 0) {
close(stream->sock_fd);
}
/* reset stream */
stream->sock_fd = -1;
stream->server_ip = NULL;
stream->port = 0;
stream->pool = NULL;
stream->close = 0;
stream->error = 0;
stream->chain = NULL;
stream->cur_chain = NULL;
stream->tcp_state = TCP_STATE_NONE;
stream->data = NULL;
stream->client = NULL;
stream->last = NULL;
stream->next = NULL;
stream->rcv_cb = NULL;
stream->snd_cb = NULL;
stream->conn_cb = NULL;
bzero(&stream->update_time, NULL);
bzero(&stream->dst_addr, NULL);
}
int main(int argc, char * argv[]) {
struct mem_pool * pool = mem_pool_new(4);
int r1,r2,r3,r4;
r1 = mem_pool_alloc(pool, 16);
printb(pool, r1);
r2 = mem_pool_alloc(pool, 32);
printb(pool, r2);
r3 = mem_pool_alloc(pool, 16);
printb(pool, r3);
mem_pool_free(pool, r1);
mem_pool_free(pool, r3);
r4 = mem_pool_alloc(pool, 32);
printb(pool, r4);
mem_pool_destroy(pool);
return 0;
}
TEST(BITMARK_TEST, HandleNoneZeroInput) {
int32 ret;
int32 value;
MEM_POOL* mem_pool = mem_pool_init(M_1M);
struct bitmark_config config;
strcpy(config.bitmark_name,"null");
strcpy(config.work_space,"/tmp/bitmark_test");
config.row_limit = 10;
system("rm -rf /tmp/bitmark_test");
init_profile(1000,mem_pool);
mkdirs(config.work_space);
struct bitmark_manager* bitmark = bitmark_init(&config, mem_pool);
//全部置零
bitmark_reset(bitmark);
//置位
//表示6行已插入该行值
ret = bitmark_set(bitmark,6);
ASSERT_EQ(ret,0);
//查询
value = bitmark_query(bitmark,6);
ASSERT_EQ(value,0);
value = bitmark_query(bitmark,8);
ASSERT_EQ(value,1);
//删除6行的值
ret = bitmark_clear(bitmark,6);
ASSERT_EQ(ret,0);
mem_pool_destroy(mem_pool);
}
int DocEngine::Command(struct mile_message_header* msg_head,
struct data_buffer* rbuf, struct data_buffer* sbuf) {
MEM_POOL_PTR mem_pool;
void* parsed_packet;
void* result;
int32_t result_code = 0;
uint16_t msg_type = msg_head->message_type;
switch (msg_type) {
case MT_MD_EXE_GET_KVS:
mem_pool = mem_pool_init(MB_SIZE);
parsed_packet = parse_get_kvs_packet(mem_pool, rbuf);
result = GetKvs((struct get_kvs_packet*) parsed_packet, mem_pool);
if(NULL == result){
result_code = ERROR_QUERY_BY_ROWID;
}else{
gen_query_result_packet((ResultSet*) result, msg_head,
sbuf);
}
mem_pool_destroy(mem_pool);
break;
case MT_CD_EXE_COMPRESS:
mem_pool = mem_pool_init(KB_SIZE);
parsed_packet = parse_compress_packet(mem_pool, rbuf);
log_debug(
"接收到compress命令:%s", ((struct compress_packet*)parsed_packet)->table_name);
db_read_lock();
result_code = db_compress(
((struct compress_packet*) parsed_packet)->table_name,
mem_pool);
db_read_unlock();
gen_dc_response_packet(result_code, msg_head, sbuf);
mem_pool_destroy(mem_pool);
break;
case MT_CD_EXE_INDEX:
mem_pool = mem_pool_init(KB_SIZE);
parsed_packet = parse_ensure_index_packet(mem_pool, rbuf);
log_debug(
"接收到ensure_index命令 %s %s %u %u", ((struct ensure_index_packet*)parsed_packet)->table_name, ((struct ensure_index_packet*)parsed_packet)->field_name, ((struct ensure_index_packet*)parsed_packet)->index_type, ((struct ensure_index_packet*)parsed_packet)->data_type);
db_read_lock();
result_code = db_ensure_index(
((struct ensure_index_packet*) parsed_packet)->table_name,
((struct ensure_index_packet*) parsed_packet)->field_name,
((struct ensure_index_packet*) parsed_packet)->index_type,
((struct ensure_index_packet*) parsed_packet)->data_type,
mem_pool);
db_read_unlock();
gen_dc_response_packet(result_code, msg_head, sbuf);
mem_pool_destroy(mem_pool);
break;
case MT_CD_EXE_UNINDEX:
mem_pool = mem_pool_init(KB_SIZE);
parsed_packet = parse_del_index_packet(mem_pool, rbuf);
log_debug(
"接收到del_index命令 %s %s %u", ((struct del_index_packet*)parsed_packet)->table_name, ((struct del_index_packet*)parsed_packet)->field_name, ((struct del_index_packet*)parsed_packet)->index_type);
db_read_lock();
result_code = db_del_index(
((struct del_index_packet*) parsed_packet)->table_name,
((struct del_index_packet*) parsed_packet)->field_name,
((struct del_index_packet*) parsed_packet)->index_type,
mem_pool);
db_read_unlock();
gen_dc_response_packet(result_code, msg_head, sbuf);
mem_pool_destroy(mem_pool);
break;
case MT_CD_EXE_LOAD:
mem_pool = mem_pool_init(KB_SIZE);
parsed_packet = parse_load_segment_packet(mem_pool, rbuf);
log_debug(
"接收到load_segment命令 %s %u %s", ((struct load_segment_packet*)parsed_packet)->table_name, ((struct load_segment_packet*)parsed_packet)->sid, ((struct load_segment_packet*)parsed_packet)->segment_dir);
db_read_lock();
result_code = db_load_segment(
((struct load_segment_packet*) parsed_packet)->table_name,
((struct load_segment_packet*) parsed_packet)->sid,
((struct load_segment_packet*) parsed_packet)->segment_dir,
mem_pool);
db_read_unlock();
gen_dc_response_packet(result_code, msg_head, sbuf);
mem_pool_destroy(mem_pool);
break;
case MT_CD_EXE_REPLACE:
mem_pool = mem_pool_init(KB_SIZE);
parsed_packet = parse_replace_segment_packet(mem_pool, rbuf);
log_debug(
"接收到replace segments命令 %s %s", ((struct replace_segment_packet*)parsed_packet)->table_name, ((struct replace_segment_packet*)parsed_packet)->segment_dir);
db_read_lock();
result_code = db_replace_all_segments(
((struct replace_segment_packet*) parsed_packet)->table_name,
((struct replace_segment_packet*) parsed_packet)->segment_dir,
mem_pool);
db_read_unlock();
gen_dc_response_packet(result_code, msg_head, sbuf);
mem_pool_destroy(mem_pool);
break;
case MT_CD_EXE_UNLOAD:
mem_pool = mem_pool_init(KB_SIZE);
parsed_packet = parse_unload_segment_packet(mem_pool, rbuf);
log_debug(
"接收到unload_segment命令 %s %u", ((struct unload_segment_packet*)parsed_packet)->table_name, ((struct unload_segment_packet*)parsed_packet)->sid);
db_read_lock();
result_code = db_unload_segment(
((struct unload_segment_packet*) parsed_packet)->table_name,
((struct unload_segment_packet*) parsed_packet)->sid, mem_pool);
db_read_unlock();
gen_dc_response_packet(result_code, msg_head, sbuf);
mem_pool_destroy(mem_pool);
break;
case MT_CD_EXE_CP:
log_debug( "get checkpoint command");
db_checkpoint();
gen_dc_response_packet(0, msg_head, sbuf);
break;
case MT_CD_STAT:
mem_pool = mem_pool_init(MB_SIZE);
parsed_packet = parse_doc_stat_packet(mem_pool, rbuf);
log_debug(
"接收到stat命令 %s %u", ((struct doc_stat_packet*)parsed_packet)->table_name, ((struct doc_stat_packet*)parsed_packet)->type);
if (((struct doc_stat_packet*) parsed_packet)->type == 1) {
uint16_t max_segment_num = 0;
struct segment_meta_data* result = NULL;
db_read_lock();
result = db_query_segment_stat(
((struct doc_stat_packet*) parsed_packet)->table_name,
&max_segment_num, mem_pool);
db_read_unlock();
gen_dc_segment_stat_packet(result, max_segment_num, msg_head, sbuf);
}
if (((struct doc_stat_packet*) parsed_packet)->type == 2) {
uint16_t index_field_count = 0;
struct index_field_meta* result = NULL;
db_read_lock();
result = db_query_index_stat(
((struct doc_stat_packet*) parsed_packet)->table_name,
&index_field_count, mem_pool);
db_read_unlock();
gen_dc_index_stat_packet(result, index_field_count, msg_head, sbuf);
}
mem_pool_destroy(mem_pool);
break;
default:
log_error("unsupported command, packet type %d", msg_type);
result_code = ERROR_UNSUPPORTED_SQL_TYPE;
break;
}
return result_code;
}
DocEngine::~DocEngine() {
db_release();
mem_pool_destroy(mem_);
mem_ = NULL;
}
TEST(HASHINDEX_TEST, HandleNoneZeroInput) {
MEM_POOL* mem_pool = mem_pool_init(MB_SIZE);
struct hash_index_config config;
system("rm -rf /tmp/hashindex_test");
char dir_path[] = "/tmp/hashindex_test";
mkdirs(dir_path);
config.row_limit = BUCKET_NUM;
strcpy(config.work_space,dir_path);
init_profile(1000,mem_pool);
int32_t ret;
//插入一个值
struct low_data_struct insert_data;
get_low_data(&insert_data,mem_pool);
struct hash_index_manager* hash_index = hash_index_init(&config,mem_pool);
ret = hash_index_insert(hash_index,&insert_data,0);
ASSERT_EQ(ret,0);
//查询
struct rowid_list* rowids;
uint32_t i;
struct rowid_list_node* p;
rowids = hash_index_query(hash_index,&insert_data,mem_pool);
for(i = 0, p = rowids->head; i < rowids->rowid_num; i++)
{
if(i != 0 && i%ROWID_ARRAY_SIZE == 0)
{
p = p->next;
}
ASSERT_EQ(p->rowid_array[i%ROWID_ARRAY_SIZE],0);
break;
}
//再插入一个值
hash_index_insert(hash_index,&insert_data,1);
rowids = hash_index_query(hash_index,&insert_data,mem_pool);
for(i = 0, p = rowids->head; i < rowids->rowid_num; i++)
{
if(i != 0 && i%ROWID_ARRAY_SIZE == 0)
{
p = p->next;
if(i==0)
ASSERT_EQ(p->rowid_array[i%ROWID_ARRAY_SIZE],1);
if(i==1)
{
ASSERT_EQ(p->rowid_array[i%ROWID_ARRAY_SIZE],0);
break;
}
}
}
//插入BUCKET_NUM个
for(i=2; i<BUCKET_NUM+1; i++)
{
sprintf((char*)insert_data.data,"ali%u",i);
ret = hash_index_insert(hash_index,&insert_data,i);
ASSERT_EQ(ret,0);
}
//再插一个报冲突了
sprintf((char*)insert_data.data,"ali%u",i);
ret = hash_index_insert(hash_index,&insert_data,i);
ASSERT_EQ(ret,ERROR_HASH_CONFLICT);
//插入一个空值，OK
insert_data.len = 0;
ret = hash_index_insert(hash_index,&insert_data,i);
ASSERT_EQ(ret,0);
rowids = hash_index_query(hash_index,&insert_data,mem_pool);
ASSERT_EQ(rowids->rowid_num,1);
ASSERT_EQ(rowids->head->rowid_array[0],5);
hash_index_release(hash_index);
mem_pool_destroy(mem_pool);
}
TEST(RECOVER_TEST_2, HandleNoneZeroInput) {
MEM_POOL* mem_pool = mem_pool_init(MB_SIZE);
struct hash_index_config config;
system("rm -rf /tmp/hashindex_test");
char dir_path[] = "/tmp/hashindex_test";
mkdirs(dir_path);
config.row_limit = ROW_LIMIT;
strcpy(config.work_space,dir_path);
init_profile(1000,mem_pool);
int32_t ret;
struct hash_index_manager* hash_index = hash_index_init(&config,mem_pool);
//插入18条数据，只恢复16条
uint32_t i;
struct low_data_struct* data = (struct low_data_struct*)mem_pool_malloc(mem_pool, sizeof(struct low_data_struct));
memset(data, 0, sizeof(struct low_data_struct));
for(i=0; i<18; i++)
{
if(i != 0 && i%2 == 1)
continue;
data->len = 5;
data->data = mem_pool_malloc(mem_pool,5);
data->type = HI_TYPE_STRING;
data->field_name = (char*)mem_pool_malloc(mem_pool,20);
memset(data->field_name,0,20);
strcpy(data->field_name,"HI_TYPE_STRING");
memset(data->data,0,5);
ret = hash_index_insert(hash_index,data,i);
ASSERT_EQ(0, ret);
ret = hash_index_insert(hash_index,data,i+1);
ASSERT_EQ(0, ret);
}
struct doc_row_unit* doc = NULL;
ASSERT_EQ(0, hash_index_recover(hash_index, 16));
for(i=0; i<ROW_LIMIT; i++)
{
doc = GET_DOC_ROW_STRUCT(hash_index->doclist, i);
}
//验证
for(i=0; i<16; i++)
{
doc = GET_DOC_ROW_STRUCT(hash_index->doclist, i);
ASSERT_EQ(i, doc->doc_id);
}
for(i=16; i<ROW_LIMIT; i++)
{
doc = GET_DOC_ROW_STRUCT(hash_index->doclist, i);
ASSERT_EQ(0, doc->doc_id);
ASSERT_EQ(0,doc->next);
}
struct hash_bucket* hbucket = NULL;
uint32_t bucket_no = 0;
for(i=0; i<ROW_LIMIT+1; i++)
{
hbucket = hash_index->mem_mmaped + i;
doc = NEXT_DOC_ROW_STRUCT(hash_index->doclist, hbucket->offset);
if(hbucket->hash_value == 0)
{
continue;
}
while(!(doc->next & 0x80000000))
{
ASSERT_GT(16, doc->doc_id);
doc = NEXT_DOC_ROW_STRUCT(hash_index->doclist, doc->next);
}
bucket_no = doc->next & 0x7fffffff;
ASSERT_EQ(i,bucket_no);
}
hash_index_release(hash_index);
mem_pool_destroy(mem_pool);
}
/******************************************************************************
**函数名称: flt_worker_get_webpage_info
**功 能: 获取网页信息
**输入参数:
** path: 网页信息文件
**输出参数:
** info: 网页信息
**返 回: 0:成功 !0:失败
**实现描述:
**注意事项:
**作 者: # Qifeng.zou # 2015.03.11 #
******************************************************************************/
static int flt_worker_get_webpage_info(
const char *path, flt_webpage_info_t *info, log_cycle_t *log)
{
xml_opt_t opt;
xml_tree_t *xml;
mem_pool_t *pool;
xml_node_t *node, *fix;
memset(&opt, 0, sizeof(opt));
/* 1. 新建内存池 */
pool = mem_pool_creat(4 * KB);
if (NULL == pool) {
log_error(log, "errmsg:[%d] %s!", errno, strerror(errno));
return FLT_ERR;
}
/* 2. 新建XML树 */
opt.log = log;
opt.pool = pool;
opt.alloc = (mem_alloc_cb_t)mem_pool_alloc;
opt.dealloc = (mem_dealloc_cb_t)mem_pool_dealloc;
xml = xml_creat(path, &opt);
if (NULL == xml) {
mem_pool_destroy(pool);
log_error(log, "Create XML failed! path:%s", path);
return FLT_ERR;
}
/* 2. 提取网页信息 */
do {
fix = xml_query(xml, ".WPI");
if (NULL == fix) {
log_error(log, "Get WPI mark failed!");
break;
}
/* 获取URI字段 */
node = xml_search(xml, fix, "URI");
if (NULL == node) {
log_error(log, "Get URI mark failed!");
break;
}
snprintf(info->uri, sizeof(info->uri), "%s", node->value.str);
/* 获取DEPTH字段 */
node = xml_search(xml, fix, "URI.DEPTH");
if (NULL == node) {
log_error(log, "Get DEPTH mark failed!");
break;
}
info->depth = atoi(node->value.str);
/* 获取IP字段 */
node = xml_search(xml, fix, "URI.IP");
if (NULL == node) {
log_error(log, "Get IP mark failed!");
break;
}
snprintf(info->ip, sizeof(info->ip), "%s", node->value.str);
/* 获取PORT字段 */
node = xml_search(xml, fix, "URI.PORT");
if (NULL == node) {
log_error(log, "Get PORT mark failed!");
break;
}
info->port = atoi(node->value.str);
/* 获取HTML字段 */
node = xml_search(xml, fix, "HTML");
if (NULL == node) {
log_error(log, "Get HTML mark failed!");
break;
}
snprintf(info->html, sizeof(info->html), "%s", node->value.str);
/* 获取HTML.SIZE字段 */
node = xml_search(xml, fix, "HTML.SIZE");
if (NULL == node) {
log_error(log, "Get HTML.SIZE mark failed!");
break;
}
info->size = atoi(node->value.str);
if (info->size <= 0) {
log_info(log, "Html size is zero!");
break;
}
snprintf(info->fname, sizeof(info->fname), "%s", path);
xml_destroy(xml);
mem_pool_destroy(pool);
return FLT_OK;
} while(0);
/* 3. 释放XML树 */
xml_destroy(xml);
mem_pool_destroy(pool);
return FLT_ERR;
}
int32_t process_mergeserver_message(struct mile_message_header* msg_head,
struct data_buffer* rbuf, struct data_buffer* sbuf,
int16_t exceed_cpu_used, uint64_t deadline_time) {
int32_t result_code = MILE_RETURN_SUCCESS;
MEM_POOL_PTR mem_pool;
void* parsed_packet;
ExecutePlan* plan = NULL;
void* result;
switch (msg_head->message_type) {
case MT_MD_EXE_INSERT:
mem_pool = mem_pool_init(MB_SIZE);
parsed_packet = parse_insert_packet(mem_pool, rbuf);
plan = SqlAnalyzer::Analyze(StorageEngine::storage,
(struct insert_packet*) parsed_packet, mem_pool);
if (plan == NULL) {
result_code = -1;
} else {
result = plan->Execute(result_code, deadline_time);
plan->~ExecutePlan();
}
if (result_code == MILE_RETURN_SUCCESS) {
gen_insert_result_packet(*(uint64_t*) result, msg_head, sbuf);
}
mem_pool_destroy(mem_pool);
break;
case MT_MD_EXE_DELETE:
if (exceed_cpu_used) {
log_warn("cpu使用率超出，拒绝UPDATE QUERY DELETE服务");
break;
}
mem_pool = mem_pool_init(MB_SIZE);
parsed_packet = parse_delete_packet(mem_pool, rbuf);
plan = SqlAnalyzer::Analyze(StorageEngine::storage,
(struct delete_packet*) parsed_packet, mem_pool);
if (plan == NULL) {
result_code = -1;
} else {
result = plan->Execute(result_code, deadline_time);
plan->~ExecutePlan();
}
if (result_code == MILE_RETURN_SUCCESS) {
gen_delete_result_packet(*(uint32_t*) result, msg_head, sbuf);
}
mem_pool_destroy(mem_pool);
break;
case MT_MD_EXE_EXPORT: {
mem_pool = mem_pool_init(MB_SIZE);
parsed_packet = parse_export_packet(mem_pool, rbuf);
plan = SqlAnalyzer::Analyze(StorageEngine::storage,
(struct export_packet *) parsed_packet, mem_pool);
if (plan == NULL) {
result_code = -1;
} else {
result = plan->Execute(result_code, deadline_time);
plan->~ExecutePlan();
}
if (result_code == MILE_RETURN_SUCCESS) {
gen_export_result_packet(*(uint64_t*) result, msg_head, sbuf);
}
mem_pool_destroy(mem_pool);
}
break;
case MT_MD_EXE_UPDATE:
if (exceed_cpu_used) {
log_warn("cpu使用率超出，拒绝UPDATE QUERY DELETE服务");
break;
}
mem_pool = mem_pool_init(MB_SIZE);
parsed_packet = parse_update_packet(mem_pool, rbuf);
plan = SqlAnalyzer::Analyze(StorageEngine::storage,
(struct update_packet*) parsed_packet, mem_pool);
if (plan == NULL) {
result_code = -1;
} else {
result = plan->Execute(result_code, deadline_time);
plan->~ExecutePlan();
}
if (result_code == MILE_RETURN_SUCCESS) {
gen_update_result_packet(*(uint32_t*) result, msg_head, sbuf);
}
mem_pool_destroy(mem_pool);
break;
case MT_MD_EXE_QUERY:
if (exceed_cpu_used) {
log_warn("cpu使用率超出，拒绝UPDATE QUERY DELETE服务");
break;
}
mem_pool = mem_pool_init(MB_SIZE);
result_code = loadavg_check(); // check overloading
if (ERROR_OS_OVERLOADING == result_code) {
log_warn(
"OS overloading, discard packet msg id %"PRIu32, msg_head->message_id);
} else if (0 == result_code) {
parsed_packet = parse_query_packet(mem_pool, rbuf);
plan = SqlAnalyzer::Analyze(StorageEngine::storage,
(struct query_packet*) parsed_packet, mem_pool);
if (plan == NULL) {
result_code = -1;
} else {
result = plan->Execute(result_code, deadline_time);
plan->~ExecutePlan();
}
}
if (result_code == MILE_RETURN_SUCCESS) {
gen_query_result_packet((ResultSet*) result, msg_head, sbuf);
}
mem_pool_destroy(mem_pool);
break;
case MT_MD_GET_STATE:
mem_pool = mem_pool_init(KB_SIZE);
parsed_packet = parse_get_state_packet(mem_pool, rbuf);
result = StateManager::QueryStates(StorageEngine::storage,
(struct get_state_packet*) parsed_packet, mem_pool);
if (result == NULL) {
result_code = ERROR_GET_DOCSERVER_STATE;
} else {
gen_state_result_packet((struct stat_info_array*) result, msg_head,
sbuf);
}
mem_pool_destroy(mem_pool);
break;
default:
result_code = StorageEngine::storage->SpecialSql(msg_head, rbuf, sbuf);
break;
}
if (result_code < 0) {
gen_error_packet(result_code, msg_head, sbuf);
}
return result_code;
}
virtual void TearDown()
{
DELETE(engine_);
mem_pool_destroy(mem_);
ClearDir();
}
rbthash_table_t *
rbthash_table_init (int buckets, rbt_hasher_t hfunc,
rbt_data_destroyer_t dfunc,
unsigned long expected_entries,
struct mem_pool *entrypool)
{
rbthash_table_t *newtab = NULL;
int ret = -1;
if (!hfunc) {
gf_log (GF_RBTHASH, GF_LOG_ERROR, "Hash function not given");
return NULL;
}
if (!entrypool && !expected_entries) {
gf_log (GF_RBTHASH, GF_LOG_ERROR,
"Both mem-pool and expected entries not provided");
return NULL;
}
if (entrypool && expected_entries) {
gf_log (GF_RBTHASH, GF_LOG_ERROR,
"Both mem-pool and expected entries are provided");
return NULL;
}
newtab = GF_CALLOC (1, sizeof (*newtab),
gf_common_mt_rbthash_table_t);
if (!newtab)
return NULL;
newtab->buckets = GF_CALLOC (buckets, sizeof (struct rbthash_bucket),
gf_common_mt_rbthash_bucket);
if (!newtab->buckets) {
goto free_newtab;
}
if (expected_entries) {
newtab->entrypool =
mem_pool_new (rbthash_entry_t, expected_entries);
if (!newtab->entrypool) {
gf_log (GF_RBTHASH, GF_LOG_ERROR,
"Failed to allocate mem-pool");
goto free_buckets;
}
newtab->pool_alloced = _gf_true;
} else {
newtab->entrypool = entrypool;
}
LOCK_INIT (&newtab->tablelock);
newtab->numbuckets = buckets;
ret = __rbthash_init_buckets (newtab, buckets);
if (ret == -1) {
gf_log (GF_RBTHASH, GF_LOG_ERROR, "Failed to init buckets");
if (newtab->pool_alloced)
mem_pool_destroy (newtab->entrypool);
} else {
gf_log (GF_RBTHASH, GF_LOG_TRACE, "Inited hash table: buckets:"
" %d", buckets);
}
newtab->hashfunc = hfunc;
newtab->dfunc = dfunc;
free_buckets:
if (ret == -1)
GF_FREE (newtab->buckets);
free_newtab:
if (ret == -1) {
GF_FREE (newtab);
newtab = NULL;
}
return newtab;
}
