Exemplo n.º 1
0
int main(int argc, char* argv[])
{
	char* cmd = NULL;
	MEM_POOL* mem_pool = mem_pool_init(MB_SIZE);
	int socket_fd;

	signal(SIGPIPE, SIG_IGN);
	signal(SIGHUP, SIG_IGN);
	signal(SIGINT, sign_handler);
	signal(SIGTERM, sign_handler);

	if((socket_fd = parse_cmd_line(argc, argv, mem_pool)) <= 0)
	{
		mem_pool_destroy(mem_pool);
		return 0;
	}
	
	while (global_start) {
		if ((cmd = readline("MILE>")) == NULL) {
			break;
		}
		parse_cmd(cmd,socket_fd,mem_pool);
		mem_pool_reset(mem_pool);
		if(cmd != NULL)
		{
			free(cmd);
			cmd = NULL;
		}
	}

	mem_pool_destroy(mem_pool);
	return 0;
}	
Exemplo n.º 2
0
/******************************************************************************
 **函数名称: mon_conf_load
 **功    能: 加载配置信息
 **输入参数:
 **     path: 配置路径
 **输出参数: NONE
 **返    回: 0:成功 !0:失败
 **实现描述: 
 **注意事项: 
 **作    者: # Qifeng.zou # 2015.03.15 #
 ******************************************************************************/
mon_conf_t *mon_conf_load(const char *path)
{
    xml_opt_t opt;
    xml_tree_t *xml;
    mon_conf_t *conf;
    mem_pool_t *pool;

    /* > 创建配置对象 */
    conf = (mon_conf_t *)calloc(1, sizeof(mon_conf_t));
    if (NULL == conf) {
        return NULL;
    }

    /* > 构建XML树 */
    pool = mem_pool_creat(4 * KB);
    if (NULL == pool) {
        free(conf);
        return NULL;
    }

    do
    {
        /* > 加载XML树 */
        memset(&opt, 0, sizeof(opt));

        opt.pool = pool;
        opt.alloc = (mem_alloc_cb_t)mem_pool_alloc;
        opt.dealloc = (mem_dealloc_cb_t)mem_pool_dealloc;

        xml = xml_creat(path, &opt);
        if (NULL == xml) {
            break;
        }

        /* > 提取配置信息 */
        if (mon_conf_load_menu(xml, conf)) {
            break;
        }

        MON_LOAD_CONF(xml, ".MONITOR.CRAWLER", &conf->crwl);
        MON_LOAD_CONF(xml, ".MONITOR.FILTER", &conf->filter);
        MON_LOAD_CONF(xml, ".MONITOR.SEARCH", &conf->search);

        /* > 释放XML树 */
        xml_destroy(xml);
        mem_pool_destroy(pool);

        return conf;
    } while(0);

    /* > 异常处理 */
    free(conf);
    xml_destroy(xml);
    mem_pool_destroy(pool);

    return NULL;
}
Exemplo n.º 3
0
int32_t main(int32_t argc, char **argv)
{
      
    struct mem_pool *test_mem_pool;
    struct mem_pool *test_mem_pool_2;
    struct mem_pool *test_mem_pool_3;
    gf_mem_init_mempool_list();
    gf_mem_acct_enable_set ();
    signals_setup();
    mem_acct_init(gf_common_mt_end+1);
    test_mem_t ** mem_array   = CALLOC(size,sizeof(test_mem_t * )); 
    test_mem_2_t ** mem_array2  = CALLOC(size,sizeof(test_mem_2_t * ));
    test_mem_3_t ** mem_array3  = CALLOC(size,sizeof(test_mem_3_t * ));
    int j;
    int i;
    CPU_TIME_START;
    test_mem_pool = mem_pool_new (test_mem_t, size);
    test_mem_pool_2 = mem_pool_new (test_mem_2_t, size);
    test_mem_pool_3 = mem_pool_new (test_mem_3_t, size);
    CPU_TIME_END_PRINT("mem pool");
    CPU_TIME_START;
    for(j=0; j<10000; j++){    
            if (!test_mem_pool)
            {
                DBG_PRINT("create mem pool error");
                return -1;
            }
        for(i=0; i<size; i++)
            mem_array[i] = (test_mem_t *)mem_get(test_mem_pool);
            mem_array2[i] = (test_mem_2_t *)mem_get(test_mem_pool_2);
            mem_array3[i] = (test_mem_3_t *)mem_get(test_mem_pool_3);
        for(i=0; i<size; i++)
            mem_put(mem_array[i]) ;
            mem_put(mem_array2[i]) ;
            mem_put(mem_array3[i]) ;
        
    }
    CPU_TIME_END_PRINT("mem pool");
    mem_pool_destroy(test_mem_pool);
    mem_pool_destroy(test_mem_pool_2);
    mem_pool_destroy(test_mem_pool_3);
    CPU_TIME_START;
    for(j=0; j<100; j++){
        for(i=0; i<size; i++)
            mem_array[i] = (test_mem_t *)MALLOC(sizeof(test_mem_t));
            mem_array2[i] = (test_mem_2_t *)MALLOC(sizeof(test_mem_2_t));
            mem_array3[i] = (test_mem_3_t *)MALLOC(sizeof(test_mem_3_t));
        for(i=0; i<size; i++)
            free (mem_array[i]);
            free (mem_array2[i]);
            free (mem_array3[i]);
    }
    CPU_TIME_END_PRINT("not mem pool");
    
    //getchar();
    return 0;
}
Exemplo n.º 4
0
int DocEngine::SpecialSql(struct mile_message_header* msg_head, struct data_buffer* rbuf, struct data_buffer* sbuf){
	MEM_POOL_PTR mem_pool;
	void* packet;
	void* result;
	int32_t result_code = MILE_RETURN_SUCCESS;
	uint16_t msg_type = msg_head->message_type;


	switch (msg_type) {
	case MT_MD_EXE_GET_KVS:
		mem_pool = mem_pool_init(MB_SIZE);
		packet = parse_get_kvs_packet(mem_pool, rbuf);
		result = GetKvs((struct get_kvs_packet*) packet, mem_pool);

		if(NULL == result){
			result_code = ERROR_QUERY_BY_ROWID;
		}else{
			gen_query_result_packet((ResultSet*) result, msg_head,
					sbuf);
		}
		mem_pool_destroy(mem_pool);
		break;
	default:
		log_error("unsupported command, packet type %d", msg_type);
		result_code = ERROR_UNSUPPORTED_SQL_TYPE;
		break;
	}

	return result_code;
}
Exemplo n.º 5
0
static void tcp_stream_close(tcp_server_stream_t *stream)
{
    tcp_server_t *server = NULL;
    if (NULL == stream) {
        return; 
    }

    server = stream->server;

    if (stream->pool) {
        mem_pool_destroy(&stream->pool);
    }

    if (stream->sock_fd >= 0) {
        close(stream->sock_fd);
    }

    /* 从busy链表中删除 */
    delete_stream_from_busy_list(server, stream); 
     
    /* 放入到idle链表尾部 */
    add_stream_into_idle_list(server, stream);  

    /* reset stream */
    stream->sock_fd = -1;
    stream->pool    = NULL;
    stream->close   = 0;
    stream->error   = 0;
    stream->data    = NULL;
    stream->server  = NULL;
}
Exemplo n.º 6
0
int32_t process_test_message(struct mile_message_header* msg_head,
		struct data_buffer* rbuf, struct data_buffer* sbuf) {
	int32_t result_code = MILE_RETURN_SUCCESS;
	MEM_POOL_PTR mem_pool;

	switch (msg_head->message_type) {
	case MT_TEST_REQ_ECHO: // echo message for testing.
	{
		mem_pool = mem_pool_init(MB_SIZE);
		const char *str = read_cstring(rbuf, mem_pool);
		log_debug("get echo message [%s]", str);

		write_int32(0, sbuf);
		write_int8(msg_head->version_major, sbuf);
		write_int8(msg_head->version_minor, sbuf);
		write_int16(MT_TEST_RES_ECHO, sbuf);
		write_int32(msg_head->message_id, sbuf);

		write_int32(strlen(str), sbuf);
		write_bytes((uint8_t *) str, strlen(str), sbuf);

		fill_int32(sbuf->data_len, sbuf, 0);
		result_code = MILE_RETURN_SUCCESS;
		mem_pool_destroy(mem_pool);
	}
		break;
	default:
		result_code = -1;
		break;
	}

	return result_code;
}
Exemplo n.º 7
0
int32_t process_master_message(struct mile_message_header* msg_head,
		struct data_buffer* rbuf, struct data_buffer* sbuf, void **bl_reader,
		MEM_POOL_PTR session_mem) {
	int32_t result_code = MILE_RETURN_SUCCESS;
	MEM_POOL_PTR mem_pool;
	void* packet;
	struct slave_sync_res *binlog_res;

	switch (msg_head->message_type) {
	case MT_SM_GET_BINLOG:
		mem_pool = mem_pool_init(MB_SIZE);
		// STATIC_ASSERT( MB_SIZE > sizeof(struct slave_sync_res) + MAX_BINLOG_DATA_PER_PACKET, MAX_BINLOG_PACKAGE_SHOULD_LESS_THAN_MEM_POOL_SIZE);
		packet = parse_slave_sync_req(mem_pool, rbuf);
		binlog_res = execute_slave_sync(mem_pool,
				(struct slave_sync_req *) packet, bl_reader, session_mem);
		assert(NULL != binlog_res);
		gen_slave_sync_res_packet(binlog_res, msg_head, sbuf);
		result_code = 0;
		mem_pool_destroy(mem_pool);
		break;
	default:
		result_code = -1;
		break;
	}

	return result_code;
}
Exemplo n.º 8
0
int32_t process_client_message(struct mile_message_header* msg_head,
		struct data_buffer* rbuf, struct data_buffer* sbuf) {
	int32_t result_code = MILE_RETURN_SUCCESS;
	MEM_POOL_PTR mem_pool;
	void* packet;
	struct slave_sync_res *binlog_res;

	switch (msg_head->message_type) {
	case MT_CD_EXE_GET_LOAD_THRESHOLD: {
		log_debug("get get_load_threshold command");
		double load = g_load_threshold;
		gen_dc_get_load_threshold_packet(load, msg_head, sbuf);
	}
		break;
	case MT_CD_EXE_SET_LOAD_THRESHOLD: {
		log_debug("get set_load_threshold command");
		double old_load = g_load_threshold;
		mem_pool = mem_pool_init(KB_SIZE);
		struct set_load_threshold_packet *p = parse_set_load_threshold_packet(
				mem_pool, rbuf);

		g_load_threshold = p->value;
		log_info(
				"change os overloading threshold from %g to %g", old_load, g_load_threshold);
		gen_dc_set_load_threshold_packet(old_load, msg_head, sbuf);
		mem_pool_destroy(mem_pool);
	}
		break;
	default:
		result_code = -1;
		break;
	}

	return result_code;
}
Exemplo n.º 9
0
void test_key_value()
{
    MemPool *pool = NULL;
    key_value_t *kv = NULL;
    key_value_node_t *node = NULL;
    char *keys[5], *vals[5];
    
    pool = mem_pool_create(160); //160M
    kv = key_value_init(pool, 20);

    keys[0] = "abcde";
    vals[0] = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
    key_value_set(pool, kv, "abcde", strlen("abcde"), 
        vals[0], strlen(vals[0]), 0);
        
    keys[1] = "qwerty";
    vals[1] = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
    key_value_set(pool, kv, keys[1], strlen(keys[1]), 
        vals[1], strlen(vals[1]), 0);

    keys[2] = "asdf";
    vals[2] = "cccccccccccc";
    key_value_set(pool, kv, keys[2], strlen(keys[2]), 
        vals[2], strlen(vals[2]), 0);

    node = key_value_get(kv, keys[2], strlen(keys[2]));
    printf("get %s node data: %s\n", keys[2], (char *)node->data);

    keys[3] = "asdf";
    vals[3] = "dddddddddddddddddddddddddddddddddddddddddddddddddddd";
    key_value_set(pool, kv, keys[3], strlen(keys[3]), 
        vals[3], strlen(vals[3]), 0);

    node = key_value_get(kv, keys[3], strlen(keys[3]));
    printf("get %s node data: %s\n", keys[3], (char *)node->data);

    keys[4] = "lkjhgfdsa";
    vals[4] = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee";
    key_value_set(pool, kv, keys[4], strlen(keys[4]), 
        vals[4], strlen(vals[4]), 0);

    node = key_value_get(kv, "aaaa", strlen("aaaa"));
    if(node)
        printf("get aaaa node data: %s\n", (char *)node->data);
    else
        printf("get aaaa node data: Not Found\n");


    printf("Key Value: used memory: %u, mem pool used memory: %lu \n", 
        kv->used_memory, pool->used_size);
    printf("Key-Value Pairs number: %u\n", kv->count);
    key_value_destroy(pool, kv);
    mem_pool_destroy(pool);
}
Exemplo n.º 10
0
int main()
{
    MemPool *pool = mem_pool_create(640); //640M
    configure_handler_t *config_handler = load_configure_file(pool, "test.conf");

    test_log(pool, config_handler);

    configure_handler_destroy(pool, config_handler);
    mem_pool_destroy(pool);
    return 0;
}
Exemplo n.º 11
0
void rs_releasemgr()
{
    struct linked_list* unload_item = g_rs.unload_items;
    while (unload_item != NULL) {
        struct linked_list* nitem = unload_item->next;
        FREE(unload_item->data);
        unload_item = nitem;
    }

    hashtable_chained_destroy(&g_rs.dict);
    mem_pool_destroy(&g_rs.dict_itempool);
    mem_pool_destroy(&g_rs.freeslot_pool);
    arr_destroy(&g_rs.ress);
    mem_pool_destroy(&g_rs.load_data_pool);

    if (g_rs.job_params.load_items != NULL)
        FREE(g_rs.job_params.load_items);
    if (g_rs.job_result.ptrs != NULL)
        FREE(g_rs.job_result.ptrs);

    log_print(LOG_TEXT, "res-mgr released.");
}
Exemplo n.º 12
0
void
rbthash_table_destroy (rbthash_table_t *tbl)
{
        if (!tbl)
                return;

        rbthash_table_destroy_buckets (tbl);
        if (tbl->pool_alloced)
                mem_pool_destroy (tbl->entrypool);

        GF_FREE (tbl->buckets);
        GF_FREE (tbl);
}
Exemplo n.º 13
0
mem_pool_t *mem_pool_create()
{
    void       *addr      = NULL;
    mem_pool_t *mem_pool  = NULL;
    int         page_size = 0;
    mem_page_t *page      = NULL;

    /* 获取内存page的大小 */ 
    page_size = getpagesize();
    
    /* 分配物理内存page */
    if(posix_memalign(&addr, page_size, page_size) != 0){
        printf("posix_memalign in mem_pool_create fails.\n");
        goto error;
    }

    /* 内存池信息和第一个page的信息都存储在第一个page的头部 */
    mem_pool             = (mem_pool_t *)addr;
    mem_pool->page_size  = page_size;
    mem_pool->page_count = 1;
    page                 = (mem_page_t *)((char *)addr + sizeof(mem_pool_t));
    mem_pool->current    = page;
    mem_pool->head       = page;
    mem_pool->tail       = page;

#ifdef MEM_POOL_LOCK
    pthread_mutex_init(&mem_pool->lock, NULL);
#endif

    /* 设置第一个page的信息 */
    page->start  = (char *)addr;
    page->last   = page->start + 
                   sizeof(mem_pool_t) + 
                   sizeof(mem_page_t);
    page->end    = (char *)addr + page_size;
    page->next   = NULL;
    page->failed = 0;

    return mem_pool;

error:
    if(mem_pool){
        mem_pool_destroy(&mem_pool);
    }
    return NULL;
}
Exemplo n.º 14
0
void test_configure()
{
    configure_handler_t *config_handler;
    char *filename = "test.conf";
    MemPool *pool = mem_pool_create(640); //640M
    char section_search[] = "[filter:dlo]";
    char key_search[] = "use";

    config_handler = load_configure_file(pool, filename);

    //print_configure(config_handler);

    printf("%s %s:  %s\n", section_search, key_search,
        get_configure_item(config_handler,section_search ,key_search ));


    mem_pool_destroy(pool);
}
Exemplo n.º 15
0
void* MainStep::Execute(TableManager* table, void* input, int32_t &result_code,
		int64_t timeout, MEM_POOL_PTR mem_pool) {
	MileIterator* iter = (MileIterator*) input;
	MileIterator* step_iter = substeps->CreateIterator();
	uint32_t i = 0;
	uint32_t cut_threshold=table->getCutThreshold();

	ExecuteSubstep* substep;
	MEM_POOL_PTR inner_mem_pool = mem_pool_init(MB_SIZE);

	for (iter->First(); !iter->IsDone(); iter->Next()) {
		for (step_iter->First(); !step_iter->IsDone(); step_iter->Next()) {
			substep = (ExecuteSubstep*) step_iter->CurrentItem();
			result_code = substep->Execute(table,
					(MileHandler*) iter->CurrentItem(), result, inner_mem_pool);
			if (result_code != 0) {
				break;
			}
		}
		mem_pool_reset(inner_mem_pool);
		if (result_code < 0) {
			break;
		}

		if((++i) % 100 == 0){
			if(get_time_msec() > timeout){
				result_code = ERROR_TIMEOUT;
				break;
			}
		}
		if( cut_threshold > 0 && i >= cut_threshold )
		{
			break;
		}
	}

	mem_pool_destroy(inner_mem_pool);

	if (result_code == WARN_EXCEED_QUERY_LIMIT || result_code >= 0) {
		result_code = MILE_RETURN_SUCCESS;
	}
	return result;
}
Exemplo n.º 16
0
xml_doc_t* xml_doc_create(){
    xml_doc_t*      doc;
    mem_pool_t*     pool;

    pool = mem_pool_create();
    if(!pool){
        return 0;
    }
    doc = (xml_doc_t*)mem_pool_alloc(pool,sizeof(xml_doc_t));
    if(!doc){
        mem_pool_destroy(pool);
        return 0;
    }
    doc->pool           = pool;
    doc->root           = 0;
    doc->_raw_item_head = xml_raw_item_create(doc->pool,item_type_none,0,0);
    doc->_raw_item_cur  = doc->_raw_item_head;
    return doc;
}
Exemplo n.º 17
0
TEST(DATA_BUFFER_TEST, READ_AND_WRITE){

	MEM_POOL_PTR pMemPool = mem_pool_init(MB_SIZE);
	struct data_buffer* buffer = init_data_buffer();
	buffer->data_len = 256;
	buffer->data = (uint8*) mem_pool_malloc(pMemPool, 256);

	uint8 tmp8;
	uint16 tmp16;
	uint32 tmp32;
	uint64 tmp64;

	char* str = (char*) mem_pool_malloc(pMemPool, 100);
	char* result = (char*) mem_pool_malloc(pMemPool, 100);
	strcpy(str, "hello world!");

	
	write_int8(122, buffer);
	write_int16(1111, buffer);
	write_int32(324, buffer);
	write_int64(2321, buffer);
	write_bytes((uint8*)str, strlen(str)+1, buffer);

	

	tmp8 = read_int8(buffer);
	ASSERT_EQ(tmp8, 122);
	tmp16 = read_int16(buffer);
	ASSERT_EQ(tmp16, 1111);
	tmp32 = read_int32(buffer);
	ASSERT_EQ(tmp32, 324);
	tmp64 = read_int64(buffer);
	ASSERT_EQ(tmp64, 2321);

	read_bytes(buffer, (uint8*)result, strlen(str)+1);
	ASSERT_EQ(0, strcmp(str, result));

	destroy_data_buffer(buffer);
	mem_pool_destroy(pMemPool);
}
Exemplo n.º 18
0
static void tcp_client_stream_reset(tcp_client_stream_t *stream)
{
    tcp_client_t *client = NULL;
    if (NULL == stream) {
        return; 
    }

    client = stream->client;

    if (stream->pool) {
        mem_pool_destroy(&stream->pool);
    }

    if (stream->sock_fd >= 0) {
        close(stream->sock_fd);
    }

    /* reset stream */
    stream->sock_fd   = -1;
    stream->server_ip = NULL;
    stream->port      = 0;
    stream->pool      = NULL;
    stream->close     = 0;
    stream->error     = 0;
    stream->chain     = NULL;
    stream->cur_chain = NULL;
    stream->tcp_state = TCP_STATE_NONE;
    stream->data      = NULL;
    stream->client    = NULL;
    stream->last      = NULL;
    stream->next      = NULL;
    stream->rcv_cb    = NULL;
    stream->snd_cb    = NULL;
    stream->conn_cb   = NULL;

    bzero(&stream->update_time, NULL);
    bzero(&stream->dst_addr, NULL);
}
Exemplo n.º 19
0
int main(int argc, char * argv[]) {
    struct mem_pool * pool = mem_pool_new(4);

    int r1,r2,r3,r4;

    r1 = mem_pool_alloc(pool, 16);
    printb(pool, r1);

    r2 = mem_pool_alloc(pool, 32);
    printb(pool, r2);

    r3 = mem_pool_alloc(pool, 16);
    printb(pool, r3);

    mem_pool_free(pool, r1);
    mem_pool_free(pool, r3);

    r4 = mem_pool_alloc(pool, 32);
    printb(pool, r4);

    mem_pool_destroy(pool);
    return 0;
}
Exemplo n.º 20
0
TEST(BITMARK_TEST, HandleNoneZeroInput)  {
    int32 ret;
    int32 value;
    MEM_POOL* mem_pool = mem_pool_init(M_1M);  
    struct bitmark_config config; 
	strcpy(config.bitmark_name,"null");
	strcpy(config.work_space,"/tmp/bitmark_test");
	config.row_limit = 10;

	system("rm -rf /tmp/bitmark_test");
	init_profile(1000,mem_pool);
	mkdirs(config.work_space);

    struct bitmark_manager* bitmark = bitmark_init(&config, mem_pool);

    //全部置零
    bitmark_reset(bitmark);

    //置位
    //表示6行已插入该行值
    ret = bitmark_set(bitmark,6);
    ASSERT_EQ(ret,0);
    
    //查询
    value = bitmark_query(bitmark,6);
    ASSERT_EQ(value,0); 
	
    value = bitmark_query(bitmark,8);
    ASSERT_EQ(value,1);

    //删除6行的值
    ret = bitmark_clear(bitmark,6);
    ASSERT_EQ(ret,0);

    mem_pool_destroy(mem_pool);
}
Exemplo n.º 21
0
int DocEngine::Command(struct mile_message_header* msg_head,
		struct data_buffer* rbuf, struct data_buffer* sbuf) {
	MEM_POOL_PTR mem_pool;
	void* parsed_packet;
	void* result;
	int32_t result_code = 0;
	uint16_t msg_type = msg_head->message_type;

	switch (msg_type) {
	case MT_MD_EXE_GET_KVS:
		mem_pool = mem_pool_init(MB_SIZE);
		parsed_packet = parse_get_kvs_packet(mem_pool, rbuf);
		result = GetKvs((struct get_kvs_packet*) parsed_packet, mem_pool);

		if(NULL == result){
			result_code = ERROR_QUERY_BY_ROWID;
		}else{
			gen_query_result_packet((ResultSet*) result, msg_head,
					sbuf);
		}
		mem_pool_destroy(mem_pool);
		break;
	case MT_CD_EXE_COMPRESS:
		mem_pool = mem_pool_init(KB_SIZE);
		parsed_packet = parse_compress_packet(mem_pool, rbuf);

		log_debug(
				"接收到compress命令:%s", ((struct compress_packet*)parsed_packet)->table_name);

		db_read_lock();
		result_code = db_compress(
				((struct compress_packet*) parsed_packet)->table_name,
				mem_pool);
		db_read_unlock();

		gen_dc_response_packet(result_code, msg_head, sbuf);
		mem_pool_destroy(mem_pool);
		break;
	case MT_CD_EXE_INDEX:
		mem_pool = mem_pool_init(KB_SIZE);
		parsed_packet = parse_ensure_index_packet(mem_pool, rbuf);

		log_debug(
				"接收到ensure_index命令 %s %s %u %u", ((struct ensure_index_packet*)parsed_packet)->table_name, ((struct ensure_index_packet*)parsed_packet)->field_name, ((struct ensure_index_packet*)parsed_packet)->index_type, ((struct ensure_index_packet*)parsed_packet)->data_type);

		db_read_lock();
		result_code = db_ensure_index(
				((struct ensure_index_packet*) parsed_packet)->table_name,
				((struct ensure_index_packet*) parsed_packet)->field_name,
				((struct ensure_index_packet*) parsed_packet)->index_type,
				((struct ensure_index_packet*) parsed_packet)->data_type,
				mem_pool);

		db_read_unlock();
		gen_dc_response_packet(result_code, msg_head, sbuf);
		mem_pool_destroy(mem_pool);
		break;

	case MT_CD_EXE_UNINDEX:
		mem_pool = mem_pool_init(KB_SIZE);
		parsed_packet = parse_del_index_packet(mem_pool, rbuf);

		log_debug(
				"接收到del_index命令 %s %s %u", ((struct del_index_packet*)parsed_packet)->table_name, ((struct del_index_packet*)parsed_packet)->field_name, ((struct del_index_packet*)parsed_packet)->index_type);

		db_read_lock();
		result_code = db_del_index(
				((struct del_index_packet*) parsed_packet)->table_name,
				((struct del_index_packet*) parsed_packet)->field_name,
				((struct del_index_packet*) parsed_packet)->index_type,
				mem_pool);
		db_read_unlock();

		gen_dc_response_packet(result_code, msg_head, sbuf);
		mem_pool_destroy(mem_pool);
		break;

	case MT_CD_EXE_LOAD:
		mem_pool = mem_pool_init(KB_SIZE);
		parsed_packet = parse_load_segment_packet(mem_pool, rbuf);

		log_debug(
				"接收到load_segment命令 %s %u %s", ((struct load_segment_packet*)parsed_packet)->table_name, ((struct load_segment_packet*)parsed_packet)->sid, ((struct load_segment_packet*)parsed_packet)->segment_dir);

		db_read_lock();
		result_code = db_load_segment(
				((struct load_segment_packet*) parsed_packet)->table_name,
				((struct load_segment_packet*) parsed_packet)->sid,
				((struct load_segment_packet*) parsed_packet)->segment_dir,
				mem_pool);
		db_read_unlock();

		gen_dc_response_packet(result_code, msg_head, sbuf);
		mem_pool_destroy(mem_pool);
		break;

	case MT_CD_EXE_REPLACE:
		mem_pool = mem_pool_init(KB_SIZE);
		parsed_packet = parse_replace_segment_packet(mem_pool, rbuf);

		log_debug(
				"接收到replace segments命令 %s %s", ((struct replace_segment_packet*)parsed_packet)->table_name, ((struct replace_segment_packet*)parsed_packet)->segment_dir);

		db_read_lock();
		result_code = db_replace_all_segments(
				((struct replace_segment_packet*) parsed_packet)->table_name,
				((struct replace_segment_packet*) parsed_packet)->segment_dir,
				mem_pool);
		db_read_unlock();

		gen_dc_response_packet(result_code, msg_head, sbuf);
		mem_pool_destroy(mem_pool);
		break;

	case MT_CD_EXE_UNLOAD:
		mem_pool = mem_pool_init(KB_SIZE);

		parsed_packet = parse_unload_segment_packet(mem_pool, rbuf);

		log_debug(
				"接收到unload_segment命令 %s %u", ((struct unload_segment_packet*)parsed_packet)->table_name, ((struct unload_segment_packet*)parsed_packet)->sid);

		db_read_lock();
		result_code = db_unload_segment(
				((struct unload_segment_packet*) parsed_packet)->table_name,
				((struct unload_segment_packet*) parsed_packet)->sid, mem_pool);
		db_read_unlock();

		gen_dc_response_packet(result_code, msg_head, sbuf);
		mem_pool_destroy(mem_pool);
		break;

	case MT_CD_EXE_CP:
		log_debug( "get checkpoint command");
		db_checkpoint();
		gen_dc_response_packet(0, msg_head, sbuf);
		break;
	case MT_CD_STAT:
		mem_pool = mem_pool_init(MB_SIZE);
		parsed_packet = parse_doc_stat_packet(mem_pool, rbuf);

		log_debug(
				"接收到stat命令 %s %u", ((struct doc_stat_packet*)parsed_packet)->table_name, ((struct doc_stat_packet*)parsed_packet)->type);

		if (((struct doc_stat_packet*) parsed_packet)->type == 1) {
			uint16_t max_segment_num = 0;
			struct segment_meta_data* result = NULL;

			db_read_lock();
			result = db_query_segment_stat(
					((struct doc_stat_packet*) parsed_packet)->table_name,
					&max_segment_num, mem_pool);
			db_read_unlock();

			gen_dc_segment_stat_packet(result, max_segment_num, msg_head, sbuf);
		}

		if (((struct doc_stat_packet*) parsed_packet)->type == 2) {
			uint16_t index_field_count = 0;
			struct index_field_meta* result = NULL;

			db_read_lock();
			result = db_query_index_stat(
					((struct doc_stat_packet*) parsed_packet)->table_name,
					&index_field_count, mem_pool);
			db_read_unlock();

			gen_dc_index_stat_packet(result, index_field_count, msg_head, sbuf);
		}

		mem_pool_destroy(mem_pool);
		break;
	default:
		log_error("unsupported command, packet type %d", msg_type);
		result_code = ERROR_UNSUPPORTED_SQL_TYPE;
		break;
	}

	return result_code;
}
Exemplo n.º 22
0
DocEngine::~DocEngine() {
	db_release();
	mem_pool_destroy(mem_);
	mem_ = NULL;
}
Exemplo n.º 23
0
TEST(HASHINDEX_TEST, HandleNoneZeroInput)  {
    MEM_POOL* mem_pool = mem_pool_init(MB_SIZE);
    struct hash_index_config config;

    system("rm -rf /tmp/hashindex_test");
    char dir_path[] = "/tmp/hashindex_test";
    mkdirs(dir_path);
    config.row_limit = BUCKET_NUM;
    strcpy(config.work_space,dir_path);

    init_profile(1000,mem_pool);
    int32_t ret;

    //插入一个值
    struct low_data_struct insert_data;
    get_low_data(&insert_data,mem_pool);

    struct hash_index_manager* hash_index = hash_index_init(&config,mem_pool);
    ret = hash_index_insert(hash_index,&insert_data,0);
    ASSERT_EQ(ret,0);

    //查询
    struct rowid_list* rowids;

    uint32_t i;
    struct rowid_list_node* p;

    rowids = hash_index_query(hash_index,&insert_data,mem_pool);

    for(i = 0, p = rowids->head; i < rowids->rowid_num; i++)
    {
        if(i != 0 && i%ROWID_ARRAY_SIZE == 0)
        {
            p = p->next;
        }
        ASSERT_EQ(p->rowid_array[i%ROWID_ARRAY_SIZE],0);
        break;
    }

    //再插入一个值
    hash_index_insert(hash_index,&insert_data,1);

    rowids = hash_index_query(hash_index,&insert_data,mem_pool);


    for(i = 0, p = rowids->head; i < rowids->rowid_num; i++)
    {
        if(i != 0 && i%ROWID_ARRAY_SIZE == 0)
        {
            p = p->next;

            if(i==0)
                ASSERT_EQ(p->rowid_array[i%ROWID_ARRAY_SIZE],1);

            if(i==1)
            {
                ASSERT_EQ(p->rowid_array[i%ROWID_ARRAY_SIZE],0);
                break;
            }
        }
    }

    //插入BUCKET_NUM个
    for(i=2; i<BUCKET_NUM+1; i++)
    {
        sprintf((char*)insert_data.data,"ali%u",i);
        ret = hash_index_insert(hash_index,&insert_data,i);
        ASSERT_EQ(ret,0);
    }

    //再插一个报冲突了
    sprintf((char*)insert_data.data,"ali%u",i);
    ret = hash_index_insert(hash_index,&insert_data,i);
    ASSERT_EQ(ret,ERROR_HASH_CONFLICT);

    //插入一个空值,OK
    insert_data.len = 0;
    ret = hash_index_insert(hash_index,&insert_data,i);
    ASSERT_EQ(ret,0);

    rowids = hash_index_query(hash_index,&insert_data,mem_pool);

    ASSERT_EQ(rowids->rowid_num,1);
    ASSERT_EQ(rowids->head->rowid_array[0],5);

    hash_index_release(hash_index);
    mem_pool_destroy(mem_pool);
}
Exemplo n.º 24
0
TEST(RECOVER_TEST_2, HandleNoneZeroInput)  {
    MEM_POOL* mem_pool = mem_pool_init(MB_SIZE);
    struct hash_index_config config;

    system("rm -rf /tmp/hashindex_test");
    char dir_path[] = "/tmp/hashindex_test";
    mkdirs(dir_path);
    config.row_limit = ROW_LIMIT;
    strcpy(config.work_space,dir_path);

    init_profile(1000,mem_pool);
    int32_t ret;

    struct hash_index_manager* hash_index = hash_index_init(&config,mem_pool);

    //插入18条数据,只恢复16条
    uint32_t i;
    struct low_data_struct* data = (struct low_data_struct*)mem_pool_malloc(mem_pool, sizeof(struct low_data_struct));
    memset(data, 0, sizeof(struct low_data_struct));

    for(i=0; i<18; i++)
    {
        if(i != 0 && i%2 == 1)
            continue;

        data->len = 5;
        data->data = mem_pool_malloc(mem_pool,5);
        data->type = HI_TYPE_STRING;
        data->field_name = (char*)mem_pool_malloc(mem_pool,20);
        memset(data->field_name,0,20);
        strcpy(data->field_name,"HI_TYPE_STRING");
        memset(data->data,0,5);
        ret = hash_index_insert(hash_index,data,i);
        ASSERT_EQ(0, ret);
        ret = hash_index_insert(hash_index,data,i+1);
        ASSERT_EQ(0, ret);
    }

    struct doc_row_unit* doc = NULL;

    ASSERT_EQ(0, hash_index_recover(hash_index, 16));

    for(i=0; i<ROW_LIMIT; i++)
    {
        doc = GET_DOC_ROW_STRUCT(hash_index->doclist, i);
    }

    //验证
    for(i=0; i<16; i++)
    {
        doc = GET_DOC_ROW_STRUCT(hash_index->doclist, i);
        ASSERT_EQ(i, doc->doc_id);
    }

    for(i=16; i<ROW_LIMIT; i++)
    {
        doc = GET_DOC_ROW_STRUCT(hash_index->doclist, i);
        ASSERT_EQ(0, doc->doc_id);
        ASSERT_EQ(0,doc->next);
    }

    struct hash_bucket* hbucket = NULL;
    uint32_t bucket_no = 0;

    for(i=0; i<ROW_LIMIT+1; i++)
    {
        hbucket = hash_index->mem_mmaped + i;
        doc = NEXT_DOC_ROW_STRUCT(hash_index->doclist, hbucket->offset);

        if(hbucket->hash_value == 0)
        {
            continue;
        }

        while(!(doc->next & 0x80000000))
        {
            ASSERT_GT(16, doc->doc_id);
            doc = NEXT_DOC_ROW_STRUCT(hash_index->doclist, doc->next);
        }

        bucket_no = doc->next & 0x7fffffff;
        ASSERT_EQ(i,bucket_no);
    }

    hash_index_release(hash_index);
    mem_pool_destroy(mem_pool);
}
Exemplo n.º 25
0
/******************************************************************************
 **函数名称: flt_worker_get_webpage_info
 **功    能: 获取网页信息
 **输入参数:
 **     path: 网页信息文件
 **输出参数:
 **     info: 网页信息
 **返    回: 0:成功 !0:失败
 **实现描述:
 **注意事项:
 **作    者: # Qifeng.zou # 2015.03.11 #
 ******************************************************************************/
static int flt_worker_get_webpage_info(
        const char *path, flt_webpage_info_t *info, log_cycle_t *log)
{
    xml_opt_t opt;
    xml_tree_t *xml;
    mem_pool_t *pool;
    xml_node_t *node, *fix;

    memset(&opt, 0, sizeof(opt));

    /* 1. 新建内存池 */
    pool = mem_pool_creat(4 * KB);
    if (NULL == pool) {
        log_error(log, "errmsg:[%d] %s!", errno, strerror(errno));
        return FLT_ERR;
    }

    /* 2. 新建XML树 */
    opt.log = log;
    opt.pool = pool;
    opt.alloc = (mem_alloc_cb_t)mem_pool_alloc;
    opt.dealloc = (mem_dealloc_cb_t)mem_pool_dealloc;

    xml = xml_creat(path, &opt);
    if (NULL == xml) {
        mem_pool_destroy(pool);
        log_error(log, "Create XML failed! path:%s", path);
        return FLT_ERR;
    }

    /* 2. 提取网页信息 */
    do {
        fix = xml_query(xml, ".WPI");
        if (NULL == fix) {
            log_error(log, "Get WPI mark failed!");
            break;
        }

        /* 获取URI字段 */
        node = xml_search(xml, fix, "URI");
        if (NULL == node) {
            log_error(log, "Get URI mark failed!");
            break;
        }

        snprintf(info->uri, sizeof(info->uri), "%s", node->value.str);

        /* 获取DEPTH字段 */
        node = xml_search(xml, fix, "URI.DEPTH");
        if (NULL == node) {
            log_error(log, "Get DEPTH mark failed!");
            break;
        }

        info->depth = atoi(node->value.str);

        /* 获取IP字段 */
        node = xml_search(xml, fix, "URI.IP");
        if (NULL == node) {
            log_error(log, "Get IP mark failed!");
            break;
        }

        snprintf(info->ip, sizeof(info->ip), "%s", node->value.str);

        /* 获取PORT字段 */
        node = xml_search(xml, fix, "URI.PORT");
        if (NULL == node) {
            log_error(log, "Get PORT mark failed!");
            break;
        }

        info->port = atoi(node->value.str);

        /* 获取HTML字段 */
        node = xml_search(xml, fix, "HTML");
        if (NULL == node) {
            log_error(log, "Get HTML mark failed!");
            break;
        }

        snprintf(info->html, sizeof(info->html), "%s", node->value.str);

        /* 获取HTML.SIZE字段 */
        node = xml_search(xml, fix, "HTML.SIZE");
        if (NULL == node) {
            log_error(log, "Get HTML.SIZE mark failed!");
            break;
        }

        info->size = atoi(node->value.str);
        if (info->size <= 0) {
            log_info(log, "Html size is zero!");
            break;
        }

        snprintf(info->fname, sizeof(info->fname), "%s", path);

        xml_destroy(xml);
        mem_pool_destroy(pool);
        return FLT_OK;
    } while(0);

    /* 3. 释放XML树 */
    xml_destroy(xml);
    mem_pool_destroy(pool);
    return FLT_ERR;
}
Exemplo n.º 26
0
int32_t process_mergeserver_message(struct mile_message_header* msg_head,
		struct data_buffer* rbuf, struct data_buffer* sbuf,
		int16_t exceed_cpu_used, uint64_t deadline_time) {
	int32_t result_code = MILE_RETURN_SUCCESS;
	MEM_POOL_PTR mem_pool;
	void* parsed_packet;
	ExecutePlan* plan = NULL;
	void* result;

	switch (msg_head->message_type) {
	case MT_MD_EXE_INSERT:
		mem_pool = mem_pool_init(MB_SIZE);
		parsed_packet = parse_insert_packet(mem_pool, rbuf);
		plan = SqlAnalyzer::Analyze(StorageEngine::storage,
				(struct insert_packet*) parsed_packet, mem_pool);
		if (plan == NULL) {
			result_code = -1;
		} else {
			result = plan->Execute(result_code, deadline_time);
			plan->~ExecutePlan();
		}
		if (result_code == MILE_RETURN_SUCCESS) {
			gen_insert_result_packet(*(uint64_t*) result, msg_head, sbuf);
		}
		mem_pool_destroy(mem_pool);
		break;
	case MT_MD_EXE_DELETE:
		if (exceed_cpu_used) {
			log_warn("cpu使用率超出,拒绝UPDATE QUERY DELETE服务");
			break;
		}
		mem_pool = mem_pool_init(MB_SIZE);
		parsed_packet = parse_delete_packet(mem_pool, rbuf);
		plan = SqlAnalyzer::Analyze(StorageEngine::storage,
				(struct delete_packet*) parsed_packet, mem_pool);
		if (plan == NULL) {
			result_code = -1;
		} else {
			result = plan->Execute(result_code, deadline_time);
			plan->~ExecutePlan();
		}

		if (result_code == MILE_RETURN_SUCCESS) {
			gen_delete_result_packet(*(uint32_t*) result, msg_head, sbuf);
		}
		mem_pool_destroy(mem_pool);
		break;
	case MT_MD_EXE_EXPORT: {
		mem_pool = mem_pool_init(MB_SIZE);
		parsed_packet = parse_export_packet(mem_pool, rbuf);
		plan = SqlAnalyzer::Analyze(StorageEngine::storage,
				(struct export_packet *) parsed_packet, mem_pool);
		if (plan == NULL) {
			result_code = -1;
		} else {
			result = plan->Execute(result_code, deadline_time);
			plan->~ExecutePlan();
		}

		if (result_code == MILE_RETURN_SUCCESS) {
			gen_export_result_packet(*(uint64_t*) result, msg_head, sbuf);
		}

		mem_pool_destroy(mem_pool);
	}
		break;
	case MT_MD_EXE_UPDATE:
		if (exceed_cpu_used) {
			log_warn("cpu使用率超出,拒绝UPDATE QUERY DELETE服务");
			break;
		}
		mem_pool = mem_pool_init(MB_SIZE);
		parsed_packet = parse_update_packet(mem_pool, rbuf);
		plan = SqlAnalyzer::Analyze(StorageEngine::storage,
				(struct update_packet*) parsed_packet, mem_pool);
		if (plan == NULL) {
			result_code = -1;
		} else {
			result = plan->Execute(result_code, deadline_time);
			plan->~ExecutePlan();
		}

		if (result_code == MILE_RETURN_SUCCESS) {
			gen_update_result_packet(*(uint32_t*) result, msg_head, sbuf);
		}
		mem_pool_destroy(mem_pool);
		break;
	case MT_MD_EXE_QUERY:
		if (exceed_cpu_used) {
			log_warn("cpu使用率超出,拒绝UPDATE QUERY DELETE服务");
			break;
		}
		mem_pool = mem_pool_init(MB_SIZE);

		result_code = loadavg_check(); // check overloading
		if (ERROR_OS_OVERLOADING == result_code) {
			log_warn(
					"OS overloading, discard packet msg id %"PRIu32, msg_head->message_id);
		} else if (0 == result_code) {
			parsed_packet = parse_query_packet(mem_pool, rbuf);
			plan = SqlAnalyzer::Analyze(StorageEngine::storage,
					(struct query_packet*) parsed_packet, mem_pool);
			if (plan == NULL) {
				result_code = -1;
			} else {
				result = plan->Execute(result_code, deadline_time);
				plan->~ExecutePlan();
			}
		}

		if (result_code == MILE_RETURN_SUCCESS) {
			gen_query_result_packet((ResultSet*) result, msg_head, sbuf);
		}
		mem_pool_destroy(mem_pool);
		break;
	case MT_MD_GET_STATE:
		mem_pool = mem_pool_init(KB_SIZE);
		parsed_packet = parse_get_state_packet(mem_pool, rbuf);
		result = StateManager::QueryStates(StorageEngine::storage,
				(struct get_state_packet*) parsed_packet, mem_pool);
		if (result == NULL) {
			result_code = ERROR_GET_DOCSERVER_STATE;
		} else {
			gen_state_result_packet((struct stat_info_array*) result, msg_head,
					sbuf);
		}
		mem_pool_destroy(mem_pool);
		break;
	default:
		result_code = StorageEngine::storage->SpecialSql(msg_head, rbuf, sbuf);
		break;
	}

	if (result_code < 0) {
		gen_error_packet(result_code, msg_head, sbuf);
	}
	return result_code;
}
Exemplo n.º 27
0
	virtual void TearDown()
	{
		DELETE(engine_);
		mem_pool_destroy(mem_);
		ClearDir();
	}
Exemplo n.º 28
0
rbthash_table_t *
rbthash_table_init (int buckets, rbt_hasher_t hfunc,
                    rbt_data_destroyer_t dfunc,
                    unsigned long expected_entries,
                    struct mem_pool *entrypool)
{
        rbthash_table_t         *newtab = NULL;
        int                     ret = -1;

        if (!hfunc) {
                gf_log (GF_RBTHASH, GF_LOG_ERROR, "Hash function not given");
                return NULL;
        }

        if (!entrypool && !expected_entries) {
                gf_log (GF_RBTHASH, GF_LOG_ERROR,
                        "Both mem-pool and expected entries not provided");
                return NULL;
        }

        if (entrypool && expected_entries) {
                gf_log (GF_RBTHASH, GF_LOG_ERROR,
                        "Both mem-pool and expected entries are provided");
                return NULL;
        }


        newtab = GF_CALLOC (1, sizeof (*newtab),
                            gf_common_mt_rbthash_table_t);
        if (!newtab)
                return NULL;

        newtab->buckets = GF_CALLOC (buckets, sizeof (struct rbthash_bucket),
                                     gf_common_mt_rbthash_bucket);
        if (!newtab->buckets) {
                goto free_newtab;
        }

        if (expected_entries) {
                newtab->entrypool =
                        mem_pool_new (rbthash_entry_t, expected_entries);
                if (!newtab->entrypool) {
                        gf_log (GF_RBTHASH, GF_LOG_ERROR,
                                "Failed to allocate mem-pool");
                        goto free_buckets;
                }
                newtab->pool_alloced = _gf_true;
        } else {
                newtab->entrypool = entrypool;
        }

        LOCK_INIT (&newtab->tablelock);
        newtab->numbuckets = buckets;
        ret = __rbthash_init_buckets (newtab, buckets);

        if (ret == -1) {
                gf_log (GF_RBTHASH, GF_LOG_ERROR, "Failed to init buckets");
                if (newtab->pool_alloced)
                        mem_pool_destroy (newtab->entrypool);
        } else {
                gf_log (GF_RBTHASH, GF_LOG_TRACE, "Inited hash table: buckets:"
                        " %d", buckets);
        }

        newtab->hashfunc = hfunc;
        newtab->dfunc = dfunc;

free_buckets:
        if (ret == -1)
                GF_FREE (newtab->buckets);

free_newtab:
        if (ret == -1) {
                GF_FREE (newtab);
                newtab = NULL;
        }

        return newtab;
}