コード例 #1
0
ファイル: client.c プロジェクト: Alibaba-boonya/mile
int32_t execute_common_handle(struct data_buffer* dbuf,int socket_fd,MEM_POOL* mem_pool)
{
	struct data_buffer rbuf;
	
	//发数据包
	if(socket_write(socket_fd,dbuf->data,dbuf->data_len) <= 0)
		return -1;

	//接受命名
	memset(&rbuf,0,sizeof(struct data_buffer));
	rbuf.data_len = sizeof(uint32_t);
	rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,sizeof(uint32_t));
	rbuf.size = rbuf.data_len;
	
	if(socket_read(socket_fd,rbuf.data,sizeof(uint32_t)) <= 0)
		return -1;

	rbuf.data_len = read_int32(&rbuf);
	rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,rbuf.data_len);
	rbuf.rpos = 0;
	rbuf.size = rbuf.data_len;

	if(socket_read(socket_fd,rbuf.data,rbuf.data_len) <= 0)
		return -1;
	rbuf.rpos += MESSAGE_HEAD_LEN;

	uint16_t type = message_type(&rbuf);
	if (type != MT_DC_EXE_RS)
		return -1;

	return read_int32(&rbuf);
}
コード例 #2
0
ファイル: AggrFunc.cpp プロジェクト: Alibaba-boonya/mile
int32_t CountDistinct(MEM_POOL_PTR mem_pool, struct low_data_struct *cur_value,
		struct low_data_struct* new_value) {

	if (cur_value == NULL || new_value == NULL) {
		return -1;
	}

	if (cur_value->data == NULL) {
		cur_value->type = HI_TYPE_SET;
		cur_value->len = 0;
		Equals * equals =
				new (mem_pool_malloc(mem_pool, sizeof(LDEquals))) LDEquals();
		HashCoding* hash =
				new (mem_pool_malloc(mem_pool, sizeof(LDHash))) LDHash();
		cur_value->data =
				new (mem_pool_malloc(mem_pool, sizeof(HashSet))) HashSet(equals,
						hash, mem_pool);
	}

	HashSet* set = (HashSet*) cur_value->data;
	if (set->Contains(new_value)) {
		return MILE_RETURN_SUCCESS;
	} else if (new_value->type != HI_TYPE_NULL) {
		struct low_data_struct* ld = (struct low_data_struct*) mem_pool_malloc(
				mem_pool, sizeof(struct low_data_struct));
		memset(ld, 0, sizeof(struct low_data_struct));
		copy_low_data_struct(mem_pool, ld, new_value);
		set->Add(ld);
		cur_value->len++;
	}

	return MILE_RETURN_SUCCESS;
}
コード例 #3
0
ファイル: AggrFunc.cpp プロジェクト: Alibaba-boonya/mile
int32_t Avg(MEM_POOL_PTR mem_pool, struct low_data_struct* cur_value,
		struct low_data_struct* new_value) {
	struct low_data_struct *sum, *count;
	MileArray* array;
	double a, b;

	if (cur_value == NULL || new_value == NULL) {
		return -1;
	}

	if (new_value->type != HI_TYPE_NULL && new_value->len > 0) {
		if (cur_value->data == NULL) {
			array =
					new (mem_pool_malloc(mem_pool, sizeof(MileArray))) MileArray(
							mem_pool);
			cur_value->type = HI_TYPE_ARRAY;
			cur_value->len = 2;
			cur_value->data = array;
			//initial the sum
			sum = (struct low_data_struct*) mem_pool_malloc(mem_pool,
					sizeof(struct low_data_struct));
			memset(sum, 0, sizeof(struct low_data_struct));
			sum->type = HI_TYPE_DOUBLE;
			sum->len = sizeof(double);
			sum->data = mem_pool_malloc(mem_pool, sizeof(double));
			*(double*) sum->data = 0;
			array->Add(sum);

			//initial the count
			count = (struct low_data_struct*) mem_pool_malloc(mem_pool,
					sizeof(struct low_data_struct));
			memset(count, 0, sizeof(struct low_data_struct));
			count->type = HI_TYPE_LONGLONG;
			count->len = sizeof(uint64_t);
			count->data = mem_pool_malloc(mem_pool, sizeof(uint64_t));
			*(uint64_t*) count->data = 0;
			array->Add(count);
		}

		array = (MileArray*) cur_value->data;
		sum = (struct low_data_struct*) array->Get(0);
		count = (struct low_data_struct*) array->Get(1);
		*(uint64_t*) count->data = *(uint64_t*) count->data + 1;

		a = ld_to_double(new_value);
		b = *(double*) sum->data;
		if (isnan(a)) {
			*(double*) sum->data = a;
			if (new_value->field_name != NULL) {
				log_error("%s列的类型不能转换为double, 无法进行avg操作", new_value->field_name);
			}
		} else if (!isnan(b)) {
			*(double*) sum->data = *(double*) sum->data + a;
		}

	}

	return MILE_RETURN_SUCCESS;
}
コード例 #4
0
ファイル: hi_db_mock.c プロジェクト: Alibaba-boonya/mile
/**
  * 在btree中做范围查询,在所有数据范围内查询
  * @param  tid 表号 
  * @param  field_id 域号
  * @param  range_condition 查询的条件
  * @param  mem_pool 内存池
  * @return 成功返回list head,上层可以通过这个遍历,取到所有段结果,失败返回NULL
  **/ 
struct list_head * db_btree_query_range(uint8 tid,	uint16 field_id, \
		struct db_range_query_condition *range_condition, MEM_POOL *pMemPool)
{
	int32 i, j, k, n;
	int8 cmp_res;

	//初始化头
	struct list_head* list_h = (struct list_head*)mem_pool_malloc(pMemPool,sizeof(struct list_head));
	
	INIT_LIST_HEAD(list_h);

	struct segment_query_rowids* seg_rowids; 

	n = 0;
	for(i = 0; i <= current_seg; i++)
	{
		if(i == current_seg)
		{
			k = current_row;
		}else{
			k = TEST_MAX_ROW_NUM;
		}

		seg_rowids = (struct segment_query_rowids*) mem_pool_malloc(pMemPool, sizeof(struct segment_query_rowids));
		seg_rowids->sid = i;
		seg_rowids->rowids = rowid_list_init(pMemPool);
		list_add(&seg_rowids->rowids_list, list_h);

		for(j = k-1; j >= 0; j--)
		{
			if(range_condition->min_flag >= 0)
			{
				cmp_res = compare(mock_data_source[i][j][field_id].data, range_condition->min_key->data, mock_fields_type_array[field_id]);
				if((cmp_res <= 0 && range_condition->min_flag == 1) || (cmp_res < 0 && range_condition->min_flag == 0))
				{
					continue;
				}
			}

			if(range_condition->max_flag >= 0)
			{
				cmp_res = compare(mock_data_source[i][j][field_id].data, range_condition->max_key->data, mock_fields_type_array[field_id]);
				if((cmp_res >= 0 && range_condition->max_flag == 1) || (cmp_res > 0 && range_condition->max_flag == 0))
				{
					continue;
				}
			}

			rowid_list_add(pMemPool, seg_rowids->rowids, j);
			n++;
		}
			
	}


	return list_h;	
}
コード例 #5
0
ファイル: client.c プロジェクト: Alibaba-boonya/mile
int32_t execute_index_stat_handle(struct data_buffer* dbuf,int socket_fd,MEM_POOL* mem_pool)
{	
	struct data_buffer rbuf;
	
	//发一个数据包长度
	if(socket_write(socket_fd,dbuf->data,dbuf->data_len) <= 0)
		return -1;

	//接受命名
	memset(&rbuf,0,sizeof(struct data_buffer));
	rbuf.data_len = sizeof(uint32_t);
	rbuf.size = rbuf.data_len;
	rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,sizeof(uint32_t));
	
	if(socket_read(socket_fd,rbuf.data,rbuf.data_len) <= 0)
		return -1;
	
	rbuf.data_len = read_int32(&rbuf);
	rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,rbuf.data_len);
	rbuf.rpos = 0;
	rbuf.size = rbuf.data_len;
	
	if(socket_read(socket_fd,rbuf.data,rbuf.data_len) <= 0) 
		return -1;

	rbuf.rpos += MESSAGE_HEAD_LEN;
	uint16_t type = message_type(&rbuf);
	if (type != MT_DC_STAT_RS)
		return -1;

	uint16_t field_count = read_int16(&rbuf);

	uint16_t i;
	char* field_name = NULL;
	fprintf(stderr,"%-20s%-20s%-20s%-20s\n","FIELD_NAME","INDEX_TYPE","DATA_TYPE","FLAG");
	uint8_t j;
	uint8_t index_num;
	for(i=0; i<field_count; i++)
	{
		field_name = read_cstring(&rbuf,mem_pool);
		index_num = read_int8(&rbuf);
		for(j=0;j < index_num; j++)
		{
			fprintf(stderr,"%-20s",field_name);
			fprintf(stderr,"%-20u",read_int8(&rbuf));
			fprintf(stderr,"%-20u",read_int8(&rbuf));
			fprintf(stderr,"%-20u\n",read_int8(&rbuf));
		}
	}

	return 0;
}
コード例 #6
0
ファイル: hashindex_test.cpp プロジェクト: rzs840707/mile
void get_low_data(struct low_data_struct* data,MEM_POOL* mem_pool)
{
    data->len = 5;
    data->data = mem_pool_malloc(mem_pool,5);
    data->type = HI_TYPE_STRING;
    data->field_name = (char*)mem_pool_malloc(mem_pool,20);
    memset(data->field_name,0,20);
    strcpy(data->field_name,"HI_TYPE_STRING");
    memset(data->data,0,5);
    strcpy((char*)data->data,"ali");

    return;
}
コード例 #7
0
ファイル: hi_db_mock.c プロジェクト: Alibaba-boonya/mile
/**
  * 根据rowid来查找对应的value,只有filter索引类型支持
  * @param  tid 表号   
  * @param  sid 段号
  * @param  row_id 行号
  * @param  field_id 域号
  * @param  mem_pool 内存池
  * @return 成功返回rowid_list,失败返回NULL
  **/ 
struct low_data_struct* db_query_by_rowid(uint8 tid, uint16 sid,uint32 row_id,uint16 field_id, MEM_POOL* mem_pool)
{
	if(mock_delete_mask[sid][row_id])
	{
		return NULL;
	}
	struct low_data_struct* result = (struct low_data_struct*) mem_pool_malloc(mem_pool, sizeof(struct low_data_struct));
	result->len = mock_data_source[sid][row_id][field_id].len;
	result->data = mem_pool_malloc(mem_pool, result->len);

	memcpy(result->data, mock_data_source[sid][row_id][field_id].data, result->len);
	return result;

}
コード例 #8
0
ファイル: vstorage.c プロジェクト: Alibaba-boonya/mile
struct vstorage_manager * vstorage_init(struct vstorage_config * config,MEM_POOL* mem_pool)
{
   struct vstorage_manager * vstorage = NULL;
   char* memap_address = NULL;
   char old_loc_name[FILENAME_MAX_LENGTH],new_loc_name[FILENAME_MAX_LENGTH];
   vstorage = (struct vstorage_manager *)mem_pool_malloc(mem_pool,sizeof(struct vstorage_manager));
   memset(vstorage,0,sizeof(struct vstorage_manager));

   //用于存储数据信息的文件
   sprintf(vstorage->data_file_name,"%s/%s.dat",config->work_space,config->vstorage_name);

   //打开文件
   vstorage->data_fd = open_file(vstorage->data_file_name,O_RDWR | O_CREAT | O_APPEND);
   if(vstorage->data_fd < 0) {
       return NULL;
   }
	  
  //分配用于存储位置信息的偏移量,需要memap到内存中
  vstorage->row_limit = config->row_limit;
  //拼接
  //  sprintf(vstorage->index_file_name,"%s/%s.loc",config->work_space,config->vstorage_name);
  sprintf(old_loc_name,"%s/%s.loc",config->work_space,config->vstorage_name);
  sprintf(new_loc_name,"%s/%s.loc4",config->work_space,config->vstorage_name);
  
  if(access(old_loc_name, W_OK) == 0)//if old index file exist then use 2 bytes length
  {
      strcpy(vstorage->index_file_name , old_loc_name);
      //memap到内存中
        
      memap_address = (char* )get_mmap_memory(vstorage->index_file_name, VSTORAGE_MMAP_SIZE(vstorage));
        
      assert(memap_address != NULL);
        
      //留8字节给偏移量
      //这个偏移量主要用来记录文件当前写入的位置
      vstorage->loc_info = (struct locate_info*)(memap_address+sizeof(uint64_t));
      
      vstorage->loc_info4 = NULL;
     
  } else { //else use 4 bytes length
 
      strcpy(vstorage->index_file_name , new_loc_name);
 
      //memap到内存中
  
      memap_address =(char* )get_mmap_memory(vstorage->index_file_name, VSTORAGE_MMAP_SIZE4(vstorage)); 

      assert(memap_address != NULL);

      //留8字节给偏移量
      //这个偏移量主要用来记录文件当前写入的位置
      vstorage->loc_info4 = (struct locate_info4*)(memap_address+sizeof(uint64_t));
      
      vstorage->loc_info = NULL;
   }
  //初始化偏移量
  vstorage->offset = (uint64_t*)memap_address;
  
  return vstorage;
}
コード例 #9
0
void verify_muti_hashvalue(struct hash_index_manager* hash_index,MEM_POOL* mem_pool)
{
	//查询性能测试
	struct rowid_list* rlist_a;
	struct rowid_list* rlist_b;
	uint64_t time_a = 0;
	uint64_t time_b = 0;
	uint64_t now;
	uint32_t i;

	struct hash_compress_manager* hash_compress = hash_compress_load(hash_index,10,mem_pool);

	struct low_data_struct* data = (struct low_data_struct*)mem_pool_malloc(mem_pool,sizeof(struct low_data_struct));
	get_low_data(data,mem_pool);
	 
	for(i=0; i<hash_index->limit;i++)
	{	sprintf((char*)data->data,"%d",i);
	
		now = get_time_usec();
		rlist_a = hash_index_query(hash_index,data,mem_pool);
		time_a += get_time_usec() - now;
	
		now = get_time_usec();
		rlist_b = hash_compress_query(hash_compress,data,mem_pool);
		time_b += get_time_usec()-now;
	
		ASSERT_EQ(rowid_list_equals(rlist_a,rlist_b),1);
	}
	
	
	 printf("hash_index %llu hash_compress %llu\n",time_a,time_b);
	 hash_compress_release(hash_compress);
}
コード例 #10
0
ファイル: rowid_list.c プロジェクト: Alibaba-boonya/mile
/**
 * 向rowid_list中添加新的rowid,注意数据库中的rowid链表是降序的
 *
 * @param pMemPool		内存池
 * @param id_list		rowid的链表
 * @param rowid
 */
void rowid_list_add(MEM_POOL_PTR pMemPool, struct rowid_list* id_list,
		uint32_t rowid) {
	if (id_list == NULL ) {
		return;
	}

	if (id_list->head == NULL || id_list->rowid_num % ROWID_ARRAY_SIZE == 0) {
		struct rowid_list_node* tmp = (struct rowid_list_node*) mem_pool_malloc(
				pMemPool, sizeof(struct rowid_list_node));
		memset(tmp, 0, sizeof(struct rowid_list_node));

		tmp->rowid_array[0] = rowid;
		tmp->next = NULL;
		id_list->rowid_num++;
		if (id_list->head == NULL ) {
			id_list->head = tmp;
			id_list->tail = tmp;
		} else {
			id_list->tail->next = tmp;
			id_list->tail = tmp;
		}
	} else {
		id_list->tail->rowid_array[id_list->rowid_num % ROWID_ARRAY_SIZE] =
				rowid;
		id_list->rowid_num++;
	}

}
コード例 #11
0
ファイル: AggrFunc.cpp プロジェクト: Alibaba-boonya/mile
int32_t SquareSum(MEM_POOL_PTR mem_pool, struct low_data_struct* cur_value,
		struct low_data_struct* new_value) {
	double a, b;

	if (cur_value == NULL || new_value == NULL) {
		return -1;
	}

	if (new_value->type != HI_TYPE_NULL && new_value->len != 0) {
		if (cur_value->type == HI_TYPE_NULL || cur_value->len == 0) {
			cur_value->type = HI_TYPE_DOUBLE;
			cur_value->len = sizeof(double);
			cur_value->data = mem_pool_malloc(mem_pool, sizeof(double));
			*(double*) cur_value->data = 0;
		}

		a = ld_to_double(new_value);
		b = *(double*) cur_value->data;
		if (isnan(a)) {
			*(double*) cur_value->data = a;
			if (new_value->field_name != NULL) {
				log_error("%s列的类型不能转换为double, 无法进行squaresum操作", new_value->field_name);
			}
		} else if (!isnan(b)) {
			*(double*) cur_value->data = b + a * a;
		}
	}

	return MILE_RETURN_SUCCESS;
}
コード例 #12
0
ファイル: storage.c プロジェクト: Alibaba-boonya/mile
struct storage_manager * storage_init(struct storage_config* config,MEM_POOL* mem_pool)
{
   struct storage_manager* storage = 
   				(struct storage_manager *)mem_pool_malloc(mem_pool,sizeof(struct storage_manager));

   assert(storage != NULL);
   
   memset(storage,0,sizeof(struct storage_manager));

   //初始化block信息
   storage->value_size = config->unit_size;
   storage->row_limit = config->row_limit;

   //拼接:文件名
   sprintf(storage->file_name,"%s/%s.dat",config->work_space,config->storage_name);

   //mmap映射处理
   storage->mem_mmaped =(char*)get_mmap_memory(storage->file_name,STORAGE_MMAP_SIZE(storage)); 

   assert(storage->mem_mmaped != NULL);

   //初始化空值标志位
   struct bitmark_config bitmark_config;
   strcpy(bitmark_config.work_space,config->work_space);
   strcpy(bitmark_config.bitmark_name,"null");
   bitmark_config.row_limit = config->row_limit;

   storage->null_bitmark = bitmark_init(&bitmark_config,mem_pool);
   
   return storage;   
}
コード例 #13
0
ファイル: rowid_list.c プロジェクト: Alibaba-boonya/mile
/**
 * 向rowid_list中添加新的rowid,注意数据库中的rowid链表是降序的
 *
 * @param pMemPool		内存池
 * @param id_list		rowid的链表
 * @param rowid
 */
void rowid_list_batch_add(MEM_POOL_PTR pMemPool, struct rowid_list* id_list,
		uint32_t* rowids, uint32_t num) {
	uint32_t insert_num = 0;
	uint32_t total = num;
	if (id_list == NULL ) {
		return;
	}

	while (num != 0) {
		insert_num = num / ROWID_ARRAY_SIZE > 0 ? ROWID_ARRAY_SIZE : num;
		struct rowid_list_node* tmp = (struct rowid_list_node*) mem_pool_malloc(
				pMemPool, sizeof(struct rowid_list_node));
		memset(tmp, 0, sizeof(struct rowid_list_node));


		memcpy(tmp->rowid_array, rowids + total - num,
				insert_num * sizeof(uint32_t));

		tmp->next = NULL;
		if (id_list->head == NULL ) {
			id_list->head = tmp;
			id_list->tail = tmp;
		} else {
			id_list->tail->next = tmp;
			id_list->tail = tmp;
		}
		num -= insert_num;
		id_list->rowid_num += insert_num;
	}
}
コード例 #14
0
TEST(DYHASH_CHECKPOINT2, dyhashindex){
	MEM_POOL* mem_pool = mem_pool_init(MB_SIZE);

	struct dyhash_index_config config;
	
	char dir_path[] = "/tmp/dyhash_index3";

	config.is_full = 0;
	config.row_limit = 2;
	config.is_full = 1;
	strcpy(config.work_space, dir_path);
	

    init_profile(1000,mem_pool);
	
	struct dyhash_index_manager* dyhash_index = dyhash_index_init(&config,mem_pool);

	struct low_data_struct data;
	data.data = mem_pool_malloc(mem_pool, strlen("hello world"));
	memset(data.data, 0, strlen("hello world"));
	strcpy((char*)data.data, "hello world");
	data.len = strlen("hello world");
	data.type = HI_TYPE_STRING;


	uint32_t count = dyhash_index_count_query(dyhash_index, &data, mem_pool);
	ASSERT_EQ(1, count);

	
	struct rowid_list* query_ret = dyhash_index_query(dyhash_index, &data, mem_pool);
	print_rowid_list(query_ret);
}
コード例 #15
0
void test_multi_hashvalue(uint32_t limit)
{
   MEM_POOL* mem_pool = mem_pool_init(MB_SIZE);
   int32_t ret;
   struct hash_index_config config;
   config.row_limit = limit;
   strcpy(config.work_space,"/tmp/hash_compress_test");
   system("rm -rf /tmp/hash_compress_test");
   mkdirs(config.work_space);
   init_profile(1000,mem_pool);
   
   struct hash_index_manager* hash_index = hash_index_init(&config,mem_pool);


   uint32_t i;
   struct low_data_struct* data = (struct low_data_struct*)mem_pool_malloc(mem_pool,sizeof(struct low_data_struct));
   get_low_data(data,mem_pool);
   
   for(i=0; i<config.row_limit;i++)
   {   sprintf((char*)data->data,"%d",i);
	   ret = hash_index_insert(hash_index,data,i);
	   ASSERT_EQ(0,ret);
   }

   verify_muti_hashvalue(hash_index,mem_pool);
   hash_index_release(hash_index);
}
コード例 #16
0
ファイル: client.c プロジェクト: Alibaba-boonya/mile
int32_t execute_segment_stat_handle(struct data_buffer* dbuf,int socket_fd,MEM_POOL* mem_pool)
{
	struct data_buffer rbuf;
	
	//发数据包
	if(socket_write(socket_fd,dbuf->data,dbuf->data_len) <= 0)
		return -1;

	//接受命名
	memset(&rbuf,0,sizeof(struct data_buffer));
	rbuf.data_len = sizeof(uint32_t);
	rbuf.size = rbuf.data_len;
	rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,sizeof(uint32_t));
	if(socket_read(socket_fd,rbuf.data,rbuf.data_len) <= 0)
		return -1;

	rbuf.data_len = read_int32(&rbuf);
	rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,rbuf.data_len);
	rbuf.rpos = 0;
	rbuf.size = rbuf.data_len;

	if(socket_read(socket_fd,rbuf.data,rbuf.data_len) <= 0)
		return -1;
	
	rbuf.rpos += MESSAGE_HEAD_LEN;
	uint16_t type = message_type(&rbuf);
	if (type != MT_DC_STAT_RS)
		return -1;

	uint16_t segment_count = read_int16(&rbuf);

	uint16_t i;
	fprintf(stderr,"%-20s%-20s%-20s%-20s%-20s%-20s%-20s\n","SEGMENT_ID","CREATE_TIME","MODIFY_TIME","CHECKPOINT_TIME","FLAG","ROW_COUNT","DEL_COUNT");
	for(i=0; i<segment_count; i++)
	{
		fprintf(stderr,"%-20u",read_int16(&rbuf));
		fprintf(stderr,"%-20llu",read_int64(&rbuf));
		fprintf(stderr,"%-20llu",read_int64(&rbuf));
		fprintf(stderr,"%-20llu",read_int64(&rbuf));
		fprintf(stderr,"%-20u",read_int8(&rbuf));
		fprintf(stderr,"%-20u",read_int32(&rbuf));
		fprintf(stderr,"%-20u\n",read_int32(&rbuf));
	}
	
	return 0;
}
コード例 #17
0
ファイル: rowid_list.c プロジェクト: Alibaba-boonya/mile
struct rowid_list* rowid_list_init(MEM_POOL_PTR pMemPool) {
	struct rowid_list* id_list = (struct rowid_list*) mem_pool_malloc(pMemPool,
			sizeof(struct rowid_list));
	id_list->rowid_num = 0;
	id_list->head = NULL;
	id_list->tail = NULL;
	return id_list;
}
コード例 #18
0
ファイル: DocEngine.cpp プロジェクト: Alibaba-boonya/mile
ResultSet* DocEngine::GetKvs(struct get_kvs_packet* packet,
		MEM_POOL_PTR mem_pool) {
	int32_t result_code = MILE_RETURN_SUCCESS;
	uint32_t i, j;
	uint16_t sid;
	uint32_t rowid;
	struct select_row_t* row;
	struct low_data_struct** sel_fields;

	Clonable* clone =
			new (mem_pool_malloc(mem_pool, sizeof(RowClone))) RowClone(
					mem_pool);
	CommonRS* result =
			new (mem_pool_malloc(mem_pool, sizeof(CommonRS))) CommonRS(packet->docid_num, clone,
					mem_pool);
	char** field_names = (char**) mem_pool_malloc(mem_pool,
			sizeof(char*) * packet->select_field.n);
	for (i = 0; i < packet->select_field.n; i++) {
		field_names[i] = packet->select_field.select_fields[i].field_name;
	}

	for (i = 0; i < packet->docid_num; i++) {
		sid = packet->docids[i] >> 32;
		rowid = (uint32_t)packet->docids[i];

		sel_fields = db_data_query_multi_col(packet->table_name, sid, rowid, field_names,
				packet->select_field.n, DATA_ACCESS_ORIGINAL, mem_pool);
		if(NULL == sel_fields){
			return NULL;
		}

		row = init_select_row_t(mem_pool, packet->select_field.n);
		for(j = 0; j < packet->select_field.n; j++){
			row->data[j] = *sel_fields[j];
			row->select_type[j] = SELECT_TYPE_ORIGINAL;
		}
		row->handler = new(mem_pool_malloc(mem_pool, sizeof(DocHandler)))DocHandler(packet->docids[i]);

		result->AddResult(row);
	}

	return result;
}
コード例 #19
0
ファイル: hi_db_mock.c プロジェクト: Alibaba-boonya/mile
/**
  * 根据value来查找对应的doc id lists,只有hash索引支持,所有段扫
  * @param  tid 表号
  * @param  field_id 域号
  * @param  data 数据
  * @param  mem_pool 内存池
  * @return 成功返回list head,上层可以通过这个遍历,取到所有段结果,失败返回NULL
  **/ 
struct list_head* db_query_by_value(uint8 tid,uint16 field_id,struct low_data_struct* data, MEM_POOL* mem_pool)
{
	int32 i, j, k, n;


	//初始化头
	struct list_head* list_h = (struct list_head*)mem_pool_malloc(mem_pool,sizeof(struct list_head));
	
	INIT_LIST_HEAD(list_h);

	struct segment_query_rowids* seg_rowids; 

	n = 0;
	for(i = 0; i <= current_seg; i++)
	{
		if(i == current_seg)
		{
			k = current_row;
		}else{
			k = TEST_MAX_ROW_NUM;
		}

		seg_rowids = (struct segment_query_rowids*) mem_pool_malloc(mem_pool, sizeof(struct segment_query_rowids));
		seg_rowids->sid = i;
		seg_rowids->rowids = rowid_list_init(mem_pool);
		list_add(&seg_rowids->rowids_list, list_h);

		for(j = k-1; j >= 0; j--)
		{
			if(compare(data->data, mock_data_source[i][j][field_id].data, mock_fields_type_array[field_id]) == 0)
			{
				rowid_list_add(mem_pool, seg_rowids->rowids, j);
				n++;
			}
		}
			
	}


	return list_h;
}
コード例 #20
0
ファイル: storage.c プロジェクト: Alibaba-boonya/mile
struct low_data_struct * storage_query(struct storage_manager * storage, uint32_t docid,MEM_POOL* mem_pool)
{
    struct low_data_struct* result = NULL;

	result = (struct low_data_struct*)mem_pool_malloc(mem_pool,sizeof(struct low_data_struct));
	memset(result,0,sizeof(struct low_data_struct));

	result->data = mem_pool_malloc(mem_pool,storage->value_size);
	memset(result->data,0,storage->value_size);

	//如果支持空值,且标记为0的话,说明此rowid的数据为空
	if(bitmark_query(storage->null_bitmark,docid) == 1)
	{
		return result;
	}

	//根据row_id*每个数据的长度,可以算到row_id对应的数据
	memcpy(result->data,storage->mem_mmaped+docid*storage->value_size,storage->value_size);
	result->len = storage->value_size;
	return result;
}
コード例 #21
0
TEST(DATA_BUFFER_TEST, READ_AND_WRITE){

	MEM_POOL_PTR pMemPool = mem_pool_init(MB_SIZE);
	struct data_buffer* buffer = init_data_buffer();
	buffer->data_len = 256;
	buffer->data = (uint8*) mem_pool_malloc(pMemPool, 256);

	uint8 tmp8;
	uint16 tmp16;
	uint32 tmp32;
	uint64 tmp64;

	char* str = (char*) mem_pool_malloc(pMemPool, 100);
	char* result = (char*) mem_pool_malloc(pMemPool, 100);
	strcpy(str, "hello world!");

	
	write_int8(122, buffer);
	write_int16(1111, buffer);
	write_int32(324, buffer);
	write_int64(2321, buffer);
	write_bytes((uint8*)str, strlen(str)+1, buffer);

	

	tmp8 = read_int8(buffer);
	ASSERT_EQ(tmp8, 122);
	tmp16 = read_int16(buffer);
	ASSERT_EQ(tmp16, 1111);
	tmp32 = read_int32(buffer);
	ASSERT_EQ(tmp32, 324);
	tmp64 = read_int64(buffer);
	ASSERT_EQ(tmp64, 2321);

	read_bytes(buffer, (uint8*)result, strlen(str)+1);
	ASSERT_EQ(0, strcmp(str, result));

	destroy_data_buffer(buffer);
	mem_pool_destroy(pMemPool);
}
コード例 #22
0
ファイル: unit_dump_test.cpp プロジェクト: rzs840707/mile
void test_with_index_dump(MEM_POOL* mem_pool)
{
    system("rm -rf /tmp/binlog");
    system("rm -rf /tmp/log");
    system("rm -rf /tmp/storage");

    db_initialize(mem_pool);

    //构造数据
    uint16 sid = 0;
    uint32 docid = 0;
    int32 ret;

    uint16 field_count = 5;
    enum field_types types[5] = {HI_TYPE_STRING,HI_TYPE_LONG,HI_TYPE_TINY,HI_TYPE_LONGLONG,HI_TYPE_DOUBLE};

    //将第一个段填满
    while(sid == 0)
    {
        ret = db_insert("hello", &sid, &docid,get_row_data(field_count,types,mem_pool), DOCID_BY_SELF, mem_pool);
        ASSERT_EQ(MILE_RETURN_SUCCESS,ret);
    }

    sid = 0;

    //建hash索引,然后unload load 验证
    ret = db_ensure_index("hello","HI_TYPE_STRING",HI_KEY_ALG_HASH,HI_TYPE_STRING, mem_pool);
    ASSERT_EQ(MILE_RETURN_SUCCESS,ret);

    struct low_data_struct* data = (struct low_data_struct*)mem_pool_malloc(mem_pool,sizeof(struct low_data_struct));
    get_low_data(data,HI_TYPE_STRING,mem_pool);

    struct list_head* index_equal_ret = db_index_equal_query("hello",get_time_hint(mem_pool),data,mem_pool);

    struct segment_query_rowids* node;
    list_for_each_entry(node,index_equal_ret,rowids_list) {
        if(node->sid == 0)
            ASSERT_EQ(node->rowids->rowid_num,10);
        else if(node->sid == 1)
            ASSERT_EQ(node->rowids->rowid_num,1);
        else
            FAIL();
    }

    ret = db_unload_segment("hello",sid,mem_pool);
    ASSERT_EQ(MILE_RETURN_SUCCESS,ret);

    index_equal_ret = db_index_equal_query("hello",get_time_hint(mem_pool),data,mem_pool);
    list_for_each_entry(node,index_equal_ret,rowids_list) {
        ASSERT_EQ(node->rowids->rowid_num,1);
        ASSERT_EQ(node->sid,1);
    }
コード例 #23
0
void test_tow_hashvalue(uint32_t limit)
{
   MEM_POOL* mem_pool = mem_pool_init(MB_SIZE);
   struct hash_index_config config;
   config.row_limit = limit;
   strcpy(config.work_space,"/tmp/hash_compress_test");
   system("rm -rf /tmp/hash_compress_test");
   mkdirs(config.work_space);
   init_profile(1000,mem_pool);
   
   struct hash_index_manager* hash_index = hash_index_init(&config,mem_pool);


   uint32_t i;
   struct low_data_struct* data = (struct low_data_struct*)mem_pool_malloc(mem_pool,sizeof(struct low_data_struct));
   get_low_data(data,mem_pool);
   
   for(i=0; i<config.row_limit/2;i++)
   {   
   	   strcpy((char*)data->data,"1");
	   hash_index_insert(hash_index,data,i);
   }

   for(i=config.row_limit/2; i<config.row_limit;i++)
   {   
   	   strcpy((char*)data->data,"2");
	   hash_index_insert(hash_index,data,i);
   }

   	struct hash_compress_manager* hash_compress = hash_compress_load(hash_index,10,mem_pool);
	
   	struct rowid_list* rlist_a;
	struct rowid_list* rlist_b;

    get_low_data(data,mem_pool);

	strcpy((char*)data->data,"1");
	rlist_a = hash_index_query(hash_index,data,mem_pool);
	rlist_b = hash_compress_query(hash_compress,data,mem_pool);

	ASSERT_EQ(rowid_list_equals(rlist_a,rlist_b),1);

	strcpy((char*)data->data,"2");
	rlist_a = hash_index_query(hash_index,data,mem_pool);
	rlist_b = hash_compress_query(hash_compress,data,mem_pool);

	ASSERT_EQ(rowid_list_equals(rlist_a,rlist_b),1);

    hash_index_release(hash_index);
    hash_compress_release(hash_compress);
}
コード例 #24
0
ファイル: DocEngine.cpp プロジェクト: Alibaba-boonya/mile
static void set_storage_dir(struct str_array_t *dirs, const char *config_value,
		MEM_POOL_PTR mem) {
	assert(NULL != config_value);

	// get dir number
	dirs->n = 0;
	for (const char *p = config_value; *p != '\0';) {
		int n = strcspn(p, ";");
		if (n > 0)
			dirs->n++;
		p += n;
		if (*p == ';')
			p++;
	}

	assert(dirs->n > 0);
	// config_value only contain ',' will fail

	// alloc memory
	dirs->strs = (char **) mem_pool_malloc(mem, sizeof(char *) * dirs->n);

	char *p = (char *) mem_pool_malloc(mem, strlen(config_value) + 1);
	strcpy(p, config_value);

	// set dir pointers
	for (int i = 0; *p != '\0';) {
		dirs->strs[i] = p;
		int n = strcspn(p, ";");
		if (p[n] == '\0')
			break;
		else {
			p[n] = '\0';
			p += n + 1;
			if (n > 0)
				i++;
		}
	}
}
コード例 #25
0
ファイル: vstorage.c プロジェクト: Alibaba-boonya/mile
struct low_data_struct*  vstorage_query(struct vstorage_manager * vstorage,uint32_t docid,MEM_POOL* mem_pool)
{	
        struct low_data_struct* result = NULL;
	struct locate_info* loc_info = NULL;
        struct locate_info4* loc_info4 = NULL;
	uint64_t offset = 0;

	//分配结构内存空间
	result = (struct low_data_struct*)mem_pool_malloc(mem_pool,sizeof(struct low_data_struct));
	memset(result,0,sizeof(struct low_data_struct));

	if(vstorage->loc_info != NULL){// still use 2 bytes length
		loc_info = vstorage->loc_info + docid;
        	result->len = loc_info->len;
		offset = loc_info->offset;		
	} else {// using 4 bytes length
		loc_info4 = vstorage->loc_info4 + docid;
		result->len = loc_info4->len;
		offset = loc_info4->offset;
	}

	if(result->len != 0)
        {
        	//分配用于存储数据的内存空间
                result->data = mem_pool_malloc(mem_pool,result->len);
                memset(result->data,0,result->len);

                //根据偏移量和长度从文件中读取数据
                sc_record_value(STAT_ITEM_VSTORE_READ, result->len);
                if(pread(vstorage->data_fd, result->data, result->len, offset) != result->len)
                {
                      log_error("读取IO有问题,文件名:%s 长度:%u 错误:%s",vstorage->data_file_name,result->len,strerror(errno));
                         return NULL;
                }
         }
	
	return result;
}
コード例 #26
0
ファイル: HashSet.cpp プロジェクト: Alibaba-boonya/mile
HashSet::HashSet(Equals* equals, HashCoding* hash, MEM_POOL_PTR mem_pool) {
	uint32_t i;

	this->bucket_size = 1000;
	this->mem_pool = mem_pool;
	this->num = 0;
	this->equal_func = equals;
	this->hash_func = hash;
	this->data_array = (MileList**) mem_pool_malloc(mem_pool,
			sizeof(MileList*) * this->bucket_size);
	for (i = 0; i < this->bucket_size; i++) {
		this->data_array[i] = NULL;
	}
}
コード例 #27
0
ファイル: client.c プロジェクト: Alibaba-boonya/mile
int execute_ldb_control_handle(struct data_buffer *dbuf, int socket_fd, MEM_POOL_PTR mem_pool)
{
	struct data_buffer rbuf;

	if (socket_write(socket_fd, dbuf->data, dbuf->data_len) <= 0)
		return -1;

	memset(&rbuf,0,sizeof(struct data_buffer));
	rbuf.data_len = sizeof(uint32_t);
	rbuf.size = rbuf.data_len;
	rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,sizeof(uint32_t));
	
	if(socket_read(socket_fd,rbuf.data,sizeof(uint32_t)) <= 0)
		return -1;

	rbuf.data_len = read_int32(&rbuf);
	rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,rbuf.data_len);
	rbuf.rpos = 0;
	rbuf.size = rbuf.data_len;

	if(socket_read(socket_fd,rbuf.data,rbuf.data_len) <= 0)
		return -1;
	rbuf.rpos += MESSAGE_HEAD_LEN;

	uint16_t type = message_type(&rbuf);
	if (type != MT_DC_LDB_CONTROL)
		return -1;

	// content
	char *content = read_cstring(&rbuf, mem_pool);
	if (content) {
		fprintf(stderr, "%s\n", content);
		return 0;
	}
	else 
		return -1;
}
コード例 #28
0
ファイル: client.c プロジェクト: Alibaba-boonya/mile
int execute_set_load_threshold_handle(struct data_buffer *dbuf, int socket_fd, MEM_POOL_PTR mem_pool)
{
	struct data_buffer rbuf;
	
	//发数据包
	if(socket_write(socket_fd,dbuf->data,dbuf->data_len) <= 0)
		return -1;

	//接受命名
	memset(&rbuf,0,sizeof(struct data_buffer));
	rbuf.data_len = sizeof(uint32_t);
	rbuf.size = rbuf.data_len;
	rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,sizeof(uint32_t));
	
	if(socket_read(socket_fd,rbuf.data,sizeof(uint32_t)) <= 0)
		return -1;

	rbuf.data_len = read_int32(&rbuf);
	rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,rbuf.data_len);
	rbuf.rpos = 0;
	rbuf.size = rbuf.data_len;

	if(socket_read(socket_fd,rbuf.data,rbuf.data_len) <= 0)
		return -1;
	rbuf.rpos += MESSAGE_HEAD_LEN;

	uint16_t type = message_type(&rbuf);
	if (type != MT_DC_EXE_RS)
		return -1;

	double load = 0;
	read_bytes(&rbuf, (uint8_t *)&load, sizeof(load));

	fprintf(stderr, "OLD VALUE: %g\n", load);
	return 0;
}
コード例 #29
0
ファイル: AggrFunc.cpp プロジェクト: Alibaba-boonya/mile
int32_t Count(MEM_POOL_PTR mem_pool, struct low_data_struct *cur_value,
		struct low_data_struct* new_value) {
	if (cur_value == NULL) {
		return -1;
	}
	if (cur_value->len == 0) {
		cur_value->len = sizeof(int64_t);
		cur_value->type = HI_TYPE_LONGLONG;
		cur_value->data = mem_pool_malloc(mem_pool, cur_value->len);
		*(int64_t*) (cur_value->data) = 1;
	} else {
		*(int64_t*) (cur_value->data) += 1;
	}
	return MILE_RETURN_SUCCESS;
}
コード例 #30
0
TEST(DYHASH_RECOVER1, dyhashindex){
	MEM_POOL* mem_pool = mem_pool_init(MB_SIZE);

	struct dyhash_index_config config;
	
	char dir_path[] = "/tmp/dyhash_index4";

	config.is_full = 0;
	config.row_limit = 8;
	strcpy(config.work_space, dir_path);
	
	
	struct dyhash_index_manager* dyhash_index = init(&config,dir_path);
	
	struct low_data_struct data;
	data.data = mem_pool_malloc(mem_pool, strlen("hello1world"));
	memset(data.data, 0, strlen("hello1world"));
	data.len = strlen("hello world");
	data.type = HI_TYPE_STRING;

	int32_t ret = 0;
	strcpy((char*)data.data, "hello1world");
    ret = dyhash_index_insert(dyhash_index,&data,0);
	ASSERT_EQ(0, ret);
	
	
	strcpy((char*)data.data, "hello1world");
    ret = dyhash_index_insert(dyhash_index,&data,1);
	ASSERT_EQ(0, ret);

	strcpy((char*)data.data, "hello1world");
    ret = dyhash_index_insert(dyhash_index,&data,2);
	ASSERT_EQ(0, ret);

	strcpy((char*)data.data, "hello1world");
    ret = dyhash_index_insert(dyhash_index,&data,3);
	ASSERT_EQ(0, ret);

	strcpy((char*)data.data, "hello2world");
    ret = dyhash_index_insert(dyhash_index,&data,4);
	ASSERT_EQ(0, ret);

	strcpy((char*)data.data, "hello2world");
    ret = dyhash_index_insert(dyhash_index,&data,5);
	ASSERT_EQ(0, ret);

	dyhash_index_release(dyhash_index);
}