int32_t execute_common_handle(struct data_buffer* dbuf,int socket_fd,MEM_POOL* mem_pool)
{
struct data_buffer rbuf;
//发数据包
if(socket_write(socket_fd,dbuf->data,dbuf->data_len) <= 0)
return -1;
//接受命名
memset(&rbuf,0,sizeof(struct data_buffer));
rbuf.data_len = sizeof(uint32_t);
rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,sizeof(uint32_t));
rbuf.size = rbuf.data_len;
if(socket_read(socket_fd,rbuf.data,sizeof(uint32_t)) <= 0)
return -1;
rbuf.data_len = read_int32(&rbuf);
rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,rbuf.data_len);
rbuf.rpos = 0;
rbuf.size = rbuf.data_len;
if(socket_read(socket_fd,rbuf.data,rbuf.data_len) <= 0)
return -1;
rbuf.rpos += MESSAGE_HEAD_LEN;
uint16_t type = message_type(&rbuf);
if (type != MT_DC_EXE_RS)
return -1;
return read_int32(&rbuf);
}
int32_t CountDistinct(MEM_POOL_PTR mem_pool, struct low_data_struct *cur_value,
struct low_data_struct* new_value) {
if (cur_value == NULL || new_value == NULL) {
return -1;
}
if (cur_value->data == NULL) {
cur_value->type = HI_TYPE_SET;
cur_value->len = 0;
Equals * equals =
new (mem_pool_malloc(mem_pool, sizeof(LDEquals))) LDEquals();
HashCoding* hash =
new (mem_pool_malloc(mem_pool, sizeof(LDHash))) LDHash();
cur_value->data =
new (mem_pool_malloc(mem_pool, sizeof(HashSet))) HashSet(equals,
hash, mem_pool);
}
HashSet* set = (HashSet*) cur_value->data;
if (set->Contains(new_value)) {
return MILE_RETURN_SUCCESS;
} else if (new_value->type != HI_TYPE_NULL) {
struct low_data_struct* ld = (struct low_data_struct*) mem_pool_malloc(
mem_pool, sizeof(struct low_data_struct));
memset(ld, 0, sizeof(struct low_data_struct));
copy_low_data_struct(mem_pool, ld, new_value);
set->Add(ld);
cur_value->len++;
}
return MILE_RETURN_SUCCESS;
}
int32_t Avg(MEM_POOL_PTR mem_pool, struct low_data_struct* cur_value,
struct low_data_struct* new_value) {
struct low_data_struct *sum, *count;
MileArray* array;
double a, b;
if (cur_value == NULL || new_value == NULL) {
return -1;
}
if (new_value->type != HI_TYPE_NULL && new_value->len > 0) {
if (cur_value->data == NULL) {
array =
new (mem_pool_malloc(mem_pool, sizeof(MileArray))) MileArray(
mem_pool);
cur_value->type = HI_TYPE_ARRAY;
cur_value->len = 2;
cur_value->data = array;
//initial the sum
sum = (struct low_data_struct*) mem_pool_malloc(mem_pool,
sizeof(struct low_data_struct));
memset(sum, 0, sizeof(struct low_data_struct));
sum->type = HI_TYPE_DOUBLE;
sum->len = sizeof(double);
sum->data = mem_pool_malloc(mem_pool, sizeof(double));
*(double*) sum->data = 0;
array->Add(sum);
//initial the count
count = (struct low_data_struct*) mem_pool_malloc(mem_pool,
sizeof(struct low_data_struct));
memset(count, 0, sizeof(struct low_data_struct));
count->type = HI_TYPE_LONGLONG;
count->len = sizeof(uint64_t);
count->data = mem_pool_malloc(mem_pool, sizeof(uint64_t));
*(uint64_t*) count->data = 0;
array->Add(count);
}
array = (MileArray*) cur_value->data;
sum = (struct low_data_struct*) array->Get(0);
count = (struct low_data_struct*) array->Get(1);
*(uint64_t*) count->data = *(uint64_t*) count->data + 1;
a = ld_to_double(new_value);
b = *(double*) sum->data;
if (isnan(a)) {
*(double*) sum->data = a;
if (new_value->field_name != NULL) {
log_error("%s列的类型不能转换为double, 无法进行avg操作", new_value->field_name);
}
} else if (!isnan(b)) {
*(double*) sum->data = *(double*) sum->data + a;
}
}
return MILE_RETURN_SUCCESS;
}
/**
* 在btree中做范围查询,在所有数据范围内查询
* @param tid 表号
* @param field_id 域号
* @param range_condition 查询的条件
* @param mem_pool 内存池
* @return 成功返回list head，上层可以通过这个遍历，取到所有段结果，失败返回NULL
**/
struct list_head * db_btree_query_range(uint8 tid, uint16 field_id, \
struct db_range_query_condition *range_condition, MEM_POOL *pMemPool)
{
int32 i, j, k, n;
int8 cmp_res;
//初始化头
struct list_head* list_h = (struct list_head*)mem_pool_malloc(pMemPool,sizeof(struct list_head));
INIT_LIST_HEAD(list_h);
struct segment_query_rowids* seg_rowids;
n = 0;
for(i = 0; i <= current_seg; i++)
{
if(i == current_seg)
{
k = current_row;
}else{
k = TEST_MAX_ROW_NUM;
}
seg_rowids = (struct segment_query_rowids*) mem_pool_malloc(pMemPool, sizeof(struct segment_query_rowids));
seg_rowids->sid = i;
seg_rowids->rowids = rowid_list_init(pMemPool);
list_add(&seg_rowids->rowids_list, list_h);
for(j = k-1; j >= 0; j--)
{
if(range_condition->min_flag >= 0)
{
cmp_res = compare(mock_data_source[i][j][field_id].data, range_condition->min_key->data, mock_fields_type_array[field_id]);
if((cmp_res <= 0 && range_condition->min_flag == 1) || (cmp_res < 0 && range_condition->min_flag == 0))
{
continue;
}
}
if(range_condition->max_flag >= 0)
{
cmp_res = compare(mock_data_source[i][j][field_id].data, range_condition->max_key->data, mock_fields_type_array[field_id]);
if((cmp_res >= 0 && range_condition->max_flag == 1) || (cmp_res > 0 && range_condition->max_flag == 0))
{
continue;
}
}
rowid_list_add(pMemPool, seg_rowids->rowids, j);
n++;
}
}
return list_h;
}
int32_t execute_index_stat_handle(struct data_buffer* dbuf,int socket_fd,MEM_POOL* mem_pool)
{
struct data_buffer rbuf;
//发一个数据包长度
if(socket_write(socket_fd,dbuf->data,dbuf->data_len) <= 0)
return -1;
//接受命名
memset(&rbuf,0,sizeof(struct data_buffer));
rbuf.data_len = sizeof(uint32_t);
rbuf.size = rbuf.data_len;
rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,sizeof(uint32_t));
if(socket_read(socket_fd,rbuf.data,rbuf.data_len) <= 0)
return -1;
rbuf.data_len = read_int32(&rbuf);
rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,rbuf.data_len);
rbuf.rpos = 0;
rbuf.size = rbuf.data_len;
if(socket_read(socket_fd,rbuf.data,rbuf.data_len) <= 0)
return -1;
rbuf.rpos += MESSAGE_HEAD_LEN;
uint16_t type = message_type(&rbuf);
if (type != MT_DC_STAT_RS)
return -1;
uint16_t field_count = read_int16(&rbuf);
uint16_t i;
char* field_name = NULL;
fprintf(stderr,"%-20s%-20s%-20s%-20s\n","FIELD_NAME","INDEX_TYPE","DATA_TYPE","FLAG");
uint8_t j;
uint8_t index_num;
for(i=0; i<field_count; i++)
{
field_name = read_cstring(&rbuf,mem_pool);
index_num = read_int8(&rbuf);
for(j=0;j < index_num; j++)
{
fprintf(stderr,"%-20s",field_name);
fprintf(stderr,"%-20u",read_int8(&rbuf));
fprintf(stderr,"%-20u",read_int8(&rbuf));
fprintf(stderr,"%-20u\n",read_int8(&rbuf));
}
}
return 0;
}
void get_low_data(struct low_data_struct* data,MEM_POOL* mem_pool)
{
data->len = 5;
data->data = mem_pool_malloc(mem_pool,5);
data->type = HI_TYPE_STRING;
data->field_name = (char*)mem_pool_malloc(mem_pool,20);
memset(data->field_name,0,20);
strcpy(data->field_name,"HI_TYPE_STRING");
memset(data->data,0,5);
strcpy((char*)data->data,"ali");
return;
}
/**
* 根据rowid来查找对应的value，只有filter索引类型支持
* @param tid 表号
* @param sid 段号
* @param row_id 行号
* @param field_id 域号
* @param mem_pool 内存池
* @return 成功返回rowid_list，失败返回NULL
**/
struct low_data_struct* db_query_by_rowid(uint8 tid, uint16 sid,uint32 row_id,uint16 field_id, MEM_POOL* mem_pool)
{
if(mock_delete_mask[sid][row_id])
{
return NULL;
}
struct low_data_struct* result = (struct low_data_struct*) mem_pool_malloc(mem_pool, sizeof(struct low_data_struct));
result->len = mock_data_source[sid][row_id][field_id].len;
result->data = mem_pool_malloc(mem_pool, result->len);
memcpy(result->data, mock_data_source[sid][row_id][field_id].data, result->len);
return result;
}
struct vstorage_manager * vstorage_init(struct vstorage_config * config,MEM_POOL* mem_pool)
{
struct vstorage_manager * vstorage = NULL;
char* memap_address = NULL;
char old_loc_name[FILENAME_MAX_LENGTH],new_loc_name[FILENAME_MAX_LENGTH];
vstorage = (struct vstorage_manager *)mem_pool_malloc(mem_pool,sizeof(struct vstorage_manager));
memset(vstorage,0,sizeof(struct vstorage_manager));
//用于存储数据信息的文件
sprintf(vstorage->data_file_name,"%s/%s.dat",config->work_space,config->vstorage_name);
//打开文件
vstorage->data_fd = open_file(vstorage->data_file_name,O_RDWR | O_CREAT | O_APPEND);
if(vstorage->data_fd < 0) {
return NULL;
}
//分配用于存储位置信息的偏移量，需要memap到内存中
vstorage->row_limit = config->row_limit;
//拼接
// sprintf(vstorage->index_file_name,"%s/%s.loc",config->work_space,config->vstorage_name);
sprintf(old_loc_name,"%s/%s.loc",config->work_space,config->vstorage_name);
sprintf(new_loc_name,"%s/%s.loc4",config->work_space,config->vstorage_name);
if(access(old_loc_name, W_OK) == 0)//if old index file exist then use 2 bytes length
{
strcpy(vstorage->index_file_name , old_loc_name);
//memap到内存中
memap_address = (char* )get_mmap_memory(vstorage->index_file_name, VSTORAGE_MMAP_SIZE(vstorage));
assert(memap_address != NULL);
//留8字节给偏移量
//这个偏移量主要用来记录文件当前写入的位置
vstorage->loc_info = (struct locate_info*)(memap_address+sizeof(uint64_t));
vstorage->loc_info4 = NULL;
} else { //else use 4 bytes length
strcpy(vstorage->index_file_name , new_loc_name);
//memap到内存中
memap_address =(char* )get_mmap_memory(vstorage->index_file_name, VSTORAGE_MMAP_SIZE4(vstorage));
assert(memap_address != NULL);
//留8字节给偏移量
//这个偏移量主要用来记录文件当前写入的位置
vstorage->loc_info4 = (struct locate_info4*)(memap_address+sizeof(uint64_t));
vstorage->loc_info = NULL;
}
//初始化偏移量
vstorage->offset = (uint64_t*)memap_address;
return vstorage;
}
void verify_muti_hashvalue(struct hash_index_manager* hash_index,MEM_POOL* mem_pool)
{
//查询性能测试
struct rowid_list* rlist_a;
struct rowid_list* rlist_b;
uint64_t time_a = 0;
uint64_t time_b = 0;
uint64_t now;
uint32_t i;
struct hash_compress_manager* hash_compress = hash_compress_load(hash_index,10,mem_pool);
struct low_data_struct* data = (struct low_data_struct*)mem_pool_malloc(mem_pool,sizeof(struct low_data_struct));
get_low_data(data,mem_pool);
for(i=0; i<hash_index->limit;i++)
{ sprintf((char*)data->data,"%d",i);
now = get_time_usec();
rlist_a = hash_index_query(hash_index,data,mem_pool);
time_a += get_time_usec() - now;
now = get_time_usec();
rlist_b = hash_compress_query(hash_compress,data,mem_pool);
time_b += get_time_usec()-now;
ASSERT_EQ(rowid_list_equals(rlist_a,rlist_b),1);
}
printf("hash_index %llu hash_compress %llu\n",time_a,time_b);
hash_compress_release(hash_compress);
}
/**
* 向rowid_list中添加新的rowid，注意数据库中的rowid链表是降序的
*
* @param pMemPool 内存池
* @param id_list rowid的链表
* @param rowid
*/
void rowid_list_add(MEM_POOL_PTR pMemPool, struct rowid_list* id_list,
uint32_t rowid) {
if (id_list == NULL ) {
return;
}
if (id_list->head == NULL || id_list->rowid_num % ROWID_ARRAY_SIZE == 0) {
struct rowid_list_node* tmp = (struct rowid_list_node*) mem_pool_malloc(
pMemPool, sizeof(struct rowid_list_node));
memset(tmp, 0, sizeof(struct rowid_list_node));
tmp->rowid_array[0] = rowid;
tmp->next = NULL;
id_list->rowid_num++;
if (id_list->head == NULL ) {
id_list->head = tmp;
id_list->tail = tmp;
} else {
id_list->tail->next = tmp;
id_list->tail = tmp;
}
} else {
id_list->tail->rowid_array[id_list->rowid_num % ROWID_ARRAY_SIZE] =
rowid;
id_list->rowid_num++;
}
}
int32_t SquareSum(MEM_POOL_PTR mem_pool, struct low_data_struct* cur_value,
struct low_data_struct* new_value) {
double a, b;
if (cur_value == NULL || new_value == NULL) {
return -1;
}
if (new_value->type != HI_TYPE_NULL && new_value->len != 0) {
if (cur_value->type == HI_TYPE_NULL || cur_value->len == 0) {
cur_value->type = HI_TYPE_DOUBLE;
cur_value->len = sizeof(double);
cur_value->data = mem_pool_malloc(mem_pool, sizeof(double));
*(double*) cur_value->data = 0;
}
a = ld_to_double(new_value);
b = *(double*) cur_value->data;
if (isnan(a)) {
*(double*) cur_value->data = a;
if (new_value->field_name != NULL) {
log_error("%s列的类型不能转换为double, 无法进行squaresum操作", new_value->field_name);
}
} else if (!isnan(b)) {
*(double*) cur_value->data = b + a * a;
}
}
return MILE_RETURN_SUCCESS;
}
struct storage_manager * storage_init(struct storage_config* config,MEM_POOL* mem_pool)
{
struct storage_manager* storage =
(struct storage_manager *)mem_pool_malloc(mem_pool,sizeof(struct storage_manager));
assert(storage != NULL);
memset(storage,0,sizeof(struct storage_manager));
//初始化block信息
storage->value_size = config->unit_size;
storage->row_limit = config->row_limit;
//拼接:文件名
sprintf(storage->file_name,"%s/%s.dat",config->work_space,config->storage_name);
//mmap映射处理
storage->mem_mmaped =(char*)get_mmap_memory(storage->file_name,STORAGE_MMAP_SIZE(storage));
assert(storage->mem_mmaped != NULL);
//初始化空值标志位
struct bitmark_config bitmark_config;
strcpy(bitmark_config.work_space,config->work_space);
strcpy(bitmark_config.bitmark_name,"null");
bitmark_config.row_limit = config->row_limit;
storage->null_bitmark = bitmark_init(&bitmark_config,mem_pool);
return storage;
}
/**
* 向rowid_list中添加新的rowid，注意数据库中的rowid链表是降序的
*
* @param pMemPool 内存池
* @param id_list rowid的链表
* @param rowid
*/
void rowid_list_batch_add(MEM_POOL_PTR pMemPool, struct rowid_list* id_list,
uint32_t* rowids, uint32_t num) {
uint32_t insert_num = 0;
uint32_t total = num;
if (id_list == NULL ) {
return;
}
while (num != 0) {
insert_num = num / ROWID_ARRAY_SIZE > 0 ? ROWID_ARRAY_SIZE : num;
struct rowid_list_node* tmp = (struct rowid_list_node*) mem_pool_malloc(
pMemPool, sizeof(struct rowid_list_node));
memset(tmp, 0, sizeof(struct rowid_list_node));
memcpy(tmp->rowid_array, rowids + total - num,
insert_num * sizeof(uint32_t));
tmp->next = NULL;
if (id_list->head == NULL ) {
id_list->head = tmp;
id_list->tail = tmp;
} else {
id_list->tail->next = tmp;
id_list->tail = tmp;
}
num -= insert_num;
id_list->rowid_num += insert_num;
}
}
TEST(DYHASH_CHECKPOINT2, dyhashindex){
MEM_POOL* mem_pool = mem_pool_init(MB_SIZE);
struct dyhash_index_config config;
char dir_path[] = "/tmp/dyhash_index3";
config.is_full = 0;
config.row_limit = 2;
config.is_full = 1;
strcpy(config.work_space, dir_path);
init_profile(1000,mem_pool);
struct dyhash_index_manager* dyhash_index = dyhash_index_init(&config,mem_pool);
struct low_data_struct data;
data.data = mem_pool_malloc(mem_pool, strlen("hello world"));
memset(data.data, 0, strlen("hello world"));
strcpy((char*)data.data, "hello world");
data.len = strlen("hello world");
data.type = HI_TYPE_STRING;
uint32_t count = dyhash_index_count_query(dyhash_index, &data, mem_pool);
ASSERT_EQ(1, count);
struct rowid_list* query_ret = dyhash_index_query(dyhash_index, &data, mem_pool);
print_rowid_list(query_ret);
}
void test_multi_hashvalue(uint32_t limit)
{
MEM_POOL* mem_pool = mem_pool_init(MB_SIZE);
int32_t ret;
struct hash_index_config config;
config.row_limit = limit;
strcpy(config.work_space,"/tmp/hash_compress_test");
system("rm -rf /tmp/hash_compress_test");
mkdirs(config.work_space);
init_profile(1000,mem_pool);
struct hash_index_manager* hash_index = hash_index_init(&config,mem_pool);
uint32_t i;
struct low_data_struct* data = (struct low_data_struct*)mem_pool_malloc(mem_pool,sizeof(struct low_data_struct));
get_low_data(data,mem_pool);
for(i=0; i<config.row_limit;i++)
{ sprintf((char*)data->data,"%d",i);
ret = hash_index_insert(hash_index,data,i);
ASSERT_EQ(0,ret);
}
verify_muti_hashvalue(hash_index,mem_pool);
hash_index_release(hash_index);
}
int32_t execute_segment_stat_handle(struct data_buffer* dbuf,int socket_fd,MEM_POOL* mem_pool)
{
struct data_buffer rbuf;
//发数据包
if(socket_write(socket_fd,dbuf->data,dbuf->data_len) <= 0)
return -1;
//接受命名
memset(&rbuf,0,sizeof(struct data_buffer));
rbuf.data_len = sizeof(uint32_t);
rbuf.size = rbuf.data_len;
rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,sizeof(uint32_t));
if(socket_read(socket_fd,rbuf.data,rbuf.data_len) <= 0)
return -1;
rbuf.data_len = read_int32(&rbuf);
rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,rbuf.data_len);
rbuf.rpos = 0;
rbuf.size = rbuf.data_len;
if(socket_read(socket_fd,rbuf.data,rbuf.data_len) <= 0)
return -1;
rbuf.rpos += MESSAGE_HEAD_LEN;
uint16_t type = message_type(&rbuf);
if (type != MT_DC_STAT_RS)
return -1;
uint16_t segment_count = read_int16(&rbuf);
uint16_t i;
fprintf(stderr,"%-20s%-20s%-20s%-20s%-20s%-20s%-20s\n","SEGMENT_ID","CREATE_TIME","MODIFY_TIME","CHECKPOINT_TIME","FLAG","ROW_COUNT","DEL_COUNT");
for(i=0; i<segment_count; i++)
{
fprintf(stderr,"%-20u",read_int16(&rbuf));
fprintf(stderr,"%-20llu",read_int64(&rbuf));
fprintf(stderr,"%-20llu",read_int64(&rbuf));
fprintf(stderr,"%-20llu",read_int64(&rbuf));
fprintf(stderr,"%-20u",read_int8(&rbuf));
fprintf(stderr,"%-20u",read_int32(&rbuf));
fprintf(stderr,"%-20u\n",read_int32(&rbuf));
}
return 0;
}
struct rowid_list* rowid_list_init(MEM_POOL_PTR pMemPool) {
struct rowid_list* id_list = (struct rowid_list*) mem_pool_malloc(pMemPool,
sizeof(struct rowid_list));
id_list->rowid_num = 0;
id_list->head = NULL;
id_list->tail = NULL;
return id_list;
}
ResultSet* DocEngine::GetKvs(struct get_kvs_packet* packet,
MEM_POOL_PTR mem_pool) {
int32_t result_code = MILE_RETURN_SUCCESS;
uint32_t i, j;
uint16_t sid;
uint32_t rowid;
struct select_row_t* row;
struct low_data_struct** sel_fields;
Clonable* clone =
new (mem_pool_malloc(mem_pool, sizeof(RowClone))) RowClone(
mem_pool);
CommonRS* result =
new (mem_pool_malloc(mem_pool, sizeof(CommonRS))) CommonRS(packet->docid_num, clone,
mem_pool);
char** field_names = (char**) mem_pool_malloc(mem_pool,
sizeof(char*) * packet->select_field.n);
for (i = 0; i < packet->select_field.n; i++) {
field_names[i] = packet->select_field.select_fields[i].field_name;
}
for (i = 0; i < packet->docid_num; i++) {
sid = packet->docids[i] >> 32;
rowid = (uint32_t)packet->docids[i];
sel_fields = db_data_query_multi_col(packet->table_name, sid, rowid, field_names,
packet->select_field.n, DATA_ACCESS_ORIGINAL, mem_pool);
if(NULL == sel_fields){
return NULL;
}
row = init_select_row_t(mem_pool, packet->select_field.n);
for(j = 0; j < packet->select_field.n; j++){
row->data[j] = *sel_fields[j];
row->select_type[j] = SELECT_TYPE_ORIGINAL;
}
row->handler = new(mem_pool_malloc(mem_pool, sizeof(DocHandler)))DocHandler(packet->docids[i]);
result->AddResult(row);
}
return result;
}
/**
* 根据value来查找对应的doc id lists，只有hash索引支持，所有段扫
* @param tid 表号
* @param field_id 域号
* @param data 数据
* @param mem_pool 内存池
* @return 成功返回list head，上层可以通过这个遍历，取到所有段结果，失败返回NULL
**/
struct list_head* db_query_by_value(uint8 tid,uint16 field_id,struct low_data_struct* data, MEM_POOL* mem_pool)
{
int32 i, j, k, n;
//初始化头
struct list_head* list_h = (struct list_head*)mem_pool_malloc(mem_pool,sizeof(struct list_head));
INIT_LIST_HEAD(list_h);
struct segment_query_rowids* seg_rowids;
n = 0;
for(i = 0; i <= current_seg; i++)
{
if(i == current_seg)
{
k = current_row;
}else{
k = TEST_MAX_ROW_NUM;
}
seg_rowids = (struct segment_query_rowids*) mem_pool_malloc(mem_pool, sizeof(struct segment_query_rowids));
seg_rowids->sid = i;
seg_rowids->rowids = rowid_list_init(mem_pool);
list_add(&seg_rowids->rowids_list, list_h);
for(j = k-1; j >= 0; j--)
{
if(compare(data->data, mock_data_source[i][j][field_id].data, mock_fields_type_array[field_id]) == 0)
{
rowid_list_add(mem_pool, seg_rowids->rowids, j);
n++;
}
}
}
return list_h;
}
struct low_data_struct * storage_query(struct storage_manager * storage, uint32_t docid,MEM_POOL* mem_pool)
{
struct low_data_struct* result = NULL;
result = (struct low_data_struct*)mem_pool_malloc(mem_pool,sizeof(struct low_data_struct));
memset(result,0,sizeof(struct low_data_struct));
result->data = mem_pool_malloc(mem_pool,storage->value_size);
memset(result->data,0,storage->value_size);
//如果支持空值，且标记为0的话，说明此rowid的数据为空
if(bitmark_query(storage->null_bitmark,docid) == 1)
{
return result;
}
//根据row_id*每个数据的长度，可以算到row_id对应的数据
memcpy(result->data,storage->mem_mmaped+docid*storage->value_size,storage->value_size);
result->len = storage->value_size;
return result;
}
TEST(DATA_BUFFER_TEST, READ_AND_WRITE){
MEM_POOL_PTR pMemPool = mem_pool_init(MB_SIZE);
struct data_buffer* buffer = init_data_buffer();
buffer->data_len = 256;
buffer->data = (uint8*) mem_pool_malloc(pMemPool, 256);
uint8 tmp8;
uint16 tmp16;
uint32 tmp32;
uint64 tmp64;
char* str = (char*) mem_pool_malloc(pMemPool, 100);
char* result = (char*) mem_pool_malloc(pMemPool, 100);
strcpy(str, "hello world!");
write_int8(122, buffer);
write_int16(1111, buffer);
write_int32(324, buffer);
write_int64(2321, buffer);
write_bytes((uint8*)str, strlen(str)+1, buffer);
tmp8 = read_int8(buffer);
ASSERT_EQ(tmp8, 122);
tmp16 = read_int16(buffer);
ASSERT_EQ(tmp16, 1111);
tmp32 = read_int32(buffer);
ASSERT_EQ(tmp32, 324);
tmp64 = read_int64(buffer);
ASSERT_EQ(tmp64, 2321);
read_bytes(buffer, (uint8*)result, strlen(str)+1);
ASSERT_EQ(0, strcmp(str, result));
destroy_data_buffer(buffer);
mem_pool_destroy(pMemPool);
}
void test_with_index_dump(MEM_POOL* mem_pool)
{
system("rm -rf /tmp/binlog");
system("rm -rf /tmp/log");
system("rm -rf /tmp/storage");
db_initialize(mem_pool);
//构造数据
uint16 sid = 0;
uint32 docid = 0;
int32 ret;
uint16 field_count = 5;
enum field_types types[5] = {HI_TYPE_STRING,HI_TYPE_LONG,HI_TYPE_TINY,HI_TYPE_LONGLONG,HI_TYPE_DOUBLE};
//将第一个段填满
while(sid == 0)
{
ret = db_insert("hello", &sid, &docid,get_row_data(field_count,types,mem_pool), DOCID_BY_SELF, mem_pool);
ASSERT_EQ(MILE_RETURN_SUCCESS,ret);
}
sid = 0;
//建hash索引，然后unload load 验证
ret = db_ensure_index("hello","HI_TYPE_STRING",HI_KEY_ALG_HASH,HI_TYPE_STRING, mem_pool);
ASSERT_EQ(MILE_RETURN_SUCCESS,ret);
struct low_data_struct* data = (struct low_data_struct*)mem_pool_malloc(mem_pool,sizeof(struct low_data_struct));
get_low_data(data,HI_TYPE_STRING,mem_pool);
struct list_head* index_equal_ret = db_index_equal_query("hello",get_time_hint(mem_pool),data,mem_pool);
struct segment_query_rowids* node;
list_for_each_entry(node,index_equal_ret,rowids_list) {
if(node->sid == 0)
ASSERT_EQ(node->rowids->rowid_num,10);
else if(node->sid == 1)
ASSERT_EQ(node->rowids->rowid_num,1);
else
FAIL();
}
ret = db_unload_segment("hello",sid,mem_pool);
ASSERT_EQ(MILE_RETURN_SUCCESS,ret);
index_equal_ret = db_index_equal_query("hello",get_time_hint(mem_pool),data,mem_pool);
list_for_each_entry(node,index_equal_ret,rowids_list) {
ASSERT_EQ(node->rowids->rowid_num,1);
ASSERT_EQ(node->sid,1);
}
void test_tow_hashvalue(uint32_t limit)
{
MEM_POOL* mem_pool = mem_pool_init(MB_SIZE);
struct hash_index_config config;
config.row_limit = limit;
strcpy(config.work_space,"/tmp/hash_compress_test");
system("rm -rf /tmp/hash_compress_test");
mkdirs(config.work_space);
init_profile(1000,mem_pool);
struct hash_index_manager* hash_index = hash_index_init(&config,mem_pool);
uint32_t i;
struct low_data_struct* data = (struct low_data_struct*)mem_pool_malloc(mem_pool,sizeof(struct low_data_struct));
get_low_data(data,mem_pool);
for(i=0; i<config.row_limit/2;i++)
{
strcpy((char*)data->data,"1");
hash_index_insert(hash_index,data,i);
}
for(i=config.row_limit/2; i<config.row_limit;i++)
{
strcpy((char*)data->data,"2");
hash_index_insert(hash_index,data,i);
}
struct hash_compress_manager* hash_compress = hash_compress_load(hash_index,10,mem_pool);
struct rowid_list* rlist_a;
struct rowid_list* rlist_b;
get_low_data(data,mem_pool);
strcpy((char*)data->data,"1");
rlist_a = hash_index_query(hash_index,data,mem_pool);
rlist_b = hash_compress_query(hash_compress,data,mem_pool);
ASSERT_EQ(rowid_list_equals(rlist_a,rlist_b),1);
strcpy((char*)data->data,"2");
rlist_a = hash_index_query(hash_index,data,mem_pool);
rlist_b = hash_compress_query(hash_compress,data,mem_pool);
ASSERT_EQ(rowid_list_equals(rlist_a,rlist_b),1);
hash_index_release(hash_index);
hash_compress_release(hash_compress);
}
static void set_storage_dir(struct str_array_t *dirs, const char *config_value,
MEM_POOL_PTR mem) {
assert(NULL != config_value);
// get dir number
dirs->n = 0;
for (const char *p = config_value; *p != '\0';) {
int n = strcspn(p, ";");
if (n > 0)
dirs->n++;
p += n;
if (*p == ';')
p++;
}
assert(dirs->n > 0);
// config_value only contain ',' will fail
// alloc memory
dirs->strs = (char **) mem_pool_malloc(mem, sizeof(char *) * dirs->n);
char *p = (char *) mem_pool_malloc(mem, strlen(config_value) + 1);
strcpy(p, config_value);
// set dir pointers
for (int i = 0; *p != '\0';) {
dirs->strs[i] = p;
int n = strcspn(p, ";");
if (p[n] == '\0')
break;
else {
p[n] = '\0';
p += n + 1;
if (n > 0)
i++;
}
}
}
struct low_data_struct* vstorage_query(struct vstorage_manager * vstorage,uint32_t docid,MEM_POOL* mem_pool)
{
struct low_data_struct* result = NULL;
struct locate_info* loc_info = NULL;
struct locate_info4* loc_info4 = NULL;
uint64_t offset = 0;
//分配结构内存空间
result = (struct low_data_struct*)mem_pool_malloc(mem_pool,sizeof(struct low_data_struct));
memset(result,0,sizeof(struct low_data_struct));
if(vstorage->loc_info != NULL){// still use 2 bytes length
loc_info = vstorage->loc_info + docid;
result->len = loc_info->len;
offset = loc_info->offset;
} else {// using 4 bytes length
loc_info4 = vstorage->loc_info4 + docid;
result->len = loc_info4->len;
offset = loc_info4->offset;
}
if(result->len != 0)
{
//分配用于存储数据的内存空间
result->data = mem_pool_malloc(mem_pool,result->len);
memset(result->data,0,result->len);
//根据偏移量和长度从文件中读取数据
sc_record_value(STAT_ITEM_VSTORE_READ, result->len);
if(pread(vstorage->data_fd, result->data, result->len, offset) != result->len)
{
log_error("读取IO有问题，文件名:%s 长度:%u 错误:%s",vstorage->data_file_name,result->len,strerror(errno));
return NULL;
}
}
return result;
}
HashSet::HashSet(Equals* equals, HashCoding* hash, MEM_POOL_PTR mem_pool) {
uint32_t i;
this->bucket_size = 1000;
this->mem_pool = mem_pool;
this->num = 0;
this->equal_func = equals;
this->hash_func = hash;
this->data_array = (MileList**) mem_pool_malloc(mem_pool,
sizeof(MileList*) * this->bucket_size);
for (i = 0; i < this->bucket_size; i++) {
this->data_array[i] = NULL;
}
}
int execute_ldb_control_handle(struct data_buffer *dbuf, int socket_fd, MEM_POOL_PTR mem_pool)
{
struct data_buffer rbuf;
if (socket_write(socket_fd, dbuf->data, dbuf->data_len) <= 0)
return -1;
memset(&rbuf,0,sizeof(struct data_buffer));
rbuf.data_len = sizeof(uint32_t);
rbuf.size = rbuf.data_len;
rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,sizeof(uint32_t));
if(socket_read(socket_fd,rbuf.data,sizeof(uint32_t)) <= 0)
return -1;
rbuf.data_len = read_int32(&rbuf);
rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,rbuf.data_len);
rbuf.rpos = 0;
rbuf.size = rbuf.data_len;
if(socket_read(socket_fd,rbuf.data,rbuf.data_len) <= 0)
return -1;
rbuf.rpos += MESSAGE_HEAD_LEN;
uint16_t type = message_type(&rbuf);
if (type != MT_DC_LDB_CONTROL)
return -1;
// content
char *content = read_cstring(&rbuf, mem_pool);
if (content) {
fprintf(stderr, "%s\n", content);
return 0;
}
else
return -1;
}
int execute_set_load_threshold_handle(struct data_buffer *dbuf, int socket_fd, MEM_POOL_PTR mem_pool)
{
struct data_buffer rbuf;
//发数据包
if(socket_write(socket_fd,dbuf->data,dbuf->data_len) <= 0)
return -1;
//接受命名
memset(&rbuf,0,sizeof(struct data_buffer));
rbuf.data_len = sizeof(uint32_t);
rbuf.size = rbuf.data_len;
rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,sizeof(uint32_t));
if(socket_read(socket_fd,rbuf.data,sizeof(uint32_t)) <= 0)
return -1;
rbuf.data_len = read_int32(&rbuf);
rbuf.data = (uint8_t *)mem_pool_malloc(mem_pool,rbuf.data_len);
rbuf.rpos = 0;
rbuf.size = rbuf.data_len;
if(socket_read(socket_fd,rbuf.data,rbuf.data_len) <= 0)
return -1;
rbuf.rpos += MESSAGE_HEAD_LEN;
uint16_t type = message_type(&rbuf);
if (type != MT_DC_EXE_RS)
return -1;
double load = 0;
read_bytes(&rbuf, (uint8_t *)&load, sizeof(load));
fprintf(stderr, "OLD VALUE: %g\n", load);
return 0;
}
int32_t Count(MEM_POOL_PTR mem_pool, struct low_data_struct *cur_value,
struct low_data_struct* new_value) {
if (cur_value == NULL) {
return -1;
}
if (cur_value->len == 0) {
cur_value->len = sizeof(int64_t);
cur_value->type = HI_TYPE_LONGLONG;
cur_value->data = mem_pool_malloc(mem_pool, cur_value->len);
*(int64_t*) (cur_value->data) = 1;
} else {
*(int64_t*) (cur_value->data) += 1;
}
return MILE_RETURN_SUCCESS;
}
TEST(DYHASH_RECOVER1, dyhashindex){
MEM_POOL* mem_pool = mem_pool_init(MB_SIZE);
struct dyhash_index_config config;
char dir_path[] = "/tmp/dyhash_index4";
config.is_full = 0;
config.row_limit = 8;
strcpy(config.work_space, dir_path);
struct dyhash_index_manager* dyhash_index = init(&config,dir_path);
struct low_data_struct data;
data.data = mem_pool_malloc(mem_pool, strlen("hello1world"));
memset(data.data, 0, strlen("hello1world"));
data.len = strlen("hello world");
data.type = HI_TYPE_STRING;
int32_t ret = 0;
strcpy((char*)data.data, "hello1world");
ret = dyhash_index_insert(dyhash_index,&data,0);
ASSERT_EQ(0, ret);
strcpy((char*)data.data, "hello1world");
ret = dyhash_index_insert(dyhash_index,&data,1);
ASSERT_EQ(0, ret);
strcpy((char*)data.data, "hello1world");
ret = dyhash_index_insert(dyhash_index,&data,2);
ASSERT_EQ(0, ret);
strcpy((char*)data.data, "hello1world");
ret = dyhash_index_insert(dyhash_index,&data,3);
ASSERT_EQ(0, ret);
strcpy((char*)data.data, "hello2world");
ret = dyhash_index_insert(dyhash_index,&data,4);
ASSERT_EQ(0, ret);
strcpy((char*)data.data, "hello2world");
ret = dyhash_index_insert(dyhash_index,&data,5);
ASSERT_EQ(0, ret);
dyhash_index_release(dyhash_index);
}