TEST(DYHASH_CHECKPOINT2, dyhashindex){ MEM_POOL* mem_pool = mem_pool_init(MB_SIZE); struct dyhash_index_config config; char dir_path[] = "/tmp/dyhash_index3"; config.is_full = 0; config.row_limit = 2; config.is_full = 1; strcpy(config.work_space, dir_path); init_profile(1000,mem_pool); struct dyhash_index_manager* dyhash_index = dyhash_index_init(&config,mem_pool); struct low_data_struct data; data.data = mem_pool_malloc(mem_pool, strlen("hello world")); memset(data.data, 0, strlen("hello world")); strcpy((char*)data.data, "hello world"); data.len = strlen("hello world"); data.type = HI_TYPE_STRING; uint32_t count = dyhash_index_count_query(dyhash_index, &data, mem_pool); ASSERT_EQ(1, count); struct rowid_list* query_ret = dyhash_index_query(dyhash_index, &data, mem_pool); print_rowid_list(query_ret); }
void test_multi_hashvalue(uint32_t limit) { MEM_POOL* mem_pool = mem_pool_init(MB_SIZE); int32_t ret; struct hash_index_config config; config.row_limit = limit; strcpy(config.work_space,"/tmp/hash_compress_test"); system("rm -rf /tmp/hash_compress_test"); mkdirs(config.work_space); init_profile(1000,mem_pool); struct hash_index_manager* hash_index = hash_index_init(&config,mem_pool); uint32_t i; struct low_data_struct* data = (struct low_data_struct*)mem_pool_malloc(mem_pool,sizeof(struct low_data_struct)); get_low_data(data,mem_pool); for(i=0; i<config.row_limit;i++) { sprintf((char*)data->data,"%d",i); ret = hash_index_insert(hash_index,data,i); ASSERT_EQ(0,ret); } verify_muti_hashvalue(hash_index,mem_pool); hash_index_release(hash_index); }
// overloaded void append_conn(string connfile, string coordfile, string coordfile_local2std, string coordfile_std2local) { // connectivity profile represented in the MNI coarser space vector<intVector> coord_map; // store the coordinate-to-index mapping vector<floatVector> coord_map_local2std; // corresponding coordinate in MNI space intVector coord(3); floatVector coord_local2std(3); // read in the coordinate mapping to memory ifstream coordstream(coordfile); ifstream coordstream_local2std(coordfile_local2std); for (int i=0; i<3; i++) coordstream >> coord[i]; while(coordstream.good()) { coord_map.push_back(coord); for (int i=0; i<3; i++) coordstream >> coord[i]; } coordstream.close(); for (int i=0; i<3; i++) coordstream_local2std >> coord_local2std[i]; while(coordstream_local2std.good()) { coord_map_local2std.push_back(coord_local2std); for (int i=0; i<3; i++) coordstream_local2std >> coord_local2std[i]; } coordstream_local2std.close(); assert(coord_map.size()==coord_map_local2std.size()); // the connectivity part ifstream connstream(connfile); int l, r, v; // left node, right node, and t=connectivity value intVector lc(3), rc(3); // coordinates of left/right nodes floatVector lc_local2std(3), rc_local2std(3); // in MNI space intVector lc_cgrid(3), rc_cgrid(3); // coordinates in the coarse grid init_profile( (conn_profile().begin()->second).size(), coord_map.size(), coordfile, conn_profile_local() ); // reset the local conn profile connstream >> l; connstream >> r; connstream >> v; while(connstream.good()) { lc = coord_map[l-1]; rc = coord_map[r-1]; lc_local2std = coord_map_local2std[l-1]; rc_local2std = coord_map_local2std[r-1]; // int MNI space fine2coarse(lc_local2std, lc_cgrid); fine2coarse(rc_local2std, rc_cgrid); if (coarse_map().count(rc_cgrid)) conn_profile_local()[lc][coarse_map()[rc_cgrid]-1] += v; if (coarse_map().count(lc_cgrid)) conn_profile_local()[rc][coarse_map()[lc_cgrid]-1] += v; connstream >> l; connstream >> r; connstream >> v; } connstream.close(); // transform local profile to MNI space profile appendpf_local2std(coordfile_std2local); }
struct dyhash_index_manager* init(struct dyhash_index_config* config,char * dir_path){ char cmd[1024]; memset(cmd, 0, sizeof(cmd)); sprintf(cmd, "rm -rf %s", dir_path); system(cmd); mkdirs(dir_path); MEM_POOL* mem_pool = mem_pool_init(MB_SIZE); init_profile(1000,mem_pool); return dyhash_index_init(config,mem_pool); }
void test_tow_hashvalue(uint32_t limit) { MEM_POOL* mem_pool = mem_pool_init(MB_SIZE); struct hash_index_config config; config.row_limit = limit; strcpy(config.work_space,"/tmp/hash_compress_test"); system("rm -rf /tmp/hash_compress_test"); mkdirs(config.work_space); init_profile(1000,mem_pool); struct hash_index_manager* hash_index = hash_index_init(&config,mem_pool); uint32_t i; struct low_data_struct* data = (struct low_data_struct*)mem_pool_malloc(mem_pool,sizeof(struct low_data_struct)); get_low_data(data,mem_pool); for(i=0; i<config.row_limit/2;i++) { strcpy((char*)data->data,"1"); hash_index_insert(hash_index,data,i); } for(i=config.row_limit/2; i<config.row_limit;i++) { strcpy((char*)data->data,"2"); hash_index_insert(hash_index,data,i); } struct hash_compress_manager* hash_compress = hash_compress_load(hash_index,10,mem_pool); struct rowid_list* rlist_a; struct rowid_list* rlist_b; get_low_data(data,mem_pool); strcpy((char*)data->data,"1"); rlist_a = hash_index_query(hash_index,data,mem_pool); rlist_b = hash_compress_query(hash_compress,data,mem_pool); ASSERT_EQ(rowid_list_equals(rlist_a,rlist_b),1); strcpy((char*)data->data,"2"); rlist_a = hash_index_query(hash_index,data,mem_pool); rlist_b = hash_compress_query(hash_compress,data,mem_pool); ASSERT_EQ(rowid_list_equals(rlist_a,rlist_b),1); hash_index_release(hash_index); hash_compress_release(hash_compress); }
int init_config( config_t *cfg) { profile_t *profile; int i; strcpy(cfg->config_filename, ""); strcpy(cfg->default_hostname, ""); cfg->profile_count = 0; for(i=0; i<CONFIG_MAX_PROFILES; i++) { profile = cfg->profiles + i; init_profile(profile); } return 0; }
/** * @brief * Called from machine independent code to do any machine * dependent initialization. * * @return Void * */ void dep_initialize() { MEMORYSTATUSEX mse; SYSTEM_INFO si; mse.dwLength = sizeof(MEMORYSTATUSEX); GlobalMemoryStatusEx(&mse); pmem_size = mse.ullTotalPhys / 1024; GetSystemInfo(&si); num_acpus = num_oscpus = num_pcpus = si.dwNumberOfProcessors; page_size = si.dwPageSize; if (!init_profile("\\System\\Processor Queue Length", &mom_prof)) { log_err(-1, "dep_initialize", "init_profile failed!"); return; } }
TEST(BITMARK_TEST, HandleNoneZeroInput) { int32 ret; int32 value; MEM_POOL* mem_pool = mem_pool_init(M_1M); struct bitmark_config config; strcpy(config.bitmark_name,"null"); strcpy(config.work_space,"/tmp/bitmark_test"); config.row_limit = 10; system("rm -rf /tmp/bitmark_test"); init_profile(1000,mem_pool); mkdirs(config.work_space); struct bitmark_manager* bitmark = bitmark_init(&config, mem_pool); //全部置零 bitmark_reset(bitmark); //置位 //表示6行已插入该行值 ret = bitmark_set(bitmark,6); ASSERT_EQ(ret,0); //查询 value = bitmark_query(bitmark,6); ASSERT_EQ(value,0); value = bitmark_query(bitmark,8); ASSERT_EQ(value,1); //删除6行的值 ret = bitmark_clear(bitmark,6); ASSERT_EQ(ret,0); mem_pool_destroy(mem_pool); }
TEST(BTREETEST, HandleNoneZeroInput) { MEM_POOL_PTR pMemPool = mem_pool_init(M_1M); BTREE_PTR pBtree = btree_init(pMemPool, "zbtree_index", "/tmp/", 3*MAX, HI_TYPE_LONGLONG); // BTREE_PTR pBtree = btree_init(pMemPool, "zbtree_index", "/tmp/", 3*MAX, HI_TYPE_LONG); BTREE_PTR pBtree2 = btree_init(pMemPool, "zbtree_index2", "/tmp/", 3*MAX, HI_TYPE_DOUBLE); init_profile(1000, pMemPool); /* struct timeval start, end, end2; double interval; gettimeofday(&start, NULL); srand((unsigned int)time(NULL)); int res,i; uint64 num; //插入10万数据 for ( i = 0; i < MAX; i += 1 ) { num = rand()%MAX; res = btree_insert(pMemPool, pBtree, num, i); ASSERT_EQ(res, 0); } gettimeofday(&end, NULL); interval = 1000000*(end.tv_sec - start.tv_sec) + (end.tv_usec - start.tv_usec); printf("insert %d data, cost %.2f ms\n", MAX, interval /1000); //范围查询10万次 BTREE_ENTRY_PTR pBtreeEntry; uint32 num2; // struct rowid_list *pRowidList; for ( i = 0; i < MAX; i += 1 ) { num = rand()%MAX+100; pBtreeEntry = btree_search(pBtree, num, CT_LE); // num2 = rand()%MAX+100; // struct rowid_list * pRowidList = btree_range_query(pBtree, num, 0, num2, 0, pMemPool); // if(pRowidList!=NULL) // { // printf("%d:", i); // print_rowid_list(pRowidList); // } // ASSERT_EQ((pBtreeEntry != NULL), 1); } gettimeofday(&end2, NULL); interval = 1000000*(end2.tv_sec - end.tv_sec) + (end2.tv_usec - end.tv_usec); printf("search %d data, cost %.2f ms\n", MAX, interval /1000); */ //插入3 7 7 9 11 2 4 5 6 8 int res = btree_insert(pMemPool, pBtree, 3, 1); ASSERT_EQ(res, 0); res = btree_insert(pMemPool, pBtree, 7, 2); ASSERT_EQ(res, 0); res = btree_insert(pMemPool, pBtree, 7, 10); ASSERT_EQ(res, 0); res = btree_insert(pMemPool, pBtree, 9, 3); ASSERT_EQ(res, 0); res = btree_insert(pMemPool, pBtree, 11, 4); ASSERT_EQ(res, 0); res = btree_insert(pMemPool, pBtree, 2, 5); ASSERT_EQ(res, 0); res = btree_insert(pMemPool, pBtree, 4, 6); ASSERT_EQ(res, 0); res = btree_insert(pMemPool, pBtree, 5, 7); ASSERT_EQ(res, 0); res = btree_insert(pMemPool, pBtree, 6, 8); ASSERT_EQ(res, 0); res = btree_insert(pMemPool, pBtree, 8, 9); ASSERT_EQ(res, 0); //等值查找3 9 8 // long long value = 3; BTREE_ENTRY_PTR pBtreeEntry = btree_search(pBtree, 3, CT_EQ); ASSERT_EQ((pBtreeEntry != NULL), 1); ASSERT_EQ(pBtreeEntry->key, 3); pBtreeEntry = btree_search(pBtree, 9, CT_EQ); ASSERT_EQ((pBtreeEntry != NULL), 1); ASSERT_EQ(pBtreeEntry->key, 9); pBtreeEntry = btree_search(pBtree, 1, CT_EQ); ASSERT_EQ((pBtreeEntry == NULL), 1); //查找<7 <6 pBtreeEntry = btree_search(pBtree, 7, CT_LT); ASSERT_EQ((pBtreeEntry != NULL), 1); ASSERT_EQ(pBtreeEntry->key, 6); pBtreeEntry = btree_search(pBtree, 6, CT_LT); ASSERT_EQ((pBtreeEntry != NULL), 1); ASSERT_EQ(pBtreeEntry->key, 5); //查找>7 >6 pBtreeEntry = btree_search(pBtree, 7, CT_GT); ASSERT_EQ((pBtreeEntry != NULL), 1); ASSERT_EQ(pBtreeEntry->key, 8); pBtreeEntry = btree_search(pBtree, 6, CT_GT); ASSERT_EQ((pBtreeEntry != NULL), 1); ASSERT_EQ(pBtreeEntry->key, 7); //测试btree next //BTREE_ENTRY_PTR pEntry = btree_search(pBtree, 2, 0); BTREE_ENTRY_PTR pEntry = btree_get_start_entry(pBtree); struct doc_row_unit *pUnit; while(pEntry != NULL) { std::cout << "key = " << pEntry->key << " docID: "; //输出doclist pUnit = doclist_next(pBtree->pDoclist, pEntry->doclist); while(pUnit != NULL) { std::cout << pUnit->doc_id << " "; pUnit = doclist_next(pBtree->pDoclist, pUnit->next); } pEntry = btree_next(pBtree, pEntry); std::cout << std::endl; } std::cout << std::endl; struct rowid_list * pRowidList = btree_range_query(pBtree, 0, 0, 10, 0, pMemPool); print_rowid_list(pRowidList); BTREE_KEY key; double value; //insert 3.1 11.11 111.111 1111.1 11111.1 11.11 value =3.1; memcpy(&key, &value, 8); res = btree_insert(pMemPool, pBtree2, key, 1); ASSERT_EQ(res, 0); value =11.11; memcpy(&key, &value, 8); res = btree_insert(pMemPool, pBtree2, key, 2); ASSERT_EQ(res, 0); value =111.111; memcpy(&key, &value, 8); res = btree_insert(pMemPool, pBtree2, key, 3); ASSERT_EQ(res, 0); value =1111.1; memcpy(&key, &value, 8); res = btree_insert(pMemPool, pBtree2, key, 4); ASSERT_EQ(res, 0); value =11111.1; memcpy(&key, &value, 8); res = btree_insert(pMemPool, pBtree2, key, 5); ASSERT_EQ(res, 0); value =11.11; memcpy(&key, &value, 8); res = btree_insert(pMemPool, pBtree2, key, 6); ASSERT_EQ(res, 0); value =1111; memcpy(&key, &value, 8); pRowidList = btree_range_query(pBtree2, 0, 0, key, 0, pMemPool); print_rowid_list(pRowidList); pEntry = btree_get_start_entry(pBtree2); while(pEntry != NULL) { std::cout << "key = " << *(double*)&pEntry->key << " docID: "; //输出doclist pUnit = doclist_next(pBtree2->pDoclist, pEntry->doclist); while(pUnit != NULL) { std::cout << pUnit->doc_id << " "; pUnit = doclist_next(pBtree2->pDoclist, pUnit->next); } pEntry = btree_next(pBtree2, pEntry); std::cout << std::endl; } std::cout << std::endl; //测试dump //btree_dump(pBtree); btree_destroy(pBtree); btree_destroy(pBtree2); }
TEST(HASHINDEX_TEST, HandleNoneZeroInput) { MEM_POOL* mem_pool = mem_pool_init(MB_SIZE); struct hash_index_config config; system("rm -rf /tmp/hashindex_test"); char dir_path[] = "/tmp/hashindex_test"; mkdirs(dir_path); config.row_limit = BUCKET_NUM; strcpy(config.work_space,dir_path); init_profile(1000,mem_pool); int32_t ret; //插入一个值 struct low_data_struct insert_data; get_low_data(&insert_data,mem_pool); struct hash_index_manager* hash_index = hash_index_init(&config,mem_pool); ret = hash_index_insert(hash_index,&insert_data,0); ASSERT_EQ(ret,0); //查询 struct rowid_list* rowids; uint32_t i; struct rowid_list_node* p; rowids = hash_index_query(hash_index,&insert_data,mem_pool); for(i = 0, p = rowids->head; i < rowids->rowid_num; i++) { if(i != 0 && i%ROWID_ARRAY_SIZE == 0) { p = p->next; } ASSERT_EQ(p->rowid_array[i%ROWID_ARRAY_SIZE],0); break; } //再插入一个值 hash_index_insert(hash_index,&insert_data,1); rowids = hash_index_query(hash_index,&insert_data,mem_pool); for(i = 0, p = rowids->head; i < rowids->rowid_num; i++) { if(i != 0 && i%ROWID_ARRAY_SIZE == 0) { p = p->next; if(i==0) ASSERT_EQ(p->rowid_array[i%ROWID_ARRAY_SIZE],1); if(i==1) { ASSERT_EQ(p->rowid_array[i%ROWID_ARRAY_SIZE],0); break; } } } //插入BUCKET_NUM个 for(i=2; i<BUCKET_NUM+1; i++) { sprintf((char*)insert_data.data,"ali%u",i); ret = hash_index_insert(hash_index,&insert_data,i); ASSERT_EQ(ret,0); } //再插一个报冲突了 sprintf((char*)insert_data.data,"ali%u",i); ret = hash_index_insert(hash_index,&insert_data,i); ASSERT_EQ(ret,ERROR_HASH_CONFLICT); //插入一个空值,OK insert_data.len = 0; ret = hash_index_insert(hash_index,&insert_data,i); ASSERT_EQ(ret,0); rowids = hash_index_query(hash_index,&insert_data,mem_pool); ASSERT_EQ(rowids->rowid_num,1); ASSERT_EQ(rowids->head->rowid_array[0],5); hash_index_release(hash_index); mem_pool_destroy(mem_pool); }
TEST(RECOVER_TEST_2, HandleNoneZeroInput) { MEM_POOL* mem_pool = mem_pool_init(MB_SIZE); struct hash_index_config config; system("rm -rf /tmp/hashindex_test"); char dir_path[] = "/tmp/hashindex_test"; mkdirs(dir_path); config.row_limit = ROW_LIMIT; strcpy(config.work_space,dir_path); init_profile(1000,mem_pool); int32_t ret; struct hash_index_manager* hash_index = hash_index_init(&config,mem_pool); //插入18条数据,只恢复16条 uint32_t i; struct low_data_struct* data = (struct low_data_struct*)mem_pool_malloc(mem_pool, sizeof(struct low_data_struct)); memset(data, 0, sizeof(struct low_data_struct)); for(i=0; i<18; i++) { if(i != 0 && i%2 == 1) continue; data->len = 5; data->data = mem_pool_malloc(mem_pool,5); data->type = HI_TYPE_STRING; data->field_name = (char*)mem_pool_malloc(mem_pool,20); memset(data->field_name,0,20); strcpy(data->field_name,"HI_TYPE_STRING"); memset(data->data,0,5); ret = hash_index_insert(hash_index,data,i); ASSERT_EQ(0, ret); ret = hash_index_insert(hash_index,data,i+1); ASSERT_EQ(0, ret); } struct doc_row_unit* doc = NULL; ASSERT_EQ(0, hash_index_recover(hash_index, 16)); for(i=0; i<ROW_LIMIT; i++) { doc = GET_DOC_ROW_STRUCT(hash_index->doclist, i); } //验证 for(i=0; i<16; i++) { doc = GET_DOC_ROW_STRUCT(hash_index->doclist, i); ASSERT_EQ(i, doc->doc_id); } for(i=16; i<ROW_LIMIT; i++) { doc = GET_DOC_ROW_STRUCT(hash_index->doclist, i); ASSERT_EQ(0, doc->doc_id); ASSERT_EQ(0,doc->next); } struct hash_bucket* hbucket = NULL; uint32_t bucket_no = 0; for(i=0; i<ROW_LIMIT+1; i++) { hbucket = hash_index->mem_mmaped + i; doc = NEXT_DOC_ROW_STRUCT(hash_index->doclist, hbucket->offset); if(hbucket->hash_value == 0) { continue; } while(!(doc->next & 0x80000000)) { ASSERT_GT(16, doc->doc_id); doc = NEXT_DOC_ROW_STRUCT(hash_index->doclist, doc->next); } bucket_no = doc->next & 0x7fffffff; ASSERT_EQ(i,bucket_no); } hash_index_release(hash_index); mem_pool_destroy(mem_pool); }
/******************** * bus_new_session ********************/ static DBusHandlerResult bus_new_session(DBusConnection *c, DBusMessage *msg, void *data) { char *address; DBusError error; (void)c; (void)data; dbus_error_init(&error); if (!dbus_message_get_args(msg, &error, DBUS_TYPE_STRING, &address, DBUS_TYPE_INVALID)) { if (dbus_error_is_set(&error)) { OHM_ERROR("Failed to parse session bus notification: %s.", error.message); dbus_error_free(&error); } else OHM_ERROR("Failed to parse session bus notification."); return DBUS_HANDLER_RESULT_NOT_YET_HANDLED; } if (!strcmp(address, "<failure>")) { OHM_INFO("profile: got session bus failure notification"); #ifndef __TEST__ OHM_INFO("profile: requesting ohm restart"); ohm_restart(0); #else OHM_INFO("profile: ignoring"); return DBUS_HANDLER_RESULT_NOT_YET_HANDLED; #endif } if (bus_conn != NULL) { OHM_INFO("profile: received new session bus address \"%s\".", address); dbus_connection_unref(bus_conn); bus_conn = NULL; } else OHM_INFO("profile: received session bus address \"%s\".", address); if ((bus_conn = dbus_connection_open(address, &error)) == NULL || !dbus_bus_register(bus_conn, &error)) { if (dbus_error_is_set(&error)) { OHM_ERROR("Failed to connect to DBUS %s (%s).", address, error.message); dbus_error_free(&error); } else OHM_ERROR("Failed to connect to DBUS %s.", address); return DBUS_HANDLER_RESULT_NOT_YET_HANDLED; } if (profile_plugin_p == NULL) { profile_plugin_p = init_profile(); if (profile_plugin_p != NULL) OHM_INFO("profile: initialized with session bus."); else OHM_ERROR("profile: failed to initialize with session bus."); } else { reconnect_profile(); OHM_INFO("profile: reinitialized with new session bus."); } return DBUS_HANDLER_RESULT_NOT_YET_HANDLED; }
int main(int argc, char* argv[]) { if (argc<2) { cout << "Usage: ./compute_profile dir" << endl; exit(1); } // directory setup string BASE_DIR = "/fs/nara-scratch/qwang37/brain_data/"+string(argv[1]); string partial_profile; string d2s, d2; // io streams on the final mean_conn_profile file ifstream in; ofstream out; //clock clock_t begin_t0; // read the dimensions int dim_profile, num_records; in.open("/fs/nara-scratch/qwang37/brain_data/scripts/dims"); in >> dim_profile; in >> num_records; in >> dimlow(); // initialize hash table string keyfile = "/fs/nara-scratch/qwang37/brain_data/scripts/keyfile"; // initialize the coarse map cout << "initializing coarse map" << endl; string coarsefile = "/fs/nara-scratch/qwang37/brain_data/scripts/coarse_map_file"; init_coarse(dim_profile, coarsefile); // compute conn_profile of subjects in a single data chunk cout << "Computing conn profile of data: " << argv[1] << endl; mkdir( (BASE_DIR + "/processed").c_str(), S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH ); d2s = exec("ls " + BASE_DIR + " -l | egrep '^d' | awk '$9~/^S/ {print $9}'"); stringstream d2s_in(d2s); d2s_in >> d2; while(d2s_in.good()) { // stringstream::goodbit is set to false when any of eofbit, failbit, or badbit is set string totaldir = BASE_DIR + '/' + d2; string connfile = totaldir + "/track_aal_90_0/fdt_matrix3.dot"; string coordfile = totaldir + "/track_aal_90_0/coords_pruned"; string coordfile_local2std = totaldir + "/track_aal_90_0/coords_standard"; string coordfile_std2local = totaldir + "/track_aal_90_0/coords_standard_in_diff"; // clear conn profile init_profile(dim_profile, num_records, keyfile, conn_profile()); // do something cout << "Processing " + totaldir << endl; begin_t0 = clock(); append_conn(connfile, coordfile, coordfile_local2std, coordfile_std2local); cout << "completed in " << float(clock()-begin_t0)/CLOCKS_PER_SEC << " seconds" << endl; // write partial_profile = BASE_DIR + '/' + d2 + "/partial_profile_std2local"; cout << "writing to disk...\n"; begin_t0 = clock(); write_profile(partial_profile); cout << "completed in " << float( (clock()-begin_t0)/CLOCKS_PER_SEC ) << " seconds" << endl; // move rename( (BASE_DIR + '/' + d2).c_str(), (BASE_DIR + "/processed/" + d2).c_str() ); d2s_in >> d2; } }