void fingerprint_cache_prefetch(int64_t id){
switch(destor.index_category[1]){
case INDEX_CATEGORY_PHYSICAL_LOCALITY:{
struct containerMeta * cm = retrieve_container_meta_by_id(id);
if (cm) {
lru_cache_insert(lru_queue, cm, NULL, NULL);
} else{
WARNING("Error! The container %lld has not been written!", id);
exit(1);
}
break;
}
case INDEX_CATEGORY_LOGICAL_LOCALITY:{
GQueue* segments = prefetch_segments(id,
destor.index_segment_prefech);
VERBOSE("Dedup phase: prefetch %d segments into %d cache",
g_queue_get_length(segments),
destor.index_cache_size);
struct segmentRecipe* sr;
while ((sr = g_queue_pop_tail(segments))) {
/* From tail to head */
if (!lru_cache_hits(lru_queue, &sr->id,
segment_recipe_check_id)) {
lru_cache_insert(lru_queue, sr, NULL, NULL);
} else {
/* Already in cache */
free_segment_recipe(sr);
}
}
g_queue_free(segments);
break;
}
}
}
static void* lru_restore_thread(void *arg) {
struct lruCache *cache;
if (destor.simulation_level >= SIMULATION_RESTORE)
cache = new_lru_cache(destor.restore_cache[1], free_container_meta,
lookup_fingerprint_in_container_meta);
else
cache = new_lru_cache(destor.restore_cache[1], free_container,
lookup_fingerprint_in_container);
struct chunk* c;
while ((c = sync_queue_pop(restore_recipe_queue))) {
if (CHECK_CHUNK(c, CHUNK_FILE_START) || CHECK_CHUNK(c, CHUNK_FILE_END)) {
sync_queue_push(restore_chunk_queue, c);
continue;
}
TIMER_DECLARE(1);
TIMER_BEGIN(1);
if (destor.simulation_level >= SIMULATION_RESTORE) {
struct containerMeta *cm = lru_cache_lookup(cache, &c->fp);
if (!cm) {
VERBOSE("Restore cache: container %lld is missed", c->id);
cm = retrieve_container_meta_by_id(c->id);
assert(lookup_fingerprint_in_container_meta(cm, &c->fp));
lru_cache_insert(cache, cm, NULL, NULL);
jcr.read_container_num++;
}
TIMER_END(1, jcr.read_chunk_time);
} else {
struct container *con = lru_cache_lookup(cache, &c->fp);
if (!con) {
VERBOSE("Restore cache: container %lld is missed", c->id);
con = retrieve_container_by_id(c->id);
lru_cache_insert(cache, con, NULL, NULL);
jcr.read_container_num++;
}
struct chunk *rc = get_chunk_in_container(con, &c->fp);
assert(rc);
TIMER_END(1, jcr.read_chunk_time);
sync_queue_push(restore_chunk_queue, rc);
}
free_chunk(c);
}
sync_queue_term(restore_chunk_queue);
free_lru_cache(cache);
return NULL;
}
/*
* Maintain a LRU cache internally to simulate recovery process when backing-up.
*/
void restore_aware_update(containerid id, int32_t chunklen) {
monitor.total_size += chunklen + CONTAINER_META_ENTRY;
struct containerRecord* record = lru_cache_lookup(monitor.cache, &id);
if (!record) {
record = (struct containerRecord*) malloc(
sizeof(struct containerRecord));
record->cid = id;
lru_cache_insert(monitor.cache, record, NULL, NULL);
monitor.ccf++;
}
monitor.ocf = (monitor.total_size + CONTAINER_SIZE - 1) / CONTAINER_SIZE;
monitor.cfl = monitor.ocf / (double) monitor.ccf;
}
Container *container_cache_insert_container(ContainerCache *cc, ContainerId cid) {
/* read container */
Container *container = 0;
TIMER_DECLARE(b, e);
TIMER_BEGIN(b);
if (cc->enable_data) {
if (simulation_level >= SIMULATION_RECOVERY) {
container = read_container_meta_only(cid);
} else {
container = read_container(cid);
}
} else {
container = read_container_meta_only(cid);
}
TIMER_END(read_container_time, b, e);
/* If this container is newly appended,
* maybe we can read nothing. */
if (container == NULL) {
return NULL;
}
/* insert */
Container *evictor = lru_cache_insert(cc->lru_cache, container);
/* evict */
if (evictor) {
int32_t chunknum = container_get_chunk_num(evictor);
Fingerprint *fingers = container_get_all_fingers(evictor);
int i = 0;
/* evict all fingers of evictor from map */
for (; i < chunknum; ++i) {
GSequence* container_list = g_hash_table_lookup(cc->map,
&fingers[i]);
/* remove the specified container from list */
GSequenceIter *iter = g_sequence_lookup(container_list, evictor,
container_cmp_des, NULL);
if (iter)
g_sequence_remove(iter);
else
dprint("Error! The sequence does not contain the container.");
if (g_sequence_get_length(container_list) == 0) {
g_hash_table_remove(cc->map, &fingers[i]);
}
}
free(fingers);
if (fragment_stream)
fprintf(fragment_stream, "%.4f\n",
1.0 * evictor->used_size / CONTAINER_SIZE);
container_free_full(evictor);
}
/* insert */
int32_t num = container_get_chunk_num(container);
Fingerprint *nfingers = container_get_all_fingers(container);
int i = 0;
for (; i < num; ++i) {
GSequence* container_list = g_hash_table_lookup(cc->map, &nfingers[i]);
if (container_list == 0) {
container_list = g_sequence_new(NULL);
Fingerprint *finger = (Fingerprint *) malloc(sizeof(Fingerprint));
memcpy(finger, &nfingers[i], sizeof(Fingerprint));
g_hash_table_insert(cc->map, finger, container_list);
}
g_sequence_insert_sorted(container_list, container, container_cmp_des,
NULL);
}
free(nfingers);
return container;
}
int main(){
printf("ok, it is done\n");
lru_cache_t * lru;
hash_map_t * hash_map;
int res = hash_map_init(&hash_map, &simple_hash_function, &key_cmp_cbf, &key_free_cbf, & key_clone_cbf);
printf("hash init result: %d\n", res);
int rest = lru_cache_init(&lru, &simple_hash_function, &key_cmp_cbf, &key_free_cbf, &key_clone_cbf);
printf("lru init result: %d\n", rest);
lru_dump(lru, &value_to_string, &key_to_string);
int i = 1;
test_key_t key;
test_value_t value;
test_value_t * vp;
int ret = 0;
while(i){
printf("0: quit\n1: insert\n2: lookup\n3: delete\nyour choice:");
scanf("%d", &i);
switch(i){
case 1:
printf("key:");
scanf("%19s", key.key);
printf("got key: %s\nvalue:", key.key);
scanf("%19s", value.value);
if(value.value[0] == '0') i=0;
ret = lru_cache_insert(lru, (const void *)(&key), (const void *)(&value), &value_clone_cbf, &value_free_cbf);
if(!ret){
printf("insert (%s, %s) failed", key.key, value.value);
}
break;
case 3:
ret = lru_delete_auto(lru);
if(!ret){
printf("delete failed\n");
}
break;
case 2:
printf("key:");
scanf("%19s", key.key);
ret = lru_cache_get(lru, (const void *) &key, (void **)(&vp));
if(!ret || !vp){
printf("key does not exist\n");
}
else{
printf("key=%s, value=%s\n", key_to_string((const void *)&key), value_to_string((const void *)vp));
}
break;
default:
i = 0;
break;
}
lru_dump(lru, value_to_string, key_to_string);
}
printf("quit cache; test hash map");
i = 1;
while(i){
printf("0: quit\n1: insert\n2: lookup\n3: delete\nyour choice:");
scanf("%d", &i);
switch(i){
case 1:
printf("key:");
scanf("%19s", key.key);
printf("got key: %s\nvalue:", key.key);
scanf("%19s", value.value);
if(value.value[0] == '0') i=0;
ret = hash_map_insert(hash_map, (const void *)(&key), (const void *)(&value), &value_clone_cbf, &value_free_cbf);
if(!ret){
printf("insert (%s, %s) failed", key.key, value.value);
}
break;
case 3:
printf("key:");
scanf("%19s", key.key);
ret = hash_map_delete_entry(hash_map, (const void *) &key);
if(!ret){
printf("delete failed\n");
}
break;
case 2:
printf("key:");
scanf("%19s", key.key);
ret = hash_map_lookup(hash_map, (const void *) &key, (void **)(&vp));
if(!ret || !vp){
printf("key does not exist\n");
}
else{
printf("key=%s, value=%s\n", key_to_string((const void *)&key), value_to_string((const void *)vp));
}
break;
default:
i = 0;
break;
}
hash_map_dump(hash_map, &key_to_string, &value_to_string);
}
hash_map_fini(hash_map);
printf("quit hash map\n");
return 0;
}
static void shard_cache_insert(shard_cache_t* dircache,
DIR_handle_t handle,
struct giga_directory* entry) {
lru_cache_insert(&(dircache->shards[shard_cache_get_shard(handle)]),
handle, entry);
}
