static void* sha1_thread(void* arg) { char code[41]; while (1) { struct chunk* c = sync_queue_pop(chunk_queue); if (c == NULL) { sync_queue_term(hash_queue); break; } if (CHECK_CHUNK(c, CHUNK_FILE_START) || CHECK_CHUNK(c, CHUNK_FILE_END)) { sync_queue_push(hash_queue, c); continue; } TIMER_DECLARE(1); TIMER_BEGIN(1); SHA_CTX ctx; SHA_Init(&ctx); SHA_Update(&ctx, c->data, c->size); SHA_Final(c->fp, &ctx); TIMER_END(1, jcr.hash_time); hash2code(c->fp, code); code[40] = 0; VERBOSE("Hash phase: %ldth chunk identified by %s", chunk_num++, code); sync_queue_push(hash_queue, c); } return NULL; }
static void* lru_restore_thread(void *arg) { struct lruCache *cache; if (destor.simulation_level >= SIMULATION_RESTORE) cache = new_lru_cache(destor.restore_cache[1], free_container_meta, lookup_fingerprint_in_container_meta); else cache = new_lru_cache(destor.restore_cache[1], free_container, lookup_fingerprint_in_container); struct chunk* c; while ((c = sync_queue_pop(restore_recipe_queue))) { if (CHECK_CHUNK(c, CHUNK_FILE_START) || CHECK_CHUNK(c, CHUNK_FILE_END)) { sync_queue_push(restore_chunk_queue, c); continue; } TIMER_DECLARE(1); TIMER_BEGIN(1); if (destor.simulation_level >= SIMULATION_RESTORE) { struct containerMeta *cm = lru_cache_lookup(cache, &c->fp); if (!cm) { VERBOSE("Restore cache: container %lld is missed", c->id); cm = retrieve_container_meta_by_id(c->id); assert(lookup_fingerprint_in_container_meta(cm, &c->fp)); lru_cache_insert(cache, cm, NULL, NULL); jcr.read_container_num++; } TIMER_END(1, jcr.read_chunk_time); } else { struct container *con = lru_cache_lookup(cache, &c->fp); if (!con) { VERBOSE("Restore cache: container %lld is missed", c->id); con = retrieve_container_by_id(c->id); lru_cache_insert(cache, con, NULL, NULL); jcr.read_container_num++; } struct chunk *rc = get_chunk_in_container(con, &c->fp); assert(rc); TIMER_END(1, jcr.read_chunk_time); sync_queue_push(restore_chunk_queue, rc); } free_chunk(c); } sync_queue_term(restore_chunk_queue); free_lru_cache(cache); return NULL; }
static void* read_recipe_thread(void *arg) { int i, j, k; for (i = 0; i < jcr.bv->number_of_files; i++) { TIMER_DECLARE(1); TIMER_BEGIN(1); struct recipeMeta *r = read_next_recipe_meta(jcr.bv); struct chunk *c = new_chunk(sdslen(r->filename) + 1); strcpy(c->data, r->filename); SET_CHUNK(c, CHUNK_FILE_START); TIMER_END(1, jcr.read_recipe_time); sync_queue_push(restore_recipe_queue, c); jcr.file_num++; for (j = 0; j < r->chunknum; j++) { TIMER_DECLARE(1); TIMER_BEGIN(1); struct chunkPointer* cp = read_next_n_chunk_pointers(jcr.bv, 1, &k); struct chunk* c = new_chunk(0); memcpy(&c->fp, &cp->fp, sizeof(fingerprint)); c->size = cp->size; c->id = cp->id; TIMER_END(1, jcr.read_recipe_time); jcr.data_size += c->size; jcr.chunk_num++; sync_queue_push(restore_recipe_queue, c); free(cp); } c = new_chunk(0); SET_CHUNK(c, CHUNK_FILE_END); sync_queue_push(restore_recipe_queue, c); free_recipe_meta(r); } sync_queue_term(restore_recipe_queue); return NULL; }
void send_segment(struct segment* s) { /* * CHUNK_SEGMENT_START and _END are used for * reconstructing the segment in filter phase. */ struct chunk* ss = new_chunk(0); SET_CHUNK(ss, CHUNK_SEGMENT_START); sync_queue_push(dedup_queue, ss); GSequenceIter *end = g_sequence_get_end_iter(s->chunks); GSequenceIter *begin = g_sequence_get_begin_iter(s->chunks); while(begin != end) { struct chunk* c = g_sequence_get(begin); if (!CHECK_CHUNK(c, CHUNK_FILE_START) && !CHECK_CHUNK(c, CHUNK_FILE_END)) { if (CHECK_CHUNK(c, CHUNK_DUPLICATE)) { if (c->id == TEMPORARY_ID) { DEBUG("Dedup phase: %ldth chunk is identical to a unique chunk", chunk_num++); } else { DEBUG("Dedup phase: %ldth chunk is duplicate in container %lld", chunk_num++, c->id); } } else { DEBUG("Dedup phase: %ldth chunk is unique", chunk_num++); } } sync_queue_push(dedup_queue, c); g_sequence_remove(begin); begin = g_sequence_get_begin_iter(s->chunks); } struct chunk* se = new_chunk(0); SET_CHUNK(se, CHUNK_SEGMENT_END); sync_queue_push(dedup_queue, se); s->chunk_num = 0; }
void write_container_async(struct container* c) { assert(c->meta.chunk_num == g_hash_table_size(c->meta.map)); if (container_empty(c)) { /* An empty container * It possibly occurs in the end of backup */ container_count--; VERBOSE("Append phase: Deny writing an empty container %lld", c->meta.id); return; } sync_queue_push(container_buffer, c); }
static void* read_trace_thread(void *argv) { FILE *trace_file = fopen(jcr.path, "r"); char line[128]; while (1) { TIMER_DECLARE(1); TIMER_BEGIN(1); fgets(line, 128, trace_file); TIMER_END(1, jcr.read_time); if (strcmp(line, "stream end") == 0) { sync_queue_term(trace_queue); break; } struct chunk* c; TIMER_BEGIN(1), assert(strncmp(line, "file start ", 11) == 0); int filenamelen; sscanf(line, "file start %d", &filenamelen); /* An additional '\n' is read */ c = new_chunk(filenamelen + 2); fgets(c->data, filenamelen + 2, trace_file); c->data[filenamelen] = 0; VERBOSE("Reading: %s", c->data); SET_CHUNK(c, CHUNK_FILE_START); TIMER_END(1, jcr.read_time); sync_queue_push(trace_queue, c); TIMER_BEGIN(1); fgets(line, 128, trace_file); while (strncmp(line, "file end", 8) != 0) { c = new_chunk(0); char code[41]; strncpy(code, line, 40); code2hash(code, c->fp); c->size = atoi(line + 41); jcr.chunk_num++; jcr.data_size += c->size; TIMER_END(1, jcr.read_time); sync_queue_push(trace_queue, c); TIMER_BEGIN(1), fgets(line, 128, trace_file); } c = new_chunk(0); SET_CHUNK(c, CHUNK_FILE_END); sync_queue_push(trace_queue, c); jcr.file_num++; } fclose(trace_file); return NULL; }
void send_fingerchunk(FingerChunk *fchunk, EigenValue* eigenvalue, BOOL update) { index_update(&fchunk->fingerprint, fchunk->container_id, eigenvalue, update); update_cfl(cfl_monitor, fchunk->container_id, fchunk->length); sync_queue_push(fingerchunk_queue, fchunk); }
void send_fc_signal() { FingerChunk *sigfc = (FingerChunk*) malloc(sizeof(FingerChunk)); sigfc->length = STREAM_END; sync_queue_push(fingerchunk_queue, sigfc); }