static void* sha1_thread(void* arg) {
char code[41];
while (1) {
struct chunk* c = sync_queue_pop(chunk_queue);
if (c == NULL) {
sync_queue_term(hash_queue);
break;
}
if (CHECK_CHUNK(c, CHUNK_FILE_START) || CHECK_CHUNK(c, CHUNK_FILE_END)) {
sync_queue_push(hash_queue, c);
continue;
}
TIMER_DECLARE(1);
TIMER_BEGIN(1);
SHA_CTX ctx;
SHA_Init(&ctx);
SHA_Update(&ctx, c->data, c->size);
SHA_Final(c->fp, &ctx);
TIMER_END(1, jcr.hash_time);
hash2code(c->fp, code);
code[40] = 0;
VERBOSE("Hash phase: %ldth chunk identified by %s", chunk_num++, code);
sync_queue_push(hash_queue, c);
}
return NULL;
}
static void* lru_restore_thread(void *arg) {
struct lruCache *cache;
if (destor.simulation_level >= SIMULATION_RESTORE)
cache = new_lru_cache(destor.restore_cache[1], free_container_meta,
lookup_fingerprint_in_container_meta);
else
cache = new_lru_cache(destor.restore_cache[1], free_container,
lookup_fingerprint_in_container);
struct chunk* c;
while ((c = sync_queue_pop(restore_recipe_queue))) {
if (CHECK_CHUNK(c, CHUNK_FILE_START) || CHECK_CHUNK(c, CHUNK_FILE_END)) {
sync_queue_push(restore_chunk_queue, c);
continue;
}
TIMER_DECLARE(1);
TIMER_BEGIN(1);
if (destor.simulation_level >= SIMULATION_RESTORE) {
struct containerMeta *cm = lru_cache_lookup(cache, &c->fp);
if (!cm) {
VERBOSE("Restore cache: container %lld is missed", c->id);
cm = retrieve_container_meta_by_id(c->id);
assert(lookup_fingerprint_in_container_meta(cm, &c->fp));
lru_cache_insert(cache, cm, NULL, NULL);
jcr.read_container_num++;
}
TIMER_END(1, jcr.read_chunk_time);
} else {
struct container *con = lru_cache_lookup(cache, &c->fp);
if (!con) {
VERBOSE("Restore cache: container %lld is missed", c->id);
con = retrieve_container_by_id(c->id);
lru_cache_insert(cache, con, NULL, NULL);
jcr.read_container_num++;
}
struct chunk *rc = get_chunk_in_container(con, &c->fp);
assert(rc);
TIMER_END(1, jcr.read_chunk_time);
sync_queue_push(restore_chunk_queue, rc);
}
free_chunk(c);
}
sync_queue_term(restore_chunk_queue);
free_lru_cache(cache);
return NULL;
}
void close_container_store() {
sync_queue_term(container_buffer);
pthread_join(append_t, NULL);
NOTICE("append phase stops successfully!");
fseek(fp, 0, SEEK_SET);
fwrite(&container_count, sizeof(container_count), 1, fp);
fclose(fp);
fp = NULL;
pthread_mutex_destroy(&mutex);
}
static void* read_recipe_thread(void *arg) {
int i, j, k;
for (i = 0; i < jcr.bv->number_of_files; i++) {
TIMER_DECLARE(1);
TIMER_BEGIN(1);
struct recipeMeta *r = read_next_recipe_meta(jcr.bv);
struct chunk *c = new_chunk(sdslen(r->filename) + 1);
strcpy(c->data, r->filename);
SET_CHUNK(c, CHUNK_FILE_START);
TIMER_END(1, jcr.read_recipe_time);
sync_queue_push(restore_recipe_queue, c);
jcr.file_num++;
for (j = 0; j < r->chunknum; j++) {
TIMER_DECLARE(1);
TIMER_BEGIN(1);
struct chunkPointer* cp = read_next_n_chunk_pointers(jcr.bv, 1, &k);
struct chunk* c = new_chunk(0);
memcpy(&c->fp, &cp->fp, sizeof(fingerprint));
c->size = cp->size;
c->id = cp->id;
TIMER_END(1, jcr.read_recipe_time);
jcr.data_size += c->size;
jcr.chunk_num++;
sync_queue_push(restore_recipe_queue, c);
free(cp);
}
c = new_chunk(0);
SET_CHUNK(c, CHUNK_FILE_END);
sync_queue_push(restore_recipe_queue, c);
free_recipe_meta(r);
}
sync_queue_term(restore_recipe_queue);
return NULL;
}
void *dedup_thread(void *arg) {
struct segment* s = NULL;
while (1) {
struct chunk *c = NULL;
if (destor.simulation_level != SIMULATION_ALL)
c = sync_queue_pop(hash_queue);
else
c = sync_queue_pop(trace_queue);
/* Add the chunk to the segment. */
s = segmenting(c);
if (!s)
continue;
/* segmenting success */
if (s->chunk_num > 0) {
VERBOSE("Dedup phase: the %lldth segment of %lld chunks", segment_num++,
s->chunk_num);
/* Each duplicate chunk will be marked. */
pthread_mutex_lock(&index_lock.mutex);
while (index_lookup(s) == 0) {
pthread_cond_wait(&index_lock.cond, &index_lock.mutex);
}
pthread_mutex_unlock(&index_lock.mutex);
} else {
VERBOSE("Dedup phase: an empty segment");
}
/* Send chunks in the segment to the next phase.
* The segment will be cleared. */
send_segment(s);
free_segment(s);
s = NULL;
if (c == NULL)
break;
}
sync_queue_term(dedup_queue);
return NULL;
}
static void* read_trace_thread(void *argv) {
FILE *trace_file = fopen(jcr.path, "r");
char line[128];
while (1) {
TIMER_DECLARE(1);
TIMER_BEGIN(1);
fgets(line, 128, trace_file);
TIMER_END(1, jcr.read_time);
if (strcmp(line, "stream end") == 0) {
sync_queue_term(trace_queue);
break;
}
struct chunk* c;
TIMER_BEGIN(1),
assert(strncmp(line, "file start ", 11) == 0);
int filenamelen;
sscanf(line, "file start %d", &filenamelen);
/* An additional '\n' is read */
c = new_chunk(filenamelen + 2);
fgets(c->data, filenamelen + 2, trace_file);
c->data[filenamelen] = 0;
VERBOSE("Reading: %s", c->data);
SET_CHUNK(c, CHUNK_FILE_START);
TIMER_END(1, jcr.read_time);
sync_queue_push(trace_queue, c);
TIMER_BEGIN(1);
fgets(line, 128, trace_file);
while (strncmp(line, "file end", 8) != 0) {
c = new_chunk(0);
char code[41];
strncpy(code, line, 40);
code2hash(code, c->fp);
c->size = atoi(line + 41);
jcr.chunk_num++;
jcr.data_size += c->size;
TIMER_END(1, jcr.read_time);
sync_queue_push(trace_queue, c);
TIMER_BEGIN(1),
fgets(line, 128, trace_file);
}
c = new_chunk(0);
SET_CHUNK(c, CHUNK_FILE_END);
sync_queue_push(trace_queue, c);
jcr.file_num++;
}
fclose(trace_file);
return NULL;
}
