static void* read_recipe_thread(void *arg) {
int i, j, k;
for (i = 0; i < jcr.bv->number_of_files; i++) {
TIMER_DECLARE(1);
TIMER_BEGIN(1);
struct recipeMeta *r = read_next_recipe_meta(jcr.bv);
struct chunk *c = new_chunk(sdslen(r->filename) + 1);
strcpy(c->data, r->filename);
SET_CHUNK(c, CHUNK_FILE_START);
TIMER_END(1, jcr.read_recipe_time);
sync_queue_push(restore_recipe_queue, c);
jcr.file_num++;
for (j = 0; j < r->chunknum; j++) {
TIMER_DECLARE(1);
TIMER_BEGIN(1);
struct chunkPointer* cp = read_next_n_chunk_pointers(jcr.bv, 1, &k);
struct chunk* c = new_chunk(0);
memcpy(&c->fp, &cp->fp, sizeof(fingerprint));
c->size = cp->size;
c->id = cp->id;
TIMER_END(1, jcr.read_recipe_time);
jcr.data_size += c->size;
jcr.chunk_num++;
sync_queue_push(restore_recipe_queue, c);
free(cp);
}
c = new_chunk(0);
SET_CHUNK(c, CHUNK_FILE_END);
sync_queue_push(restore_recipe_queue, c);
free_recipe_meta(r);
}
sync_queue_term(restore_recipe_queue);
return NULL;
}
static void* sha1_thread(void* arg) {
char code[41];
while (1) {
struct chunk* c = sync_queue_pop(chunk_queue);
if (c == NULL) {
sync_queue_term(hash_queue);
break;
}
if (CHECK_CHUNK(c, CHUNK_FILE_START) || CHECK_CHUNK(c, CHUNK_FILE_END)) {
sync_queue_push(hash_queue, c);
continue;
}
TIMER_DECLARE(1);
TIMER_BEGIN(1);
SHA_CTX ctx;
SHA_Init(&ctx);
SHA_Update(&ctx, c->data, c->size);
SHA_Final(c->fp, &ctx);
TIMER_END(1, jcr.hash_time);
hash2code(c->fp, code);
code[40] = 0;
VERBOSE("Hash phase: %ldth chunk identified by %s", chunk_num++, code);
sync_queue_push(hash_queue, c);
}
return NULL;
}
static void* lru_restore_thread(void *arg) {
struct lruCache *cache;
if (destor.simulation_level >= SIMULATION_RESTORE)
cache = new_lru_cache(destor.restore_cache[1], free_container_meta,
lookup_fingerprint_in_container_meta);
else
cache = new_lru_cache(destor.restore_cache[1], free_container,
lookup_fingerprint_in_container);
struct chunk* c;
while ((c = sync_queue_pop(restore_recipe_queue))) {
if (CHECK_CHUNK(c, CHUNK_FILE_START) || CHECK_CHUNK(c, CHUNK_FILE_END)) {
sync_queue_push(restore_chunk_queue, c);
continue;
}
TIMER_DECLARE(1);
TIMER_BEGIN(1);
if (destor.simulation_level >= SIMULATION_RESTORE) {
struct containerMeta *cm = lru_cache_lookup(cache, &c->fp);
if (!cm) {
VERBOSE("Restore cache: container %lld is missed", c->id);
cm = retrieve_container_meta_by_id(c->id);
assert(lookup_fingerprint_in_container_meta(cm, &c->fp));
lru_cache_insert(cache, cm, NULL, NULL);
jcr.read_container_num++;
}
TIMER_END(1, jcr.read_chunk_time);
} else {
struct container *con = lru_cache_lookup(cache, &c->fp);
if (!con) {
VERBOSE("Restore cache: container %lld is missed", c->id);
con = retrieve_container_by_id(c->id);
lru_cache_insert(cache, con, NULL, NULL);
jcr.read_container_num++;
}
struct chunk *rc = get_chunk_in_container(con, &c->fp);
assert(rc);
TIMER_END(1, jcr.read_chunk_time);
sync_queue_push(restore_chunk_queue, rc);
}
free_chunk(c);
}
sync_queue_term(restore_chunk_queue);
free_lru_cache(cache);
return NULL;
}
/*
* We must ensure a container is either in the buffer or written to disks.
*/
static void* append_thread(void *arg) {
while (1) {
struct container *c = sync_queue_get_top(container_buffer);
if (c == NULL)
break;
TIMER_DECLARE(1);
TIMER_BEGIN(1);
write_container(c);
TIMER_END(1, jcr.write_time);
sync_queue_pop(container_buffer);
free_container(c);
}
return NULL;
}
void har_monitor_update(containerid id, int32_t size) {
TIMER_DECLARE(1);
TIMER_BEGIN(1);
struct containerRecord* record = g_hash_table_lookup(
container_utilization_monitor, &id);
if (record) {
record->size += size;
} else {
record = (struct containerRecord*) malloc(
sizeof(struct containerRecord));
record->cid = id;
record->size = 0;
g_hash_table_insert(container_utilization_monitor,
&record->cid, record);
record->size += size;
}
TIMER_END(1, jcr.rewrite_time);
}
static void* read_trace_thread(void *argv) {
FILE *trace_file = fopen(jcr.path, "r");
char line[128];
while (1) {
TIMER_DECLARE(1);
TIMER_BEGIN(1);
fgets(line, 128, trace_file);
TIMER_END(1, jcr.read_time);
if (strcmp(line, "stream end") == 0) {
sync_queue_term(trace_queue);
break;
}
struct chunk* c;
TIMER_BEGIN(1),
assert(strncmp(line, "file start ", 11) == 0);
int filenamelen;
sscanf(line, "file start %d", &filenamelen);
/* An additional '\n' is read */
c = new_chunk(filenamelen + 2);
fgets(c->data, filenamelen + 2, trace_file);
c->data[filenamelen] = 0;
VERBOSE("Reading: %s", c->data);
SET_CHUNK(c, CHUNK_FILE_START);
TIMER_END(1, jcr.read_time);
sync_queue_push(trace_queue, c);
TIMER_BEGIN(1);
fgets(line, 128, trace_file);
while (strncmp(line, "file end", 8) != 0) {
c = new_chunk(0);
char code[41];
strncpy(code, line, 40);
code2hash(code, c->fp);
c->size = atoi(line + 41);
jcr.chunk_num++;
jcr.data_size += c->size;
TIMER_END(1, jcr.read_time);
sync_queue_push(trace_queue, c);
TIMER_BEGIN(1),
fgets(line, 128, trace_file);
}
c = new_chunk(0);
SET_CHUNK(c, CHUNK_FILE_END);
sync_queue_push(trace_queue, c);
jcr.file_num++;
}
fclose(trace_file);
return NULL;
}
void do_backup(char *path) {
init_recipe_store();
init_container_store();
init_index();
init_backup_jcr(path);
NOTICE("\n\n==== backup begin ====");
TIMER_DECLARE(1);
TIMER_BEGIN(1);
time_t start = time(NULL);
if (destor.simulation_level == SIMULATION_ALL) {
start_read_trace_phase();
} else {
start_read_phase();
start_chunk_phase();
start_hash_phase();
}
start_dedup_phase();
start_rewrite_phase();
start_filter_phase();
do{
sleep(5);
/*time_t now = time(NULL);*/
fprintf(stderr,"job %" PRId32 ", data size %" PRId64 " bytes, %" PRId32 " chunks, %d files processed\r",
jcr.id, jcr.data_size, jcr.chunk_num, jcr.file_num);
}while(jcr.status == JCR_STATUS_RUNNING || jcr.status != JCR_STATUS_DONE);
fprintf(stderr,"job %" PRId32 ", data size %" PRId64 " bytes, %" PRId32 " chunks, %d files processed\n",
jcr.id, jcr.data_size, jcr.chunk_num, jcr.file_num);
if (destor.simulation_level == SIMULATION_ALL) {
stop_read_trace_phase();
} else {
stop_read_phase();
stop_chunk_phase();
stop_hash_phase();
}
stop_dedup_phase();
stop_rewrite_phase();
stop_filter_phase();
TIMER_END(1, jcr.total_time);
close_index();
close_container_store();
close_recipe_store();
update_backup_version(jcr.bv);
free_backup_version(jcr.bv);
printf("\n\njob id: %" PRId32 "\n", jcr.id);
printf("index method: %d.(Remark 0: NO; 1: DDFS; 2: Extreme binning; 3: Silo; 4: Sparse; 5: Sampled; 6: Block; 7: Learn)\n",
destor.index_specific);
printf("sampling method: %d (%d) (Remark 1:Random; 2: Min; 3: Uniform; 4: Optimized_min)\n", destor.index_sampling_method[0], destor.index_sampling_method[1]);
printf("segment method: %d (%d) (Remark 0: Fixed; 1: Content; 2: File)\n", destor.index_segment_algorithm[0], destor.index_segment_algorithm[1]);
printf("prefetch # of segments: %d (Remark 1 for sparse index)\n", destor.index_segment_prefech);
printf("segment selection method: %d (%d)(Remark 0: Base; 1: Top; 2: Mix)\n", destor.index_segment_selection_method[0], destor.index_segment_selection_method[1]);
printf("backup path: %s\n", jcr.path);
printf("number of files: %d\n", jcr.file_num);
printf("number of chunks: %" PRId32 " (%" PRId64 " bytes on average)\n", jcr.chunk_num,
jcr.data_size / jcr.chunk_num);
printf("number of unique chunks: %" PRId32 "\n", jcr.unique_chunk_num);
printf("total size(B): %" PRId64 "\n", jcr.data_size);
printf("stored data size(B): %" PRId64 "\n",
jcr.unique_data_size + jcr.rewritten_chunk_size);
printf("deduplication ratio: %.4f, %.4f\n",
jcr.data_size != 0 ?
(jcr.data_size - jcr.unique_data_size
- jcr.rewritten_chunk_size)
/ (double) (jcr.data_size) :
0,
jcr.data_size
/ (double) (jcr.unique_data_size + jcr.rewritten_chunk_size));
printf("total time(s): %.3f\n", jcr.total_time / 1000000);
printf("the index memory footprint (B): %" PRId32 "\n", destor.index_memory_footprint);
printf("throughput(MB/s): %.2f\n",
(double) jcr.data_size * 1000000 / (1024 * 1024 * jcr.total_time));
printf("number of zero chunks: %" PRId32 "\n", jcr.zero_chunk_num);
printf("size of zero chunks: %" PRId64 "\n", jcr.zero_chunk_size);
printf("number of rewritten chunks: %" PRId32 "\n", jcr.rewritten_chunk_num);
printf("size of rewritten chunks: %" PRId64 "\n", jcr.rewritten_chunk_size);
printf("rewritten rate in size: %.3f\n",
jcr.rewritten_chunk_size / (double) jcr.data_size);
destor.data_size += jcr.data_size;
destor.stored_data_size += jcr.unique_data_size + jcr.rewritten_chunk_size;
destor.chunk_num += jcr.chunk_num;
destor.stored_chunk_num += jcr.unique_chunk_num + jcr.rewritten_chunk_num;
destor.zero_chunk_num += jcr.zero_chunk_num;
destor.zero_chunk_size += jcr.zero_chunk_size;
destor.rewritten_chunk_num += jcr.rewritten_chunk_num;
destor.rewritten_chunk_size += jcr.rewritten_chunk_size;
printf("read_time : %.3fs, %.2fMB/s\n", jcr.read_time / 1000000,
jcr.data_size * 1000000 / jcr.read_time / 1024 / 1024);
printf("chunk_time : %.3fs, %.2fMB/s\n", jcr.chunk_time / 1000000,
jcr.data_size * 1000000 / jcr.chunk_time / 1024 / 1024);
printf("hash_time : %.3fs, %.2fMB/s\n", jcr.hash_time / 1000000,
jcr.data_size * 1000000 / jcr.hash_time / 1024 / 1024);
printf("dedup_time : %.3fs, %.2fMB/s\n",
jcr.dedup_time / 1000000,
jcr.data_size * 1000000 / jcr.dedup_time / 1024 / 1024);
printf("rewrite_time : %.3fs, %.2fMB/s\n", jcr.rewrite_time / 1000000,
jcr.data_size * 1000000 / jcr.rewrite_time / 1024 / 1024);
printf("filter_time : %.3fs, %.2fMB/s\n",
jcr.filter_time / 1000000,
jcr.data_size * 1000000 / jcr.filter_time / 1024 / 1024);
printf("write_time : %.3fs, %.2fMB/s\n", jcr.write_time / 1000000,
jcr.data_size * 1000000 / jcr.write_time / 1024 / 1024);
//double seek_time = 0.005; //5ms
//double bandwidth = 120 * 1024 * 1024; //120MB/s
/* double index_lookup_throughput = jcr.data_size
/ (index_read_times * seek_time
+ index_read_entry_counter * 24 / bandwidth) / 1024 / 1024;
double write_data_throughput = 1.0 * jcr.data_size * bandwidth
/ (jcr->unique_chunk_num) / 1024 / 1024;
double index_read_throughput = 1.0 * jcr.data_size / 1024 / 1024
/ (index_read_times * seek_time
+ index_read_entry_counter * 24 / bandwidth);
double index_write_throughput = 1.0 * jcr.data_size / 1024 / 1024
/ (index_write_times * seek_time
+ index_write_entry_counter * 24 / bandwidth);*/
/* double estimated_throughput = write_data_throughput;
if (estimated_throughput > index_read_throughput)
estimated_throughput = index_read_throughput;*/
/*if (estimated_throughput > index_write_throughput)
estimated_throughput = index_write_throughput;*/
char logfile[] = "backup.log";
FILE *fp = fopen(logfile, "a");
/*
* job id,
* the size of backup
* accumulative consumed capacity,
* deduplication rate,
* rewritten rate,
* total container number,
* sparse container number,
* inherited container number,
* 4 * index overhead (4 * int)
* throughput,
*/
fprintf(fp, "%" PRId32 " %" PRId64 " %" PRId64 " %.4f %.4f %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32" %" PRId32 " %" PRId32" %" PRId32" %.2f\n",
jcr.id,
jcr.data_size,
destor.stored_data_size,
jcr.data_size != 0 ?
(jcr.data_size - jcr.rewritten_chunk_size - jcr.unique_data_size)/(double) (jcr.data_size)
: 0,
jcr.data_size != 0 ? (double) (jcr.rewritten_chunk_size) / (double) (jcr.data_size) : 0,
jcr.total_container_num,
jcr.sparse_container_num,
jcr.inherited_sparse_num,
index_overhead.lookup_requests,
index_overhead.lookup_requests_for_unique,
index_overhead.update_requests,
index_overhead.read_prefetching_units,
(double) jcr.data_size * 1000000 / (1024 * 1024 * jcr.total_time));
fclose(fp);
}
void Shutdown(int flags)
{
if ((flags & SHUTDOWN_FROM_CONFIG))
goto from_config;
EndGame();
SAFE_DELETE(g_XmppClient);
ShutdownPs();
TIMER_BEGIN(L"shutdown TexMan");
delete &g_TexMan;
TIMER_END(L"shutdown TexMan");
// destroy renderer
TIMER_BEGIN(L"shutdown Renderer");
delete &g_Renderer;
g_VBMan.Shutdown();
TIMER_END(L"shutdown Renderer");
g_Profiler2.ShutdownGPU();
#if OS_WIN
TIMER_BEGIN(L"shutdown wmi");
wmi_Shutdown();
TIMER_END(L"shutdown wmi");
#endif
// Free cursors before shutting down SDL, as they may depend on SDL.
cursor_shutdown();
TIMER_BEGIN(L"shutdown SDL");
ShutdownSDL();
TIMER_END(L"shutdown SDL");
g_VideoMode.Shutdown();
TIMER_BEGIN(L"shutdown UserReporter");
g_UserReporter.Deinitialize();
TIMER_END(L"shutdown UserReporter");
// JS debugger temporarily disabled during the SpiderMonkey upgrade (check trac ticket #2348 for details)
//TIMER_BEGIN(L"shutdown DebuggingServer (if active)");
//delete g_DebuggingServer;
//TIMER_END(L"shutdown DebuggingServer (if active)");
delete &g_L10n;
from_config:
TIMER_BEGIN(L"shutdown ConfigDB");
delete &g_ConfigDB;
TIMER_END(L"shutdown ConfigDB");
SAFE_DELETE(g_Console);
// This is needed to ensure that no callbacks from the JSAPI try to use
// the profiler when it's already destructed
g_ScriptRuntime.reset();
// resource
// first shut down all resource owners, and then the handle manager.
TIMER_BEGIN(L"resource modules");
ISoundManager::SetEnabled(false);
g_VFS.reset();
// this forcibly frees all open handles (thus preventing real leaks),
// and makes further access to h_mgr impossible.
h_mgr_shutdown();
file_stats_dump();
TIMER_END(L"resource modules");
TIMER_BEGIN(L"shutdown misc");
timer_DisplayClientTotals();
CNetHost::Deinitialize();
// should be last, since the above use them
SAFE_DELETE(g_Logger);
delete &g_Profiler;
delete &g_ProfileViewer;
SAFE_DELETE(g_ScriptStatsTable);
TIMER_END(L"shutdown misc");
}
void Shutdown(int UNUSED(flags))
{
EndGame();
ShutdownPs(); // Must delete g_GUI before g_ScriptingHost
in_reset_handlers();
TIMER_BEGIN(L"shutdown TexMan");
delete &g_TexMan;
TIMER_END(L"shutdown TexMan");
// destroy renderer
TIMER_BEGIN(L"shutdown Renderer");
delete &g_Renderer;
g_VBMan.Shutdown();
TIMER_END(L"shutdown Renderer");
tex_codec_unregister_all();
g_Profiler2.ShutdownGPU();
// Free cursors before shutting down SDL, as they may depend on SDL.
cursor_shutdown();
TIMER_BEGIN(L"shutdown SDL");
ShutdownSDL();
TIMER_END(L"shutdown SDL");
g_VideoMode.Shutdown();
TIMER_BEGIN(L"shutdown UserReporter");
g_UserReporter.Deinitialize();
TIMER_END(L"shutdown UserReporter");
TIMER_BEGIN(L"shutdown ScriptingHost");
delete &g_ScriptingHost;
TIMER_END(L"shutdown ScriptingHost");
TIMER_BEGIN(L"shutdown ConfigDB");
delete &g_ConfigDB;
TIMER_END(L"shutdown ConfigDB");
// resource
// first shut down all resource owners, and then the handle manager.
TIMER_BEGIN(L"resource modules");
#if CONFIG2_AUDIO
if (g_SoundManager)
delete g_SoundManager;
#endif
g_VFS.reset();
// this forcibly frees all open handles (thus preventing real leaks),
// and makes further access to h_mgr impossible.
h_mgr_shutdown();
file_stats_dump();
TIMER_END(L"resource modules");
TIMER_BEGIN(L"shutdown misc");
timer_DisplayClientTotals();
CNetHost::Deinitialize();
SAFE_DELETE(g_ScriptStatsTable);
// should be last, since the above use them
SAFE_DELETE(g_Logger);
delete &g_Profiler;
delete &g_ProfileViewer;
TIMER_END(L"shutdown misc");
#if OS_WIN
TIMER_BEGIN(L"shutdown wmi");
wmi_Shutdown();
TIMER_END(L"shutdown wmi");
#endif
}
Container *container_cache_insert_container(ContainerCache *cc, ContainerId cid) {
/* read container */
Container *container = 0;
TIMER_DECLARE(b, e);
TIMER_BEGIN(b);
if (cc->enable_data) {
if (simulation_level >= SIMULATION_RECOVERY) {
container = read_container_meta_only(cid);
} else {
container = read_container(cid);
}
} else {
container = read_container_meta_only(cid);
}
TIMER_END(read_container_time, b, e);
/* If this container is newly appended,
* maybe we can read nothing. */
if (container == NULL) {
return NULL;
}
/* insert */
Container *evictor = lru_cache_insert(cc->lru_cache, container);
/* evict */
if (evictor) {
int32_t chunknum = container_get_chunk_num(evictor);
Fingerprint *fingers = container_get_all_fingers(evictor);
int i = 0;
/* evict all fingers of evictor from map */
for (; i < chunknum; ++i) {
GSequence* container_list = g_hash_table_lookup(cc->map,
&fingers[i]);
/* remove the specified container from list */
GSequenceIter *iter = g_sequence_lookup(container_list, evictor,
container_cmp_des, NULL);
if (iter)
g_sequence_remove(iter);
else
dprint("Error! The sequence does not contain the container.");
if (g_sequence_get_length(container_list) == 0) {
g_hash_table_remove(cc->map, &fingers[i]);
}
}
free(fingers);
if (fragment_stream)
fprintf(fragment_stream, "%.4f\n",
1.0 * evictor->used_size / CONTAINER_SIZE);
container_free_full(evictor);
}
/* insert */
int32_t num = container_get_chunk_num(container);
Fingerprint *nfingers = container_get_all_fingers(container);
int i = 0;
for (; i < num; ++i) {
GSequence* container_list = g_hash_table_lookup(cc->map, &nfingers[i]);
if (container_list == 0) {
container_list = g_sequence_new(NULL);
Fingerprint *finger = (Fingerprint *) malloc(sizeof(Fingerprint));
memcpy(finger, &nfingers[i], sizeof(Fingerprint));
g_hash_table_insert(cc->map, finger, container_list);
}
g_sequence_insert_sorted(container_list, container, container_cmp_des,
NULL);
}
free(nfingers);
return container;
}
void do_restore(int revision, char *path) {
init_recipe_store();
init_container_store();
init_restore_jcr(revision, path);
destor_log(DESTOR_NOTICE, "job id: %d", jcr.id);
destor_log(DESTOR_NOTICE, "backup path: %s", jcr.bv->path);
destor_log(DESTOR_NOTICE, "restore to: %s", jcr.path);
restore_chunk_queue = sync_queue_new(100);
restore_recipe_queue = sync_queue_new(100);
TIMER_DECLARE(1);
TIMER_BEGIN(1);
puts("==== restore begin ====");
pthread_t recipe_t, read_t;
pthread_create(&recipe_t, NULL, read_recipe_thread, NULL);
if (destor.restore_cache[0] == RESTORE_CACHE_LRU) {
destor_log(DESTOR_NOTICE, "restore cache is LRU");
pthread_create(&read_t, NULL, lru_restore_thread, NULL);
} else if (destor.restore_cache[0] == RESTORE_CACHE_OPT) {
destor_log(DESTOR_NOTICE, "restore cache is OPT");
pthread_create(&read_t, NULL, optimal_restore_thread, NULL);
} else if (destor.restore_cache[0] == RESTORE_CACHE_ASM) {
destor_log(DESTOR_NOTICE, "restore cache is ASM");
pthread_create(&read_t, NULL, assembly_restore_thread, NULL);
} else {
fprintf(stderr, "Invalid restore cache.\n");
exit(1);
}
write_restore_data();
assert(sync_queue_size(restore_chunk_queue) == 0);
assert(sync_queue_size(restore_recipe_queue) == 0);
free_backup_version(jcr.bv);
TIMER_END(1, jcr.total_time);
puts("==== restore end ====");
printf("job id: %d\n", jcr.id);
printf("restore path: %s\n", jcr.path);
printf("number of files: %d\n", jcr.file_num);
printf("number of chunks: %d\n", jcr.chunk_num);
printf("total size(B): %ld\n", jcr.data_size);
printf("total time(s): %.3f\n", jcr.total_time / 1000000);
printf("throughput(MB/s): %.2f\n",
jcr.data_size * 1000000 / (1024.0 * 1024 * jcr.total_time));
printf("speed factor: %.2f\n",
jcr.data_size / (1024.0 * 1024 * jcr.read_container_num));
printf("read_recipe_time : %.3fs, %.2fMB/s\n",
jcr.read_recipe_time / 1000000,
jcr.data_size * 1000000 / jcr.read_recipe_time / 1024 / 1024);
printf("read_chunk_time : %.3fs, %.2fMB/s\n", jcr.read_chunk_time / 1000000,
jcr.data_size * 1000000 / jcr.read_chunk_time / 1024 / 1024);
printf("write_chunk_time : %.3fs, %.2fMB/s\n",
jcr.write_chunk_time / 1000000,
jcr.data_size * 1000000 / jcr.write_chunk_time / 1024 / 1024);
char logfile[] = "restore.log";
FILE *fp = fopen(logfile, "a");
/*
* job id,
* chunk num,
* data size,
* actually read container number,
* speed factor,
* throughput
*/
fprintf(fp, "%d %lld %d %.4f %.4f\n", jcr.id, jcr.data_size,
jcr.read_container_num,
jcr.data_size / (1024.0 * 1024 * jcr.read_container_num),
jcr.data_size * 1000000 / (1024 * 1024 * jcr.total_time));
fclose(fp);
close_container_store();
close_recipe_store();
}
void write_restore_data() {
char *p, *q;
q = jcr.path + 1;/* ignore the first char*/
/*
* recursively make directory
*/
while ((p = strchr(q, '/'))) {
if (*p == *(p - 1)) {
q++;
continue;
}
*p = 0;
if (access(jcr.path, 0) != 0) {
mkdir(jcr.path, S_IRWXU | S_IRWXG | S_IRWXO);
}
*p = '/';
q = p + 1;
}
struct chunk *c = NULL;
FILE *fp = NULL;
while ((c = sync_queue_pop(restore_chunk_queue))) {
TIMER_DECLARE(1);
TIMER_BEGIN(1);
if (CHECK_CHUNK(c, CHUNK_FILE_START)) {
NOTICE("Restoring: %s", c->data);
sds filepath = sdsdup(jcr.path);
filepath = sdscat(filepath, c->data);
int len = sdslen(jcr.path);
char *q = filepath + len;
char *p;
while ((p = strchr(q, '/'))) {
if (*p == *(p - 1)) {
q++;
continue;
}
*p = 0;
if (access(filepath, 0) != 0) {
mkdir(filepath, S_IRWXU | S_IRWXG | S_IRWXO);
}
*p = '/';
q = p + 1;
}
if (destor.simulation_level == SIMULATION_NO) {
assert(fp == NULL);
fp = fopen(filepath, "w");
}
sdsfree(filepath);
} else if (CHECK_CHUNK(c, CHUNK_FILE_END)) {
if (fp)
fclose(fp);
fp = NULL;
} else {
assert(destor.simulation_level == SIMULATION_NO);
VERBOSE("Restoring %d bytes", c->size);
fwrite(c->data, c->size, 1, fp);
}
free_chunk(c);
TIMER_END(1, jcr.write_chunk_time);
}
}
int main( int argc, char* argv[] ) {
gnd::opsm::cmap_t cmap; // counting map
gnd::opsm::map_t map; // scan matching map
gnd::vector::fixed_column<3> pos; // matching result
FILE *fp; // matched scan data (input)
gnd::conf::parameter_array<char, 256> cmap_dir = {
"cmap-dir", // item name
"./", // default value
"cmap directory path" // comment
};
gnd::conf::parameter_array<double, 3> pos_ini = {
"init-pos", // item name
{0, 0, 0}, // default value
"initial position for scan matching (x[m], y[m], theta[deg])" // comment
};
gnd::conf::parameter_array<char, 256> scan_file = {
"scan-data", // item name
"scan.txt", // default value
"scan data file path" // comment
};
{ // ---> debug mode
gnd::opsm::debug_set_log_level(2);
gnd::gridmap::debug_set_log_level(2);
} // <--- debug mode
{ // ---> initialize
int ret;
gnd::conf::file_stream fs;
// set parameter item and default parameter
if( argc < 2 ) {
static const char fname[] = "_scan-matching.conf";
::fprintf(stderr, " error: missing file operand\n");
::fprintf(stderr, " please input configuration file path\n");
::fprintf(stderr, " e.g.: %s scan-matching.conf\n", argv[0]);
::fprintf(stderr, " ... \n");
// write configuration file
gnd::conf::set_parameter(&fs, &cmap_dir);
gnd::conf::set_parameter(&fs, &pos_ini);
gnd::conf::set_parameter(&fs, &scan_file);
fs.write(fname);
::fprintf(stderr, " => generate configuration file template \"%s\"\n", fname);
return -1;
}
// configuration file read
ret = fs.read(argv[1]);
if( ret < 0 ) {
::fprintf(stderr, " error: fail to read configuration file\n");
return -1;
}
// get parameter value
gnd::conf::get_parameter(&fs, &cmap_dir);
gnd::conf::get_parameter(&fs, &pos_ini);
gnd::conf::get_parameter(&fs, &scan_file);
if( !cmap_dir.value[0] ) {
::fprintf(stdout, " error: missing counting map\n");
::fprintf(stdout, " please set counting map directory\n");
return -1;
}
// read counting map
ret = gnd::opsm::read_counting_map( &cmap, cmap_dir.value );
if( ret < 0 ) {
::fprintf(stderr, " error: fail to read counting map form directory \"%s\"\n", cmap_dir.value);
return 0;
}
// build scan matching map
ret = gnd::opsm::build_map( &map, &cmap );
if( !scan_file.value[0] ) {
::fprintf(stdout, " error: missing scan data\n");
::fprintf(stdout, " please set scan data file\n");
return -1;
}
// open scan data file
fp = ::fopen( scan_file.value, "r" );
if( !fp ) {
::fprintf(stdout, " error: fail to open \"%s\"\n", scan_file.value);
return -1;
}
} // <--- initialize
{ // ---> operation
int ret;
gnd::opsm::mcl mcl; // quasi monte calro method class
gnd::opsm::mcl::initial_parameter mcl_ini;
gnd::vector::fixed_column<3> delta;
double likelihood;
int cnt = 0;
// timer init (for linux)
TIMER_INIT();
// set map
mcl.set_map(&map);
// set random seed
gnd::random_set_seed();
// initial position
mcl_ini.pos[0] = pos_ini.value[0];
mcl_ini.pos[1] = pos_ini.value[1];
mcl_ini.pos[2] = pos_ini.value[2] * 3.141592 / 180;
// number of particle
mcl_ini.n = 300;
// initial particles distribution
// co-variance matrix
gnd::matrix::set_zero( &mcl_ini.var_ini );
mcl_ini.var_ini[0][0] = 0.1 * 0.1; // x's standard deviation is 0.1
mcl_ini.var_ini[1][1] = 0.1 * 0.1; // y's standard deviation is 0.1
mcl_ini.var_ini[2][2] = (10 * 3.141592 / 180) * (10 * 3.141592 / 180); // orientation's standard deviation is 5 [deg]
// set initial parameter
mcl.begin( &mcl_ini );
{// ---> set scan data (robot coordinate) and file out scan data on initial position
FILE *fp_ini; // scan data on initial position (file out)
double cosv = cos(mcl_ini.pos[2]);
double sinv = sin(mcl_ini.pos[2]);
// open output file (scan on initial position)
fp_ini = ::fopen("scan-on-init-pos.txt", "w");
if( !fp_ini ) {
::fprintf(stdout, " error: fail to open \"%s\"\n", "scan-on-init-pos.txt");
return -1;
}
while( !::feof(fp) ) {
double x, y;
// read data file
ret = ::fscanf(fp, "%lf %lf\n", &x, &y);
// when read all data, break
if( ret < 0 ) break;
// set scan data
mcl.set_scan_point( x, y );
// file out scan data on initial position
::fprintf(fp_ini, "%lf %lf\n",
x * cosv - y * sinv + mcl_ini.pos[0],
x * sinv + y * cosv + mcl_ini.pos[1]);
}
::fclose(fp_ini);
} // ---> set scan data (robot coordinate) and file out scan data on initial position
// set convergence test parameter
// difference from previous position 0.001m in distance and 0.1deg in orientation
mcl.set_converge_threshold( 0.001, 0.01 * 3.141592 / 180 );
// show initial position
::fprintf(stdout, " init pos = %lf, %lf, %lf\n", mcl_ini.pos[0], mcl_ini.pos[1], mcl_ini.pos[2] * 180 / 3.141592);
::fprintf(stdout, " scan point num = %d\n", mcl.nscan_point());
::fprintf(stdout, "\n");
::fprintf(stdout, " => Scan Matching Begin\n");
// iterative optimization
double t = 0, l = 0;
do {
// optimize
TIMER_BEGIN();
mcl.iterate( &delta, &pos, &likelihood);
TIMER_REC(&t, &l);
cnt++;
// show result
::fprintf(stdout, "-------- optimization loop %d --------\n", cnt);
::fprintf(stdout, " delta = %lf, %lf, %lf\n", delta[0], delta[1], delta[2] * 180 / 3.141592);
::fprintf(stdout, " pos = %lf, %lf, %lf\n", pos[0], pos[1], pos[2] * 180 / 3.141592);
::fprintf(stdout, " likelihood = %lf\n", likelihood);
TIMER_SHOW(&t, &l);
::fprintf(stdout, "\n");
::fprintf(stdout, "\n");
// convergence test
} while( !mcl.converge_test() || cnt < 5);
::fprintf(stdout, " ... Finish\n");
} // <--- operation
{ // ---> finalize
{ // ---> output matched scan data (global coordinate)
int ret;
double cosv = cos(pos[2]);
double sinv = sin(pos[2]);
FILE *fp_m; // scan data on matching position (file out)
::fprintf(stdout, " => file-out matched scan points on global coordinate\n");
// open output file (scan on matching position)
fp_m = ::fopen("scan-on-match-pos.txt", "w");
if( !fp_m ) {
::fprintf(stdout, " error: fail to open \"%s\"\n", "scan-on-match-pos.txt");
return -1;
}
// ---> fileout
::fseek(fp, 0, SEEK_SET);
while( !::feof(fp) ) {
double x, y;
// read data file
ret = ::fscanf(fp, "%lf %lf\n", &x, &y);
// when read all data, break
if( ret < 0 ) break;
::fprintf(fp_m, "%lf %lf\n",
x * cosv - y * sinv + pos[0],
x * sinv + y * cosv + pos[1]
);
} // ---> fileout
::fprintf(stdout, " ... output \"matched-scan-data.txt\"\n");
// file close
::fclose(fp_m);
} // <--- output matched scan data (global coordinate)
// file close
::fclose(fp);
gnd::opsm::destroy_counting_map(&cmap);
gnd::opsm::destroy_map(&map);
} // <--- finalize
return 0;
}
/*
* When a container buffer is full, we push it into container_queue.
*/
static void* filter_thread(void *arg) {
int enable_rewrite = 1;
struct fileRecipeMeta* r = NULL;
while (1) {
struct chunk* c = sync_queue_pop(rewrite_queue);
if (c == NULL)
/* backup job finish */
break;
/* reconstruct a segment */
struct segment* s = new_segment();
/* segment head */
assert(CHECK_CHUNK(c, CHUNK_SEGMENT_START));
free_chunk(c);
c = sync_queue_pop(rewrite_queue);
while (!(CHECK_CHUNK(c, CHUNK_SEGMENT_END))) {
g_sequence_append(s->chunks, c);
if (!CHECK_CHUNK(c, CHUNK_FILE_START)
&& !CHECK_CHUNK(c, CHUNK_FILE_END))
s->chunk_num++;
c = sync_queue_pop(rewrite_queue);
}
free_chunk(c);
/* For self-references in a segment.
* If we find an early copy of the chunk in this segment has been rewritten,
* the rewrite request for it will be denied to avoid repeat rewriting. */
GHashTable *recently_rewritten_chunks = g_hash_table_new_full(g_int64_hash,
g_fingerprint_equal, NULL, free_chunk);
GHashTable *recently_unique_chunks = g_hash_table_new_full(g_int64_hash,
g_fingerprint_equal, NULL, free_chunk);
pthread_mutex_lock(&index_lock.mutex);
TIMER_DECLARE(1);
TIMER_BEGIN(1);
/* This function will check the fragmented chunks
* that would be rewritten later.
* If we find an early copy of the chunk in earlier segments,
* has been rewritten,
* the rewrite request for it will be denied. */
index_check_buffer(s);
GSequenceIter *iter = g_sequence_get_begin_iter(s->chunks);
GSequenceIter *end = g_sequence_get_end_iter(s->chunks);
for (; iter != end; iter = g_sequence_iter_next(iter)) {
c = g_sequence_get(iter);
if (CHECK_CHUNK(c, CHUNK_FILE_START) || CHECK_CHUNK(c, CHUNK_FILE_END))
continue;
VERBOSE("Filter phase: %dth chunk in %s container %lld", chunk_num,
CHECK_CHUNK(c, CHUNK_OUT_OF_ORDER) ? "out-of-order" : "", c->id);
/* Cache-Aware Filter */
if (destor.rewrite_enable_cache_aware && restore_aware_contains(c->id)) {
assert(c->id != TEMPORARY_ID);
VERBOSE("Filter phase: %dth chunk is cached", chunk_num);
SET_CHUNK(c, CHUNK_IN_CACHE);
}
/* A cfl-switch for rewriting out-of-order chunks. */
if (destor.rewrite_enable_cfl_switch) {
double cfl = restore_aware_get_cfl();
if (enable_rewrite && cfl > destor.rewrite_cfl_require) {
VERBOSE("Filter phase: Turn OFF the (out-of-order) rewrite switch of %.3f",
cfl);
enable_rewrite = 0;
} else if (!enable_rewrite && cfl < destor.rewrite_cfl_require) {
VERBOSE("Filter phase: Turn ON the (out-of-order) rewrite switch of %.3f",
cfl);
enable_rewrite = 1;
}
}
if(CHECK_CHUNK(c, CHUNK_DUPLICATE) && c->id == TEMPORARY_ID){
struct chunk* ruc = g_hash_table_lookup(recently_unique_chunks, &c->fp);
assert(ruc);
c->id = ruc->id;
}
struct chunk* rwc = g_hash_table_lookup(recently_rewritten_chunks, &c->fp);
if(rwc){
c->id = rwc->id;
SET_CHUNK(c, CHUNK_REWRITE_DENIED);
}
/* A fragmented chunk will be denied if it has been rewritten recently */
if (!CHECK_CHUNK(c, CHUNK_DUPLICATE) || (!CHECK_CHUNK(c, CHUNK_REWRITE_DENIED)
&& (CHECK_CHUNK(c, CHUNK_SPARSE)
|| (enable_rewrite && CHECK_CHUNK(c, CHUNK_OUT_OF_ORDER)
&& !CHECK_CHUNK(c, CHUNK_IN_CACHE))))) {
/*
* If the chunk is unique, or be fragmented and not denied,
* we write it to a container.
* Fragmented indicates: sparse, or out of order and not in cache,
*/
if (storage_buffer.container_buffer == NULL){
storage_buffer.container_buffer = create_container();
if(destor.index_category[1] == INDEX_CATEGORY_PHYSICAL_LOCALITY)
storage_buffer.chunks = g_sequence_new(free_chunk);
}
if (container_overflow(storage_buffer.container_buffer, c->size)) {
if(destor.index_category[1] == INDEX_CATEGORY_PHYSICAL_LOCALITY){
/*
* TO-DO
* Update_index for physical locality
*/
GHashTable *features = sampling(storage_buffer.chunks,
g_sequence_get_length(storage_buffer.chunks));
index_update(features, get_container_id(storage_buffer.container_buffer));
g_hash_table_destroy(features);
g_sequence_free(storage_buffer.chunks);
storage_buffer.chunks = g_sequence_new(free_chunk);
}
TIMER_END(1, jcr.filter_time);
write_container_async(storage_buffer.container_buffer);
TIMER_BEGIN(1);
storage_buffer.container_buffer = create_container();
}
if(add_chunk_to_container(storage_buffer.container_buffer, c)){
struct chunk* wc = new_chunk(0);
memcpy(&wc->fp, &c->fp, sizeof(fingerprint));
wc->id = c->id;
if (!CHECK_CHUNK(c, CHUNK_DUPLICATE)) {
jcr.unique_chunk_num++;
jcr.unique_data_size += c->size;
g_hash_table_insert(recently_unique_chunks, &wc->fp, wc);
VERBOSE("Filter phase: %dth chunk is recently unique, size %d", chunk_num,
g_hash_table_size(recently_unique_chunks));
} else {
jcr.rewritten_chunk_num++;
jcr.rewritten_chunk_size += c->size;
g_hash_table_insert(recently_rewritten_chunks, &wc->fp, wc);
}
if(destor.index_category[1] == INDEX_CATEGORY_PHYSICAL_LOCALITY){
struct chunk* ck = new_chunk(0);
memcpy(&ck->fp, &c->fp, sizeof(fingerprint));
g_sequence_append(storage_buffer.chunks, ck);
}
VERBOSE("Filter phase: Write %dth chunk to container %lld",
chunk_num, c->id);
}else{
VERBOSE("Filter phase: container %lld already has this chunk", c->id);
assert(destor.index_category[0] != INDEX_CATEGORY_EXACT
|| destor.rewrite_algorithm[0]!=REWRITE_NO);
}
}else{
if(CHECK_CHUNK(c, CHUNK_REWRITE_DENIED)){
VERBOSE("Filter phase: %lldth fragmented chunk is denied", chunk_num);
}else if (CHECK_CHUNK(c, CHUNK_OUT_OF_ORDER)) {
VERBOSE("Filter phase: %lldth chunk in out-of-order container %lld is already cached",
chunk_num, c->id);
}
}
assert(c->id != TEMPORARY_ID);
/* Collect historical information. */
har_monitor_update(c->id, c->size);
/* Restore-aware */
restore_aware_update(c->id, c->size);
chunk_num++;
}
int full = index_update_buffer(s);
/* Write a SEGMENT_BEGIN */
segmentid sid = append_segment_flag(jcr.bv, CHUNK_SEGMENT_START, s->chunk_num);
/* Write recipe */
iter = g_sequence_get_begin_iter(s->chunks);
end = g_sequence_get_end_iter(s->chunks);
for (; iter != end; iter = g_sequence_iter_next(iter)) {
c = g_sequence_get(iter);
if(r == NULL){
assert(CHECK_CHUNK(c,CHUNK_FILE_START));
r = new_file_recipe_meta(c->data);
}else if(!CHECK_CHUNK(c,CHUNK_FILE_END)){
struct chunkPointer cp;
cp.id = c->id;
assert(cp.id>=0);
memcpy(&cp.fp, &c->fp, sizeof(fingerprint));
cp.size = c->size;
append_n_chunk_pointers(jcr.bv, &cp ,1);
r->chunknum++;
r->filesize += c->size;
jcr.chunk_num++;
jcr.data_size += c->size;
}else{
assert(CHECK_CHUNK(c,CHUNK_FILE_END));
append_file_recipe_meta(jcr.bv, r);
free_file_recipe_meta(r);
r = NULL;
jcr.file_num++;
}
}
/* Write a SEGMENT_END */
append_segment_flag(jcr.bv, CHUNK_SEGMENT_END, 0);
if(destor.index_category[1] == INDEX_CATEGORY_LOGICAL_LOCALITY){
/*
* TO-DO
* Update_index for logical locality
*/
s->features = sampling(s->chunks, s->chunk_num);
if(destor.index_category[0] == INDEX_CATEGORY_EXACT){
/*
* For exact deduplication,
* unique fingerprints are inserted.
*/
VERBOSE("Filter phase: add %d unique fingerprints to %d features",
g_hash_table_size(recently_unique_chunks),
g_hash_table_size(s->features));
GHashTableIter iter;
gpointer key, value;
g_hash_table_iter_init(&iter, recently_unique_chunks);
while(g_hash_table_iter_next(&iter, &key, &value)){
struct chunk* uc = value;
fingerprint *ft = malloc(sizeof(fingerprint));
memcpy(ft, &uc->fp, sizeof(fingerprint));
g_hash_table_insert(s->features, ft, NULL);
}
/*
* OPTION:
* It is still an open problem whether we need to update
* rewritten fingerprints.
* It would increase index update overhead, while the benefit
* remains unclear.
* More experiments are required.
*/
VERBOSE("Filter phase: add %d rewritten fingerprints to %d features",
g_hash_table_size(recently_rewritten_chunks),
g_hash_table_size(s->features));
g_hash_table_iter_init(&iter, recently_rewritten_chunks);
while(g_hash_table_iter_next(&iter, &key, &value)){
struct chunk* uc = value;
fingerprint *ft = malloc(sizeof(fingerprint));
memcpy(ft, &uc->fp, sizeof(fingerprint));
g_hash_table_insert(s->features, ft, NULL);
}
}
index_update(s->features, sid);
}
free_segment(s);
if(index_lock.wait_threshold > 0 && full == 0){
pthread_cond_broadcast(&index_lock.cond);
}
TIMER_END(1, jcr.filter_time);
pthread_mutex_unlock(&index_lock.mutex);
g_hash_table_destroy(recently_rewritten_chunks);
g_hash_table_destroy(recently_unique_chunks);
}
if (storage_buffer.container_buffer
&& !container_empty(storage_buffer.container_buffer)){
if(destor.index_category[1] == INDEX_CATEGORY_PHYSICAL_LOCALITY){
/*
* TO-DO
* Update_index for physical locality
*/
GHashTable *features = sampling(storage_buffer.chunks,
g_sequence_get_length(storage_buffer.chunks));
index_update(features, get_container_id(storage_buffer.container_buffer));
g_hash_table_destroy(features);
g_sequence_free(storage_buffer.chunks);
}
write_container_async(storage_buffer.container_buffer);
}
/* All files done */
jcr.status = JCR_STATUS_DONE;
return NULL;
}
