TEST_F(Benchmark, ZeroMQ)
{
TIMER_START(1000000);
::zmq::message_t *msg = new ::zmq::message_t();
delete msg;
TIMER_END("zmq.empty_message_allocation");
TIMER_START(1000000);
::zmq::message_t *msg = new ::zmq::message_t(100);
delete msg;
TIMER_END("zmq.100_byte_message_allocation");
TIMER_START(1000000);
::zmq::message_t *msg = new ::zmq::message_t(1000);
delete msg;
TIMER_END("zmq.1000_byte_message_allocation");
TIMER_START(100000);
::zmq::message_t *msg = new ::zmq::message_t(10000);
delete msg;
TIMER_END("zmq.10000_byte_message_allocation");
TIMER_START(10000);
::zmq::message_t *msg = new ::zmq::message_t(100000);
delete msg;
TIMER_END("zmq.100000_byte_message_allocation");
zmq_socket_send_recv("inproc://00", ZMQ_PUSH, ZMQ_PULL, 0, 1000000, "zmq.socket.inproc.pushpull.empty");
zmq_socket_send_recv("inproc://01", ZMQ_PUSH, ZMQ_PULL, 100, 1000000, "zmq.socket.inproc.pushpull.100_byte_message");
zmq_socket_send_recv("inproc://02", ZMQ_PUSH, ZMQ_PULL, 1000, 1000000, "zmq.socket.inproc.pushpull.1000_byte_message");
zmq_socket_send_recv("inproc://03", ZMQ_PUSH, ZMQ_PULL, 10000, 1000000, "zmq.socket.inproc.pushpull.10000_byte_message");
zmq_socket_send_recv("inproc://04", ZMQ_PUSH, ZMQ_PULL, 100000, 100000, "zmq.socket.inproc.pushpull.100000_byte_message");
zmq_socket_send_recv("inproc://05", ZMQ_PUSH, ZMQ_PULL, 1000000, 100000, "zmq.socket.inproc.pushpull.1000000_byte_message");
}
static void* lru_restore_thread(void *arg) {
struct lruCache *cache;
if (destor.simulation_level >= SIMULATION_RESTORE)
cache = new_lru_cache(destor.restore_cache[1], free_container_meta,
lookup_fingerprint_in_container_meta);
else
cache = new_lru_cache(destor.restore_cache[1], free_container,
lookup_fingerprint_in_container);
struct chunk* c;
while ((c = sync_queue_pop(restore_recipe_queue))) {
if (CHECK_CHUNK(c, CHUNK_FILE_START) || CHECK_CHUNK(c, CHUNK_FILE_END)) {
sync_queue_push(restore_chunk_queue, c);
continue;
}
TIMER_DECLARE(1);
TIMER_BEGIN(1);
if (destor.simulation_level >= SIMULATION_RESTORE) {
struct containerMeta *cm = lru_cache_lookup(cache, &c->fp);
if (!cm) {
VERBOSE("Restore cache: container %lld is missed", c->id);
cm = retrieve_container_meta_by_id(c->id);
assert(lookup_fingerprint_in_container_meta(cm, &c->fp));
lru_cache_insert(cache, cm, NULL, NULL);
jcr.read_container_num++;
}
TIMER_END(1, jcr.read_chunk_time);
} else {
struct container *con = lru_cache_lookup(cache, &c->fp);
if (!con) {
VERBOSE("Restore cache: container %lld is missed", c->id);
con = retrieve_container_by_id(c->id);
lru_cache_insert(cache, con, NULL, NULL);
jcr.read_container_num++;
}
struct chunk *rc = get_chunk_in_container(con, &c->fp);
assert(rc);
TIMER_END(1, jcr.read_chunk_time);
sync_queue_push(restore_chunk_queue, rc);
}
free_chunk(c);
}
sync_queue_term(restore_chunk_queue);
free_lru_cache(cache);
return NULL;
}
static void* read_recipe_thread(void *arg) {
int i, j, k;
for (i = 0; i < jcr.bv->number_of_files; i++) {
TIMER_DECLARE(1);
TIMER_BEGIN(1);
struct recipeMeta *r = read_next_recipe_meta(jcr.bv);
struct chunk *c = new_chunk(sdslen(r->filename) + 1);
strcpy(c->data, r->filename);
SET_CHUNK(c, CHUNK_FILE_START);
TIMER_END(1, jcr.read_recipe_time);
sync_queue_push(restore_recipe_queue, c);
jcr.file_num++;
for (j = 0; j < r->chunknum; j++) {
TIMER_DECLARE(1);
TIMER_BEGIN(1);
struct chunkPointer* cp = read_next_n_chunk_pointers(jcr.bv, 1, &k);
struct chunk* c = new_chunk(0);
memcpy(&c->fp, &cp->fp, sizeof(fingerprint));
c->size = cp->size;
c->id = cp->id;
TIMER_END(1, jcr.read_recipe_time);
jcr.data_size += c->size;
jcr.chunk_num++;
sync_queue_push(restore_recipe_queue, c);
free(cp);
}
c = new_chunk(0);
SET_CHUNK(c, CHUNK_FILE_END);
sync_queue_push(restore_recipe_queue, c);
free_recipe_meta(r);
}
sync_queue_term(restore_recipe_queue);
return NULL;
}
TEST_F(Benchmark, EnvelopeEmptyAllocations)
{
TIMER_START(1000000);
Envelope *e = new Envelope();
delete e;
TIMER_END("zippylog.envelope.empty_allocations");
}
TEST_F(Benchmark, EnvelopeEmptyCopyConstructor)
{
Envelope empty;
TIMER_START(1000000);
Envelope e(empty);
TIMER_END("zippylog.envelope.empty_copy_constructor");
}
LocInfo* parse_add_locinfov(Parse *p,char *filename, int lineno, char *line, char *tag, char *referrer, char *context_in,va_list args)
{
char buf[128];
char *ctxt = context_in;
LocInfo *l;
TIMER_START();
if(ctxt)
{
vsnprintf(buf,DIMOF(buf),ctxt,args);
buf[DIMOF(buf)-1] = 0;
ctxt = buf;
}
p->n_locs++;
l = ali_push(&p->locs);
l->tag = parse_find_add_str(p,tag);
l->referrer = parse_find_add_str(p,referrer);
l->context = parse_find_add_str(p,ctxt);
l->fname = parse_find_add_str(p,filename);
l->lineno = lineno;
l->line = parse_find_add_str(p,line);
TIMER_END(locinfo_timer);
return l;
}
static void* sha1_thread(void* arg) {
char code[41];
while (1) {
struct chunk* c = sync_queue_pop(chunk_queue);
if (c == NULL) {
sync_queue_term(hash_queue);
break;
}
if (CHECK_CHUNK(c, CHUNK_FILE_START) || CHECK_CHUNK(c, CHUNK_FILE_END)) {
sync_queue_push(hash_queue, c);
continue;
}
TIMER_DECLARE(1);
TIMER_BEGIN(1);
SHA_CTX ctx;
SHA_Init(&ctx);
SHA_Update(&ctx, c->data, c->size);
SHA_Final(c->fp, &ctx);
TIMER_END(1, jcr.hash_time);
hash2code(c->fp, code);
code[40] = 0;
VERBOSE("Hash phase: %ldth chunk identified by %s", chunk_num++, code);
sync_queue_push(hash_queue, c);
}
return NULL;
}
TEST_F(Benchmark, EnvelopeEmptySerialize)
{
Envelope e;
string s;
TIMER_START(1000000);
e.Serialize(s);
TIMER_END("zippylog.envelope.empty_serialize");
}
TEST_F(Benchmark, EnvelopeEmptyZmqProtocolSerialize)
{
Envelope e;
::zmq::message_t msg;
TIMER_START(1000000);
e.ToProtocolZmqMessage(msg);
TIMER_END("zippylog.envelope.empty_zmq_protocol_serialize");
}
TEST_F(Benchmark, EnvelopeEmptyConstructFromString)
{
Envelope empty;
string serialized;
empty.Serialize(serialized);
TIMER_START(1000000);
Envelope e(serialized);
TIMER_END("zippylog.envelope.empty_construct_from_string");
}
TEST_F(Benchmark, EnvelopeEmptyConstructFromZmq)
{
Envelope empty;
::zmq::message_t msg;
empty.ToZmqMessage(msg);
TIMER_START(1000000);
Envelope e(msg);
TIMER_END("zippylog.envelope.empty.construct_from_zmq");
}
TEST_F(Benchmark, EnvelopeEmptyConstructFromBuffer)
{
Envelope empty;
string serialized;
empty.Serialize(serialized);
void * buffer = (void *)serialized.data();
int length = serialized.length();
TIMER_START(1000000);
Envelope e(buffer, length);
TIMER_END("zippylog.envelope.empty_construct_from_buffer");
}
TEST_F(Benchmark, LuaFunctionCalls)
{
lua_State *L = luaL_newstate();
const char *function = "function test()\n return nil\nend";
assert(!luaL_loadstring(L, function));
TIMER_START(1000000);
lua_getglobal(L, "test");
lua_pcall(L, 0, 1, 0);
lua_pop(L, 1);
TIMER_END("lua.pcall_empty_function");
}
TEST_F(Benchmark, EnvelopeAddMessage)
{
TIMER_START(1000000);
Envelope e;
::zippylog::request_processor::Destroy msg;
msg.add_to_envelope(e);
TIMER_END("zippylog.envelope.add_1_message");
TIMER_START(1000000);
Envelope e;
::zippylog::request_processor::Destroy msg;
msg.add_to_envelope(e);
msg.add_to_envelope(e);
TIMER_END("zippylog.envelope.add_2_messages");
TIMER_START(1000000);
Envelope e;
::zippylog::request_processor::Destroy msg;
msg.add_to_envelope(e);
msg.add_to_envelope(e);
msg.add_to_envelope(e);
TIMER_END("zippylog.envelope.add_3_messages");
TIMER_START(100000);
Envelope e;
::zippylog::request_processor::Destroy msg;
for (int i = 5; i; --i) {
msg.add_to_envelope(e);
}
TIMER_END("zippylog.envelope.add_5_messages");
TIMER_START(100000);
Envelope e;
::zippylog::request_processor::Destroy msg;
for (int i = 10; i; --i) {
msg.add_to_envelope(e);
}
TIMER_END("zippylog.envelope.add_10_messages");
TIMER_START(10000);
Envelope e;
::zippylog::request_processor::Destroy msg;
for (int i = 100; i; --i) {
msg.add_to_envelope(e);
}
TIMER_END("zippylog.envelope.add_100_messages");
TIMER_START(10000);
Envelope e;
::zippylog::request_processor::Destroy msg;
for (int i = 1000; i; --i) {
msg.add_to_envelope(e);
}
TIMER_END("zippylog.envelope.add_1000_messages");
}
void test_data(int socket,char *filename)
{
TIMER_DECLARE(start,end);
TIMER_DECLARE(Rstart,Rend);
TIMER_DECLARE(Sstart,Send);
char buf[SOCKET_BUF_SIZE+1] = {0};
int readlen=0;
double total_time=0;
double send_time=0;
double read_time=0;
double total_len=0;
int fd=-1;
if ((fd=open(filename, O_RDONLY)) < 0) {
printf("%s,%d open file error!\n",__FILE__,__LINE__);
return;
}
TIMER_START(start);
while(1){
TIMER_START(Rstart);
if((readlen=readn(fd, buf, SOCKET_BUF_SIZE))<=0)
break;
TIMER_END(Rend);
TIMER_DIFF(read_time,Rstart,Rend);
TIMER_START(Sstart);
bnet_send(socket,buf,readlen);
TIMER_END(Send);
TIMER_DIFF(send_time,Sstart,Send);
total_len+=readlen;
}
TIMER_END(end);
TIMER_DIFF(total_time,start,end);
close(fd);
close(socket);
printf("read time=%.4f %.4fMB/s\n",read_time,total_len/read_time/1036288);
printf("send time=%.4f %.4fMB/s\n",send_time,total_len/send_time/1036288);
printf("total time=%.4f %.4fMB/s\n", total_time,total_len/total_time/1036288);
}
TEST_F(Benchmark, LuaLoadString)
{
LuaState l;
string error;
l.LoadLuaCode("function zippylog_load_string(s)\n"
" e = zippylog.envelope.new()\n"
" e:set_string_value(s)\n"
" return e\n"
"end", error);
::zippylog::lua::LoadStringResult result;
const string input = "foo bar 2k";
TIMER_START(10000);
l.ExecuteLoadString(input, result);
TIMER_END("zippylog.lua.load_string_return_simple_envelope");
}
void zmq_socket_send_recv(const char *address, int sender_type, int receiver_type, int message_size, int iterations, const char *name)
{
::zmq::context_t ctx(1);
::zmq::socket_t sender(ctx, sender_type);
::zmq::socket_t receiver(ctx, receiver_type);
sender.bind(address);
receiver.connect(address);
TIMER_START(iterations);
::zmq::message_t msg(message_size);
::zmq::message_t received;
sender.send(msg, 0);
receiver.recv(&received, 0);
TIMER_END(name);
}
int parse_print_search(Parse *p,char *tag, LocInfoField flds)
{
char *err_str = 0;
int err_val = 0;
pcre *re;
int res = 0;
int i;
int matches[10];
TIMER_START();
if(!flds)
flds = LocInfoField_tag;
re = pcre_compile(tag, PCRE_CASELESS, &err_str, &err_val, NULL);
if(!re)
{
if(err_str)
pcre_free(err_str);
return -1;
}
for(i = 0; i < p->n_locs; ++i)
{
LocInfo *li = p->locs+i;
// why didn't I integrate the bitfield check into the macro? :P
#define TAG_MATCH(T) (T && pcre_exec(re,NULL,T,strlen(T),0,0,matches,DIMOF(matches))>=0)
if(((flds & LocInfoField_tag) && TAG_MATCH(li->tag))
|| ((flds & LocInfoField_referrer) && TAG_MATCH(li->referrer))
|| ((flds & LocInfoField_context) && TAG_MATCH(li->context))
|| ((flds & LocInfoField_referrer) && TAG_MATCH(li->referrer))
|| ((flds & LocInfoField_file) && TAG_MATCH(li->fname))
|| ((flds & LocInfoField_line) && TAG_MATCH(li->line))
)
{
res++;
locinfo_print(li,NULL);
}
#undef TAG_MATCH
}
TIMER_END(locinfo_timer);
pcre_free(re);
return res;
}
void har_monitor_update(containerid id, int32_t size) {
TIMER_DECLARE(1);
TIMER_BEGIN(1);
struct containerRecord* record = g_hash_table_lookup(
container_utilization_monitor, &id);
if (record) {
record->size += size;
} else {
record = (struct containerRecord*) malloc(
sizeof(struct containerRecord));
record->cid = id;
record->size = 0;
g_hash_table_insert(container_utilization_monitor,
&record->cid, record);
record->size += size;
}
TIMER_END(1, jcr.rewrite_time);
}
/*
* We must ensure a container is either in the buffer or written to disks.
*/
static void* append_thread(void *arg) {
while (1) {
struct container *c = sync_queue_get_top(container_buffer);
if (c == NULL)
break;
TIMER_DECLARE(1);
TIMER_BEGIN(1);
write_container(c);
TIMER_END(1, jcr.write_time);
sync_queue_pop(container_buffer);
free_container(c);
}
return NULL;
}
void Shutdown(int flags)
{
if ((flags & SHUTDOWN_FROM_CONFIG))
goto from_config;
EndGame();
SAFE_DELETE(g_XmppClient);
ShutdownPs();
TIMER_BEGIN(L"shutdown TexMan");
delete &g_TexMan;
TIMER_END(L"shutdown TexMan");
// destroy renderer
TIMER_BEGIN(L"shutdown Renderer");
delete &g_Renderer;
g_VBMan.Shutdown();
TIMER_END(L"shutdown Renderer");
g_Profiler2.ShutdownGPU();
#if OS_WIN
TIMER_BEGIN(L"shutdown wmi");
wmi_Shutdown();
TIMER_END(L"shutdown wmi");
#endif
// Free cursors before shutting down SDL, as they may depend on SDL.
cursor_shutdown();
TIMER_BEGIN(L"shutdown SDL");
ShutdownSDL();
TIMER_END(L"shutdown SDL");
g_VideoMode.Shutdown();
TIMER_BEGIN(L"shutdown UserReporter");
g_UserReporter.Deinitialize();
TIMER_END(L"shutdown UserReporter");
// JS debugger temporarily disabled during the SpiderMonkey upgrade (check trac ticket #2348 for details)
//TIMER_BEGIN(L"shutdown DebuggingServer (if active)");
//delete g_DebuggingServer;
//TIMER_END(L"shutdown DebuggingServer (if active)");
delete &g_L10n;
from_config:
TIMER_BEGIN(L"shutdown ConfigDB");
delete &g_ConfigDB;
TIMER_END(L"shutdown ConfigDB");
SAFE_DELETE(g_Console);
// This is needed to ensure that no callbacks from the JSAPI try to use
// the profiler when it's already destructed
g_ScriptRuntime.reset();
// resource
// first shut down all resource owners, and then the handle manager.
TIMER_BEGIN(L"resource modules");
ISoundManager::SetEnabled(false);
g_VFS.reset();
// this forcibly frees all open handles (thus preventing real leaks),
// and makes further access to h_mgr impossible.
h_mgr_shutdown();
file_stats_dump();
TIMER_END(L"resource modules");
TIMER_BEGIN(L"shutdown misc");
timer_DisplayClientTotals();
CNetHost::Deinitialize();
// should be last, since the above use them
SAFE_DELETE(g_Logger);
delete &g_Profiler;
delete &g_ProfileViewer;
SAFE_DELETE(g_ScriptStatsTable);
TIMER_END(L"shutdown misc");
}
TEST_F(Benchmark, SimpleStoreIO)
{
{
const int32 n = 1000000;
Envelope e;
Store * s = this->GetTemporaryStore();
ASSERT_TRUE(s->CreateBucket("foo"));
ASSERT_TRUE(s->CreateStreamset("foo", "bar"));
TIMER_START(n);
s->WriteEnvelope("foo", "bar", e);
TIMER_END("zippylog.store.write_1M_empty_envelopes");
}
{
const int32 n = 1000000;
Envelope e;
string path = ::zippylog::platform::PathJoin(this->GetTemporaryDirectory(), "s.zippylog");
{
::zippylog::FileOutputStream fos(path);
for (int32 i = 0; i < n; i++) {
fos.WriteEnvelope(e);
}
}
::zippylog::FileInputStream fis(path);
uint32 read;
TIMER_START(n);
fis.ReadEnvelope(e, read);
TIMER_END("zippylog.FileInputStream.read_1M_empty_envelopes");
}
{
const int32 n = 1000000;
Envelope *envelopes = new Envelope[n];
for (int32 i = 0; i < n; i++) {
envelopes[i] = this->GetRandomEnvelope();
}
{
Store * s = this->GetTemporaryStore();
ASSERT_TRUE(s->CreateBucket("foo"));
ASSERT_TRUE(s->CreateStreamset("foo", "bar"));
TIMER_START_SINGLE();
for (int32 i = 0; i < n; i++) {
s->WriteEnvelope("foo", "bar", envelopes[i]);
}
TIMER_END_COUNT("zippylog.store.write_1M_random_envelopes", n);
}
{
string path = ::zippylog::platform::PathJoin(this->GetTemporaryDirectory(), "s.zippylog");
{
::zippylog::FileOutputStream fos(path);
TIMER_START(n);
fos.WriteEnvelope(envelopes[TIMER_LOOP_VALUE - 1]);
TIMER_END("zippylog.FileOutputStream.write_1M_random_envelopes");
}
Envelope e;
::zippylog::FileInputStream fis(path);
uint32 read;
TIMER_START(n);
fis.ReadEnvelope(e, read);
TIMER_END("zippylog.FileInputStream.read_1M_random_envelopes");
}
delete [] envelopes;
}
}
void do_backup(char *path) {
init_recipe_store();
init_container_store();
init_index();
init_backup_jcr(path);
NOTICE("\n\n==== backup begin ====");
TIMER_DECLARE(1);
TIMER_BEGIN(1);
time_t start = time(NULL);
if (destor.simulation_level == SIMULATION_ALL) {
start_read_trace_phase();
} else {
start_read_phase();
start_chunk_phase();
start_hash_phase();
}
start_dedup_phase();
start_rewrite_phase();
start_filter_phase();
do{
sleep(5);
/*time_t now = time(NULL);*/
fprintf(stderr,"job %" PRId32 ", data size %" PRId64 " bytes, %" PRId32 " chunks, %d files processed\r",
jcr.id, jcr.data_size, jcr.chunk_num, jcr.file_num);
}while(jcr.status == JCR_STATUS_RUNNING || jcr.status != JCR_STATUS_DONE);
fprintf(stderr,"job %" PRId32 ", data size %" PRId64 " bytes, %" PRId32 " chunks, %d files processed\n",
jcr.id, jcr.data_size, jcr.chunk_num, jcr.file_num);
if (destor.simulation_level == SIMULATION_ALL) {
stop_read_trace_phase();
} else {
stop_read_phase();
stop_chunk_phase();
stop_hash_phase();
}
stop_dedup_phase();
stop_rewrite_phase();
stop_filter_phase();
TIMER_END(1, jcr.total_time);
close_index();
close_container_store();
close_recipe_store();
update_backup_version(jcr.bv);
free_backup_version(jcr.bv);
printf("\n\njob id: %" PRId32 "\n", jcr.id);
printf("index method: %d.(Remark 0: NO; 1: DDFS; 2: Extreme binning; 3: Silo; 4: Sparse; 5: Sampled; 6: Block; 7: Learn)\n",
destor.index_specific);
printf("sampling method: %d (%d) (Remark 1:Random; 2: Min; 3: Uniform; 4: Optimized_min)\n", destor.index_sampling_method[0], destor.index_sampling_method[1]);
printf("segment method: %d (%d) (Remark 0: Fixed; 1: Content; 2: File)\n", destor.index_segment_algorithm[0], destor.index_segment_algorithm[1]);
printf("prefetch # of segments: %d (Remark 1 for sparse index)\n", destor.index_segment_prefech);
printf("segment selection method: %d (%d)(Remark 0: Base; 1: Top; 2: Mix)\n", destor.index_segment_selection_method[0], destor.index_segment_selection_method[1]);
printf("backup path: %s\n", jcr.path);
printf("number of files: %d\n", jcr.file_num);
printf("number of chunks: %" PRId32 " (%" PRId64 " bytes on average)\n", jcr.chunk_num,
jcr.data_size / jcr.chunk_num);
printf("number of unique chunks: %" PRId32 "\n", jcr.unique_chunk_num);
printf("total size(B): %" PRId64 "\n", jcr.data_size);
printf("stored data size(B): %" PRId64 "\n",
jcr.unique_data_size + jcr.rewritten_chunk_size);
printf("deduplication ratio: %.4f, %.4f\n",
jcr.data_size != 0 ?
(jcr.data_size - jcr.unique_data_size
- jcr.rewritten_chunk_size)
/ (double) (jcr.data_size) :
0,
jcr.data_size
/ (double) (jcr.unique_data_size + jcr.rewritten_chunk_size));
printf("total time(s): %.3f\n", jcr.total_time / 1000000);
printf("the index memory footprint (B): %" PRId32 "\n", destor.index_memory_footprint);
printf("throughput(MB/s): %.2f\n",
(double) jcr.data_size * 1000000 / (1024 * 1024 * jcr.total_time));
printf("number of zero chunks: %" PRId32 "\n", jcr.zero_chunk_num);
printf("size of zero chunks: %" PRId64 "\n", jcr.zero_chunk_size);
printf("number of rewritten chunks: %" PRId32 "\n", jcr.rewritten_chunk_num);
printf("size of rewritten chunks: %" PRId64 "\n", jcr.rewritten_chunk_size);
printf("rewritten rate in size: %.3f\n",
jcr.rewritten_chunk_size / (double) jcr.data_size);
destor.data_size += jcr.data_size;
destor.stored_data_size += jcr.unique_data_size + jcr.rewritten_chunk_size;
destor.chunk_num += jcr.chunk_num;
destor.stored_chunk_num += jcr.unique_chunk_num + jcr.rewritten_chunk_num;
destor.zero_chunk_num += jcr.zero_chunk_num;
destor.zero_chunk_size += jcr.zero_chunk_size;
destor.rewritten_chunk_num += jcr.rewritten_chunk_num;
destor.rewritten_chunk_size += jcr.rewritten_chunk_size;
printf("read_time : %.3fs, %.2fMB/s\n", jcr.read_time / 1000000,
jcr.data_size * 1000000 / jcr.read_time / 1024 / 1024);
printf("chunk_time : %.3fs, %.2fMB/s\n", jcr.chunk_time / 1000000,
jcr.data_size * 1000000 / jcr.chunk_time / 1024 / 1024);
printf("hash_time : %.3fs, %.2fMB/s\n", jcr.hash_time / 1000000,
jcr.data_size * 1000000 / jcr.hash_time / 1024 / 1024);
printf("dedup_time : %.3fs, %.2fMB/s\n",
jcr.dedup_time / 1000000,
jcr.data_size * 1000000 / jcr.dedup_time / 1024 / 1024);
printf("rewrite_time : %.3fs, %.2fMB/s\n", jcr.rewrite_time / 1000000,
jcr.data_size * 1000000 / jcr.rewrite_time / 1024 / 1024);
printf("filter_time : %.3fs, %.2fMB/s\n",
jcr.filter_time / 1000000,
jcr.data_size * 1000000 / jcr.filter_time / 1024 / 1024);
printf("write_time : %.3fs, %.2fMB/s\n", jcr.write_time / 1000000,
jcr.data_size * 1000000 / jcr.write_time / 1024 / 1024);
//double seek_time = 0.005; //5ms
//double bandwidth = 120 * 1024 * 1024; //120MB/s
/* double index_lookup_throughput = jcr.data_size
/ (index_read_times * seek_time
+ index_read_entry_counter * 24 / bandwidth) / 1024 / 1024;
double write_data_throughput = 1.0 * jcr.data_size * bandwidth
/ (jcr->unique_chunk_num) / 1024 / 1024;
double index_read_throughput = 1.0 * jcr.data_size / 1024 / 1024
/ (index_read_times * seek_time
+ index_read_entry_counter * 24 / bandwidth);
double index_write_throughput = 1.0 * jcr.data_size / 1024 / 1024
/ (index_write_times * seek_time
+ index_write_entry_counter * 24 / bandwidth);*/
/* double estimated_throughput = write_data_throughput;
if (estimated_throughput > index_read_throughput)
estimated_throughput = index_read_throughput;*/
/*if (estimated_throughput > index_write_throughput)
estimated_throughput = index_write_throughput;*/
char logfile[] = "backup.log";
FILE *fp = fopen(logfile, "a");
/*
* job id,
* the size of backup
* accumulative consumed capacity,
* deduplication rate,
* rewritten rate,
* total container number,
* sparse container number,
* inherited container number,
* 4 * index overhead (4 * int)
* throughput,
*/
fprintf(fp, "%" PRId32 " %" PRId64 " %" PRId64 " %.4f %.4f %" PRId32 " %" PRId32 " %" PRId32 " %" PRId32" %" PRId32 " %" PRId32" %" PRId32" %.2f\n",
jcr.id,
jcr.data_size,
destor.stored_data_size,
jcr.data_size != 0 ?
(jcr.data_size - jcr.rewritten_chunk_size - jcr.unique_data_size)/(double) (jcr.data_size)
: 0,
jcr.data_size != 0 ? (double) (jcr.rewritten_chunk_size) / (double) (jcr.data_size) : 0,
jcr.total_container_num,
jcr.sparse_container_num,
jcr.inherited_sparse_num,
index_overhead.lookup_requests,
index_overhead.lookup_requests_for_unique,
index_overhead.update_requests,
index_overhead.read_prefetching_units,
(double) jcr.data_size * 1000000 / (1024 * 1024 * jcr.total_time));
fclose(fp);
}
int AnalyseLaterBoard(
int leadHand,
moveType * move,
int hint,
int hintDir,
futureTricks * futp,
int thrId)
{
// Specialized function for PlayAnalyser for cards after the
// opening lead. No further parameter checks! This function
// makes heavy reuse of parameters that are already stored in
// various places. It corresponds to:
// target == -1, solutions == 1, mode == 2.
// The function only needs to return fut.score[0].
localVarType * thrp = &localVar[thrId];
int iniDepth = --thrp->iniDepth;
int cardCount = iniDepth + 4;
int trick = (iniDepth + 3) >> 2;
int handRelFirst = (48 - iniDepth) % 4;
thrp->trickNodes = 0;
thrp->analysisFlag = true;
int handToPlay = handId(leadHand, handRelFirst);
if (handToPlay == 0 || handToPlay == 2)
{
thrp->nodeTypeStore[0] = MAXNODE;
thrp->nodeTypeStore[1] = MINNODE;
thrp->nodeTypeStore[2] = MAXNODE;
thrp->nodeTypeStore[3] = MINNODE;
}
else
{
thrp->nodeTypeStore[0] = MINNODE;
thrp->nodeTypeStore[1] = MAXNODE;
thrp->nodeTypeStore[2] = MINNODE;
thrp->nodeTypeStore[3] = MAXNODE;
}
if (handRelFirst == 0)
{
thrp->moves.MakeSpecific(move, trick + 1, 3);
unsigned short int ourWinRanks[DDS_SUITS]; // Unused here
Make3(&thrp->lookAheadPos, ourWinRanks, iniDepth + 1, move, thrp);
}
else if (handRelFirst == 1)
{
thrp->moves.MakeSpecific(move, trick, 0);
Make0(&thrp->lookAheadPos, iniDepth + 1, move);
}
else if (handRelFirst == 2)
{
thrp->moves.MakeSpecific(move, trick, 1);
Make1(&thrp->lookAheadPos, iniDepth + 1, move);
}
else
{
thrp->moves.MakeSpecific(move, trick, 2);
Make2(&thrp->lookAheadPos, iniDepth + 1, move);
}
if (cardCount <= 4)
{
// Last trick.
evalType eval = Evaluate(&thrp->lookAheadPos, thrp->trump, thrp);
futp->score[0] = eval.tricks;
futp->nodes = 0;
return RETURN_NO_FAULT;
}
#ifdef DDS_AB_STATS
thrp->ABStats.Reset();
thrp->ABStats.ResetCum();
#endif
#ifdef DDS_TOP_LEVEL
thrp->nodes = 0;
#endif
int guess = hint,
lowerbound,
upperbound;
if (hintDir == 0)
{
lowerbound = hint;
upperbound = 13;
}
else
{
lowerbound = 0;
upperbound = hint;
}
do
{
ResetBestMoves(thrp);
TIMER_START(TIMER_AB + iniDepth);
thrp->val = (* AB_ptr_trace_list[handRelFirst])(
&thrp->lookAheadPos,
guess,
iniDepth,
thrp);
TIMER_END(TIMER_AB + iniDepth);
#ifdef DDS_TOP_LEVEL
DumpTopLevel(thrp, guess, lowerbound, upperbound, 1);
#endif
if (thrp->val)
lowerbound = guess++;
else
upperbound = --guess;
}
while (lowerbound < upperbound);
futp->score[0] = lowerbound;
futp->nodes = thrp->trickNodes;
thrp->memUsed = thrp->transTable.MemoryInUse() +
ThreadMemoryUsed();
#ifdef DDS_TIMING
thrp->timer.PrintStats();
#endif
#ifdef DDS_TT_STATS
thrp->transTable.PrintSummarySuitStats();
thrp->transTable.PrintSummaryEntryStats();
#endif
#ifdef DDS_MOVES
thrp->moves.PrintTrickStats();
#ifdef DDS_MOVES_DETAILS
thrp->moves.PrintTrickDetails();
#endif
thrp->moves.PrintFunctionStats();
#endif
#ifdef DDS_MEMORY_LEAKS_WIN32
_CrtDumpMemoryLeaks();
#endif
return RETURN_NO_FAULT;
}
void write_restore_data() {
char *p, *q;
q = jcr.path + 1;/* ignore the first char*/
/*
* recursively make directory
*/
while ((p = strchr(q, '/'))) {
if (*p == *(p - 1)) {
q++;
continue;
}
*p = 0;
if (access(jcr.path, 0) != 0) {
mkdir(jcr.path, S_IRWXU | S_IRWXG | S_IRWXO);
}
*p = '/';
q = p + 1;
}
struct chunk *c = NULL;
FILE *fp = NULL;
while ((c = sync_queue_pop(restore_chunk_queue))) {
TIMER_DECLARE(1);
TIMER_BEGIN(1);
if (CHECK_CHUNK(c, CHUNK_FILE_START)) {
NOTICE("Restoring: %s", c->data);
sds filepath = sdsdup(jcr.path);
filepath = sdscat(filepath, c->data);
int len = sdslen(jcr.path);
char *q = filepath + len;
char *p;
while ((p = strchr(q, '/'))) {
if (*p == *(p - 1)) {
q++;
continue;
}
*p = 0;
if (access(filepath, 0) != 0) {
mkdir(filepath, S_IRWXU | S_IRWXG | S_IRWXO);
}
*p = '/';
q = p + 1;
}
if (destor.simulation_level == SIMULATION_NO) {
assert(fp == NULL);
fp = fopen(filepath, "w");
}
sdsfree(filepath);
} else if (CHECK_CHUNK(c, CHUNK_FILE_END)) {
if (fp)
fclose(fp);
fp = NULL;
} else {
assert(destor.simulation_level == SIMULATION_NO);
VERBOSE("Restoring %d bytes", c->size);
fwrite(c->data, c->size, 1, fp);
}
free_chunk(c);
TIMER_END(1, jcr.write_chunk_time);
}
}
int STDCALL SolveBoard(
deal dl,
int target,
int solutions,
int mode,
futureTricks * futp,
int thrId)
{
localVarType * thrp = &localVar[thrId];
// ----------------------------------------------------------
// Formal parameter checks.
// ----------------------------------------------------------
int ret = BoardRangeChecks(&dl, target, solutions, mode, thrId);
if (ret != RETURN_NO_FAULT)
return ret;
// ----------------------------------------------------------
// Count and classify deal.
// ----------------------------------------------------------
bool newDeal = false;
bool newTrump = false;
unsigned diffDeal = 0;
unsigned aggDeal = 0;
bool similarDeal;
int cardCount = 0;
int ind, forb, noMoves;
for (int h = 0; h < DDS_HANDS; h++)
{
for (int s = 0; s < DDS_SUITS; s++)
{
unsigned int c = dl.remainCards[h][s] >> 2;
cardCount += counttable[c];
diffDeal += (c ^ (thrp->suit[h][s]));
aggDeal += c;
if (thrp->suit[h][s] != c)
{
thrp->suit[h][s] = static_cast<unsigned short>(c);
newDeal = true;
}
}
}
if (newDeal)
{
if (diffDeal == 0)
similarDeal = true;
else if ((aggDeal / diffDeal) > SIMILARDEALLIMIT)
similarDeal = true;
else
similarDeal = false;
}
else
similarDeal = false;
if (dl.trump != thrp->trump)
newTrump = true;
// ----------------------------------------------------------
// Generic initialization.
// ----------------------------------------------------------
thrp->trump = dl.trump;
thrp->iniDepth = cardCount - 4;
int iniDepth = thrp->iniDepth;
int trick = (iniDepth + 3) >> 2;
int handRelFirst = (48 - iniDepth) % 4;
int handToPlay = handId(dl.first, handRelFirst);
thrp->trickNodes = 0;
thrp->lookAheadPos.handRelFirst = handRelFirst;
thrp->lookAheadPos.first[iniDepth] = dl.first;
thrp->lookAheadPos.tricksMAX = 0;
moveType mv = {0, 0, 0, 0};
for (int k = 0; k <= 13; k++)
{
thrp->forbiddenMoves[k].rank = 0;
thrp->forbiddenMoves[k].suit = 0;
}
// ----------------------------------------------------------
// Consistency checks.
// ----------------------------------------------------------
ret = BoardValueChecks(&dl, target, solutions, mode, thrp);
if (ret != RETURN_NO_FAULT)
return ret;
// ----------------------------------------------------------
// Last trick, easy to solve.
// ----------------------------------------------------------
if (cardCount <= 4)
{
int leadRank, leadSuit, leadSideWins;
LastTrickWinner(&dl, thrp, handToPlay, handRelFirst,
&leadRank, &leadSuit, &leadSideWins);
futp->nodes = 0;
futp->cards = 1;
futp->suit[0] = leadSuit;
futp->rank[0] = leadRank;
futp->equals[0] = 0;
futp->score[0] = (target == 0 && solutions < 3 ? 0 : leadSideWins);
goto SOLVER_DONE;
}
// ----------------------------------------------------------
// More detailed initialization.
// ----------------------------------------------------------
if ((mode != 2) &&
(((newDeal) && (! similarDeal)) ||
newTrump ||
(thrp->nodes > SIMILARMAXWINNODES)))
{
thrp->transTable.ResetMemory();
}
if (newDeal)
{
SetDeal(thrp);
SetDealTables(thrp);
}
else if (thrp->analysisFlag)
{
SetDeal(thrp);
}
thrp->analysisFlag = false;
if (handToPlay == 0 || handToPlay == 2)
{
thrp->nodeTypeStore[0] = MAXNODE;
thrp->nodeTypeStore[1] = MINNODE;
thrp->nodeTypeStore[2] = MAXNODE;
thrp->nodeTypeStore[3] = MINNODE;
}
else
{
thrp->nodeTypeStore[0] = MINNODE;
thrp->nodeTypeStore[1] = MAXNODE;
thrp->nodeTypeStore[2] = MINNODE;
thrp->nodeTypeStore[3] = MAXNODE;
}
for (int k = 0; k < handRelFirst; k++)
{
mv.rank = dl.currentTrickRank[k];
mv.suit = dl.currentTrickSuit[k];
mv.sequence = 0;
thrp->moves.Init(
trick,
k,
dl.currentTrickRank,
dl.currentTrickSuit,
thrp->lookAheadPos.rankInSuit,
thrp->trump,
thrp->lookAheadPos.first[iniDepth]);
if (k == 0)
thrp->moves.MoveGen0(
trick,
&thrp->lookAheadPos,
&thrp->bestMove[iniDepth],
&thrp->bestMoveTT[iniDepth],
thrp->rel);
else
thrp->moves.MoveGen123(
trick,
k,
&thrp->lookAheadPos);
thrp->lookAheadPos.move[iniDepth + handRelFirst - k] = mv;
thrp->moves.MakeSpecific(&mv, trick, k);
}
InitWinners(&dl, &thrp->lookAheadPos, thrp);
#ifdef DDS_AB_STATS
thrp->ABStats.Reset();
thrp->ABStats.ResetCum();
#endif
#ifdef DDS_TOP_LEVEL
thrp->nodes = 0;
#endif
thrp->moves.Init(
trick,
handRelFirst,
dl.currentTrickRank,
dl.currentTrickSuit,
thrp->lookAheadPos.rankInSuit,
thrp->trump,
thrp->lookAheadPos.first[iniDepth]);
if (handRelFirst == 0)
thrp->moves.MoveGen0(
trick,
&thrp->lookAheadPos,
&thrp->bestMove[iniDepth],
&thrp->bestMoveTT[iniDepth],
thrp->rel);
else
thrp->moves.MoveGen123(
trick,
handRelFirst,
&thrp->lookAheadPos);
noMoves = thrp->moves.GetLength(trick, handRelFirst);
// ----------------------------------------------------------
// mode == 0: Check whether there is only one possible move
// ----------------------------------------------------------
if (mode == 0 && noMoves == 1)
{
moveType * mp = thrp->moves.MakeNextSimple(trick, handRelFirst);
futp->nodes = 0;
futp->cards = 1;
futp->suit[0] = mp->suit;
futp->rank[0] = mp->rank;
futp->equals[0] = mp->sequence << 2;
futp->score[0] = -2;
goto SOLVER_DONE;
}
// ----------------------------------------------------------
// solutions == 3: Target and mode don't matter; all cards
// ----------------------------------------------------------
if (solutions == 3)
{
// 7 for hand 0 and 2, 6 for hand 1 and 3
int guess = 7 - (handToPlay & 0x1);
int upperbound = 13;
int lowerbound = 0;
futp->cards = noMoves;
for (int mno = 0; mno < noMoves; mno++)
{
do
{
ResetBestMoves(thrp);
TIMER_START(TIMER_AB + iniDepth);
thrp->val = (* AB_ptr_list[handRelFirst])(
&thrp->lookAheadPos,
guess,
iniDepth,
thrp);
TIMER_END(TIMER_AB + iniDepth);
#ifdef DDS_TOP_LEVEL
DumpTopLevel(thrp, guess, lowerbound, upperbound, 1);
#endif
if (thrp->val)
{
mv = thrp->bestMove[iniDepth];
lowerbound = guess++;
}
else
upperbound = --guess;
}
while (lowerbound < upperbound);
if (lowerbound)
{
thrp->bestMove[iniDepth] = mv;
futp->suit[mno] = mv.suit;
futp->rank[mno] = mv.rank;
futp->equals[mno] = mv.sequence << 2;
futp->score[mno] = lowerbound;
thrp->forbiddenMoves[mno + 1].suit = mv.suit;
thrp->forbiddenMoves[mno + 1].rank = mv.rank;
guess = lowerbound;
lowerbound = 0;
}
else
{
int noLeft = thrp->moves.GetLength(trick, handRelFirst);
thrp->moves.Rewind(trick, handRelFirst);
moveType * mp;
for (int j = 0; j < noLeft; j++)
{
mp = thrp->moves.MakeNextSimple(trick, handRelFirst);
futp->suit[mno + j] = mp->suit;
futp->rank[mno + j] = mp->rank;
futp->equals[mno + j] = mp->sequence << 2;
futp->score[mno + j] = 0;
}
break;
}
}
goto SOLVER_STATS;
}
// ----------------------------------------------------------
// target == 0: Only cards required, no scoring
// ----------------------------------------------------------
else if (target == 0)
{
futp->nodes = 0;
futp->cards = (solutions == 1 ? 1 : noMoves);
for (int mno = 0; mno < noMoves; mno++)
{
moveType * mp = thrp->moves.MakeNextSimple(trick, handRelFirst);
futp->suit[mno] = mp->suit;
futp->rank[mno] = mp->rank;
futp->equals[mno] = mp->sequence << 2;
futp->score[mno] = 0;
}
goto SOLVER_DONE;
}
// ----------------------------------------------------------
// target == -1: Find optimum score and 1 or more cards
// ----------------------------------------------------------
else if (target == -1)
{
// 7 for hand 0 and 2, 6 for hand 1 and 3
int guess = 7 - (handToPlay & 0x1);
int upperbound = 13;
int lowerbound = 0;
do
{
ResetBestMoves(thrp);
TIMER_START(TIMER_AB + iniDepth);
thrp->val = (* AB_ptr_list[handRelFirst])(&thrp->lookAheadPos,
guess,
iniDepth,
thrp);
TIMER_END(TIMER_AB + iniDepth);
#ifdef DDS_TOP_LEVEL
DumpTopLevel(thrp, guess, lowerbound, upperbound, 1);
#endif
if (thrp->val)
{
mv = thrp->bestMove[iniDepth];
lowerbound = guess++;
}
else
upperbound = --guess;
}
while (lowerbound < upperbound);
thrp->bestMove[iniDepth] = mv;
if (lowerbound == 0)
{
// ALL the other moves must also have payoff 0.
if (solutions == 1) // We only need one of them
futp->cards = 1;
else // solutions == 2, so return all cards
futp->cards = noMoves;
moveType * mp;
thrp->moves.Rewind(trick, handRelFirst);
for (int i = 0; i < noMoves; i++)
{
mp = thrp->moves.MakeNextSimple(trick, handRelFirst);
futp->score[i] = 0;
futp->suit[i] = mp->suit;
futp->rank[i] = mp->rank;
futp->equals[i] = mp->sequence << 2;
}
goto SOLVER_STATS;
}
else // payoff > 0
{
futp->cards = 1;
futp->score[0] = lowerbound;
futp->suit[0] = mv.suit;
futp->rank[0] = mv.rank;
futp->equals[0] = mv.sequence << 2;
if (solutions != 2)
goto SOLVER_STATS;
}
}
// ----------------------------------------------------------
// target >= 1: Find optimum card(s) achieving user's target
// ----------------------------------------------------------
else
{
TIMER_START(TIMER_AB + iniDepth);
thrp->val = (* AB_ptr_list[handRelFirst])(
&thrp->lookAheadPos,
target,
iniDepth,
thrp);
TIMER_END(TIMER_AB + iniDepth);
#ifdef DDS_TOP_LEVEL
DumpTopLevel(thrp, target, -1, -1, 0);
#endif
if (! thrp->val)
{
// No move. If target was 1, then we are sure that in
// fact no tricks can be won. If target was > 1, then
// it is still possible that no tricks can't be won.
// We don't know. So that's arguably a small bug.
futp->cards = 0;
futp->score[0] = (target > 1 ? -1 : 0);
goto SOLVER_STATS;
}
else
{
futp->cards = 1;
futp->suit[0] = thrp->bestMove[iniDepth].suit;
futp->rank[0] = thrp->bestMove[iniDepth].rank;
futp->equals[0] = thrp->bestMove[iniDepth].sequence << 2;
futp->score[0] = target;
if (solutions != 2)
goto SOLVER_STATS;
}
}
// ----------------------------------------------------------
// solution == 2 && payoff > 0: Find other cards with score.
// This applies both to target == -1 and target >= 1.
// ----------------------------------------------------------
moveType * mp;
forb = 1;
ind = 1;
while (ind < noMoves)
{
// Moves up to and including bestMove are now forbidden.
thrp->moves.Rewind(trick, handRelFirst);
int num = thrp->moves.GetLength(trick, handRelFirst);
for (int k = 0; k < num; k++)
{
mp = thrp->moves.MakeNextSimple(trick, handRelFirst);
thrp->forbiddenMoves[forb] = * mp;
forb++;
if ((thrp->bestMove[iniDepth].suit == mp->suit) &&
(thrp->bestMove[iniDepth].rank == mp->rank))
break;
}
ResetBestMoves(thrp);
TIMER_START(TIMER_AB + iniDepth);
thrp->val = (* AB_ptr_list[handRelFirst])(
&thrp->lookAheadPos,
futp->score[0],
iniDepth,
thrp);
TIMER_END(TIMER_AB + iniDepth);
#ifdef DDS_TOP_LEVEL
DumpTopLevel(thrp, target, -1, -1, 2);
#endif
if (! thrp->val)
break;
futp->cards = ind + 1;
futp->suit[ind] = thrp->bestMove[iniDepth].suit;
futp->rank[ind] = thrp->bestMove[iniDepth].rank;
futp->equals[ind] = thrp->bestMove[iniDepth].sequence << 2;
futp->score[ind] = futp->score[0];
ind++;
}
SOLVER_STATS:
for (int k = 0; k <= 13; k++)
{
thrp->forbiddenMoves[k].rank = 0;
thrp->forbiddenMoves[k].suit = 0;
}
#ifdef DDS_TIMING
thrp->timer.PrintStats();
#endif
#ifdef DDS_TT_STATS
// thrp->transTable.PrintAllSuits();
// thrp->transTable.PrintEntries(10, 0);
thrp->transTable.PrintSummarySuitStats();
thrp->transTable.PrintSummaryEntryStats();
// thrp->transTable.PrintPageSummary();
#endif
#ifdef DDS_MOVES
thrp->moves.PrintTrickStats();
#ifdef DDS_MOVES_DETAILS
thrp->moves.PrintTrickDetails();
#endif
thrp->moves.PrintFunctionStats();
#endif
SOLVER_DONE:
thrp->memUsed = thrp->transTable.MemoryInUse() +
ThreadMemoryUsed();
futp->nodes = thrp->trickNodes;
#ifdef DDS_MEMORY_LEAKS_WIN32
_CrtDumpMemoryLeaks();
#endif
return RETURN_NO_FAULT;
}
Container *container_cache_insert_container(ContainerCache *cc, ContainerId cid) {
/* read container */
Container *container = 0;
TIMER_DECLARE(b, e);
TIMER_BEGIN(b);
if (cc->enable_data) {
if (simulation_level >= SIMULATION_RECOVERY) {
container = read_container_meta_only(cid);
} else {
container = read_container(cid);
}
} else {
container = read_container_meta_only(cid);
}
TIMER_END(read_container_time, b, e);
/* If this container is newly appended,
* maybe we can read nothing. */
if (container == NULL) {
return NULL;
}
/* insert */
Container *evictor = lru_cache_insert(cc->lru_cache, container);
/* evict */
if (evictor) {
int32_t chunknum = container_get_chunk_num(evictor);
Fingerprint *fingers = container_get_all_fingers(evictor);
int i = 0;
/* evict all fingers of evictor from map */
for (; i < chunknum; ++i) {
GSequence* container_list = g_hash_table_lookup(cc->map,
&fingers[i]);
/* remove the specified container from list */
GSequenceIter *iter = g_sequence_lookup(container_list, evictor,
container_cmp_des, NULL);
if (iter)
g_sequence_remove(iter);
else
dprint("Error! The sequence does not contain the container.");
if (g_sequence_get_length(container_list) == 0) {
g_hash_table_remove(cc->map, &fingers[i]);
}
}
free(fingers);
if (fragment_stream)
fprintf(fragment_stream, "%.4f\n",
1.0 * evictor->used_size / CONTAINER_SIZE);
container_free_full(evictor);
}
/* insert */
int32_t num = container_get_chunk_num(container);
Fingerprint *nfingers = container_get_all_fingers(container);
int i = 0;
for (; i < num; ++i) {
GSequence* container_list = g_hash_table_lookup(cc->map, &nfingers[i]);
if (container_list == 0) {
container_list = g_sequence_new(NULL);
Fingerprint *finger = (Fingerprint *) malloc(sizeof(Fingerprint));
memcpy(finger, &nfingers[i], sizeof(Fingerprint));
g_hash_table_insert(cc->map, finger, container_list);
}
g_sequence_insert_sorted(container_list, container, container_cmp_des,
NULL);
}
free(nfingers);
return container;
}
void Shutdown(int UNUSED(flags))
{
EndGame();
ShutdownPs(); // Must delete g_GUI before g_ScriptingHost
in_reset_handlers();
TIMER_BEGIN(L"shutdown TexMan");
delete &g_TexMan;
TIMER_END(L"shutdown TexMan");
// destroy renderer
TIMER_BEGIN(L"shutdown Renderer");
delete &g_Renderer;
g_VBMan.Shutdown();
TIMER_END(L"shutdown Renderer");
tex_codec_unregister_all();
g_Profiler2.ShutdownGPU();
// Free cursors before shutting down SDL, as they may depend on SDL.
cursor_shutdown();
TIMER_BEGIN(L"shutdown SDL");
ShutdownSDL();
TIMER_END(L"shutdown SDL");
g_VideoMode.Shutdown();
TIMER_BEGIN(L"shutdown UserReporter");
g_UserReporter.Deinitialize();
TIMER_END(L"shutdown UserReporter");
TIMER_BEGIN(L"shutdown ScriptingHost");
delete &g_ScriptingHost;
TIMER_END(L"shutdown ScriptingHost");
TIMER_BEGIN(L"shutdown ConfigDB");
delete &g_ConfigDB;
TIMER_END(L"shutdown ConfigDB");
// resource
// first shut down all resource owners, and then the handle manager.
TIMER_BEGIN(L"resource modules");
#if CONFIG2_AUDIO
if (g_SoundManager)
delete g_SoundManager;
#endif
g_VFS.reset();
// this forcibly frees all open handles (thus preventing real leaks),
// and makes further access to h_mgr impossible.
h_mgr_shutdown();
file_stats_dump();
TIMER_END(L"resource modules");
TIMER_BEGIN(L"shutdown misc");
timer_DisplayClientTotals();
CNetHost::Deinitialize();
SAFE_DELETE(g_ScriptStatsTable);
// should be last, since the above use them
SAFE_DELETE(g_Logger);
delete &g_Profiler;
delete &g_ProfileViewer;
TIMER_END(L"shutdown misc");
#if OS_WIN
TIMER_BEGIN(L"shutdown wmi");
wmi_Shutdown();
TIMER_END(L"shutdown wmi");
#endif
}
static void* read_trace_thread(void *argv) {
FILE *trace_file = fopen(jcr.path, "r");
char line[128];
while (1) {
TIMER_DECLARE(1);
TIMER_BEGIN(1);
fgets(line, 128, trace_file);
TIMER_END(1, jcr.read_time);
if (strcmp(line, "stream end") == 0) {
sync_queue_term(trace_queue);
break;
}
struct chunk* c;
TIMER_BEGIN(1),
assert(strncmp(line, "file start ", 11) == 0);
int filenamelen;
sscanf(line, "file start %d", &filenamelen);
/* An additional '\n' is read */
c = new_chunk(filenamelen + 2);
fgets(c->data, filenamelen + 2, trace_file);
c->data[filenamelen] = 0;
VERBOSE("Reading: %s", c->data);
SET_CHUNK(c, CHUNK_FILE_START);
TIMER_END(1, jcr.read_time);
sync_queue_push(trace_queue, c);
TIMER_BEGIN(1);
fgets(line, 128, trace_file);
while (strncmp(line, "file end", 8) != 0) {
c = new_chunk(0);
char code[41];
strncpy(code, line, 40);
code2hash(code, c->fp);
c->size = atoi(line + 41);
jcr.chunk_num++;
jcr.data_size += c->size;
TIMER_END(1, jcr.read_time);
sync_queue_push(trace_queue, c);
TIMER_BEGIN(1),
fgets(line, 128, trace_file);
}
c = new_chunk(0);
SET_CHUNK(c, CHUNK_FILE_END);
sync_queue_push(trace_queue, c);
jcr.file_num++;
}
fclose(trace_file);
return NULL;
}
void do_restore(int revision, char *path) {
init_recipe_store();
init_container_store();
init_restore_jcr(revision, path);
destor_log(DESTOR_NOTICE, "job id: %d", jcr.id);
destor_log(DESTOR_NOTICE, "backup path: %s", jcr.bv->path);
destor_log(DESTOR_NOTICE, "restore to: %s", jcr.path);
restore_chunk_queue = sync_queue_new(100);
restore_recipe_queue = sync_queue_new(100);
TIMER_DECLARE(1);
TIMER_BEGIN(1);
puts("==== restore begin ====");
pthread_t recipe_t, read_t;
pthread_create(&recipe_t, NULL, read_recipe_thread, NULL);
if (destor.restore_cache[0] == RESTORE_CACHE_LRU) {
destor_log(DESTOR_NOTICE, "restore cache is LRU");
pthread_create(&read_t, NULL, lru_restore_thread, NULL);
} else if (destor.restore_cache[0] == RESTORE_CACHE_OPT) {
destor_log(DESTOR_NOTICE, "restore cache is OPT");
pthread_create(&read_t, NULL, optimal_restore_thread, NULL);
} else if (destor.restore_cache[0] == RESTORE_CACHE_ASM) {
destor_log(DESTOR_NOTICE, "restore cache is ASM");
pthread_create(&read_t, NULL, assembly_restore_thread, NULL);
} else {
fprintf(stderr, "Invalid restore cache.\n");
exit(1);
}
write_restore_data();
assert(sync_queue_size(restore_chunk_queue) == 0);
assert(sync_queue_size(restore_recipe_queue) == 0);
free_backup_version(jcr.bv);
TIMER_END(1, jcr.total_time);
puts("==== restore end ====");
printf("job id: %d\n", jcr.id);
printf("restore path: %s\n", jcr.path);
printf("number of files: %d\n", jcr.file_num);
printf("number of chunks: %d\n", jcr.chunk_num);
printf("total size(B): %ld\n", jcr.data_size);
printf("total time(s): %.3f\n", jcr.total_time / 1000000);
printf("throughput(MB/s): %.2f\n",
jcr.data_size * 1000000 / (1024.0 * 1024 * jcr.total_time));
printf("speed factor: %.2f\n",
jcr.data_size / (1024.0 * 1024 * jcr.read_container_num));
printf("read_recipe_time : %.3fs, %.2fMB/s\n",
jcr.read_recipe_time / 1000000,
jcr.data_size * 1000000 / jcr.read_recipe_time / 1024 / 1024);
printf("read_chunk_time : %.3fs, %.2fMB/s\n", jcr.read_chunk_time / 1000000,
jcr.data_size * 1000000 / jcr.read_chunk_time / 1024 / 1024);
printf("write_chunk_time : %.3fs, %.2fMB/s\n",
jcr.write_chunk_time / 1000000,
jcr.data_size * 1000000 / jcr.write_chunk_time / 1024 / 1024);
char logfile[] = "restore.log";
FILE *fp = fopen(logfile, "a");
/*
* job id,
* chunk num,
* data size,
* actually read container number,
* speed factor,
* throughput
*/
fprintf(fp, "%d %lld %d %.4f %.4f\n", jcr.id, jcr.data_size,
jcr.read_container_num,
jcr.data_size / (1024.0 * 1024 * jcr.read_container_num),
jcr.data_size * 1000000 / (1024 * 1024 * jcr.total_time));
fclose(fp);
close_container_store();
close_recipe_store();
}
