void syssim_report_completion(double t, struct disksim_request *r, void *ctx) { completed = 1; now = t; add_statistics(&st, t - r->start); }
/*@null@*/ static void do_slabs_stats(struct default_engine *engine, ADD_STAT add_stats, const void *cookie) { unsigned int i; unsigned int total = 0; for(i = POWER_SMALLEST; i <= engine->slabs.power_largest; i++) { slabclass_t *p = &engine->slabs.slabclass[i]; if (p->slabs != 0) { uint32_t perslab, slabs; slabs = p->slabs; perslab = p->perslab; add_statistics(cookie, add_stats, NULL, i, "chunk_size", "%u", p->size); add_statistics(cookie, add_stats, NULL, i, "chunks_per_page", "%u", perslab); add_statistics(cookie, add_stats, NULL, i, "total_pages", "%u", slabs); add_statistics(cookie, add_stats, NULL, i, "total_chunks", "%u", slabs * perslab); add_statistics(cookie, add_stats, NULL, i, "used_chunks", "%u", slabs*perslab - p->sl_curr - p->end_page_free); add_statistics(cookie, add_stats, NULL, i, "free_chunks", "%u", p->sl_curr); add_statistics(cookie, add_stats, NULL, i, "free_chunks_end", "%u", p->end_page_free); add_statistics(cookie, add_stats, NULL, i, "mem_requested", "%"PRIu64, (uint64_t)p->requested); total++; } } /* add overall slab stats and append terminator */ add_statistics(cookie, add_stats, NULL, -1, "active_slabs", "%d", total); add_statistics(cookie, add_stats, NULL, -1, "total_malloced", "%"PRIu64, (uint64_t)engine->slabs.mem_malloced); }
static void do_item_stats(struct default_engine *engine, ADD_STAT add_stats, const void *c) { int i; rel_time_t current_time = engine->server.core->get_current_time(); for (i = 0; i < POWER_LARGEST; i++) { if (engine->items.tails[i] != NULL) { int search = search_items; while (search > 0 && engine->items.tails[i] != NULL && ((engine->config.oldest_live != 0 && /* Item flushd */ engine->config.oldest_live <= current_time && engine->items.tails[i]->time <= engine->config.oldest_live) || (engine->items.tails[i]->exptime != 0 && /* and not expired */ engine->items.tails[i]->exptime < current_time))) { --search; if (engine->items.tails[i]->refcount == 0) { do_item_unlink(engine, engine->items.tails[i]); } else { break; } } if (engine->items.tails[i] == NULL) { /* We removed all of the items in this slab class */ continue; } const char *prefix = "items"; add_statistics(c, add_stats, prefix, i, "number", "%u", engine->items.sizes[i]); add_statistics(c, add_stats, prefix, i, "age", "%u", engine->items.tails[i]->time); add_statistics(c, add_stats, prefix, i, "evicted", "%u", engine->items.itemstats[i].evicted); add_statistics(c, add_stats, prefix, i, "evicted_nonzero", "%u", engine->items.itemstats[i].evicted_nonzero); add_statistics(c, add_stats, prefix, i, "evicted_time", "%u", engine->items.itemstats[i].evicted_time); add_statistics(c, add_stats, prefix, i, "outofmemory", "%u", engine->items.itemstats[i].outofmemory); add_statistics(c, add_stats, prefix, i, "tailrepairs", "%u", engine->items.itemstats[i].tailrepairs);; add_statistics(c, add_stats, prefix, i, "reclaimed", "%u", engine->items.itemstats[i].reclaimed);; } } }
/*@null@*/ static void do_slabs_stats(struct default_engine *engine, ADD_STAT add_stats, const void *cookie) { unsigned int i, total; /* Get the per-thread stats which contain some interesting aggregates */ #ifdef FUTURE struct conn *conn = (struct conn*)cookie; struct thread_stats thread_stats; threadlocal_stats_aggregate(c, &thread_stats); #endif total = 0; for(i = POWER_SMALLEST; i <= engine->slabs.power_largest; i++) { slabclass_t *p = &engine->slabs.slabclass[i]; if (p->slabs != 0) { uint32_t perslab, slabs; slabs = p->slabs; perslab = p->perslab; add_statistics(cookie, add_stats, NULL, i, "chunk_size", "%u", p->size); add_statistics(cookie, add_stats, NULL, i, "chunks_per_page", "%u", perslab); add_statistics(cookie, add_stats, NULL, i, "total_pages", "%u", slabs); add_statistics(cookie, add_stats, NULL, i, "total_chunks", "%u", slabs * perslab); add_statistics(cookie, add_stats, NULL, i, "used_chunks", "%u", slabs*perslab - p->sl_curr - p->end_page_free); add_statistics(cookie, add_stats, NULL, i, "free_chunks", "%u", p->sl_curr); add_statistics(cookie, add_stats, NULL, i, "free_chunks_end", "%u", p->end_page_free); add_statistics(cookie, add_stats, NULL, i, "mem_requested", "%"PRIu64, (uint64_t)p->requested); #ifdef FUTURE add_statistics(cookie, add_stats, NULL, i, "get_hits", "%"PRIu64, thread_stats.slab_stats[i].get_hits); add_statistics(cookie, add_stats, NULL, i, "cmd_set", "%"PRIu64, thread_stats.slab_stats[i].set_cmds); add_statistics(cookie, add_stats, NULL, i, "delete_hits", "%"PRIu64, thread_stats.slab_stats[i].delete_hits); add_statistics(cookie, add_stats, NULL, i, "cas_hits", "%"PRIu64, thread_stats.slab_stats[i].cas_hits); add_statistics(cookie, add_stats, NULL, i, "cas_badval", "%"PRIu64, thread_stats.slab_stats[i].cas_badval); #endif total++; } } /* add overall slab stats and append terminator */ add_statistics(cookie, add_stats, NULL, -1, "active_slabs", "%d", total); add_statistics(cookie, add_stats, NULL, -1, "total_malloced", "%"PRIu64, (uint64_t)engine->slabs.mem_malloced); }
/** * Computes training by extracting statistics from a tagged corpus file. */ void do_training(U_FILE* input_text,U_FILE* rforms_file,U_FILE* iforms_file){ /* these two hash tables are respectively for simple and compound entries */ struct string_hash_ptr* rforms_table = NULL, *iforms_table = NULL; if(rforms_file != NULL){ rforms_table = new_string_hash_ptr(200000); } if(iforms_file != NULL){ iforms_table = new_string_hash_ptr(200000); } /* we initialize a contextual matrix */ struct corpus_entry** context = new_context_matrix(); initialize_context_matrix(context); unichar line[MAX_TAGGED_CORPUS_LINE]; /* check the format of the corpus */ long previous_file_position = ftell(input_text); if(u_fgets(line,input_text) == EOF){ fatal_error("File is empty"); } fseek(input_text,previous_file_position,SEEK_SET); int format_corpus = check_corpus_entry(line); if(format_corpus == 0){ // the corpus is in the Tagger format, one word per line where line=word/tag while(u_fgets(line,input_text) !=EOF){ if(u_strlen(line) == 0){ initialize_context_matrix(context); } else{ corpus_entry* entry = new_corpus_entry(line); if(u_strchr(line,'_')!=NULL && line[0]!='_'){ corpus_entry** entries = extract_simple_words(entry); free_corpus_entry(entry); for(int i=0;entries[i]!=NULL;i++){ push_corpus_entry(entries[i],context); add_statistics(context,rforms_table,iforms_table); } free(entries); } else { push_corpus_entry(entry,context); add_statistics(context,rforms_table,iforms_table); } } } } else { // the corpus is in the Unitex tagged format, one sentence per line where token={word,lemma.tag} unichar *tmp,*s = (unichar*)malloc(sizeof(unichar)*(MAX_TAGGED_CORPUS_LINE)); int current_len,len; unsigned int i; while(u_fgets(line,input_text) != EOF){ current_len = 0, len = 0; /* extract each token of the sentence */ for (;;) { len = 1+u_strlen(line+current_len)-u_strlen(u_strchr(line+current_len,'}')); tmp = u_strcpy_sized(s,len-1,line+current_len+1); u_strcat(tmp,"\0"); if(u_strcmp(s,"S") == 0) break; //particular case: '\},\}.PONCT' if(line[current_len+2] == '}'){ int start = current_len+3; do{ tmp = u_strchr(line+start,'}'); start += 1+u_strlen(line+start)-u_strlen(tmp); } while(*(tmp+1) != ' '); tmp = u_strcpy_sized(s,start-current_len-1,line+current_len+1); u_strcat(tmp,"\0"); len += start-current_len-3; } /* format the {XX.YY} into standard tagger format, XX/YY */ unichar* newline = (unichar*)malloc(sizeof(unichar)*(8096)); if(u_strchr(s,',')[1] == ','){ u_strcpy(newline,","); } else u_strcpy_sized(newline,1+u_strlen(s)-u_strlen(u_strchr(s,',')),s); u_sprintf(newline,"%S/%S\0",newline,s+u_strrchr(s,'.')+1); for(i=0;i<u_strlen(newline);i++){ if(newline[i] == ' ') newline[i] = '_'; } //create corpus entry corpus_entry* entry = new_corpus_entry(newline); if(u_strchr(newline,'_') != NULL && newline[0] != '_'){ corpus_entry** entries = extract_simple_words(entry); free_corpus_entry(entry); for(int j=0;entries[j]!=NULL;j++){ push_corpus_entry(entries[j],context); add_statistics(context,rforms_table,iforms_table); } free(entries); } else { push_corpus_entry(entry,context); add_statistics(context,rforms_table,iforms_table); } free(newline); current_len += len+1; } initialize_context_matrix(context); } free(s); } free_context_matrix(context); /* we fill dictionary files with pairs (tuple,value) and then * we add a special line "CODE\tFEATURES,.value" in order to * specify whether the dictionary contains inflected or raw form tuples*/ unichar* str = u_strdup(""); if(rforms_table != NULL){ write_keys_values(rforms_table,rforms_table->hash->root,str,rforms_file); u_fprintf(rforms_file,"%s,.%d\n","CODE\tFEATURES",0); free_string_hash_ptr(rforms_table,NULL); } if(iforms_table != NULL){ write_keys_values(iforms_table,iforms_table->hash->root,str,iforms_file); u_fprintf(iforms_file,"%s,.%d\n","CODE\tFEATURES",1); free_string_hash_ptr(iforms_table,NULL); } free(str); }
/*@null@*/ static void do_slabs_stats(struct default_engine *engine, ADD_STAT add_stats, const void *cookie) { int i, total; /* Get the per-thread stats which contain some interesting aggregates */ #ifdef FUTURE struct conn *conn = (struct conn*)cookie; struct thread_stats thread_stats; threadlocal_stats_aggregate(c, &thread_stats); #endif /* small memory classes */ add_statistics(cookie, add_stats, "SM", -1, "used_num_classes", "%d", sm_anchor.used_num_classes); add_statistics(cookie, add_stats, "SM", -1, "free_num_classes", "%d", sm_anchor.free_num_classes); add_statistics(cookie, add_stats, "SM", -1, "used_min_classid", "%d", sm_anchor.used_minid); add_statistics(cookie, add_stats, "SM", -1, "used_max_classid", "%d", sm_anchor.used_maxid); add_statistics(cookie, add_stats, "SM", -1, "free_min_classid", "%d", sm_anchor.free_minid); add_statistics(cookie, add_stats, "SM", -1, "free_max_classid", "%d", sm_anchor.free_maxid); add_statistics(cookie, add_stats, "SM", -1, "free_big_slot_count", "%"PRIu64, sm_anchor.free_slist[SMMGR_NUM_CLASSES-1].count); add_statistics(cookie, add_stats, "SM", -1, "used_total_space", "%"PRIu64, sm_anchor.used_total_space); add_statistics(cookie, add_stats, "SM", -1, "free_small_space", "%"PRIu64, sm_anchor.free_small_space); add_statistics(cookie, add_stats, "SM", -1, "free_avail_space", "%"PRIu64, sm_anchor.free_avail_space); add_statistics(cookie, add_stats, "SM", -1, "free_chunk_space", "%"PRIu64, sm_anchor.free_chunk_space); add_statistics(cookie, add_stats, "SM", -1, "free_limit_space", "%"PRIu64, sm_anchor.free_limit_space); add_statistics(cookie, add_stats, "SM", -1, "space_shortage_level", "%d", sm_anchor.space_shortage_level); total = 0; int min_slab_id = POWER_SMALLEST; min_slab_id = sm_anchor.blck_clsid; for (i = min_slab_id; i <= engine->slabs.power_largest; i++) { slabclass_t *p = &engine->slabs.slabclass[i]; if (p->slabs != 0) { uint32_t perslab, slabs; slabs = p->slabs; perslab = p->perslab; add_statistics(cookie, add_stats, NULL, i, "chunk_size", "%u", p->size); add_statistics(cookie, add_stats, NULL, i, "chunks_per_page", "%u", perslab); add_statistics(cookie, add_stats, NULL, i, "reserved_pages", "%u", p->rsvd_slabs); add_statistics(cookie, add_stats, NULL, i, "total_pages", "%u", slabs); add_statistics(cookie, add_stats, NULL, i, "total_chunks", "%u", slabs*perslab); add_statistics(cookie, add_stats, NULL, i, "used_chunks", "%u", (slabs*perslab)-p->sl_curr-p->end_page_free); add_statistics(cookie, add_stats, NULL, i, "free_chunks", "%u", p->sl_curr); add_statistics(cookie, add_stats, NULL, i, "free_chunks_end", "%u", p->end_page_free); add_statistics(cookie, add_stats, NULL, i, "mem_requested", "%zu", p->requested); #ifdef FUTURE add_statistics(cookie, add_stats, NULL, i, "get_hits", "%"PRIu64, thread_stats.slab_stats[i].get_hits); add_statistics(cookie, add_stats, NULL, i, "cmd_set", "%"PRIu64, thread_stats.slab_stats[i].set_cmds); add_statistics(cookie, add_stats, NULL, i, "delete_hits", "%"PRIu64, thread_stats.slab_stats[i].delete_hits); add_statistics(cookie, add_stats, NULL, i, "cas_hits", "%"PRIu64, thread_stats.slab_stats[i].cas_hits); add_statistics(cookie, add_stats, NULL, i, "cas_badval", "%"PRIu64, thread_stats.slab_stats[i].cas_badval); #endif total++; } } /* add overall slab stats and append terminator */ add_statistics(cookie, add_stats, NULL, -1, "active_slabs", "%d", total); add_statistics(cookie, add_stats, NULL, -1, "memory_limit", "%zu", engine->slabs.mem_limit); add_statistics(cookie, add_stats, NULL, -1, "total_malloced", "%zu", engine->slabs.mem_malloced); }