void
syssim_report_completion(double t, struct disksim_request *r, void *ctx)
{
completed = 1;
now = t;
add_statistics(&st, t - r->start);
}
/*@null@*/
static void do_slabs_stats(struct default_engine *engine, ADD_STAT add_stats, const void *cookie) {
unsigned int i;
unsigned int total = 0;
for(i = POWER_SMALLEST; i <= engine->slabs.power_largest; i++) {
slabclass_t *p = &engine->slabs.slabclass[i];
if (p->slabs != 0) {
uint32_t perslab, slabs;
slabs = p->slabs;
perslab = p->perslab;
add_statistics(cookie, add_stats, NULL, i, "chunk_size", "%u",
p->size);
add_statistics(cookie, add_stats, NULL, i, "chunks_per_page", "%u",
perslab);
add_statistics(cookie, add_stats, NULL, i, "total_pages", "%u",
slabs);
add_statistics(cookie, add_stats, NULL, i, "total_chunks", "%u",
slabs * perslab);
add_statistics(cookie, add_stats, NULL, i, "used_chunks", "%u",
slabs*perslab - p->sl_curr - p->end_page_free);
add_statistics(cookie, add_stats, NULL, i, "free_chunks", "%u",
p->sl_curr);
add_statistics(cookie, add_stats, NULL, i, "free_chunks_end", "%u",
p->end_page_free);
add_statistics(cookie, add_stats, NULL, i, "mem_requested", "%"PRIu64,
(uint64_t)p->requested);
total++;
}
}
/* add overall slab stats and append terminator */
add_statistics(cookie, add_stats, NULL, -1, "active_slabs", "%d", total);
add_statistics(cookie, add_stats, NULL, -1, "total_malloced", "%"PRIu64,
(uint64_t)engine->slabs.mem_malloced);
}
static void do_item_stats(struct default_engine *engine,
ADD_STAT add_stats, const void *c) {
int i;
rel_time_t current_time = engine->server.core->get_current_time();
for (i = 0; i < POWER_LARGEST; i++) {
if (engine->items.tails[i] != NULL) {
int search = search_items;
while (search > 0 &&
engine->items.tails[i] != NULL &&
((engine->config.oldest_live != 0 && /* Item flushd */
engine->config.oldest_live <= current_time &&
engine->items.tails[i]->time <= engine->config.oldest_live) ||
(engine->items.tails[i]->exptime != 0 && /* and not expired */
engine->items.tails[i]->exptime < current_time))) {
--search;
if (engine->items.tails[i]->refcount == 0) {
do_item_unlink(engine, engine->items.tails[i]);
} else {
break;
}
}
if (engine->items.tails[i] == NULL) {
/* We removed all of the items in this slab class */
continue;
}
const char *prefix = "items";
add_statistics(c, add_stats, prefix, i, "number", "%u",
engine->items.sizes[i]);
add_statistics(c, add_stats, prefix, i, "age", "%u",
engine->items.tails[i]->time);
add_statistics(c, add_stats, prefix, i, "evicted",
"%u", engine->items.itemstats[i].evicted);
add_statistics(c, add_stats, prefix, i, "evicted_nonzero",
"%u", engine->items.itemstats[i].evicted_nonzero);
add_statistics(c, add_stats, prefix, i, "evicted_time",
"%u", engine->items.itemstats[i].evicted_time);
add_statistics(c, add_stats, prefix, i, "outofmemory",
"%u", engine->items.itemstats[i].outofmemory);
add_statistics(c, add_stats, prefix, i, "tailrepairs",
"%u", engine->items.itemstats[i].tailrepairs);;
add_statistics(c, add_stats, prefix, i, "reclaimed",
"%u", engine->items.itemstats[i].reclaimed);;
}
}
}
/*@null@*/
static void do_slabs_stats(struct default_engine *engine, ADD_STAT add_stats, const void *cookie) {
unsigned int i, total;
/* Get the per-thread stats which contain some interesting aggregates */
#ifdef FUTURE
struct conn *conn = (struct conn*)cookie;
struct thread_stats thread_stats;
threadlocal_stats_aggregate(c, &thread_stats);
#endif
total = 0;
for(i = POWER_SMALLEST; i <= engine->slabs.power_largest; i++) {
slabclass_t *p = &engine->slabs.slabclass[i];
if (p->slabs != 0) {
uint32_t perslab, slabs;
slabs = p->slabs;
perslab = p->perslab;
add_statistics(cookie, add_stats, NULL, i, "chunk_size", "%u",
p->size);
add_statistics(cookie, add_stats, NULL, i, "chunks_per_page", "%u",
perslab);
add_statistics(cookie, add_stats, NULL, i, "total_pages", "%u",
slabs);
add_statistics(cookie, add_stats, NULL, i, "total_chunks", "%u",
slabs * perslab);
add_statistics(cookie, add_stats, NULL, i, "used_chunks", "%u",
slabs*perslab - p->sl_curr - p->end_page_free);
add_statistics(cookie, add_stats, NULL, i, "free_chunks", "%u",
p->sl_curr);
add_statistics(cookie, add_stats, NULL, i, "free_chunks_end", "%u",
p->end_page_free);
add_statistics(cookie, add_stats, NULL, i, "mem_requested", "%"PRIu64,
(uint64_t)p->requested);
#ifdef FUTURE
add_statistics(cookie, add_stats, NULL, i, "get_hits", "%"PRIu64,
thread_stats.slab_stats[i].get_hits);
add_statistics(cookie, add_stats, NULL, i, "cmd_set", "%"PRIu64,
thread_stats.slab_stats[i].set_cmds);
add_statistics(cookie, add_stats, NULL, i, "delete_hits", "%"PRIu64,
thread_stats.slab_stats[i].delete_hits);
add_statistics(cookie, add_stats, NULL, i, "cas_hits", "%"PRIu64,
thread_stats.slab_stats[i].cas_hits);
add_statistics(cookie, add_stats, NULL, i, "cas_badval", "%"PRIu64,
thread_stats.slab_stats[i].cas_badval);
#endif
total++;
}
}
/* add overall slab stats and append terminator */
add_statistics(cookie, add_stats, NULL, -1, "active_slabs", "%d", total);
add_statistics(cookie, add_stats, NULL, -1, "total_malloced", "%"PRIu64,
(uint64_t)engine->slabs.mem_malloced);
}
/**
* Computes training by extracting statistics from a tagged corpus file.
*/
void do_training(U_FILE* input_text,U_FILE* rforms_file,U_FILE* iforms_file){
/* these two hash tables are respectively for simple and compound entries */
struct string_hash_ptr* rforms_table = NULL, *iforms_table = NULL;
if(rforms_file != NULL){
rforms_table = new_string_hash_ptr(200000);
}
if(iforms_file != NULL){
iforms_table = new_string_hash_ptr(200000);
}
/* we initialize a contextual matrix */
struct corpus_entry** context = new_context_matrix();
initialize_context_matrix(context);
unichar line[MAX_TAGGED_CORPUS_LINE];
/* check the format of the corpus */
long previous_file_position = ftell(input_text);
if(u_fgets(line,input_text) == EOF){
fatal_error("File is empty");
}
fseek(input_text,previous_file_position,SEEK_SET);
int format_corpus = check_corpus_entry(line);
if(format_corpus == 0){
// the corpus is in the Tagger format, one word per line where line=word/tag
while(u_fgets(line,input_text) !=EOF){
if(u_strlen(line) == 0){
initialize_context_matrix(context);
}
else{
corpus_entry* entry = new_corpus_entry(line);
if(u_strchr(line,'_')!=NULL && line[0]!='_'){
corpus_entry** entries = extract_simple_words(entry);
free_corpus_entry(entry);
for(int i=0;entries[i]!=NULL;i++){
push_corpus_entry(entries[i],context);
add_statistics(context,rforms_table,iforms_table);
}
free(entries);
}
else {
push_corpus_entry(entry,context);
add_statistics(context,rforms_table,iforms_table);
}
}
}
}
else {
// the corpus is in the Unitex tagged format, one sentence per line where token={word,lemma.tag}
unichar *tmp,*s = (unichar*)malloc(sizeof(unichar)*(MAX_TAGGED_CORPUS_LINE));
int current_len,len;
unsigned int i;
while(u_fgets(line,input_text) != EOF){
current_len = 0, len = 0;
/* extract each token of the sentence */
for (;;) {
len = 1+u_strlen(line+current_len)-u_strlen(u_strchr(line+current_len,'}'));
tmp = u_strcpy_sized(s,len-1,line+current_len+1);
u_strcat(tmp,"\0");
if(u_strcmp(s,"S") == 0)
break;
//particular case: '\},\}.PONCT'
if(line[current_len+2] == '}'){
int start = current_len+3;
do{
tmp = u_strchr(line+start,'}');
start += 1+u_strlen(line+start)-u_strlen(tmp);
}
while(*(tmp+1) != ' ');
tmp = u_strcpy_sized(s,start-current_len-1,line+current_len+1);
u_strcat(tmp,"\0");
len += start-current_len-3;
}
/* format the {XX.YY} into standard tagger format, XX/YY */
unichar* newline = (unichar*)malloc(sizeof(unichar)*(8096));
if(u_strchr(s,',')[1] == ','){
u_strcpy(newline,",");
}
else
u_strcpy_sized(newline,1+u_strlen(s)-u_strlen(u_strchr(s,',')),s);
u_sprintf(newline,"%S/%S\0",newline,s+u_strrchr(s,'.')+1);
for(i=0;i<u_strlen(newline);i++){
if(newline[i] == ' ')
newline[i] = '_';
}
//create corpus entry
corpus_entry* entry = new_corpus_entry(newline);
if(u_strchr(newline,'_') != NULL && newline[0] != '_'){
corpus_entry** entries = extract_simple_words(entry);
free_corpus_entry(entry);
for(int j=0;entries[j]!=NULL;j++){
push_corpus_entry(entries[j],context);
add_statistics(context,rforms_table,iforms_table);
}
free(entries);
}
else {
push_corpus_entry(entry,context);
add_statistics(context,rforms_table,iforms_table);
}
free(newline);
current_len += len+1;
}
initialize_context_matrix(context);
}
free(s);
}
free_context_matrix(context);
/* we fill dictionary files with pairs (tuple,value) and then
* we add a special line "CODE\tFEATURES,.value" in order to
* specify whether the dictionary contains inflected or raw form tuples*/
unichar* str = u_strdup("");
if(rforms_table != NULL){
write_keys_values(rforms_table,rforms_table->hash->root,str,rforms_file);
u_fprintf(rforms_file,"%s,.%d\n","CODE\tFEATURES",0);
free_string_hash_ptr(rforms_table,NULL);
}
if(iforms_table != NULL){
write_keys_values(iforms_table,iforms_table->hash->root,str,iforms_file);
u_fprintf(iforms_file,"%s,.%d\n","CODE\tFEATURES",1);
free_string_hash_ptr(iforms_table,NULL);
}
free(str);
}
/*@null@*/
static void do_slabs_stats(struct default_engine *engine, ADD_STAT add_stats, const void *cookie)
{
int i, total;
/* Get the per-thread stats which contain some interesting aggregates */
#ifdef FUTURE
struct conn *conn = (struct conn*)cookie;
struct thread_stats thread_stats;
threadlocal_stats_aggregate(c, &thread_stats);
#endif
/* small memory classes */
add_statistics(cookie, add_stats, "SM", -1, "used_num_classes", "%d", sm_anchor.used_num_classes);
add_statistics(cookie, add_stats, "SM", -1, "free_num_classes", "%d", sm_anchor.free_num_classes);
add_statistics(cookie, add_stats, "SM", -1, "used_min_classid", "%d", sm_anchor.used_minid);
add_statistics(cookie, add_stats, "SM", -1, "used_max_classid", "%d", sm_anchor.used_maxid);
add_statistics(cookie, add_stats, "SM", -1, "free_min_classid", "%d", sm_anchor.free_minid);
add_statistics(cookie, add_stats, "SM", -1, "free_max_classid", "%d", sm_anchor.free_maxid);
add_statistics(cookie, add_stats, "SM", -1, "free_big_slot_count", "%"PRIu64, sm_anchor.free_slist[SMMGR_NUM_CLASSES-1].count);
add_statistics(cookie, add_stats, "SM", -1, "used_total_space", "%"PRIu64, sm_anchor.used_total_space);
add_statistics(cookie, add_stats, "SM", -1, "free_small_space", "%"PRIu64, sm_anchor.free_small_space);
add_statistics(cookie, add_stats, "SM", -1, "free_avail_space", "%"PRIu64, sm_anchor.free_avail_space);
add_statistics(cookie, add_stats, "SM", -1, "free_chunk_space", "%"PRIu64, sm_anchor.free_chunk_space);
add_statistics(cookie, add_stats, "SM", -1, "free_limit_space", "%"PRIu64, sm_anchor.free_limit_space);
add_statistics(cookie, add_stats, "SM", -1, "space_shortage_level", "%d", sm_anchor.space_shortage_level);
total = 0;
int min_slab_id = POWER_SMALLEST;
min_slab_id = sm_anchor.blck_clsid;
for (i = min_slab_id; i <= engine->slabs.power_largest; i++) {
slabclass_t *p = &engine->slabs.slabclass[i];
if (p->slabs != 0) {
uint32_t perslab, slabs;
slabs = p->slabs;
perslab = p->perslab;
add_statistics(cookie, add_stats, NULL, i, "chunk_size", "%u", p->size);
add_statistics(cookie, add_stats, NULL, i, "chunks_per_page", "%u", perslab);
add_statistics(cookie, add_stats, NULL, i, "reserved_pages", "%u", p->rsvd_slabs);
add_statistics(cookie, add_stats, NULL, i, "total_pages", "%u", slabs);
add_statistics(cookie, add_stats, NULL, i, "total_chunks", "%u", slabs*perslab);
add_statistics(cookie, add_stats, NULL, i, "used_chunks", "%u", (slabs*perslab)-p->sl_curr-p->end_page_free);
add_statistics(cookie, add_stats, NULL, i, "free_chunks", "%u", p->sl_curr);
add_statistics(cookie, add_stats, NULL, i, "free_chunks_end", "%u", p->end_page_free);
add_statistics(cookie, add_stats, NULL, i, "mem_requested", "%zu", p->requested);
#ifdef FUTURE
add_statistics(cookie, add_stats, NULL, i, "get_hits", "%"PRIu64, thread_stats.slab_stats[i].get_hits);
add_statistics(cookie, add_stats, NULL, i, "cmd_set", "%"PRIu64, thread_stats.slab_stats[i].set_cmds);
add_statistics(cookie, add_stats, NULL, i, "delete_hits", "%"PRIu64, thread_stats.slab_stats[i].delete_hits);
add_statistics(cookie, add_stats, NULL, i, "cas_hits", "%"PRIu64, thread_stats.slab_stats[i].cas_hits);
add_statistics(cookie, add_stats, NULL, i, "cas_badval", "%"PRIu64, thread_stats.slab_stats[i].cas_badval);
#endif
total++;
}
}
/* add overall slab stats and append terminator */
add_statistics(cookie, add_stats, NULL, -1, "active_slabs", "%d", total);
add_statistics(cookie, add_stats, NULL, -1, "memory_limit", "%zu", engine->slabs.mem_limit);
add_statistics(cookie, add_stats, NULL, -1, "total_malloced", "%zu", engine->slabs.mem_malloced);
}
