static void dump_qstats_to_db(void) { /* qstats hashtable */ char *kk; qstats_t *s; struct hashtable *qstats_htp = NULL; qstats_htp = create_hashtable(64, hashfromkey, equalkeys); assert(qstats_htp != NULL); /* cp hashtable to stats table, this is very fast in-memory */ pthread_rwlock_rdlock(&qlist_ht_lock); char *k; queue_t *q; int result; struct hashtable_itr *itr = NULL; itr = hashtable_iterator(qlist_htp); assert(itr != NULL); if (hashtable_count(qlist_htp) > 0) { do { k = hashtable_iterator_key(itr); q = hashtable_iterator_value(itr); pthread_mutex_lock(&(q->lock)); if (q->old_set_hits == q->set_hits && q->old_get_hits == q->get_hits) { pthread_mutex_unlock(&(q->lock)); continue; } q->old_set_hits = q->set_hits; q->old_get_hits = q->get_hits; pthread_mutex_unlock(&(q->lock)); kk = strdup(k); assert(kk); s = calloc(1, sizeof(qstats_t)); assert(s); s->set_hits = q->old_set_hits; s->get_hits = q->old_get_hits; result = hashtable_insert(qstats_htp, (void *)kk, (void *)s); assert(result != 0); } while (hashtable_iterator_advance(itr)); } free(itr); itr = NULL; pthread_rwlock_unlock(&qlist_ht_lock); /* dump stats hashtable to db */ DBT dbkey, dbdata; int ret; DB_TXN *txnp = NULL; ret = envp->txn_begin(envp, NULL, &txnp, 0); CHECK_DB_RET(ret); itr = hashtable_iterator(qstats_htp); assert(itr != NULL); if (hashtable_count(qstats_htp) > 0) { do { kk = hashtable_iterator_key(itr); s = hashtable_iterator_value(itr); BDB_CLEANUP_DBT(); dbkey.data = (void *)kk; dbkey.size = strlen(kk) + 1; dbdata.data = (void *)s; dbdata.size = sizeof(qstats_t); ret = qlist_dbp->put(qlist_dbp, txnp, &dbkey, &dbdata, 0); CHECK_DB_RET(ret); fprintf(stderr, "dump stats[%s], set_hits: %lld, get_hits: %lld \n", kk, s->set_hits, s->get_hits); } while (hashtable_iterator_advance(itr)); } free(itr); itr = NULL; ret = txnp->commit(txnp, 0); CHECK_DB_RET(ret); hashtable_destroy(qstats_htp, 1); qstats_htp = NULL; return; dberr: if(itr != NULL) { free(itr); } if (qstats_htp != NULL) { hashtable_destroy(qstats_htp, 1); } if (txnp != NULL){ txnp->abort(txnp); } if (settings.verbose > 1) { fprintf(stderr, "dump_qstats_to_db: %s\n", db_strerror(ret)); } }
void db_open(struct config *conf) { int i; int ret; unsigned int cache_gb, cache_b; int oflags; /* so gprof works right */ getitimer(ITIMER_PROF, &global_itimer); if ((ret = db_env_create(&env, 0)) != 0) { fatal("ENV CREATE: %s\n", db_strerror(ret)); exit (1); } /* with such large keys we need more cache to avoid constant disk seeks. */ cache_gb = conf->cache_size / 1000; cache_b = (conf->cache_size % 1000) * 1e6; info("allocating %iGB and %iB cache\n", cache_gb, cache_b); if ((ret = env->set_cachesize(env, cache_gb, cache_b, 0)) != 0) { fatal("Error allocating cache error: %s\n", db_strerror(ret)); exit(1); } /* set the number of transactions to the number of threads we might have, plus one for the commit thread. default is 20. */ if ((ret = env->set_tx_max(env, MAXCONCURRENCY+1)) != 0) { fatal("set_tx_max: %s\n", db_strerror(ret)); exit(1); } oflags = DB_INIT_MPOOL | DB_CREATE | DB_THREAD | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_TXN | DB_RECOVER | DB_REGISTER; if ((ret = env->open(env, conf->data_dir, oflags, 0)) != 0) { fatal("ENV OPEN: %s\n", db_strerror(ret)); exit(1); } if ((ret = env->set_flags(env, DB_TXN_NOSYNC, 1)) != 0) { fatal("set flags: %s\n", db_strerror(ret)); exit(1); } for (i = 0; i < MAX_SUBSTREAMS; i++) { oflags = DB_CREATE | DB_THREAD | DB_AUTO_COMMIT; snprintf(dbs[i].dbfile, sizeof(dbs[i].dbfile), "readings-%i.db", i); info("Opening '%s'\n", dbs[i].dbfile); if ((ret = db_create(&dbs[i].dbp, env, 0)) != 0) { fatal("CREATE: %i: %s", i, db_strerror(ret)); exit(1); } if ((ret = dbs[i].dbp->set_pagesize(dbs[i].dbp, DEFAULT_PAGESIZE)) != 0) { warn("set_pagesize: dbid: %i: %s\n", i, db_strerror(ret)); } if ((ret = dbs[i].dbp->open(dbs[i].dbp, NULL, dbs[i].dbfile, NULL, DB_BTREE, oflags, 0644)) != 0) { fatal("db->open: %s\n", db_strerror(ret)); exit(1); } pthread_mutex_init(&dbs[i].lock, NULL); dbs[i].dirty_data = create_hashtable(100, hash_streamid, eqfn_streamid); } }
int main(int argc, char* argv[]){ FILE *fp; setlocale(LC_ALL, "zh_TW.UTF-8"); wchar_t wstring[20]; wchar_t term[3]={'\0'}; long count; float ratio; node ** hashtable; node * tempNode; wchar_t wchar, pre_wchar; hashtable = create_hashtable(hashtable, HASHTABLE_SLOTSIZE); fp = fopen(argv[argc-3],"r"); while(fgetws(wstring,20,fp)!=NULL){ // wprintf(L"%lc-%ld\n",wchar,wchar); swscanf(wstring, L"%lc%lc %ld %f",&term[0],&term[1], &count, &ratio); insert_node(hashtable, HASHTABLE_SLOTSIZE, term, count, ratio); } fclose(fp); //load stopwords fp = fopen(argv[argc-2],"r"); stopwordNode * stopwordList = loadStopwordFromFile(fp); fclose(fp); //load into article wordnode * article_wordlist; node ** articleHashTable; int article_wordcount=0; long term_totalCount=0; double ratio_total=0; int ratio_count=0; float ratio_max=-1000; double filterRate; keywordnode * keywordlist; keywordnode * keywordlistFlag; char *buf, *sflag, *eflag, *bufflag; char *bflag; //char char_buffer[2000]; char *article_buffer; long content_size; long buffer_size = 10*bytesPerMB; int i; size_t frsize=1; fpos_t fppos; articleHashTable = create_hashtable(articleHashTable, 2000); fp = fopen(argv[argc-1],"rb"); if(fp==NULL) exit(1); buf = (char*)malloc(buffer_size); do{ fgetpos(fp, &fppos); frsize = fread(buf, 1, buffer_size, fp); bufflag = buf; while((sflag=strstr(bufflag,"@GAISRec:"))!=NULL && sflag-buf<frsize){ eflag=strstr(sflag+2,"@GAISRec:"); if(eflag==NULL){ if(!feof(fp)){ fsetpos(fp,&fppos); fseek(fp, sflag-buf, SEEK_CUR); break; } else{ if((bflag=strstr(sflag,"\n@B:"))!=NULL){ content_size = buf+frsize-bflag-4; //strncpy(char_buffer, bflag+4, 2000); //swprintf(article_buffer, 2000, L"%hs", char_buffer); article_buffer = (char*)malloc(content_size+sizeof(char)); strncpy(article_buffer, bflag+4, content_size/sizeof(char)); article_buffer[content_size/sizeof(char)]='\0'; ratio_total=0; ratio_count=0; ratio_max=-1000; article_wordlist = article_loadIntoHt(article_buffer, &article_wordcount, &term_totalCount, articleHashTable, stopwordList); article_ratioCount(articleHashTable, 2000, term_totalCount); article_countWeight(articleHashTable,hashtable,article_wordlist,article_wordcount, &ratio_total, &ratio_count, &ratio_max); filterRate = kmeans_rate(article_wordlist, article_wordcount, ratio_count, ratio_max); //printf("filterRate: %lf\n",filterRate); //filterRate = ((ratio_total/(double)ratio_count))*4; //printf("%lf\n",filterRate); keywordlist=article_countKeyword(article_wordlist, article_wordcount, filterRate); article_outputKeyword(keywordlist); // printf("\ntotal term:%d\nratio ave:%lf\nlog count:%lf\nratio max:%lf\nfilter rate:%lf\n",ratio_count,ratio_total/(double)ratio_count,log10((double)ratio_count),ratio_max, filterRate); for(i=0;i<2000;i++) articleHashTable[i]=NULL; //memset(articleHashTable, 0, sizeof(node*)*2000); free(article_wordlist); free(keywordlist); free(article_buffer); article_wordlist=NULL; keywordlist=NULL; } break; } } else{ if((bflag=strstr(sflag,"\n@B:"))!=NULL){ content_size = eflag-bflag-4; //fwrite(bflag+4, 1, content_size,stdout); //strncpy(char_buffer, bflag+4, 2000); //swprintf(article_buffer, 2000, L"%hs", char_buffer); article_buffer = (char*)malloc(content_size+sizeof(char)); strncpy(article_buffer, bflag+4, content_size/sizeof(char)); article_buffer[content_size/sizeof(char)]='\0'; ratio_total=0; ratio_count=0; ratio_max=-1000; article_wordlist =article_loadIntoHt(article_buffer, &article_wordcount, &term_totalCount, articleHashTable, stopwordList); article_ratioCount(articleHashTable, 2000, term_totalCount); article_countWeight(articleHashTable,hashtable,article_wordlist,article_wordcount, &ratio_total, &ratio_count, &ratio_max); filterRate = kmeans_rate(article_wordlist, article_wordcount, ratio_count, ratio_max); //printf("filterRate: %lf\n",filterRate); //filterRate = ((ratio_total/(double)ratio_count))*4; //printf("%lf\n",filterRate); keywordlist=article_countKeyword(article_wordlist, article_wordcount, filterRate); article_outputKeyword(keywordlist); // printf("\ntotal term:%d\nratio ave:%lf\nlog count:%lf\nratio max:%lf\nfilter rate:%lf\n",ratio_count,ratio_total/(double)ratio_count,log10((double)ratio_count),ratio_max, filterRate); for(i=0;i<2000;i++) articleHashTable[i]=NULL; //memset(articleHashTable, 0, sizeof(node*)*2000); free(article_wordlist); free(keywordlist); free(article_buffer); article_wordlist=NULL; keywordlist=NULL; } } bufflag = eflag; } }while(!feof(fp)); fclose(fp); return 0; }
int bbdocument_init(container **attrkeys) { DIR *dirp; FILE *filep; char buf[512]; char path[512]; struct dirent *dp; char lines[512]; char **splitdata; int TokCount; struct fileFilterFormat *fileFilter = NULL; char fileFilterName[] = "fileFilter"; perl_embed_init(NULL, 1); //chtbl_init(&htbl, PRIME_TBLSIZ, bbdocument_h, bbdocument_hmatch, free); h_fileFilter = create_hashtable(PRIME_TBLSIZ, bbdocument_h, bbdocument_hmatch); printf("opening %s\n",bfile(fileFilterName)); if ((dirp = opendir(bfile(fileFilterName))) == NULL) { fprintf(stderr,"warn: cant open fileFilter \"%s\". Cant use fileFilters\n",bfile(fileFilterName)); return 1; } while ((dp = readdir(dirp)) != NULL) { if (dp->d_name[0] == '.') { continue; } sprintf(path,"%s/%s/",bfile(fileFilterName),dp->d_name); sprintf(buf,"%sruninfo",path); printf("%s\n",buf); if ((filep = fopen(buf,"r")) == NULL) { printf("no runinfo file for \"%s\"\n",dp->d_name); continue; } printf("loading \"%s\"\n",dp->d_name); while ((!feof(filep)) && (fgets(lines,sizeof(lines) -1,filep) != NULL)) { //blanke linjer og komentarer som starter på # if ((lines[0] == '\n') || (lines[0] == '#')) { continue; } //void chomp(char string[]) chomp(lines); //printf("line %s\n",lines); TokCount = split(lines, ": ", &splitdata); //printf("\tfound %d token(s):\n", TokCount); /* if (TokCount != 2) { printf("bad config line \"%s\". Splitet in %i elements\n",lines,TokCount); continue; } */ if (strcmp(splitdata[0],"documentstype") == 0) { //legger til det gamle filteret if (fileFilter != NULL) { if (NULL != hashtable_search(h_fileFilter,fileFilter->documentstype )) { printf("####################### BUG ################################\n"); printf("allredy have a filter for \"%s\"!\n",fileFilter->documentstype); printf("#######################/BUG ################################\n"); } //add to hash printf("inserting %s\n",(*fileFilter).documentstype); //chtbl_insert(&htbl,(void *)fileFilter); if (!hashtable_insert(h_fileFilter,fileFilter->documentstype,fileFilter) ) { printf("cant insert\n"); exit(-1); } printf("end inserting\n"); } //begynner på et nytt filter fileFilter = malloc(sizeof(struct fileFilterFormat)); fileFilter->attrwhitelist = NULL; //ikke alle filfiltere har sat alle opsjoner, så vi nulstiller alt, slik at det er lett og strcmp()'e //etter en verdi, uten at vi må tenke på at den kansje ikke er satt. memset(fileFilter,'\0',sizeof(struct fileFilterFormat)); // default til FILTER_EXEOC fileFilter->filtertype = FILTER_EXEOC; strcpy((*fileFilter).documentstype,splitdata[1]); strlcpy(fileFilter->path, path, sizeof fileFilter->path); } else if (strcmp(splitdata[0],"command") == 0) { //vi kan ha : i komandoen. Kopierer derfor først inn hele, så fjerner vi command: //strcpy((*fileFilter).command,splitdata[1]); strscpy((*fileFilter).command,lines,sizeof((*fileFilter).command)); strcasesandr((*fileFilter).command,sizeof((*fileFilter).command),"command: ",""); //leger til path der vi har sakt vi skal ha lokal path ( ./ ) strcasesandr((*fileFilter).command,sizeof((*fileFilter).command),"./",path); printf(".command %s\n",(*fileFilter).command); } else if (strcmp(splitdata[0],"comment") == 0) { strscpy((*fileFilter).comment,splitdata[1],sizeof((*fileFilter).comment)); } else if (strcmp(splitdata[0],"format") == 0) { strscpy((*fileFilter).format,splitdata[1],sizeof((*fileFilter).format)); } else if (strcmp(splitdata[0],"outputtype") == 0) { //stdio, file, osv,, strcpy((*fileFilter).outputtype,splitdata[1]); } else if (strcmp(splitdata[0],"outputformat") == 0) { //text, html strcpy((*fileFilter).outputformat,splitdata[1]); } else if (strcmp(splitdata[0], "filtertype") == 0) { if (strcmp(splitdata[1], FILTER_EXEOC_STR) == 0) fileFilter->filtertype = FILTER_EXEOC; else if (strcmp(splitdata[1], FILTER_PERL_PLUGIN_STR) == 0) fileFilter->filtertype = FILTER_PERL_PLUGIN; else errx(1, "Unknown filtertype %s\n", splitdata[1]); } else if (strcmp(splitdata[0], "attrwhitelist") == 0) { // TODO: Free fileFilter->attrwhitelist if (!split(splitdata[1], ",", &fileFilter->attrwhitelist)) warnx("attrwhitelist was empty."); } else { printf("unknown command \"%s\"\n",lines); } //clean FreeSplitList(splitdata); } if (fileFilter != NULL) { //add to hash printf("inserting %s\n",(*fileFilter).documentstype); //chtbl_insert(&htbl,(void *)fileFilter); if (!hashtable_insert(h_fileFilter,fileFilter->documentstype,fileFilter) ) { printf("cant insert\n"); exit(-1); } printf("end inserting\n"); } //markerer at vi har lagt det til fileFilter = NULL; fclose(filep); } closedir(dirp); if (attrkeys != NULL) { *attrkeys = ropen(); } return 1; }
Mote::Mote(nesc_app_t* n) { app = n; varTable = create_hashtable(128, tossim_hash, tossim_hash_eq); }
void transaction_clear_readset(transaction* t) { if (hashtable_count(t->rs) > 0) { hashtable_destroy(t->rs, 1); t->rs = create_hashtable(hashtable_init); } }
int main(int argc, char *argv[]){ int numtasks, rank, rc,tag=1; MPI_Comm parentcomm; MPI_Status stat; int a=1; char *buffer; int buffer_size; MPI_Datatype MPI_ENTRY_WORD, oldtypes[2]; int blockcounts[2]; int i,j; /* MPI_Aint type used to be consistent with syntax of */ /* MPI_Type_extent routine */ MPI_Aint offsets[2], extent; /******** creare hashtable **********/ Hashtable hashtable; hashtable = create_hashtable(500); rc = MPI_Init(&argc,&argv); if (rc != MPI_SUCCESS) { printf ("Error starting MPI program. Terminating.\n"); MPI_Abort(MPI_COMM_WORLD, rc); } MPI_Comm_size(MPI_COMM_WORLD,&numtasks); MPI_Comm_rank(MPI_COMM_WORLD,&rank); //commit structura mpi offsets[0] = 0; oldtypes[0] = MPI_INT; blockcounts[0] = ENTRIES + 1; /* Setup description of the 2 MPI_INT fields n, type */ /* Need to first figure offset by getting size of MPI_FLOAT */ MPI_Type_extent(MPI_INT, &extent); offsets[1] = (ENTRIES+1) * extent; oldtypes[1] = MPI_CHAR; blockcounts[1] = ENTRIES * 50; /* Now define structured type and commit it */ MPI_Type_struct(2, blockcounts, offsets, oldtypes, &MPI_ENTRY_WORD); MPI_Type_commit(&MPI_ENTRY_WORD); MPI_Comm_get_parent( &parentcomm ); //printf ("Reducer: Number of tasks= %d My rank= %d\n", numtasks,rank); MPI_Send(&a,1,MPI_INT,0,tag,parentcomm); MPI_Recv(&buffer_size, 1, MPI_INT, MPI_ANY_SOURCE, tag, parentcomm, &stat); //alocam spatiu si pentru terminatorul de sir de caractere buffer = (char*)malloc((buffer_size+1)*sizeof(char)); MPI_Recv(&buffer[0],buffer_size,MPI_CHAR,MPI_ANY_SOURCE,tag,parentcomm,&stat); //se dauaga terminatorul buffer[buffer_size]=0; //printf("Reducer %d: string :|%s|\n",rank,buffer); //printf("Reduceri %d:Am primit de la parintele %d valoarea %d \n",rank,stat.MPI_SOURCE,buffer_size); /******* do some work *******/ /*********** process work *************/ process_chunck(buffer,hashtable); entry.numar_intrari = 0; for(i=0;i<hashtable->size;i++){ for(j=0;j<hashtable->buckets[i].current;j++){ if(entry.numar_intrari == ENTRIES){ MPI_Send(&entry,1,MPI_ENTRY_WORD,0,tag,parentcomm); entry.numar_intrari=0; } else { entry.aparitii[entry.numar_intrari] = hashtable->buckets[i].entries[j].number; //if(strcmp(hashtable->buckets[i].entries[j].value,"said")==0) //printf("Reducer %d: has said %d \n",rank,hashtable->buckets[i].entries[j].number); strcpy(entry.cuvinte[entry.numar_intrari],hashtable->buckets[i].entries[j].value); entry.numar_intrari++; } } } if(entry.numar_intrari > 0){ MPI_Send(&entry,1,MPI_ENTRY_WORD,0,tag,parentcomm); entry.numar_intrari=0; } /************ Terminare trimitere hashtable *************/ //printf("Reducer %d: Send exit to mapper\n",rank); entry.aparitii[0] = -1; MPI_Send(&entry,1,MPI_ENTRY_WORD,0,tag,parentcomm); MPI_Finalize(); free(hashtable); return 0; }
static int interpret_table_entry(const char *line) { char buf[1024], type, *path = NULL, *name = NULL; int len; struct path_htbl_element *ph_elt = NULL; struct name_htbl_element *nh_elt = NULL; unsigned int mode = 0755, uid = 0, gid = 0, major = 0, minor = 0; unsigned int start = 0, increment = 0, count = 0; if (sscanf(line, "%1023s %c %o %u %u %u %u %u %u %u", buf, &type, &mode, &uid, &gid, &major, &minor, &start, &increment, &count) < 0) return sys_err_msg("sscanf failed"); dbg_msg(3, "name %s, type %c, mode %o, uid %u, gid %u, major %u, " "minor %u, start %u, inc %u, cnt %u", buf, type, mode, uid, gid, major, minor, start, increment, count); len = strnlen(buf, 1024); if (len == 1024) return err_msg("too long path"); if (!strcmp(buf, "/")) return err_msg("device table entries require absolute paths"); if (buf[1] == '\0') return err_msg("root directory cannot be created"); if (strstr(buf, "//")) return err_msg("'//' cannot be used in the path"); if (buf[len - 1] == '/') return err_msg("do not put '/' at the end"); if (strstr(buf, "/./") || strstr(buf, "/../") || !strcmp(buf + len - 2, "/.") || !strcmp(buf + len - 3, "/..")) return err_msg("'.' and '..' cannot be used in the path"); switch (type) { case 'd': mode |= S_IFDIR; break; case 'f': mode |= S_IFREG; break; case 'p': mode |= S_IFIFO; break; case 'c': mode |= S_IFCHR; break; case 'b': mode |= S_IFBLK; break; default: return err_msg("unsupported file type '%c'", type); } if (separate_last(buf, len, &path, &name)) return -1; /* * Check if this path already exist in the path hash table and add it * if it is not. */ ph_elt = hashtable_search(path_htbl, path); if (!ph_elt) { dbg_msg(3, "inserting '%s' into path hash table", path); ph_elt = malloc(sizeof(struct path_htbl_element)); if (!ph_elt) { err_msg("cannot allocate %zd bytes of memory", sizeof(struct path_htbl_element)); goto out_free; } if (!hashtable_insert(path_htbl, path, ph_elt)) { err_msg("cannot insert into path hash table"); goto out_free; } ph_elt->path = path; path = NULL; ph_elt->name_htbl = create_hashtable(128, &r5_hash, &is_equivalent); if (!ph_elt->name_htbl) { err_msg("cannot create name hash table"); goto out_free; } } if (increment != 0 && count == 0) return err_msg("count cannot be zero if increment is non-zero"); /* * Add the file/directory/device node (last component of the path) to * the name hashtable. The name hashtable resides in the corresponding * path hashtable element. */ if (count == 0) { /* This entry does not require any iterating */ nh_elt = malloc(sizeof(struct name_htbl_element)); if (!nh_elt) { err_msg("cannot allocate %zd bytes of memory", sizeof(struct name_htbl_element)); goto out_free; } nh_elt->mode = mode; nh_elt->uid = uid; nh_elt->gid = gid; nh_elt->dev = makedev(major, minor); dbg_msg(3, "inserting '%s' into name hash table (major %d, minor %d)", name, major(nh_elt->dev), minor(nh_elt->dev)); if (hashtable_search(ph_elt->name_htbl, name)) return err_msg("'%s' is referred twice", buf); nh_elt->name = name; if (!hashtable_insert(ph_elt->name_htbl, name, nh_elt)) { err_msg("cannot insert into name hash table"); goto out_free; } } else { int i, num = start + increment * count, len = strlen(name) + 20; char *nm; for (i = start; i < num; i++) { nh_elt = malloc(sizeof(struct name_htbl_element)); if (!nh_elt) { err_msg("cannot allocate %zd bytes of memory", sizeof(struct name_htbl_element)); goto out_free; } nh_elt->mode = mode; nh_elt->uid = uid; nh_elt->gid = gid; nh_elt->dev = makedev(major, minor + i - start); nm = malloc(len); if (!nm) { err_msg("cannot allocate %d bytes of memory", len); goto out_free; } sprintf(nm, "%s%d", name, i); nh_elt->name = nm; dbg_msg(3, "inserting '%s' into name hash table (major %d, minor %d)", nm, major(nh_elt->dev), minor(nh_elt->dev)); if (hashtable_search(ph_elt->name_htbl, nm)) { err_msg("'%s' is referred twice", buf); free (nm); goto out_free; } if (!hashtable_insert(ph_elt->name_htbl, nm, nh_elt)) { err_msg("cannot insert into name hash table"); free (nm); goto out_free; } } free(name); name = NULL; } return 0; out_free: free(ph_elt); free(nh_elt); free(path); free(name); return -1; }
/** * parse_devtable - parse the device table. * @tbl_file: device table file name * * This function parses the device table and prepare the hash table which will * later be used by mkfs.ubifs to create the specified files/device nodes. * Returns zero in case of success and a negative error code in case of * failure. */ int parse_devtable(const char *tbl_file) { FILE *f; char *line = NULL; struct stat st; size_t len; dbg_msg(1, "parsing device table file '%s'", tbl_file); path_htbl = create_hashtable(128, &r5_hash, &is_equivalent); if (!path_htbl) return err_msg("cannot create path hash table"); f = fopen(tbl_file, "r"); if (!f) return sys_err_msg("cannot open '%s'", tbl_file); if (fstat(fileno(f), &st) < 0) { sys_err_msg("cannot stat '%s'", tbl_file); goto out_close; } if (st.st_size < 10) { sys_err_msg("'%s' is too short", tbl_file); goto out_close; } /* * The general plan now is to read in one line at a time, check for * leading comment delimiters ('#'), then try and parse the line as a * device table */ while (getline(&line, &len, f) != -1) { /* First trim off any white-space */ len = strlen(line); /* Trim trailing white-space */ while (len > 0 && isspace(line[len - 1])) line[--len] = '\0'; /* Trim leading white-space */ memmove(line, &line[strspn(line, " \n\r\t\v")], len); /* How long are we after trimming? */ len = strlen(line); /* If this is not a comment line, try to interpret it */ if (len && *line != '#') { if (interpret_table_entry(line)) { err_msg("cannot parse '%s'", line); goto out_close; } } free(line); line = NULL; } dbg_msg(1, "finished parsing"); fclose(f); return 0; out_close: fclose(f); free_devtable_info(); return -1; }
hashtable_t* create_int_hashtable(int capacity) { return create_hashtable(capacity, sizeof(int)); }
int main(int argc, char *argv[]){ int numtasks, rank, rc,tag=1; FILE *configFile,*fileOut; char *command; char **argv2; command = "./mapper_program"; char configFileName[] = "config.in"; int maxMappers; int maxReducers; MPI_Status stat; MPI_Comm intercomm; Hashtable hashtable; int i,j; char fileInName[100]; char fileOutName[100]; MPI_Datatype MPI_ENTRY_WORD, oldtypes[2]; int blockcounts[2]; /* MPI_Aint type used to be consistent with syntax of */ /* MPI_Type_extent routine */ MPI_Aint offsets[2], extent; hashtable = create_hashtable(1000); configFile = fopen(configFileName,"r"); if(configFile == NULL) printf("Error opening %s file for configurations\n",configFileName); //citire fisier de configuratii fscanf(configFile,"%d\n",&maxMappers); fscanf(configFile,"%d\n",&maxReducers); fscanf(configFile,"%s\n",fileInName); fscanf(configFile,"%s\n",fileOutName); fileOut = fopen(fileOutName,"w"); printf("Master: mappers %d reducers %d\n",maxMappers,maxReducers); char buff[20]; sprintf(buff,"%d",maxReducers); argv2=(char **)malloc(3 * sizeof(char *)); argv2[0] = (char*)malloc(100*sizeof(char)); strcpy(argv2[0],fileInName); argv2[1] = buff; argv2[2] = NULL; printf("Master: Fisierul de citit %s | %s ------------------\n",argv2[1],argv2[0]); int *errcodes = (int*)malloc(maxMappers * sizeof(int)); rc = MPI_Init(&argc,&argv); if (rc != MPI_SUCCESS) { printf ("Error starting MPI program. Terminating.\n"); MPI_Abort(MPI_COMM_WORLD, rc); } //commit structura mpi offsets[0] = 0; oldtypes[0] = MPI_INT; blockcounts[0] = ENTRIES + 1; /* Setup description of the 2 MPI_INT fields n, type */ /* Need to first figure offset by getting size of MPI_FLOAT */ MPI_Type_extent(MPI_INT, &extent); offsets[1] = (ENTRIES + 1) * extent; oldtypes[1] = MPI_CHAR; blockcounts[1] = ENTRIES * 50; /* Now define structured type and commit it */ MPI_Type_struct(2, blockcounts, offsets, oldtypes, &MPI_ENTRY_WORD); MPI_Type_commit(&MPI_ENTRY_WORD); MPI_Comm_size(MPI_COMM_WORLD,&numtasks); MPI_Comm_rank(MPI_COMM_WORLD,&rank); //proces principal ce va face spawn proceselor map MPI_Comm_spawn(command,argv2,maxMappers,MPI_INFO_NULL,0,MPI_COMM_WORLD,&intercomm,errcodes); //for(i=0;i<maxMappers;i++) // MPI_Send(&i,1,MPI_INT,i,tag,intercomm); /******* do some work *******/ int ok=0; while(ok<maxMappers){ MPI_Recv(&entry,1,MPI_ENTRY_WORD,MPI_ANY_SOURCE,tag,intercomm,&stat); if(entry.aparitii[0] == -1){ ok++; printf("Master:Finalize from %d\n",stat.MPI_SOURCE); } else for(i=0;i<entry.numar_intrari;i++) merge_elements(hashtable,entry.cuvinte[i],entry.aparitii[i]); } MPI_Finalize(); //print_hashtable(hashtable,fileOut); Entry *list; int listSize = 1000; int listPoz = 0; list = (Entry*) malloc(sizeof(Entry)*listSize); printf("Master: start to make list\n"); for(i=0;i<hashtable->size;i++){ if(hashtable->buckets[i].size > 0){ for(j=0;j<hashtable->buckets[i].current;j++){ if(listPoz == listSize){ list = (Entry*)realloc(list,sizeof(Entry)*(listSize+1000)); listSize += 1000; } list[listPoz].value = hashtable->buckets[i].entries[j].value; list[listPoz].number = hashtable->buckets[i].entries[j].number; listPoz++; } } } printf ("Master: %d am terminat\n",rank); // sort the list using quicksort quicksort(list,0,listPoz-1); // print the result //printf("The list after sorting using quicksort algorithm:\n"); printlist(list,listPoz,fileOut); free_hashtable(hashtable); fclose(fileOut); return 0; }
/* create_string_hashtable */ hashtable * create_string_hashtable() { return create_hashtable(16, hash_str, str_eq); }
int main(int argc, char *argv[]){ int numtasks, rank, rc,tag=1,i,j; int maxReducers; FILE *file; char *command; command = "./reduce_program"; MPI_File mpi_file; MPI_Status status,stat; int bufsize,filesize,chuncksize; char *buf; int offset; int chunck; int aux; MPI_Comm intercomm_reducers,parentcomm; Hashtable hashtable; hashtable = create_hashtable(500); MPI_Datatype MPI_ENTRY_WORD, oldtypes[2]; int blockcounts[2]; /* MPI_Aint type used to be consistent with syntax of */ /* MPI_Type_extent routine */ MPI_Aint offsets[2], extent; //aflare dimensiune fisier file = fopen(argv[1],"r"); filesize = fsize(file); //printf("Mapper: Fisierul are dimensiunea %d\n",filesize); fclose(file); maxReducers = atoi(argv[2]); int *errcodes = (int*)malloc(maxReducers * sizeof(int)); //printf("Mapper: Fisierul de intrare este %s ---- %s|\n",argv[1],argv[2]); rc = MPI_Init(&argc,&argv); if (rc != MPI_SUCCESS) { printf ("Error starting MPI program. Terminating.\n"); MPI_Abort(MPI_COMM_WORLD, rc); } MPI_Comm_size(MPI_COMM_WORLD,&numtasks); MPI_Comm_rank(MPI_COMM_WORLD,&rank); MPI_Comm_get_parent( &parentcomm ); //commit structura mpi offsets[0] = 0; oldtypes[0] = MPI_INT; blockcounts[0] = ENTRIES + 1; /*Commit structura definita pentru comunicatia informatiilor intre reduceri si mapperi si mapperi si master */ MPI_Type_extent(MPI_INT, &extent); offsets[1] = (ENTRIES + 1) * extent; oldtypes[1] = MPI_CHAR; blockcounts[1] = ENTRIES * 50; /* Now define structured type and commit it */ MPI_Type_struct(2, blockcounts, offsets, oldtypes, &MPI_ENTRY_WORD); MPI_Type_commit(&MPI_ENTRY_WORD); //printf("Redy for spawn"); //proces principal ce va face spawn proceselor map MPI_Comm_spawn(command,MPI_ARGV_NULL,maxReducers,MPI_INFO_NULL,0,MPI_COMM_SELF,&intercomm_reducers,errcodes); for(i=0;i<maxReducers;i++) if(errcodes[i] == MPI_SUCCESS) printf("Mapper %d: spwan reducer %d\n",rank,i); else printf("Mapper %d:error spwn reducer %d\n",rank,i); bufsize = filesize/numtasks; if(rank == numtasks-1){ bufsize = filesize - rank*bufsize; } chuncksize = bufsize/maxReducers; buf = (char*)malloc(sizeof(char)*bufsize); MPI_File_open(MPI_COMM_WORLD, argv[1], MPI_MODE_RDONLY, MPI_INFO_NULL, &mpi_file); MPI_File_read_at(mpi_file, rank*bufsize, buf, bufsize, MPI_CHAR, &status); MPI_File_close(&mpi_file); //printf("Mapper %d: {%s}\n",rank,buf); /******* do some work *******/ chunck = chuncksize; offset=0; //printf("Mapper %d: asteapta procesele reduce\n",rank); /********** asteapta ca toti reducerii sa fie activi**************/ for(i=0;i<maxReducers;i++){ MPI_Recv(&aux,1,MPI_INT,MPI_ANY_SOURCE,tag,intercomm_reducers,&stat); } /********** raspandire chunk-uri catre procesele reduce *************/ for(i=0;i<maxReducers;i++){ if(i == maxReducers-1) { chunck=bufsize-i*chunck; } //printf("Mapper %d: prepare to send data to reducer %d\n",rank,i); MPI_Send(&chunck, 1, MPI_INT, i, tag, intercomm_reducers); MPI_Send(&buf[offset], chunck, MPI_CHAR, i, tag, intercomm_reducers); offset += chunck; } int ok=0; while(ok<maxReducers){ MPI_Recv(&entry,1,MPI_ENTRY_WORD,MPI_ANY_SOURCE,tag,intercomm_reducers,&stat); if(entry.aparitii[0] == -1) ok++; else for(i=0;i<entry.numar_intrari;i++) merge_elements(hashtable,entry.cuvinte[i],entry.aparitii[i]); } printf("Mapper %d: Am toate informatiile de la procesele reduce\n",rank); entry.numar_intrari=0; /******* Trimite informatiile la procesul principal **************/ for(i=0;i<hashtable->size;i++){ for(j=0;j<hashtable->buckets[i].current;j++){ if(entry.numar_intrari == ENTRIES){ MPI_Send(&entry,1,MPI_ENTRY_WORD,0,tag,parentcomm); //printf("Mapper %d: trimite hash date\n",rank); entry.numar_intrari=0; } else { //printf("Mapper %d: Cuv %s\n",rank,hashtable->buckets[i].entries[j].value); entry.aparitii[entry.numar_intrari] = hashtable->buckets[i].entries[j].number; strcpy(entry.cuvinte[entry.numar_intrari],hashtable->buckets[i].entries[j].value); entry.numar_intrari++; } } } if(entry.numar_intrari > 0){ MPI_Send(&entry,1,MPI_ENTRY_WORD,0,tag,parentcomm); //printf("Mapper %d: trimite hash date\n",rank); entry.numar_intrari=0; } /************ Terminare trimitere hashtable *************/ entry.aparitii[0] = -1; //printf("Mapper %d: send finish to main process\n ",rank); MPI_Send(&entry,1,MPI_ENTRY_WORD,0,tag,parentcomm); //printf("Mapper %d: %d %s",rank,bufsize,buf); printf ("Mapper: %d am terminat\n", rank); //print_hashtable(hashtable,stdout); MPI_Finalize(); free_hashtable(hashtable); return 0; }
void *sync_src(void *arg) { sync_thread_arg_t *tmparg = arg; int sock_fd = tmparg->sock_fd; io_sync_ctrl_block_t *dcp = tmparg->dcp; int ret; //iterate berkeley db, retrieving each record in turn. struct list_head key_head; struct hashtable *key_hash; DBC *cursorp; DBT dbt_key, dbt_data; dcp->dbp->cursor(dcp->dbp, NULL, &cursorp, 0); memset(&dbt_key, 0, sizeof(DBT)); memset(&dbt_data, 0, sizeof(DBT)); pthread_mutex_lock(&(dcp->io_lock)); key_hash = create_hashtable(10000, hash_from_key_fn, keys_equal_fn); INIT_LIST_HEAD(&key_head); while ((ret = cursorp->c_get(cursorp, &dbt_key, &dbt_data, DB_NEXT)) == 0) { key_list_t *tmp = (key_list_t *)malloc(sizeof(key_list_t)); memcpy(tmp->key, dbt_key.data, KEY_SIZE); list_add_tail(&(tmp->list_node), &key_head); if (!hashtable_insert(key_hash, tmp->key, tmp)) { perror("hash insert error!"); return NULL; } } //close if (cursorp != NULL) cursorp->c_close(cursorp); pthread_mutex_unlock(&(dcp->io_lock)); unsigned int max_packet_size = sizeof(msg_sync_t) + SECTION_SIZE; unsigned char *sock_buf = (unsigned char *)malloc(max_packet_size); DBT key, data; MD5_CTX md5_ctx; unsigned char tmp[BLOCK_INDEX_SIZE]; msg_sync_t msg; unsigned int bi, fi = 0; off_t off1 = 0, off2 = 0; key_list_t *valuep; void *buf; if (posix_memalign(&buf, 512, SECTION_SIZE) != 0) { perror("posix_memalign error"); return NULL; } int is_data_end = 0, is_block_end = 1; while (!is_data_end) { if (recv_task(sock_fd, sock_buf, max_packet_size) <= 0) { perror("rece_task error"); return NULL; } msg = *((msg_sync_t *)sock_buf); unsigned char *sock_data; switch (msg.type) { //recv section md5 code case SYNC_SECTION_MD5: if (is_block_end || off2 >= BLOCK_SIZE) { perror("wrong section msg"); return NULL; } sock_data = sock_buf + sizeof(msg_sync_t); //read src section data, and calc md5 if (lseek(dcp->fd[fi], off1 + off2, SEEK_SET) == (off_t)-1) { perror("seek error"); return NULL; } if (read(dcp->fd[fi], buf, SECTION_SIZE) != SECTION_SIZE) { perror("seek error"); return NULL; } MD5Calc(&md5_ctx, (unsigned char *)buf, SECTION_SIZE); #ifdef _SYNC_SRC_DEBUG__ unsigned int kk; for (kk = 0; kk < 16; ++kk) printf("%02X", md5_ctx.digest[kk]); printf("\n"); #endif off2 += SECTION_SIZE; // compare to dest md5 if (memcmp(md5_ctx.digest, sock_data, 16) == 0) { SEND_MSG(0, buf, SYNC_SECTION_SAME, "Can't send msg to client"); } else { SEND_MSG(SECTION_SIZE, buf, SYNC_SECTION_DATA, "Can't send msg to client"); } break; case SYNC_BLOCK_KEY: sock_data = sock_buf + sizeof(msg_sync_t); #ifdef _SYNC_SRC_DEBUG__ printf("%u %u ", msg.size, msg.type); int jj; for (jj = 0; jj < KEY_SIZE; ++jj) { printf("%u ", sock_data[jj]); } printf("\n"); #endif if ((valuep = hashtable_remove(key_hash, sock_data)) != NULL) { list_del(&(valuep->list_node)); free(valuep); } //get block index by key memset(&key, 0, sizeof(DBT)); key.data = sock_data; key.size = KEY_SIZE; memset(&data, 0, sizeof(DBT)); data.data = tmp; data.ulen = BLOCK_INDEX_SIZE; data.flags = DB_DBT_USERMEM; ret = dcp->dbp->get(dcp->dbp, NULL, &key, &data, 0); if (ret == DB_NOTFOUND) { //reply SYNC_BLOCK_DELETE SEND_MSG(0, buf, SYNC_BLOCK_DELETE, "Can't send msg to client"); } else if (ret) { printf("dbp->get error - %d\n", ret); perror("dbp->get error"); return NULL; } else { //reply SYNC_BLOCK_CONTINUE bi = *((unsigned int *)tmp); fi = *(unsigned int *)(tmp + 4); off1 = bi; off1 = off1 * BLOCK_SIZE; off2 = 0; is_block_end = 0; SEND_MSG(0, buf, SYNC_BLOCK_CONTINUE, "log_deliver send error -can't send msg to client"); } break; case SYNC_BLOCK_END: if (off2 != BLOCK_SIZE) { perror("wrong section"); return NULL; } is_block_end = 1; SEND_MSG(0, buf, SYNC_OK, "Can't send msg to client"); break; case SYNC_DATA_END: is_data_end = 1; break; default: perror("unknown sync msg code"); return NULL; } } //destroy hashtable but not free key and value hashtable_destroy(key_hash, 2); key_list_t *v; while (!list_empty(&key_head)) { v = container_of(key_head.next, key_list_t , list_node); #ifdef _SYNC_SRC_H__ int kk; for (kk = 0; kk < 8; ++kk) printf("%u ", v->key[kk]); printf("\n"); #endif memset(&key, 0, sizeof(DBT)); key.data = v->key; key.size = KEY_SIZE; memset(&data, 0, sizeof(DBT)); data.data = tmp; data.ulen = BLOCK_INDEX_SIZE; data.flags = DB_DBT_USERMEM; ret = dcp->dbp->get(dcp->dbp, NULL, &key, &data, 0); if (ret == DB_NOTFOUND) { //do something here //key may be deleted list_del(key_head.next); continue; } bi = *(unsigned int *)tmp; fi = *(unsigned int *)(tmp + 4); //send keys SEND_MSG(KEY_SIZE, v->key, SYNC_BLOCK_KEY, "Can't send msg to client"); //recv reply RECV_MSG(SYNC_BLOCK_CONTINUE, "recv key_reply error", "return type error"); //new block, send section data off1 = bi; off1 = off1 * BLOCK_SIZE; for (off2 = 0; off2 < BLOCK_SIZE; off2 += SECTION_SIZE) { if (lseek(dcp->fd[fi], off1 + off2, SEEK_SET) == (off_t)-1) { perror("seek error"); return NULL; } if (read(dcp->fd[fi], buf, SECTION_SIZE) != SECTION_SIZE) { perror("read error"); return NULL; } //new block, send section data SEND_MSG(SECTION_SIZE, buf, SYNC_SECTION_DATA, "send section data failed"); RECV_MSG(SYNC_OK, "recv section_reply error", "return type error"); } //new block, send block end SEND_MSG(0, buf, SYNC_BLOCK_END, "send block end failed"); RECV_MSG(SYNC_OK, "recv data_end_reply", "return type error"); list_del(key_head.next); } //new block, send data end SEND_MSG(0, buf, SYNC_DATA_END, "send data end failed"); close(sock_fd); free(sock_buf); //EXIT: return NULL; }
int main(int argc, char* argv[]){ FILE *fp; setlocale(LC_ALL, "zh_TW.UTF-8"); wint_t wchar, pre_wchar=0; node ** hashtable; node * tempNode; wint_t newTerm[2]; long term_totalcount=0; hashtable = create_hashtable(hashtable, HASHTABLE_SLOTSIZE); fp = fopen(argv[argc-1],"r"); while((wchar=fgetwc(fp))!=WEOF){ // wprintf(L"%lc-%ld\n",wchar,wchar); if(wchar==65292 || wchar==12290 || wchar==10 || wchar==12288 || wchar==65306 || wchar==65307 || wchar==65311 || wchar==65281 || wchar==12289 || wchar==65295 || wchar==12300 || wchar==12301 || wchar==12302 || wchar==12303 || wchar==12304 || wchar==12305 || wchar==12308 || wchar==12309 || wchar==65288 || wchar==65289 || wchar==183 || wchar<=128) { pre_wchar=0; } else { if(pre_wchar==0) pre_wchar = wchar; else{ //wprintf(L"%lc%lc\n",pre_wchar,wchar); newTerm[0]=pre_wchar; newTerm[1]= wchar; if((tempNode=find_node(hashtable, newTerm))!=NULL){ tempNode->count++; term_totalcount++; } else{ insert_node(hashtable, newTerm); term_totalcount++; } pre_wchar=wchar; } } } fclose(fp); hashtable_generateResult(hashtable, HASHTABLE_SLOTSIZE, term_totalcount); return 0; }
/* file delta coding according to chunk algorithms */ int file_delta(char *src_filename, char *chunk_filename, char *delta_filename, int chunk_algo) { int fd_src, fd_chunk, fd_delta; int rwsize; hashtable *htab_md5 = NULL; hashtable *htab_csum = NULL; chunk_file_header chunk_file_hdr; chunk_block_entry chunk_bentry; delta_file_header delta_file_hdr; int i, ret = 0; /* open files */ fd_src = open(src_filename, O_RDONLY); if (fd_src == -1) return -1; fd_chunk = open(chunk_filename, O_RDONLY); if (fd_chunk == -1) { ret = -1; goto _FILE_DELTA_EXIT; } fd_delta = open(delta_filename, O_WRONLY|O_CREAT|O_TRUNC, 0644); if (fd_delta == -1) { ret = -1; goto _FILE_DELTA_EXIT; } /* build hashtable from chunk file */ htab_md5 = create_hashtable(HASHTABLE_BUCKET_SZ); htab_csum = create_hashtable(HASHTABLE_BUCKET_SZ); if (htab_md5 == NULL || htab_csum == NULL) { ret = -1; goto _FILE_DELTA_EXIT; } rwsize = read(fd_chunk, &chunk_file_hdr, CHUNK_FILE_HEADER_SZ); if (rwsize == -1 || rwsize != CHUNK_FILE_HEADER_SZ) { ret = -1; goto _FILE_DELTA_EXIT; } for(i = 0; i < chunk_file_hdr.block_nr; i++) { rwsize = read(fd_chunk, &chunk_bentry, CHUNK_BLOCK_ENTRY_SZ); if (rwsize == -1 || rwsize != CHUNK_BLOCK_ENTRY_SZ) { ret = -1; goto _FILE_DELTA_EXIT; } hash_checkin(htab_md5, chunk_bentry.md5, chunk_bentry); hash_checkin(htab_csum, chunk_bentry.csum, chunk_bentry); } /* pre-write delte file header */ delta_file_hdr.block_nr = 0; rwsize = write(fd_delta, &delta_file_hdr, DELTA_FILE_HEADER_SZ); if (rwsize == -1 || rwsize != DELTA_FILE_HEADER_SZ) { ret = -1; goto _FILE_DELTA_EXIT; } /* generate delta file according to chunk algorithms */ switch(chunk_algo) { case CHUNK_FSP: ret = file_delta_fsp(htab_md5, htab_csum, fd_src, fd_delta, &delta_file_hdr); break; case CHUNK_CDC: ret = file_delta_cdc(htab_md5, htab_csum, fd_src, fd_delta, &delta_file_hdr); break; case CHUNK_SBC: ret = file_delta_sbc(htab_md5, htab_csum, fd_src, fd_delta, &delta_file_hdr); break; } /* write delta file header */ if (ret == 0) { if (-1 == lseek(fd_delta, 0, SEEK_SET)) { ret = -1; goto _FILE_DELTA_EXIT; } rwsize = write(fd_delta, &delta_file_hdr, DELTA_FILE_HEADER_SZ); if (rwsize == -1 || rwsize != DELTA_FILE_HEADER_SZ) ret = -1; } _FILE_DELTA_EXIT: close(fd_src); close(fd_chunk); close(fd_delta); unlink(chunk_filename); if (htab_md5 != NULL) hash_free(htab_md5); if (htab_csum != NULL) hash_free(htab_csum); return ret; }