Exemple #1
0
static void dump_qstats_to_db(void) {
    /* qstats hashtable */
    char *kk;
    qstats_t *s;
    struct hashtable *qstats_htp = NULL;    
    qstats_htp = create_hashtable(64, hashfromkey, equalkeys);
    assert(qstats_htp != NULL);
    
    /* cp hashtable to stats table, this is very fast in-memory */
    pthread_rwlock_rdlock(&qlist_ht_lock);
    char *k;
    queue_t *q;
    int result;
    struct hashtable_itr *itr = NULL;
    itr = hashtable_iterator(qlist_htp);
    assert(itr != NULL);
    if (hashtable_count(qlist_htp) > 0)
    {
        do {
            k = hashtable_iterator_key(itr);
            q = hashtable_iterator_value(itr);
            pthread_mutex_lock(&(q->lock));
            if (q->old_set_hits == q->set_hits &&
                q->old_get_hits == q->get_hits) {
                pthread_mutex_unlock(&(q->lock));
                continue;
            }
            q->old_set_hits = q->set_hits;
            q->old_get_hits = q->get_hits;
            pthread_mutex_unlock(&(q->lock));
            kk = strdup(k);
            assert(kk);
            s = calloc(1, sizeof(qstats_t));
            assert(s);
            s->set_hits = q->old_set_hits;
            s->get_hits = q->old_get_hits;
            result = hashtable_insert(qstats_htp, (void *)kk, (void *)s);
            assert(result != 0);
        } while (hashtable_iterator_advance(itr));
    }
    free(itr);
    itr = NULL;
    pthread_rwlock_unlock(&qlist_ht_lock);
    
    /* dump stats hashtable to db */
    DBT dbkey, dbdata;
    int ret;
    DB_TXN *txnp = NULL;
    ret = envp->txn_begin(envp, NULL, &txnp, 0);
    CHECK_DB_RET(ret);
    itr = hashtable_iterator(qstats_htp);
    assert(itr != NULL);
    if (hashtable_count(qstats_htp) > 0)
    {
        do {
            kk = hashtable_iterator_key(itr);
            s = hashtable_iterator_value(itr);
            BDB_CLEANUP_DBT();
            dbkey.data = (void *)kk;
            dbkey.size = strlen(kk) + 1;
            dbdata.data = (void *)s;
            dbdata.size = sizeof(qstats_t);
            ret = qlist_dbp->put(qlist_dbp, txnp, &dbkey, &dbdata, 0);
            CHECK_DB_RET(ret);
            fprintf(stderr, "dump stats[%s], set_hits: %lld, get_hits: %lld \n",
                    kk, s->set_hits, s->get_hits);
        } while (hashtable_iterator_advance(itr));
    }
    free(itr);
    itr = NULL;
    ret = txnp->commit(txnp, 0);
    CHECK_DB_RET(ret);

    hashtable_destroy(qstats_htp, 1);
    qstats_htp = NULL;
    return;
dberr:
    if(itr != NULL) {
        free(itr);
    }
    if (qstats_htp != NULL) {
        hashtable_destroy(qstats_htp, 1);
    }
    if (txnp != NULL){
        txnp->abort(txnp);
    }
    if (settings.verbose > 1) {
        fprintf(stderr, "dump_qstats_to_db: %s\n", db_strerror(ret));
    }
}
Exemple #2
0
void db_open(struct config *conf) {
  int i;
  int ret;
  unsigned int cache_gb, cache_b;
  int oflags;

  /* so gprof works right */
  getitimer(ITIMER_PROF, &global_itimer);

  if ((ret = db_env_create(&env, 0)) != 0) {
    fatal("ENV CREATE: %s\n", db_strerror(ret));
    exit (1);
  }

  /* with such large keys we need more cache to avoid constant disk
     seeks. */
  cache_gb = conf->cache_size / 1000;
  cache_b = (conf->cache_size % 1000) * 1e6;
  info("allocating %iGB and %iB cache\n", cache_gb, cache_b);
  if ((ret = env->set_cachesize(env, cache_gb, cache_b, 0)) != 0) {
    fatal("Error allocating cache error: %s\n", db_strerror(ret));
    exit(1);
  }

  /* set the number of transactions to the number of threads we might
     have, plus one for the commit thread. default is 20. */
  if ((ret = env->set_tx_max(env, MAXCONCURRENCY+1)) != 0) {
    fatal("set_tx_max: %s\n", db_strerror(ret));
    exit(1);
  }

  oflags =  DB_INIT_MPOOL | DB_CREATE | DB_THREAD | DB_INIT_LOCK |
    DB_INIT_LOG |  DB_INIT_TXN | DB_RECOVER | DB_REGISTER;

  if ((ret = env->open(env, conf->data_dir, oflags, 0)) != 0) {
    fatal("ENV OPEN: %s\n", db_strerror(ret));
    exit(1);
  }
  
  if ((ret = env->set_flags(env, DB_TXN_NOSYNC, 1)) != 0) {
    fatal("set flags: %s\n", db_strerror(ret));
    exit(1);
  }

  for (i = 0; i < MAX_SUBSTREAMS; i++) {
    oflags = DB_CREATE | DB_THREAD | DB_AUTO_COMMIT;

    snprintf(dbs[i].dbfile, sizeof(dbs[i].dbfile), "readings-%i.db", i);
    info("Opening '%s'\n", dbs[i].dbfile);
    
    if ((ret = db_create(&dbs[i].dbp, env, 0)) != 0) {
      fatal("CREATE: %i: %s", i, db_strerror(ret));
      exit(1);
    }

    if ((ret = dbs[i].dbp->set_pagesize(dbs[i].dbp, DEFAULT_PAGESIZE)) != 0) {
      warn("set_pagesize: dbid: %i: %s\n", i, db_strerror(ret));
    }

    if ((ret = dbs[i].dbp->open(dbs[i].dbp,
                                NULL, dbs[i].dbfile, NULL, 
                                DB_BTREE, oflags, 0644)) != 0) {
      fatal("db->open: %s\n", db_strerror(ret));
      exit(1);
    }
    
    pthread_mutex_init(&dbs[i].lock, NULL);
    dbs[i].dirty_data = create_hashtable(100,
                                         hash_streamid,
                                         eqfn_streamid);
  }
}
Exemple #3
0
int main(int argc, char* argv[]){

	FILE *fp;
	setlocale(LC_ALL, "zh_TW.UTF-8");

	wchar_t wstring[20];
	wchar_t term[3]={'\0'};
	long count;
	float ratio;
	node ** hashtable;
	node * tempNode;
	wchar_t wchar, pre_wchar;

	hashtable = create_hashtable(hashtable, HASHTABLE_SLOTSIZE);

	fp = fopen(argv[argc-3],"r");

	while(fgetws(wstring,20,fp)!=NULL){
//		wprintf(L"%lc-%ld\n",wchar,wchar);		
	
		swscanf(wstring, L"%lc%lc %ld %f",&term[0],&term[1], &count, &ratio);
		insert_node(hashtable, HASHTABLE_SLOTSIZE, term, count, ratio);	

	}
	
	fclose(fp);


//load stopwords
	fp = fopen(argv[argc-2],"r");
	stopwordNode * stopwordList = loadStopwordFromFile(fp);
	fclose(fp);

//load into article
	wordnode * article_wordlist;
	node ** articleHashTable;
	int article_wordcount=0;
	long term_totalCount=0;

	double ratio_total=0;
	int ratio_count=0;
	float ratio_max=-1000;
	double filterRate;

	keywordnode * keywordlist;
	keywordnode * keywordlistFlag;


	char *buf, *sflag, *eflag, *bufflag;
	char *bflag;
	//char char_buffer[2000];
	char *article_buffer;
	long content_size;
	long buffer_size = 10*bytesPerMB;
	int i;
	size_t frsize=1;
	fpos_t fppos;

	articleHashTable = create_hashtable(articleHashTable, 2000);

	fp = fopen(argv[argc-1],"rb");
	
	if(fp==NULL)
		exit(1);

	buf = (char*)malloc(buffer_size);

	do{
		fgetpos(fp, &fppos);
		frsize = fread(buf, 1, buffer_size, fp);
		bufflag = buf;

		while((sflag=strstr(bufflag,"@GAISRec:"))!=NULL && sflag-buf<frsize){

			eflag=strstr(sflag+2,"@GAISRec:");

			if(eflag==NULL){
				if(!feof(fp)){
					fsetpos(fp,&fppos);
					fseek(fp, sflag-buf, SEEK_CUR);
					break;
				}
				else{
					if((bflag=strstr(sflag,"\[email protected]:"))!=NULL){
						content_size = buf+frsize-bflag-4;
						//strncpy(char_buffer, bflag+4, 2000);
						//swprintf(article_buffer, 2000, L"%hs", char_buffer);
						article_buffer = (char*)malloc(content_size+sizeof(char));
						strncpy(article_buffer, bflag+4, content_size/sizeof(char));
						article_buffer[content_size/sizeof(char)]='\0';

						ratio_total=0;
						ratio_count=0;
						ratio_max=-1000;
					
						article_wordlist = article_loadIntoHt(article_buffer, &article_wordcount, &term_totalCount, articleHashTable, stopwordList);
						article_ratioCount(articleHashTable, 2000, term_totalCount);
						article_countWeight(articleHashTable,hashtable,article_wordlist,article_wordcount, &ratio_total, &ratio_count, &ratio_max);
						
						filterRate = kmeans_rate(article_wordlist, article_wordcount, ratio_count, ratio_max);
						//printf("filterRate: %lf\n",filterRate);
						
						//filterRate = ((ratio_total/(double)ratio_count))*4;
						//printf("%lf\n",filterRate);
						
						keywordlist=article_countKeyword(article_wordlist, article_wordcount, filterRate);
						article_outputKeyword(keywordlist);

	//	printf("\ntotal term:%d\nratio ave:%lf\nlog count:%lf\nratio max:%lf\nfilter rate:%lf\n",ratio_count,ratio_total/(double)ratio_count,log10((double)ratio_count),ratio_max, filterRate);

						for(i=0;i<2000;i++)
							articleHashTable[i]=NULL;

	//memset(articleHashTable, 0, sizeof(node*)*2000);
							free(article_wordlist);
							free(keywordlist);
							free(article_buffer);
							article_wordlist=NULL;
							keywordlist=NULL;

					}
					break;
				}

			}
			else{
				if((bflag=strstr(sflag,"\[email protected]:"))!=NULL){
					content_size = eflag-bflag-4;
					//fwrite(bflag+4, 1, content_size,stdout);
						//strncpy(char_buffer, bflag+4, 2000);
						//swprintf(article_buffer, 2000, L"%hs", char_buffer);
						article_buffer = (char*)malloc(content_size+sizeof(char));
						strncpy(article_buffer, bflag+4, content_size/sizeof(char));
						article_buffer[content_size/sizeof(char)]='\0';

					ratio_total=0;
					ratio_count=0;
					ratio_max=-1000;
					
					article_wordlist =article_loadIntoHt(article_buffer, &article_wordcount, &term_totalCount, articleHashTable, stopwordList);
					article_ratioCount(articleHashTable, 2000, term_totalCount);
					article_countWeight(articleHashTable,hashtable,article_wordlist,article_wordcount, &ratio_total, &ratio_count, &ratio_max);
					
					filterRate = kmeans_rate(article_wordlist, article_wordcount, ratio_count, ratio_max);
					//printf("filterRate: %lf\n",filterRate);
					
					//filterRate = ((ratio_total/(double)ratio_count))*4;
					//printf("%lf\n",filterRate);

					keywordlist=article_countKeyword(article_wordlist, article_wordcount, filterRate);
					article_outputKeyword(keywordlist);
					
//	printf("\ntotal term:%d\nratio ave:%lf\nlog count:%lf\nratio max:%lf\nfilter rate:%lf\n",ratio_count,ratio_total/(double)ratio_count,log10((double)ratio_count),ratio_max, filterRate);

						for(i=0;i<2000;i++)
							articleHashTable[i]=NULL;
							
	//memset(articleHashTable, 0, sizeof(node*)*2000);
							free(article_wordlist);
							free(keywordlist);
							free(article_buffer);
							article_wordlist=NULL;
							keywordlist=NULL;

				}
	
			}
			bufflag = eflag;

		}

	}while(!feof(fp));


	fclose(fp);


	
	
	
	return 0;

}
int bbdocument_init(container **attrkeys) {

	DIR *dirp;
	FILE *filep;
	char buf[512];
	char path[512];
	struct dirent *dp;
	char lines[512];
	char **splitdata;
	int TokCount;
	struct fileFilterFormat *fileFilter = NULL;

	char fileFilterName[] = "fileFilter";
	perl_embed_init(NULL, 1);


	//chtbl_init(&htbl, PRIME_TBLSIZ, bbdocument_h, bbdocument_hmatch, free);
	h_fileFilter = create_hashtable(PRIME_TBLSIZ, bbdocument_h, bbdocument_hmatch);

	printf("opening %s\n",bfile(fileFilterName));
	if ((dirp = opendir(bfile(fileFilterName))) == NULL) {
		fprintf(stderr,"warn: cant open fileFilter \"%s\". Cant use fileFilters\n",bfile(fileFilterName));
		return 1;
	}  
	while ((dp = readdir(dirp)) != NULL) {
		if (dp->d_name[0] == '.') {
			continue;
		}
		sprintf(path,"%s/%s/",bfile(fileFilterName),dp->d_name);
		sprintf(buf,"%sruninfo",path);
		printf("%s\n",buf);
		if ((filep = fopen(buf,"r")) == NULL) {
			printf("no runinfo file for \"%s\"\n",dp->d_name);	
			continue;
		}
		
		printf("loading \"%s\"\n",dp->d_name);


		while ((!feof(filep)) && (fgets(lines,sizeof(lines) -1,filep) != NULL)) {
			

			//blanke linjer og komentarer som starter på #
			if ((lines[0] == '\n') || (lines[0] == '#')) {
				continue;
			}

			//void chomp(char string[])
			chomp(lines);

			//printf("line %s\n",lines);
			TokCount = split(lines, ": ", &splitdata);			
			//printf("\tfound %d token(s):\n", TokCount);

			/*
			if (TokCount != 2) {
				printf("bad config line \"%s\". Splitet in %i elements\n",lines,TokCount);
				continue;
			}
			*/

			if (strcmp(splitdata[0],"documentstype") == 0) {

				//legger til det gamle filteret
				if (fileFilter != NULL) {
					if (NULL != hashtable_search(h_fileFilter,fileFilter->documentstype )) {
						printf("####################### BUG ################################\n");
						printf("allredy have a filter for \"%s\"!\n",fileFilter->documentstype);
						printf("#######################/BUG ################################\n");
					}
					//add to hash
					printf("inserting %s\n",(*fileFilter).documentstype);
					//chtbl_insert(&htbl,(void *)fileFilter);
					if (!hashtable_insert(h_fileFilter,fileFilter->documentstype,fileFilter) ) {
                        		        printf("cant insert\n");
                		        	exit(-1);
		                        }

					printf("end inserting\n");
				}
				//begynner på et nytt filter

				fileFilter = malloc(sizeof(struct fileFilterFormat));
				fileFilter->attrwhitelist = NULL;

				//ikke alle filfiltere har sat alle opsjoner, så vi nulstiller alt, slik at det er lett og strcmp()'e
				//etter en verdi, uten at vi må tenke på at den kansje ikke er satt.
				memset(fileFilter,'\0',sizeof(struct fileFilterFormat));

				// default til FILTER_EXEOC
				fileFilter->filtertype = FILTER_EXEOC;

				strcpy((*fileFilter).documentstype,splitdata[1]);
				
				strlcpy(fileFilter->path, path, sizeof fileFilter->path);
				

			}
			else if (strcmp(splitdata[0],"command") == 0) {
				//vi kan ha : i komandoen. Kopierer derfor først inn hele, så fjerner vi command:
				//strcpy((*fileFilter).command,splitdata[1]);
			
				strscpy((*fileFilter).command,lines,sizeof((*fileFilter).command));
				strcasesandr((*fileFilter).command,sizeof((*fileFilter).command),"command: ","");
				//leger til path der vi har sakt vi skal ha lokal path ( ./ )
				strcasesandr((*fileFilter).command,sizeof((*fileFilter).command),"./",path);
				printf(".command %s\n",(*fileFilter).command);
			}
			else if (strcmp(splitdata[0],"comment") == 0) {
				strscpy((*fileFilter).comment,splitdata[1],sizeof((*fileFilter).comment));
			}
			else if (strcmp(splitdata[0],"format") == 0) {
				strscpy((*fileFilter).format,splitdata[1],sizeof((*fileFilter).format));
			}
			else if (strcmp(splitdata[0],"outputtype") == 0) {
				//stdio, file, osv,,
				strcpy((*fileFilter).outputtype,splitdata[1]);
			}
			else if (strcmp(splitdata[0],"outputformat") == 0) {
				//text, html
				strcpy((*fileFilter).outputformat,splitdata[1]);
			}
			else if (strcmp(splitdata[0], "filtertype") == 0) {
			
				if (strcmp(splitdata[1], FILTER_EXEOC_STR) == 0)
					fileFilter->filtertype = FILTER_EXEOC;

				else if (strcmp(splitdata[1], FILTER_PERL_PLUGIN_STR) == 0) 
					fileFilter->filtertype = FILTER_PERL_PLUGIN;

				else 
					errx(1, "Unknown filtertype %s\n", splitdata[1]);
			
			}
			else if (strcmp(splitdata[0], "attrwhitelist") == 0) {
				// TODO: Free fileFilter->attrwhitelist
				if (!split(splitdata[1], ",", &fileFilter->attrwhitelist))
					warnx("attrwhitelist was empty.");

			}
			else {
				printf("unknown command \"%s\"\n",lines);
			}

			//clean
			FreeSplitList(splitdata);

		}


		if (fileFilter != NULL) {
			//add to hash
			printf("inserting %s\n",(*fileFilter).documentstype);
			//chtbl_insert(&htbl,(void *)fileFilter);
			if (!hashtable_insert(h_fileFilter,fileFilter->documentstype,fileFilter) ) {
                                printf("cant insert\n");
                        	exit(-1);
                        }
			printf("end inserting\n");
		}
		//markerer at vi har lagt det til
		fileFilter = NULL;

		fclose(filep);
	}
	closedir(dirp);

	if (attrkeys != NULL) {
		*attrkeys = ropen();
	}

	return 1;
}
Exemple #5
0
Mote::Mote(nesc_app_t* n) {
  app = n;
  varTable = create_hashtable(128, tossim_hash, tossim_hash_eq);
}
Exemple #6
0
void transaction_clear_readset(transaction* t) {
	if (hashtable_count(t->rs) > 0) {
    	hashtable_destroy(t->rs, 1);
		t->rs = create_hashtable(hashtable_init);
	}
}
int main(int argc, char *argv[]){

	int  numtasks, rank, rc,tag=1;
	MPI_Comm parentcomm;
	MPI_Status stat;
	int a=1;
	char *buffer;
	int buffer_size;


	MPI_Datatype MPI_ENTRY_WORD, oldtypes[2]; 
	int          blockcounts[2];
	int i,j;
	/* MPI_Aint type used to be consistent with syntax of */
	/* MPI_Type_extent routine */
	MPI_Aint    offsets[2], extent;


	/******** creare hashtable **********/
	Hashtable hashtable;
	hashtable = create_hashtable(500);
	
	rc = MPI_Init(&argc,&argv);
	if (rc != MPI_SUCCESS) {
		printf ("Error starting MPI program. Terminating.\n");
		MPI_Abort(MPI_COMM_WORLD, rc);
	}

	MPI_Comm_size(MPI_COMM_WORLD,&numtasks);
	MPI_Comm_rank(MPI_COMM_WORLD,&rank);

	//commit structura mpi 
	
	offsets[0] = 0;
	oldtypes[0] = MPI_INT;
	blockcounts[0] = ENTRIES + 1;

	/* Setup description of the 2 MPI_INT fields n, type */
	/* Need to first figure offset by getting size of MPI_FLOAT */
	MPI_Type_extent(MPI_INT, &extent);
	offsets[1] = (ENTRIES+1) * extent;
	oldtypes[1] = MPI_CHAR;
	blockcounts[1] = ENTRIES * 50;

	/* Now define structured type and commit it */
	MPI_Type_struct(2, blockcounts, offsets, oldtypes, &MPI_ENTRY_WORD);
	MPI_Type_commit(&MPI_ENTRY_WORD);

	MPI_Comm_get_parent( &parentcomm );

	//printf ("Reducer: Number of tasks= %d My rank= %d\n", numtasks,rank);
	MPI_Send(&a,1,MPI_INT,0,tag,parentcomm);

	MPI_Recv(&buffer_size, 1, MPI_INT, MPI_ANY_SOURCE, tag, parentcomm, &stat);
	//alocam spatiu si pentru terminatorul de sir de caractere
	buffer = (char*)malloc((buffer_size+1)*sizeof(char));
			
	MPI_Recv(&buffer[0],buffer_size,MPI_CHAR,MPI_ANY_SOURCE,tag,parentcomm,&stat);
	//se dauaga terminatorul
	buffer[buffer_size]=0;		
	//printf("Reducer %d: string :|%s|\n",rank,buffer);
	//printf("Reduceri %d:Am primit de la parintele %d valoarea %d \n",rank,stat.MPI_SOURCE,buffer_size);
	

   /*******  do some work *******/

	/*********** process work *************/
	process_chunck(buffer,hashtable);

	entry.numar_intrari = 0;
	
	for(i=0;i<hashtable->size;i++){
			
		for(j=0;j<hashtable->buckets[i].current;j++){
			
			if(entry.numar_intrari == ENTRIES){
			
				MPI_Send(&entry,1,MPI_ENTRY_WORD,0,tag,parentcomm);
				entry.numar_intrari=0;
				
							
			}
			else
			{
				entry.aparitii[entry.numar_intrari] = hashtable->buckets[i].entries[j].number;
				//if(strcmp(hashtable->buckets[i].entries[j].value,"said")==0)
					//printf("Reducer %d: has said %d \n",rank,hashtable->buckets[i].entries[j].number);				
				strcpy(entry.cuvinte[entry.numar_intrari],hashtable->buckets[i].entries[j].value);
				entry.numar_intrari++;				
				
			}
			
		}
		
	}

		if(entry.numar_intrari > 0){
			MPI_Send(&entry,1,MPI_ENTRY_WORD,0,tag,parentcomm);
			entry.numar_intrari=0;
		}

	/************ Terminare trimitere hashtable *************/
	//printf("Reducer %d: Send exit to mapper\n",rank);
	entry.aparitii[0] = -1;
	
	MPI_Send(&entry,1,MPI_ENTRY_WORD,0,tag,parentcomm);
	
	MPI_Finalize();
	free(hashtable);
	return 0;
}
static int interpret_table_entry(const char *line)
{
	char buf[1024], type, *path = NULL, *name = NULL;
	int len;
	struct path_htbl_element *ph_elt = NULL;
	struct name_htbl_element *nh_elt = NULL;
	unsigned int mode = 0755, uid = 0, gid = 0, major = 0, minor = 0;
	unsigned int start = 0, increment = 0, count = 0;

	if (sscanf(line, "%1023s %c %o %u %u %u %u %u %u %u",
		   buf, &type, &mode, &uid, &gid, &major, &minor,
		   &start, &increment, &count) < 0)
		return sys_err_msg("sscanf failed");

	dbg_msg(3, "name %s, type %c, mode %o, uid %u, gid %u, major %u, "
		"minor %u, start %u, inc %u, cnt %u",
		buf, type, mode, uid, gid, major, minor, start,
		increment, count);

	len = strnlen(buf, 1024);
	if (len == 1024)
		return err_msg("too long path");

	if (!strcmp(buf, "/"))
		return err_msg("device table entries require absolute paths");
	if (buf[1] == '\0')
		return err_msg("root directory cannot be created");
	if (strstr(buf, "//"))
		return err_msg("'//' cannot be used in the path");
	if (buf[len - 1] == '/')
		return err_msg("do not put '/' at the end");

	if (strstr(buf, "/./") || strstr(buf, "/../") ||
	    !strcmp(buf + len - 2, "/.") || !strcmp(buf + len - 3, "/.."))
		return err_msg("'.' and '..' cannot be used in the path");

	switch (type) {
		case 'd':
			mode |= S_IFDIR;
			break;
		case 'f':
			mode |= S_IFREG;
			break;
		case 'p':
			mode |= S_IFIFO;
			break;
		case 'c':
			mode |= S_IFCHR;
			break;
		case 'b':
			mode |= S_IFBLK;
			break;
		default:
			return err_msg("unsupported file type '%c'", type);
	}

	if (separate_last(buf, len, &path, &name))
		return -1;

	/*
	 * Check if this path already exist in the path hash table and add it
	 * if it is not.
	 */
	ph_elt = hashtable_search(path_htbl, path);
	if (!ph_elt) {
		dbg_msg(3, "inserting '%s' into path hash table", path);
		ph_elt = malloc(sizeof(struct path_htbl_element));
		if (!ph_elt) {
			err_msg("cannot allocate %zd bytes of memory",
				sizeof(struct path_htbl_element));
			goto out_free;
		}

		if (!hashtable_insert(path_htbl, path, ph_elt)) {
			err_msg("cannot insert into path hash table");
			goto out_free;
		}

		ph_elt->path = path;
		path = NULL;
		ph_elt->name_htbl = create_hashtable(128, &r5_hash,
						     &is_equivalent);
		if (!ph_elt->name_htbl) {
			err_msg("cannot create name hash table");
			goto out_free;
		}
	}

	if (increment != 0 && count == 0)
		return err_msg("count cannot be zero if increment is non-zero");

	/*
	 * Add the file/directory/device node (last component of the path) to
	 * the name hashtable. The name hashtable resides in the corresponding
	 * path hashtable element.
	 */

	if (count == 0) {
		/* This entry does not require any iterating */
		nh_elt = malloc(sizeof(struct name_htbl_element));
		if (!nh_elt) {
			err_msg("cannot allocate %zd bytes of memory",
				sizeof(struct name_htbl_element));
			goto out_free;
		}

		nh_elt->mode = mode;
		nh_elt->uid = uid;
		nh_elt->gid = gid;
		nh_elt->dev = makedev(major, minor);

		dbg_msg(3, "inserting '%s' into name hash table (major %d, minor %d)",
			name, major(nh_elt->dev), minor(nh_elt->dev));

		if (hashtable_search(ph_elt->name_htbl, name))
			return err_msg("'%s' is referred twice", buf);

		nh_elt->name = name;
		if (!hashtable_insert(ph_elt->name_htbl, name, nh_elt)) {
			err_msg("cannot insert into name hash table");
			goto out_free;
		}
	} else {
		int i, num = start + increment * count, len = strlen(name) + 20;
		char *nm;

		for (i = start; i < num; i++) {
			nh_elt = malloc(sizeof(struct name_htbl_element));
			if (!nh_elt) {
				err_msg("cannot allocate %zd bytes of memory",
					sizeof(struct name_htbl_element));
				goto out_free;
			}

			nh_elt->mode = mode;
			nh_elt->uid = uid;
			nh_elt->gid = gid;
			nh_elt->dev = makedev(major, minor + i - start);

			nm = malloc(len);
			if (!nm) {
				err_msg("cannot allocate %d bytes of memory", len);
				goto out_free;
			}

			sprintf(nm, "%s%d", name, i);
			nh_elt->name = nm;

			dbg_msg(3, "inserting '%s' into name hash table (major %d, minor %d)",
			        nm, major(nh_elt->dev), minor(nh_elt->dev));

			if (hashtable_search(ph_elt->name_htbl, nm)) {
				err_msg("'%s' is referred twice", buf);
				free (nm);
				goto out_free;
			}

			if (!hashtable_insert(ph_elt->name_htbl, nm, nh_elt)) {
				err_msg("cannot insert into name hash table");
				free (nm);
				goto out_free;
			}
		}
		free(name);
		name = NULL;
	}

	return 0;

out_free:
	free(ph_elt);
	free(nh_elt);
	free(path);
	free(name);
	return -1;
}
/**
 * parse_devtable - parse the device table.
 * @tbl_file: device table file name
 *
 * This function parses the device table and prepare the hash table which will
 * later be used by mkfs.ubifs to create the specified files/device nodes.
 * Returns zero in case of success and a negative error code in case of
 * failure.
 */
int parse_devtable(const char *tbl_file)
{
	FILE *f;
	char *line = NULL;
	struct stat st;
	size_t len;

	dbg_msg(1, "parsing device table file '%s'", tbl_file);

	path_htbl = create_hashtable(128, &r5_hash, &is_equivalent);
	if (!path_htbl)
		return err_msg("cannot create path hash table");

	f = fopen(tbl_file, "r");
	if (!f)
		return sys_err_msg("cannot open '%s'", tbl_file);

	if (fstat(fileno(f), &st) < 0) {
		sys_err_msg("cannot stat '%s'", tbl_file);
		goto out_close;
	}

	if (st.st_size < 10) {
		sys_err_msg("'%s' is too short", tbl_file);
		goto out_close;
	}

	/*
	 * The general plan now is to read in one line at a time, check for
	 * leading comment delimiters ('#'), then try and parse the line as a
	 * device table
	 */
	while (getline(&line, &len, f) != -1) {
		/* First trim off any white-space */
		len = strlen(line);

		/* Trim trailing white-space */
		while (len > 0 && isspace(line[len - 1]))
			line[--len] = '\0';
		/* Trim leading white-space */
		memmove(line, &line[strspn(line, " \n\r\t\v")], len);

		/* How long are we after trimming? */
		len = strlen(line);

		/* If this is not a comment line, try to interpret it */
		if (len && *line != '#') {
			if (interpret_table_entry(line)) {
				err_msg("cannot parse '%s'", line);
				goto out_close;
			}
		}

		free(line);
		line = NULL;
	}

	dbg_msg(1, "finished parsing");
	fclose(f);
	return 0;

out_close:
	fclose(f);
	free_devtable_info();
	return -1;
}
Exemple #10
0
hashtable_t* create_int_hashtable(int capacity)
{
	return create_hashtable(capacity, sizeof(int));
}
int main(int argc, char *argv[]){
	int  numtasks, rank, rc,tag=1;

	FILE *configFile,*fileOut;

	char *command; 
	char **argv2; 
	command = "./mapper_program"; 
		
	char configFileName[] = "config.in";
	int maxMappers;
	int maxReducers;	
	MPI_Status stat;
	MPI_Comm intercomm;
	Hashtable hashtable;
	int i,j;
	
	char fileInName[100];
	char fileOutName[100];
	MPI_Datatype MPI_ENTRY_WORD, oldtypes[2]; 
	int          blockcounts[2];

	/* MPI_Aint type used to be consistent with syntax of */
	/* MPI_Type_extent routine */
	MPI_Aint    offsets[2], extent;

	
	hashtable = create_hashtable(1000);
	
	configFile = fopen(configFileName,"r");
	if(configFile == NULL)
		printf("Error opening %s file for configurations\n",configFileName);
	
	//citire fisier de configuratii
	fscanf(configFile,"%d\n",&maxMappers);
	fscanf(configFile,"%d\n",&maxReducers);
	

	fscanf(configFile,"%s\n",fileInName);
	fscanf(configFile,"%s\n",fileOutName);

	fileOut = fopen(fileOutName,"w");
	printf("Master: mappers %d reducers %d\n",maxMappers,maxReducers);
	

	
	
	
	char buff[20];
	sprintf(buff,"%d",maxReducers);
		
	argv2=(char **)malloc(3 * sizeof(char *)); 
	argv2[0] = (char*)malloc(100*sizeof(char));
	strcpy(argv2[0],fileInName);	
	argv2[1] = buff; 
	

	argv2[2] = NULL; 
	
	
	printf("Master: Fisierul de citit %s | %s ------------------\n",argv2[1],argv2[0]);

	int *errcodes = (int*)malloc(maxMappers * sizeof(int));
	
	
	rc = MPI_Init(&argc,&argv);
	if (rc != MPI_SUCCESS) {
		printf ("Error starting MPI program. Terminating.\n");
		MPI_Abort(MPI_COMM_WORLD, rc);
	}

	//commit structura mpi 
	
	offsets[0] = 0;
	oldtypes[0] = MPI_INT;
	blockcounts[0] = ENTRIES + 1;

	/* Setup description of the 2 MPI_INT fields n, type */
	/* Need to first figure offset by getting size of MPI_FLOAT */
	MPI_Type_extent(MPI_INT, &extent);
	offsets[1] = (ENTRIES + 1) * extent;
	oldtypes[1] = MPI_CHAR;
	blockcounts[1] = ENTRIES * 50;

	/* Now define structured type and commit it */
	MPI_Type_struct(2, blockcounts, offsets, oldtypes, &MPI_ENTRY_WORD);
	MPI_Type_commit(&MPI_ENTRY_WORD);


	MPI_Comm_size(MPI_COMM_WORLD,&numtasks);
	MPI_Comm_rank(MPI_COMM_WORLD,&rank);
		
	//proces principal ce va face spawn proceselor map		
	MPI_Comm_spawn(command,argv2,maxMappers,MPI_INFO_NULL,0,MPI_COMM_WORLD,&intercomm,errcodes);
		
	

	
	//for(i=0;i<maxMappers;i++)
	//	MPI_Send(&i,1,MPI_INT,i,tag,intercomm);

	
	
	
   /*******  do some work *******/

	int ok=0;

	while(ok<maxMappers){

		MPI_Recv(&entry,1,MPI_ENTRY_WORD,MPI_ANY_SOURCE,tag,intercomm,&stat);
		
		if(entry.aparitii[0] == -1){
			ok++;
			printf("Master:Finalize from %d\n",stat.MPI_SOURCE);
		}
		else
			for(i=0;i<entry.numar_intrari;i++)
				merge_elements(hashtable,entry.cuvinte[i],entry.aparitii[i]);

	}

	

	
	MPI_Finalize();
	//print_hashtable(hashtable,fileOut);
	
	
   Entry *list;
	int listSize = 1000;
	int listPoz = 0;
	list = (Entry*) malloc(sizeof(Entry)*listSize);
	
   printf("Master: start to make list\n");
	for(i=0;i<hashtable->size;i++){

		if(hashtable->buckets[i].size > 0){
			for(j=0;j<hashtable->buckets[i].current;j++){
				if(listPoz == listSize){
					list = (Entry*)realloc(list,sizeof(Entry)*(listSize+1000));
					listSize += 1000;				
				}
				list[listPoz].value = hashtable->buckets[i].entries[j].value;
				list[listPoz].number = 	hashtable->buckets[i].entries[j].number;
				listPoz++;			
					
			}
			
		}
				
	}
	printf ("Master: %d am terminat\n",rank);

   // sort the list using quicksort
   quicksort(list,0,listPoz-1);

   // print the result
   //printf("The list after sorting using quicksort algorithm:\n");
   printlist(list,listPoz,fileOut);




	free_hashtable(hashtable);
	fclose(fileOut);
	return 0;
}
Exemple #12
0
/* create_string_hashtable */
hashtable *
create_string_hashtable()
{
	return create_hashtable(16, hash_str, str_eq);
}
int main(int argc, char *argv[]){

	int  numtasks, rank, rc,tag=1,i,j;
	int maxReducers;
	FILE *file;
	char *command; 
	command = "./reduce_program"; 
	MPI_File mpi_file;
	MPI_Status status,stat;
  	int bufsize,filesize,chuncksize;
  	char *buf;
	int offset;
	int chunck;
	int aux;
	MPI_Comm intercomm_reducers,parentcomm;
	
	Hashtable hashtable;
	
	hashtable = create_hashtable(500);
		
	
	

	
	MPI_Datatype MPI_ENTRY_WORD, oldtypes[2]; 
	int          blockcounts[2];

	/* MPI_Aint type used to be consistent with syntax of */
	/* MPI_Type_extent routine */
	MPI_Aint    offsets[2], extent;


	
	//aflare dimensiune fisier
	file = fopen(argv[1],"r");
	filesize = fsize(file);
	//printf("Mapper: Fisierul are dimensiunea %d\n",filesize);	
	fclose(file);
	maxReducers = atoi(argv[2]);
	int *errcodes = (int*)malloc(maxReducers * sizeof(int));

	//printf("Mapper: Fisierul de intrare este %s ---- %s|\n",argv[1],argv[2]);

	rc = MPI_Init(&argc,&argv);
	if (rc != MPI_SUCCESS) {
		printf ("Error starting MPI program. Terminating.\n");
		MPI_Abort(MPI_COMM_WORLD, rc);
	}

	
	MPI_Comm_size(MPI_COMM_WORLD,&numtasks);
	MPI_Comm_rank(MPI_COMM_WORLD,&rank);
	MPI_Comm_get_parent( &parentcomm );

	
	//commit structura mpi 
	
	offsets[0] = 0;
	oldtypes[0] = MPI_INT;
	blockcounts[0] = ENTRIES + 1;

	/*Commit structura definita pentru comunicatia informatiilor intre reduceri si mapperi si mapperi si master
	*/
	
	MPI_Type_extent(MPI_INT, &extent);
	offsets[1] = (ENTRIES + 1) * extent;
	oldtypes[1] = MPI_CHAR;
	blockcounts[1] = ENTRIES * 50;

	/* Now define structured type and commit it */
	MPI_Type_struct(2, blockcounts, offsets, oldtypes, &MPI_ENTRY_WORD);
	MPI_Type_commit(&MPI_ENTRY_WORD);

	//printf("Redy for spawn");
		
	//proces principal ce va face spawn proceselor map		
	MPI_Comm_spawn(command,MPI_ARGV_NULL,maxReducers,MPI_INFO_NULL,0,MPI_COMM_SELF,&intercomm_reducers,errcodes);
	for(i=0;i<maxReducers;i++)
		if(errcodes[i] == MPI_SUCCESS)
			printf("Mapper %d: spwan reducer %d\n",rank,i);
		else
			printf("Mapper %d:error spwn reducer %d\n",rank,i);
		
 	bufsize = filesize/numtasks;

	if(rank == numtasks-1){
		bufsize = filesize - rank*bufsize;
	}	
	chuncksize = bufsize/maxReducers;
	

	 buf = (char*)malloc(sizeof(char)*bufsize);
  
 	 MPI_File_open(MPI_COMM_WORLD, argv[1], MPI_MODE_RDONLY, MPI_INFO_NULL, &mpi_file);
 	 MPI_File_read_at(mpi_file, rank*bufsize, buf, bufsize, MPI_CHAR, &status);
 	 MPI_File_close(&mpi_file);

		//printf("Mapper %d: {%s}\n",rank,buf);
	   /*******  do some work *******/
	chunck = chuncksize;
	offset=0;
	//printf("Mapper %d: asteapta procesele reduce\n",rank);
	/********** asteapta ca toti reducerii sa fie activi**************/	
	for(i=0;i<maxReducers;i++){
		MPI_Recv(&aux,1,MPI_INT,MPI_ANY_SOURCE,tag,intercomm_reducers,&stat);
	}

	/********** raspandire chunk-uri catre procesele reduce *************/	
	for(i=0;i<maxReducers;i++){
	
		if(i == maxReducers-1)
		{
			chunck=bufsize-i*chunck;
		}

		//printf("Mapper %d: prepare to send data to reducer %d\n",rank,i);
		MPI_Send(&chunck, 1, MPI_INT, i, tag, intercomm_reducers);
		MPI_Send(&buf[offset], chunck, MPI_CHAR, i, tag, intercomm_reducers);
		
		offset += chunck;
	}
	 	

	int ok=0;
	while(ok<maxReducers){

		MPI_Recv(&entry,1,MPI_ENTRY_WORD,MPI_ANY_SOURCE,tag,intercomm_reducers,&stat);
		
		
		if(entry.aparitii[0] == -1)
			ok++;
		else
			for(i=0;i<entry.numar_intrari;i++)
				merge_elements(hashtable,entry.cuvinte[i],entry.aparitii[i]);
		
	}
	printf("Mapper %d: Am toate informatiile de la procesele reduce\n",rank);


	entry.numar_intrari=0;
	/******* Trimite informatiile la procesul principal  **************/
	for(i=0;i<hashtable->size;i++){
			
		for(j=0;j<hashtable->buckets[i].current;j++){
			
			if(entry.numar_intrari == ENTRIES){
			
				
				MPI_Send(&entry,1,MPI_ENTRY_WORD,0,tag,parentcomm);
				//printf("Mapper %d: trimite hash date\n",rank);
				entry.numar_intrari=0;
							
			}
			else
			{
				//printf("Mapper %d: Cuv %s\n",rank,hashtable->buckets[i].entries[j].value);
				entry.aparitii[entry.numar_intrari] = hashtable->buckets[i].entries[j].number;
				strcpy(entry.cuvinte[entry.numar_intrari],hashtable->buckets[i].entries[j].value);
				entry.numar_intrari++;				
				
			}
			
		}
		
	}

		if(entry.numar_intrari > 0){
			MPI_Send(&entry,1,MPI_ENTRY_WORD,0,tag,parentcomm);
			//printf("Mapper %d: trimite hash date\n",rank);
			entry.numar_intrari=0;
		}

	/************ Terminare trimitere hashtable *************/
	entry.aparitii[0] = -1;
	
	//printf("Mapper %d: send finish to main process\n ",rank);
	MPI_Send(&entry,1,MPI_ENTRY_WORD,0,tag,parentcomm);


	//printf("Mapper %d: %d %s",rank,bufsize,buf);

	printf ("Mapper:  %d am terminat\n", rank);


	//print_hashtable(hashtable,stdout);
	MPI_Finalize();
	free_hashtable(hashtable);

	return 0;
}
Exemple #14
0
void *sync_src(void *arg)
{
	sync_thread_arg_t *tmparg = arg;
	int sock_fd = tmparg->sock_fd;
	io_sync_ctrl_block_t *dcp = tmparg->dcp;

	int ret;

	//iterate berkeley db, retrieving each record in turn.
	struct list_head key_head;
	struct hashtable *key_hash;
	DBC *cursorp;
	DBT dbt_key, dbt_data;
	dcp->dbp->cursor(dcp->dbp, NULL, &cursorp, 0);
	memset(&dbt_key, 0, sizeof(DBT));
	memset(&dbt_data, 0, sizeof(DBT));

	pthread_mutex_lock(&(dcp->io_lock));
	key_hash = create_hashtable(10000, hash_from_key_fn, keys_equal_fn);
	INIT_LIST_HEAD(&key_head);
	while ((ret = cursorp->c_get(cursorp, &dbt_key, &dbt_data, DB_NEXT)) == 0) {
		key_list_t *tmp = (key_list_t *)malloc(sizeof(key_list_t));
		memcpy(tmp->key, dbt_key.data, KEY_SIZE);
		list_add_tail(&(tmp->list_node), &key_head);
		if (!hashtable_insert(key_hash, tmp->key, tmp)) {
			perror("hash insert error!");
			return NULL;
		}
	}

	//close
	if (cursorp != NULL)
		cursorp->c_close(cursorp);

	pthread_mutex_unlock(&(dcp->io_lock));

	unsigned int max_packet_size = sizeof(msg_sync_t) + SECTION_SIZE;
	unsigned char *sock_buf = (unsigned char *)malloc(max_packet_size);
	DBT key, data;
	MD5_CTX md5_ctx;
	unsigned char tmp[BLOCK_INDEX_SIZE];
	msg_sync_t msg;
	unsigned int bi, fi = 0;
	off_t off1 = 0, off2 = 0;
	key_list_t *valuep;

	void *buf;
	if (posix_memalign(&buf, 512, SECTION_SIZE) != 0) {
		perror("posix_memalign error");
		return NULL;
	}

	int is_data_end = 0, is_block_end = 1;
	while (!is_data_end) {

		if (recv_task(sock_fd, sock_buf, max_packet_size) <= 0) {
			perror("rece_task error");
			return NULL;
		}

		msg = *((msg_sync_t *)sock_buf);
		unsigned char *sock_data;
		switch (msg.type) {
		//recv section md5 code
		case SYNC_SECTION_MD5:
			if (is_block_end || off2 >= BLOCK_SIZE) {
				perror("wrong section msg");
				return NULL;
			}

			sock_data = sock_buf + sizeof(msg_sync_t);

			//read src section data, and calc md5
			if (lseek(dcp->fd[fi], off1 + off2, SEEK_SET) == (off_t)-1) {
				perror("seek error");
				return NULL;
			}

			if (read(dcp->fd[fi], buf, SECTION_SIZE) != SECTION_SIZE) {
				perror("seek error");
				return NULL;
			}

			MD5Calc(&md5_ctx, (unsigned char *)buf, SECTION_SIZE);
#ifdef _SYNC_SRC_DEBUG__
			unsigned int kk;
			for (kk = 0; kk < 16; ++kk)
				printf("%02X", md5_ctx.digest[kk]);
			printf("\n");
#endif
			off2 += SECTION_SIZE;

			// compare to dest md5
			if (memcmp(md5_ctx.digest, sock_data, 16) == 0) {
				SEND_MSG(0, buf, SYNC_SECTION_SAME, "Can't send msg to client");
			} else {
				SEND_MSG(SECTION_SIZE, buf, SYNC_SECTION_DATA, "Can't send msg to client");
			}

			break;

		case SYNC_BLOCK_KEY:
			sock_data = sock_buf + sizeof(msg_sync_t);
#ifdef _SYNC_SRC_DEBUG__
			printf("%u %u ", msg.size, msg.type);
			int jj;
			for (jj = 0; jj < KEY_SIZE; ++jj) {
				printf("%u ", sock_data[jj]);
			}
			printf("\n");
#endif

			if ((valuep = hashtable_remove(key_hash, sock_data)) != NULL) {
				list_del(&(valuep->list_node));
				free(valuep);
			}

			//get block index by key
			memset(&key, 0, sizeof(DBT));
			key.data = sock_data;
			key.size = KEY_SIZE;

			memset(&data, 0, sizeof(DBT));
			data.data = tmp;
			data.ulen = BLOCK_INDEX_SIZE;
			data.flags = DB_DBT_USERMEM;

			ret = dcp->dbp->get(dcp->dbp, NULL, &key, &data, 0);
			if (ret == DB_NOTFOUND) {
				//reply SYNC_BLOCK_DELETE
				SEND_MSG(0, buf, SYNC_BLOCK_DELETE, "Can't send msg to client");
			} else if (ret) {
				printf("dbp->get error - %d\n", ret);
				perror("dbp->get error");
				return NULL;
			} else {
				//reply SYNC_BLOCK_CONTINUE
				bi = *((unsigned int *)tmp);
				fi = *(unsigned int *)(tmp + 4);
				off1 = bi;
				off1 = off1 * BLOCK_SIZE;
				off2 = 0;
				is_block_end = 0;

				SEND_MSG(0, buf, SYNC_BLOCK_CONTINUE,
					"log_deliver send error -can't send msg to client");
			}

			break;

		case SYNC_BLOCK_END:
			if (off2 != BLOCK_SIZE) {
				perror("wrong section");
				return NULL;
			}
			is_block_end = 1;
			SEND_MSG(0, buf, SYNC_OK, "Can't send msg to client");
			break;

		case SYNC_DATA_END:
			is_data_end = 1;
			break;

		default:
			perror("unknown sync msg code");
			return NULL;
		}
	}

   	//destroy hashtable but not free key and value
    hashtable_destroy(key_hash, 2);

	key_list_t *v;
    while (!list_empty(&key_head)) {
    	v = container_of(key_head.next, key_list_t , list_node);
#ifdef _SYNC_SRC_H__
		int kk;
		for (kk = 0; kk < 8; ++kk) printf("%u ", v->key[kk]);
		printf("\n");
#endif

		memset(&key, 0, sizeof(DBT));
		key.data = v->key;
		key.size = KEY_SIZE;

		memset(&data, 0, sizeof(DBT));
		data.data = tmp;
		data.ulen = BLOCK_INDEX_SIZE;
		data.flags = DB_DBT_USERMEM;

		ret = dcp->dbp->get(dcp->dbp, NULL, &key, &data, 0);
		if (ret == DB_NOTFOUND) {
			//do something here
			//key may be deleted
			list_del(key_head.next);
			continue;
		}

		bi = *(unsigned int *)tmp;
		fi = *(unsigned int *)(tmp + 4);

		//send keys
		SEND_MSG(KEY_SIZE, v->key, SYNC_BLOCK_KEY, "Can't send msg to client");

		//recv reply
		RECV_MSG(SYNC_BLOCK_CONTINUE, "recv key_reply error", "return type error");

		//new block, send section data
		off1 = bi;
		off1 = off1 * BLOCK_SIZE;
		for (off2 = 0; off2 < BLOCK_SIZE; off2 += SECTION_SIZE) {
			if (lseek(dcp->fd[fi], off1 + off2, SEEK_SET) == (off_t)-1) {
				perror("seek error");
				return NULL;
			}

			if (read(dcp->fd[fi], buf, SECTION_SIZE) != SECTION_SIZE) {
				perror("read error");
				return NULL;
			}

			//new block, send section data
			SEND_MSG(SECTION_SIZE, buf, SYNC_SECTION_DATA, "send section data failed");

			RECV_MSG(SYNC_OK, "recv section_reply error", "return type error");
		}

		//new block, send block end
		SEND_MSG(0, buf, SYNC_BLOCK_END, "send block end failed");
		
		RECV_MSG(SYNC_OK, "recv data_end_reply", "return type error");

    	list_del(key_head.next);
    }

	//new block, send data end
	SEND_MSG(0, buf, SYNC_DATA_END, "send data end failed");

	close(sock_fd);
	free(sock_buf);

//EXIT:

	return NULL;
}
Exemple #15
0
int main(int argc, char* argv[]){

	FILE *fp;
	setlocale(LC_ALL, "zh_TW.UTF-8");

	wint_t wchar, pre_wchar=0;

	node ** hashtable;
	node * tempNode;
	wint_t newTerm[2];

	long term_totalcount=0;

	hashtable = create_hashtable(hashtable, HASHTABLE_SLOTSIZE);

	fp = fopen(argv[argc-1],"r");

	while((wchar=fgetwc(fp))!=WEOF){
//		wprintf(L"%lc-%ld\n",wchar,wchar);		

		if(wchar==65292 || wchar==12290 || wchar==10 || wchar==12288 || wchar==65306 || wchar==65307 ||
                   wchar==65311 || wchar==65281 || wchar==12289 || wchar==65295 ||
		   wchar==12300 || wchar==12301 || wchar==12302 || wchar==12303 ||
		   wchar==12304 || wchar==12305 || wchar==12308 || wchar==12309 ||
		   wchar==65288 || wchar==65289 || wchar==183 || wchar<=128)
		{
			pre_wchar=0;
		}
		else
		{

			if(pre_wchar==0)
				pre_wchar = wchar;
			else{
			//wprintf(L"%lc%lc\n",pre_wchar,wchar);

				newTerm[0]=pre_wchar;
				newTerm[1]= wchar;

				if((tempNode=find_node(hashtable, newTerm))!=NULL){
					tempNode->count++;
					term_totalcount++;
				}
				else{
					insert_node(hashtable, newTerm);
					term_totalcount++;
				}
				
			pre_wchar=wchar;
			}
		}

	}
	
	fclose(fp);


	hashtable_generateResult(hashtable, HASHTABLE_SLOTSIZE, term_totalcount);
	return 0;

}
Exemple #16
0
/* file delta coding according to chunk algorithms */
int file_delta(char *src_filename, char *chunk_filename, char *delta_filename, int chunk_algo)
{
	int fd_src, fd_chunk, fd_delta;
	int rwsize;
	hashtable *htab_md5 = NULL;
	hashtable *htab_csum = NULL;
	chunk_file_header chunk_file_hdr;
	chunk_block_entry chunk_bentry;
	delta_file_header delta_file_hdr;
	int i, ret = 0;

	/* open files */
	fd_src = open(src_filename, O_RDONLY);
	if (fd_src == -1)
		return -1;

	fd_chunk = open(chunk_filename, O_RDONLY);
	if (fd_chunk == -1) {
		ret = -1;
		goto _FILE_DELTA_EXIT;
	}

	fd_delta = open(delta_filename, O_WRONLY|O_CREAT|O_TRUNC, 0644);
	if (fd_delta == -1) {
		ret = -1;
		goto _FILE_DELTA_EXIT;
	}

	/* build hashtable from chunk file */
	htab_md5 = create_hashtable(HASHTABLE_BUCKET_SZ);
	htab_csum = create_hashtable(HASHTABLE_BUCKET_SZ);
	if (htab_md5 == NULL || htab_csum == NULL) {
		ret = -1;
		goto _FILE_DELTA_EXIT;
	}

	rwsize = read(fd_chunk, &chunk_file_hdr, CHUNK_FILE_HEADER_SZ);
	if (rwsize == -1 || rwsize != CHUNK_FILE_HEADER_SZ) {
		ret = -1;
		goto _FILE_DELTA_EXIT;
	}

	for(i = 0; i < chunk_file_hdr.block_nr; i++) {
		rwsize = read(fd_chunk, &chunk_bentry, CHUNK_BLOCK_ENTRY_SZ);
		if (rwsize == -1 || rwsize != CHUNK_BLOCK_ENTRY_SZ) {
			ret = -1;
			goto _FILE_DELTA_EXIT;
		}
		hash_checkin(htab_md5, chunk_bentry.md5, chunk_bentry);
		hash_checkin(htab_csum, chunk_bentry.csum, chunk_bentry);
	}

	/* pre-write delte file header */
	delta_file_hdr.block_nr = 0;
	rwsize = write(fd_delta, &delta_file_hdr, DELTA_FILE_HEADER_SZ);
	if (rwsize == -1 || rwsize != DELTA_FILE_HEADER_SZ) {
		ret = -1;
		goto _FILE_DELTA_EXIT;
	}

	/* generate delta file according to chunk algorithms */
	switch(chunk_algo) {
	case CHUNK_FSP:
		ret = file_delta_fsp(htab_md5, htab_csum, fd_src, fd_delta, &delta_file_hdr);
		break;
	case CHUNK_CDC:
		ret = file_delta_cdc(htab_md5, htab_csum, fd_src, fd_delta, &delta_file_hdr);
		break;
	case CHUNK_SBC:
		ret = file_delta_sbc(htab_md5, htab_csum, fd_src, fd_delta, &delta_file_hdr);
		break;
	}

	/* write delta file header */
	if (ret == 0) {
		if (-1 == lseek(fd_delta, 0, SEEK_SET)) {
			ret = -1;
			goto _FILE_DELTA_EXIT;
		}
		rwsize = write(fd_delta, &delta_file_hdr, DELTA_FILE_HEADER_SZ);
		if (rwsize == -1 || rwsize != DELTA_FILE_HEADER_SZ)
			ret = -1;
	}

_FILE_DELTA_EXIT:
	close(fd_src);
	close(fd_chunk);
	close(fd_delta);
	unlink(chunk_filename);
	if (htab_md5 != NULL) hash_free(htab_md5);
	if (htab_csum != NULL) hash_free(htab_csum);

	return ret;
}