Exemplo n.º 1
0
/**
* Entry point.
*/
int main(int argc,char** argv){
	//Seed the timer...
	srand(time(0));
		
	/**
	* Does the user need help?
	*/
	if(wasArgSpecified("--help",argv,argc)!=0){
		printHelp();
		return 0;
	}

	//Initialize with default configuration.
	BloomOptions_t bloomOptions_t;
	setDefault(&bloomOptions_t);
	//Parset he user's configuration
	getConfiguration(&bloomOptions_t,argv,argc);
	//Show the user the configuration.
	if(!wasArgSpecified("--silent",argv,argc)!=0)
		showDetails(&bloomOptions_t);
	//Do we need to generate new test files?
	if(wasArgSpecified("--generate",argv,argc)!=0){
		generateFiles(bloomOptions_t.numBatches,bloomOptions_t.batchSize);
		generateFilesPrefix(bloomOptions_t.falseBatches,bloomOptions_t.batchSize,
			(char*)"q");
		return 0;
	}
	//Create the bloom filter eing used, and initailize with all 0's.
	char* bloom = (char*)calloc(sizeof(char)*bloomOptions_t.size,sizeof(char));
	int totalNumWords = 0;
	int i = 0;
	for(i = 0;i<bloomOptions_t.numBatches;i++){
		//Do we need to insert it multiple times?
		if(i<bloomOptions_t.trueBatches){
			int y = 0;
			for(;y<bloomOptions_t.numTrueBatchInsertions;y++){
				WordAttributes* wordAttributes = loadFile(i);
				if(y == 0)
					totalNumWords+=wordAttributes->numWords;
				insertWords(bloom,&bloomOptions_t,wordAttributes,bloomOptions_t.prob);
				freeWordAttributes(wordAttributes);
			}
		}else{
			//Only insert it once.
			WordAttributes* wordAttributes = loadFile(i);
			totalNumWords+=wordAttributes->numWords;
			insertWords(bloom,&bloomOptions_t,wordAttributes,bloomOptions_t.prob);
			freeWordAttributes(wordAttributes);
		}
	}

	FILE* pbfOutput = 0;
	if(bloomOptions_t.pbfOutput){
		pbfOutput = fopen(bloomOptions_t.pbfOutput,"w+");
	}

	//Get the stats...
	int numTrueOnesCalculated = 0;
	int numFalseOnesCalculated = 0;
	i = 0;
	for(;i<bloomOptions_t.trueBatches;i++){
		WordAttributes* wordAttributes = loadFile(i);
		int* results = (int*)calloc(sizeof(int)*wordAttributes->numWords,
			sizeof(int));
		queryWords(bloom,&bloomOptions_t,wordAttributes,results);
		if(bloomOptions_t.pbfOutput)
			writeStats(pbfOutput,i,results,wordAttributes->numWords,
				bloomOptions_t.numHashes,bloomOptions_t.prob,wordAttributes->numWords,	
				bloomOptions_t.size);

		free(results);
		freeWordAttributes(wordAttributes);
	}

	if(bloomOptions_t.pbfOutput)
		fprintf(pbfOutput,"false\n");

	for(i = 0;i<bloomOptions_t.falseBatches;i++){
		WordAttributes* wordAttributes = loadFileByPrefix(i,(char*)"q");
		int* results = (int*)calloc(sizeof(int)*wordAttributes->numWords,
			sizeof(int));
		queryWords(bloom,&bloomOptions_t,wordAttributes,results);
		if(bloomOptions_t.pbfOutput){
			writeStats(pbfOutput,i,results,wordAttributes->numWords,
				bloomOptions_t.numHashes,bloomOptions_t.prob,wordAttributes->numWords,	
				bloomOptions_t.size);
		}

		free(results);
		freeWordAttributes(wordAttributes);
	}	

	if(pbfOutput)
		fclose(pbfOutput);

	if(bloomOptions_t.fileName!=0){
		writeBloomFilterToFile(&bloomOptions_t,bloom);
	}

	free(bloom);

	return 0;
}
Exemplo n.º 2
0
int main(int argc,char** argv){
	
	//Seed the timer.
	srand(time(0));
	//printf("Begin GPU PBF..\n");

	//Does the user need help?
	if(wasArgSpecified("--help",argv,argc)!=0){
		printHelp();
		return 0;
	}
	
	//Initialize with default configuration.
	BloomOptions_t bloomOptions_t;
	setDefault(&bloomOptions_t);	
	//Parse the user's configuration.
	getConfiguration(&bloomOptions_t,argv,argc);
	bloomOptions_t.prob = calculateProb(bloomOptions_t.freq,bloomOptions_t.numKeys);
	bloomOptions_t.size = calculateSize(bloomOptions_t.numHashes,bloomOptions_t.numKeys,bloomOptions_t.prob);

	//showDetails(&bloomOptions_t);

	//Create the bloom filter being used, and initialize it with all 0's.
	char* bloom = (char*)malloc(sizeof(char)*bloomOptions_t.size);
	memset(bloom,0,bloomOptions_t.size);

	//Allocate the GPU bloom filter.
	char* dev_bloom = allocateAndCopyChar(bloom,bloomOptions_t.size); 
	if(dev_bloom==0){
		printf("Could not allocate the bloom filter \n");
		return -1;	
	}

	//Read input keys and insert to PBF
	char* fileName = (char*)malloc(sizeof(char)*50);
	sprintf(fileName,"./data/total_keys.txt");
	WordAttributes* allKeys = loadFileByName(fileName);
	int total_keys = allKeys->numWords;	
	int randOffset = rand()%2432+10;			
	insertWordsPBF(dev_bloom,bloomOptions_t.size,allKeys->currentWords,allKeys->positions,allKeys->numWords,allKeys->numBytes,bloomOptions_t.numHashes,bloomOptions_t.device,bloomOptions_t.prob,randOffset);
	freeWordAttributes(allKeys);
			
	//Query PBF
	sprintf(fileName,"./data/distinct_keys.txt");
	WordAttributes* distinctKeys = loadFileByName(fileName);
	int distinct_keys = distinctKeys->numWords;
	//printf("distinct keys = %d, total keys = %d\n",distinct_keys,total_keys);
	int* results = (int*)calloc(sizeof(int)*distinct_keys,sizeof(int));
	queryWordsPBF(dev_bloom,bloomOptions_t.size,distinctKeys->currentWords,distinctKeys->positions,distinctKeys->numWords,
distinctKeys->numBytes,bloomOptions_t.numHashes,bloomOptions_t.device,results);
	freeWordAttributes(distinctKeys);

	//Read the actual frequency of keys
	int* actual = (int*)calloc(sizeof(int)*distinct_keys,sizeof(int));
	FILE* actualFreqFile = fopen("./data/freq.txt","r");
	if(!actualFreqFile){
		printf("Can not open actual frequency file!\n");
		return -1;
	}
	for(int i = 0; i < distinct_keys; i++){
		char* temp = (char*)malloc(sizeof(char)*15);
		if(fgets(temp,15,actualFreqFile)!=NULL){
			int len = strlen(temp);
			temp[len-1]='\0'; 		
			actual[i]=atoi(temp);
			//printf("%d: actual = %d\n",i,actual[i]);
		}
		free(temp);
	}
	
	//Copy the bloom filter to main memory.
	//copyCharsToHost(bloom,dev_bloom,bloomOptions_t.size);

	//Write the result to output file
	//data format: index, number of 1s, calculated frequency, actual frequency, relative error
	if(bloomOptions_t.pbfOutput){
		FILE* outputFile = fopen(bloomOptions_t.pbfOutput,"w");
		writeStats(outputFile,actual,results,distinct_keys, bloomOptions_t.numHashes,bloomOptions_t.prob,total_keys,bloomOptions_t.size);
		fclose(outputFile);
	}
	
	free(fileName);
	free(actual);
	free(results);
	freeChars(dev_bloom);	
	free(bloom);
	//printf("\n");
	return 0;
}