/*-------------------------------------------------------------------------*/
Bool
sl_close (object_t *ob)
/* For object <ob>, find and close the database connection.
* Return TRUE on success, FALSE if there wasn't one.
*/
{
sqlite_dbs_t *db = find_db(ob);
ob->open_sqlite_db = MY_FALSE;
if (!db)
return MY_FALSE;
sqlite3_close(db->db);
remove_db(db);
return MY_TRUE;
} /* sl_close() */
int main (int argc, char *argv[]){
//directory or disk device
int devicetype=-1;
//path to the device
char devicepath[100];
//path of databases folder
int dbfolder=0;
char dbfolderpath[100];
while ((argc > 1) && (argv[1][0] == '-'))
{
switch (argv[1][1])
{
case 'f':
//Test if -d is not being used also
if(devicetype!=DEVICE)
devicetype=FOLDER;
else{
printf("Cannot use both -f and -d\n");
usage();
}
break;
case 'd':
//test if -f is not being used also
if(devicetype!=FOLDER)
devicetype=DEVICE;
else{
printf("Cannot use both -f and -d\n\n");
usage();
}
break;
case 'p':
strcpy(devicepath,&argv[1][2]);
break;
case 'b':
str_split(&argv[1][2],FIXEDBLOCKS);
break;
case 'r':
str_split(&argv[1][2],RABIN);
break;
case 'z':
dbfolder=1;
strcpy(dbfolderpath,&argv[1][2]);
break;
case 'h':
help();
break;
default:
printf("Wrong Argument: %s\n", argv[1]);
usage();
exit(0);
break;
}
++argv;
--argc;
}
//test if iotype is defined
if(devicetype!=FOLDER && devicetype!=DEVICE){
printf("missing -f or -d\n\n");
usage();
exit(0);
}
//test if testype is defined
if(strlen(devicepath)==0){
printf("missing -p<value>\n\n");
usage();
exit(0);
}
if(nr_sizes_proc_rabin==0 && nr_sizes_proc==0){
printf("missing -b<value> or -r<value>\n\n");
usage();
exit(0);
}
//Initialize variables
total_blocks=malloc(sizeof(uint64_t)*(nr_sizes_proc+nr_sizes_proc_rabin));
eq=malloc(sizeof(uint64_t)*(nr_sizes_proc+nr_sizes_proc_rabin));
incomplete_blocks=malloc(sizeof(uint64_t)*(nr_sizes_proc+nr_sizes_proc_rabin));
incomplete_space=malloc(sizeof(uint64_t)*(nr_sizes_proc+nr_sizes_proc_rabin));
dif=malloc(sizeof(uint64_t)*(nr_sizes_proc+nr_sizes_proc_rabin));
distinctdup=malloc(sizeof(uint64_t)*(nr_sizes_proc+nr_sizes_proc_rabin));
//zeroed_blocks=malloc(sizeof(uint64_t)*nr_sizes_proc);
space=malloc(sizeof(uint64_t)*(nr_sizes_proc+nr_sizes_proc_rabin));
dbporiginal=malloc(sizeof(DB**)*(nr_sizes_proc+nr_sizes_proc_rabin));
envporiginal=malloc(sizeof(DB_ENV**)*(nr_sizes_proc+nr_sizes_proc_rabin));
dbprinter=malloc(sizeof(DB**)*(nr_sizes_proc+nr_sizes_proc_rabin));
envprinter=malloc(sizeof(DB_ENV**)*(nr_sizes_proc+nr_sizes_proc_rabin));
int aux=0;
for(aux=0;aux<(nr_sizes_proc+nr_sizes_proc_rabin);aux++){
dbporiginal[aux]=malloc(sizeof(DB *));
envporiginal[aux]=malloc(sizeof(DB_ENV *));
dbprinter[aux]=malloc(sizeof(DB *));
envprinter[aux]=malloc(sizeof(DB_ENV *));
char printdbpath[100];
char duplicatedbpath[100];
char sizeid[20];
if(aux<nr_sizes_proc){
sprintf(sizeid,"fixed_%d",sizes_proc[aux]);
}else{
sprintf(sizeid,"rabin_%d",sizes_proc_rabin[aux-nr_sizes_proc]);
}
//if a folder were specified for databases
if(dbfolder==1){
strcpy(printdbpath,PRINTDB);
strcat(printdbpath,sizeid);
strcpy(duplicatedbpath,dbfolderpath);
strcat(duplicatedbpath,sizeid);
}
else{
strcpy(printdbpath,PRINTDB);
strcat(printdbpath,sizeid);
strcpy(duplicatedbpath,DUPLICATEDB);
strcat(duplicatedbpath,sizeid);
}
char mkcmd[200];
sprintf(mkcmd, "mkdir -p %s", printdbpath);
int ress = system(mkcmd);
sprintf(mkcmd, "mkdir -p %s", duplicatedbpath);
ress=system(mkcmd);
if(ress<0)
perror("Error creating folders for databases\n");
printf("Removing old databases\n");
//remove databases if exist
remove_db(duplicatedbpath,dbporiginal[aux],envporiginal[aux]);
remove_db(printdbpath,dbprinter[aux],envprinter[aux]);
printf("Initing new database\n");
init_db(duplicatedbpath,dbporiginal[aux],envporiginal[aux]);
}
//initialize analyzis variables
bzero(incomplete_blocks,(nr_sizes_proc+nr_sizes_proc_rabin)*(sizeof(uint64_t)));
bzero(incomplete_space,(nr_sizes_proc+nr_sizes_proc_rabin)*(sizeof(uint64_t)));
//initialize analyzis variables
bzero(total_blocks,(nr_sizes_proc+nr_sizes_proc_rabin)*(sizeof(uint64_t)));
//identical chunks (that could be eliminated)
bzero(eq,(nr_sizes_proc+nr_sizes_proc_rabin)*(sizeof(uint64_t)));
//distinct chunks
bzero(dif,(nr_sizes_proc+nr_sizes_proc_rabin)*(sizeof(uint64_t)));
//distinct chunks with duplicates
bzero(distinctdup,(nr_sizes_proc+nr_sizes_proc_rabin)*(sizeof(uint64_t)));
//chunks that were appended with zeros due to their size
//bzero(zeroed_blocks,nr_sizes_proc*(sizeof(uint64_t)));
//duplicated disk space
bzero(space,(nr_sizes_proc+nr_sizes_proc_rabin)*(sizeof(uint64_t)));
//initialize rabin block
initialize_rabin_polynomial_defaults();
cur_block=malloc(sizeof(struct rab_block_info *)*nr_sizes_proc_rabin);
int auxc=0;
for(auxc=0;auxc<nr_sizes_proc_rabin;auxc++){
cur_block[auxc]=NULL;
}
//check if it is a folder or device and start processing
if(devicetype==FOLDER){
printf("start processing folder %s\n",devicepath);
search_dir(devicepath);
}
else{
printf("start processing device %s\n",devicepath);
extract_blocks(devicepath);
}
for(aux=0;aux<(nr_sizes_proc+nr_sizes_proc_rabin);aux++){
if(aux>=nr_sizes_proc)
fprintf(stderr,"\n\n\nRabin Results for %d\n",sizes_proc_rabin[aux-nr_sizes_proc]);
else
fprintf(stderr,"\n\n\nFixed Size Results for %d\n",sizes_proc[aux]);
fprintf(stderr,"files scanned %llu\n",(unsigned long long int)nfiles);
fprintf(stderr,"space scanned %llu Bytes (including incomplete blocks)\n",(unsigned long long int)total_space);
fprintf(stderr,"Complete and Incomplete Block statistics:\ntotal blocks scanned %llu\n",(unsigned long long int)total_blocks[aux]);
//fprintf(stderr,"total blocks with zeros appended %llu\n",(unsigned long long int)zeroed_blocks[aux]);
//blocks without any duplicate are the distinct block minus the distinct blocks with duplicates
uint64_t zerodups=dif[aux]-distinctdup[aux];
fprintf(stderr,"blocks without duplicates %llu\n",(unsigned long long int)zerodups);
fprintf(stderr,"distinct blocks with duplicates %llu\n",(unsigned long long int)distinctdup[aux]);
fprintf(stderr,"duplicate blocks %llu\n",(unsigned long long int)eq[aux]);
fprintf(stderr,"space saved %llu Bytes\n",(unsigned long long int)space[aux]);
fprintf(stderr,"Incomplete Block statistics:\nincomplete blocks %llu\n",(unsigned long long int)incomplete_blocks[aux]);
fprintf(stderr,"incomplete blocks space %llu Bytes\n",(unsigned long long int)incomplete_space[aux]);
close_db(dbporiginal[aux],envporiginal[aux]);
//TODO this is not removed to keep the database for dedisgen-utils
//remove_db(duplicatedbpath,dbporiginal,envporiginal);
}
//free memory
free(incomplete_space);
free(incomplete_blocks);
free(total_blocks);
free(eq);
free(dif);
free(distinctdup);
//free(zeroed_blocks);
free(space);
free(dbporiginal);
free(envporiginal);
free(dbprinter);
free(envprinter);
return 0;
}
int main (int argc, char *argv[]){
//directory or disk device
int devicetype=-1;
//path to the device
char devicepath[100];
//path output log
int outputfile=0;
char outputpath[100];
//path of databases folder
int dbfolder=0;
char dbfolderpath[100];
while ((argc > 1) && (argv[1][0] == '-'))
{
switch (argv[1][1])
{
case 'f':
//Test if -d is not being used also
if(devicetype!=DEVICE)
devicetype=FOLDER;
else{
printf("Cannot use both -f and -d\n");
usage();
}
break;
case 'd':
//test if -f is not being used also
if(devicetype!=FOLDER)
devicetype=DEVICE;
else{
printf("Cannot use both -f and -d\n\n");
usage();
}
break;
case 'p':
strcpy(devicepath,&argv[1][2]);
break;
case 'o':
outputfile=1;
strcpy(outputpath,&argv[1][2]);
break;
case 'b':
str_split(&argv[1][2]);
break;
case 'z':
dbfolder=1;
strcpy(dbfolderpath,&argv[1][2]);
break;
case 'h':
help();
break;
default:
printf("Wrong Argument: %s\n", argv[1]);
usage();
exit(0);
break;
}
++argv;
--argc;
}
//test if iotype is defined
if(devicetype!=FOLDER && devicetype!=DEVICE){
printf("missing -f or -d\n\n");
usage();
exit(0);
}
//test if testype is defined
if(strlen(devicepath)==0){
printf("missing -p<value>\n\n");
usage();
exit(0);
}
//test if blocksize >0
if(nr_sizes_proc==0){
nr_sizes_proc=1;
sizes_proc=malloc(sizeof(int));
sizes_proc[0]=4096;
}
//Initialize variables
total_blocks=malloc(sizeof(uint64_t)*nr_sizes_proc);
eq=malloc(sizeof(uint64_t)*nr_sizes_proc);
dif=malloc(sizeof(uint64_t)*nr_sizes_proc);
distinctdup=malloc(sizeof(uint64_t)*nr_sizes_proc);
//zeroed_blocks=malloc(sizeof(uint64_t)*nr_sizes_proc);
space=malloc(sizeof(uint64_t)*nr_sizes_proc);
dbporiginal=malloc(sizeof(DB**)*nr_sizes_proc);
envporiginal=malloc(sizeof(DB_ENV**)*nr_sizes_proc);
dbprinter=malloc(sizeof(DB**)*nr_sizes_proc);
envprinter=malloc(sizeof(DB_ENV**)*nr_sizes_proc);
int aux=0;
for(aux=0;aux<nr_sizes_proc;aux++){
dbporiginal[aux]=malloc(sizeof(DB *));
envporiginal[aux]=malloc(sizeof(DB_ENV *));
dbprinter[aux]=malloc(sizeof(DB *));
envprinter[aux]=malloc(sizeof(DB_ENV *));
char printdbpath[100];
char duplicatedbpath[100];
char sizeid[5];
sprintf(sizeid,"%d",aux);
//if a folder were specified for databases
if(dbfolder==1){
strcpy(printdbpath,PRINTDB);
strcat(printdbpath,sizeid);
strcpy(duplicatedbpath,dbfolderpath);
strcat(duplicatedbpath,sizeid);
}
else{
strcpy(printdbpath,PRINTDB);
strcat(printdbpath,sizeid);
strcpy(duplicatedbpath,DUPLICATEDB);
strcat(duplicatedbpath,sizeid);
}
char mkcmd[200];
sprintf(mkcmd, "mkdir -p %s", printdbpath);
int ress = system(mkcmd);
sprintf(mkcmd, "mkdir -p %s", duplicatedbpath);
ress=system(mkcmd);
if(ress<0)
perror("Error creating folders for databases\n");
printf("Removing old databases\n");
//remove databases if exist
remove_db(duplicatedbpath,dbporiginal[aux],envporiginal[aux]);
remove_db(printdbpath,dbprinter[aux],envprinter[aux]);
printf("Initing new database\n");
init_db(duplicatedbpath,dbporiginal[aux],envporiginal[aux]);
if(outputfile==1){
init_db(printdbpath,dbprinter[aux],envprinter[aux]);
}
}
//initialize analysis variables
bzero(total_blocks,nr_sizes_proc*(sizeof(uint64_t)));
//identical chunks (that could be eliminated)
bzero(eq,nr_sizes_proc*(sizeof(uint64_t)));
//distinct chunks
bzero(dif,nr_sizes_proc*(sizeof(uint64_t)));
//distinct chunks with duplicates
bzero(distinctdup,nr_sizes_proc*(sizeof(uint64_t)));
//chunks that were appended with zeros due to their size
//bzero(zeroed_blocks,nr_sizes_proc*(sizeof(uint64_t)));
//duplicated disk space
bzero(space,nr_sizes_proc*(sizeof(uint64_t)));
//check if it is a folder or device and start processing
if(devicetype==FOLDER){
printf("start processing folder %s\n",devicepath);
search_dir(devicepath);
}
else{
printf("start processing device %s\n",devicepath);
extract_blocks(devicepath);
}
for(aux=0;aux<nr_sizes_proc;aux++){
fprintf(stderr,"\n\n\nResults for %d\n",sizes_proc[aux]);
fprintf(stderr,"files scanned %llu\n",(unsigned long long int)nfiles);
fprintf(stderr,"total blocks scanned %llu\n",(unsigned long long int)total_blocks[aux]);
//fprintf(stderr,"total blocks with zeros appended %llu\n",(unsigned long long int)zeroed_blocks[aux]);
//blocks without any duplicate are the distinct block minus the distinct blocks with duplicates
uint64_t zerodups=dif[aux]-distinctdup[aux];
fprintf(stderr,"blocks without duplicates %llu\n",(unsigned long long int)zerodups);
fprintf(stderr,"distinct blocks with duplicates %llu\n",(unsigned long long int)distinctdup[aux]);
fprintf(stderr,"duplicate blocks %llu\n",(unsigned long long int)eq[aux]);
fprintf(stderr,"space saved %llu Bytes\n",(unsigned long long int)space[aux]);
//if outputdist was chosen and specified generate it
if(outputfile==1){
printf("before generating output dist\n");
gen_output(dbporiginal[aux],envporiginal[aux],dbprinter[aux],envprinter[aux]);
printf("before printing output dist\n");
char outputfilename[100];
char sizeid[10];
sprintf(sizeid,"%d",sizes_proc[aux]);
strcpy(outputfilename,outputpath);
strcat(outputfilename,sizeid);
FILE* fpp=fopen(outputfilename,"w");
print_elements_print(dbprinter[aux], envprinter[aux],fpp);
fclose(fpp);
close_db(dbprinter[aux],envprinter[aux]);
//TODO this is not removed now but in the future it can be...
//remove_db(printdbpath,dbprinter[aux],envprinter[aux]);
}
close_db(dbporiginal[aux],envporiginal[aux]);
//TODO this is not removed to keep the database for dedisgen-utils
//remove_db(duplicatedbpath,dbporiginal,envporiginal);
}
//free memory
free(total_blocks);
free(eq);
free(dif);
free(distinctdup);
//free(zeroed_blocks);
free(space);
free(dbporiginal);
free(envporiginal);
free(dbprinter);
free(envprinter);
return 0;
}
