void map_path(string path, string fileid) {
//cout << "in map_path" << endl;
//cout << "path: " << path << " fileid: " << fileid << endl;
string token = "";
string attr = "";
stringstream ss2(path.c_str());
while(getline(ss2, token, '/')) {
//cout << "got token " << token << endl;
if(strcmp(token.c_str(),"null")!=0) {
if(attr.length()>0) {
//cout << "mapping " << attr << " to " << token << endl;
database_setval(fileid, attr, token);
if(attr=="location") {
int pos=find(server_ids.begin(),server_ids.end(),token)-server_ids.begin();
if( pos < server_ids.size() ) {
database_setval(fileid, "server", servers.at(pos));
}
}
attr = "";
} else {
attr = token;
}
}
}
//cout << "finished map_path" << endl << endl;
}
void process_file(string server, string fileid) {
string file = database_getval(fileid, "name");
string ext = database_getval(fileid, "ext");
file = server + "/" + file;
string attrs=database_getval(ext,"attrs");
string token="";
stringstream ss2(attrs.c_str());
while(getline(ss2,token,':')){
if(strcmp(token.c_str(),"null")!=0){
if(token == "name") {
continue;
}
//cout << "========= looking at attr = " << token <<endl;
string cmd=database_getval(token+"gen","command");
if(cmd=="null") {
//cout << "command is null, skipping" << endl;
continue;
}
string msg2=(cmd+" \""+file+"\"").c_str();
//cout << "========= issuing command = " << msg2 <<endl;
FILE* stream=popen(msg2.c_str(),"r");
if(fgets(msg,200,stream)!=0){
//cout << "========= attr value = " << msg <<endl;
database_setval(fileid,token,msg);
}
pclose(stream);
}
}
}
int khan_create(const char *path, mode_t mode, struct fuse_file_info *fi)
{
create_calls++;
string fileid=database_getval("name",basename(strdup(path)));
if(strcmp(fileid.c_str(),"null")==0){
fileid=database_setval("null","name",basename(strdup(path)));
database_setval(fileid, "server", servers.at(0));
string ext = strrchr(basename(strdup(path)),'.')+1;
database_setval(fileid, "ext", ext);
}
string server = database_getval(fileid, "server");
process_file(server, fileid);
map_path(resolve_selectors(path), fileid);
return 0;
}
/* Init the database values. Return 0 on failure, 1 on success */
int init_database_values(simple_rec & data)
{
std::string original_fp;
std::string shared_fn;
char exp_id_str[10];
if (sprintf(exp_id_str, "%d", data.exp_id) <= 0)
{
log_err("Copy of file id string failed.");
return 0;
}
original_fp = data.file_path;
shared_fn = "/dev/shm/" + original_fp.substr(49, strlen(data.file_path) - 49);
log_info("Extract attributed for %s", shared_fn.c_str());
std::string ext = strrchr(original_fp.c_str(),'.')+1;
std::string filename=strrchr(original_fp.c_str(),'/')+1;
std::string fileid = database_setval("null","name",filename);
if(!fileid.compare("fail"))
{
log_err("Failure to set filename value in database");
return 0;
}
data.db_id = strdup(fileid.c_str());
//test1 and test2 are non-descript here, we need to ask Akash about there purpose
database_setval(fileid,"ext",ext);
database_setval(fileid,"server","test1");
database_setval(fileid,"location","test2");
database_setval(fileid,"file_path", original_fp);
database_setval(fileid,"experiment_id", exp_id_str);
return 1;
}
static int xmp_rename(const char *from, const char *to) {
//cout << endl << endl << endl << "Entering Rename Function" << endl;
double start_time = 0;
struct timeval start_tv;
gettimeofday(&start_tv, NULL);
start_time = start_tv.tv_sec;
start_time += (start_tv.tv_usec/1000000.0);
start_times << fixed << start_time << endl << flush;
string src = basename(strdup(from));
string dst = basename(strdup(to));
string fileid = database_getval("name", src);
//cout << fileid << endl;
database_remove_val(fileid,"name",src);
//cout << src << endl;
database_setval(fileid,"name",dst);
//cout << dst << endl;
string orig_path = append_path2(src);
string orig_loc = database_getval(fileid,"location");
map_path(resolve_selectors(to), fileid);
string new_path = append_path2(dst);
string new_loc = database_getval(fileid,"location");
if(new_loc!=orig_loc) {
if(new_loc=="google_music") {
//upload
cloud_upload(orig_path);
} else if(orig_loc=="google_music") {
//download
cloud_download(src, new_path);
} else {
//file system rename
rename(orig_path.c_str(), new_path.c_str());
}
}
double rename_time = 0;
struct timeval end_tv;
gettimeofday(&end_tv, NULL);
rename_time = end_tv.tv_sec - start_tv.tv_sec;
rename_time += (end_tv.tv_usec - start_tv.tv_usec) / 1000000.0;
rename_times << fixed << rename_time << endl << flush;
//cout << "Exiting Rename Function" << endl << endl << endl << endl;
return 0;
}
void process_transducers(string server) {
if(server == "cloud") {
return;
}
string line;
ifstream transducers_file((server+"/transducers.txt").c_str());
getline(transducers_file, line);
while(transducers_file.good()){
//cout << "=============== got type = " << line <<endl;
//add line to vold as file type
database_setval("allfiles","types",line);
database_setval(line,"attrs","name");
database_setval(line,"attrs","tags");
database_setval(line,"attrs","location");
database_setval("namegen","command","basename");
database_setval(line,"attrs","ext");
string ext=line;
getline(transducers_file,line);
const char *firstchar=line.c_str();
while(firstchar[0]=='-') {
//add line to vold under filetype as vold
stringstream ss(line.c_str());
string attr;
getline(ss,attr,'-');
getline(ss,attr,':');
string command;
getline(ss,command,':');
//cout << "============ checking attr = "<<attr<<endl;
//cout << "============ checking command = "<<command<<endl;
attr=trim(attr);
database_setval(ext,"attrs",attr);
database_setval(attr+"gen","command",command);
getline(transducers_file,line);
firstchar=line.c_str();
}
}
}
static int xmp_mkdir(const char *path, mode_t mode) {
struct timespec mkdir_start, mkdir_stop;
string strpath=path;
if(strpath.find("localize")!=string::npos) {
if(strpath.find("usage")!=string::npos) {
usage_localize();
} else {
//cout << "LOCALIZING" << endl;
//cout << strpath << endl;
//check location
string filename = "winter.mp3";
string fileid = database_getval("name", filename);
string location = get_location(fileid);
string server = database_getval(fileid, "server");
//cout << "======== LOCATION: " << location << endl << endl;
//if not current
if(location.compare(server)!=0) {
// move to new location
//cout << " MUST MOVE "<<server<<" TO "<<location<<endl;
database_setval(fileid,"server",location);
string from = server + "/" + filename;
string to = location + "/" + filename;
string command = "mv " + from + " " + to;
FILE* stream=popen(command.c_str(),"r");
pclose(stream);
}
}
//cout << "LOCALIZATION TIME:" << localize_time << endl <<endl;
return -1;
}
if(strpath.find("stats")!=string::npos){
//print stats and reset
ofstream stfile;
stfile.open(stats_file.c_str(), ofstream::out);
stfile << "TOT TIME :" << tot_time << endl;
stfile << "Vold Calls :" << vold_calls << endl;
stfile << " Avg Time:" << vold_avg_time << endl;
stfile << "Readdir Calls:" << readdir_calls << endl;
stfile << " Avg Time:" << readdir_avg_time << endl;
stfile << "Access Calls :" << access_calls << endl;
stfile << " Avg Time:" << access_avg_time << endl;
stfile << "Read Calls :" << read_calls << endl;
stfile << " Avg Time:" << read_avg_time << endl;
stfile << "Getattr Calls:" << getattr_calls << endl;
stfile << " Avg Time:" << getattr_avg_time << endl;
stfile << "Write Calls :" << write_calls << endl;
stfile << " Avg Time:" << write_avg_time << endl;
stfile << "Create Calls :" << create_calls << endl;
stfile << " Avg Time:" << create_avg_time << endl;
stfile << "Rename Calls :" << rename_calls << endl;
stfile << " Avg Time:" << rename_avg_time << endl;
stfile.close();
//cout << "TOT TIME :" << tot_time << endl;
//cout << "Vold Calls :" << vold_calls << endl;
//cout << " Avg Time:" << vold_avg_time << endl;
//cout << "Readdir Calls:" << readdir_calls << endl;
//cout << " Avg Time:" << readdir_avg_time << endl;
//cout << "Access Calls :" << access_calls << endl;
//cout << " Avg Time:" << access_avg_time << endl;
//cout << "Read Calls :" << read_calls << endl;
//cout << " Avg Time:" << read_avg_time << endl;
//cout << "Getattr Calls:" << getattr_calls << endl;
//cout << " Avg Time:" << getattr_avg_time << endl;
//cout << "Write Calls :" << write_calls << endl;
//cout << " Avg Time:" << write_avg_time << endl;
//cout << "Create Calls :" << create_calls << endl;
//cout << " Avg Time:" << create_avg_time << endl;
//cout << "Rename Calls :" << rename_calls << endl;
//cout << " Avg Time:" << rename_avg_time << endl;
vold_calls=0;
readdir_calls=0;
access_calls=0;
getattr_calls=0;
read_calls=0;
write_calls=0;
create_calls=0;
rename_calls=0;
tot_time=0;
vold_avg_time=0;
readdir_avg_time=0;
access_avg_time=0;
getattr_avg_time=0;
read_avg_time=0;
write_avg_time=0;
create_avg_time=0;
rename_avg_time=0;
return -1;
}
//log_msg("xmp_mkdir");
sprintf(msg,"khan_mkdir for path=%s\n",path);
//log_msg(msg);
struct stat *st;
if(khan_getattr(path, st)<0) {
//add path
database_setval("alldirs","paths",path);
//and break into attr/val pair and add to vold
} else {
//log_msg("Directory exists\n");
}
return 0;
}
int initializing_khan(char * mnt_dir) {
//log_msg("In initialize\n");
unmounting(mnt_dir);
//Opening root directory and creating if not present
//cout<<"khan_root[0] is "<<servers.at(0)<<endl;
if(NULL == opendir(servers.at(0).c_str())) {
sprintf(msg,"Error msg on opening directory : %s\n",strerror(errno));
//log_msg(msg);
//log_msg("Root directory might not exist..Creating\n");
string command = "mkdir " + servers.at(0);
if (system(command.c_str()) < 0) {
log_msg("Unable to create storage directory...Aborting\n");
exit(1);
}
} else {
fprintf(stderr, "directory opened successfully\n");
}
init_database();
//check if we've loaded metadata before
string output=database_getval("setup","value");
if(output.compare("true")==0){
log_msg("Database was previously initialized.");
tot_time+=(stop.tv_sec-start.tv_sec)+(stop.tv_nsec-start.tv_nsec)/BILLION;
return 0; //setup has happened before
}
//if we have not setup, do so now
//log_msg("it hasnt happened, setvalue then setup");
database_setval("setup","value","true");
//load metadata associatons
for(int i=0; i<servers.size(); i++){
process_transducers(servers.at(i));
}
//load metadata for each file on each server
string types=database_getval("allfiles","types");
//cout << "================= types to look for ="<<types<<endl;
for(int i=0; i<servers.size(); i++) {
if(servers.at(i) == "cloud") {
PyObject* myFunction = PyObject_GetAttrString(cloud_interface,(char*)"get_all_titles");
PyObject* myResult = PyObject_CallObject(myFunction, NULL);
if(myResult==NULL) {
PyErr_PrintEx(0);
continue;
}
int n = PyList_Size(myResult);
//cout << "SIZE = " << n << endl << flush;
for(int j = 0; j<n; j++) {
PyObject* title = PyList_GetItem(myResult, j);
char* temp = PyString_AsString(title);
if(temp==NULL) {
PyErr_PrintEx(0);
continue;
}
string filename = temp;
//cout << "Checking " << filename << " ... " << endl << flush;
if(database_getval("name",filename)=="null") {
string fileid = database_setval("null","name",filename);
string ext = strrchr(filename.c_str(),'.')+1;
database_setval(fileid,"ext",ext);
database_setval(fileid,"server",servers.at(i));
database_setval(fileid,"location",server_ids.at(i));
string attrs=database_getval(ext,"attrs");
string token="";
stringstream ss2(attrs.c_str());
PyObject* myFunction = PyObject_GetAttrString(cloud_interface,(char*)"get_metadata");
while(getline(ss2,token,':')){
if(strcmp(token.c_str(),"null")!=0){
//cout << "========= looking at attr = " << token << endl << flush;
PyObject* arglist = PyTuple_New(2);
PyTuple_SetItem(arglist, 0, PyString_FromString(filename.c_str()));
PyTuple_SetItem(arglist, 1, PyString_FromString(token.c_str()));
PyObject* myResult = PyObject_CallObject(myFunction, arglist);
//cout << myResult << endl << flush;
if(myResult==NULL) {
PyErr_PrintEx(0);
continue;
}
char* msg = PyString_AsString(myResult);
if(!msg) {
PyErr_PrintEx(0);
continue;
}
string val = msg;
Py_DECREF(arglist);
Py_DECREF(myResult);
//cout << "========= got val = " << val << endl << flush;
if(val!="na") {
database_setval(fileid,token,val);
}
}
}
} else {
string fileid = database_getval("name", filename);
database_setval(fileid,"server",servers.at(i));
database_setval(fileid,"location",server_ids.at(i));
}
}
} else {
glob_t files;
glob((servers.at(i)+"/*.*").c_str(),0,NULL,&files);
for(int j=0; j<files.gl_pathc; j++) {//for each file
string file = files.gl_pathv[j];
string ext = strrchr(file.c_str(),'.')+1;
string filename=strrchr(file.c_str(),'/')+1;
if(database_getval("name", filename) == "null") {
string fileid = database_setval("null","name",filename);
database_setval(fileid,"ext",ext);
database_setval(fileid,"server",servers.at(i));
database_setval(fileid,"location",server_ids.at(i));
for(int k=0; k<server_ids.size(); k++) {
database_setval(fileid, server_ids.at(k), "0");
}
if(j%10==0) {
cout << "processed file " << j << "\n";
}
process_file(servers.at(i), fileid);
} else {
string fileid = database_getval("name",filename);
database_setval(fileid,"server",servers.at(i));
database_setval(fileid,"location",server_ids.at(i));
}
}
}
}
//log_msg("At the end of initialize\n");
return 0;
}
