int main(int argc, char** argv)
{
if ((argc != 2) && (argc != 3))
{
help();
return -1;
}
Sector client;
Session s;
s.loadInfo("../conf/client.conf");
int result = client.init();
if (result < 0)
{
Utility::print_error(result);
return -1;
}
string passwd = s.m_ClientConf.m_strPassword;
if (s.m_ClientConf.m_strUserName != "root")
{
cout << "please input root password:";
cin >> passwd;
}
int main(int argc, char** argv)
{
if (argc != 2)
{
cerr << "USAGE: mkdir <dir>\n";
return -1;
}
Sector client;
Session s;
s.loadInfo("../conf/client.conf");
if (client.init(s.m_ClientConf.m_strMasterIP, s.m_ClientConf.m_iMasterPort) < 0)
return -1;
if (client.login(s.m_ClientConf.m_strUserName, s.m_ClientConf.m_strPassword, s.m_ClientConf.m_strCertificate.c_str()) < 0)
return -1;
int r = client.mkdir(argv[1]);
if (r < 0)
cerr << "ERROR: " << r << " " << SectorError::getErrorMsg(r) << endl;
client.logout();
client.close();
return 1;
}
int main(int argc, char** argv)
{
if (argc != 2)
{
cout << "USAGE: stat file\n";
return -1;
}
Sector client;
Session s;
s.loadInfo("../conf/client.conf");
if (client.init(s.m_ClientConf.m_strMasterIP, s.m_ClientConf.m_iMasterPort) < 0)
return -1;
if (client.login(s.m_ClientConf.m_strUserName, s.m_ClientConf.m_strPassword, s.m_ClientConf.m_strCertificate.c_str()) < 0)
return -1;
SNode attr;
int r = client.stat(argv[1], attr);
if (r < 0)
{
cout << "ERROR: " << r << " " << SectorError::getErrorMsg(r) << endl;
}
else
{
cout << "FILE NAME: " << attr.m_strName << endl;
if (attr.m_bIsDir)
cout << "DIR: TRUE\n";
else
cout << "DIR: FALSE\n";
cout << "SIZE: " << attr.m_llSize << " bytes" << endl;
time_t ft = attr.m_llTimeStamp;
cout << "LAST MODIFIED: " << ctime(&ft);
cout << "LOCATION: ";
for (set<Address, AddrComp>::iterator i = attr.m_sLocation.begin(); i != attr.m_sLocation.end(); ++ i)
{
cout << i->m_strIP << ":" << i->m_iPort << " ";
}
cout << endl;
}
client.logout();
client.close();
return 1;
}
int main(int argc, char** argv)
{
if (argc != 2)
{
cerr << "usage: text.idx text_file_name" << endl;
return -1;
}
Sector client;
Session s;
s.loadInfo("../conf/client.conf");
if (client.init(s.m_ClientConf.m_strMasterIP, s.m_ClientConf.m_iMasterPort) < 0)
return -1;
if (client.login(s.m_ClientConf.m_strUserName, s.m_ClientConf.m_strPassword, s.m_ClientConf.m_strCertificate.c_str()) < 0)
return -1;
vector<string> files;
files.insert(files.end(), argv[1]);
SphereStream input;
if (input.init(files) < 0)
{
cout << "unable to locate input data files. quit.\n";
return -1;
}
SphereStream output;
output.init(0);
SphereProcess* myproc = client.createSphereProcess();
if (myproc->loadOperator("./funcs/gen_idx" SECTOR_DYNLIB_EXT) < 0)
return -1;
timeval t;
gettimeofday(&t, 0);
cout << "start time " << t.tv_sec << endl;
if (myproc->run(input, output, "gen_idx", 0) < 0)
{
cout << "failed to find any computing resources." << endl;
return -1;
}
timeval t1, t2;
gettimeofday(&t1, 0);
t2 = t1;
while (true)
{
SphereResult* res = NULL;
if (myproc->read(res) <= 0)
{
if (myproc->checkProgress() < 0)
{
cerr << "all SPEs failed\n";
break;
}
if (myproc->checkProgress() == 100)
break;
}
else
{
delete res;
}
gettimeofday(&t2, 0);
if (t2.tv_sec - t1.tv_sec > 60)
{
cout << "PROGRESS: " << myproc->checkProgress() << "%" << endl;
t1 = t2;
}
}
gettimeofday(&t, 0);
cout << "mission accomplished " << t.tv_sec << endl;
myproc->close();
client.releaseSphereProcess(myproc);
client.logout();
client.close();
return 0;
}
int main(int argc, char** argv)
{
if (1 != argc)
{
cout << "usage: wordcount" << endl;
return 0;
}
Sector client;
Session se;
se.loadInfo("../conf/client.conf");
if (client.init(se.m_ClientConf.m_strMasterIP, se.m_ClientConf.m_iMasterPort) < 0)
return -1;
if (client.login(se.m_ClientConf.m_strUserName, se.m_ClientConf.m_strPassword, se.m_ClientConf.m_strCertificate.c_str()) < 0)
return -1;
vector<string> files;
files.insert(files.end(), "/html");
SphereStream s;
if (s.init(files) < 0)
{
cout << "unable to locate input data files. quit.\n";
return -1;
}
SphereStream temp;
temp.setOutputPath("/wordcount", "word_bucket");
temp.init(256);
SphereProcess* myproc = client.createSphereProcess();
if (myproc->loadOperator("./funcs/wordbucket" SECTOR_DYNLIB_EXT) < 0)
return -1;
timeval t;
gettimeofday(&t, 0);
cout << "start time " << t.tv_sec << endl;
if (myproc->run(s, temp, "wordbucket", 0) < 0)
{
cout << "failed to find any computing resources." << endl;
return -1;
}
timeval t1, t2;
gettimeofday(&t1, 0);
t2 = t1;
while (true)
{
SphereResult* res = NULL;
if (myproc->read(res) <= 0)
{
if (myproc->checkProgress() < 0)
{
cerr << "all SPEs failed\n";
break;
}
if (myproc->checkProgress() == 100)
break;
}
else
{
delete res;
}
gettimeofday(&t2, 0);
if (t2.tv_sec - t1.tv_sec > 60)
{
cout << "PROGRESS: " << myproc->checkProgress() << "%" << endl;
t1 = t2;
}
}
gettimeofday(&t, 0);
cout << "stage 1 accomplished " << t.tv_sec << endl;
/*
//NOT FINISHED. PROCESS EACH BUCKET AND GENERATE INDEX
SphereStream output;
output.init(0);
myproc->setProcNumPerNode(2);
if (myproc->run(temp, output, "index", 0, NULL, 0) < 0)
{
cout << "failed to find any computing resources." << endl;
return -1;
}
gettimeofday(&t1, 0);
t2 = t1;
while (true)
{
SphereResult* res = NULL;
if (-1 == myproc->read(res))
{
if (myproc->checkProgress() < 0)
{
cerr << "all SPEs failed\n";
break;
}
if (myproc->checkProgress() == 100)
break;
}
else
{
delete res;
}
gettimeofday(&t2, 0);
if (t2.tv_sec - t1.tv_sec > 60)
{
cout << "PROGRESS: " << myproc.checkProgress() << "%" << endl;
t1 = t2;
}
}
gettimeofday(&t, 0);
cout << "stage 2 accomplished " << t.tv_sec << endl;
*/
cout << "SPE COMPLETED " << endl;
myproc->close();
client.releaseSphereProcess(myproc);
client.logout();
client.close();
return 0;
}
int main(int argc, char** argv)
{
if (argc != 2)
{
cout << "USAGE: rm <dir>\n";
return -1;
}
Sector client;
Session s;
s.loadInfo("../conf/client.conf");
if (client.init(s.m_ClientConf.m_strMasterIP, s.m_ClientConf.m_iMasterPort) < 0)
return -1;
if (client.login(s.m_ClientConf.m_strUserName, s.m_ClientConf.m_strPassword, s.m_ClientConf.m_strCertificate.c_str()) < 0)
return -1;
string path = argv[1];
bool wc = WildCard::isWildCard(path);
if (!wc)
{
int r = client.remove(path);
if (r == SectorError::E_NOEMPTY)
{
if (isRecursive(path))
client.rmr(path);
}
else if (r < 0)
cout << "ERROR: " << r << " " << SectorError::getErrorMsg(r) << endl;
}
else
{
string orig = path;
size_t p = path.rfind('/');
if (p == string::npos)
path = "/";
else
{
path = path.substr(0, p);
orig = orig.substr(p + 1, orig.length() - p);
}
vector<SNode> filelist;
int r = client.list(path, filelist);
if (r < 0)
cout << "ERROR: " << r << " " << SectorError::getErrorMsg(r) << endl;
bool recursive = false;
vector<string> filtered;
for (vector<SNode>::iterator i = filelist.begin(); i != filelist.end(); ++ i)
{
if (WildCard::match(orig, i->m_strName))
{
if (recursive)
client.rmr(path + "/" + i->m_strName);
else
{
r = client.remove(path + "/" + i->m_strName);
if (r == SectorError::E_NOEMPTY)
{
recursive = isRecursive(path + "/" + i->m_strName);
if (recursive)
client.rmr(path + "/" + i->m_strName);
}
else if (r < 0)
cout << "ERROR: " << r << " " << SectorError::getErrorMsg(r) << endl;
}
}
}
}
client.logout();
client.close();
return 1;
}
int main(int argc, char** argv)
{
if (3 != argc)
{
cout << "usage: upload <src file/dir> <dst dir>" << endl;
return 0;
}
Sector client;
Session s;
s.loadInfo("../conf/client.conf");
if (client.init(s.m_ClientConf.m_strMasterIP, s.m_ClientConf.m_iMasterPort) < 0)
return -1;
if (client.login(s.m_ClientConf.m_strUserName, s.m_ClientConf.m_strPassword, s.m_ClientConf.m_strCertificate.c_str()) < 0)
return -1;
vector<string> fl;
bool wc = WildCard::isWildCard(argv[1]);
if (!wc)
{
struct stat64 st;
if (stat64(argv[1], &st) < 0)
{
cout << "ERROR: source file does not exist.\n";
return -1;
}
getFileList(argv[1], fl);
}
else
{
string path = argv[1];
string orig = path;
size_t p = path.rfind('/');
if (p == string::npos)
path = "/";
else
{
path = path.substr(0, p);
orig = orig.substr(p + 1, orig.length() - p);
}
dirent **namelist;
int n = scandir(path.c_str(), &namelist, 0, alphasort);
if (n < 0)
return -1;
for (int i = 0; i < n; ++ i)
{
// skip "." and ".." and hidden directory
if (namelist[i]->d_name[0] == '.')
{
free(namelist[i]);
continue;
}
if (WildCard::match(orig, namelist[i]->d_name))
getFileList(path + "/" + namelist[i]->d_name, fl);
}
}
string olddir;
for (int i = strlen(argv[1]) - 1; i >= 0; -- i)
{
if (argv[1][i] != '/')
{
olddir = string(argv[1]).substr(0, i);
break;
}
}
size_t p = olddir.rfind('/');
if (p == string::npos)
olddir = "";
else
olddir = olddir.substr(0, p);
string newdir = argv[2];
SNode attr;
int r = client.stat(newdir, attr);
if ((r < 0) || (!attr.m_bIsDir))
{
cout << "destination directory on Sector does not exist.\n";
return -1;
}
for (vector<string>::const_iterator i = fl.begin(); i != fl.end(); ++ i)
{
string dst = *i;
if (olddir.length() > 0)
dst.replace(0, olddir.length(), newdir);
else
dst = newdir + "/" + dst;
struct stat64 s;
if (stat64(i->c_str(), &s) < 0)
continue;
if (S_ISDIR(s.st_mode))
client.mkdir(dst);
else
upload(i->c_str(), dst.c_str(), client);
}
client.logout();
client.close();
return 1;
}
int main(int argc, char** argv)
{
if (1 != argc)
{
cout << "usage: mrsort" << endl;
return 0;
}
Sector client;
Session s;
s.loadInfo("../conf/client.conf");
if (client.init(s.m_ClientConf.m_strMasterIP, s.m_ClientConf.m_iMasterPort) < 0)
return -1;
if (client.login(s.m_ClientConf.m_strUserName, s.m_ClientConf.m_strPassword, s.m_ClientConf.m_strCertificate.c_str()) < 0)
return -1;
vector<string> files;
files.insert(files.end(), "/html");
SphereStream input;
if (input.init(files) < 0)
{
cout << "unable to locate input data files. quit.\n";
return -1;
}
SphereStream output;
output.setOutputPath("/mrword", "inverted_index");
output.init(256);
SphereProcess* myproc = client.createSphereProcess();
if (myproc->loadOperator("./funcs/mr_word.so") < 0)
return -1;
timeval t;
gettimeofday(&t, 0);
cout << "start time " << t.tv_sec << endl;
if (myproc->run_mr(input, output, "mr_word", 0) < 0)
{
cout << "failed to find any computing resources." << endl;
return -1;
}
timeval t1, t2;
gettimeofday(&t1, 0);
t2 = t1;
while (true)
{
SphereResult* res = NULL;
if (myproc->read(res) <= 0)
{
if (myproc->checkMapProgress() <= 0)
{
cerr << "all SPEs failed\n";
break;
}
if (myproc->checkMapProgress() == 100)
break;
}
else
{
delete res;
}
gettimeofday(&t2, 0);
if (t2.tv_sec - t1.tv_sec > 60)
{
cout << "MAP PROGRESS: " << myproc->checkProgress() << "%" << endl;
t1 = t2;
}
}
while (myproc->checkReduceProgress() < 100)
{
usleep(10);
}
gettimeofday(&t, 0);
cout << "mapreduce sort accomplished " << t.tv_sec << endl;
cout << "SPE COMPLETED " << endl;
myproc->close();
client.releaseSphereProcess(myproc);
client.logout();
client.close();
return 0;
}
