int main(int argc, char** argv)
{
if (argc != 2)
{
cerr << "usage: text.idx text_file_name" << endl;
return -1;
}
Sector client;
Session s;
s.loadInfo("../conf/client.conf");
if (client.init(s.m_ClientConf.m_strMasterIP, s.m_ClientConf.m_iMasterPort) < 0)
return -1;
if (client.login(s.m_ClientConf.m_strUserName, s.m_ClientConf.m_strPassword, s.m_ClientConf.m_strCertificate.c_str()) < 0)
return -1;
vector<string> files;
files.insert(files.end(), argv[1]);
SphereStream input;
if (input.init(files) < 0)
{
cout << "unable to locate input data files. quit.\n";
return -1;
}
SphereStream output;
output.init(0);
SphereProcess* myproc = client.createSphereProcess();
if (myproc->loadOperator("./funcs/gen_idx" SECTOR_DYNLIB_EXT) < 0)
return -1;
timeval t;
gettimeofday(&t, 0);
cout << "start time " << t.tv_sec << endl;
if (myproc->run(input, output, "gen_idx", 0) < 0)
{
cout << "failed to find any computing resources." << endl;
return -1;
}
timeval t1, t2;
gettimeofday(&t1, 0);
t2 = t1;
while (true)
{
SphereResult* res = NULL;
if (myproc->read(res) <= 0)
{
if (myproc->checkProgress() < 0)
{
cerr << "all SPEs failed\n";
break;
}
if (myproc->checkProgress() == 100)
break;
}
else
{
delete res;
}
gettimeofday(&t2, 0);
if (t2.tv_sec - t1.tv_sec > 60)
{
cout << "PROGRESS: " << myproc->checkProgress() << "%" << endl;
t1 = t2;
}
}
gettimeofday(&t, 0);
cout << "mission accomplished " << t.tv_sec << endl;
myproc->close();
client.releaseSphereProcess(myproc);
client.logout();
client.close();
return 0;
}
int main(int argc, char** argv)
{
if (1 != argc)
{
cout << "usage: mrsort" << endl;
return 0;
}
Sector client;
Session s;
s.loadInfo("../conf/client.conf");
if (client.init(s.m_ClientConf.m_strMasterIP, s.m_ClientConf.m_iMasterPort) < 0)
return -1;
if (client.login(s.m_ClientConf.m_strUserName, s.m_ClientConf.m_strPassword, s.m_ClientConf.m_strCertificate.c_str()) < 0)
return -1;
vector<string> files;
files.insert(files.end(), "/html");
SphereStream input;
if (input.init(files) < 0)
{
cout << "unable to locate input data files. quit.\n";
return -1;
}
SphereStream output;
output.setOutputPath("/mrword", "inverted_index");
output.init(256);
SphereProcess* myproc = client.createSphereProcess();
if (myproc->loadOperator("./funcs/mr_word.so") < 0)
return -1;
timeval t;
gettimeofday(&t, 0);
cout << "start time " << t.tv_sec << endl;
if (myproc->run_mr(input, output, "mr_word", 0) < 0)
{
cout << "failed to find any computing resources." << endl;
return -1;
}
timeval t1, t2;
gettimeofday(&t1, 0);
t2 = t1;
while (true)
{
SphereResult* res = NULL;
if (myproc->read(res) <= 0)
{
if (myproc->checkMapProgress() <= 0)
{
cerr << "all SPEs failed\n";
break;
}
if (myproc->checkMapProgress() == 100)
break;
}
else
{
delete res;
}
gettimeofday(&t2, 0);
if (t2.tv_sec - t1.tv_sec > 60)
{
cout << "MAP PROGRESS: " << myproc->checkProgress() << "%" << endl;
t1 = t2;
}
}
while (myproc->checkReduceProgress() < 100)
{
usleep(10);
}
gettimeofday(&t, 0);
cout << "mapreduce sort accomplished " << t.tv_sec << endl;
cout << "SPE COMPLETED " << endl;
myproc->close();
client.releaseSphereProcess(myproc);
client.logout();
client.close();
return 0;
}
int main(int argc, char** argv)
{
if (1 != argc)
{
cout << "usage: wordcount" << endl;
return 0;
}
Sector client;
Session se;
se.loadInfo("../conf/client.conf");
if (client.init(se.m_ClientConf.m_strMasterIP, se.m_ClientConf.m_iMasterPort) < 0)
return -1;
if (client.login(se.m_ClientConf.m_strUserName, se.m_ClientConf.m_strPassword, se.m_ClientConf.m_strCertificate.c_str()) < 0)
return -1;
vector<string> files;
files.insert(files.end(), "/html");
SphereStream s;
if (s.init(files) < 0)
{
cout << "unable to locate input data files. quit.\n";
return -1;
}
SphereStream temp;
temp.setOutputPath("/wordcount", "word_bucket");
temp.init(256);
SphereProcess* myproc = client.createSphereProcess();
if (myproc->loadOperator("./funcs/wordbucket" SECTOR_DYNLIB_EXT) < 0)
return -1;
timeval t;
gettimeofday(&t, 0);
cout << "start time " << t.tv_sec << endl;
if (myproc->run(s, temp, "wordbucket", 0) < 0)
{
cout << "failed to find any computing resources." << endl;
return -1;
}
timeval t1, t2;
gettimeofday(&t1, 0);
t2 = t1;
while (true)
{
SphereResult* res = NULL;
if (myproc->read(res) <= 0)
{
if (myproc->checkProgress() < 0)
{
cerr << "all SPEs failed\n";
break;
}
if (myproc->checkProgress() == 100)
break;
}
else
{
delete res;
}
gettimeofday(&t2, 0);
if (t2.tv_sec - t1.tv_sec > 60)
{
cout << "PROGRESS: " << myproc->checkProgress() << "%" << endl;
t1 = t2;
}
}
gettimeofday(&t, 0);
cout << "stage 1 accomplished " << t.tv_sec << endl;
/*
//NOT FINISHED. PROCESS EACH BUCKET AND GENERATE INDEX
SphereStream output;
output.init(0);
myproc->setProcNumPerNode(2);
if (myproc->run(temp, output, "index", 0, NULL, 0) < 0)
{
cout << "failed to find any computing resources." << endl;
return -1;
}
gettimeofday(&t1, 0);
t2 = t1;
while (true)
{
SphereResult* res = NULL;
if (-1 == myproc->read(res))
{
if (myproc->checkProgress() < 0)
{
cerr << "all SPEs failed\n";
break;
}
if (myproc->checkProgress() == 100)
break;
}
else
{
delete res;
}
gettimeofday(&t2, 0);
if (t2.tv_sec - t1.tv_sec > 60)
{
cout << "PROGRESS: " << myproc.checkProgress() << "%" << endl;
t1 = t2;
}
}
gettimeofday(&t, 0);
cout << "stage 2 accomplished " << t.tv_sec << endl;
*/
cout << "SPE COMPLETED " << endl;
myproc->close();
client.releaseSphereProcess(myproc);
client.logout();
client.close();
return 0;
}
int main(int argc, char** argv)
{
if (1 != argc)
{
cout << "usage: wordcount" << endl;
return 0;
}
Sector client;
if (Utility::login(client) < 0)
return -1;
vector<string> files;
files.insert(files.end(), "/html");
SphereStream s;
if (s.init(files) < 0)
{
cout << "unable to locate input data files. quit.\n";
Utility::logout(client);
return -1;
}
SphereStream temp;
temp.setOutputPath("/wordcount", "word_bucket");
temp.init(256);
SphereProcess* myproc = client.createSphereProcess();
if (myproc->loadOperator("./funcs/wordbucket" SECTOR_DYNLIB_EXT) < 0)
{
cout << "cannot find workbucket.so\n";
Utility::logout(client);
return -1;
}
timeval t;
gettimeofday(&t, 0);
cout << "start time " << t.tv_sec << endl;
int result = myproc->run(s, temp, "wordbucket", 0);
if (result < 0)
{
Utility::print_error(result);
Utility::logout(client);
return -1;
}
myproc->waitForCompletion();
gettimeofday(&t, 0);
cout << "stage 1 accomplished " << t.tv_sec << endl;
for (vector<string>::iterator i = temp.m_vFiles.begin(); i != temp.m_vFiles.end(); ++ i)
cout << *i << endl;
for (vector<int64_t>::iterator i = temp.m_vSize.begin(); i != temp.m_vSize.end(); ++ i)
cout << *i << endl;
/*
//NOT FINISHED. PROCESS EACH BUCKET AND GENERATE INDEX
SphereStream output;
output.init(0);
myproc->setProcNumPerNode(2);
if (myproc->run(temp, output, "index", 0, NULL, 0) < 0)
{
cout << "failed to find any computing resources." << endl;
return -1;
}
myproc->waitForCompletion();
gettimeofday(&t, 0);
cout << "stage 2 accomplished " << t.tv_sec << endl;
*/
cout << "SPE COMPLETED " << endl;
myproc->close();
client.releaseSphereProcess(myproc);
Utility::logout(client);
return 0;
}
