int benchmarkInsert() {
double start = 0;
double end = 0;
start = TimeUtil::getTime_msec();
int errCount = 0;
int c = 0;
vector<string>::iterator it;
for (it = pkgList.begin(); it != pkgList.end(); it++) {
c++;
string pkg_str = *it;
Package pkg;
pkg.ParseFromString(pkg_str);
int ret = zc.insert(pkg.virtualpath(), pkg_str);
if (ret < 0) {
errCount++;
}
}
end = TimeUtil::getTime_msec();
char buf[200];
sprintf(buf, "Inserted packages, %d, %d, cost(ms), %f", numOfOps - errCount,
numOfOps, end - start);
cout << buf << endl;
return 0;
}
int c_zht_insert_std(ZHTClient_c zhtClient, const char *pair) {
ZHTClient * zhtcppClient = (ZHTClient *) zhtClient;
string str(pair);
return zhtcppClient->insert(str);
}
int zht_insert_meta(ZHTClient_c zhtClient, struct metadata * meta){
ZHTClient * zhtcppClient = (ZHTClient *) zhtClient;
Package package;
string keyStr(meta->key);
//1. General infos on the file/storage
package.set_virtualpath(keyStr); //as key
package.set_isdir(false);
package.set_replicano(1);
package.set_operation(3); //1 for look up, 2 for remove, 3 for insert
package.set_realfullpath(keyStr);
//2. General infos on the encoding
package.set_k(meta->k);
package.set_m(meta->m);
package.set_encodinglib(meta->encodingLib);
package.set_filesize(meta->fileSize);
package.set_bufsize(meta->bufsize);
//3. Chunks/Replicas locations
struct comTransfer * current = meta->loc->transfers;
Package_Location* location;
while(current != NULL){
location = package.add_location();
location->set_hostname(string(current->hostName));
location->set_port(current->port);
location->set_distantchunkname(string(current->distantChunkName));
current=current->next;
}
//4. Insertion
std::string serialized = package.SerializeAsString();
dbgprintf("Package Length:%i\n",serialized.size());
//TEST
Package package4;
package4.ParseFromString(serialized);
dbgprintf("Serialized: %s \n",serialized.c_str());
dbgprintf("key: %s \n",(package4.virtualpath()).c_str());
dbgprintf("k:%i,m:%i,encodinglib:%i,filesize:%i \n",package4.k(),package4.m(),package4.encodinglib(),package4.filesize());
for (int j = 0; j < package4.location_size(); j++) {
const Package_Location& location = package4.location(j);
dbgprintf("chunk:%s, port:%i\n",location.distantchunkname().c_str(), location.port());
}
//END TEST
int res = zhtcppClient->insert(serialized);
return res;
}
int benchmarkInsert(string cfgFile, string memberList, vector<string> &pkgList,
ZHTClient &clientRet, int numTest, int lenString) {
// if (clientRet.initialize(cfgFile, memberList) != 0) { //zhouxb
if (clientRet.initialize(cfgFile, memberList, TCP) != 0) {
cout << "Crap! ZHTClient initialization failed, program exits." << endl;
return -1;
}
for (int i = 0; i < numTest; i++) {
Package package, package_ret;
package.set_virtualpath(randomString(lenString)); //as key
package.set_isdir(true);
package.set_replicano(5); //orginal--Note: never let it be nagative!!!
package.set_operation(3); // 3 for insert, 1 for look up, 2 for remove
package.set_realfullpath(
"Some-Real-longer-longer-and-longer-Paths--------");
package.add_listitem("item-----2");
package.add_listitem("item-----3");
package.add_listitem("item-----4");
package.add_listitem("item-----5");
package.add_listitem("item-----6");
string str = package.SerializeAsString();
// cout << "package size = " << str.size() << endl;
// cout<<"Client.cpp:benchmarkInsert: "<<endl;
// cout<<"string: "<<str<<endl;
// cout<<"Insert str: "<<str.c_str()<<endl;
// cout<<"data(): "<< str.data()<<endl;
pkgList.push_back(str);
}
double start = 0;
double end = 0;
start = getTime_msec();
int errCount = 0;
vector<string>::iterator it;
int c = 0;
// cout << "-----2" << endl;
for (it = pkgList.begin(); it != pkgList.end(); it++) {
// cerr <<"insert count "<< c << endl;
c++;
string str_ins = *it;
//cout << "-----1" << endl;
int ret = clientRet.insert(str_ins);
//cout << "-----2" << endl;
if (ret < 0) {
errCount++;
}
}
//close(sock);
end = getTime_msec();
cout << "Inserted " << numTest - errCount << " packages out of " << numTest
<< ", cost " << end - start << " ms" << endl;
return 0;
}
int c_zht_insert_std(ZHTClient_c zhtClient, const char *key,
const char *value) {
#ifdef IND_MUTEX
LockGuard lock(&c_zht_insert_mutex);
#elif SHARED_MUTEX
LockGuard lock(&c_zht_client_mutex);
#else
#endif
ZHTClient * zhtcppClient = (ZHTClient *) zhtClient;
string skey(key);
string sval(value);
return zhtcppClient->insert(skey, sval);
}
int c_zht_insert2_std(ZHTClient_c zhtClient, const char *key,
const char *value) {
ZHTClient * zhtcppClient = (ZHTClient *) zhtClient;
string sKey(key);
string sValue(value);
Package package;
package.set_virtualpath(sKey); //as key
package.set_isdir(true);
package.set_replicano(5);
package.set_operation(3); //1 for look up, 2 for remove, 3 for insert, 4 append
if (!sValue.empty())
package.set_realfullpath(sValue);
return zhtcppClient->insert(package.SerializeAsString());
}
void test_insert()
{
string key = "response";
string val = "y";
while(val != "q")
{
printf("Enter the value to be inserted,(Enter \"q\" Quit) Value = ");
cin>>val;
int rc = zc.insert(key, val);
if (rc == 0)
printf("INSERT OK, rc(%d)\n", rc);
else
printf("INSERT ERR, rc(%d)\n", rc);
}
}
int c_zht_insert2(const char *key, const char *value) {
string keyStr(key);
string valueStr(value);
if (keyStr.empty()) //empty key not allowed.
return -1;
Package package;
package.set_virtualpath(keyStr); //as key
package.set_isdir(true);
package.set_replicano(5);
package.set_operation(3); //1 for look up, 2 for remove, 3 for insert
if (!valueStr.empty())
package.set_realfullpath(valueStr);
return zhtClient.insert(package.SerializeAsString());
}
int benmarkTimeAnalize(string cfgFile, string memberList,
vector<string> &pkgList, ZHTClient &clientRet, int numTest,
int lenString, string Recordpath) {
ZHTClient client;
// if (client.initialize(cfgFile, memberList) != 0) {
if (client.initialize(cfgFile, memberList, TCP) != 0) {
cout << "Crap! ZHTClient initialization failed, program exits." << endl;
return -1;
}
double timeRecord[numTest]; //={0};
// bzero(timeRecord, sizeof(timeRecord));
clientRet = client; //reserve this client object for other benchmark(lookup/remove) to use.
//vector<string> pkgList;
int i = 0;
for (i = 0; i < numTest; i++) {
Package package, package_ret;
package.set_virtualpath(randomString(lenString)); //as key
package.set_isdir(true);
package.set_replicano(5); //orginal--Note: never let it be nagative!!!
package.set_operation(3); // 3 for insert, 1 for look up, 2 for remove
package.set_realfullpath(
"Some-Real-longer-longer-and-longer-Paths--------");
package.add_listitem("item-----1");
package.add_listitem("item-----2");
package.add_listitem("item-----3");
package.add_listitem("item-----4");
package.add_listitem("item-----5");
string str = package.SerializeAsString();
// cout << "package size = " << str.size() << endl;
// cout<<"Client.cpp:benchmarkInsert: "<<endl;
// cout<<"string: "<<str<<endl;
// cout<<"Insert str: "<<str.c_str()<<endl;
// cout<<"data(): "<< str.data()<<endl;
pkgList.push_back(str);
}
double start = 0;
double end = 0;
double istart = 0;
double iend = 0;
int errCount = 0;
vector<string>::iterator it;
int c = 0;
ofstream record;
record.open(Recordpath.c_str());
start = getTime_msec();
for (it = pkgList.begin(); it != pkgList.end(); it++) {
// cout<<c<<endl;
c++;
double interval = 0;
istart = getTime_usec();
int op_ret = client.insert((*it));
iend = getTime_usec();
if (op_ret < 0) {
errCount++;
interval = -1;
} else
interval = iend - istart;
record << interval << endl;
timeRecord[c] = interval;
}
end = getTime_msec();
record.close();
cout << "Inserted " << numTest - errCount << " packages out of " << numTest
<< ", cost " << end - start << " ms" << endl;
return 0;
}
/*
* Serializes the metadata of the file and then inserts it in ZHT
*/
int insertMetadata(string cfgFile, string memberList, vector<string> &pkgList,
ZHTClient &clientRet, int numTest, int lenString, string localPath, int codingId, int k, int m, int bufsize) {
if (clientRet.initialize(cfgFile, memberList) != 0) {
cout << "Crap! ZHTClient initialization failed, program exits." << endl;
return -1;
}
// Define the package for the file, the chunk ids and more
Package package, package_ret;
package.set_virtualpath(randomString(lenString)); // as key TODO
package.set_isdir(true);
package.set_replicano(5); // original--Note: never let it be negative!!!
package.set_operation(3); // 3 for insert, 1 for look up, 2 for remove
package.set_realfullpath(localPath);
// Assign the chunk ids to the metadata
// TODO: Each insertion for just one chunk or all of them?
package.set_ecChunkIds(chunkId);
package.set_ecCoding(codingId);
package.set_ecK(k);
package.set_ecM(m);
package.set_ecBufSize(bufsize);
string str = package.SerializeAsString();
// cout << "package size = " << str.size() << endl;
// cout<<"Client.cpp:insertMetadata: "<<endl;
// cout<<"string: "<<str<<endl;
// cout<<"Insert str: "<<str.c_str()<<endl;
// cout<<"data(): "<< str.data()<<endl;
pkgList.push_back(str);
//clientRet = client; //reserve this client object for other benchmark(lookup/remove) to use.
//vector<string> pkgList;
/*
int i = 0;
for (i = 0; i < numTest; i++) {
Package package, package_ret;
package.set_virtualpath(randomString(lenString)); //as key
package.set_isdir(true);
package.set_replicano(5); //orginal--Note: never let it be nagative!!!
package.set_operation(3); // 3 for insert, 1 for look up, 2 for remove
package.set_realfullpath(
"Some-Real-longer-longer-and-longer-Paths--------");
package.add_listitem("item-----2");
package.add_listitem("item-----3");
package.add_listitem("item-----4");
package.add_listitem("item-----5");
package.add_listitem("item-----6");
string str = package.SerializeAsString();
//cout << "package size = " << str.size() << endl;
//cout<<"Client.cpp:insertMetadata: "<<endl;
//cout<<"string: "<<str<<endl;
//cout<<"Insert str: "<<str.c_str()<<endl;
//cout<<"data(): "<< str.data()<<endl;
pkgList.push_back(str);
}
*/
double start = 0;
double end = 0;
start = getTime_msec();
int errCount = 0;
vector<string>::iterator it;
int c = 0;
//cout << "-----2" << endl;
//string sampleString = *(pkgList.begin());
//struct HostEntity aHost = client.str2Host(sampleString);
/*
int sock = makeClientSocket("localhost", 50000, 1);
cout<<"client sock = "<< sock<<endl;
reuseSock(sock);
*/
for (it = pkgList.begin(); it != pkgList.end(); it++) {
//cout <<"insert count "<< c << endl;
c++;
string str_ins = *it;
//cout << "-----1" << endl;
int ret = clientRet.insert(str_ins);
//cout << "-----2" << endl;
if (ret < 0) {
errCount++;
}
}
//close(sock);
end = getTime_msec();
cout << "Inserted " << numTest - errCount << " packages out of " << numTest
<< ", cost " << end - start << " ms" << endl;
return 0;
}
int c_zht_insert(const char *pair) {
string str(pair);
return zhtClient.insert(str);
}
void *execTask(void *popTask)
{
string result, update, task, key, cmd, dataTask[4] = "";
int i, k = 0, rc, data;
char intstr[10];
task = (char *) popTask;
for(i = 0; i < strlen(task.c_str()); i++)
{
if (task[i] == ',')
k = k + 1;
else
dataTask[k] = dataTask[k] + task[i];
//key = key + task[i];
}
key = dataTask[0];
cmd = dataTask[1] + " " + dataTask[2];
//rc = system("netstat | grep 50000 | wc -l");
rc = system(cmd.c_str());
pthread_mutex_lock( &mutex1 );
if ( rc == 0)
{
rc = zc.lookup(key, result);
cout << "Lookup Result: " << (atoi(result.c_str()) - 1000) << endl;
if ( rc == 0)
{
data = atoi(result.c_str());
data = data + 1;
sprintf(intstr, "%d", data);
update = string(intstr);
rc = zc.insert(key, update);
cout << "Updating the Key with the Value: " << (atoi(update.c_str()) - 1000) << endl;
if (rc == 0)
printf("INSERT OK, rc(%d)\n", rc);
else
printf("INSERT ERR, rc(%d)\n", rc);
}
else
{
rc = zc.insert(key, "1001");
}
}
else
{
cout << "********************************************************************" << endl;
cout << "Job Failed execution: " << task << endl;
cout << "Pushing the failed job back to queue" << endl;
rc = zc.push((key + ".x" +dataTask[3]), task, "q1", result);
if (rc == 0)
printf("PUSH OK, rc(%d)\n", rc);
else
printf("PUSH ERR, rc(%d)\n", rc);
}
threadCount = threadCount - 1;
pthread_mutex_unlock( &mutex1 );
}
