void *parserThread(void *argv)
{
string tmpstr;
int ret;
while(!IsExit)
{
jsonsemc.wait();
if(IsExit)
break;
tmpstr=Qjson.pop();
jsonsemp.post();
statelog.write("JsonGrab parser %s Begin\n", tmpstr.c_str());
statelog.flush();
ret=jsongrabmysql(tmpstr.c_str());
if(ret == 1)
{
statelog.write("JsonGrab parser error %s\n", tmpstr.c_str());
statelog.flush();
}
else
{
statelog.write("JsonGrab parser successful %s\n", tmpstr.c_str());
statelog.flush();
}
}
}
void sig_usr2(int signo)
{
weibolog.flush();
statelog.flush();
queueinfolog.write("JsonQueue size:\t%d\n", Qjson.size());
queueinfolog.write("WeiboQueue size:\t %d\n", Qweibo.size());
queueinfolog.flush();
}
void flushWeibo()
{
int i;
try
{
hashclient.hashBlogId(outmblogid, inmblogid);
}
catch(...)
{
statelog.write("Hash RPC Error\n");
sig_usr(-1);
}
sort(outmblogid.begin(), outmblogid.end());
for(i=0; i<inmblogid.size(); i++)
{
if(minfohash[i].lastitem)
{
delfnames.push_back(minfohash[i].filename);
}
if(binary_search(outmblogid.begin(), outmblogid.end(), inmblogid[i]))
{
weibolog.write("%ld %d %d\n", minfohash[i].basicinfo_uid, minfohash[i].basicinfo_rtnum, minfohash[i].basicinfo_cmtnum);
storemblog(minfohash[i], mhash[i]);
}
}
outmblogid.clear();
inmblogid.clear();
for(i=0; i<delfnames.size(); i++)
{
unlink(delfnames[i].c_str());
statelog.write("Parser file %s successful\n", delfnames[i].c_str());
statelog.flush();
}
delfnames.clear();
weibolog.flush();
}
void SendJsonFileName(const std::vector<std::string> & filename) {
int i;
for(i=0; i<filename.size(); i++)
{
jsonsemp.wait();
jsonmutex.lock();
Qjson.push(filename[i]);
jsonmutex.unlock();
jsonsemc.post();
statelog.write("Client Send Jsonfile %s\n", filename[i].c_str());
statelog.flush();
}
}
void * insertThread(void *argv)
{
int i;
while(!IsExit)
{
for(i=0; i < HASHSENDTOTAL;i++)
{
weibosemc.wait();
if(IsExit)
break;
minfohash[i]=Qweiboinfo.pop();
mhash[i]=Qweibo.pop();
weibosemp.post();
inmblogid.push_back(minfohash[i].mblogid);
}
statelog.write("Flush Weibo begin\n");
statelog.flush();
flushWeibo();
statelog.write("Flush Weibo over\n");
statelog.flush();
}
}
int initVal()
{
string tmpstr;
IsExit=0;
signal(SIGUSR1, sig_usr);
signal(SIGUSR2, sig_usr2);
if(weibolog.initFile(WEIBOLOGDIR, WEIBOLOGPREFIX))
{
printf("weibolog init error\n");
return -1;
}
if(statelog.initFile(STATELOGDIR, STATELOGPREFIX))
{
printf("statelog init error\n");
return -1;
}
statelog.write("ParserServer start\n");
statelog.flush();
if(queueinfolog.initFile(STATELOGDIR, QUEUEINFOLOGPREFIX))
{
printf("queueinfolog init error\n");
return -1;
}
hbasetransport->open();
hashtransport->open();
mysql_init(&mysql);
if(!mysql_real_connect(&mysql, "localhost", "chenmin", "chenmin", "sina", 3306, NULL, 0))
{
printf("mysql connect error\n");
return -1;
}
return 0;
}
int storemblog(weiboinfo& minfo, weibo &mblog)
{
vector<Mutation>v1, v2;
Mutation muta;
char tmpstr[32];
char mysqlstr[512];
muta.isDelete=0;
muta.column="basicinfo:uid";
sprintf(tmpstr, "%ld", minfo.basicinfo_uid);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="basicinfo:time";
sprintf(tmpstr, "%ld", minfo.basicinfo_time);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="basicinfo:source";
sprintf(tmpstr, "%ld", minfo.basicinfo_source);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="basicinfo:rtnum";
sprintf(tmpstr, "%d", minfo.basicinfo_rtnum);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="basicinfo:cmtnum";
sprintf(tmpstr, "%d", minfo.basicinfo_cmtnum);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="basicinfo:topic";
muta.value=minfo.basicinfo_topic;
v1.push_back(muta);
muta.column="atusers";
muta.value=minfo.atusers;
v1.push_back(muta);
if(minfo.rtinfo_rootid.size() != 0)
{
muta.column="rtinfo:rootuid";
sprintf(tmpstr, "%ld", minfo.rtinfo_rootuid);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="rtinfo:fromuid";
sprintf(tmpstr, "%ld", minfo.rtinfo_rootuid);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="rtinfo:fromid";
muta.value=minfo.rtinfo_fromid;
v1.push_back(muta);
muta.column="rtinfo:rootid";
muta.value=minfo.rtinfo_rootid;
v1.push_back(muta);
muta.column="rtinfo:rootrtnum";
sprintf(tmpstr, "%d", minfo.rtinfo_rootrtnum);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="rtinfo:rtreason";
muta.value=minfo.rtinfo_rtreason;
v1.push_back(muta);
}
try
{
hbaseclient.mutateRow("weiboinfo", minfo.mblogid, v1);
}
catch(IOError &e)
{
statelog.write("Hbase RPC Error insert table weiboinfo %s\n", e.what());
sig_usr(-1);
}
muta.column="content";
muta.value=mblog.content;
v2.push_back(muta);
try
{
hbaseclient.mutateRow("weibo", minfo.mblogid, v2);
}
catch(IOError &e)
{
statelog.write("Hbase RPC Error insert table weibo %s\n", e.what());
sig_usr(-1);
}
sprintf(mysqlstr, "insert into weiboNum values(\"%s\", %d, %d, %d, \"%ld\", \"%s\");", minfo.mblogid.c_str(), minfo.basicinfo_rtnum, minfo.basicinfo_cmtnum, minfo.rtinfo_rootrtnum, minfo.basicinfo_time, minfo.basicinfo_topic.c_str());
if(mysql_query(&mysql, mysqlstr))
{
statelog.write("MySQL insert table weiboNum Error %s\n", mysqlstr);
statelog.flush();
}
sprintf(mysqlstr, "insert into weiboUser values(\"%s\", \"%ld\",\"%s\",\"%ld\", \"%s\", \"%ld\", \"%s\");",minfo.mblogid.c_str(), minfo.basicinfo_uid, minfo.rtinfo_fromid.c_str(),minfo.rtinfo_fromuid, minfo.rtinfo_rootid.c_str(), minfo.rtinfo_rootuid, minfo.atusers.c_str());
if(mysql_query(&mysql, mysqlstr))
{
statelog.write("MySQL insert table weiboUser Error %s\n", mysqlstr);
statelog.flush();
}
}
int main()
{
// Initialize the winsock library
MyTcpSocket::initialize();
// get client's information (assume neither the name nor the address is given)
winLog << endl;
winLog << "Retrieve the localHost [CLIENT] name and address:" << endl;
winLog.flush();
//以下是客户端程序，所有服务器IP从配置文件中读取
//*********************************************************************************
// 【①】 客户端HostInfo(hostent)
MyHostInfo clientInfo;
string clientName = clientInfo.getHostName();
string clientIPAddress = clientInfo.getHostIPAddress();
cout << "Name: " << clientName << endl;
cout << "Address: " << clientIPAddress << endl;
winLog << " ==> Name: " << clientName << endl;
winLog << " ==> Address: " << clientIPAddress << endl;
//*********************************************************************************
// get server's IP address and name 【服务器IP，从配置文件读取】
string serverIPAddress = "";
readServerConfig(serverIPAddress);
winLog << endl;
winLog << "Retrieve the remoteHost [SERVER] name and address:" << endl;
winLog << " ==> the given address is " << serverIPAddress << endl;
// 【②】 服务器端HostInfo(hostent)
MyHostInfo serverInfo(serverIPAddress,ADDRESS);
string serverName = serverInfo.getHostName();
cout << "Name: " << serverName << endl;
cout << "Address: " << serverIPAddress << endl;
winLog << " ==> Name: " << serverName << endl;
winLog << " ==> Address: " << serverIPAddress << endl;
//*********************************************************************************
// create the socket for client
MyTcpSocket myClient(PORTNUM);
cout << myClient;
winLog << "client configuation: " << endl;
winLog << myClient;
//*********************************************************************************
// connect to the server.
cout << "connecting to the server [" << serverName << "] ... " << endl;
winLog << "connecting to the server [" << serverName << "] ... " << endl;
myClient.connectToServer(serverIPAddress,ADDRESS);
int recvBytes = 0;
while (1)
{
// send message to server
char messageToServer[MAX_MSG_LEN+1];
memset(messageToServer,0,sizeof(messageToServer));
cout << "[SEND] ";
cin.getline(messageToServer,MAX_MSG_LEN);
winLog << "[SEND] " << messageToServer << endl;
myClient.sendMessage(string(messageToServer));
if ( !string(messageToServer).compare("Quit") || !string(messageToServer).compare("quit") ) break;
// receive message from server
string messageFromServer = "";
recvBytes = myClient.recieveMessage(messageFromServer);
if ( recvBytes == -99 ) break;
cout << "[RECV:" << serverName << "]: " << messageFromServer << endl;
winLog << "[RECV:" << serverName << "]: " << messageFromServer << endl;
}
return EXIT_SUCCESS;
}