void *parserThread(void *argv)
{
string tmpstr;
int ret;
while(!IsExit)
{
jsonsemc.wait();
if(IsExit)
break;
tmpstr=Qjson.pop();
jsonsemp.post();
statelog.write("JsonGrab parser %s Begin\n", tmpstr.c_str());
statelog.flush();
ret=jsongrabmysql(tmpstr.c_str());
if(ret == 1)
{
statelog.write("JsonGrab parser error %s\n", tmpstr.c_str());
statelog.flush();
}
else
{
statelog.write("JsonGrab parser successful %s\n", tmpstr.c_str());
statelog.flush();
}
}
}
void sig_usr2(int signo)
{
weibolog.flush();
statelog.flush();
queueinfolog.write("JsonQueue size:\t%d\n", Qjson.size());
queueinfolog.write("WeiboQueue size:\t %d\n", Qweibo.size());
queueinfolog.flush();
}
void deinitVal(int fail)
{
hashtransport->close();
hbasetransport->close();
mysql_close(&mysql);
weibolog.close();
if(!fail)
{
statelog.write("ParserServer stop\n");
}
statelog.close();
queueinfolog.close();
}
void flushWeibo()
{
int i;
try
{
hashclient.hashBlogId(outmblogid, inmblogid);
}
catch(...)
{
statelog.write("Hash RPC Error\n");
sig_usr(-1);
}
sort(outmblogid.begin(), outmblogid.end());
for(i=0; i<inmblogid.size(); i++)
{
if(minfohash[i].lastitem)
{
delfnames.push_back(minfohash[i].filename);
}
if(binary_search(outmblogid.begin(), outmblogid.end(), inmblogid[i]))
{
weibolog.write("%ld %d %d\n", minfohash[i].basicinfo_uid, minfohash[i].basicinfo_rtnum, minfohash[i].basicinfo_cmtnum);
storemblog(minfohash[i], mhash[i]);
}
}
outmblogid.clear();
inmblogid.clear();
for(i=0; i<delfnames.size(); i++)
{
unlink(delfnames[i].c_str());
statelog.write("Parser file %s successful\n", delfnames[i].c_str());
statelog.flush();
}
delfnames.clear();
weibolog.flush();
}
void SendJsonFileName(const std::vector<std::string> & filename) {
int i;
for(i=0; i<filename.size(); i++)
{
jsonsemp.wait();
jsonmutex.lock();
Qjson.push(filename[i]);
jsonmutex.unlock();
jsonsemc.post();
statelog.write("Client Send Jsonfile %s\n", filename[i].c_str());
statelog.flush();
}
}
void * insertThread(void *argv)
{
int i;
while(!IsExit)
{
for(i=0; i < HASHSENDTOTAL;i++)
{
weibosemc.wait();
if(IsExit)
break;
minfohash[i]=Qweiboinfo.pop();
mhash[i]=Qweibo.pop();
weibosemp.post();
inmblogid.push_back(minfohash[i].mblogid);
}
statelog.write("Flush Weibo begin\n");
statelog.flush();
flushWeibo();
statelog.write("Flush Weibo over\n");
statelog.flush();
}
}
int initVal()
{
string tmpstr;
IsExit=0;
signal(SIGUSR1, sig_usr);
signal(SIGUSR2, sig_usr2);
if(weibolog.initFile(WEIBOLOGDIR, WEIBOLOGPREFIX))
{
printf("weibolog init error\n");
return -1;
}
if(statelog.initFile(STATELOGDIR, STATELOGPREFIX))
{
printf("statelog init error\n");
return -1;
}
statelog.write("ParserServer start\n");
statelog.flush();
if(queueinfolog.initFile(STATELOGDIR, QUEUEINFOLOGPREFIX))
{
printf("queueinfolog init error\n");
return -1;
}
hbasetransport->open();
hashtransport->open();
mysql_init(&mysql);
if(!mysql_real_connect(&mysql, "localhost", "chenmin", "chenmin", "sina", 3306, NULL, 0))
{
printf("mysql connect error\n");
return -1;
}
return 0;
}
int storemblog(weiboinfo& minfo, weibo &mblog)
{
vector<Mutation>v1, v2;
Mutation muta;
char tmpstr[32];
char mysqlstr[512];
muta.isDelete=0;
muta.column="basicinfo:uid";
sprintf(tmpstr, "%ld", minfo.basicinfo_uid);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="basicinfo:time";
sprintf(tmpstr, "%ld", minfo.basicinfo_time);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="basicinfo:source";
sprintf(tmpstr, "%ld", minfo.basicinfo_source);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="basicinfo:rtnum";
sprintf(tmpstr, "%d", minfo.basicinfo_rtnum);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="basicinfo:cmtnum";
sprintf(tmpstr, "%d", minfo.basicinfo_cmtnum);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="basicinfo:topic";
muta.value=minfo.basicinfo_topic;
v1.push_back(muta);
muta.column="atusers";
muta.value=minfo.atusers;
v1.push_back(muta);
if(minfo.rtinfo_rootid.size() != 0)
{
muta.column="rtinfo:rootuid";
sprintf(tmpstr, "%ld", minfo.rtinfo_rootuid);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="rtinfo:fromuid";
sprintf(tmpstr, "%ld", minfo.rtinfo_rootuid);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="rtinfo:fromid";
muta.value=minfo.rtinfo_fromid;
v1.push_back(muta);
muta.column="rtinfo:rootid";
muta.value=minfo.rtinfo_rootid;
v1.push_back(muta);
muta.column="rtinfo:rootrtnum";
sprintf(tmpstr, "%d", minfo.rtinfo_rootrtnum);
muta.value=tmpstr;
v1.push_back(muta);
muta.column="rtinfo:rtreason";
muta.value=minfo.rtinfo_rtreason;
v1.push_back(muta);
}
try
{
hbaseclient.mutateRow("weiboinfo", minfo.mblogid, v1);
}
catch(IOError &e)
{
statelog.write("Hbase RPC Error insert table weiboinfo %s\n", e.what());
sig_usr(-1);
}
muta.column="content";
muta.value=mblog.content;
v2.push_back(muta);
try
{
hbaseclient.mutateRow("weibo", minfo.mblogid, v2);
}
catch(IOError &e)
{
statelog.write("Hbase RPC Error insert table weibo %s\n", e.what());
sig_usr(-1);
}
sprintf(mysqlstr, "insert into weiboNum values(\"%s\", %d, %d, %d, \"%ld\", \"%s\");", minfo.mblogid.c_str(), minfo.basicinfo_rtnum, minfo.basicinfo_cmtnum, minfo.rtinfo_rootrtnum, minfo.basicinfo_time, minfo.basicinfo_topic.c_str());
if(mysql_query(&mysql, mysqlstr))
{
statelog.write("MySQL insert table weiboNum Error %s\n", mysqlstr);
statelog.flush();
}
sprintf(mysqlstr, "insert into weiboUser values(\"%s\", \"%ld\",\"%s\",\"%ld\", \"%s\", \"%ld\", \"%s\");",minfo.mblogid.c_str(), minfo.basicinfo_uid, minfo.rtinfo_fromid.c_str(),minfo.rtinfo_fromuid, minfo.rtinfo_rootid.c_str(), minfo.rtinfo_rootuid, minfo.atusers.c_str());
if(mysql_query(&mysql, mysqlstr))
{
statelog.write("MySQL insert table weiboUser Error %s\n", mysqlstr);
statelog.flush();
}
}