int main(int argc, char** argv) {
shared_ptr<TTransport> socket(new TSocket("localhost", 9090));
shared_ptr<TTransport> transport(new TBufferedTransport(socket));
shared_ptr<TProtocol> protocol(new TBinaryProtocol(transport));
SpiderWebServiceClient client(protocol);
try {
transport->open();
load("urls.txt", client);
HttpRequest rqst;
rqst.__set_url("http://www.baidu.com/");
rqst.__set_userdata("my user data");
client.submit(rqst);
std::string url = "http://m.hongxiu.com";
client.submit_url(url);
transport->close();
} catch (TException &tx) {
printf("ERROR: %s\n", tx.what());
}
}
void load(std::string file, SpiderWebServiceClient& client) {
int fd = open(file.c_str(), O_RDONLY);
if(fd == -1) return ;
char buf[1024*1024];
ssize_t n = 0;
ssize_t left = 0;
while((n=read(fd, buf+left, sizeof(buf)-left)) > 0)
{
char* p = buf;
char* begin = buf;
left += n;
while(p < buf+left){
if(*p == '\r' || *p == '\n'){
std::string url = std::string(begin, p-begin);
std::cout<<url<<std::endl;
static int i = 0;
if(i++ % 2 == 0)
{
HttpRequest rqst;
rqst.__set_url(url);
rqst.__set_userdata("USER-DATA:" + url);
client.submit(rqst);
}
else
client.submit_url(url);
while(p<buf+left) {
if(*p != '\r' && *p != '\n')
break;
p++;
}
begin = p;
continue;
}
p++;
}
if(begin != buf && p > begin)
memmove(buf, begin, p-begin);
}
}