int Parse(char * pagefile, char *resultfile)
{
char docBuffer[65536];
int i=0,ret=0;
int sock = net_tcp_connect(g_host,g_port);
assert(sock>0);
FILE *fp = fopen(pagefile,"r");
FILE *fpw = fopen(resultfile,"w");
if(fp==NULL)
{
printf("%s can not be opened\n",pagefile);
return -1;
}
assert(fpw!=NULL);
ClientBuffer_t* pClientBuffer=CreateClientBuffer(1);
assert(pClientBuffer!=NULL);
i = 0;
char out[65536]="";
while(fgets(docBuffer, 65535, fp))
{
int len = strlen(docBuffer);
assert(len>0);
docBuffer[len - 1] = 0;
strcpy(out,docBuffer);
time_t time_b = time(0);
ret = AddDocSrc(pClientBuffer, 0,docBuffer,len);
if(ret < 0){
fprintf(stderr, "adddocsrc err:%d\n", ret);
continue;
}
ret = DoClientDocs(sock, pClientBuffer, 1);
if(ret < 0)
{
fprintf(stderr, "doclientdoc err:%d\n", ret);
continue;
}
unsigned long retdocID;
char *keyStr=NULL;
int keyLen;
ret = GetDocResult(pClientBuffer, 0, &retdocID,&keyStr, &keyLen);
fprintf(fpw,"%s\t-->%s\n\n",out,keyStr);
//fprintf(fpw,"%s\n",keyStr);
ResetClientBuffer(pClientBuffer);
time_t time_e = time(0);
time_t timediff=time_e-time_b;
if(timediff>3)
{
fprintf(stderr,"timeout\n");
}
}
fclose(fp);
fclose(fpw);
return 0;
}
void CSpoofRemote::StartConnection(void *pUserData)
{
#if defined(CONF_SPOOFING)
CSpoofRemote *pSelf = (CSpoofRemote *)pUserData;
pSelf->m_ConnState = CONNSTATE_CONNECTING;
pSelf->Console()->Print(0, "spfrmt", "Connecting to zervor...", false);
NETADDR BindAddr;
mem_zero(&pSelf->m_HostAddress, sizeof(pSelf->m_HostAddress));
mem_zero(&BindAddr, sizeof(BindAddr));
// lookup
if(net_host_lookup(g_Config.m_ClSpoofSrvIP, &pSelf->m_HostAddress, NETTYPE_IPV4) != 0)
{
pSelf->Console()->Printf(IConsole::OUTPUT_LEVEL_STANDARD, "spfrmt", "ERROR: Can't resolve %s", g_Config.m_ClSpoofSrvIP);
pSelf->m_ConnState = CONNSTATE_CONNECTING;
return;
}
pSelf->m_HostAddress.port = (unsigned short)g_Config.m_ClSpoofSrvPort;
// connect
BindAddr.type = NETTYPE_IPV4;
pSelf->m_Socket = net_tcp_create(BindAddr);
if(net_tcp_connect(pSelf->m_Socket, &pSelf->m_HostAddress) != 0)
{
net_tcp_close(pSelf->m_Socket);
char aBuf[128];
net_addr_str(&pSelf->m_HostAddress, aBuf, sizeof(aBuf), 0);
pSelf->Console()->Printf(IConsole::OUTPUT_LEVEL_STANDARD, "spfrmt", "ERROR: Can't connect to '%s:%d' on type=%i", aBuf, pSelf->m_HostAddress.port, pSelf->m_HostAddress.type);
pSelf->m_ConnState = CONNSTATE_DISCONNECTED;
int error = net_errno();
#if defined(CONF_FAMILY_WINDOWS)
dbg_msg("spfrmt", " : (errorcode=%d)", error);
#else
dbg_msg("spfrmt", " : (%d '%s')", error, strerror(error));
#endif
return;
}
pSelf->m_LastAck = time_get();
pSelf->Console()->Print(IConsole::OUTPUT_LEVEL_DEBUG, "spfrmt", "connected, creating threads...", false);
pSelf->m_ConnState = CONNSTATE_CONNECTED;
pSelf->m_pWorkerThread = thread_init(CSpoofRemote::Worker, pUserData);
pSelf->m_pListenerThread = thread_init(CSpoofRemote::Listener, pUserData);
#endif
}
/* called once in the master process */
void xnbd_initialize(struct xnbd_info *xnbd)
{
if (xnbd->proxymode) {
int remotefd;
if (!(xnbd->remotehost && xnbd->remoteport
&& xnbd->cachepath && xnbd->cbitmappath))
err("insuffcient info");
remotefd = net_tcp_connect(xnbd->remotehost, xnbd->remoteport);
if (remotefd < 0)
err("connecting %s:%s failed", xnbd->remotehost, xnbd->remoteport);
/* check the remote server and get a disksize */
xnbd->disksize = nbd_negotiate_with_server(remotefd);
nbd_client_send_disc_request(remotefd);
close(remotefd);
xnbd->nblocks = get_disk_nblocks(xnbd->disksize);
xnbd->cbitmap = bitmap_open_file(xnbd->cbitmappath, xnbd->nblocks, &xnbd->cbitmaplen, 0, 1);
// xnbd->cbitmapopened = 1;
/* setup cachefile */
setup_cachedisk(xnbd, xnbd->disksize, xnbd->cachepath);
info("proxymode mode %s %s cache %s cachebitmap %s",
xnbd->remotehost, xnbd->remoteport,
xnbd->cachepath, xnbd->cbitmappath);
} else {
if (!xnbd->diskpath)
err("insuffcient info");
if (xnbd->cow) {
xnbd->ds = open_cow_disk(xnbd->diskpath, 1, 0);
xnbd->disksize = xnbd->ds->disksize;
} else
setup_disk(xnbd->diskpath, xnbd);
xnbd->nblocks = get_disk_nblocks(xnbd->disksize);
}
// monitor_init(xnbd->nblocks);
info("xnbd master initialization done");
}
void xnbd_session_initialize_connections(struct xnbd_info *xnbd, struct xnbd_session *ses)
{
if (xnbd->proxymode) {
off_t disksize = 0;
ses->remotefd = net_tcp_connect(xnbd->remotehost, xnbd->remoteport);
if (ses->remotefd < 0)
err("connecting %s:%s failed", xnbd->remotehost, xnbd->remoteport);
/* negotiate and get disksize from remote server */
disksize = nbd_negotiate_with_server(ses->remotefd);
if (disksize != xnbd->disksize)
err("The remote host answered a different disksize.");
}
}
/**
* @brief Begin connect to MQTT broker
* @param client: MQTT_Client reference
* @retval None
*/
void FUNCTION_ATTRIBUTE
MQTT_Connect(MQTT_Client *mqttClient)
{
MQTT_Disconnect(mqttClient);
mqttClient->pCon = (struct pando_tcp_conn *)pd_malloc(sizeof(struct pando_tcp_conn));
pd_memset(mqttClient->pCon, 0, sizeof(struct pando_tcp_conn));
(mqttClient->pCon)->reverse = mqttClient;
mqttClient->pCon->secure = mqttClient->security;
net_tcp_register_connected_callback(mqttClient->pCon, mqtt_tcpclient_connect_cb);
//no reconnection call back. TODO
mqttClient->keepAliveTick = 0;
mqttClient->reconnectTick = 0;
mqttClient->connectTick = 0;
mqttClient->heart_beat_flag = 1;
mqttClient->mqttTimer.interval = 1000;
mqttClient->mqttTimer.timer_no = 1;
mqttClient->mqttTimer.repeated = 1;
mqttClient->mqttTimer.arg = mqttClient;
mqttClient->mqttTimer.timer_cb = mqtt_timer;
pando_timer_init(&(mqttClient->mqttTimer));
pando_timer_stop(&(mqttClient->mqttTimer));
pando_timer_start(&(mqttClient->mqttTimer));
(mqttClient->pCon)->remote_port = mqttClient->port;
if(UTILS_StrToIP(mqttClient->host, &mqttClient->pCon->remote_ip)) {
INFO("TCP: Connect to ip %s:%d\r\n", mqttClient->host, mqttClient->port);
net_tcp_connect(mqttClient->pCon, mqttClient->sendTimeout);
}
else {
INFO("TCP: Connect to domain %s:%d\r\n", mqttClient->host, mqttClient->port);
//need a host name function. TODO
//espconn_gethostbyname(mqttClient->pCon, mqttClient->host, &mqttClient->ip, mqtt_dns_found);
}
mqttClient->connState = TCP_CONNECTING;
}
IGeoIP::GeoInfo CGeoIP::GetInfo(std::string ip)
{
dbg_msg("GeoIP", "Searching geolocation of '%s'...", ip.c_str());
NETADDR bindAddr;
mem_zero(&bindAddr, sizeof(bindAddr));
char aNetBuff[1024];
std::string jsonData;
IGeoIP::GeoInfo rInfo;
//Connect
bindAddr.type = NETTYPE_IPV4;
m_Socket = net_tcp_create(bindAddr);
net_set_non_blocking(m_Socket);
if(net_tcp_connect(m_Socket, &m_HostAddress) < 0)
{
if (net_errno() != EINPROGRESS)
{
dbg_msg("GeoIP","ERROR: Can't connect.");
net_tcp_close(m_Socket);
return rInfo;
}
}
net_socket_read_wait(m_Socket, 1);
net_set_blocking(m_Socket);
//Send request
str_format(aNetBuff, sizeof(aNetBuff), "GET /geoip/%s HTTP/1.0\r\nHost: www.telize.com\r\n\r\n", ip.c_str());
net_tcp_send(m_Socket, aNetBuff, strlen(aNetBuff));
//read data
bool errors = true;
std::string NetData;
int TotalRecv = 0;
int TotalBytes = 0;
int CurrentRecv = 0;
bool isHead = true;
int enterCtrl = 0;
while ((CurrentRecv = net_tcp_recv(m_Socket, aNetBuff, sizeof(aNetBuff))) > 0)
{
for (int i=0; i<CurrentRecv ; i++)
{
if (isHead)
{
if (aNetBuff[i]=='\n')
{
enterCtrl++;
if (enterCtrl == 2)
{
isHead = false;
NetData.clear();
continue;
}
std::transform(NetData.begin(), NetData.end(), NetData.begin(), ::tolower);
if (NetData.find("content-length:") != std::string::npos)
{
sscanf(NetData.c_str(), "content-length:%d", &TotalBytes);
if (TotalBytes == 0)
sscanf(NetData.c_str(), "content-length: %d", &TotalBytes);
}
NetData.clear();
continue;
}
else if (aNetBuff[i]!='\r')
enterCtrl=0;
NetData+=aNetBuff[i];
}
else
{
if (TotalBytes == 0)
{
net_tcp_close(m_Socket);
dbg_msg("GeoIP","ERROR: Error with size received data.");
break;
}
jsonData += aNetBuff[i];
TotalRecv++;
if (TotalRecv == TotalBytes)
{
errors = false;
break;
}
}
}
}
//Finish
net_tcp_close(m_Socket);
if (!errors)
{
// parse json data
json_settings JsonSettings;
mem_zero(&JsonSettings, sizeof(JsonSettings));
char aError[256];
json_value *pJsonData = json_parse_ex(&JsonSettings, jsonData.c_str(), jsonData.length(), aError);
if (pJsonData == 0)
{
dbg_msg("GeoIP", "Error: %s", aError);
return rInfo;
}
// generate configurations
const json_value &countryCode = (*pJsonData)["country_code"];
if (countryCode.type == json_string) str_copy(rInfo.m_aCountryCode, (const char *)countryCode, sizeof(rInfo.m_aCountryCode));
const json_value &countryName = (*pJsonData)["country"];
if (countryName.type == json_string) str_copy(rInfo.m_aCountryName, (const char *)countryName, sizeof(rInfo.m_aCountryName));
//const json_value &isp = (*pJsonData)["isp"];
//if (isp.type == json_string) geoInfo->m_Isp = (const char *)isp;
}
return rInfo;
}
static int xnbd_connect(const char *devpath, unsigned long blocksize, unsigned int timeout, GList *dst_list, int max_retry, const char *recovery_command)
{
int sockfd;
off_t disksize;
uint32_t flags;
int retry = 0;
for (;;) {
int connected = 0;
for (GList *list = g_list_first(dst_list); list != NULL; list = g_list_next(list)) {
struct dst_info *dst = (struct dst_info *) list->data;
const char *host = dst->host;
const char *port = dst->port;
info("connecting to %s(%s)", host, port);
sockfd = net_tcp_connect(host, port);
if (sockfd < 0) {
warn("cannot connect to %s(%s)", host, port);
continue;
}
int ret = nbd_negotiate_with_server2(sockfd, &disksize, &flags);
if (ret < 0) {
warn("negotiation with %s:%s failed", host, port);
continue;
}
info("connected to %s(%s)", host, port);
connected = 1;
break;
}
if (connected)
break;
if (max_retry == 0 || retry < max_retry) {
info("sleep for a moment and try again ...");
sleep(5);
retry += 1;
continue;
} else
err("cannot establish a NBD session with any server");
}
/* so far, we get a negotiated socket */
int retcode = -3;
int nbd = open(devpath, O_RDWR);
if (nbd < 0)
err("open %s, ret %d, %m", devpath, nbd);
nbddev_set_sockfd(nbd, sockfd);
nbddev_set_sizes(nbd, (uint64_t) disksize, blocksize);
if (flags & NBD_FLAG_READ_ONLY)
nbddev_set_readonly(nbd);
nbddev_set_timeout(nbd, timeout);
/*
* signal(7) says the default disposition of SIGCHLD is "Ignore".
* Here, make sure the parent does not ignore SIGCHLD.
*/
sigset_t set, oldset;
sigemptyset(&set);
sigemptyset(&oldset);
sigaddset(&set, SIGCHLD);
sigaddset(&set, SIGUSR1);
sigprocmask(SIG_BLOCK, &set, &oldset);
pid_t pid = fork();
if (pid < 0)
err("fork() %m");
if (pid != 0) {
/* parent */
for (;;) {
sigset_t sigs;
sigpending(&sigs);
if (sigismember(&sigs, SIGCHLD))
err("xnbd-client (child) was terminated due to an internal error");
pid_t nbd_pid = get_nbd_pid(devpath);
if (nbd_pid < 0) {
info("wait for a moment ...");
sleep(1);
} else
break;
}
/* re-read partition table */
int ret = ioctl(nbd, BLKRRPART);
if (ret < 0) {
warn("ioctl(BLKRRPART), ret %d, %m", ret);
close(nbd);
open(devpath, O_RDONLY);
}
close(sockfd);
close(nbd);
exit(EXIT_SUCCESS);
}
/* set to the default */
sigprocmask(SIG_BLOCK, &oldset, NULL);
/*
* Before /sys/block/nbd0/pid is created by calling ioctl(NBD_DO_IT),
* read/write to /dev/nbd0 fails.
**/
int ret = ioctl(nbd, NBD_DO_IT);
if (ret < 0) {
sigset_t sigs;
sigpending(&sigs);
if (sigismember(&sigs, SIGUSR1)) {
info("%s connection was probably terminated by a user", devpath);
retcode = 0;
} else {
warn("unexpected disconnect. NBD_DO_IT returned, ret %d, %m", ret);
if (recovery_command) {
if (recovery_command == recovery_command_internal_reboot_call) {
info("restart the system by reboot(2)");
ret = reboot(RB_AUTOBOOT);
if (ret < 0)
warn("internal reboot call failed, %m");
/* should never return */
}
info("spawn recovery command, \"%s\"", recovery_command);
GError *error = NULL;
g_spawn_command_line_async(recovery_command, &error);
if (error)
warn("%s", error->message);
}
retcode = -2;
}
}
ioctl(nbd, NBD_CLEAR_QUE);
ioctl(nbd, NBD_CLEAR_SOCK);
close(sockfd);
close(nbd);
/*
* retcode == 0: terminated by a user (after NBD_DO_IT)
* -2: abnormal termination; for example, server
* termination, and transport error.
* I/O errors have probably occurred. (after NBD_DO_IT)
* -3: never happen with Linux 2.6.x; we should quit. (after NBD_DO_IT)
*/
return retcode;
}
bool CHttpDownloader::GetToFile(const char *url, const char *dest, unsigned timeOut, unsigned downloadSpeed)
{
if (!dest || dest[0] == 0)
return false;
int64 downloadTime = time_get();
unsigned chunkBytes = 0;
NETSOCKET sock;
NETADDR nadd, bindAddr;
mem_zero(&nadd, sizeof(nadd));
mem_zero(&bindAddr, sizeof(bindAddr));
bindAddr.type = NETTYPE_IPV4;
NETURL NetUrl = CreateUrl(url);
if (net_host_lookup(NetUrl.m_aHost, &nadd, NETTYPE_IPV4) != 0)
{
dbg_msg("HttpDownloader", "Error can't found '%s'...", NetUrl.m_aHost);
return false;
}
nadd.port = 80;
sock = net_tcp_create(bindAddr);
if (net_tcp_connect(sock, &nadd) != 0)
{
dbg_msg("HttpDownloader", "Error can't connect with '%s'...", NetUrl.m_aHost);
net_tcp_close(sock);
return false;
}
net_socket_rcv_timeout(sock, timeOut);
char aBuff[512] = {0};
str_format(aBuff, sizeof(aBuff), "GET %s HTTP/1.0\r\nHost: %s\r\n\r\n", NetUrl.m_aSlug, NetUrl.m_aHost);
net_tcp_send(sock, aBuff, str_length(aBuff));
std::string NetData;
unsigned TotalRecv = 0;
unsigned TotalBytes = 0;
int CurrentRecv = 0;
unsigned nlCount = 0;
char aNetBuff[1024] = {0};
IOHANDLE dstFile = NULL;
do
{
// Limit Speed
if (downloadSpeed > 0)
{
int64 ctime = time_get();
if (ctime - downloadTime <= time_freq())
{
if (chunkBytes >= downloadSpeed)
{
int tdff = (time_freq() - (ctime - downloadTime)) / 1000;
thread_sleep(tdff);
continue;
}
}
else
{
chunkBytes = 0;
downloadTime = time_get();
}
}
//
CurrentRecv = net_tcp_recv(sock, aNetBuff, sizeof(aNetBuff));
chunkBytes += CurrentRecv;
for (int i=0; i<CurrentRecv ; i++)
{
if (nlCount < 2)
{
if (aNetBuff[i] == '\r' || aNetBuff[i] == '\n')
{
++nlCount;
if (NetData.size() > 0)
{
std::transform(NetData.begin(), NetData.end(), NetData.begin(), ::tolower);
if (NetData.find("404 not found") != std::string::npos)
{
dbg_msg("HttpDownloader", "ERROR 404: '%s' not found...", NetUrl.m_aFile);
net_tcp_close(sock);
return false;
}
else if (NetData.find("content-length:") != std::string::npos)
{
char aFileSize[64];
str_copy(aFileSize, NetData.substr(15).c_str(), sizeof(aFileSize));
str_trim(aFileSize);
TotalBytes = atoi(aFileSize);
}
NetData.clear();
}
if (aNetBuff[i] == '\r') ++i;
continue;
}
nlCount = 0;
NetData += aNetBuff[i];
}
else
{
if (nlCount == 2)
{
if (TotalBytes <= 0)
{
dbg_msg("HttpDownloader", "Error downloading '%s'...", NetUrl.m_aFile);
net_tcp_close(sock);
return false;
}
dstFile = io_open(dest, IOFLAG_WRITE);
if(!dstFile)
{
dbg_msg("HttpDownloader", "Error creating '%s'...", dest);
net_tcp_close(sock);
return false;
}
++nlCount;
}
io_write(dstFile, &aNetBuff[i], 1);
TotalRecv++;
if (TotalRecv == TotalBytes)
break;
}
}
} while (CurrentRecv > 0);
net_tcp_close(sock);
if (TotalRecv > 0)
{
io_close(dstFile);
return true;
}
return false;
}
unsigned CHttpDownloader::GetFileSize(const char *url, unsigned timeOut)
{
NETSOCKET sock;
NETADDR nadd, bindAddr;
mem_zero(&nadd, sizeof(nadd));
mem_zero(&bindAddr, sizeof(bindAddr));
bindAddr.type = NETTYPE_IPV4;
NETURL NetUrl = CreateUrl(url);
if (net_host_lookup(NetUrl.m_aHost, &nadd, NETTYPE_IPV4) != 0)
{
dbg_msg("HttpDownloader", "Error can't found '%s'...", NetUrl.m_aHost);
return false;
}
nadd.port = 80;
sock = net_tcp_create(bindAddr);
net_socket_rcv_timeout(sock, timeOut);
if (net_tcp_connect(sock, &nadd) != 0)
{
dbg_msg("HttpDownloader", "Error can't connect with '%s'...", NetUrl.m_aHost);
net_tcp_close(sock);
return false;
}
char aBuff[512] = {0};
str_format(aBuff, sizeof(aBuff), "GET %s HTTP/1.0\r\nHost: %s\r\n\r\n", NetUrl.m_aSlug, NetUrl.m_aHost);
net_tcp_send(sock, aBuff, str_length(aBuff));
std::string NetData;
int TotalBytes = 0;
int CurrentRecv = 0;
int nlCount = 0;
char aNetBuff[1024] = {0};
do
{
CurrentRecv = net_tcp_recv(sock, aNetBuff, sizeof(aNetBuff));
for (int i=0; i<CurrentRecv ; i++)
{
if (nlCount < 2)
{
if (aNetBuff[i] == '\r' || aNetBuff[i] == '\n')
{
++nlCount;
if (NetData.size() > 0)
{
std::transform(NetData.begin(), NetData.end(), NetData.begin(), ::tolower);
if (NetData.find("404 not found") != std::string::npos)
{
dbg_msg("HttpDownloader", "ERROR 404: '%s' not found...", NetUrl.m_aFile);
net_tcp_close(sock);
return false;
}
else if (NetData.find("content-length:") != std::string::npos)
{
char aFileSize[64];
str_copy(aFileSize, NetData.substr(15).c_str(), sizeof(aFileSize));
str_trim(aFileSize);
TotalBytes = atoi(aFileSize);
if (TotalBytes <= 0)
{
dbg_msg("HttpDownloader", "Error getting size of '%s'...", NetUrl.m_aFile);
net_tcp_close(sock);
return 0;
}
net_tcp_close(sock);
return TotalBytes;
}
NetData.clear();
}
if (aNetBuff[i] == '\r') ++i;
continue;
}
nlCount = 0;
NetData += aNetBuff[i];
}
else
{
dbg_msg("HttpDownloader", "Error getting size of '%s'...", NetUrl.m_aFile);
net_tcp_close(sock);
return 0;
}
}
} while (CurrentRecv > 0);
return 0;
}
// TODO: Ugly but works
bool CUpdater::GetFile(const char *url, const char *path)
{
NETSOCKET Socket = invalid_socket;
NETADDR HostAddress, BindAddr;
mem_zero(&HostAddress, sizeof(HostAddress));
mem_zero(&BindAddr, sizeof(BindAddr));
char aNetBuff[1024];
//Lookup
if(net_host_lookup(UPDATES_HOST, &HostAddress, NETTYPE_IPV4) != 0)
{
dbg_msg("autoupdate", "Error running host lookup");
return false;
}
HostAddress.port = 80;
//Connect
BindAddr.type = NETTYPE_IPV4;
Socket = net_tcp_create(BindAddr);
if(net_tcp_connect(Socket, &HostAddress) != 0)
{
net_tcp_close(Socket);
dbg_msg("autoupdate","Error connecting to host");
return false;
}
//Send request
str_format(aNetBuff, sizeof(aNetBuff), "GET "UPDATES_BASE_PATH"%s HTTP/1.0\r\nHost: "UPDATES_HOST"\r\n\r\n", url);
net_tcp_send(Socket, aNetBuff, str_length(aNetBuff));
//read data
IOHANDLE dstFile = 0;
std::string NetData;
int TotalRecv = 0, TotalBytes = 0, CurrentRecv = 0, enterCtrl = 0;
bool isHead = true, isStatusLine = true;
while ((CurrentRecv = net_tcp_recv(Socket, aNetBuff, sizeof(aNetBuff))) > 0)
{
for (int i=0; i<CurrentRecv ; i++)
{
if (isHead)
{
if (aNetBuff[i]=='\n')
{
if (++enterCtrl == 2) // Go To Body Part
{
dstFile = io_open(path, IOFLAG_WRITE);
if (!dstFile)
{
net_tcp_close(Socket);
dbg_msg("autoupdate","Error writing to disk");
return false;
}
str_copy(m_CurrentDownloadFileName, url, sizeof(m_CurrentDownloadFileName));
isHead = false;
NetData.clear();
continue;
}
std::transform(NetData.begin(), NetData.end(), NetData.begin(), ::tolower);
// Check Status
if (isStatusLine)
{
bool isCodeOk = NetData.find("200") != std::string::npos;
bool isStatusOk = NetData.find("ok") != std::string::npos;
if (!isCodeOk || !isStatusOk)
{
net_tcp_close(Socket);
dbg_msg("autoupdate","Server Host returns error code");
return false;
}
}
isStatusLine = false;
// Get File Size
std::size_t fpos = std::string::npos;
if ((fpos=NetData.find("content-length:")) != std::string::npos)
sscanf(NetData.substr(fpos+15).c_str(), "%d", &TotalBytes);
NetData.clear();
continue;
}
else if (aNetBuff[i]!='\r')
enterCtrl=0;
NetData+=aNetBuff[i];
}
else // Body Part
{
if (TotalBytes <= 0)
{
io_close(dstFile);
net_tcp_close(Socket);
dbg_msg("autoupdate","Error receiving file");
return false;
}
m_CurrentDownloadProgress = (float)TotalRecv/(float)TotalBytes;
io_write(dstFile, &aNetBuff[i], 1);
TotalRecv++;
if (TotalRecv == TotalBytes)
break;
}
}
}
//Finish
io_close(dstFile);
net_tcp_close(Socket);
return true;
}
int Parse(char * pagefile, char *resultfile)
{
char docBuffer[65536];
int i=0,ret=0;
int sock = net_tcp_connect(g_host,g_port);
assert(sock>0);
WorkData_t *workData = (WorkData_t*)calloc(sizeof(WorkData_t),1) ;
assert(workData!=NULL);
workData->ptree = CreateJsonTree();
assert(workData->ptree!=NULL);
workData->ptext = CreateJsonText(1000);
assert(workData->ptext != NULL);
FILE *fp = fopen(pagefile,"r");
FILE *fpw = fopen(resultfile,"w");
if(fp==NULL)
{
printf("%s can not be opened\n",pagefile);
return -1;
}
assert(fpw!=NULL);
ClientBuffer_t* pClientBuffer=CreateClientBuffer(1);
assert(pClientBuffer!=NULL);
i = 0;
char out[65536];
while(fgets(docBuffer, 65535, fp))
{
int len = strlen(docBuffer);
assert(len>0);
snprintf(out,len+1,"%s",docBuffer);
time_t time_b = time(0);
ret = AddDocSrc(pClientBuffer, 0,docBuffer,len);
if(ret < 0){
fprintf(stderr, "adddocsrc err:%d\n", ret);
continue;
}
ret = DoClientDocs(sock, pClientBuffer, 1);
if(ret < 0)
{
fprintf(stderr, "doclientdoc err:%d\n", ret);
continue;
}
unsigned long retdocID;
char *keyStr=NULL;
char output[65536];
int keyLen;
PairNode_t *ppair;
ret = GetDocResult(pClientBuffer, 0, &retdocID,&keyStr, &keyLen);
//ret = ParseJson((char*)keyStr, keyLen, workData->ptree);
keyStr[keyLen]='\0';
fprintf(fpw,"%s-->%s\n",out,keyStr);
/*
ForEachPairNode(workData->ptree, 0, ppair)
{
if((strcmp(ppair->keyStr,"term")==0))
{
//if(ppair->v_type != V_STR)
//{
// return -1;
//}
fprintf(fpw,"\"%s\":\"%s\",",ppair->keyStr,ppair->pStr);
}
else if((strcmp(ppair->keyStr,"df")==0))
{
//if(ppair->v_type != V_STR)
//{
// return -1;
//}
fprintf(fpw,"\"%s\":\"%s\",",ppair->keyStr,ppair->pStr);
}
else if((strcmp(ppair->keyStr,"cooc")==0))
{
//if(ppair->v_type != V_STR)
//{
// return -1;
//}
fprintf(fpw,"\"%s\":\"%s\",",ppair->keyStr,ppair->pStr);
}
}
fprintf(fpw,"\n");
*/
//fprintf(fpw,"%s\n",keyStr);
ResetClientBuffer(pClientBuffer);
time_t time_e = time(0);
time_t timediff=time_e-time_b;
if(timediff>3)
{
fprintf(stderr,"timeout\n");
}
}
fclose(fp);
fclose(fpw);
return 0;
}