/*******************************************************************
* Function : DoGetInternal()
* Parameters: url - The URL to which an HTTP GET is to be done
* resp - The response object which will store the HTTP
* response received from the website
* Returns : TRUE - GET succeeded (could have any HTTP errors,
* including 404 errors)
* FALSE - The GET failed most likely because a
* connection coul not be established
* Purpose : To do an HTTP GET
*******************************************************************/
int CHttpClient::DoGetInternal(const char *url, CHttpClientResponse &resp)
{
//m_bIsBusy = true;
CURLcode res;
MemoryStruct chunk;
CScutString strProxy;
CScutString strProxyAuth;
chunk.memory = NULL;
chunk.size = 0;
chunk.session = session;
chunk.headers.clear();
if (curl_handle == NULL)
Initialize();
resp.SetRequestUrl(url);
CScutString cookies = session->GetCookies(this);
if (cookies != "")
{
//logFile.Write(LOG_COOKIE, "Setting curl cookie to: %s", LPCTSTR(cookies.c_str()));
curl_easy_setopt(curl_handle, CURLOPT_COOKIE, LPCTSTR(cookies.c_str()));
}
curl_easy_setopt(curl_handle, CURLOPT_URL, url);
curl_easy_setopt(curl_handle, CURLOPT_FILE, static_cast<void *>(&resp));
curl_easy_setopt(curl_handle, CURLOPT_WRITEHEADER, static_cast<void *>(&chunk));
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, headers);
//Ignore the Content-Length header. This is useful for Apache 1.x (and similar servers) which will report incorrect content length for files over 2 gigabytes. If this option is used, curl will not be able to accurately report progress, and will simply stop the download when the server ends the connection.
//curl_easy_setopt(curl_handle, CURLOPT_IGNORE_CONTENT_LENGTH, TRUE);
curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 1);
//安全和代理服务器设置
if (settings.m_bProxyNTLM)
curl_easy_setopt(curl_handle, CURLOPT_HTTPAUTH, CURLAUTH_NTLM);
if (settings.m_bProxyAuthNTLM)
curl_easy_setopt(curl_handle, CURLOPT_PROXYAUTH, CURLAUTH_NTLM);
//超时时间
if (m_nTimeOut > 0)
{
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, m_nTimeOut);
}
//断点续传
if (resp.GetUseDataResume() && resp.GetTarget()->GetSize() > 0)
{
//curl_easy_setopt(curl_handle, CURLOPT_RESUME_FROM_LARGE, resp.GetTarget()->GetSize());
CScutString strTemp;
strTemp.Format("%d-", resp.GetTarget()->GetSize());
curl_easy_setopt(curl_handle, CURLOPT_RANGE, strTemp.c_str());
}
if(CScutString::NoCaseCmp(url, "https://", 8) == 0)
{
if(bUseHttpsProxy)
{
strProxy.Format("%s:%d", httpsProxyHost, httpsProxyPort);
curl_easy_setopt(curl_handle, CURLOPT_PROXY, strProxy.c_str());
}
}
else if(bUseHttpProxy)
{
strProxy.Format("%s:%d", httpProxyHost, httpProxyPort);
curl_easy_setopt(curl_handle, CURLOPT_PROXY, strProxy.c_str());
}
if(settings.m_bProxyAuth == TRUE)
{
strProxyAuth.Format("%s:%s", settings.m_strProxyAuthUsername.c_str(), settings.m_strProxyAuthPassword.c_str());
curl_easy_setopt(curl_handle, CURLOPT_PROXYUSERPWD, strProxyAuth.c_str());
}
//进度设置，异步才启用进度显示功能
if (m_bUseProgressReport && m_bAsyncProcessing)
{
curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0);
curl_easy_setopt(curl_handle, CURLOPT_PROGRESSFUNCTION, ProgressReportProc);
curl_easy_setopt(curl_handle, CURLOPT_PROGRESSDATA, this);
}
res = curl_easy_perform(curl_handle);
char *effectiveUrl = NULL;
curl_easy_getinfo(curl_handle, CURLINFO_EFFECTIVE_URL, &effectiveUrl);
referer = effectiveUrl;
//logFile.Write(LOG_ADVANCED, "last URL visited = %s", effectiveUrl);
GetUrlHost(effectiveUrl, host);
resp.SetLastResponseUrl(effectiveUrl);
//获取状态码
int nStatus = 0;
curl_easy_getinfo(curl_handle, CURLINFO_HTTP_CODE, &nStatus);
resp.SetStatusCode(nStatus);
char *conttype = NULL;
curl_easy_getinfo(curl_handle, CURLINFO_CONTENT_TYPE, &conttype);
//logFile.Write(LOG_ADVANCED, "Content-Type = %s", nvl(conttype,"(null)"));
//resp.SetData(chunk.memory, chunk.size);
resp.SetContentType(conttype);
//处理压缩数据
if (IsGzipOrDeflateData(chunk) && resp.GetTarget())
{
GZipUnZipStream(resp.GetTarget());
}
/*
if (res == CURLE_OK)
{
if (nStatus == 200)
{
//判断长度是否正确
double dContentLength = 0;
CURLcode code = curl_easy_getinfo(curl_handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD , &dContentLength);
if (code == CURLE_OK) //判断长度并解压
{
if (resp.GetTarget())
{
if (resp.GetTarget()->GetSize() == (int)dContentLength)
{
//处理压缩数据
if (IsGzipOrDeflateData(chunk))
{
GZipUnZipStream(resp.GetTarget());
}
}
else
res = CURLE_RECV_ERROR;
}
}
else //直接解压
{
//处理压缩数据
if (IsGzipOrDeflateData(chunk) && resp.GetTarget())
{
GZipUnZipStream(resp.GetTarget());
}
}
}
else
res = CURLE_RECV_ERROR;
}
*/
free(chunk.memory);
if(res == CURLE_PARTIAL_FILE)
{
//logFile.Write(LOG_BASIC, "WARNING: HTTP GET returned CURL error: %s (%d) - ignoring",curl_easy_strerror(res), res);
}
if(res != CURLE_OK)
{
const char* strerror = curl_easy_strerror(res);
if (strerror != NULL)
{
//m_pLogFile->Write(LOG_BASIC, strerror);
}
//logFile.Write(LOG_BASIC, "HTTP GET failed. CURL error: %s (%d)",curl_easy_strerror(res), res);
}
//m_bIsBusy = false;
return res;
}
int CHttpClient::DoPostInternal(const char *url, const void * postData, int nPostDataSize, CHttpClientResponse &resp, bool formflag)
{
//m_bIsBusy = true;
CURLcode res;
MemoryStruct chunk;
CScutString strProxy;
CScutString strProxyAuth;
chunk.memory = NULL; /* we expect realloc(NULL, size) to work */
chunk.size = 0; /* no data at this point */
chunk.session = session;
chunk.headers.clear();
/* init the curl session if necessary */
if (curl_handle == NULL)
Initialize();
resp.SetRequestUrl(url);
CScutString cookies = session->GetCookies(this);
if (cookies != "")
{
//logFile.Write(LOG_COOKIE, "Setting curl cookie to: %s", LPCTSTR(cookies.c_str()));
curl_easy_setopt(curl_handle, CURLOPT_COOKIE, LPCTSTR(cookies.c_str()));
}
/* specify URL to get */
curl_easy_setopt(curl_handle, CURLOPT_URL, url);
/* send all data to this function */
//curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(curl_handle, CURLOPT_FILE, static_cast<void *>(&resp));
curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, HttpHeaderData);
curl_easy_setopt(curl_handle, CURLOPT_WRITEHEADER, static_cast<void *>(&chunk));
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, headers);
//Apache服务器需要设定此项
//curl_easy_setopt(curl_handle, CURLOPT_IGNORE_CONTENT_LENGTH, TRUE);
/* Set the operation to POST */
if (formflag)
{
curl_easy_setopt(curl_handle, CURLOPT_HTTPPOST, postData);
}
else
{
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, postData);
if (nPostDataSize != -1)
{
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, nPostDataSize);
}
}
/* set proxy stuff */
if (settings.m_bProxyNTLM)
curl_easy_setopt(curl_handle, CURLOPT_HTTPAUTH, CURLAUTH_NTLM);
if (settings.m_bProxyAuthNTLM)
curl_easy_setopt(curl_handle, CURLOPT_PROXYAUTH, CURLAUTH_NTLM);
//超时处理
if (m_nTimeOut > 0)
{
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, m_nTimeOut);
}
if(CScutString::NoCaseCmp(url, "https://", 8) == 0)
{
if(bUseHttpsProxy)
{
strProxy.Format("%s:%d", httpsProxyHost, httpsProxyPort);
curl_easy_setopt(curl_handle, CURLOPT_PROXY, strProxy.c_str());
}
}
else if(bUseHttpProxy)
{
strProxy.Format("%s:%d", httpProxyHost, httpProxyPort);
curl_easy_setopt(curl_handle, CURLOPT_PROXY, strProxy.c_str());
}
if(settings.m_bProxyAuth == TRUE)
{
strProxyAuth.Format("%s:%s", settings.m_strProxyAuthUsername.c_str(), settings.m_strProxyAuthPassword.c_str());
curl_easy_setopt(curl_handle, CURLOPT_PROXYUSERPWD, strProxyAuth.c_str());
}
/* get it! */
res = curl_easy_perform(curl_handle);
char *effectiveUrl = NULL;
curl_easy_getinfo(curl_handle, CURLINFO_EFFECTIVE_URL, &effectiveUrl);
referer = effectiveUrl;
//logFile.Write(LOG_ADVANCED, "last URL visited = %s", effectiveUrl);
GetUrlHost(effectiveUrl, host);
resp.SetLastResponseUrl(effectiveUrl);
/* cleanup curl stuff */
/*curl_easy_cleanup(curl_handle);
curl_handle = NULL;*/
//resp.SetData(chunk.memory, chunk.size);
//处理压缩数据
if (IsGzipOrDeflateData(chunk) && resp.GetTarget())
{
GZipUnZipStream(resp.GetTarget());
}
free(chunk.memory);
if(res == CURLE_PARTIAL_FILE)
{
//logFile.Write(LOG_BASIC, "WARNING: HTTP POST returned CURL error: %s (%d) - ignoring",curl_easy_strerror(res), res);
}
if(res != CURLE_OK)
{
//logFile.Write(LOG_BASIC, "HTTP POST failed. CURL error: %s (%d)",curl_easy_strerror(res), res);
}
//m_bIsBusy = false;
return res;
}