bool CFileCurl::Download(const CStdString& strURL, const CStdString& strFileName, LPDWORD pdwSize)
{
CLog::Log(LOGINFO, "Download: %s->%s", strURL.c_str(), strFileName.c_str());
CStdString strData;
if (!Get(strURL, strData))
return false;
XFILE::CFile file;
if (!file.OpenForWrite(strFileName, true))
{
CLog::Log(LOGERROR, "Unable to open file %s: %u",
strFileName.c_str(), GetLastError());
return false;
}
if (strData.size())
file.Write(strData.data(), strData.size());
file.Close();
if (pdwSize != NULL)
{
*pdwSize = strData.size();
}
return true;
}
bool CScraperUrl::Get(const SUrlEntry& scrURL, string& strHTML, CHTTP& http)
{
CURL url(scrURL.m_url);
http.SetReferer(scrURL.m_spoof);
CStdString strCachePath;
if (!scrURL.m_cache.IsEmpty())
{
CUtil::AddFileToFolder(g_advancedSettings.m_cachePath,"scrapers\\"+scrURL.m_cache,strCachePath);
if (XFILE::CFile::Exists(strCachePath))
{
XFILE::CFile file;
file.Open(strCachePath);
char* temp = new char[(int)file.GetLength()];
file.Read(temp,file.GetLength());
strHTML.append(temp,temp+file.GetLength());
file.Close();
delete[] temp;
return true;
}
}
if (scrURL.m_post)
{
CStdString strOptions = url.GetOptions();
strOptions = strOptions.substr(1);
url.SetOptions("");
CStdString strUrl;
url.GetURL(strUrl);
if (!http.Post(strUrl, strOptions, strHTML))
return false;
}
else if (!http.Get(scrURL.m_url, strHTML))
return false;
if (scrURL.m_url.Find(".zip") > -1)
{
XFILE::CFileZip file;
CStdString strBuffer;
int iSize = file.UnpackFromMemory(strBuffer,strHTML);
if (iSize)
{
strHTML.clear();
strHTML.append(strBuffer.c_str(),strBuffer.data()+iSize);
}
}
if (!scrURL.m_cache.IsEmpty())
{
CStdString strCachePath;
CUtil::AddFileToFolder(g_advancedSettings.m_cachePath,"scrapers\\"+scrURL.m_cache,strCachePath);
XFILE::CFile file;
if (file.OpenForWrite(strCachePath,true,true))
file.Write(strHTML.data(),strHTML.size());
file.Close();
}
return true;
}
bool CScraperUrl::Get(const SUrlEntry& scrURL, std::string& strHTML, XFILE::CCurlFile& http, const CStdString& cacheContext)
{
CURL url(scrURL.m_url);
http.SetReferer(scrURL.m_spoof);
CStdString strCachePath;
if (scrURL.m_isgz)
http.SetContentEncoding("gzip");
if (!scrURL.m_cache.IsEmpty())
{
URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath,
"scrapers/"+cacheContext+"/"+scrURL.m_cache,
strCachePath);
if (XFILE::CFile::Exists(strCachePath))
{
XFILE::CFile file;
if (file.Open(strCachePath))
{
char* temp = new char[(int)file.GetLength()];
file.Read(temp,file.GetLength());
strHTML.clear();
strHTML.append(temp,temp+file.GetLength());
file.Close();
delete[] temp;
return true;
}
}
}
CStdString strHTML1(strHTML);
if (scrURL.m_post)
{
CStdString strOptions = url.GetOptions();
strOptions = strOptions.substr(1);
url.SetOptions("");
if (!http.Post(url.Get(), strOptions, strHTML1))
return false;
}
else
if (!http.Get(url.Get(), strHTML1))
return false;
strHTML = strHTML1;
if (scrURL.m_url.Find(".zip") > -1 )
{
XFILE::CZipFile file;
CStdString strBuffer;
int iSize = file.UnpackFromMemory(strBuffer,strHTML,scrURL.m_isgz);
if (iSize)
{
strHTML.clear();
strHTML.append(strBuffer.c_str(),strBuffer.data()+iSize);
}
}
if (!scrURL.m_cache.IsEmpty())
{
CStdString strCachePath;
URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath,
"scrapers/"+cacheContext+"/"+scrURL.m_cache,
strCachePath);
XFILE::CFile file;
if (file.OpenForWrite(strCachePath,true))
file.Write(strHTML.data(),strHTML.size());
file.Close();
}
return true;
}
bool CFileCurl::Service(const CStdString& strURL, const CStdString& strPostData, CStdString& strHTML, bool bCloseConnection)
{
XBMC::HttpCacheHandle cacheHandle = g_application.GetHttpCacheManager().Open();
XBMC::CHttpCacheHandleGuard guard(cacheHandle);
m_postdata = strPostData;
std::string strLocalCacheName;
m_etag.clear();
m_httpTimeModified = 0;
if (m_postdata.empty())
{
XBMC::HttpCacheReturnCode rc = g_application.GetHttpCacheManager().StartCachingURL(cacheHandle, strURL.c_str(), strLocalCacheName, m_etag, m_httpTimeModified);
if (rc == XBMC::HTTP_CACHE_ALREADY_IN_PROGRESS)
{
if ( g_application.GetHttpCacheManager().WaitForURL(cacheHandle, strURL.c_str(), strLocalCacheName, 5000) == XBMC::HTTP_CACHE_OK )
rc = XBMC::HTTP_CACHE_ALREADY_EXISTS;
else
rc = g_application.GetHttpCacheManager().StartCachingURL(cacheHandle, strURL.c_str(), strLocalCacheName, m_etag, m_httpTimeModified);
}
if (rc == XBMC::HTTP_CACHE_ALREADY_EXISTS)
{
if (ReadFile(strLocalCacheName, strHTML))
return true;
strLocalCacheName.clear();
}
else if (rc != XBMC::HTTP_CACHE_OK)
{
strLocalCacheName.clear();
cacheHandle = NULL;
m_etag.clear();
m_httpTimeModified = 0;
}
}
if (Open(strURL))
{
if (ReadData(strHTML))
{
long nLastCode=200;
g_curlInterface.easy_getinfo(m_state->m_easyHandle, CURLINFO_RESPONSE_CODE, &nLastCode);
if (bCloseConnection)
Close();
if (nLastCode == 304)
{
ReadFile(strLocalCacheName, strHTML);
strLocalCacheName.clear();
}
if (!strLocalCacheName.empty() && strHTML.size() && cacheHandle)
{
XFILE::CFile file;
if (file.OpenForWrite(strLocalCacheName, true))
{
file.Write(strHTML.data(), strHTML.size());
file.Close();
XBMC::HttpCacheHeaders headers;
const char **ptr = g_application.GetHttpCacheManager().GetUsedReponseHeaders();
while (ptr && *ptr)
{
std::string strHeader = m_state->m_httpheader.GetValue(*ptr).c_str();
if (!strHeader.empty())
headers[*ptr] = strHeader;
ptr++;
}
g_application.GetHttpCacheManager().DoneCachingURL(cacheHandle, strURL.c_str(), nLastCode, headers);
}
}
return true;
}
}
Close();
return false;
}
bool CScraperUrl::Get(const SUrlEntry& scrURL, std::string& strHTML, XFILE::CCurlFile& http, const CStdString& cacheContext)
{
CURL url(scrURL.m_url);
http.SetReferer(scrURL.m_spoof);
CStdString strCachePath;
if (scrURL.m_isgz)
http.SetContentEncoding("gzip");
if (!scrURL.m_cache.IsEmpty())
{
strCachePath = URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath,
"scrapers/" + cacheContext + "/" + scrURL.m_cache);
if (XFILE::CFile::Exists(strCachePath))
{
XFILE::CFile file;
XFILE::auto_buffer buffer;
if (file.LoadFile(strCachePath, buffer))
{
strHTML.assign(buffer.get(), buffer.length());
return true;
}
}
}
CStdString strHTML1(strHTML);
if (scrURL.m_post)
{
CStdString strOptions = url.GetOptions();
strOptions = strOptions.substr(1);
url.SetOptions("");
if (!http.Post(url.Get(), strOptions, strHTML1))
return false;
}
else
if (!http.Get(url.Get(), strHTML1))
return false;
strHTML = strHTML1;
std::string fileCharset(http.GetServerReportedCharset());
if (scrURL.m_url.Find(".zip") > -1 )
{
XFILE::CZipFile file;
CStdString strBuffer;
int iSize = file.UnpackFromMemory(strBuffer,strHTML,scrURL.m_isgz);
if (iSize)
{
fileCharset.clear();
strHTML.clear();
strHTML.append(strBuffer.c_str(),strBuffer.data()+iSize);
}
}
if (!fileCharset.empty() && fileCharset != "UTF-8")
{
std::string converted;
if (g_charsetConverter.ToUtf8(fileCharset, strHTML, converted) && !converted.empty())
strHTML = converted;
}
if (!scrURL.m_cache.IsEmpty())
{
CStdString strCachePath = URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath,
"scrapers/" + cacheContext + "/" + scrURL.m_cache);
XFILE::CFile file;
if (file.OpenForWrite(strCachePath,true))
file.Write(strHTML.data(),strHTML.size());
file.Close();
}
return true;
}
