void CInputCodingTableBaiduPY::Process()
{
while (!m_bStop) //Make sure we don't exit the thread
{
AbortableWait(m_Event, -1); //Wait for work to appear
while (!m_bStop) //Process all queued work before going back to wait on the event
{
CSingleLock lock(m_CS);
if (m_work.empty())
break;
auto work = m_work.front();
m_work.pop_front();
lock.Leave();
std::string data;
XFILE::CCurlFile http;
std::string strUrl;
strUrl = StringUtils::Format(m_url.c_str(), work.c_str(), m_api_begin, m_api_end);
if (http.Get(strUrl, data))
HandleResponse(work, data);
}
}
}
bool CRepository::FetchIndex(const DirInfo& repo, VECADDONS& addons) noexcept
{
XFILE::CCurlFile http;
http.SetAcceptEncoding("gzip");
std::string response;
if (!http.Get(repo.info, response))
{
CLog::Log(LOGERROR, "CRepository: failed to read %s", repo.info.c_str());
return false;
}
if (URIUtils::HasExtension(repo.info, ".gz")
|| CMime::GetFileTypeFromMime(http.GetProperty(XFILE::FILE_PROPERTY_MIME_TYPE)) == CMime::EFileType::FileTypeGZip)
{
CLog::Log(LOGDEBUG, "CRepository '%s' is gzip. decompressing", repo.info.c_str());
std::string buffer;
if (!CZipFile::DecompressGzip(response, buffer))
{
CLog::Log(LOGERROR, "CRepository: failed to decompress gzip from '%s'", repo.info.c_str());
return false;
}
response = std::move(buffer);
}
return CServiceBroker::GetAddonMgr().AddonsFromRepoXML(repo, response, addons);
}
bool CScraperUrl::DownloadThumbnail(const CStdString &thumb, const CScraperUrl::SUrlEntry& entry)
{
if (entry.m_url.IsEmpty())
return false;
CURL url(entry.m_url);
if (url.GetProtocol() != "http")
{ // do a direct file copy
try
{
return CPicture::CreateThumbnail(entry.m_url, thumb);
}
catch (...)
{
XFILE::CFile::Delete(thumb);
}
return false;
}
XFILE::CCurlFile http;
http.SetReferer(entry.m_spoof);
CStdString thumbData;
if (http.Get(entry.m_url, thumbData))
{
try
{
return CPicture::CreateThumbnailFromMemory((const BYTE *)thumbData.c_str(), thumbData.size(), URIUtils::GetExtension(entry.m_url), thumb);
}
catch (...)
{
XFILE::CFile::Delete(thumb);
}
}
return false;
}
bool CRepository::FetchIndex(const DirInfo& repo, std::string const& digest, VECADDONS& addons) noexcept
{
XFILE::CCurlFile http;
http.SetAcceptEncoding("gzip");
std::string response;
if (!http.Get(repo.info, response))
{
CLog::Log(LOGERROR, "CRepository: failed to read %s", repo.info.c_str());
return false;
}
if (repo.checksumType != CDigest::Type::INVALID)
{
std::string actualDigest = CDigest::Calculate(repo.checksumType, response);
if (!StringUtils::EqualsNoCase(digest, actualDigest))
{
CLog::Log(LOGERROR, "CRepository: {} index has wrong digest {}, expected: {}", repo.info, actualDigest, digest);
return false;
}
}
if (URIUtils::HasExtension(repo.info, ".gz")
|| CMime::GetFileTypeFromMime(http.GetProperty(XFILE::FILE_PROPERTY_MIME_TYPE)) == CMime::EFileType::FileTypeGZip)
{
CLog::Log(LOGDEBUG, "CRepository '%s' is gzip. decompressing", repo.info.c_str());
std::string buffer;
if (!CZipFile::DecompressGzip(response, buffer))
{
CLog::Log(LOGERROR, "CRepository: failed to decompress gzip from '%s'", repo.info.c_str());
return false;
}
response = std::move(buffer);
}
return CServiceBroker::GetAddonMgr().AddonsFromRepoXML(repo, response, addons);
}
bool CScraperUrl::Get(const SUrlEntry& scrURL, std::string& strHTML, XFILE::CCurlFile& http, const CStdString& cacheContext)
{
CURL url(scrURL.m_url);
http.SetReferer(scrURL.m_spoof);
CStdString strCachePath;
if (scrURL.m_isgz)
http.SetContentEncoding("gzip");
if (!scrURL.m_cache.empty())
{
strCachePath = URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath,
"scrapers/" + cacheContext + "/" + scrURL.m_cache);
if (XFILE::CFile::Exists(strCachePath))
{
XFILE::CFile file;
XFILE::auto_buffer buffer;
if (file.LoadFile(strCachePath, buffer))
{
strHTML.assign(buffer.get(), buffer.length());
return true;
}
}
}
CStdString strHTML1(strHTML);
if (scrURL.m_post)
{
CStdString strOptions = url.GetOptions();
strOptions = strOptions.substr(1);
url.SetOptions("");
if (!http.Post(url.Get(), strOptions, strHTML1))
return false;
}
else
if (!http.Get(url.Get(), strHTML1))
return false;
strHTML = strHTML1;
std::string mimeType(http.GetMimeType());
CMime::EFileType ftype = CMime::GetFileTypeFromMime(mimeType);
if (ftype == CMime::FileTypeUnknown)
ftype = CMime::GetFileTypeFromContent(strHTML);
if (ftype == CMime::FileTypeZip || ftype == CMime::FileTypeGZip)
{
XFILE::CZipFile file;
std::string strBuffer;
int iSize = file.UnpackFromMemory(strBuffer,strHTML,scrURL.m_isgz); // FIXME: use FileTypeGZip instead of scrURL.m_isgz?
if (iSize > 0)
{
strHTML = strBuffer;
CLog::Log(LOGDEBUG, "%s: Archive \"%s\" was unpacked in memory", __FUNCTION__, scrURL.m_url.c_str());
}
else
CLog::Log(LOGWARNING, "%s: \"%s\" looks like archive, but cannot be unpacked", __FUNCTION__, scrURL.m_url.c_str());
}
std::string reportedCharset(http.GetServerReportedCharset());
if (ftype == CMime::FileTypeHtml)
{
std::string realHtmlCharset, converted;
if (!CCharsetDetection::ConvertHtmlToUtf8(strHTML, converted, reportedCharset, realHtmlCharset))
CLog::Log(LOGWARNING, "%s: Can't find precise charset for HTML \"%s\", using \"%s\" as fallback", __FUNCTION__, scrURL.m_url.c_str(), realHtmlCharset.c_str());
else
CLog::Log(LOGDEBUG, "%s: Using \"%s\" charset for HTML \"%s\"", __FUNCTION__, realHtmlCharset.c_str(), scrURL.m_url.c_str());
strHTML = converted;
}
else if (ftype == CMime::FileTypeXml)
{
CXBMCTinyXML xmlDoc;
xmlDoc.Parse(strHTML, reportedCharset);
std::string realXmlCharset(xmlDoc.GetUsedCharset());
if (!realXmlCharset.empty())
{
CLog::Log(LOGDEBUG, "%s: Using \"%s\" charset for XML \"%s\"", __FUNCTION__, realXmlCharset.c_str(), scrURL.m_url.c_str());
std::string converted;
g_charsetConverter.ToUtf8(realXmlCharset, strHTML, converted);
strHTML = converted;
}
}
else if (ftype == CMime::FileTypePlainText || StringUtils::CompareNoCase(mimeType.substr(0, 5), "text/") == 0)
{
std::string realTextCharset, converted;
CCharsetDetection::ConvertPlainTextToUtf8(strHTML, converted, reportedCharset, realTextCharset);
strHTML = converted;
if (reportedCharset != realTextCharset)
CLog::Log(LOGWARNING, "%s: Using \"%s\" charset for plain text \"%s\" instead of server reported \"%s\" charset", __FUNCTION__, realTextCharset.c_str(), scrURL.m_url.c_str(), reportedCharset.c_str());
else
CLog::Log(LOGDEBUG, "%s: Using \"%s\" charset for plain text \"%s\"", __FUNCTION__, realTextCharset.c_str(), scrURL.m_url.c_str());
}
else if (!reportedCharset.empty())
{
CLog::Log(LOGDEBUG, "%s: Using \"%s\" charset for \"%s\"", __FUNCTION__, reportedCharset.c_str(), scrURL.m_url.c_str());
if (reportedCharset != "UTF-8")
{
std::string converted;
g_charsetConverter.ToUtf8(reportedCharset, strHTML, converted);
strHTML = converted;
}
}
else
CLog::Log(LOGDEBUG, "%s: Using content of \"%s\" as binary or text with \"UTF-8\" charset", __FUNCTION__, scrURL.m_url.c_str());
if (!scrURL.m_cache.empty())
{
CStdString strCachePath = URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath,
"scrapers/" + cacheContext + "/" + scrURL.m_cache);
XFILE::CFile file;
if (file.OpenForWrite(strCachePath,true))
file.Write(strHTML.data(),strHTML.size());
file.Close();
}
return true;
}
bool CScraperUrl::Get(const SUrlEntry& scrURL, std::string& strHTML, XFILE::CCurlFile& http, const CStdString& cacheContext)
{
CURL url(scrURL.m_url);
http.SetReferer(scrURL.m_spoof);
CStdString strCachePath;
if (scrURL.m_isgz)
http.SetContentEncoding("gzip");
if (!scrURL.m_cache.IsEmpty())
{
URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath,
"scrapers/"+cacheContext+"/"+scrURL.m_cache,
strCachePath);
if (XFILE::CFile::Exists(strCachePath))
{
XFILE::CFile file;
if (file.Open(strCachePath))
{
char* temp = new char[(int)file.GetLength()];
file.Read(temp,file.GetLength());
strHTML.clear();
strHTML.append(temp,temp+file.GetLength());
file.Close();
delete[] temp;
return true;
}
}
}
CStdString strHTML1(strHTML);
if (scrURL.m_post)
{
CStdString strOptions = url.GetOptions();
strOptions = strOptions.substr(1);
url.SetOptions("");
if (!http.Post(url.Get(), strOptions, strHTML1))
return false;
}
else
if (!http.Get(url.Get(), strHTML1))
return false;
strHTML = strHTML1;
if (scrURL.m_url.Find(".zip") > -1 )
{
XFILE::CZipFile file;
CStdString strBuffer;
int iSize = file.UnpackFromMemory(strBuffer,strHTML,scrURL.m_isgz);
if (iSize)
{
strHTML.clear();
strHTML.append(strBuffer.c_str(),strBuffer.data()+iSize);
}
}
if (!scrURL.m_cache.IsEmpty())
{
CStdString strCachePath;
URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath,
"scrapers/"+cacheContext+"/"+scrURL.m_cache,
strCachePath);
XFILE::CFile file;
if (file.OpenForWrite(strCachePath,true))
file.Write(strHTML.data(),strHTML.size());
file.Close();
}
return true;
}
bool CPlexClient::ParseSections(PlexSectionParsing parser)
{
bool rtn = false;
XFILE::CCurlFile plex;
plex.SetBufferSize(32768*10);
plex.SetTimeout(10);
CURL curl(m_url);
curl.SetFileName(curl.GetFileName() + "library/sections");
std::string strResponse;
if (plex.Get(curl.Get(), strResponse))
{
#if defined(PLEX_DEBUG_VERBOSE)
if (parser == PlexSectionParsing::newSection || parser == PlexSectionParsing::checkSection)
CLog::Log(LOGDEBUG, "CPlexClient::ParseSections %d, %s", parser, strResponse.c_str());
#endif
if (parser == PlexSectionParsing::updateSection)
{
{
CSingleLock lock(m_criticalMovies);
m_movieSectionsContents.clear();
}
{
CSingleLock lock(m_criticalTVShow);
m_showSectionsContents.clear();
}
m_needUpdate = false;
}
TiXmlDocument xml;
xml.Parse(strResponse.c_str());
TiXmlElement* MediaContainer = xml.RootElement();
if (MediaContainer)
{
const TiXmlElement* DirectoryNode = MediaContainer->FirstChildElement("Directory");
while (DirectoryNode)
{
PlexSectionsContent content;
content.uuid = XMLUtils::GetAttribute(DirectoryNode, "uuid");
content.path = XMLUtils::GetAttribute(DirectoryNode, "path");
content.type = XMLUtils::GetAttribute(DirectoryNode, "type");
content.title = XMLUtils::GetAttribute(DirectoryNode, "title");
content.updatedAt = XMLUtils::GetAttribute(DirectoryNode, "updatedAt");
std::string key = XMLUtils::GetAttribute(DirectoryNode, "key");
content.section = "library/sections/" + key;
content.thumb = XMLUtils::GetAttribute(DirectoryNode, "composite");
std::string art = XMLUtils::GetAttribute(DirectoryNode, "art");
if (m_local)
content.art = art;
else
content.art = content.section + "/resources/" + URIUtils::GetFileName(art);
if (content.type == "movie")
{
if (parser == PlexSectionParsing::checkSection)
{
CSingleLock lock(m_criticalMovies);
for (const auto &contents : m_movieSectionsContents)
{
if (contents.uuid == content.uuid)
{
if (contents.updatedAt != content.updatedAt)
{
#if defined(PLEX_DEBUG_VERBOSE)
CLog::Log(LOGDEBUG, "CPlexClient::ParseSections need update on %s:%s",
m_serverName.c_str(), content.title.c_str());
#endif
m_needUpdate = true;
}
}
}
}
else
{
CSingleLock lock(m_criticalMovies);
m_movieSectionsContents.push_back(content);
}
}
else if (content.type == "show")
{
if (parser == PlexSectionParsing::checkSection)
{
CSingleLock lock(m_criticalTVShow);
for (const auto &contents : m_showSectionsContents)
{
if (contents.uuid == content.uuid)
{
if (contents.updatedAt != content.updatedAt)
{
#if defined(PLEX_DEBUG_VERBOSE)
CLog::Log(LOGDEBUG, "CPlexClient::ParseSections need update on %s:%s",
m_serverName.c_str(), content.title.c_str());
#endif
m_needUpdate = true;
}
}
}
}
else
{
CSingleLock lock(m_criticalTVShow);
m_showSectionsContents.push_back(content);
}
}
else if (content.type == "artist")
{
if (parser == PlexSectionParsing::checkSection)
{
CSingleLock lock(m_criticalArtist);
for (const auto &contents : m_artistSectionsContents)
{
if (contents.uuid == content.uuid)
{
if (contents.updatedAt != content.updatedAt)
{
#if defined(PLEX_DEBUG_VERBOSE)
CLog::Log(LOGDEBUG, "CPlexClient::ParseSections need update on %s:%s",
m_serverName.c_str(), content.title.c_str());
#endif
m_needUpdate = true;
}
}
}
}
else
{
CSingleLock lock(m_criticalArtist);
m_artistSectionsContents.push_back(content);
}
}
else if (content.type == "photo")
{
if (parser == PlexSectionParsing::checkSection)
{
CSingleLock lock(m_criticalPhoto);
for (const auto &contents : m_photoSectionsContents)
{
if (contents.uuid == content.uuid)
{
if (contents.updatedAt != content.updatedAt)
{
#if defined(PLEX_DEBUG_VERBOSE)
CLog::Log(LOGDEBUG, "CPlexClient::ParseSections need update on %s:%s",
m_serverName.c_str(), content.title.c_str());
#endif
m_needUpdate = true;
}
}
}
}
else
{
CSingleLock lock(m_criticalPhoto);
m_photoSectionsContents.push_back(content);
}
}
else
{
CLog::Log(LOGDEBUG, "CPlexClient::ParseSections %s found unhandled content type %s",
m_serverName.c_str(), content.type.c_str());
}
DirectoryNode = DirectoryNode->NextSiblingElement("Directory");
}
CLog::Log(LOGDEBUG, "CPlexClient::ParseSections %s found %d movie sections",
m_serverName.c_str(), (int)m_movieSectionsContents.size());
CLog::Log(LOGDEBUG, "CPlexClient::ParseSections %s found %d shows sections",
m_serverName.c_str(), (int)m_showSectionsContents.size());
CLog::Log(LOGDEBUG, "CPlexClient::ParseSections %s found %d artist sections",
m_serverName.c_str(), (int)m_artistSectionsContents.size());
CLog::Log(LOGDEBUG, "CPlexClient::ParseSections %s found %d photo sections",
m_serverName.c_str(), (int)m_photoSectionsContents.size());
rtn = true;
}
else
{
CLog::Log(LOGDEBUG, "CPlexClient::ParseSections no MediaContainer found");
}
}
else
{
// 401's are attempts to access a local server that is also in PMS
// and these require an access token. Only local servers that are
// not is PMS can be accessed via GDM.
if (plex.GetResponseCode() != 401)
CLog::Log(LOGDEBUG, "CPlexClient::ParseSections failed %s", strResponse.c_str());
rtn = false;
}
return rtn;
}
bool CScraperUrl::Get(const SUrlEntry& scrURL, std::string& strHTML, XFILE::CCurlFile& http, const CStdString& cacheContext)
{
CURL url(scrURL.m_url);
http.SetReferer(scrURL.m_spoof);
CStdString strCachePath;
if (scrURL.m_isgz)
http.SetContentEncoding("gzip");
if (!scrURL.m_cache.IsEmpty())
{
strCachePath = URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath,
"scrapers/" + cacheContext + "/" + scrURL.m_cache);
if (XFILE::CFile::Exists(strCachePath))
{
XFILE::CFile file;
XFILE::auto_buffer buffer;
if (file.LoadFile(strCachePath, buffer))
{
strHTML.assign(buffer.get(), buffer.length());
return true;
}
}
}
CStdString strHTML1(strHTML);
if (scrURL.m_post)
{
CStdString strOptions = url.GetOptions();
strOptions = strOptions.substr(1);
url.SetOptions("");
if (!http.Post(url.Get(), strOptions, strHTML1))
return false;
}
else
if (!http.Get(url.Get(), strHTML1))
return false;
strHTML = strHTML1;
std::string fileCharset(http.GetServerReportedCharset());
if (scrURL.m_url.Find(".zip") > -1 )
{
XFILE::CZipFile file;
CStdString strBuffer;
int iSize = file.UnpackFromMemory(strBuffer,strHTML,scrURL.m_isgz);
if (iSize)
{
fileCharset.clear();
strHTML.clear();
strHTML.append(strBuffer.c_str(),strBuffer.data()+iSize);
}
}
if (!fileCharset.empty() && fileCharset != "UTF-8")
{
std::string converted;
if (g_charsetConverter.ToUtf8(fileCharset, strHTML, converted) && !converted.empty())
strHTML = converted;
}
if (!scrURL.m_cache.IsEmpty())
{
CStdString strCachePath = URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath,
"scrapers/" + cacheContext + "/" + scrURL.m_cache);
XFILE::CFile file;
if (file.OpenForWrite(strCachePath,true))
file.Write(strHTML.data(),strHTML.size());
file.Close();
}
return true;
}