bool CRepository::FetchIndex(const DirInfo& repo, VECADDONS& addons) noexcept { XFILE::CCurlFile http; http.SetAcceptEncoding("gzip"); std::string response; if (!http.Get(repo.info, response)) { CLog::Log(LOGERROR, "CRepository: failed to read %s", repo.info.c_str()); return false; } if (URIUtils::HasExtension(repo.info, ".gz") || CMime::GetFileTypeFromMime(http.GetProperty(XFILE::FILE_PROPERTY_MIME_TYPE)) == CMime::EFileType::FileTypeGZip) { CLog::Log(LOGDEBUG, "CRepository '%s' is gzip. decompressing", repo.info.c_str()); std::string buffer; if (!CZipFile::DecompressGzip(response, buffer)) { CLog::Log(LOGERROR, "CRepository: failed to decompress gzip from '%s'", repo.info.c_str()); return false; } response = std::move(buffer); } return CServiceBroker::GetAddonMgr().AddonsFromRepoXML(repo, response, addons); }
bool CScraperUrl::DownloadThumbnail(const CStdString &thumb, const CScraperUrl::SUrlEntry& entry) { if (entry.m_url.IsEmpty()) return false; CURL url(entry.m_url); if (url.GetProtocol() != "http") { // do a direct file copy try { return CPicture::CreateThumbnail(entry.m_url, thumb); } catch (...) { XFILE::CFile::Delete(thumb); } return false; } XFILE::CCurlFile http; http.SetReferer(entry.m_spoof); CStdString thumbData; if (http.Get(entry.m_url, thumbData)) { try { return CPicture::CreateThumbnailFromMemory((const BYTE *)thumbData.c_str(), thumbData.size(), URIUtils::GetExtension(entry.m_url), thumb); } catch (...) { XFILE::CFile::Delete(thumb); } } return false; }
void CInputCodingTableBaiduPY::Process() { while (!m_bStop) //Make sure we don't exit the thread { AbortableWait(m_Event, -1); //Wait for work to appear while (!m_bStop) //Process all queued work before going back to wait on the event { CSingleLock lock(m_CS); if (m_work.empty()) break; auto work = m_work.front(); m_work.pop_front(); lock.Leave(); std::string data; XFILE::CCurlFile http; std::string strUrl; strUrl = StringUtils::Format(m_url.c_str(), work.c_str(), m_api_begin, m_api_end); if (http.Get(strUrl, data)) HandleResponse(work, data); } } }
CSysData::INTERNET_STATE CSysInfoJob::GetInternetState() { // Internet connection state! XFILE::CCurlFile http; if (http.IsInternet()) return CSysData::CONNECTED; return CSysData::DISCONNECTED; }
void CLastFMDirectory::Run() { XFILE::CCurlFile http; if (!http.Download(m_strSource, m_strDestination)) m_Error=true; m_Downloaded=true; }
bool CRepository::FetchIndex(const DirInfo& repo, std::string const& digest, VECADDONS& addons) noexcept { XFILE::CCurlFile http; http.SetAcceptEncoding("gzip"); std::string response; if (!http.Get(repo.info, response)) { CLog::Log(LOGERROR, "CRepository: failed to read %s", repo.info.c_str()); return false; } if (repo.checksumType != CDigest::Type::INVALID) { std::string actualDigest = CDigest::Calculate(repo.checksumType, response); if (!StringUtils::EqualsNoCase(digest, actualDigest)) { CLog::Log(LOGERROR, "CRepository: {} index has wrong digest {}, expected: {}", repo.info, actualDigest, digest); return false; } } if (URIUtils::HasExtension(repo.info, ".gz") || CMime::GetFileTypeFromMime(http.GetProperty(XFILE::FILE_PROPERTY_MIME_TYPE)) == CMime::EFileType::FileTypeGZip) { CLog::Log(LOGDEBUG, "CRepository '%s' is gzip. decompressing", repo.info.c_str()); std::string buffer; if (!CZipFile::DecompressGzip(response, buffer)) { CLog::Log(LOGERROR, "CRepository: failed to decompress gzip from '%s'", repo.info.c_str()); return false; } response = std::move(buffer); } return CServiceBroker::GetAddonMgr().AddonsFromRepoXML(repo, response, addons); }
bool CScraperUrl::Get(const SUrlEntry& scrURL, std::string& strHTML, XFILE::CCurlFile& http, const CStdString& cacheContext) { CURL url(scrURL.m_url); http.SetReferer(scrURL.m_spoof); CStdString strCachePath; if (scrURL.m_isgz) http.SetContentEncoding("gzip"); if (!scrURL.m_cache.empty()) { strCachePath = URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath, "scrapers/" + cacheContext + "/" + scrURL.m_cache); if (XFILE::CFile::Exists(strCachePath)) { XFILE::CFile file; XFILE::auto_buffer buffer; if (file.LoadFile(strCachePath, buffer)) { strHTML.assign(buffer.get(), buffer.length()); return true; } } } CStdString strHTML1(strHTML); if (scrURL.m_post) { CStdString strOptions = url.GetOptions(); strOptions = strOptions.substr(1); url.SetOptions(""); if (!http.Post(url.Get(), strOptions, strHTML1)) return false; } else if (!http.Get(url.Get(), strHTML1)) return false; strHTML = strHTML1; std::string mimeType(http.GetMimeType()); CMime::EFileType ftype = CMime::GetFileTypeFromMime(mimeType); if (ftype == CMime::FileTypeUnknown) ftype = CMime::GetFileTypeFromContent(strHTML); if (ftype == CMime::FileTypeZip || ftype == CMime::FileTypeGZip) { XFILE::CZipFile file; std::string strBuffer; int iSize = file.UnpackFromMemory(strBuffer,strHTML,scrURL.m_isgz); // FIXME: use FileTypeGZip instead of scrURL.m_isgz? if (iSize > 0) { strHTML = strBuffer; CLog::Log(LOGDEBUG, "%s: Archive \"%s\" was unpacked in memory", __FUNCTION__, scrURL.m_url.c_str()); } else CLog::Log(LOGWARNING, "%s: \"%s\" looks like archive, but cannot be unpacked", __FUNCTION__, scrURL.m_url.c_str()); } std::string reportedCharset(http.GetServerReportedCharset()); if (ftype == CMime::FileTypeHtml) { std::string realHtmlCharset, converted; if (!CCharsetDetection::ConvertHtmlToUtf8(strHTML, converted, reportedCharset, realHtmlCharset)) CLog::Log(LOGWARNING, "%s: Can't find precise charset for HTML \"%s\", using \"%s\" as fallback", __FUNCTION__, scrURL.m_url.c_str(), realHtmlCharset.c_str()); else CLog::Log(LOGDEBUG, "%s: Using \"%s\" charset for HTML \"%s\"", __FUNCTION__, realHtmlCharset.c_str(), scrURL.m_url.c_str()); strHTML = converted; } else if (ftype == CMime::FileTypeXml) { CXBMCTinyXML xmlDoc; xmlDoc.Parse(strHTML, reportedCharset); std::string realXmlCharset(xmlDoc.GetUsedCharset()); if (!realXmlCharset.empty()) { CLog::Log(LOGDEBUG, "%s: Using \"%s\" charset for XML \"%s\"", __FUNCTION__, realXmlCharset.c_str(), scrURL.m_url.c_str()); std::string converted; g_charsetConverter.ToUtf8(realXmlCharset, strHTML, converted); strHTML = converted; } } else if (ftype == CMime::FileTypePlainText || StringUtils::CompareNoCase(mimeType.substr(0, 5), "text/") == 0) { std::string realTextCharset, converted; CCharsetDetection::ConvertPlainTextToUtf8(strHTML, converted, reportedCharset, realTextCharset); strHTML = converted; if (reportedCharset != realTextCharset) CLog::Log(LOGWARNING, "%s: Using \"%s\" charset for plain text \"%s\" instead of server reported \"%s\" charset", __FUNCTION__, realTextCharset.c_str(), scrURL.m_url.c_str(), reportedCharset.c_str()); else CLog::Log(LOGDEBUG, "%s: Using \"%s\" charset for plain text \"%s\"", __FUNCTION__, realTextCharset.c_str(), scrURL.m_url.c_str()); } else if (!reportedCharset.empty()) { CLog::Log(LOGDEBUG, "%s: Using \"%s\" charset for \"%s\"", __FUNCTION__, reportedCharset.c_str(), scrURL.m_url.c_str()); if (reportedCharset != "UTF-8") { std::string converted; g_charsetConverter.ToUtf8(reportedCharset, strHTML, converted); strHTML = converted; } } else CLog::Log(LOGDEBUG, "%s: Using content of \"%s\" as binary or text with \"UTF-8\" charset", __FUNCTION__, scrURL.m_url.c_str()); if (!scrURL.m_cache.empty()) { CStdString strCachePath = URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath, "scrapers/" + cacheContext + "/" + scrURL.m_cache); XFILE::CFile file; if (file.OpenForWrite(strCachePath,true)) file.Write(strHTML.data(),strHTML.size()); file.Close(); } return true; }
bool CScraperUrl::Get(const SUrlEntry& scrURL, std::string& strHTML, XFILE::CCurlFile& http, const CStdString& cacheContext) { CURL url(scrURL.m_url); http.SetReferer(scrURL.m_spoof); CStdString strCachePath; if (scrURL.m_isgz) http.SetContentEncoding("gzip"); if (!scrURL.m_cache.IsEmpty()) { URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath, "scrapers/"+cacheContext+"/"+scrURL.m_cache, strCachePath); if (XFILE::CFile::Exists(strCachePath)) { XFILE::CFile file; if (file.Open(strCachePath)) { char* temp = new char[(int)file.GetLength()]; file.Read(temp,file.GetLength()); strHTML.clear(); strHTML.append(temp,temp+file.GetLength()); file.Close(); delete[] temp; return true; } } } CStdString strHTML1(strHTML); if (scrURL.m_post) { CStdString strOptions = url.GetOptions(); strOptions = strOptions.substr(1); url.SetOptions(""); if (!http.Post(url.Get(), strOptions, strHTML1)) return false; } else if (!http.Get(url.Get(), strHTML1)) return false; strHTML = strHTML1; if (scrURL.m_url.Find(".zip") > -1 ) { XFILE::CZipFile file; CStdString strBuffer; int iSize = file.UnpackFromMemory(strBuffer,strHTML,scrURL.m_isgz); if (iSize) { strHTML.clear(); strHTML.append(strBuffer.c_str(),strBuffer.data()+iSize); } } if (!scrURL.m_cache.IsEmpty()) { CStdString strCachePath; URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath, "scrapers/"+cacheContext+"/"+scrURL.m_cache, strCachePath); XFILE::CFile file; if (file.OpenForWrite(strCachePath,true)) file.Write(strHTML.data(),strHTML.size()); file.Close(); } return true; }
bool CPlexClient::ParseSections(PlexSectionParsing parser) { bool rtn = false; XFILE::CCurlFile plex; plex.SetBufferSize(32768*10); plex.SetTimeout(10); CURL curl(m_url); curl.SetFileName(curl.GetFileName() + "library/sections"); std::string strResponse; if (plex.Get(curl.Get(), strResponse)) { #if defined(PLEX_DEBUG_VERBOSE) if (parser == PlexSectionParsing::newSection || parser == PlexSectionParsing::checkSection) CLog::Log(LOGDEBUG, "CPlexClient::ParseSections %d, %s", parser, strResponse.c_str()); #endif if (parser == PlexSectionParsing::updateSection) { { CSingleLock lock(m_criticalMovies); m_movieSectionsContents.clear(); } { CSingleLock lock(m_criticalTVShow); m_showSectionsContents.clear(); } m_needUpdate = false; } TiXmlDocument xml; xml.Parse(strResponse.c_str()); TiXmlElement* MediaContainer = xml.RootElement(); if (MediaContainer) { const TiXmlElement* DirectoryNode = MediaContainer->FirstChildElement("Directory"); while (DirectoryNode) { PlexSectionsContent content; content.uuid = XMLUtils::GetAttribute(DirectoryNode, "uuid"); content.path = XMLUtils::GetAttribute(DirectoryNode, "path"); content.type = XMLUtils::GetAttribute(DirectoryNode, "type"); content.title = XMLUtils::GetAttribute(DirectoryNode, "title"); content.updatedAt = XMLUtils::GetAttribute(DirectoryNode, "updatedAt"); std::string key = XMLUtils::GetAttribute(DirectoryNode, "key"); content.section = "library/sections/" + key; content.thumb = XMLUtils::GetAttribute(DirectoryNode, "composite"); std::string art = XMLUtils::GetAttribute(DirectoryNode, "art"); if (m_local) content.art = art; else content.art = content.section + "/resources/" + URIUtils::GetFileName(art); if (content.type == "movie") { if (parser == PlexSectionParsing::checkSection) { CSingleLock lock(m_criticalMovies); for (const auto &contents : m_movieSectionsContents) { if (contents.uuid == content.uuid) { if (contents.updatedAt != content.updatedAt) { #if defined(PLEX_DEBUG_VERBOSE) CLog::Log(LOGDEBUG, "CPlexClient::ParseSections need update on %s:%s", m_serverName.c_str(), content.title.c_str()); #endif m_needUpdate = true; } } } } else { CSingleLock lock(m_criticalMovies); m_movieSectionsContents.push_back(content); } } else if (content.type == "show") { if (parser == PlexSectionParsing::checkSection) { CSingleLock lock(m_criticalTVShow); for (const auto &contents : m_showSectionsContents) { if (contents.uuid == content.uuid) { if (contents.updatedAt != content.updatedAt) { #if defined(PLEX_DEBUG_VERBOSE) CLog::Log(LOGDEBUG, "CPlexClient::ParseSections need update on %s:%s", m_serverName.c_str(), content.title.c_str()); #endif m_needUpdate = true; } } } } else { CSingleLock lock(m_criticalTVShow); m_showSectionsContents.push_back(content); } } else if (content.type == "artist") { if (parser == PlexSectionParsing::checkSection) { CSingleLock lock(m_criticalArtist); for (const auto &contents : m_artistSectionsContents) { if (contents.uuid == content.uuid) { if (contents.updatedAt != content.updatedAt) { #if defined(PLEX_DEBUG_VERBOSE) CLog::Log(LOGDEBUG, "CPlexClient::ParseSections need update on %s:%s", m_serverName.c_str(), content.title.c_str()); #endif m_needUpdate = true; } } } } else { CSingleLock lock(m_criticalArtist); m_artistSectionsContents.push_back(content); } } else if (content.type == "photo") { if (parser == PlexSectionParsing::checkSection) { CSingleLock lock(m_criticalPhoto); for (const auto &contents : m_photoSectionsContents) { if (contents.uuid == content.uuid) { if (contents.updatedAt != content.updatedAt) { #if defined(PLEX_DEBUG_VERBOSE) CLog::Log(LOGDEBUG, "CPlexClient::ParseSections need update on %s:%s", m_serverName.c_str(), content.title.c_str()); #endif m_needUpdate = true; } } } } else { CSingleLock lock(m_criticalPhoto); m_photoSectionsContents.push_back(content); } } else { CLog::Log(LOGDEBUG, "CPlexClient::ParseSections %s found unhandled content type %s", m_serverName.c_str(), content.type.c_str()); } DirectoryNode = DirectoryNode->NextSiblingElement("Directory"); } CLog::Log(LOGDEBUG, "CPlexClient::ParseSections %s found %d movie sections", m_serverName.c_str(), (int)m_movieSectionsContents.size()); CLog::Log(LOGDEBUG, "CPlexClient::ParseSections %s found %d shows sections", m_serverName.c_str(), (int)m_showSectionsContents.size()); CLog::Log(LOGDEBUG, "CPlexClient::ParseSections %s found %d artist sections", m_serverName.c_str(), (int)m_artistSectionsContents.size()); CLog::Log(LOGDEBUG, "CPlexClient::ParseSections %s found %d photo sections", m_serverName.c_str(), (int)m_photoSectionsContents.size()); rtn = true; } else { CLog::Log(LOGDEBUG, "CPlexClient::ParseSections no MediaContainer found"); } } else { // 401's are attempts to access a local server that is also in PMS // and these require an access token. Only local servers that are // not is PMS can be accessed via GDM. if (plex.GetResponseCode() != 401) CLog::Log(LOGDEBUG, "CPlexClient::ParseSections failed %s", strResponse.c_str()); rtn = false; } return rtn; }
bool CScraperUrl::Get(const SUrlEntry& scrURL, std::string& strHTML, XFILE::CCurlFile& http, const CStdString& cacheContext) { CURL url(scrURL.m_url); http.SetReferer(scrURL.m_spoof); CStdString strCachePath; if (scrURL.m_isgz) http.SetContentEncoding("gzip"); if (!scrURL.m_cache.IsEmpty()) { strCachePath = URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath, "scrapers/" + cacheContext + "/" + scrURL.m_cache); if (XFILE::CFile::Exists(strCachePath)) { XFILE::CFile file; XFILE::auto_buffer buffer; if (file.LoadFile(strCachePath, buffer)) { strHTML.assign(buffer.get(), buffer.length()); return true; } } } CStdString strHTML1(strHTML); if (scrURL.m_post) { CStdString strOptions = url.GetOptions(); strOptions = strOptions.substr(1); url.SetOptions(""); if (!http.Post(url.Get(), strOptions, strHTML1)) return false; } else if (!http.Get(url.Get(), strHTML1)) return false; strHTML = strHTML1; std::string fileCharset(http.GetServerReportedCharset()); if (scrURL.m_url.Find(".zip") > -1 ) { XFILE::CZipFile file; CStdString strBuffer; int iSize = file.UnpackFromMemory(strBuffer,strHTML,scrURL.m_isgz); if (iSize) { fileCharset.clear(); strHTML.clear(); strHTML.append(strBuffer.c_str(),strBuffer.data()+iSize); } } if (!fileCharset.empty() && fileCharset != "UTF-8") { std::string converted; if (g_charsetConverter.ToUtf8(fileCharset, strHTML, converted) && !converted.empty()) strHTML = converted; } if (!scrURL.m_cache.IsEmpty()) { CStdString strCachePath = URIUtils::AddFileToFolder(g_advancedSettings.m_cachePath, "scrapers/" + cacheContext + "/" + scrURL.m_cache); XFILE::CFile file; if (file.OpenForWrite(strCachePath,true)) file.Write(strHTML.data(),strHTML.size()); file.Close(); } return true; }