void MetaBox::addMdatItem(const std::uint32_t itemId,
FourCCInt type,
const String& name,
const std::uint64_t baseOffset)
{
addItem(itemId, type, name);
ItemLocation itemLocation;
itemLocation.setItemID(itemId);
itemLocation.setBaseOffset(baseOffset);
itemLocation.setConstructionMethod(ItemLocation::ConstructionMethod::FILE_OFFSET);
mItemLocationBox.addLocation(itemLocation);
}
void MetaBox::addIdatItem(const std::uint32_t itemId, FourCCInt type, const String& name, const Vector<uint8_t>& data)
{
const uint64_t offset = mItemDataBox.addData(data);
addItem(itemId, type, name);
ItemLocationExtent locationExtent;
locationExtent.mExtentOffset = offset;
locationExtent.mExtentLength = data.size();
ItemLocation itemLocation;
itemLocation.setItemID(itemId);
itemLocation.addExtent(locationExtent);
itemLocation.setConstructionMethod(ItemLocation::ConstructionMethod::IDAT_OFFSET);
mItemLocationBox.addLocation(itemLocation);
}
void ItemsManagerWorker::OnFirstTabReceived() {
QNetworkReply *reply = qobject_cast<QNetworkReply *>(QObject::sender());
QByteArray bytes = reply->readAll();
rapidjson::Document doc;
doc.Parse(bytes.constData());
int index = 0;
if (!doc.IsObject()) {
QLOG_ERROR() << "Can't even fetch first tab. Failed to update items.";
updating_ = false;
return;
}
if (!doc.HasMember("tabs") || doc["tabs"].Size() == 0) {
QLOG_WARN() << "There are no tabs, this should not happen, bailing out.";
updating_ = false;
return;
}
tabs_as_string_ = Util::RapidjsonSerialize(doc["tabs"]);
QLOG_DEBUG() << "Received tabs list, there are" << doc["tabs"].Size() << "tabs";
tabs_.clear();
for (auto &tab : doc["tabs"]) {
std::string label = tab["n"].GetString();
tabs_.push_back(label);
if (index > 0) {
ItemLocation location;
location.set_type(ItemLocationType::STASH);
location.set_tab_id(index);
location.set_tab_label(label);
if (!tab.HasMember("hidden") || !tab["hidden"].GetBool())
QueueRequest(MakeTabRequest(index), location);
}
++index;
}
ItemLocation first_tab_location;
first_tab_location.set_type(ItemLocationType::STASH);
first_tab_location.set_tab_id(0);
first_tab_location.set_tab_label(tabs_[0]);
if (!doc["tabs"][0].HasMember("hidden") || !doc["tabs"][0]["hidden"].GetBool())
ParseItems(&doc["items"], first_tab_location, doc.GetAllocator());
total_needed_ = queue_.size() + 1;
total_completed_ = 1;
FetchItems(kThrottleRequests - 1);
connect(signal_mapper_, SIGNAL(mapped(int)), this, SLOT(OnTabReceived(int)));
reply->deleteLater();
}
void MetaBox::addIloc(const std::uint32_t itemId,
const std::uint64_t offset,
const std::uint64_t length,
const std::uint64_t baseOffset)
{
ItemLocationExtent locationExtent;
locationExtent.mExtentOffset = offset;
locationExtent.mExtentLength = length;
ItemLocation itemLocation;
itemLocation.setItemID(itemId);
itemLocation.setBaseOffset(baseOffset);
itemLocation.addExtent(locationExtent);
mItemLocationBox.addLocation(itemLocation);
}
void ItemsManagerWorker::QueueRequest(const QNetworkRequest &request, const ItemLocation &location) {
QLOG_INFO() << "Queued" << location.GetHeader().c_str();
ItemsRequest items_request;
items_request.network_request = request;
items_request.id = queue_id_++;
items_request.location = location;
queue_.push(items_request);
}
void ItemsManagerWorker::OnCharacterListReceived() {
QNetworkReply *reply = qobject_cast<QNetworkReply *>(QObject::sender());
QByteArray bytes = reply->readAll();
rapidjson::Document doc;
doc.Parse(bytes.constData());
if (doc.HasParseError() || !doc.IsArray()) {
QLOG_ERROR() << "Received invalid reply instead of character list. The reply was"
<< bytes.constData();
if (doc.HasParseError()) {
QLOG_ERROR() << "The error was" << rapidjson::GetParseError_En(doc.GetParseError());
}
updating_ = false;
return;
}
QLOG_DEBUG() << "Received character list, there are" << doc.Size() << "characters";
for (auto &character : doc) {
if (!character.HasMember("league") || !character.HasMember("name") || !character["league"].IsString() || !character["name"].IsString()) {
QLOG_ERROR() << "Malformed character entry, the reply is most likely invalid" << bytes.constData();
continue;
}
if (character["league"].GetString() == league_) {
std::string name = character["name"].GetString();
ItemLocation location;
location.set_type(ItemLocationType::CHARACTER);
location.set_character(name);
QueueRequest(MakeCharacterRequest(name, location), location);
}
}
// Fetch a single tab and also request tabs list. We can fetch any tab here with tabs list
// appended, so prefer one that the user has already 'checked'. Default to index '1' which is
// first user visible tab.
first_fetch_tab_ = 1;
for (auto const &tab : tabs_) {
if (bo_manager_.GetRefreshChecked(tab)) {
first_fetch_tab_ = tab.get_tab_id();
break;
}
}
QNetworkReply *first_tab = network_manager_.get(MakeTabRequest(first_fetch_tab_, ItemLocation(), true));
connect(first_tab, SIGNAL(finished()), this, SLOT(OnFirstTabReceived()));
reply->deleteLater();
}
QNetworkRequest TabCache::Request(const QUrl &url, const ItemLocation &location, Flags flags) {
QNetworkRequest request{url};
// The cache policy exists so it can override normal behavior for a given refresh.
// Based on the current policy we may ignore refresh requests, or force refreshes
// even if none was specifed with 'flags'.
switch (cache_policy_) {
case DefaultCache:
// This is the default policy, where we honor cache policy as specified by 'flags'
// Evict this request from the cache if refresh is requested
if (flags.testFlag(Refresh))
remove(url);
break;
case AlwaysCache:
// By default we always try to hit in the cache, so this policy just allows
// the normal cache mechanism to do its job and it will basically grab everything
// from the cache that is available for all network requests
break;
case NeverCache:
// We've already fully flushed the cache in OnPolicyUpdate, so nothing to do here.
break;
case ManualCache:
// The case involves refreshing only an explicitily specified set of named tabs, customers
// use AddManualRefresh to indicate what set of tabs to refresh before triggering a refresh
// all other network requests will come from the cache if available
if (location.IsValid() && manual_refresh_.count(location.GetUniqueHash()))
remove(url);
break;
default:
QLOG_ERROR() << "TabCache::Request Failed to handle all cache policy cases";
break;
};
// At this point we've evicted any request that should be refreshed, so we always
// tell the 'real' request to prefer but not require the entry be in the cache.
// If it is not in the cache it will be fetched from the network regardless.
request.setAttribute(QNetworkRequest::CacheSaveControlAttribute, QNetworkRequest::PreferCache);
return request;
}
QNetworkRequest ItemsManagerWorker::MakeCharacterRequest(const std::string &name, const ItemLocation &location) {
QUrlQuery query;
query.addQueryItem("character", name.c_str());
query.addQueryItem("accountName", account_name_.c_str());
QUrl url(kCharacterItemsUrl);
url.setQuery(query);
TabCache::Flags flags;
if (!location.IsValid() || bo_manager_.GetRefreshChecked(location))
flags |= TabCache::Refresh;
return Request(url, location, flags);
}
void ItemsManagerWorker::OnCharacterListReceived() {
QNetworkReply *reply = qobject_cast<QNetworkReply *>(QObject::sender());
QByteArray bytes = reply->readAll();
rapidjson::Document doc;
doc.Parse(bytes.constData());
if (doc.HasParseError() || !doc.IsArray()) {
QLOG_ERROR() << "Received invalid reply instead of character list. The reply was"
<< bytes.constData();
if (doc.HasParseError()) {
QLOG_ERROR() << "The error was" << rapidjson::GetParseError_En(doc.GetParseError());
}
updating_ = false;
StatusFinished();
return;
}
QLOG_INFO() << "Received character list, there are" << doc.Size() << "characters";
for (auto &character : doc) {
if (!character.HasMember("league") || !character.HasMember("name") || !character["league"].IsString() || !character["name"].IsString()) {
QLOG_ERROR() << "Malformed character entry, the reply is most likely invalid" << bytes.constData();
continue;
}
if (character["league"].GetString() == league_) {
std::string name = character["name"].GetString();
ItemLocation location;
location.set_type(ItemLocationType::CHARACTER);
location.set_character(name);
QueueRequest(MakeCharacterRequest(name), location);
}
}
// now get first tab and tab list
QNetworkReply *first_tab = network_manager_.get(MakeTabRequest(0, true));
connect(first_tab, SIGNAL(finished()), this, SLOT(OnFirstTabReceived()));
reply->deleteLater();
}
QNetworkRequest ItemsManagerWorker::MakeTabRequest(int tab_index, const ItemLocation &location, bool tabs) {
QUrlQuery query;
query.addQueryItem("league", league_.c_str());
query.addQueryItem("tabs", tabs ? "1" : "0");
query.addQueryItem("tabIndex", QString::number(tab_index));
query.addQueryItem("accountName", account_name_.c_str());
QUrl url(kStashItemsUrl);
url.setQuery(query);
TabCache::Flags flags;
if (tabs) flags |= TabCache::Refresh;
if (!location.IsValid() || bo_manager_.GetRefreshChecked(location))
flags |= TabCache::Refresh;
return Request(url, location, flags);
}
void BuyoutManager::SetRefreshLocked(const ItemLocation &loc) {
refresh_locked_.insert(loc.GetUniqueHash());
}
bool BuyoutManager::GetRefreshLocked(const ItemLocation &loc) const {
return refresh_locked_.count(loc.GetUniqueHash());
}
bool BuyoutManager::GetRefreshChecked(const ItemLocation &loc) const {
auto it = refresh_checked_.find(loc.GetUniqueHash());
bool refresh_checked = (it != refresh_checked_.end()) ? it->second : true;
return (refresh_checked || GetRefreshLocked(loc));
}
void BuyoutManager::SetRefreshChecked(const ItemLocation &loc, bool value) {
save_needed_ = true;
refresh_checked_[loc.GetUniqueHash()] = value;
}
void ItemsManagerWorker::OnFirstTabReceived() {
QNetworkReply *reply = qobject_cast<QNetworkReply *>(QObject::sender());
QByteArray bytes = reply->readAll();
rapidjson::Document doc;
doc.Parse(bytes.constData());
int index = 0;
if (!doc.IsObject()) {
QLOG_ERROR() << "Can't even fetch first tab. Failed to update items.";
updating_ = false;
StatusFinished();
return;
}
if (!doc.HasMember("tabs") || doc["tabs"].Size() == 0) {
QLOG_WARN() << "There are no tabs, this should not happen, bailing out.";
updating_ = false;
StatusFinished();
return;
}
tabs_as_string_ = Util::RapidjsonSerialize(doc["tabs"]);
QLOG_INFO() << "Received tabs list, there are" << doc["tabs"].Size() << "tabs";
// Setup tab exclusions
std::string exclusions = data_manager_.Get("tab_exclusions");
QList<QRegularExpression> expressions;
rapidjson::Document exclusionDoc;
exclusionDoc.Parse(exclusions.c_str());
if (exclusionDoc.IsArray()) {
for (auto &excl : exclusionDoc) {
QRegularExpression expr(excl.GetString());
if (!expr.pattern().isEmpty() && expr.isValid()) expressions.append(expr);
}
}
int tabsParsed = 0;
tabs_.clear();
for (auto &tab : doc["tabs"]) {
std::string label = tab["n"].GetString();
bool skip = false;
for (QRegularExpression expr : expressions) {
QRegularExpressionMatch match = expr.match(QString::fromStdString(label));
if (match.isValid() && match.hasMatch()) {
skip = true;
break;
}
}
if (!skip) {
tabs_.push_back(label);
ItemLocation location;
location.set_type(ItemLocationType::STASH);
location.set_tab_id(index);
location.set_tab_label(label);
if (index == 0) {
// We already have this tab (it's the first one we get).
// Parse the items now that we have them
if (!doc["tabs"][0].HasMember("hidden") || !doc["tabs"][0]["hidden"].GetBool()) {
ParseItems(&doc["items"], location, doc.GetAllocator());
++tabsParsed;
}
}
else if (!tab.HasMember("hidden") || !tab["hidden"].GetBool()) {
// If it's not hidden, then we need to download
QueueRequest(MakeTabRequest(index), location);
}
}
++index;
}
if (tabs_.size() == 0) {
QLOG_WARN() << "There are no tabs to be downloaded. Try clearing your tab exclusions list.";
updating_ = false;
StatusFinished();
return;
}
// tabsParsed will be 1 if the first tab was included in the initial download
total_needed_ = queue_.size() + tabsParsed;
total_completed_ = tabsParsed;
FetchItems(kThrottleRequests - tabsParsed);
connect(signal_mapper_, SIGNAL(mapped(int)), this, SLOT(OnTabReceived(int)));
reply->deleteLater();
}
void ItemsManagerWorker::OnFirstTabReceived() {
QNetworkReply *reply = qobject_cast<QNetworkReply *>(QObject::sender());
QByteArray bytes = reply->readAll();
rapidjson::Document doc;
doc.Parse(bytes.constData());
int index = 0;
if (!doc.IsObject()) {
QLOG_ERROR() << "Can't even fetch first tab. Failed to update items.";
updating_ = false;
return;
}
if (!doc.HasMember("tabs") || doc["tabs"].Size() == 0) {
QLOG_WARN() << "There are no tabs, this should not happen, bailing out.";
updating_ = false;
return;
}
tabs_as_string_ = Util::RapidjsonSerialize(doc["tabs"]);
QLOG_INFO() << "Received tabs list, there are" << doc["tabs"].Size() << "tabs";
// Setup tab exclusions
std::string exclusions = data_manager_.Get("tab_exclusions");
QList<QRegularExpression> expressions;
rapidjson::Document exclusionDoc;
exclusionDoc.Parse(exclusions.c_str());
if (exclusionDoc.IsArray()) {
for (auto &excl : exclusionDoc) {
expressions.append(QRegularExpression(excl.GetString()));
}
}
tabs_.clear();
for (auto &tab : doc["tabs"]) {
std::string label = tab["n"].GetString();
bool skip = false;
for (QRegularExpression expr : expressions) {
QRegularExpressionMatch match = expr.match(QString::fromStdString(label));
if (match.isValid() && match.hasMatch()) {
skip = true;
break;
}
}
tabs_.push_back(label);
if (index > 0 || skip) {
ItemLocation location;
location.set_type(ItemLocationType::STASH);
location.set_tab_id(index);
location.set_tab_label(label);
if (!tab.HasMember("hidden") || !tab["hidden"].GetBool())
QueueRequest(MakeTabRequest(skip ? 0 : index), location);
}
++index;
}
ItemLocation first_tab_location;
first_tab_location.set_type(ItemLocationType::STASH);
first_tab_location.set_tab_id(0);
first_tab_location.set_tab_label(tabs_[0]);
if (!doc["tabs"][0].HasMember("hidden") || !doc["tabs"][0]["hidden"].GetBool())
ParseItems(&doc["items"], first_tab_location, doc.GetAllocator());
total_needed_ = queue_.size() + 1;
total_completed_ = 1;
FetchItems(kThrottleRequests - 1);
connect(signal_mapper_, SIGNAL(mapped(int)), this, SLOT(OnTabReceived(int)));
reply->deleteLater();
}
void TabCache::AddManualRefresh( const ItemLocation &location) {
if (location.IsValid())
manual_refresh_.insert(location.GetUniqueHash());
}
