void HLDirectory::GetItem(UInt32 inIndex, FSRef& outFSRef)
{
FetchItems();
ItemList::iterator theIterator = mItems->begin();
std::advance(theIterator, inIndex);
ThrowIf(theIterator == mItems->end(), CAException(paramErr), "HLDirectory::GetItem: index out of range");
memcpy(&outFSRef, &(*theIterator), sizeof(FSRef));
}
lldb::SBQueueItem
GetPendingItemAtIndex (uint32_t idx)
{
SBQueueItem result;
FetchItems();
if (m_pending_items_fetched && idx < m_pending_items.size())
{
result.SetQueueItem (m_pending_items[idx]);
}
return result;
}
uint32_t
GetNumItems ()
{
uint32_t result = 0;
FetchItems();
if (m_queue_items_fetched)
{
result = m_items.size();
}
return result;
}
void ItemsManagerWorker::OnFirstTabReceived() {
QNetworkReply *reply = qobject_cast<QNetworkReply *>(QObject::sender());
QByteArray bytes = reply->readAll();
rapidjson::Document doc;
doc.Parse(bytes.constData());
int index = 0;
if (!doc.IsObject()) {
QLOG_ERROR() << "Can't even fetch first tab. Failed to update items.";
updating_ = false;
return;
}
if (!doc.HasMember("tabs") || doc["tabs"].Size() == 0) {
QLOG_WARN() << "There are no tabs, this should not happen, bailing out.";
updating_ = false;
return;
}
tabs_as_string_ = Util::RapidjsonSerialize(doc["tabs"]);
QLOG_DEBUG() << "Received tabs list, there are" << doc["tabs"].Size() << "tabs";
tabs_.clear();
for (auto &tab : doc["tabs"]) {
std::string label = tab["n"].GetString();
tabs_.push_back(label);
if (index > 0) {
ItemLocation location;
location.set_type(ItemLocationType::STASH);
location.set_tab_id(index);
location.set_tab_label(label);
if (!tab.HasMember("hidden") || !tab["hidden"].GetBool())
QueueRequest(MakeTabRequest(index), location);
}
++index;
}
ItemLocation first_tab_location;
first_tab_location.set_type(ItemLocationType::STASH);
first_tab_location.set_tab_id(0);
first_tab_location.set_tab_label(tabs_[0]);
if (!doc["tabs"][0].HasMember("hidden") || !doc["tabs"][0]["hidden"].GetBool())
ParseItems(&doc["items"], first_tab_location, doc.GetAllocator());
total_needed_ = queue_.size() + 1;
total_completed_ = 1;
FetchItems(kThrottleRequests - 1);
connect(signal_mapper_, SIGNAL(mapped(int)), this, SLOT(OnTabReceived(int)));
reply->deleteLater();
}
void ItemsManagerWorker::OnFirstTabReceived() {
QNetworkReply *reply = qobject_cast<QNetworkReply *>(QObject::sender());
QByteArray bytes = reply->readAll();
rapidjson::Document doc;
doc.Parse(bytes.constData());
if (!doc.IsObject()) {
QLOG_ERROR() << "Can't even fetch first tab. Failed to update items.";
updating_ = false;
return;
}
if (!doc.HasMember("tabs") || doc["tabs"].Size() == 0) {
QLOG_WARN() << "There are no tabs, this should not happen, bailing out.";
updating_ = false;
return;
}
tabs_as_string_ = Util::RapidjsonSerialize(doc["tabs"]);
QLOG_DEBUG() << "Received tabs list, there are" << doc["tabs"].Size() << "tabs";
tabs_.clear();
// Create tab location objects
for (auto &tab : doc["tabs"]) {
std::string label = tab["n"].GetString();
auto index = tab["i"].GetInt();
// Ignore hidden locations
if (!doc["tabs"][index].HasMember("hidden") || !doc["tabs"][index]["hidden"].GetBool())
tabs_.push_back(ItemLocation(index, label, ItemLocationType::STASH));
}
// Immediately parse items received from this tab (first_fetch_tab_) and Queue requests for the others
for (auto const &tab: tabs_) {
auto index = tab.get_tab_id();
if (index == first_fetch_tab_) {
ParseItems(&doc["items"], tab, doc.GetAllocator());
} else {
QueueRequest(MakeTabRequest(index, tab), tab);
}
}
total_needed_ = queue_.size() + 1;
total_completed_ = 1;
total_cached_ = reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool() ? 1:0;
FetchItems(kThrottleRequests - 1);
connect(signal_mapper_, SIGNAL(mapped(int)), this, SLOT(OnTabReceived(int)));
reply->deleteLater();
}
void ItemsManagerWorker::OnTabReceived(int request_id) {
if (!replies_.count(request_id)) {
QLOG_WARN() << "Received a reply for request" << request_id << "that was not requested.";
return;
}
ItemsReply reply = replies_[request_id];
QLOG_INFO() << "Received a reply for" << reply.request.location.GetHeader().c_str();
QByteArray bytes = reply.network_reply->readAll();
rapidjson::Document doc;
doc.Parse(bytes.constData());
bool error = false;
if (!doc.IsObject()) {
QLOG_WARN() << request_id << "got a non-object response";
error = true;
} else if (doc.HasMember("error")) {
// this can happen if user is browsing stash in background and we can't know about it
QLOG_WARN() << request_id << "got 'error' instead of stash tab contents";
error = true;
}
// re-queue a failed request
if (error)
QueueRequest(reply.request.network_request, reply.request.location);
++requests_completed_;
if (!error)
++total_completed_;
bool throttled = false;
if (requests_completed_ == requests_needed_ && queue_.size() > 0) {
throttled = true;
QLOG_INFO() << "Sleeping one minute to prevent throttling.";
QTimer::singleShot(kThrottleSleep * 1000, this, SLOT(FetchItems()));
}
CurrentStatusUpdate status = CurrentStatusUpdate();
status.state = throttled ? ProgramState::ItemsPaused : ProgramState::ItemsReceive;
status.progress = total_completed_;
status.total = total_needed_;
if (total_completed_ == total_needed_)
status.state = ProgramState::ItemsCompleted;
emit StatusUpdate(status);
if (error)
return;
ParseItems(&doc["items"], reply.request.location, doc.GetAllocator());
if (total_completed_ == total_needed_) {
// all requests completed
emit ItemsRefreshed(items_, tabs_, false);
// since we build items_as_string_ in a hackish way inside ParseItems last character will either be
// ' ' when no items were parsed or ',' when at least one item is parsed, and the first character is '['
items_as_string_[items_as_string_.size() - 1] = ']';
// DataManager is thread safe so it's ok to call it here
data_manager_.Set("items", items_as_string_);
data_manager_.Set("tabs", tabs_as_string_);
updating_ = false;
QLOG_INFO() << "Finished updating stash.";
// if we're at the verge of getting throttled, sleep so we don't
if (requests_completed_ == kThrottleRequests)
QTimer::singleShot(kThrottleSleep, this, SLOT(PreserveSelectedCharacter()));
else
PreserveSelectedCharacter();
}
reply.network_reply->deleteLater();
}
void ItemsManagerWorker::OnFirstTabReceived() {
QNetworkReply *reply = qobject_cast<QNetworkReply *>(QObject::sender());
QByteArray bytes = reply->readAll();
rapidjson::Document doc;
doc.Parse(bytes.constData());
int index = 0;
if (!doc.IsObject()) {
QLOG_ERROR() << "Can't even fetch first tab. Failed to update items.";
updating_ = false;
StatusFinished();
return;
}
if (!doc.HasMember("tabs") || doc["tabs"].Size() == 0) {
QLOG_WARN() << "There are no tabs, this should not happen, bailing out.";
updating_ = false;
StatusFinished();
return;
}
tabs_as_string_ = Util::RapidjsonSerialize(doc["tabs"]);
QLOG_INFO() << "Received tabs list, there are" << doc["tabs"].Size() << "tabs";
// Setup tab exclusions
std::string exclusions = data_manager_.Get("tab_exclusions");
QList<QRegularExpression> expressions;
rapidjson::Document exclusionDoc;
exclusionDoc.Parse(exclusions.c_str());
if (exclusionDoc.IsArray()) {
for (auto &excl : exclusionDoc) {
QRegularExpression expr(excl.GetString());
if (!expr.pattern().isEmpty() && expr.isValid()) expressions.append(expr);
}
}
int tabsParsed = 0;
tabs_.clear();
for (auto &tab : doc["tabs"]) {
std::string label = tab["n"].GetString();
bool skip = false;
for (QRegularExpression expr : expressions) {
QRegularExpressionMatch match = expr.match(QString::fromStdString(label));
if (match.isValid() && match.hasMatch()) {
skip = true;
break;
}
}
if (!skip) {
tabs_.push_back(label);
ItemLocation location;
location.set_type(ItemLocationType::STASH);
location.set_tab_id(index);
location.set_tab_label(label);
if (index == 0) {
// We already have this tab (it's the first one we get).
// Parse the items now that we have them
if (!doc["tabs"][0].HasMember("hidden") || !doc["tabs"][0]["hidden"].GetBool()) {
ParseItems(&doc["items"], location, doc.GetAllocator());
++tabsParsed;
}
}
else if (!tab.HasMember("hidden") || !tab["hidden"].GetBool()) {
// If it's not hidden, then we need to download
QueueRequest(MakeTabRequest(index), location);
}
}
++index;
}
if (tabs_.size() == 0) {
QLOG_WARN() << "There are no tabs to be downloaded. Try clearing your tab exclusions list.";
updating_ = false;
StatusFinished();
return;
}
// tabsParsed will be 1 if the first tab was included in the initial download
total_needed_ = queue_.size() + tabsParsed;
total_completed_ = tabsParsed;
FetchItems(kThrottleRequests - tabsParsed);
connect(signal_mapper_, SIGNAL(mapped(int)), this, SLOT(OnTabReceived(int)));
reply->deleteLater();
}
UInt32 HLDirectory::GetNumberItems()
{
FetchItems();
return mItems->size();
}
void ItemsManagerWorker::OnFirstTabReceived() {
QNetworkReply *reply = qobject_cast<QNetworkReply *>(QObject::sender());
QByteArray bytes = reply->readAll();
rapidjson::Document doc;
doc.Parse(bytes.constData());
int index = 0;
if (!doc.IsObject()) {
QLOG_ERROR() << "Can't even fetch first tab. Failed to update items.";
updating_ = false;
return;
}
if (!doc.HasMember("tabs") || doc["tabs"].Size() == 0) {
QLOG_WARN() << "There are no tabs, this should not happen, bailing out.";
updating_ = false;
return;
}
tabs_as_string_ = Util::RapidjsonSerialize(doc["tabs"]);
QLOG_INFO() << "Received tabs list, there are" << doc["tabs"].Size() << "tabs";
// Setup tab exclusions
std::string exclusions = data_manager_.Get("tab_exclusions");
QList<QRegularExpression> expressions;
rapidjson::Document exclusionDoc;
exclusionDoc.Parse(exclusions.c_str());
if (exclusionDoc.IsArray()) {
for (auto &excl : exclusionDoc) {
expressions.append(QRegularExpression(excl.GetString()));
}
}
tabs_.clear();
for (auto &tab : doc["tabs"]) {
std::string label = tab["n"].GetString();
bool skip = false;
for (QRegularExpression expr : expressions) {
QRegularExpressionMatch match = expr.match(QString::fromStdString(label));
if (match.isValid() && match.hasMatch()) {
skip = true;
break;
}
}
tabs_.push_back(label);
if (index > 0 || skip) {
ItemLocation location;
location.set_type(ItemLocationType::STASH);
location.set_tab_id(index);
location.set_tab_label(label);
if (!tab.HasMember("hidden") || !tab["hidden"].GetBool())
QueueRequest(MakeTabRequest(skip ? 0 : index), location);
}
++index;
}
ItemLocation first_tab_location;
first_tab_location.set_type(ItemLocationType::STASH);
first_tab_location.set_tab_id(0);
first_tab_location.set_tab_label(tabs_[0]);
if (!doc["tabs"][0].HasMember("hidden") || !doc["tabs"][0]["hidden"].GetBool())
ParseItems(&doc["items"], first_tab_location, doc.GetAllocator());
total_needed_ = queue_.size() + 1;
total_completed_ = 1;
FetchItems(kThrottleRequests - 1);
connect(signal_mapper_, SIGNAL(mapped(int)), this, SLOT(OnTabReceived(int)));
reply->deleteLater();
}
void ItemsManagerWorker::OnTabReceived(int request_id) {
if (!replies_.count(request_id)) {
QLOG_WARN() << "Received a reply for request" << request_id << "that was not requested.";
return;
}
ItemsReply reply = replies_[request_id];
bool cache_status = reply.network_reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool();
if (cache_status) {
QLOG_DEBUG() << "Received a cached reply for" << reply.request.location.GetHeader().c_str();
++cached_requests_completed_;
++total_cached_;
} else {
QLOG_DEBUG() << "Received a reply for" << reply.request.location.GetHeader().c_str();
}
QByteArray bytes = reply.network_reply->readAll();
rapidjson::Document doc;
doc.Parse(bytes.constData());
bool error = false;
if (!doc.IsObject()) {
QLOG_WARN() << request_id << "got a non-object response";
error = true;
} else if (doc.HasMember("error")) {
// this can happen if user is browsing stash in background and we can't know about it
QLOG_WARN() << request_id << "got 'error' instead of stash tab contents";
error = true;
}
// re-queue a failed request
if (error) {
// We can 'cache' error response document so make sure we remove it
// before reque
tab_cache_->remove(reply.request.network_request.url());
QueueRequest(reply.request.network_request, reply.request.location);
}
++requests_completed_;
if (!error)
++total_completed_;
bool throttled = false;
if (requests_completed_ == requests_needed_ && queue_.size() > 0) {
if (cached_requests_completed_ > 0) {
// We basically don't want cached requests to count against throttle limit
// so if we did get any cached requests fetch up to that number without a
// large delay
QTimer::singleShot(1, [&]() {
FetchItems(cached_requests_completed_);
});
} else {
throttled = true;
QLOG_DEBUG() << "Sleeping one minute to prevent throttling.";
QTimer::singleShot(kThrottleSleep * 1000, this, SLOT(FetchItems()));
}
}
CurrentStatusUpdate status = CurrentStatusUpdate();
status.state = throttled ? ProgramState::ItemsPaused : ProgramState::ItemsReceive;
status.progress = total_completed_;
status.total = total_needed_;
status.cached = total_cached_;
if (total_completed_ == total_needed_)
status.state = ProgramState::ItemsCompleted;
emit StatusUpdate(status);
if (error)
return;
ParseItems(&doc["items"], reply.request.location, doc.GetAllocator());
if (total_completed_ == total_needed_) {
// It's possible that we receive character vs stash tabs out of order, or users
// move items around in a tab and we get them in a different order. For
// consistency we want to present the tab data in a deterministic way to the rest
// of the application. Especially so we don't try to update shop when nothing actually
// changed. So sort items_ here before emitting and then generate
// item list as strings.
std::sort(begin(items_), end(items_), [](const std::shared_ptr<Item> &a, const std::shared_ptr<Item> &b) {
return b->location() < a->location();
});
QStringList tmp;
for (auto const &item: items_) {
tmp.push_back(item->json().c_str());
}
auto items_as_string = std::string("[") + tmp.join(",").toStdString() + "]";
// all requests completed
emit ItemsRefreshed(items_, tabs_, false);
// DataStore is thread safe so it's ok to call it here
data_.Set("items", items_as_string);
data_.Set("tabs", tabs_as_string_);
updating_ = false;
QLOG_DEBUG() << "Finished updating stash.";
// if we're at the verge of getting throttled, sleep so we don't
if (requests_completed_ == kThrottleRequests)
QTimer::singleShot(kThrottleSleep, this, SLOT(PreserveSelectedCharacter()));
else
PreserveSelectedCharacter();
}
reply.network_reply->deleteLater();
}