void LLInventoryModelBackgroundFetch::start(const LLUUID& id, BOOL recursive)
{
LLViewerInventoryCategory* cat = gInventory.getCategory(id);
if (cat || (id.isNull() && !isEverythingFetched()))
{ // it's a folder, do a bulk fetch
LL_DEBUGS("InventoryFetch") << "Start fetching category: " << id << ", recursive: " << recursive << LL_ENDL;
mFolderFetchActive = true;
if (id.isNull())
{
if (!mRecursiveInventoryFetchStarted)
{
mRecursiveInventoryFetchStarted |= recursive;
mFetchQueue.push_back(FetchQueueInfo(gInventory.getRootFolderID(), recursive));
gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
mBackgroundFetchActive = TRUE;
}
if (!mRecursiveLibraryFetchStarted)
{
mRecursiveLibraryFetchStarted |= recursive;
mFetchQueue.push_back(FetchQueueInfo(gInventory.getLibraryRootFolderID(), recursive));
gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
mBackgroundFetchActive = TRUE;
}
}
else
{
// Specific folder requests go to front of queue.
if (mFetchQueue.empty() || mFetchQueue.front().mUUID != id)
{
mFetchQueue.push_front(FetchQueueInfo(id, recursive));
gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
mBackgroundFetchActive = TRUE;
}
if (id == gInventory.getLibraryRootFolderID())
{
mRecursiveLibraryFetchStarted |= recursive;
}
if (id == gInventory.getRootFolderID())
{
mRecursiveInventoryFetchStarted |= recursive;
}
}
}
else if (LLViewerInventoryItem* itemp = gInventory.getItem(id))
{
if (!itemp->mIsComplete && (mFetchQueue.empty() || mFetchQueue.front().mUUID != id))
{
mBackgroundFetchActive = TRUE;
mFetchQueue.push_front(FetchQueueInfo(id, false, false));
gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
}
}
}
void LLInventoryModelBackgroundFetch::findLostItems()
{
mBackgroundFetchActive = TRUE;
mFolderFetchActive = true;
mFetchQueue.push_back(FetchQueueInfo(LLUUID::null, TRUE));
gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
}
void LLInventoryModelBackgroundFetch::start(const LLUUID& cat_id, BOOL recursive)
{
if (!mAllFoldersFetched)
{
LL_DEBUGS("InventoryFetch") << "Start fetching category: " << cat_id << ", recursive: " << recursive << LL_ENDL;
mBackgroundFetchActive = TRUE;
if (cat_id.isNull())
{
if (!mRecursiveInventoryFetchStarted)
{
mRecursiveInventoryFetchStarted |= recursive;
mFetchQueue.push_back(FetchQueueInfo(gInventory.getRootFolderID(), recursive));
gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
}
if (!mRecursiveLibraryFetchStarted)
{
mRecursiveLibraryFetchStarted |= recursive;
mFetchQueue.push_back(FetchQueueInfo(gInventory.getLibraryRootFolderID(), recursive));
gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
}
}
else
{
// Specific folder requests go to front of queue.
if (mFetchQueue.empty() || mFetchQueue.front().mCatUUID != cat_id)
{
mFetchQueue.push_front(FetchQueueInfo(cat_id, recursive));
gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL);
}
if (cat_id == gInventory.getLibraryRootFolderID())
{
mRecursiveLibraryFetchStarted |= recursive;
}
if (cat_id == gInventory.getRootFolderID())
{
mRecursiveInventoryFetchStarted |= recursive;
}
}
}
}
// Bundle up a bunch of requests to send all at once.
// static
void LLInventoryModelBackgroundFetch::bulkFetch(std::string url)
{
//Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped.
//If there are items in mFetchQueue, we want to check the time since the last bulkFetch was
//sent. If it exceeds our retry time, go ahead and fire off another batch.
//Stopbackgroundfetch will be run from the Responder instead of here.
S16 max_concurrent_fetches=8;
F32 new_min_time = 0.5f; //HACK! Clean this up when old code goes away entirely.
if (mMinTimeBetweenFetches < new_min_time)
{
mMinTimeBetweenFetches=new_min_time; //HACK! See above.
}
if (gDisconnected ||
(mBulkFetchCount > max_concurrent_fetches) ||
(mFetchTimer.getElapsedTimeF32() < mMinTimeBetweenFetches))
{
return; // just bail if we are disconnected
}
U32 folder_count=0;
U32 max_batch_size=5;
U32 sort_order = gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1;
uuid_vec_t recursive_cats;
LLSD body;
LLSD body_lib;
while (!(mFetchQueue.empty()) && (folder_count < max_batch_size))
{
const FetchQueueInfo& fetch_info = mFetchQueue.front();
const LLUUID &cat_id = fetch_info.mCatUUID;
if (cat_id.isNull()) //DEV-17797
{
LLSD folder_sd;
folder_sd["folder_id"] = LLUUID::null.asString();
folder_sd["owner_id"] = gAgent.getID();
folder_sd["sort_order"] = (LLSD::Integer)sort_order;
folder_sd["fetch_folders"] = (LLSD::Boolean)FALSE;
folder_sd["fetch_items"] = (LLSD::Boolean)TRUE;
body["folders"].append(folder_sd);
folder_count++;
}
else
{
const LLViewerInventoryCategory* cat = gInventory.getCategory(cat_id);
if (cat)
{
if (LLViewerInventoryCategory::VERSION_UNKNOWN == cat->getVersion())
{
LLSD folder_sd;
folder_sd["folder_id"] = cat->getUUID();
folder_sd["owner_id"] = cat->getOwnerID();
folder_sd["sort_order"] = (LLSD::Integer)sort_order;
folder_sd["fetch_folders"] = TRUE; //(LLSD::Boolean)sFullFetchStarted;
folder_sd["fetch_items"] = (LLSD::Boolean)TRUE;
if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID())
body_lib["folders"].append(folder_sd);
else
body["folders"].append(folder_sd);
folder_count++;
}
// May already have this folder, but append child folders to list.
if (fetch_info.mRecursive)
{
LLInventoryModel::cat_array_t* categories;
LLInventoryModel::item_array_t* items;
gInventory.getDirectDescendentsOf(cat->getUUID(), categories, items);
for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
it != categories->end();
++it)
{
mFetchQueue.push_back(FetchQueueInfo((*it)->getUUID(), fetch_info.mRecursive));
}
}
}
}
if (fetch_info.mRecursive)
recursive_cats.push_back(cat_id);
mFetchQueue.pop_front();
}
if (folder_count > 0)
{
mBulkFetchCount++;
if (body["folders"].size())
{
LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(body, recursive_cats);
LLHTTPClient::post(url, body, fetcher, 300.0);
}
if (body_lib["folders"].size())
{
std::string url_lib = gAgent.getRegion()->getCapability("FetchLibDescendents2");
LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(body_lib, recursive_cats);
LLHTTPClient::post(url_lib, body_lib, fetcher, 300.0);
}
mFetchTimer.reset();
}
else if (isBulkFetchProcessingComplete())
{
setAllFoldersFetched();
}
}
void LLInventoryModelBackgroundFetch::backgroundFetch()
{
if (mBackgroundFetchActive && gAgent.getRegion())
{
// If we'll be using the capability, we'll be sending batches and the background thing isn't as important.
std::string url = gAgent.getRegion()->getCapability("FetchInventoryDescendents2");
if (gSavedSettings.getBOOL("UseHTTPInventory") && !url.empty())
{
bulkFetch(url);
return;
}
#if 1
//--------------------------------------------------------------------------------
// DEPRECATED OLD CODE
//
// No more categories to fetch, stop fetch process.
if (mFetchQueue.empty())
{
llinfos << "Inventory fetch completed" << llendl;
setAllFoldersFetched();
return;
}
F32 fast_fetch_time = lerp(mMinTimeBetweenFetches, mMaxTimeBetweenFetches, 0.1f);
F32 slow_fetch_time = lerp(mMinTimeBetweenFetches, mMaxTimeBetweenFetches, 0.5f);
if (mTimelyFetchPending && mFetchTimer.getElapsedTimeF32() > slow_fetch_time)
{
// Double timeouts on failure.
mMinTimeBetweenFetches = llmin(mMinTimeBetweenFetches * 2.f, 10.f);
mMaxTimeBetweenFetches = llmin(mMaxTimeBetweenFetches * 2.f, 120.f);
llinfos << "Inventory fetch times grown to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << llendl;
// fetch is no longer considered "timely" although we will wait for full time-out.
mTimelyFetchPending = FALSE;
}
while(1)
{
if (mFetchQueue.empty())
{
break;
}
if(gDisconnected)
{
// Just bail if we are disconnected.
break;
}
const FetchQueueInfo info = mFetchQueue.front();
LLViewerInventoryCategory* cat = gInventory.getCategory(info.mCatUUID);
// Category has been deleted, remove from queue.
if (!cat)
{
mFetchQueue.pop_front();
continue;
}
if (mFetchTimer.getElapsedTimeF32() > mMinTimeBetweenFetches &&
LLViewerInventoryCategory::VERSION_UNKNOWN == cat->getVersion())
{
// Category exists but has no children yet, fetch the descendants
// for now, just request every time and rely on retry timer to throttle.
if (cat->fetch())
{
mFetchTimer.reset();
mTimelyFetchPending = TRUE;
}
else
{
// The catagory also tracks if it has expired and here it says it hasn't
// yet. Get out of here because nothing is going to happen until we
// update the timers.
break;
}
}
// Do I have all my children?
else if (gInventory.isCategoryComplete(info.mCatUUID))
{
// Finished with this category, remove from queue.
mFetchQueue.pop_front();
// Add all children to queue.
LLInventoryModel::cat_array_t* categories;
LLInventoryModel::item_array_t* items;
gInventory.getDirectDescendentsOf(cat->getUUID(), categories, items);
for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
it != categories->end();
++it)
{
mFetchQueue.push_back(FetchQueueInfo((*it)->getUUID(),info.mRecursive));
}
// We received a response in less than the fast time.
if (mTimelyFetchPending && mFetchTimer.getElapsedTimeF32() < fast_fetch_time)
{
// Shrink timeouts based on success.
mMinTimeBetweenFetches = llmax(mMinTimeBetweenFetches * 0.8f, 0.3f);
mMaxTimeBetweenFetches = llmax(mMaxTimeBetweenFetches * 0.8f, 10.f);
//llinfos << "Inventory fetch times shrunk to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << llendl;
}
mTimelyFetchPending = FALSE;
continue;
}
else if (mFetchTimer.getElapsedTimeF32() > mMaxTimeBetweenFetches)
{
// Received first packet, but our num descendants does not match db's num descendants
// so try again later.
mFetchQueue.pop_front();
if (mNumFetchRetries++ < MAX_FETCH_RETRIES)
{
// push on back of queue
mFetchQueue.push_back(info);
}
mTimelyFetchPending = FALSE;
mFetchTimer.reset();
break;
}
// Not enough time has elapsed to do a new fetch
break;
}
//
// DEPRECATED OLD CODE
//--------------------------------------------------------------------------------
#endif
}
}
// Bundle up a bunch of requests to send all at once.
// static
void LLInventoryModelBackgroundFetch::bulkFetch()
{
//Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped.
//If there are items in mFetchQueue, we want to check the time since the last bulkFetch was
//sent. If it exceeds our retry time, go ahead and fire off another batch.
LLViewerRegion* region = gAgent.getRegion();
if (!region) return;
S16 max_concurrent_fetches=8;
F32 new_min_time = 0.5f; //HACK! Clean this up when old code goes away entirely.
if (mMinTimeBetweenFetches < new_min_time)
{
mMinTimeBetweenFetches=new_min_time; //HACK! See above.
}
if (gDisconnected ||
(mFetchCount > max_concurrent_fetches) ||
(mFetchTimer.getElapsedTimeF32() < mMinTimeBetweenFetches))
{
return; // just bail if we are disconnected
}
U32 item_count=0;
U32 folder_count=0;
U32 max_batch_size=5;
U32 sort_order = gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1;
uuid_vec_t recursive_cats;
LLSD folder_request_body;
LLSD folder_request_body_lib;
LLSD item_request_body;
LLSD item_request_body_lib;
while (!mFetchQueue.empty()
&& (item_count + folder_count) < max_batch_size)
{
const FetchQueueInfo& fetch_info = mFetchQueue.front();
if (fetch_info.mIsCategory)
{
const LLUUID &cat_id = fetch_info.mUUID;
if (cat_id.isNull()) //DEV-17797
{
LLSD folder_sd;
folder_sd["folder_id"] = LLUUID::null.asString();
folder_sd["owner_id"] = gAgent.getID();
folder_sd["sort_order"] = (LLSD::Integer)sort_order;
folder_sd["fetch_folders"] = (LLSD::Boolean)FALSE;
folder_sd["fetch_items"] = (LLSD::Boolean)TRUE;
folder_request_body["folders"].append(folder_sd);
folder_count++;
}
else
{
const LLViewerInventoryCategory* cat = gInventory.getCategory(cat_id);
if (cat)
{
if (LLViewerInventoryCategory::VERSION_UNKNOWN == cat->getVersion())
{
LLSD folder_sd;
folder_sd["folder_id"] = cat->getUUID();
folder_sd["owner_id"] = cat->getOwnerID();
folder_sd["sort_order"] = (LLSD::Integer)sort_order;
folder_sd["fetch_folders"] = TRUE; //(LLSD::Boolean)sFullFetchStarted;
folder_sd["fetch_items"] = (LLSD::Boolean)TRUE;
if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID())
folder_request_body_lib["folders"].append(folder_sd);
else
folder_request_body["folders"].append(folder_sd);
folder_count++;
}
// May already have this folder, but append child folders to list.
if (fetch_info.mRecursive)
{
LLInventoryModel::cat_array_t* categories;
LLInventoryModel::item_array_t* items;
gInventory.getDirectDescendentsOf(cat->getUUID(), categories, items);
for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
it != categories->end();
++it)
{
mFetchQueue.push_back(FetchQueueInfo((*it)->getUUID(), fetch_info.mRecursive));
}
}
}
}
if (fetch_info.mRecursive)
recursive_cats.push_back(cat_id);
}
else
{
LLViewerInventoryItem* itemp = gInventory.getItem(fetch_info.mUUID);
if (itemp)
{
LLSD item_sd;
item_sd["owner_id"] = itemp->getPermissions().getOwner();
item_sd["item_id"] = itemp->getUUID();
if (itemp->getPermissions().getOwner() == gAgent.getID())
{
item_request_body.append(item_sd);
}
else
{
item_request_body_lib.append(item_sd);
}
//itemp->fetchFromServer();
item_count++;
}
}
mFetchQueue.pop_front();
}
if (item_count + folder_count > 0)
{
if (folder_count)
{
std::string url = region->getCapability("FetchInventoryDescendents2");
mFetchCount++;
if (folder_request_body["folders"].size())
{
LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(folder_request_body, recursive_cats);
LLHTTPClient::post(url, folder_request_body, fetcher, 300.0);
}
if (folder_request_body_lib["folders"].size())
{
std::string url_lib = gAgent.getRegion()->getCapability("FetchLibDescendents2");
LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(folder_request_body_lib, recursive_cats);
LLHTTPClient::post(url_lib, folder_request_body_lib, fetcher, 300.0);
}
}
if (item_count)
{
std::string url;
if (item_request_body.size())
{
mFetchCount++;
url = region->getCapability("FetchInventory2");
if (!url.empty())
{
LLSD body;
body["agent_id"] = gAgent.getID();
body["items"] = item_request_body;
LLHTTPClient::post(url, body, new LLInventoryModelFetchItemResponder(body));
}
//else
//{
// LLMessageSystem* msg = gMessageSystem;
// msg->newMessage("FetchInventory");
// msg->nextBlock("AgentData");
// msg->addUUID("AgentID", gAgent.getID());
// msg->addUUID("SessionID", gAgent.getSessionID());
// msg->nextBlock("InventoryData");
// msg->addUUID("OwnerID", mPermissions.getOwner());
// msg->addUUID("ItemID", mUUID);
// gAgent.sendReliableMessage();
//}
}
if (item_request_body_lib.size())
{
mFetchCount++;
url = region->getCapability("FetchLib2");
if (!url.empty())
{
LLSD body;
body["agent_id"] = gAgent.getID();
body["items"] = item_request_body_lib;
LLHTTPClient::post(url, body, new LLInventoryModelFetchItemResponder(body));
}
}
}
mFetchTimer.reset();
}
else if (isBulkFetchProcessingComplete())
{
setAllFoldersFetched();
}
}
// Bundle up a bunch of requests to send all at once.
void LLInventoryModelBackgroundFetch::bulkFetch()
{
//Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped.
//If there are items in mFetchQueue, we want to check the time since the last bulkFetch was
//sent. If it exceeds our retry time, go ahead and fire off another batch.
LLViewerRegion* region = gAgent.getRegion();
if (gDisconnected || !region) return;
U32 const max_batch_size = 5;
U32 sort_order = gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1;
uuid_vec_t recursive_cats;
U32 folder_count=0;
U32 folder_lib_count=0;
U32 item_count=0;
U32 item_lib_count=0;
// This function can do up to four requests at once.
LLPointer<AIPerService::Approvement> approved_folder;
LLPointer<AIPerService::Approvement> approved_folder_lib;
LLPointer<AIPerService::Approvement> approved_item;
LLPointer<AIPerService::Approvement> approved_item_lib;
LLSD folder_request_body;
LLSD folder_request_body_lib;
LLSD item_request_body;
LLSD item_request_body_lib;
while (!mFetchQueue.empty())
{
const FetchQueueInfo& fetch_info = mFetchQueue.front();
if (fetch_info.mIsCategory)
{
const LLUUID &cat_id = fetch_info.mUUID;
if (!cat_id.isNull())
{
const LLViewerInventoryCategory* cat = gInventory.getCategory(cat_id);
if (cat)
{
if (LLViewerInventoryCategory::VERSION_UNKNOWN == cat->getVersion())
{
LLSD folder_sd;
folder_sd["folder_id"] = cat->getUUID();
folder_sd["owner_id"] = cat->getOwnerID();
folder_sd["sort_order"] = (LLSD::Integer)sort_order;
folder_sd["fetch_folders"] = TRUE; //(LLSD::Boolean)sFullFetchStarted;
folder_sd["fetch_items"] = (LLSD::Boolean)TRUE;
if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID())
{
if (folder_lib_count == max_batch_size ||
(folder_lib_count == 0 &&
!(approved_folder_lib = AIPerService::approveHTTPRequestFor(mPerServicePtr, cap_inventory))))
{
break;
}
folder_request_body_lib["folders"].append(folder_sd);
++folder_lib_count;
}
else
{
if (folder_count == max_batch_size ||
(folder_count == 0 &&
!(approved_folder = AIPerService::approveHTTPRequestFor(mPerServicePtr, cap_inventory))))
{
break;
}
folder_request_body["folders"].append(folder_sd);
++folder_count;
}
}
// May already have this folder, but append child folders to list.
if (fetch_info.mRecursive)
{
LLInventoryModel::cat_array_t* categories;
LLInventoryModel::item_array_t* items;
gInventory.getDirectDescendentsOf(cat->getUUID(), categories, items);
for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin();
it != categories->end();
++it)
{
mFetchQueue.push_back(FetchQueueInfo((*it)->getUUID(), fetch_info.mRecursive));
}
}
}
if (fetch_info.mRecursive)
recursive_cats.push_back(cat_id);
}
}
else
{
LLViewerInventoryItem* itemp = gInventory.getItem(fetch_info.mUUID);
if (itemp)
{
LLSD item_sd;
item_sd["owner_id"] = itemp->getPermissions().getOwner();
item_sd["item_id"] = itemp->getUUID();
if (itemp->getPermissions().getOwner() == gAgent.getID())
{
if (item_count == max_batch_size ||
(item_count == 0 &&
!(approved_item = AIPerService::approveHTTPRequestFor(mPerServicePtr, cap_inventory))))
{
break;
}
item_request_body.append(item_sd);
++item_count;
}
else
{
if (item_lib_count == max_batch_size ||
(item_lib_count == 0 &&
!(approved_item_lib = AIPerService::approveHTTPRequestFor(mPerServicePtr, cap_inventory))))
{
break;
}
item_request_body_lib.append(item_sd);
++item_lib_count;
}
}
}
mFetchQueue.pop_front();
}
if (item_count + folder_count + item_lib_count + folder_lib_count > 0)
{
if (folder_count)
{
std::string url = region->getCapability("FetchInventoryDescendents2");
llassert(!url.empty());
++mFetchCount;
LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(folder_request_body, recursive_cats);
LLHTTPClient::post_approved(url, folder_request_body, fetcher, approved_folder);
}
if (folder_lib_count)
{
std::string url = gAgent.getRegion()->getCapability("FetchLibDescendents2");
llassert(!url.empty());
++mFetchCount;
LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(folder_request_body_lib, recursive_cats);
LLHTTPClient::post_approved(url, folder_request_body_lib, fetcher, approved_folder_lib);
}
if (item_count)
{
std::string url = region->getCapability("FetchInventory2");
llassert(!url.empty());
++mFetchCount;
LLSD body;
body["agent_id"] = gAgent.getID();
body["items"] = item_request_body;
LLHTTPClient::post_approved(url, body, new LLInventoryModelFetchItemResponder(body), approved_item);
}
if (item_lib_count)
{
std::string url = region->getCapability("FetchLib2");
llassert(!url.empty());
++mFetchCount;
LLSD body;
body["agent_id"] = gAgent.getID();
body["items"] = item_request_body_lib;
LLHTTPClient::post_approved(url, body, new LLInventoryModelFetchItemResponder(body), approved_item_lib);
}
mFetchTimer.reset();
}
else if (isBulkFetchProcessingComplete())
{
llinfos << "Inventory fetch completed" << llendl;
setAllFoldersFetched();
}
}
