void output_multi_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
//make sure we get the one passed in
if(!ways_done_tracker->is_marked(id) && id_tracker::is_valid(id)) {
job_queue.push(pending_job_t(id, output_id));
added++;
}
//grab the first one or bail if its not valid
osmid_t popped = ways_pending_tracker->pop_mark();
if(!id_tracker::is_valid(popped))
return;
//get all the ones up to the id that was passed in
while (popped < id) {
if (!ways_done_tracker->is_marked(popped)) {
job_queue.push(pending_job_t(popped, output_id));
added++;
}
popped = ways_pending_tracker->pop_mark();
}
//make sure to get this one as well and move to the next
if(popped == id) {
popped = ways_pending_tracker->pop_mark();
}
if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) {
job_queue.push(pending_job_t(popped, output_id));
added++;
}
}
bool isRequestPending(const LLUUID& public_key)
{
bool isPending = false;
const F64 PENDING_TIMEOUT_SECS = 5.0 * 60.0;
pending_queue_t::const_iterator it = sPendingQueue.find(public_key);
if(it != sPendingQueue.end())
{
F64 expire_time = LLFrameTimer::getTotalSeconds() - PENDING_TIMEOUT_SECS;
isPending = (it->second > expire_time);
}
return isPending;
}
void processExperience( const LLUUID& public_key, const LLSD& experience )
{
sCache[public_key]=experience;
LLSD & row = sCache[public_key];
if(row.has(EXPIRES))
{
row[EXPIRES] = row[EXPIRES].asReal() + LLFrameTimer::getTotalSeconds();
}
if(row.has(EXPERIENCE_ID))
{
sPendingQueue.erase(row[EXPERIENCE_ID].asUUID());
}
//signal
signal_map_t::iterator sig_it = sSignalMap.find(public_key);
if (sig_it != sSignalMap.end())
{
callback_signal_t* signal = sig_it->second;
(*signal)(experience);
sSignalMap.erase(public_key);
delete signal;
}
}
void output_pgsql_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
osmid_t const prev = ways_pending_tracker.last_returned();
if (id_tracker::is_valid(prev) && prev >= id) {
if (prev > id) {
job_queue.push(pending_job_t(id, output_id));
}
// already done the job
return;
}
//make sure we get the one passed in
if(!ways_done_tracker->is_marked(id) && id_tracker::is_valid(id)) {
job_queue.push(pending_job_t(id, output_id));
added++;
}
//grab the first one or bail if its not valid
osmid_t popped = ways_pending_tracker.pop_mark();
if(!id_tracker::is_valid(popped))
return;
//get all the ones up to the id that was passed in
while (popped < id) {
if (!ways_done_tracker->is_marked(popped)) {
job_queue.push(pending_job_t(popped, output_id));
added++;
}
popped = ways_pending_tracker.pop_mark();
}
//make sure to get this one as well and move to the next
if(popped > id) {
if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) {
job_queue.push(pending_job_t(popped, output_id));
added++;
}
}
}
namespace LLExperienceCache
{
typedef std::map<LLUUID, LLUUID> KeyMap;
KeyMap privateToPublicKeyMap;
void mapKeys(const LLSD& legacyKeys);
std::string sLookupURL;
typedef std::map<LLUUID, std::string> ask_queue_t;
ask_queue_t sAskQueue;
typedef std::map<LLUUID, F64> pending_queue_t;
pending_queue_t sPendingQueue;
cache_t sCache;
int sMaximumLookups = 10;
LLFrameTimer sRequestTimer;
// Periodically clean out expired entries from the cache
LLFrameTimer sEraseExpiredTimer;
// May have multiple callbacks for a single ID, which are
// represented as multiple slots bound to the signal.
// Avoid copying signals via pointers.
typedef std::map<LLUUID, callback_signal_t*> signal_map_t;
signal_map_t sSignalMap;
bool max_age_from_cache_control(const std::string& cache_control, S32 *max_age);
void eraseExpired();
void processExperience( const LLUUID& public_key, const LLSD& experience )
{
sCache[public_key]=experience;
LLSD & row = sCache[public_key];
if(row.has(EXPIRES))
{
row[EXPIRES] = row[EXPIRES].asReal() + LLFrameTimer::getTotalSeconds();
}
if(row.has(EXPERIENCE_ID))
{
sPendingQueue.erase(row[EXPERIENCE_ID].asUUID());
}
//signal
signal_map_t::iterator sig_it = sSignalMap.find(public_key);
if (sig_it != sSignalMap.end())
{
callback_signal_t* signal = sig_it->second;
(*signal)(experience);
sSignalMap.erase(public_key);
delete signal;
}
}
void initClass( )
{
}
const cache_t& getCached()
{
return sCache;
}
void setMaximumLookups( int maximumLookups)
{
sMaximumLookups = maximumLookups;
}
void bootstrap(const LLSD& legacyKeys, int initialExpiration)
{
mapKeys(legacyKeys);
LLSD::array_const_iterator it = legacyKeys.beginArray();
for(/**/; it != legacyKeys.endArray(); ++it)
{
LLSD experience = *it;
if(experience.has(EXPERIENCE_ID))
{
if(!experience.has(EXPIRES))
{
experience[EXPIRES] = initialExpiration;
}
processExperience(experience[EXPERIENCE_ID].asUUID(), experience);
}
else
{
LL_WARNS("ExperienceCache")
<< "Skipping bootstrap entry which is missing " << EXPERIENCE_ID
<< LL_ENDL;
}
}
}
bool expirationFromCacheControl(LLSD headers, F64 *expires)
{
// Allow the header to override the default
LLSD cache_control_header = headers["cache-control"];
if (cache_control_header.isDefined())
{
S32 max_age = 0;
std::string cache_control = cache_control_header.asString();
if (max_age_from_cache_control(cache_control, &max_age))
{
LL_WARNS("ExperienceCache")
<< "got EXPIRES from headers, max_age " << max_age
<< LL_ENDL;
F64 now = LLFrameTimer::getTotalSeconds();
*expires = now + (F64)max_age;
return true;
}
}
return false;
}
static const std::string MAX_AGE("max-age");
static const boost::char_separator<char> EQUALS_SEPARATOR("=");
static const boost::char_separator<char> COMMA_SEPARATOR(",");
bool max_age_from_cache_control(const std::string& cache_control, S32 *max_age)
{
// Split the string on "," to get a list of directives
typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
tokenizer directives(cache_control, COMMA_SEPARATOR);
tokenizer::iterator token_it = directives.begin();
for ( ; token_it != directives.end(); ++token_it)
{
// Tokens may have leading or trailing whitespace
std::string token = *token_it;
LLStringUtil::trim(token);
if (token.compare(0, MAX_AGE.size(), MAX_AGE) == 0)
{
// ...this token starts with max-age, so let's chop it up by "="
tokenizer subtokens(token, EQUALS_SEPARATOR);
tokenizer::iterator subtoken_it = subtokens.begin();
// Must have a token
if (subtoken_it == subtokens.end()) return false;
std::string subtoken = *subtoken_it;
// Must exactly equal "max-age"
LLStringUtil::trim(subtoken);
if (subtoken != MAX_AGE) return false;
// Must have another token
++subtoken_it;
if (subtoken_it == subtokens.end()) return false;
subtoken = *subtoken_it;
// Must be a valid integer
// *NOTE: atoi() returns 0 for invalid values, so we have to
// check the string first.
// *TODO: Do servers ever send "0000" for zero? We don't handle it
LLStringUtil::trim(subtoken);
if (subtoken == "0")
{
*max_age = 0;
return true;
}
S32 val = atoi( subtoken.c_str() );
if (val > 0 && val < S32_MAX)
{
*max_age = val;
return true;
}
return false;
}
}
return false;
}
void importFile(std::istream& istr)
{
LLSD data;
S32 parse_count = LLSDSerialize::fromXMLDocument(data, istr);
if(parse_count < 1) return;
LLSD experiences = data["experiences"];
LLUUID public_key;
LLSD::map_const_iterator it = experiences.beginMap();
for(; it != experiences.endMap() ; ++it)
{
public_key.set(it->first);
sCache[public_key]=it->second;
}
LL_DEBUGS("ExperienceCache") << "importFile() loaded " << sCache.size() << LL_ENDL;
}
void exportFile(std::ostream& ostr)
{
LLSD experiences;
cache_t::const_iterator it =sCache.begin();
for( ; it != sCache.end() ; ++it)
{
if(!it->second.has(EXPERIENCE_ID) || it->second[EXPERIENCE_ID].asUUID().isNull() ||
it->second.has("DoesNotExist") || (it->second.has(PROPERTIES) && it->second[PROPERTIES].asInteger() & PROPERTY_INVALID))
continue;
experiences[it->first.asString()] = it->second;
}
LLSD data;
data["experiences"] = experiences;
LLSDSerialize::toPrettyXML(data, ostr);
}
class LLExperienceResponder : public LLHTTPClient::Responder
{
public:
LLExperienceResponder(const ask_queue_t& keys)
:mKeys(keys)
{
}
/*virtual*/ void httpCompleted()
{
LLSD experiences = getContent()["experience_keys"];
LLSD::array_const_iterator it = experiences.beginArray();
for( /**/ ; it != experiences.endArray(); ++it)
{
const LLSD& row = *it;
LLUUID public_key = row[EXPERIENCE_ID].asUUID();
LL_DEBUGS("ExperienceCache") << "Received result for " << public_key
<< " display '" << row[LLExperienceCache::NAME].asString() << "'" << LL_ENDL ;
processExperience(public_key, row);
}
LLSD error_ids = getContent()["error_ids"];
LLSD::array_const_iterator errIt = error_ids.beginArray();
for( /**/ ; errIt != error_ids.endArray() ; ++errIt )
{
LLUUID id = errIt->asUUID();
LLSD exp;
exp[EXPIRES]=DEFAULT_EXPIRATION;
exp[EXPERIENCE_ID] = id;
exp[PROPERTIES]=PROPERTY_INVALID;
exp[MISSING]=true;
exp[QUOTA] = DEFAULT_QUOTA;
processExperience(id, exp);
LL_WARNS("ExperienceCache") << "LLExperienceResponder::result() error result for " << id << LL_ENDL ;
}
LL_DEBUGS("ExperienceCache") << sCache.size() << " cached experiences" << LL_ENDL;
}
/*virtual*/ void httpFailure()
{
LL_WARNS("ExperienceCache") << "Request failed "<<getStatus()<<" "<<getReason()<< LL_ENDL;
// We're going to construct a dummy record and cache it for a while,
// either briefly for a 503 Service Unavailable, or longer for other
// errors.
F64 retry_timestamp = errorRetryTimestamp(getStatus());
// Add dummy records for all agent IDs in this request
ask_queue_t::const_iterator it = mKeys.begin();
for ( ; it != mKeys.end(); ++it)
{
LLSD exp = get(it->first);
//leave the properties alone if we already have a cache entry for this xp
if(exp.isUndefined())
{
exp[PROPERTIES]=PROPERTY_INVALID;
}
exp[EXPIRES]=retry_timestamp;
exp[EXPERIENCE_ID] = it->first;
exp["key_type"] = it->second;
exp["uuid"] = it->first;
exp["error"] = (LLSD::Integer)getStatus();
exp[QUOTA] = DEFAULT_QUOTA;
LLExperienceCache::processExperience(it->first, exp);
}
}
// Return time to retry a request that generated an error, based on
// error type and headers. Return value is seconds-since-epoch.
F64 errorRetryTimestamp(S32 status)
{
// Retry-After takes priority
LLSD retry_after = getResponseHeaders()["retry-after"];
if (retry_after.isDefined())
{
// We only support the delta-seconds type
S32 delta_seconds = retry_after.asInteger();
if (delta_seconds > 0)
{
// ...valid delta-seconds
return F64(delta_seconds);
}
}
// If no Retry-After, look for Cache-Control max-age
F64 expires = 0.0;
if (LLExperienceCache::expirationFromCacheControl(getResponseHeaders(), &expires))
{
return expires;
}
// No information in header, make a guess
if (status == 503)
{
// ...service unavailable, retry soon
const F64 SERVICE_UNAVAILABLE_DELAY = 600.0; // 10 min
return SERVICE_UNAVAILABLE_DELAY;
}
else if (status == 499)
{
// ...we were probably too busy, retry quickly
const F64 BUSY_DELAY = 10.0; // 10 seconds
return BUSY_DELAY;
}
else
{
// ...other unexpected error
const F64 DEFAULT_DELAY = 3600.0; // 1 hour
return DEFAULT_DELAY;
}
}
private:
ask_queue_t mKeys;
};
void requestExperiences()
{
if(sAskQueue.empty() || sLookupURL.empty())
return;
F64 now = LLFrameTimer::getTotalSeconds();
const U32 EXP_URL_SEND_THRESHOLD = 3000;
const U32 PAGE_SIZE = EXP_URL_SEND_THRESHOLD/UUID_STR_LENGTH;
std::ostringstream ostr;
ask_queue_t keys;
ostr << sLookupURL << "?page_size=" << PAGE_SIZE;
int request_count = 0;
while(!sAskQueue.empty() && request_count < sMaximumLookups)
{
ask_queue_t::iterator it = sAskQueue.begin();
const LLUUID& key = it->first;
const std::string& key_type = it->second;
ostr << '&' << key_type << '=' << key.asString() ;
keys[key]=key_type;
request_count++;
sPendingQueue[key] = now;
if(ostr.tellp() > EXP_URL_SEND_THRESHOLD)
{
LL_DEBUGS("ExperienceCache") << "requestExperiences() query: " << ostr.str() << LL_ENDL;
LLHTTPClient::get(ostr.str(), new LLExperienceResponder(keys));
ostr.clear();
ostr.str(sLookupURL);
ostr << "?page_size=" << PAGE_SIZE;
keys.clear();
}
sAskQueue.erase(it);
}
if(ostr.tellp() > sLookupURL.size())
{
LL_DEBUGS("ExperienceCache") << "requestExperiences() query 2: " << ostr.str() << LL_ENDL;
LLHTTPClient::get(ostr.str(), new LLExperienceResponder(keys));
}
}
bool isRequestPending(const LLUUID& public_key)
{
bool isPending = false;
const F64 PENDING_TIMEOUT_SECS = 5.0 * 60.0;
pending_queue_t::const_iterator it = sPendingQueue.find(public_key);
if(it != sPendingQueue.end())
{
F64 expire_time = LLFrameTimer::getTotalSeconds() - PENDING_TIMEOUT_SECS;
isPending = (it->second > expire_time);
}
return isPending;
}
void setLookupURL( const std::string& lookup_url )
{
sLookupURL = lookup_url;
if(!sLookupURL.empty())
{
sLookupURL += "id/";
}
}
bool hasLookupURL()
{
return !sLookupURL.empty();
}
void idle()
{
const F32 SECS_BETWEEN_REQUESTS = 0.1f;
if (!sRequestTimer.checkExpirationAndReset(SECS_BETWEEN_REQUESTS))
{
return;
}
// Must be large relative to above
const F32 ERASE_EXPIRED_TIMEOUT = 60.f; // seconds
if (sEraseExpiredTimer.checkExpirationAndReset(ERASE_EXPIRED_TIMEOUT))
{
eraseExpired();
}
if(!sAskQueue.empty())
{
requestExperiences();
}
}
void erase( const LLUUID& key )
{
cache_t::iterator it = sCache.find(key);
if(it != sCache.end())
{
sCache.erase(it);
}
}
void eraseExpired()
{
F64 now = LLFrameTimer::getTotalSeconds();
cache_t::iterator it = sCache.begin();
while (it != sCache.end())
{
cache_t::iterator cur = it;
LLSD& exp = cur->second;
++it;
if(exp.has(EXPIRES) && exp[EXPIRES].asReal() < now)
{
if(!exp.has(EXPERIENCE_ID))
{
LL_WARNS("ExperienceCache") << "Removing experience with no id " << LL_ENDL ;
sCache.erase(cur);
}
else
{
LLUUID id = exp[EXPERIENCE_ID].asUUID();
LLUUID private_key = exp.has(LLExperienceCache::PRIVATE_KEY) ? exp[LLExperienceCache::PRIVATE_KEY].asUUID():LLUUID::null;
if(private_key.notNull() || !exp.has("DoesNotExist"))
{
fetch(id, true);
}
else
{
LL_WARNS("ExperienceCache") << "Removing invalid experience " << id << LL_ENDL ;
sCache.erase(cur);
}
}
}
}
}
bool fetch( const LLUUID& key, bool refresh/* = true*/ )
{
if(!key.isNull() && !isRequestPending(key) && (refresh || sCache.find(key)==sCache.end()))
{
LL_DEBUGS("ExperienceCache") << " queue request for " << EXPERIENCE_ID << " " << key << LL_ENDL ;
sAskQueue[key]=EXPERIENCE_ID;
return true;
}
return false;
}
void insert(const LLSD& experience_data )
{
if(experience_data.has(EXPERIENCE_ID))
{
processExperience(experience_data[EXPERIENCE_ID].asUUID(), experience_data);
}
else
{
LL_WARNS("ExperienceCache") << ": Ignoring cache insert of experience which is missing " << EXPERIENCE_ID << LL_ENDL;
}
}
static LLSD empty;
const LLSD& get(const LLUUID& key)
{
if(key.isNull()) return empty;
cache_t::const_iterator it = sCache.find(key);
if (it != sCache.end())
{
return it->second;
}
fetch(key);
return empty;
}
void get( const LLUUID& key, callback_slot_t slot )
{
if(key.isNull()) return;
cache_t::const_iterator it = sCache.find(key);
if (it != sCache.end())
{
// ...name already exists in cache, fire callback now
callback_signal_t signal;
signal.connect(slot);
signal(it->second);
return;
}
fetch(key);
// always store additional callback, even if request is pending
signal_map_t::iterator sig_it = sSignalMap.find(key);
if (sig_it == sSignalMap.end())
{
// ...new callback for this id
callback_signal_t* signal = new callback_signal_t();
signal->connect(slot);
sSignalMap[key] = signal;
}
else
{
// ...existing callback, bind additional slot
callback_signal_t* signal = sig_it->second;
signal->connect(slot);
}
}
}