예제 #1
0
void output_multi_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
    //make sure we get the one passed in
    if(!ways_done_tracker->is_marked(id) && id_tracker::is_valid(id)) {
        job_queue.push(pending_job_t(id, output_id));
        added++;
    }

    //grab the first one or bail if its not valid
    osmid_t popped = ways_pending_tracker->pop_mark();
    if(!id_tracker::is_valid(popped))
        return;

    //get all the ones up to the id that was passed in
    while (popped < id) {
        if (!ways_done_tracker->is_marked(popped)) {
            job_queue.push(pending_job_t(popped, output_id));
            added++;
        }
        popped = ways_pending_tracker->pop_mark();
    }

    //make sure to get this one as well and move to the next
    if(popped == id) {
        popped = ways_pending_tracker->pop_mark();
    }
    if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) {
        job_queue.push(pending_job_t(popped, output_id));
        added++;
    }
}
예제 #2
0
	bool isRequestPending(const LLUUID& public_key)
	{
		bool isPending = false;
		const F64 PENDING_TIMEOUT_SECS = 5.0 * 60.0;

		pending_queue_t::const_iterator it = sPendingQueue.find(public_key);

		if(it != sPendingQueue.end())
		{
			F64 expire_time = LLFrameTimer::getTotalSeconds() - PENDING_TIMEOUT_SECS;
			isPending = (it->second > expire_time);
		}

		return isPending;
	}
예제 #3
0
	void processExperience( const LLUUID& public_key, const LLSD& experience ) 
	{
		sCache[public_key]=experience;
		LLSD & row = sCache[public_key];

		if(row.has(EXPIRES))
		{
			row[EXPIRES] = row[EXPIRES].asReal() + LLFrameTimer::getTotalSeconds();
		}

		if(row.has(EXPERIENCE_ID))
		{
			sPendingQueue.erase(row[EXPERIENCE_ID].asUUID());
		}

		//signal
		signal_map_t::iterator sig_it =	sSignalMap.find(public_key);
		if (sig_it != sSignalMap.end())
		{
			callback_signal_t* signal = sig_it->second;
			(*signal)(experience);

			sSignalMap.erase(public_key);

			delete signal;
		}
	}
예제 #4
0
void output_pgsql_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
    osmid_t const prev = ways_pending_tracker.last_returned();
    if (id_tracker::is_valid(prev) && prev >= id) {
        if (prev > id) {
            job_queue.push(pending_job_t(id, output_id));
        }
        // already done the job
        return;
    }

    //make sure we get the one passed in
    if(!ways_done_tracker->is_marked(id) && id_tracker::is_valid(id)) {
        job_queue.push(pending_job_t(id, output_id));
        added++;
    }

    //grab the first one or bail if its not valid
    osmid_t popped = ways_pending_tracker.pop_mark();
    if(!id_tracker::is_valid(popped))
        return;

    //get all the ones up to the id that was passed in
    while (popped < id) {
        if (!ways_done_tracker->is_marked(popped)) {
            job_queue.push(pending_job_t(popped, output_id));
            added++;
        }
        popped = ways_pending_tracker.pop_mark();
    }

    //make sure to get this one as well and move to the next
    if(popped > id) {
        if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) {
            job_queue.push(pending_job_t(popped, output_id));
            added++;
        }
    }
}
예제 #5
0
namespace LLExperienceCache
{

    typedef std::map<LLUUID, LLUUID> KeyMap;
    KeyMap privateToPublicKeyMap;

    void mapKeys(const LLSD& legacyKeys);

	std::string sLookupURL;

	typedef std::map<LLUUID, std::string> ask_queue_t;
	ask_queue_t sAskQueue;

	typedef std::map<LLUUID, F64> pending_queue_t;
	pending_queue_t sPendingQueue;

	cache_t sCache;
	int sMaximumLookups = 10;

	LLFrameTimer sRequestTimer;

	// Periodically clean out expired entries from the cache
	LLFrameTimer sEraseExpiredTimer;

	// May have multiple callbacks for a single ID, which are
	// represented as multiple slots bound to the signal.
	// Avoid copying signals via pointers.
	typedef std::map<LLUUID, callback_signal_t*> signal_map_t;
	signal_map_t sSignalMap;



	bool max_age_from_cache_control(const std::string& cache_control, S32 *max_age);
	void eraseExpired();

	void processExperience( const LLUUID& public_key, const LLSD& experience ) 
	{
		sCache[public_key]=experience;
		LLSD & row = sCache[public_key];

		if(row.has(EXPIRES))
		{
			row[EXPIRES] = row[EXPIRES].asReal() + LLFrameTimer::getTotalSeconds();
		}

		if(row.has(EXPERIENCE_ID))
		{
			sPendingQueue.erase(row[EXPERIENCE_ID].asUUID());
		}

		//signal
		signal_map_t::iterator sig_it =	sSignalMap.find(public_key);
		if (sig_it != sSignalMap.end())
		{
			callback_signal_t* signal = sig_it->second;
			(*signal)(experience);

			sSignalMap.erase(public_key);

			delete signal;
		}
	}

	void initClass( )
	{
	}

	const cache_t& getCached()
	{
		return sCache;
	}

	void setMaximumLookups( int maximumLookups)
	{
		sMaximumLookups = maximumLookups;
	}

	void bootstrap(const LLSD& legacyKeys, int initialExpiration)
	{
        mapKeys(legacyKeys);
		LLSD::array_const_iterator it = legacyKeys.beginArray();
		for(/**/; it != legacyKeys.endArray(); ++it)
		{
			LLSD experience = *it;
			if(experience.has(EXPERIENCE_ID))
			{
				if(!experience.has(EXPIRES))
				{
					experience[EXPIRES] = initialExpiration;
				}
				processExperience(experience[EXPERIENCE_ID].asUUID(), experience);
			}
			else
			{
				LL_WARNS("ExperienceCache") 
					<< "Skipping bootstrap entry which is missing " << EXPERIENCE_ID 
					<< LL_ENDL;
			}
		}		
	}



	bool expirationFromCacheControl(LLSD headers, F64 *expires)
	{
		// Allow the header to override the default
		LLSD cache_control_header = headers["cache-control"];
		if (cache_control_header.isDefined())
		{
			S32 max_age = 0;
			std::string cache_control = cache_control_header.asString();
			if (max_age_from_cache_control(cache_control, &max_age))
			{
				LL_WARNS("ExperienceCache") 
					<< "got EXPIRES from headers, max_age " << max_age 
					<< LL_ENDL;
				F64 now = LLFrameTimer::getTotalSeconds();
				*expires = now + (F64)max_age;
				return true;
			}
		}
		return false;
	}


	static const std::string MAX_AGE("max-age");
	static const boost::char_separator<char> EQUALS_SEPARATOR("=");
	static const boost::char_separator<char> COMMA_SEPARATOR(",");

	bool max_age_from_cache_control(const std::string& cache_control, S32 *max_age)
	{
		// Split the string on "," to get a list of directives
		typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
		tokenizer directives(cache_control, COMMA_SEPARATOR);

		tokenizer::iterator token_it = directives.begin();
		for ( ; token_it != directives.end(); ++token_it)
		{
			// Tokens may have leading or trailing whitespace
			std::string token = *token_it;
			LLStringUtil::trim(token);

			if (token.compare(0, MAX_AGE.size(), MAX_AGE) == 0)
			{
				// ...this token starts with max-age, so let's chop it up by "="
				tokenizer subtokens(token, EQUALS_SEPARATOR);
				tokenizer::iterator subtoken_it = subtokens.begin();

				// Must have a token
				if (subtoken_it == subtokens.end()) return false;
				std::string subtoken = *subtoken_it;

				// Must exactly equal "max-age"
				LLStringUtil::trim(subtoken);
				if (subtoken != MAX_AGE) return false;

				// Must have another token
				++subtoken_it;
				if (subtoken_it == subtokens.end()) return false;
				subtoken = *subtoken_it;

				// Must be a valid integer
				// *NOTE: atoi() returns 0 for invalid values, so we have to
				// check the string first.
				// *TODO: Do servers ever send "0000" for zero?  We don't handle it
				LLStringUtil::trim(subtoken);
				if (subtoken == "0")
				{
					*max_age = 0;
					return true;
				}
				S32 val = atoi( subtoken.c_str() );
				if (val > 0 && val < S32_MAX)
				{
					*max_age = val;
					return true;
				}
				return false;
			}
		}
		return false;
	}


	void importFile(std::istream& istr)
	{
		LLSD data;
		S32 parse_count = LLSDSerialize::fromXMLDocument(data, istr);
		if(parse_count < 1) return;

		LLSD experiences = data["experiences"];

		LLUUID public_key;
		LLSD::map_const_iterator it = experiences.beginMap();
		for(; it != experiences.endMap() ; ++it)
		{
			public_key.set(it->first);
			sCache[public_key]=it->second;
		}

		LL_DEBUGS("ExperienceCache") << "importFile() loaded " << sCache.size() << LL_ENDL;
	}

	void exportFile(std::ostream& ostr)
	{
		LLSD experiences;

		cache_t::const_iterator it =sCache.begin();
		for( ; it != sCache.end() ; ++it)
		{
			if(!it->second.has(EXPERIENCE_ID) || it->second[EXPERIENCE_ID].asUUID().isNull() ||
				it->second.has("DoesNotExist") || (it->second.has(PROPERTIES) && it->second[PROPERTIES].asInteger() & PROPERTY_INVALID))
				continue;

			experiences[it->first.asString()] = it->second;
		}

		LLSD data;
		data["experiences"] = experiences;

		LLSDSerialize::toPrettyXML(data, ostr);
	}

	class LLExperienceResponder : public LLHTTPClient::Responder
	{
	public:
		LLExperienceResponder(const ask_queue_t& keys)
			:mKeys(keys)
		{

		}

		/*virtual*/ void httpCompleted()
		{
			LLSD experiences = getContent()["experience_keys"];
			LLSD::array_const_iterator it = experiences.beginArray();
			for( /**/ ; it != experiences.endArray(); ++it)
			{
				const LLSD& row = *it;
				LLUUID public_key = row[EXPERIENCE_ID].asUUID();


				LL_DEBUGS("ExperienceCache") << "Received result for " << public_key 
					<< " display '" << row[LLExperienceCache::NAME].asString() << "'" << LL_ENDL ;

				processExperience(public_key, row);
			}

			LLSD error_ids = getContent()["error_ids"];
			LLSD::array_const_iterator errIt = error_ids.beginArray();
			for( /**/ ; errIt != error_ids.endArray() ; ++errIt )
			{
				LLUUID id = errIt->asUUID();		
				LLSD exp;
				exp[EXPIRES]=DEFAULT_EXPIRATION;
				exp[EXPERIENCE_ID] = id;
				exp[PROPERTIES]=PROPERTY_INVALID;
				exp[MISSING]=true;
                exp[QUOTA] = DEFAULT_QUOTA;

				processExperience(id, exp);
				LL_WARNS("ExperienceCache") << "LLExperienceResponder::result() error result for " << id << LL_ENDL ;
			}

			LL_DEBUGS("ExperienceCache") << sCache.size() << " cached experiences" << LL_ENDL;
		}

		/*virtual*/ void httpFailure()
		{
 			LL_WARNS("ExperienceCache") << "Request failed "<<getStatus()<<" "<<getReason()<< LL_ENDL;
 			// We're going to construct a dummy record and cache it for a while,
 			// either briefly for a 503 Service Unavailable, or longer for other
 			// errors.
 			F64 retry_timestamp = errorRetryTimestamp(getStatus());
 
 
 			// Add dummy records for all agent IDs in this request
 			ask_queue_t::const_iterator it = mKeys.begin();
 			for ( ; it != mKeys.end(); ++it)
			{

				LLSD exp = get(it->first);
                //leave the properties alone if we already have a cache entry for this xp
                if(exp.isUndefined())
                {
                    exp[PROPERTIES]=PROPERTY_INVALID;
                }
				exp[EXPIRES]=retry_timestamp;
				exp[EXPERIENCE_ID] = it->first;
				exp["key_type"] = it->second;
				exp["uuid"] = it->first;
				exp["error"] = (LLSD::Integer)getStatus();
                exp[QUOTA] = DEFAULT_QUOTA;

 				LLExperienceCache::processExperience(it->first, exp);
 			}

		}

		// Return time to retry a request that generated an error, based on
		// error type and headers.  Return value is seconds-since-epoch.
		F64 errorRetryTimestamp(S32 status)
		{

			// Retry-After takes priority
			LLSD retry_after = getResponseHeaders()["retry-after"];
			if (retry_after.isDefined())
			{
				// We only support the delta-seconds type
				S32 delta_seconds = retry_after.asInteger();
				if (delta_seconds > 0)
				{
					// ...valid delta-seconds
					return F64(delta_seconds);
				}
			}

			// If no Retry-After, look for Cache-Control max-age
			F64 expires = 0.0;
			if (LLExperienceCache::expirationFromCacheControl(getResponseHeaders(), &expires))
			{
				return expires;
			}

			// No information in header, make a guess
			if (status == 503)
			{
				// ...service unavailable, retry soon
				const F64 SERVICE_UNAVAILABLE_DELAY = 600.0; // 10 min
				return SERVICE_UNAVAILABLE_DELAY;
			}
			else if (status == 499)
			{
				// ...we were probably too busy, retry quickly
				const F64 BUSY_DELAY = 10.0; // 10 seconds
				return BUSY_DELAY;

			}
			else
			{
				// ...other unexpected error
				const F64 DEFAULT_DELAY = 3600.0; // 1 hour
				return DEFAULT_DELAY;
			}
		}

	private:
		ask_queue_t mKeys;
	};

	void requestExperiences() 
	{
		if(sAskQueue.empty() || sLookupURL.empty())
			return;

		F64 now = LLFrameTimer::getTotalSeconds();

		const U32 EXP_URL_SEND_THRESHOLD = 3000;
		const U32 PAGE_SIZE = EXP_URL_SEND_THRESHOLD/UUID_STR_LENGTH;

		std::ostringstream ostr;

		ask_queue_t keys;

		ostr << sLookupURL << "?page_size=" << PAGE_SIZE;


		int request_count = 0;
		while(!sAskQueue.empty() && request_count < sMaximumLookups)
		{
			ask_queue_t::iterator it = sAskQueue.begin();
			const LLUUID& key = it->first;
			const std::string& key_type = it->second;

			ostr << '&' << key_type << '=' << key.asString() ;
		
			keys[key]=key_type;
			request_count++;

			sPendingQueue[key] = now;
			
			if(ostr.tellp() > EXP_URL_SEND_THRESHOLD)
			{
				LL_DEBUGS("ExperienceCache") <<  "requestExperiences() query: " << ostr.str() << LL_ENDL;
				LLHTTPClient::get(ostr.str(), new LLExperienceResponder(keys));
				ostr.clear();
				ostr.str(sLookupURL);
				ostr << "?page_size=" << PAGE_SIZE;
				keys.clear();
			}
			sAskQueue.erase(it);
		}

		if(ostr.tellp() > sLookupURL.size())
		{
			LL_DEBUGS("ExperienceCache") <<  "requestExperiences() query 2: " << ostr.str() << LL_ENDL;
			LLHTTPClient::get(ostr.str(), new LLExperienceResponder(keys));
		}
	}

	bool isRequestPending(const LLUUID& public_key)
	{
		bool isPending = false;
		const F64 PENDING_TIMEOUT_SECS = 5.0 * 60.0;

		pending_queue_t::const_iterator it = sPendingQueue.find(public_key);

		if(it != sPendingQueue.end())
		{
			F64 expire_time = LLFrameTimer::getTotalSeconds() - PENDING_TIMEOUT_SECS;
			isPending = (it->second > expire_time);
		}

		return isPending;
	}


	void setLookupURL( const std::string& lookup_url )
	{
		sLookupURL = lookup_url;
		if(!sLookupURL.empty())
		{
			sLookupURL += "id/";
		}
	}

	bool hasLookupURL()
	{
		return !sLookupURL.empty();
	}

	void idle()
	{

		const F32 SECS_BETWEEN_REQUESTS = 0.1f;
		if (!sRequestTimer.checkExpirationAndReset(SECS_BETWEEN_REQUESTS))
		{
			return;
		}

		// Must be large relative to above
		const F32 ERASE_EXPIRED_TIMEOUT = 60.f; // seconds
		if (sEraseExpiredTimer.checkExpirationAndReset(ERASE_EXPIRED_TIMEOUT))
		{
			eraseExpired();
		}


		if(!sAskQueue.empty())
		{
			requestExperiences();
		}
	}

	void erase( const LLUUID& key )
	{
		cache_t::iterator it = sCache.find(key);
				
		if(it != sCache.end())
		{
			sCache.erase(it);
		}
	}

	void eraseExpired()
	{
		F64 now = LLFrameTimer::getTotalSeconds();
		cache_t::iterator it = sCache.begin();
		while (it != sCache.end())
		{
			cache_t::iterator cur = it;
			LLSD& exp = cur->second;
			++it;

			if(exp.has(EXPIRES) && exp[EXPIRES].asReal() < now)
			{
                if(!exp.has(EXPERIENCE_ID))
				{
                    LL_WARNS("ExperienceCache") << "Removing experience with no id " << LL_ENDL ;
                    sCache.erase(cur);
					}
                else
                {
                    LLUUID id = exp[EXPERIENCE_ID].asUUID();
                    LLUUID private_key = exp.has(LLExperienceCache::PRIVATE_KEY) ? exp[LLExperienceCache::PRIVATE_KEY].asUUID():LLUUID::null;
                    if(private_key.notNull() || !exp.has("DoesNotExist"))
					{
						fetch(id, true);
					}
					else
					{
                        LL_WARNS("ExperienceCache") << "Removing invalid experience " << id << LL_ENDL ;
						sCache.erase(cur);
					}
				}
			}
		}
	}

	
	bool fetch( const LLUUID& key, bool refresh/* = true*/ ) 
	{
		if(!key.isNull() && !isRequestPending(key) && (refresh || sCache.find(key)==sCache.end()))
		{
			LL_DEBUGS("ExperienceCache") << " queue request for " << EXPERIENCE_ID << " " << key << LL_ENDL ;
			sAskQueue[key]=EXPERIENCE_ID;

			return true;
		}
		return false;
	}

	void insert(const LLSD& experience_data )
	{
		if(experience_data.has(EXPERIENCE_ID))
		{
            processExperience(experience_data[EXPERIENCE_ID].asUUID(), experience_data);
		}
		else
		{
			LL_WARNS("ExperienceCache") << ": Ignoring cache insert of experience which is missing " << EXPERIENCE_ID << LL_ENDL;
		}
	}
	static LLSD empty;
	const LLSD& get(const LLUUID& key)
	{
		if(key.isNull()) return empty;
		cache_t::const_iterator it = sCache.find(key);

		if (it != sCache.end())
		{
			return it->second;
		}

		fetch(key);

		return empty;
	}
	void get( const LLUUID& key, callback_slot_t slot )
	{
		if(key.isNull()) return;

		cache_t::const_iterator it = sCache.find(key);
		if (it != sCache.end())
		{
			// ...name already exists in cache, fire callback now
			callback_signal_t signal;
			signal.connect(slot);
			
			signal(it->second);
			return;
		}

		fetch(key);

		// always store additional callback, even if request is pending
		signal_map_t::iterator sig_it = sSignalMap.find(key);
		if (sig_it == sSignalMap.end())
		{
			// ...new callback for this id
			callback_signal_t* signal = new callback_signal_t();
			signal->connect(slot);
			sSignalMap[key] = signal;
		}
		else
		{
			// ...existing callback, bind additional slot
			callback_signal_t* signal = sig_it->second;
			signal->connect(slot);
		}
	}

}