int Ardb::BlockForKeys(Context& ctx, const StringArray& keys, const AnyArray& vals, KeyType ktype, uint32 mstimeout) { if (ctx.keyslocked) { FATAL_LOG("Can not modify block dataset when key locked."); } if (mstimeout > 0) { ctx.GetBPop().timeout = (uint64) mstimeout * 1000 + get_current_epoch_micros(); } ctx.GetBPop().block_keytype = ktype; ctx.client->client->BlockRead(); LockGuard<SpinMutexLock> guard(m_block_keys_lock); for (size_t i = 0; i < keys.size(); i++) { KeyPrefix prefix; prefix.ns = ctx.ns; prefix.key.SetString(keys[i], false); if(vals.size() != keys.size()) { ctx.GetBPop().keys.insert(BlockingState::BlockKeyTable::value_type(prefix, NULL)); }else { ctx.GetBPop().keys.insert(BlockingState::BlockKeyTable::value_type(prefix, vals[i])); } m_blocked_ctxs[prefix].insert(&ctx); } return 0; }
/** * wait for specified timeout value, if timeout < 0, then wait for ever */ bool Wait(uint64 timeout = 0, TimeUnit unit = MILLIS) { if (timeout <= 0) { return 0 == pthread_cond_wait(&m_cond, &m_mutex); } else { uint64 now = get_current_epoch_micros(); uint64 inc = microstime(timeout, unit); struct timespec ts; init_timespec(now + inc, MICROS, ts); return 0 == pthread_cond_timedwait(&m_cond, &m_mutex, &ts); } }
void Run() { while (running) { DBID firstDB = 0, lastDB = 0; if (0 == adb->LastDB(lastDB) && 0 == adb->FirstDB(firstDB)) { for (DBID db = firstDB; db <= lastDB; db++) { DBID nexdb = db; if (adb->DBExist(db, nexdb)) { adb->CheckExpireKey(db); } else { if (nexdb == db) { break; } adb->CheckExpireKey(nexdb); db = nexdb; } } } uint64 end = get_current_epoch_micros(); uint64 sleep = check_period; if (adb->m_min_expireat > end) { sleep = (adb->m_min_expireat - end) / 1000; if (sleep > check_period) { sleep = check_period; } if (sleep == 0) { sleep = 1; } } Thread::Sleep(sleep); } }
void Run() { while (running) { DBID dbid = 0; while (dbid < ARDB_GLOBAL_DB) { DBID nexdb = 0; if (adb->DBExist(dbid, nexdb)) { adb->CheckExpireKey(dbid); dbid++; } else { if (nexdb == dbid || nexdb == ARDB_GLOBAL_DB) { break; } adb->CheckExpireKey(nexdb); dbid = nexdb + 1; } } uint64 end = get_current_epoch_micros(); uint64 sleep = check_period; if (adb->m_min_expireat > end) { sleep = (adb->m_min_expireat - end) / 1000; if (sleep > check_period) { sleep = check_period; } if (sleep == 0) { sleep = 1; } } Thread::Sleep(sleep); } }
int Ardb::GeoSearchByOptions(Context& ctx, ValueObject& meta, GeoSearchOptions& options) { uint64 start_time = get_current_epoch_micros(); int ret = 0; double x = options.x, y = options.y; if (options.by_member) { Data element, score; element.SetString(options.member, true); ret = ZSetScore(ctx, meta, element, score); if (0 != ret || score.IsNil()) { return -1; } GeoHashHelper::GetMercatorXYByHash(score.value.iv, x, y); //GeoHashHelper::GetXYByHash(score.value.iv, x, y); } else { if (options.coord_type != GEO_MERCATOR_TYPE) { x = GeoHashHelper::GetMercatorX(options.x); y = GeoHashHelper::GetMercatorY(options.y); } } DEBUG_LOG("####Step1: Cost %lluus", get_current_epoch_micros() - start_time); GeoPointArray points; ZSetRangeByScoreOptions fetch_options; fetch_options.withscores = false; fetch_options.op = OP_GET; fetch_options.fill_reply = false; fetch_options.fetch_geo_location = true; if (options.in_members) { StringSet::iterator it = options.submembers.begin(); while (it != options.submembers.end()) { Data element, score; element.SetString(*it, true); Location loc; ret = ZSetScore(ctx, meta, element, score, &loc); if (0 == ret) { fetch_options.results.push_back(element); fetch_options.locs.push_back(loc); } it++; } } else { GeoHashBitsSet ress; GeoHashHelper::GetAreasByRadiusV2(GEO_MERCATOR_TYPE, y, x, options.radius, ress); /* * Merge areas if possible to avoid disk search */ std::vector<ZRangeSpec> range_array; GeoHashBitsSet::iterator rit = ress.begin(); typedef TreeMap<uint64, uint64>::Type HashRangeMap; HashRangeMap tmp; while (rit != ress.end()) { GeoHashBits& hash = *rit; GeoHashBits next = hash; next.bits++; tmp[GeoHashHelper::Allign60Bits(hash)] = GeoHashHelper::Allign60Bits(next); rit++; } HashRangeMap::iterator tit = tmp.begin(); HashRangeMap::iterator nit = tmp.begin(); nit++; while (tit != tmp.end()) { ZRangeSpec range; range.contain_min = true; range.contain_max = true; range.min.SetInt64(tit->first); range.max.SetInt64(tit->second); while (nit != tmp.end() && nit->first == range.max.value.iv) { range.max.SetInt64(nit->second); nit++; tit++; } range_array.push_back(range); nit++; tit++; } DEBUG_LOG("After areas merging, reduce searching area size from %u to %u", ress.size(), range_array.size()); std::vector<ZRangeSpec>::iterator hit = range_array.begin(); ZSetIterator* iter = NULL; while (hit != range_array.end()) { ZRangeSpec& range = *hit; uint64 t1 = get_current_epoch_millis(); ZSetRangeByScore(ctx, meta, range, fetch_options, iter); uint64 t2 = get_current_epoch_millis(); DEBUG_LOG("####Cost %llums to range fetch", t2 - t1); hit++; } DELETE(iter); } DEBUG_LOG("####Step2: Cost %lluus", get_current_epoch_micros() - start_time); uint32 outrange = 0; LocationDeque::iterator lit = fetch_options.locs.begin(); DataArray::iterator vit = fetch_options.results.begin(); while (vit != fetch_options.results.end()) { Location& loc = *lit; GeoPoint point; point.x = loc.x; point.y = loc.y; /* * distance accuracy is 0.2m */ if (GeoHashHelper::GetDistanceSquareIfInRadius(GEO_MERCATOR_TYPE, x, y, point.x, point.y, options.radius, point.distance, 0.2)) { vit->GetDecodeString(point.value); /* * filter by exclude/include */ if (!options.includes.empty() || !options.excludes.empty()) { Data subst; subst.SetString(point.value, false); bool matched = options.includes.empty() ? true : false; if (!options.includes.empty()) { StringStringMap::const_iterator sit = options.includes.begin(); while (sit != options.includes.end()) { Data mv; if (0 != MatchValueByPattern(ctx, sit->first, sit->second, subst, mv)) { matched = false; break; } else { matched = true; } sit++; } } if (matched && !options.excludes.empty()) { StringStringMap::const_iterator sit = options.excludes.begin(); while (sit != options.excludes.end()) { Data mv; if (0 == MatchValueByPattern(ctx, sit->first, sit->second, subst, mv)) { matched = false; break; } else { matched = true; } sit++; } } if (matched) { points.push_back(point); } } else { points.push_back(point); } } else { outrange++; } vit++; lit++; } DEBUG_LOG("###Result size:%d, outrange:%d", points.size(), outrange); DEBUG_LOG("####Step3: Cost %lluus", get_current_epoch_micros() - start_time); if (!options.nosort) { std::sort(points.begin(), points.end(), options.asc ? less_by_distance : great_by_distance); } DEBUG_LOG("####Step3.5: Cost %lluus", get_current_epoch_micros() - start_time); if (options.offset > 0) { if ((uint32) options.offset > points.size()) { points.clear(); } else { GeoPointArray::iterator start = points.begin() + options.offset; points.erase(points.begin(), start); } } if (options.limit > 0) { if ((uint32) options.limit < points.size()) { GeoPointArray::iterator end = points.begin() + options.limit; points.erase(end, points.end()); } } DEBUG_LOG("####Step4: Cost %lluus", get_current_epoch_micros() - start_time); ValueObjectMap meta_cache; GeoPointArray::iterator pit = points.begin(); while (pit != points.end()) { RedisReply& r = ctx.reply.AddMember(); fill_str_reply(r, pit->value); GeoGetOptionArray::const_iterator ait = options.get_patterns.begin(); while (ait != options.get_patterns.end()) { if (ait->get_distances) { RedisReply& rr = ctx.reply.AddMember(); rr.type = REDIS_REPLY_STRING; char dbuf[128]; int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", sqrt(pit->distance)); rr.str.assign(dbuf, dlen); } else if (ait->get_coodinates) { if (options.coord_type == GEO_WGS84_TYPE) { pit->x = GeoHashHelper::GetWGS84X(pit->x); pit->y = GeoHashHelper::GetWGS84Y(pit->y); } RedisReply& rr1 = ctx.reply.AddMember(); RedisReply& rr2 = ctx.reply.AddMember(); if (options.coord_type == GEO_WGS84_TYPE) { fill_double_reply(rr1, pit->x); fill_double_reply(rr2, pit->y); } else { char dbuf[128]; int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->x); rr1.type = REDIS_REPLY_STRING; rr1.str.assign(dbuf, dlen); dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->y); rr2.type = REDIS_REPLY_STRING; rr2.str.assign(dbuf, dlen); } } else if (ait->hgetall) { std::string keystr(ait->get_pattern.data(), ait->get_pattern.size()); string_replace(keystr, "*", pit->value); RedisReply& rr = ctx.reply.AddMember(); rr.type = REDIS_REPLY_ARRAY; HashGetAll(ctx, keystr, rr); } else { Data v, attr; v.SetString(pit->value, false); GetValueByPattern(ctx, ait->get_pattern, v, attr, &meta_cache); RedisReply& rr = ctx.reply.AddMember(); fill_value_reply(rr, attr); } ait++; } pit++; } DEBUG_LOG("####Step5: Cost %lluus", get_current_epoch_micros() - start_time); uint64 end_time = get_current_epoch_micros(); DEBUG_LOG("Cost %llu microseconds to search.", end_time - start_time); return points.size(); }
int Ardb::GeoSearch(const DBID& db, const Slice& key, const GeoSearchOptions& options, ValueDataDeque& results) { uint64 start_time = get_current_epoch_micros(); GeoHashBitsSet ress; double x = options.x, y = options.y; if (options.coord_type == GEO_WGS84_TYPE) { x = GeoHashHelper::GetMercatorX(options.x); y = GeoHashHelper::GetMercatorY(options.y); } if (options.by_member) { ValueData score, attr; int err = ZGetNodeValue(db, key, options.member, score, attr); if (0 != err) { return err; } Buffer attr_content(const_cast<char*>(attr.bytes_value.data()), 0, attr.bytes_value.size()); GeoPoint point; point.Decode(attr_content); x = point.x; y = point.y; } ZSetCacheElementSet subset; if (options.in_members) { StringSet::const_iterator sit = options.submembers.begin(); while (sit != options.submembers.end()) { ZSetCaheElement ele; ValueData score, attr; if (0 == ZGetNodeValue(db, key, *sit, score, attr)) { ele.score = score.NumberValue(); Buffer buf1, buf2; ValueData vv; vv.SetValue(*sit, true); vv.Encode(buf1); attr.Encode(buf2); ele.value.assign(buf1.GetRawReadBuffer(), buf1.ReadableBytes()); ele.attr.assign(buf2.GetRawReadBuffer(), buf2.ReadableBytes()); subset.insert(ele); } sit++; } } GeoHashHelper::GetAreasByRadius(GEO_MERCATOR_TYPE, y, x, options.radius, ress); /* * Merge areas if possible to avoid disk search */ std::vector<ZRangeSpec> range_array; GeoHashBitsSet::iterator rit = ress.begin(); GeoHashBitsSet::iterator next_it = ress.begin(); next_it++; while (rit != ress.end()) { GeoHashBits& hash = *rit; GeoHashBits next = hash; next.bits++; while (next_it != ress.end() && next.bits == next_it->bits) { next.bits++; next_it++; rit++; } ZRangeSpec range; range.contain_min = true; range.contain_max = false; range.min.SetIntValue(GeoHashHelper::Allign52Bits(hash)); range.max.SetIntValue(GeoHashHelper::Allign52Bits(next)); range_array.push_back(range); rit++; next_it++; } DEBUG_LOG("After areas merging, reduce searching area size from %u to %u", ress.size(), range_array.size()); GeoPointArray points; std::vector<ZRangeSpec>::iterator hit = range_array.begin(); Iterator* iter = NULL; while (hit != range_array.end()) { ZSetQueryOptions z_options; z_options.withscores = false; z_options.withattr = true; ValueDataArray values; if (options.in_members) { ZSetCache::GetRangeInZSetCache(subset, *hit, z_options.withscores, z_options.withattr, ZSetValueStoreCallback, &values); } else { ZRangeByScoreRange(db, key, *hit, iter, z_options, true, ZSetValueStoreCallback, &values); } ValueDataArray::iterator vit = values.begin(); while (vit != values.end()) { GeoPoint point; vit->ToString(point.value); //attributes vit++; Buffer content(const_cast<char*>(vit->bytes_value.data()), 0, vit->bytes_value.size()); if (point.Decode(content)) { if (GeoHashHelper::GetDistanceSquareIfInRadius(GEO_MERCATOR_TYPE, x, y, point.x, point.y, options.radius, point.distance)) { /* * filter by exclude/include */ if (!options.includes.empty() || !options.excludes.empty()) { ValueData subst; subst.SetValue(point.value, false); bool matched = options.includes.empty() ? true : false; if (!options.includes.empty()) { StringStringMap::const_iterator sit = options.includes.begin(); while (sit != options.includes.end()) { ValueData mv; if (0 != MatchValueByPattern(db, sit->first, sit->second, subst, mv)) { matched = false; break; } else { matched = true; } sit++; } } if (matched && !options.excludes.empty()) { StringStringMap::const_iterator sit = options.excludes.begin(); while (sit != options.excludes.end()) { ValueData mv; if (0 == MatchValueByPattern(db, sit->first, sit->second, subst, mv)) { matched = false; break; } else { matched = true; } sit++; } } if (matched) { points.push_back(point); } } else { points.push_back(point); } } else { //DEBUG_LOG("%s is not in radius:%.2fm", point.value.c_str(), options.radius); } } else { WARN_LOG("Failed to decode geo point."); } vit++; } hit++; } DELETE(iter); if (!options.nosort) { std::sort(points.begin(), points.end(), options.asc ? less_by_distance : great_by_distance); } if (options.offset > 0) { if ((uint32) options.offset > points.size()) { points.clear(); } else { GeoPointArray::iterator start = points.begin() + options.offset; points.erase(points.begin(), start); } } if (options.limit > 0) { if ((uint32) options.limit < points.size()) { GeoPointArray::iterator end = points.begin() + options.limit; points.erase(end, points.end()); } } GeoPointArray::iterator pit = points.begin(); while (pit != points.end()) { ValueData v; v.SetValue(pit->value, false); results.push_back(v); GeoGetOptionDeque::const_iterator ait = options.get_patterns.begin(); while (ait != options.get_patterns.end()) { ValueData attr; if (ait->get_distances) { attr.SetDoubleValue(sqrt(pit->distance)); results.push_back(attr); } else if (ait->get_coodinates) { if (options.coord_type == GEO_WGS84_TYPE) { pit->x = GeoHashHelper::GetWGS84X(pit->x); pit->y = GeoHashHelper::GetWGS84Y(pit->y); } attr.SetDoubleValue(pit->x); results.push_back(attr); attr.SetDoubleValue(pit->y); results.push_back(attr); } else if (ait->get_attr) { StringStringMap::iterator found = pit->attrs.find(ait->get_pattern); if (found != pit->attrs.end()) { attr.SetValue(found->second, false); } results.push_back(attr); } else { GetValueByPattern(db, ait->get_pattern, v, attr); results.push_back(attr); } ait++; } pit++; } uint64 end_time = get_current_epoch_micros(); DEBUG_LOG("Cost %llu microseconds to search.", end_time - start_time); return 0; }