void test_geo_common(Context& ctx, Ardb& db)
{
db.GetConfig().zset_max_ziplist_entries = 16;
RedisCommandFrame del;
del.SetFullCommand("del mygeo");
db.Call(ctx, del, 0);
double x = 300.3;
double y = 300.3;
double p_x = 1000.0;
double p_y = 1000.0;
uint32 raius = 1000;
uint32 total = 100000;
GeoPointArray cmp;
for (uint32 i = 0; i < total; i++)
{
char name[100];
sprintf(name, "p%u", i);
/*
* min accuracy is 0.2meters
*/
double xx = x + i * 0.3;
double yy = y + i * 0.3;
if (((xx - p_x) * (xx - p_x) + (yy - p_y) * (yy - p_y)) <= raius * raius)
{
GeoPoint p;
p.x = xx;
p.y = yy;
cmp.push_back(p);
}
RedisCommandFrame geoadd;
geoadd.SetFullCommand("geoadd mygeo MERCATOR %.2f %.2f %s", xx, yy, name);
db.Call(ctx, geoadd, 0);
}
RedisCommandFrame zcard;
zcard.SetFullCommand("zcard mygeo");
db.Call(ctx, zcard, 0);
CHECK_FATAL(ctx.reply.integer != total, "geoadd failed");
RedisCommandFrame geosearch;
geosearch.SetFullCommand("geosearch mygeo MERCATOR %.2f %.2f radius %d ASC WITHCOORDINATES WITHDISTANCES", p_x, p_y,
raius);
db.Call(ctx, geosearch, 0);
CHECK_FATAL(ctx.reply.MemberSize() != cmp.size() * 4, "geosearch failed");
}
int Ardb::GeoSearch(const DBID& db, const Slice& key, const GeoSearchOptions& options, ValueDataDeque& results)
{
uint64 start_time = get_current_epoch_micros();
GeoHashBitsSet ress;
double x = options.x, y = options.y;
if (options.coord_type == GEO_WGS84_TYPE)
{
x = GeoHashHelper::GetMercatorX(options.x);
y = GeoHashHelper::GetMercatorY(options.y);
}
if (options.by_member)
{
ValueData score, attr;
int err = ZGetNodeValue(db, key, options.member, score, attr);
if (0 != err)
{
return err;
}
Buffer attr_content(const_cast<char*>(attr.bytes_value.data()), 0, attr.bytes_value.size());
GeoPoint point;
point.Decode(attr_content);
x = point.x;
y = point.y;
}
ZSetCacheElementSet subset;
if (options.in_members)
{
StringSet::const_iterator sit = options.submembers.begin();
while (sit != options.submembers.end())
{
ZSetCaheElement ele;
ValueData score, attr;
if (0 == ZGetNodeValue(db, key, *sit, score, attr))
{
ele.score = score.NumberValue();
Buffer buf1, buf2;
ValueData vv;
vv.SetValue(*sit, true);
vv.Encode(buf1);
attr.Encode(buf2);
ele.value.assign(buf1.GetRawReadBuffer(), buf1.ReadableBytes());
ele.attr.assign(buf2.GetRawReadBuffer(), buf2.ReadableBytes());
subset.insert(ele);
}
sit++;
}
}
GeoHashHelper::GetAreasByRadius(GEO_MERCATOR_TYPE, y, x, options.radius, ress);
/*
* Merge areas if possible to avoid disk search
*/
std::vector<ZRangeSpec> range_array;
GeoHashBitsSet::iterator rit = ress.begin();
GeoHashBitsSet::iterator next_it = ress.begin();
next_it++;
while (rit != ress.end())
{
GeoHashBits& hash = *rit;
GeoHashBits next = hash;
next.bits++;
while (next_it != ress.end() && next.bits == next_it->bits)
{
next.bits++;
next_it++;
rit++;
}
ZRangeSpec range;
range.contain_min = true;
range.contain_max = false;
range.min.SetIntValue(GeoHashHelper::Allign52Bits(hash));
range.max.SetIntValue(GeoHashHelper::Allign52Bits(next));
range_array.push_back(range);
rit++;
next_it++;
}
DEBUG_LOG("After areas merging, reduce searching area size from %u to %u", ress.size(), range_array.size());
GeoPointArray points;
std::vector<ZRangeSpec>::iterator hit = range_array.begin();
Iterator* iter = NULL;
while (hit != range_array.end())
{
ZSetQueryOptions z_options;
z_options.withscores = false;
z_options.withattr = true;
ValueDataArray values;
if (options.in_members)
{
ZSetCache::GetRangeInZSetCache(subset, *hit, z_options.withscores, z_options.withattr,
ZSetValueStoreCallback, &values);
}
else
{
ZRangeByScoreRange(db, key, *hit, iter, z_options, true, ZSetValueStoreCallback, &values);
}
ValueDataArray::iterator vit = values.begin();
while (vit != values.end())
{
GeoPoint point;
vit->ToString(point.value);
//attributes
vit++;
Buffer content(const_cast<char*>(vit->bytes_value.data()), 0, vit->bytes_value.size());
if (point.Decode(content))
{
if (GeoHashHelper::GetDistanceSquareIfInRadius(GEO_MERCATOR_TYPE, x, y, point.x, point.y,
options.radius, point.distance))
{
/*
* filter by exclude/include
*/
if (!options.includes.empty() || !options.excludes.empty())
{
ValueData subst;
subst.SetValue(point.value, false);
bool matched = options.includes.empty() ? true : false;
if (!options.includes.empty())
{
StringStringMap::const_iterator sit = options.includes.begin();
while (sit != options.includes.end())
{
ValueData mv;
if (0 != MatchValueByPattern(db, sit->first, sit->second, subst, mv))
{
matched = false;
break;
}
else
{
matched = true;
}
sit++;
}
}
if (matched && !options.excludes.empty())
{
StringStringMap::const_iterator sit = options.excludes.begin();
while (sit != options.excludes.end())
{
ValueData mv;
if (0 == MatchValueByPattern(db, sit->first, sit->second, subst, mv))
{
matched = false;
break;
}
else
{
matched = true;
}
sit++;
}
}
if (matched)
{
points.push_back(point);
}
}
else
{
points.push_back(point);
}
}
else
{
//DEBUG_LOG("%s is not in radius:%.2fm", point.value.c_str(), options.radius);
}
}
else
{
WARN_LOG("Failed to decode geo point.");
}
vit++;
}
hit++;
}
DELETE(iter);
if (!options.nosort)
{
std::sort(points.begin(), points.end(), options.asc ? less_by_distance : great_by_distance);
}
if (options.offset > 0)
{
if ((uint32) options.offset > points.size())
{
points.clear();
}
else
{
GeoPointArray::iterator start = points.begin() + options.offset;
points.erase(points.begin(), start);
}
}
if (options.limit > 0)
{
if ((uint32) options.limit < points.size())
{
GeoPointArray::iterator end = points.begin() + options.limit;
points.erase(end, points.end());
}
}
GeoPointArray::iterator pit = points.begin();
while (pit != points.end())
{
ValueData v;
v.SetValue(pit->value, false);
results.push_back(v);
GeoGetOptionDeque::const_iterator ait = options.get_patterns.begin();
while (ait != options.get_patterns.end())
{
ValueData attr;
if (ait->get_distances)
{
attr.SetDoubleValue(sqrt(pit->distance));
results.push_back(attr);
}
else if (ait->get_coodinates)
{
if (options.coord_type == GEO_WGS84_TYPE)
{
pit->x = GeoHashHelper::GetWGS84X(pit->x);
pit->y = GeoHashHelper::GetWGS84Y(pit->y);
}
attr.SetDoubleValue(pit->x);
results.push_back(attr);
attr.SetDoubleValue(pit->y);
results.push_back(attr);
}
else if (ait->get_attr)
{
StringStringMap::iterator found = pit->attrs.find(ait->get_pattern);
if (found != pit->attrs.end())
{
attr.SetValue(found->second, false);
}
results.push_back(attr);
}
else
{
GetValueByPattern(db, ait->get_pattern, v, attr);
results.push_back(attr);
}
ait++;
}
pit++;
}
uint64 end_time = get_current_epoch_micros();
DEBUG_LOG("Cost %llu microseconds to search.", end_time - start_time);
return 0;
}
int Ardb::GeoSearchByOptions(Context& ctx, ValueObject& meta, GeoSearchOptions& options)
{
uint64 start_time = get_current_epoch_micros();
int ret = 0;
double x = options.x, y = options.y;
if (options.by_member)
{
Data element, score;
element.SetString(options.member, true);
ret = ZSetScore(ctx, meta, element, score);
if (0 != ret || score.IsNil())
{
return -1;
}
GeoHashHelper::GetMercatorXYByHash(score.value.iv, x, y);
//GeoHashHelper::GetXYByHash(score.value.iv, x, y);
}
else
{
if (options.coord_type != GEO_MERCATOR_TYPE)
{
x = GeoHashHelper::GetMercatorX(options.x);
y = GeoHashHelper::GetMercatorY(options.y);
}
}
DEBUG_LOG("####Step1: Cost %lluus", get_current_epoch_micros() - start_time);
GeoPointArray points;
ZSetRangeByScoreOptions fetch_options;
fetch_options.withscores = false;
fetch_options.op = OP_GET;
fetch_options.fill_reply = false;
fetch_options.fetch_geo_location = true;
if (options.in_members)
{
StringSet::iterator it = options.submembers.begin();
while (it != options.submembers.end())
{
Data element, score;
element.SetString(*it, true);
Location loc;
ret = ZSetScore(ctx, meta, element, score, &loc);
if (0 == ret)
{
fetch_options.results.push_back(element);
fetch_options.locs.push_back(loc);
}
it++;
}
}
else
{
GeoHashBitsSet ress;
GeoHashHelper::GetAreasByRadiusV2(GEO_MERCATOR_TYPE, y, x, options.radius, ress);
/*
* Merge areas if possible to avoid disk search
*/
std::vector<ZRangeSpec> range_array;
GeoHashBitsSet::iterator rit = ress.begin();
typedef TreeMap<uint64, uint64>::Type HashRangeMap;
HashRangeMap tmp;
while (rit != ress.end())
{
GeoHashBits& hash = *rit;
GeoHashBits next = hash;
next.bits++;
tmp[GeoHashHelper::Allign60Bits(hash)] = GeoHashHelper::Allign60Bits(next);
rit++;
}
HashRangeMap::iterator tit = tmp.begin();
HashRangeMap::iterator nit = tmp.begin();
nit++;
while (tit != tmp.end())
{
ZRangeSpec range;
range.contain_min = true;
range.contain_max = true;
range.min.SetInt64(tit->first);
range.max.SetInt64(tit->second);
while (nit != tmp.end() && nit->first == range.max.value.iv)
{
range.max.SetInt64(nit->second);
nit++;
tit++;
}
range_array.push_back(range);
nit++;
tit++;
}
DEBUG_LOG("After areas merging, reduce searching area size from %u to %u", ress.size(), range_array.size());
std::vector<ZRangeSpec>::iterator hit = range_array.begin();
ZSetIterator* iter = NULL;
while (hit != range_array.end())
{
ZRangeSpec& range = *hit;
uint64 t1 = get_current_epoch_millis();
ZSetRangeByScore(ctx, meta, range, fetch_options, iter);
uint64 t2 = get_current_epoch_millis();
DEBUG_LOG("####Cost %llums to range fetch", t2 - t1);
hit++;
}
DELETE(iter);
}
DEBUG_LOG("####Step2: Cost %lluus", get_current_epoch_micros() - start_time);
uint32 outrange = 0;
LocationDeque::iterator lit = fetch_options.locs.begin();
DataArray::iterator vit = fetch_options.results.begin();
while (vit != fetch_options.results.end())
{
Location& loc = *lit;
GeoPoint point;
point.x = loc.x;
point.y = loc.y;
/*
* distance accuracy is 0.2m
*/
if (GeoHashHelper::GetDistanceSquareIfInRadius(GEO_MERCATOR_TYPE, x, y, point.x, point.y, options.radius,
point.distance, 0.2))
{
vit->GetDecodeString(point.value);
/*
* filter by exclude/include
*/
if (!options.includes.empty() || !options.excludes.empty())
{
Data subst;
subst.SetString(point.value, false);
bool matched = options.includes.empty() ? true : false;
if (!options.includes.empty())
{
StringStringMap::const_iterator sit = options.includes.begin();
while (sit != options.includes.end())
{
Data mv;
if (0 != MatchValueByPattern(ctx, sit->first, sit->second, subst, mv))
{
matched = false;
break;
}
else
{
matched = true;
}
sit++;
}
}
if (matched && !options.excludes.empty())
{
StringStringMap::const_iterator sit = options.excludes.begin();
while (sit != options.excludes.end())
{
Data mv;
if (0 == MatchValueByPattern(ctx, sit->first, sit->second, subst, mv))
{
matched = false;
break;
}
else
{
matched = true;
}
sit++;
}
}
if (matched)
{
points.push_back(point);
}
}
else
{
points.push_back(point);
}
}
else
{
outrange++;
}
vit++;
lit++;
}
DEBUG_LOG("###Result size:%d, outrange:%d", points.size(), outrange);
DEBUG_LOG("####Step3: Cost %lluus", get_current_epoch_micros() - start_time);
if (!options.nosort)
{
std::sort(points.begin(), points.end(), options.asc ? less_by_distance : great_by_distance);
}
DEBUG_LOG("####Step3.5: Cost %lluus", get_current_epoch_micros() - start_time);
if (options.offset > 0)
{
if ((uint32) options.offset > points.size())
{
points.clear();
}
else
{
GeoPointArray::iterator start = points.begin() + options.offset;
points.erase(points.begin(), start);
}
}
if (options.limit > 0)
{
if ((uint32) options.limit < points.size())
{
GeoPointArray::iterator end = points.begin() + options.limit;
points.erase(end, points.end());
}
}
DEBUG_LOG("####Step4: Cost %lluus", get_current_epoch_micros() - start_time);
ValueObjectMap meta_cache;
GeoPointArray::iterator pit = points.begin();
while (pit != points.end())
{
RedisReply& r = ctx.reply.AddMember();
fill_str_reply(r, pit->value);
GeoGetOptionArray::const_iterator ait = options.get_patterns.begin();
while (ait != options.get_patterns.end())
{
if (ait->get_distances)
{
RedisReply& rr = ctx.reply.AddMember();
rr.type = REDIS_REPLY_STRING;
char dbuf[128];
int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", sqrt(pit->distance));
rr.str.assign(dbuf, dlen);
}
else if (ait->get_coodinates)
{
if (options.coord_type == GEO_WGS84_TYPE)
{
pit->x = GeoHashHelper::GetWGS84X(pit->x);
pit->y = GeoHashHelper::GetWGS84Y(pit->y);
}
RedisReply& rr1 = ctx.reply.AddMember();
RedisReply& rr2 = ctx.reply.AddMember();
if (options.coord_type == GEO_WGS84_TYPE)
{
fill_double_reply(rr1, pit->x);
fill_double_reply(rr2, pit->y);
}
else
{
char dbuf[128];
int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->x);
rr1.type = REDIS_REPLY_STRING;
rr1.str.assign(dbuf, dlen);
dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->y);
rr2.type = REDIS_REPLY_STRING;
rr2.str.assign(dbuf, dlen);
}
}
else if (ait->hgetall)
{
std::string keystr(ait->get_pattern.data(), ait->get_pattern.size());
string_replace(keystr, "*", pit->value);
RedisReply& rr = ctx.reply.AddMember();
rr.type = REDIS_REPLY_ARRAY;
HashGetAll(ctx, keystr, rr);
}
else
{
Data v, attr;
v.SetString(pit->value, false);
GetValueByPattern(ctx, ait->get_pattern, v, attr, &meta_cache);
RedisReply& rr = ctx.reply.AddMember();
fill_value_reply(rr, attr);
}
ait++;
}
pit++;
}
DEBUG_LOG("####Step5: Cost %lluus", get_current_epoch_micros() - start_time);
uint64 end_time = get_current_epoch_micros();
DEBUG_LOG("Cost %llu microseconds to search.", end_time - start_time);
return points.size();
}
void test_geo(Ardb& db)
{
DBID dbid = 0;
db.ZClear(dbid, "mygeo");
double x = 300.3;
double y = 300.3;
double p_x = 1000.0;
double p_y = 1000.0;
double raius = 1000;
GeoPointArray cmp;
for (uint32 i = 0; i < 100000; i++)
{
char name[100];
sprintf(name, "p%u", i);
double xx = x + i * 0.1;
double yy = y + i * 0.1;
if (((xx - p_x) * (xx - p_x) + (yy - p_y) * (yy - p_y)) < raius * raius)
{
GeoPoint p;
p.x = xx;
p.y = yy;
cmp.push_back(p);
}
GeoAddOptions add;
add.coord_type = 2;
add.x = x + i * 0.1;
add.y = y + i * 0.1;
add.value = name;
db.GeoAdd(dbid, "mygeo", add);
}
if(db.GetL1Cache() != NULL)
{
db.GetL1Cache()->SyncLoad(dbid, "mygeo");
}
GeoSearchOptions options;
StringArray args;
std::string err;
string_to_string_array("MERCATOR 1000.0 1000.0 RADIUS 1000 ASC WITHCOORDINATES WITHDISTANCES", args);
options.Parse(args, err);
ValueDataDeque result;
db.GeoSearch(dbid, "mygeo", options, result);
CHECK_FATAL(cmp.size() != result.size() / 4, "Search failed with %u elements while expected %u", result.size() / 4,
cmp.size());
uint64 start = get_current_epoch_millis();
for (uint32 i = 0; i < 10000; i++)
{
result.clear();
options.x = p_x + i * 0.1;
options.y = p_y + i * 0.1;
options.radius = 100;
db.GeoSearch(dbid, "mygeo", options, result);
}
uint64 end = get_current_epoch_millis();
for (uint32 i = 0; i < result.size(); i += 4)
{
INFO_LOG("GeoPoint:%s x:%.2f, y:%.2f, distance:%.2f", result[i].bytes_value.c_str(),
result[i + 1].NumberValue(), result[i + 2].NumberValue(), result[i + 3].NumberValue());
}
INFO_LOG("Found %d points for search while expected %d points", result.size() / 4, cmp.size());
INFO_LOG("Cost %lldms to geo search 100000 zset elements 10000 times", (end - start));
}