int Ardb::MatchValueByPattern(Context& ctx, const Slice& key_pattern, const Slice& value_pattern, Data& subst,
Data& value)
{
if (0 != GetValueByPattern(ctx, key_pattern, subst, value))
{
return -1;
}
std::string str;
subst.GetDecodeString(str);
std::string vpattern(value_pattern.data(), value_pattern.size());
return stringmatchlen(vpattern.c_str(), vpattern.size(), str.c_str(),str.size(), 0) == 1 ? 0 : -1;
}
int Ardb::GeoSearchByOptions(Context& ctx, ValueObject& meta, GeoSearchOptions& options)
{
uint64 start_time = get_current_epoch_micros();
int ret = 0;
double x = options.x, y = options.y;
if (options.by_member)
{
Data element, score;
element.SetString(options.member, true);
ret = ZSetScore(ctx, meta, element, score);
if (0 != ret || score.IsNil())
{
return -1;
}
GeoHashHelper::GetMercatorXYByHash(score.value.iv, x, y);
//GeoHashHelper::GetXYByHash(score.value.iv, x, y);
}
else
{
if (options.coord_type != GEO_MERCATOR_TYPE)
{
x = GeoHashHelper::GetMercatorX(options.x);
y = GeoHashHelper::GetMercatorY(options.y);
}
}
DEBUG_LOG("####Step1: Cost %lluus", get_current_epoch_micros() - start_time);
GeoPointArray points;
ZSetRangeByScoreOptions fetch_options;
fetch_options.withscores = false;
fetch_options.op = OP_GET;
fetch_options.fill_reply = false;
fetch_options.fetch_geo_location = true;
if (options.in_members)
{
StringSet::iterator it = options.submembers.begin();
while (it != options.submembers.end())
{
Data element, score;
element.SetString(*it, true);
Location loc;
ret = ZSetScore(ctx, meta, element, score, &loc);
if (0 == ret)
{
fetch_options.results.push_back(element);
fetch_options.locs.push_back(loc);
}
it++;
}
}
else
{
GeoHashBitsSet ress;
GeoHashHelper::GetAreasByRadiusV2(GEO_MERCATOR_TYPE, y, x, options.radius, ress);
/*
* Merge areas if possible to avoid disk search
*/
std::vector<ZRangeSpec> range_array;
GeoHashBitsSet::iterator rit = ress.begin();
typedef TreeMap<uint64, uint64>::Type HashRangeMap;
HashRangeMap tmp;
while (rit != ress.end())
{
GeoHashBits& hash = *rit;
GeoHashBits next = hash;
next.bits++;
tmp[GeoHashHelper::Allign60Bits(hash)] = GeoHashHelper::Allign60Bits(next);
rit++;
}
HashRangeMap::iterator tit = tmp.begin();
HashRangeMap::iterator nit = tmp.begin();
nit++;
while (tit != tmp.end())
{
ZRangeSpec range;
range.contain_min = true;
range.contain_max = true;
range.min.SetInt64(tit->first);
range.max.SetInt64(tit->second);
while (nit != tmp.end() && nit->first == range.max.value.iv)
{
range.max.SetInt64(nit->second);
nit++;
tit++;
}
range_array.push_back(range);
nit++;
tit++;
}
DEBUG_LOG("After areas merging, reduce searching area size from %u to %u", ress.size(), range_array.size());
std::vector<ZRangeSpec>::iterator hit = range_array.begin();
ZSetIterator* iter = NULL;
while (hit != range_array.end())
{
ZRangeSpec& range = *hit;
uint64 t1 = get_current_epoch_millis();
ZSetRangeByScore(ctx, meta, range, fetch_options, iter);
uint64 t2 = get_current_epoch_millis();
DEBUG_LOG("####Cost %llums to range fetch", t2 - t1);
hit++;
}
DELETE(iter);
}
DEBUG_LOG("####Step2: Cost %lluus", get_current_epoch_micros() - start_time);
uint32 outrange = 0;
LocationDeque::iterator lit = fetch_options.locs.begin();
DataArray::iterator vit = fetch_options.results.begin();
while (vit != fetch_options.results.end())
{
Location& loc = *lit;
GeoPoint point;
point.x = loc.x;
point.y = loc.y;
/*
* distance accuracy is 0.2m
*/
if (GeoHashHelper::GetDistanceSquareIfInRadius(GEO_MERCATOR_TYPE, x, y, point.x, point.y, options.radius,
point.distance, 0.2))
{
vit->GetDecodeString(point.value);
/*
* filter by exclude/include
*/
if (!options.includes.empty() || !options.excludes.empty())
{
Data subst;
subst.SetString(point.value, false);
bool matched = options.includes.empty() ? true : false;
if (!options.includes.empty())
{
StringStringMap::const_iterator sit = options.includes.begin();
while (sit != options.includes.end())
{
Data mv;
if (0 != MatchValueByPattern(ctx, sit->first, sit->second, subst, mv))
{
matched = false;
break;
}
else
{
matched = true;
}
sit++;
}
}
if (matched && !options.excludes.empty())
{
StringStringMap::const_iterator sit = options.excludes.begin();
while (sit != options.excludes.end())
{
Data mv;
if (0 == MatchValueByPattern(ctx, sit->first, sit->second, subst, mv))
{
matched = false;
break;
}
else
{
matched = true;
}
sit++;
}
}
if (matched)
{
points.push_back(point);
}
}
else
{
points.push_back(point);
}
}
else
{
outrange++;
}
vit++;
lit++;
}
DEBUG_LOG("###Result size:%d, outrange:%d", points.size(), outrange);
DEBUG_LOG("####Step3: Cost %lluus", get_current_epoch_micros() - start_time);
if (!options.nosort)
{
std::sort(points.begin(), points.end(), options.asc ? less_by_distance : great_by_distance);
}
DEBUG_LOG("####Step3.5: Cost %lluus", get_current_epoch_micros() - start_time);
if (options.offset > 0)
{
if ((uint32) options.offset > points.size())
{
points.clear();
}
else
{
GeoPointArray::iterator start = points.begin() + options.offset;
points.erase(points.begin(), start);
}
}
if (options.limit > 0)
{
if ((uint32) options.limit < points.size())
{
GeoPointArray::iterator end = points.begin() + options.limit;
points.erase(end, points.end());
}
}
DEBUG_LOG("####Step4: Cost %lluus", get_current_epoch_micros() - start_time);
ValueObjectMap meta_cache;
GeoPointArray::iterator pit = points.begin();
while (pit != points.end())
{
RedisReply& r = ctx.reply.AddMember();
fill_str_reply(r, pit->value);
GeoGetOptionArray::const_iterator ait = options.get_patterns.begin();
while (ait != options.get_patterns.end())
{
if (ait->get_distances)
{
RedisReply& rr = ctx.reply.AddMember();
rr.type = REDIS_REPLY_STRING;
char dbuf[128];
int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", sqrt(pit->distance));
rr.str.assign(dbuf, dlen);
}
else if (ait->get_coodinates)
{
if (options.coord_type == GEO_WGS84_TYPE)
{
pit->x = GeoHashHelper::GetWGS84X(pit->x);
pit->y = GeoHashHelper::GetWGS84Y(pit->y);
}
RedisReply& rr1 = ctx.reply.AddMember();
RedisReply& rr2 = ctx.reply.AddMember();
if (options.coord_type == GEO_WGS84_TYPE)
{
fill_double_reply(rr1, pit->x);
fill_double_reply(rr2, pit->y);
}
else
{
char dbuf[128];
int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->x);
rr1.type = REDIS_REPLY_STRING;
rr1.str.assign(dbuf, dlen);
dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->y);
rr2.type = REDIS_REPLY_STRING;
rr2.str.assign(dbuf, dlen);
}
}
else if (ait->hgetall)
{
std::string keystr(ait->get_pattern.data(), ait->get_pattern.size());
string_replace(keystr, "*", pit->value);
RedisReply& rr = ctx.reply.AddMember();
rr.type = REDIS_REPLY_ARRAY;
HashGetAll(ctx, keystr, rr);
}
else
{
Data v, attr;
v.SetString(pit->value, false);
GetValueByPattern(ctx, ait->get_pattern, v, attr, &meta_cache);
RedisReply& rr = ctx.reply.AddMember();
fill_value_reply(rr, attr);
}
ait++;
}
pit++;
}
DEBUG_LOG("####Step5: Cost %lluus", get_current_epoch_micros() - start_time);
uint64 end_time = get_current_epoch_micros();
DEBUG_LOG("Cost %llu microseconds to search.", end_time - start_time);
return points.size();
}
int Ardb::GeoSearch(const DBID& db, const Slice& key, const GeoSearchOptions& options, ValueDataDeque& results)
{
uint64 start_time = get_current_epoch_micros();
GeoHashBitsSet ress;
double x = options.x, y = options.y;
if (options.coord_type == GEO_WGS84_TYPE)
{
x = GeoHashHelper::GetMercatorX(options.x);
y = GeoHashHelper::GetMercatorY(options.y);
}
if (options.by_member)
{
ValueData score, attr;
int err = ZGetNodeValue(db, key, options.member, score, attr);
if (0 != err)
{
return err;
}
Buffer attr_content(const_cast<char*>(attr.bytes_value.data()), 0, attr.bytes_value.size());
GeoPoint point;
point.Decode(attr_content);
x = point.x;
y = point.y;
}
ZSetCacheElementSet subset;
if (options.in_members)
{
StringSet::const_iterator sit = options.submembers.begin();
while (sit != options.submembers.end())
{
ZSetCaheElement ele;
ValueData score, attr;
if (0 == ZGetNodeValue(db, key, *sit, score, attr))
{
ele.score = score.NumberValue();
Buffer buf1, buf2;
ValueData vv;
vv.SetValue(*sit, true);
vv.Encode(buf1);
attr.Encode(buf2);
ele.value.assign(buf1.GetRawReadBuffer(), buf1.ReadableBytes());
ele.attr.assign(buf2.GetRawReadBuffer(), buf2.ReadableBytes());
subset.insert(ele);
}
sit++;
}
}
GeoHashHelper::GetAreasByRadius(GEO_MERCATOR_TYPE, y, x, options.radius, ress);
/*
* Merge areas if possible to avoid disk search
*/
std::vector<ZRangeSpec> range_array;
GeoHashBitsSet::iterator rit = ress.begin();
GeoHashBitsSet::iterator next_it = ress.begin();
next_it++;
while (rit != ress.end())
{
GeoHashBits& hash = *rit;
GeoHashBits next = hash;
next.bits++;
while (next_it != ress.end() && next.bits == next_it->bits)
{
next.bits++;
next_it++;
rit++;
}
ZRangeSpec range;
range.contain_min = true;
range.contain_max = false;
range.min.SetIntValue(GeoHashHelper::Allign52Bits(hash));
range.max.SetIntValue(GeoHashHelper::Allign52Bits(next));
range_array.push_back(range);
rit++;
next_it++;
}
DEBUG_LOG("After areas merging, reduce searching area size from %u to %u", ress.size(), range_array.size());
GeoPointArray points;
std::vector<ZRangeSpec>::iterator hit = range_array.begin();
Iterator* iter = NULL;
while (hit != range_array.end())
{
ZSetQueryOptions z_options;
z_options.withscores = false;
z_options.withattr = true;
ValueDataArray values;
if (options.in_members)
{
ZSetCache::GetRangeInZSetCache(subset, *hit, z_options.withscores, z_options.withattr,
ZSetValueStoreCallback, &values);
}
else
{
ZRangeByScoreRange(db, key, *hit, iter, z_options, true, ZSetValueStoreCallback, &values);
}
ValueDataArray::iterator vit = values.begin();
while (vit != values.end())
{
GeoPoint point;
vit->ToString(point.value);
//attributes
vit++;
Buffer content(const_cast<char*>(vit->bytes_value.data()), 0, vit->bytes_value.size());
if (point.Decode(content))
{
if (GeoHashHelper::GetDistanceSquareIfInRadius(GEO_MERCATOR_TYPE, x, y, point.x, point.y,
options.radius, point.distance))
{
/*
* filter by exclude/include
*/
if (!options.includes.empty() || !options.excludes.empty())
{
ValueData subst;
subst.SetValue(point.value, false);
bool matched = options.includes.empty() ? true : false;
if (!options.includes.empty())
{
StringStringMap::const_iterator sit = options.includes.begin();
while (sit != options.includes.end())
{
ValueData mv;
if (0 != MatchValueByPattern(db, sit->first, sit->second, subst, mv))
{
matched = false;
break;
}
else
{
matched = true;
}
sit++;
}
}
if (matched && !options.excludes.empty())
{
StringStringMap::const_iterator sit = options.excludes.begin();
while (sit != options.excludes.end())
{
ValueData mv;
if (0 == MatchValueByPattern(db, sit->first, sit->second, subst, mv))
{
matched = false;
break;
}
else
{
matched = true;
}
sit++;
}
}
if (matched)
{
points.push_back(point);
}
}
else
{
points.push_back(point);
}
}
else
{
//DEBUG_LOG("%s is not in radius:%.2fm", point.value.c_str(), options.radius);
}
}
else
{
WARN_LOG("Failed to decode geo point.");
}
vit++;
}
hit++;
}
DELETE(iter);
if (!options.nosort)
{
std::sort(points.begin(), points.end(), options.asc ? less_by_distance : great_by_distance);
}
if (options.offset > 0)
{
if ((uint32) options.offset > points.size())
{
points.clear();
}
else
{
GeoPointArray::iterator start = points.begin() + options.offset;
points.erase(points.begin(), start);
}
}
if (options.limit > 0)
{
if ((uint32) options.limit < points.size())
{
GeoPointArray::iterator end = points.begin() + options.limit;
points.erase(end, points.end());
}
}
GeoPointArray::iterator pit = points.begin();
while (pit != points.end())
{
ValueData v;
v.SetValue(pit->value, false);
results.push_back(v);
GeoGetOptionDeque::const_iterator ait = options.get_patterns.begin();
while (ait != options.get_patterns.end())
{
ValueData attr;
if (ait->get_distances)
{
attr.SetDoubleValue(sqrt(pit->distance));
results.push_back(attr);
}
else if (ait->get_coodinates)
{
if (options.coord_type == GEO_WGS84_TYPE)
{
pit->x = GeoHashHelper::GetWGS84X(pit->x);
pit->y = GeoHashHelper::GetWGS84Y(pit->y);
}
attr.SetDoubleValue(pit->x);
results.push_back(attr);
attr.SetDoubleValue(pit->y);
results.push_back(attr);
}
else if (ait->get_attr)
{
StringStringMap::iterator found = pit->attrs.find(ait->get_pattern);
if (found != pit->attrs.end())
{
attr.SetValue(found->second, false);
}
results.push_back(attr);
}
else
{
GetValueByPattern(db, ait->get_pattern, v, attr);
results.push_back(attr);
}
ait++;
}
pit++;
}
uint64 end_time = get_current_epoch_micros();
DEBUG_LOG("Cost %llu microseconds to search.", end_time - start_time);
return 0;
}
int Ardb::Sort(const DBID& db, const Slice& key, const StringArray& args,
ValueArray& values)
{
SortOptions options;
if (parse_sort_options(options, args) < 0)
{
DEBUG_LOG("Failed to parse sort options.");
return ERR_INVALID_ARGS;
}
int type = Type(db, key);
ValueArray sortvals;
switch (type)
{
case LIST_META:
{
LRange(db, key, 0, -1, sortvals);
break;
}
case SET_ELEMENT:
{
SMembers(db, key, sortvals);
break;
}
case ZSET_ELEMENT_SCORE:
{
QueryOptions tmp;
ZRange(db, key, 0, -1, sortvals, tmp);
if(NULL == options.by)
{
options.nosort = true;
}
break;
}
default:
{
return ERR_INVALID_TYPE;
}
}
if (sortvals.empty())
{
return 0;
}
if (options.with_limit)
{
if (options.limit_offset < 0)
{
options.limit_offset = 0;
}
if ((uint32) options.limit_offset > sortvals.size())
{
values.clear();
return 0;
}
if (options.limit_count < 0)
{
options.limit_count = sortvals.size();
}
}
std::vector<SortValue> sortvec;
if (!options.nosort)
{
if (NULL != options.by)
{
sortvec.reserve(sortvals.size());
}
for (uint32 i = 0; i < sortvals.size(); i++)
{
if (NULL != options.by)
{
sortvec.push_back(SortValue(&sortvals[i]));
if (GetValueByPattern(db, options.by, sortvals[i],
sortvec[i].cmp) < 0)
{
DEBUG_LOG("Failed to get value by pattern:%s", options.by);
sortvec[i].cmp.Clear();
continue;
}
}
if (options.with_alpha)
{
if (NULL != options.by)
{
value_convert_to_raw(sortvec[i].cmp);
}
else
{
value_convert_to_raw(sortvals[i]);
}
}
else
{
if (NULL != options.by)
{
value_convert_to_number(sortvec[i].cmp);
}
else
{
value_convert_to_number(sortvals[i]);
}
}
}
if (NULL != options.by)
{
if (!options.is_desc)
{
std::sort(sortvec.begin(), sortvec.end(),
less_value<SortValue>);
}
else
{
std::sort(sortvec.begin(), sortvec.end(),
greater_value<SortValue>);
}
}
else
{
if (!options.is_desc)
{
std::sort(sortvals.begin(), sortvals.end(),
less_value<ValueObject>);
}
else
{
std::sort(sortvals.begin(), sortvals.end(),
greater_value<ValueObject>);
}
}
}
if (!options.with_limit)
{
options.limit_offset = 0;
options.limit_count = sortvals.size();
}
uint32 count = 0;
for (uint32 i = options.limit_offset;
i < sortvals.size() && count < (uint32) options.limit_count;
i++, count++)
{
ValueObject* patternObj = NULL;
if (NULL != options.by)
{
patternObj = sortvec[i].value;
}
else
{
patternObj = &(sortvals[i]);
}
if (options.get_patterns.empty())
{
values.push_back(*patternObj);
}
else
{
for (uint32 j = 0; j < options.get_patterns.size(); j++)
{
ValueObject vo;
if (GetValueByPattern(db, options.get_patterns[j],
*patternObj, vo) < 0)
{
DEBUG_LOG("Failed to get value by pattern for:%s", options.get_patterns[j]);
vo.Clear();
}
values.push_back(vo);
}
}
}
if (options.store_dst != NULL && !values.empty())
{
BatchWriteGuard guard(GetEngine());
LClear(db, options.store_dst);
ValueArray::iterator it = values.begin();
uint64 score = 0;
while (it != values.end())
{
if (it->type != EMPTY)
{
ListKeyObject lk(options.store_dst, score, db);
SetValue(lk, *it);
score++;
}
it++;
}
ListMetaValue meta;
meta.min_score = 0;
meta.max_score = (score - 1);
meta.size = score;
SetListMetaValue(db, options.store_dst, meta);
}
return 0;
}
int Ardb::SortCommand(Context& ctx, const Slice& key, SortOptions& options, DataArray& values)
{
values.clear();
KeyType keytype = KEY_END;
GetType(ctx, key, keytype);
switch (keytype)
{
case LIST_META:
{
ListRange(ctx, key, 0, -1);
break;
}
case SET_META:
{
SetMembers(ctx, key);
break;
}
case ZSET_META:
{
ZSetRange(ctx, key, 0, -1, false, false, OP_GET);
if (NULL == options.by)
{
options.nosort = true;
}
break;
}
default:
{
return ERR_INVALID_TYPE;
}
}
DataArray sortvals;
if (ctx.reply.MemberSize() > 0)
{
for (uint32 i = 0; i < ctx.reply.MemberSize(); i++)
{
Data v;
v.SetString(ctx.reply.MemberAt(i).str, true);
sortvals.push_back(v);
}
}
if (sortvals.empty())
{
return 0;
}
if (options.with_limit)
{
if (options.limit_offset < 0)
{
options.limit_offset = 0;
}
if ((uint32) options.limit_offset > sortvals.size())
{
values.clear();
return 0;
}
if (options.limit_count < 0)
{
options.limit_count = sortvals.size();
}
}
std::vector<SortValue> sortvec;
if (!options.nosort)
{
if (NULL != options.by)
{
sortvec.reserve(sortvals.size());
}
for (uint32 i = 0; i < sortvals.size(); i++)
{
if (NULL != options.by)
{
sortvec.push_back(SortValue(&sortvals[i]));
if (GetValueByPattern(ctx, options.by, sortvals[i], sortvec[i].cmp) < 0)
{
DEBUG_LOG("Failed to get value by pattern:%s", options.by);
sortvec[i].cmp.Clear();
continue;
}
}
if (options.with_alpha)
{
if (NULL != options.by)
{
sortvec[i].cmp.ToString();
}
else
{
sortvals[i].ToString();
}
}
}
if (NULL != options.by)
{
if (!options.is_desc)
{
std::sort(sortvec.begin(), sortvec.end(), less_value<SortValue>);
}
else
{
std::sort(sortvec.begin(), sortvec.end(), greater_value<SortValue>);
}
}
else
{
if (!options.is_desc)
{
std::sort(sortvals.begin(), sortvals.end(), less_value<Data>);
}
else
{
std::sort(sortvals.begin(), sortvals.end(), greater_value<Data>);
}
}
}
if (!options.with_limit)
{
options.limit_offset = 0;
options.limit_count = sortvals.size();
}
uint32 count = 0;
for (uint32 i = options.limit_offset; i < sortvals.size() && count < (uint32) options.limit_count; i++, count++)
{
Data* patternObj = NULL;
if (NULL != options.by)
{
patternObj = sortvec[i].value;
}
else
{
patternObj = &(sortvals[i]);
}
if (options.get_patterns.empty())
{
values.push_back(*patternObj);
}
else
{
for (uint32 j = 0; j < options.get_patterns.size(); j++)
{
Data vo;
if (GetValueByPattern(ctx, options.get_patterns[j], *patternObj, vo) < 0)
{
DEBUG_LOG("Failed to get value by pattern for:%s", options.get_patterns[j]);
vo.Clear();
}
values.push_back(vo);
}
}
}
uint32 step = options.get_patterns.empty() ? 1 : options.get_patterns.size();
switch (options.aggregate)
{
case AGGREGATE_SUM:
case AGGREGATE_AVG:
{
DataArray result;
result.resize(step);
for (uint32 i = 0; i < result.size(); i++)
{
for (uint32 j = i; j < values.size(); j += step)
{
result[i].IncrBy(values[j]);
}
}
if (options.aggregate == AGGREGATE_AVG)
{
size_t count = values.size() / step;
for (uint32 i = 0; i < result.size(); i++)
{
result[i].SetDouble(result[i].NumberValue() / count);
}
}
values.assign(result.begin(), result.end());
break;
}
case AGGREGATE_MAX:
case AGGREGATE_MIN:
{
DataArray result;
result.resize(step);
for (uint32 i = 0; i < result.size(); i++)
{
for (uint32 j = i; j < values.size(); j += step)
{
if (result[i].IsNil())
{
result[i] = values[j];
}
else
{
if (options.aggregate == AGGREGATE_MIN)
{
if (values[j] < result[i])
{
result[i] = values[j];
}
}
else
{
if (values[j] > result[i])
{
result[i] = values[j];
}
}
}
}
}
values.assign(result.begin(), result.end());
break;
}
case AGGREGATE_COUNT:
{
size_t size = values.size() / step;
values.clear();
Data v;
v.SetInt64(size);
values.push_back(v);
break;
}
default:
{
break;
}
}
if (options.store_dst != NULL && !values.empty())
{
DeleteKey(ctx, options.store_dst);
ValueObject list_meta;
list_meta.key.key = options.store_dst;
list_meta.key.type = KEY_META;
list_meta.key.db = ctx.currentDB;
list_meta.type = LIST_META;
list_meta.meta.SetEncoding(COLLECTION_ECODING_ZIPLIST);
BatchWriteGuard guard(GetKeyValueEngine());
DataArray::iterator it = values.begin();
while (it != values.end())
{
if (!it->IsNil())
{
std::string tmp;
it->GetDecodeString(tmp);
ListInsert(ctx, list_meta, NULL, tmp, false, false);
}
it++;
}
SetKeyValue(ctx, list_meta);
}
return 0;
}