int Ardb::ListInsert(Context& ctx, ValueObject& meta, const std::string* match, const std::string& value, bool head,
bool abort_nonexist)
{
if (WakeBlockList(ctx, meta.key.key, value))
{
fill_int_reply(ctx.reply, 1);
return 0;
}
if (NULL != match)
{
if (meta.meta.Encoding() == COLLECTION_ENCODING_ZIPLIST)
{
Data element;
element.SetString(value, true);
DataArray::iterator zit = meta.meta.ziplist.begin();
while (zit != meta.meta.ziplist.end())
{
std::string tmp;
zit->GetDecodeString(tmp);
if (tmp == *match)
{
break;
}
zit++;
}
if (zit == meta.meta.ziplist.end())
{
fill_int_reply(ctx.reply, 0);
return 0;
}
if (head)
{
meta.meta.ziplist.insert(zit, element);
}
else
{
zit++;
if (zit != meta.meta.ziplist.end())
{
meta.meta.ziplist.insert(zit, element);
}
else
{
meta.meta.ziplist.push_back(element);
}
}
if (meta.meta.Length() > 1
&& (meta.meta.Length() >= m_cfg.list_max_ziplist_entries
|| element.StringLength() >= m_cfg.list_max_ziplist_value))
{
//convert to non ziplist
ZipListConvert(ctx, meta);
}
}
else
{
ListIterator iter;
ListIter(ctx, meta, iter, false);
std::string tmp;
Data prev, next;
Data current;
bool matched = false;
while (iter.Valid())
{
if (iter.Element()->GetDecodeString(tmp) == (*match))
{
current = *(iter.Score());
matched = true;
if (head)
{
break;
}
}
if (head)
{
prev = *(iter.Score());
iter.Next();
}
else
{
if (matched)
{
next = *(iter.Score());
break;
}
}
iter.Next();
}
if (!matched)
{
fill_int_reply(ctx.reply, 0);
return 0;
}
Data score;
if (head)
{
if (prev.IsNil())
{
score = current.IncrBy(-1);
}
else
{
score.SetDouble((prev.NumberValue() + current.NumberValue()) / 2);
meta.meta.SetFlag(COLLECTION_FLAG_NORMAL);
}
}
else
{
if (next.IsNil())
{
score = current.IncrBy(1);
}
else
{
score.SetDouble((next.NumberValue() + current.NumberValue()) / 2);
meta.meta.SetFlag(COLLECTION_FLAG_NORMAL);
}
}
meta.meta.len++;
ValueObject v;
v.type = LIST_ELEMENT;
v.element.SetString(value, true);
v.key.db = meta.key.db;
v.key.key = meta.key.key;
v.key.type = LIST_ELEMENT;
v.key.score = score;
SetKeyValue(ctx, v);
}
fill_int_reply(ctx.reply, meta.meta.Length());
return 0;
}
else
{
if (meta.meta.Encoding() == COLLECTION_ENCODING_ZIPLIST)
{
Data element;
element.SetString(value, true);
if (head)
{
meta.meta.ziplist.push_front(element);
}
else
{
meta.meta.ziplist.push_back(element);
}
if (meta.meta.Length() >= m_cfg.list_max_ziplist_entries
|| element.StringLength() >= m_cfg.list_max_ziplist_value)
{
//convert to non ziplist
ZipListConvert(ctx, meta);
}
}
else
{
meta.meta.len++;
ValueObject v;
v.type = LIST_ELEMENT;
v.element.SetString(value, true);
v.key.db = meta.key.db;
v.key.key = meta.key.key;
v.key.type = LIST_ELEMENT;
if (head)
{
v.key.score = meta.meta.min_index.IncrBy(-1);
}
else
{
v.key.score = meta.meta.max_index.IncrBy(1);
}
SetKeyValue(ctx, v);
}
fill_int_reply(ctx.reply, meta.meta.Length());
return 0;
}
}
int Ardb::GeoSearchByOptions(Context& ctx, ValueObject& meta, GeoSearchOptions& options)
{
uint64 start_time = get_current_epoch_micros();
int ret = 0;
double x = options.x, y = options.y;
if (options.by_member)
{
Data element, score;
element.SetString(options.member, true);
ret = ZSetScore(ctx, meta, element, score);
if (0 != ret || score.IsNil())
{
return -1;
}
GeoHashHelper::GetMercatorXYByHash(score.value.iv, x, y);
//GeoHashHelper::GetXYByHash(score.value.iv, x, y);
}
else
{
if (options.coord_type != GEO_MERCATOR_TYPE)
{
x = GeoHashHelper::GetMercatorX(options.x);
y = GeoHashHelper::GetMercatorY(options.y);
}
}
DEBUG_LOG("####Step1: Cost %lluus", get_current_epoch_micros() - start_time);
GeoPointArray points;
ZSetRangeByScoreOptions fetch_options;
fetch_options.withscores = false;
fetch_options.op = OP_GET;
fetch_options.fill_reply = false;
fetch_options.fetch_geo_location = true;
if (options.in_members)
{
StringSet::iterator it = options.submembers.begin();
while (it != options.submembers.end())
{
Data element, score;
element.SetString(*it, true);
Location loc;
ret = ZSetScore(ctx, meta, element, score, &loc);
if (0 == ret)
{
fetch_options.results.push_back(element);
fetch_options.locs.push_back(loc);
}
it++;
}
}
else
{
GeoHashBitsSet ress;
GeoHashHelper::GetAreasByRadiusV2(GEO_MERCATOR_TYPE, y, x, options.radius, ress);
/*
* Merge areas if possible to avoid disk search
*/
std::vector<ZRangeSpec> range_array;
GeoHashBitsSet::iterator rit = ress.begin();
typedef TreeMap<uint64, uint64>::Type HashRangeMap;
HashRangeMap tmp;
while (rit != ress.end())
{
GeoHashBits& hash = *rit;
GeoHashBits next = hash;
next.bits++;
tmp[GeoHashHelper::Allign60Bits(hash)] = GeoHashHelper::Allign60Bits(next);
rit++;
}
HashRangeMap::iterator tit = tmp.begin();
HashRangeMap::iterator nit = tmp.begin();
nit++;
while (tit != tmp.end())
{
ZRangeSpec range;
range.contain_min = true;
range.contain_max = true;
range.min.SetInt64(tit->first);
range.max.SetInt64(tit->second);
while (nit != tmp.end() && nit->first == range.max.value.iv)
{
range.max.SetInt64(nit->second);
nit++;
tit++;
}
range_array.push_back(range);
nit++;
tit++;
}
DEBUG_LOG("After areas merging, reduce searching area size from %u to %u", ress.size(), range_array.size());
std::vector<ZRangeSpec>::iterator hit = range_array.begin();
ZSetIterator* iter = NULL;
while (hit != range_array.end())
{
ZRangeSpec& range = *hit;
uint64 t1 = get_current_epoch_millis();
ZSetRangeByScore(ctx, meta, range, fetch_options, iter);
uint64 t2 = get_current_epoch_millis();
DEBUG_LOG("####Cost %llums to range fetch", t2 - t1);
hit++;
}
DELETE(iter);
}
DEBUG_LOG("####Step2: Cost %lluus", get_current_epoch_micros() - start_time);
uint32 outrange = 0;
LocationDeque::iterator lit = fetch_options.locs.begin();
DataArray::iterator vit = fetch_options.results.begin();
while (vit != fetch_options.results.end())
{
Location& loc = *lit;
GeoPoint point;
point.x = loc.x;
point.y = loc.y;
/*
* distance accuracy is 0.2m
*/
if (GeoHashHelper::GetDistanceSquareIfInRadius(GEO_MERCATOR_TYPE, x, y, point.x, point.y, options.radius,
point.distance, 0.2))
{
vit->GetDecodeString(point.value);
/*
* filter by exclude/include
*/
if (!options.includes.empty() || !options.excludes.empty())
{
Data subst;
subst.SetString(point.value, false);
bool matched = options.includes.empty() ? true : false;
if (!options.includes.empty())
{
StringStringMap::const_iterator sit = options.includes.begin();
while (sit != options.includes.end())
{
Data mv;
if (0 != MatchValueByPattern(ctx, sit->first, sit->second, subst, mv))
{
matched = false;
break;
}
else
{
matched = true;
}
sit++;
}
}
if (matched && !options.excludes.empty())
{
StringStringMap::const_iterator sit = options.excludes.begin();
while (sit != options.excludes.end())
{
Data mv;
if (0 == MatchValueByPattern(ctx, sit->first, sit->second, subst, mv))
{
matched = false;
break;
}
else
{
matched = true;
}
sit++;
}
}
if (matched)
{
points.push_back(point);
}
}
else
{
points.push_back(point);
}
}
else
{
outrange++;
}
vit++;
lit++;
}
DEBUG_LOG("###Result size:%d, outrange:%d", points.size(), outrange);
DEBUG_LOG("####Step3: Cost %lluus", get_current_epoch_micros() - start_time);
if (!options.nosort)
{
std::sort(points.begin(), points.end(), options.asc ? less_by_distance : great_by_distance);
}
DEBUG_LOG("####Step3.5: Cost %lluus", get_current_epoch_micros() - start_time);
if (options.offset > 0)
{
if ((uint32) options.offset > points.size())
{
points.clear();
}
else
{
GeoPointArray::iterator start = points.begin() + options.offset;
points.erase(points.begin(), start);
}
}
if (options.limit > 0)
{
if ((uint32) options.limit < points.size())
{
GeoPointArray::iterator end = points.begin() + options.limit;
points.erase(end, points.end());
}
}
DEBUG_LOG("####Step4: Cost %lluus", get_current_epoch_micros() - start_time);
ValueObjectMap meta_cache;
GeoPointArray::iterator pit = points.begin();
while (pit != points.end())
{
RedisReply& r = ctx.reply.AddMember();
fill_str_reply(r, pit->value);
GeoGetOptionArray::const_iterator ait = options.get_patterns.begin();
while (ait != options.get_patterns.end())
{
if (ait->get_distances)
{
RedisReply& rr = ctx.reply.AddMember();
rr.type = REDIS_REPLY_STRING;
char dbuf[128];
int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", sqrt(pit->distance));
rr.str.assign(dbuf, dlen);
}
else if (ait->get_coodinates)
{
if (options.coord_type == GEO_WGS84_TYPE)
{
pit->x = GeoHashHelper::GetWGS84X(pit->x);
pit->y = GeoHashHelper::GetWGS84Y(pit->y);
}
RedisReply& rr1 = ctx.reply.AddMember();
RedisReply& rr2 = ctx.reply.AddMember();
if (options.coord_type == GEO_WGS84_TYPE)
{
fill_double_reply(rr1, pit->x);
fill_double_reply(rr2, pit->y);
}
else
{
char dbuf[128];
int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->x);
rr1.type = REDIS_REPLY_STRING;
rr1.str.assign(dbuf, dlen);
dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->y);
rr2.type = REDIS_REPLY_STRING;
rr2.str.assign(dbuf, dlen);
}
}
else if (ait->hgetall)
{
std::string keystr(ait->get_pattern.data(), ait->get_pattern.size());
string_replace(keystr, "*", pit->value);
RedisReply& rr = ctx.reply.AddMember();
rr.type = REDIS_REPLY_ARRAY;
HashGetAll(ctx, keystr, rr);
}
else
{
Data v, attr;
v.SetString(pit->value, false);
GetValueByPattern(ctx, ait->get_pattern, v, attr, &meta_cache);
RedisReply& rr = ctx.reply.AddMember();
fill_value_reply(rr, attr);
}
ait++;
}
pit++;
}
DEBUG_LOG("####Step5: Cost %lluus", get_current_epoch_micros() - start_time);
uint64 end_time = get_current_epoch_micros();
DEBUG_LOG("Cost %llu microseconds to search.", end_time - start_time);
return points.size();
}
int Ardb::SortCommand(Context& ctx, const Slice& key, SortOptions& options, DataArray& values)
{
values.clear();
KeyType keytype = KEY_END;
GetType(ctx, key, keytype);
switch (keytype)
{
case LIST_META:
{
ListRange(ctx, key, 0, -1);
break;
}
case SET_META:
{
SetMembers(ctx, key);
break;
}
case ZSET_META:
{
ZSetRange(ctx, key, 0, -1, false, false, OP_GET);
if (NULL == options.by)
{
options.nosort = true;
}
break;
}
default:
{
return ERR_INVALID_TYPE;
}
}
DataArray sortvals;
if (ctx.reply.MemberSize() > 0)
{
for (uint32 i = 0; i < ctx.reply.MemberSize(); i++)
{
Data v;
v.SetString(ctx.reply.MemberAt(i).str, true);
sortvals.push_back(v);
}
}
if (sortvals.empty())
{
return 0;
}
if (options.with_limit)
{
if (options.limit_offset < 0)
{
options.limit_offset = 0;
}
if ((uint32) options.limit_offset > sortvals.size())
{
values.clear();
return 0;
}
if (options.limit_count < 0)
{
options.limit_count = sortvals.size();
}
}
std::vector<SortValue> sortvec;
if (!options.nosort)
{
if (NULL != options.by)
{
sortvec.reserve(sortvals.size());
}
for (uint32 i = 0; i < sortvals.size(); i++)
{
if (NULL != options.by)
{
sortvec.push_back(SortValue(&sortvals[i]));
if (GetValueByPattern(ctx, options.by, sortvals[i], sortvec[i].cmp) < 0)
{
DEBUG_LOG("Failed to get value by pattern:%s", options.by);
sortvec[i].cmp.Clear();
continue;
}
}
if (options.with_alpha)
{
if (NULL != options.by)
{
sortvec[i].cmp.ToString();
}
else
{
sortvals[i].ToString();
}
}
}
if (NULL != options.by)
{
if (!options.is_desc)
{
std::sort(sortvec.begin(), sortvec.end(), less_value<SortValue>);
}
else
{
std::sort(sortvec.begin(), sortvec.end(), greater_value<SortValue>);
}
}
else
{
if (!options.is_desc)
{
std::sort(sortvals.begin(), sortvals.end(), less_value<Data>);
}
else
{
std::sort(sortvals.begin(), sortvals.end(), greater_value<Data>);
}
}
}
if (!options.with_limit)
{
options.limit_offset = 0;
options.limit_count = sortvals.size();
}
uint32 count = 0;
for (uint32 i = options.limit_offset; i < sortvals.size() && count < (uint32) options.limit_count; i++, count++)
{
Data* patternObj = NULL;
if (NULL != options.by)
{
patternObj = sortvec[i].value;
}
else
{
patternObj = &(sortvals[i]);
}
if (options.get_patterns.empty())
{
values.push_back(*patternObj);
}
else
{
for (uint32 j = 0; j < options.get_patterns.size(); j++)
{
Data vo;
if (GetValueByPattern(ctx, options.get_patterns[j], *patternObj, vo) < 0)
{
DEBUG_LOG("Failed to get value by pattern for:%s", options.get_patterns[j]);
vo.Clear();
}
values.push_back(vo);
}
}
}
uint32 step = options.get_patterns.empty() ? 1 : options.get_patterns.size();
switch (options.aggregate)
{
case AGGREGATE_SUM:
case AGGREGATE_AVG:
{
DataArray result;
result.resize(step);
for (uint32 i = 0; i < result.size(); i++)
{
for (uint32 j = i; j < values.size(); j += step)
{
result[i].IncrBy(values[j]);
}
}
if (options.aggregate == AGGREGATE_AVG)
{
size_t count = values.size() / step;
for (uint32 i = 0; i < result.size(); i++)
{
result[i].SetDouble(result[i].NumberValue() / count);
}
}
values.assign(result.begin(), result.end());
break;
}
case AGGREGATE_MAX:
case AGGREGATE_MIN:
{
DataArray result;
result.resize(step);
for (uint32 i = 0; i < result.size(); i++)
{
for (uint32 j = i; j < values.size(); j += step)
{
if (result[i].IsNil())
{
result[i] = values[j];
}
else
{
if (options.aggregate == AGGREGATE_MIN)
{
if (values[j] < result[i])
{
result[i] = values[j];
}
}
else
{
if (values[j] > result[i])
{
result[i] = values[j];
}
}
}
}
}
values.assign(result.begin(), result.end());
break;
}
case AGGREGATE_COUNT:
{
size_t size = values.size() / step;
values.clear();
Data v;
v.SetInt64(size);
values.push_back(v);
break;
}
default:
{
break;
}
}
if (options.store_dst != NULL && !values.empty())
{
DeleteKey(ctx, options.store_dst);
ValueObject list_meta;
list_meta.key.key = options.store_dst;
list_meta.key.type = KEY_META;
list_meta.key.db = ctx.currentDB;
list_meta.type = LIST_META;
list_meta.meta.SetEncoding(COLLECTION_ECODING_ZIPLIST);
BatchWriteGuard guard(GetKeyValueEngine());
DataArray::iterator it = values.begin();
while (it != values.end())
{
if (!it->IsNil())
{
std::string tmp;
it->GetDecodeString(tmp);
ListInsert(ctx, list_meta, NULL, tmp, false, false);
}
it++;
}
SetKeyValue(ctx, list_meta);
}
return 0;
}