int Ardb::ListInsert(Context& ctx, ValueObject& meta, const std::string* match, const std::string& value, bool head, bool abort_nonexist) { if (WakeBlockList(ctx, meta.key.key, value)) { fill_int_reply(ctx.reply, 1); return 0; } if (NULL != match) { if (meta.meta.Encoding() == COLLECTION_ENCODING_ZIPLIST) { Data element; element.SetString(value, true); DataArray::iterator zit = meta.meta.ziplist.begin(); while (zit != meta.meta.ziplist.end()) { std::string tmp; zit->GetDecodeString(tmp); if (tmp == *match) { break; } zit++; } if (zit == meta.meta.ziplist.end()) { fill_int_reply(ctx.reply, 0); return 0; } if (head) { meta.meta.ziplist.insert(zit, element); } else { zit++; if (zit != meta.meta.ziplist.end()) { meta.meta.ziplist.insert(zit, element); } else { meta.meta.ziplist.push_back(element); } } if (meta.meta.Length() > 1 && (meta.meta.Length() >= m_cfg.list_max_ziplist_entries || element.StringLength() >= m_cfg.list_max_ziplist_value)) { //convert to non ziplist ZipListConvert(ctx, meta); } } else { ListIterator iter; ListIter(ctx, meta, iter, false); std::string tmp; Data prev, next; Data current; bool matched = false; while (iter.Valid()) { if (iter.Element()->GetDecodeString(tmp) == (*match)) { current = *(iter.Score()); matched = true; if (head) { break; } } if (head) { prev = *(iter.Score()); iter.Next(); } else { if (matched) { next = *(iter.Score()); break; } } iter.Next(); } if (!matched) { fill_int_reply(ctx.reply, 0); return 0; } Data score; if (head) { if (prev.IsNil()) { score = current.IncrBy(-1); } else { score.SetDouble((prev.NumberValue() + current.NumberValue()) / 2); meta.meta.SetFlag(COLLECTION_FLAG_NORMAL); } } else { if (next.IsNil()) { score = current.IncrBy(1); } else { score.SetDouble((next.NumberValue() + current.NumberValue()) / 2); meta.meta.SetFlag(COLLECTION_FLAG_NORMAL); } } meta.meta.len++; ValueObject v; v.type = LIST_ELEMENT; v.element.SetString(value, true); v.key.db = meta.key.db; v.key.key = meta.key.key; v.key.type = LIST_ELEMENT; v.key.score = score; SetKeyValue(ctx, v); } fill_int_reply(ctx.reply, meta.meta.Length()); return 0; } else { if (meta.meta.Encoding() == COLLECTION_ENCODING_ZIPLIST) { Data element; element.SetString(value, true); if (head) { meta.meta.ziplist.push_front(element); } else { meta.meta.ziplist.push_back(element); } if (meta.meta.Length() >= m_cfg.list_max_ziplist_entries || element.StringLength() >= m_cfg.list_max_ziplist_value) { //convert to non ziplist ZipListConvert(ctx, meta); } } else { meta.meta.len++; ValueObject v; v.type = LIST_ELEMENT; v.element.SetString(value, true); v.key.db = meta.key.db; v.key.key = meta.key.key; v.key.type = LIST_ELEMENT; if (head) { v.key.score = meta.meta.min_index.IncrBy(-1); } else { v.key.score = meta.meta.max_index.IncrBy(1); } SetKeyValue(ctx, v); } fill_int_reply(ctx.reply, meta.meta.Length()); return 0; } }
int Ardb::GeoSearchByOptions(Context& ctx, ValueObject& meta, GeoSearchOptions& options) { uint64 start_time = get_current_epoch_micros(); int ret = 0; double x = options.x, y = options.y; if (options.by_member) { Data element, score; element.SetString(options.member, true); ret = ZSetScore(ctx, meta, element, score); if (0 != ret || score.IsNil()) { return -1; } GeoHashHelper::GetMercatorXYByHash(score.value.iv, x, y); //GeoHashHelper::GetXYByHash(score.value.iv, x, y); } else { if (options.coord_type != GEO_MERCATOR_TYPE) { x = GeoHashHelper::GetMercatorX(options.x); y = GeoHashHelper::GetMercatorY(options.y); } } DEBUG_LOG("####Step1: Cost %lluus", get_current_epoch_micros() - start_time); GeoPointArray points; ZSetRangeByScoreOptions fetch_options; fetch_options.withscores = false; fetch_options.op = OP_GET; fetch_options.fill_reply = false; fetch_options.fetch_geo_location = true; if (options.in_members) { StringSet::iterator it = options.submembers.begin(); while (it != options.submembers.end()) { Data element, score; element.SetString(*it, true); Location loc; ret = ZSetScore(ctx, meta, element, score, &loc); if (0 == ret) { fetch_options.results.push_back(element); fetch_options.locs.push_back(loc); } it++; } } else { GeoHashBitsSet ress; GeoHashHelper::GetAreasByRadiusV2(GEO_MERCATOR_TYPE, y, x, options.radius, ress); /* * Merge areas if possible to avoid disk search */ std::vector<ZRangeSpec> range_array; GeoHashBitsSet::iterator rit = ress.begin(); typedef TreeMap<uint64, uint64>::Type HashRangeMap; HashRangeMap tmp; while (rit != ress.end()) { GeoHashBits& hash = *rit; GeoHashBits next = hash; next.bits++; tmp[GeoHashHelper::Allign60Bits(hash)] = GeoHashHelper::Allign60Bits(next); rit++; } HashRangeMap::iterator tit = tmp.begin(); HashRangeMap::iterator nit = tmp.begin(); nit++; while (tit != tmp.end()) { ZRangeSpec range; range.contain_min = true; range.contain_max = true; range.min.SetInt64(tit->first); range.max.SetInt64(tit->second); while (nit != tmp.end() && nit->first == range.max.value.iv) { range.max.SetInt64(nit->second); nit++; tit++; } range_array.push_back(range); nit++; tit++; } DEBUG_LOG("After areas merging, reduce searching area size from %u to %u", ress.size(), range_array.size()); std::vector<ZRangeSpec>::iterator hit = range_array.begin(); ZSetIterator* iter = NULL; while (hit != range_array.end()) { ZRangeSpec& range = *hit; uint64 t1 = get_current_epoch_millis(); ZSetRangeByScore(ctx, meta, range, fetch_options, iter); uint64 t2 = get_current_epoch_millis(); DEBUG_LOG("####Cost %llums to range fetch", t2 - t1); hit++; } DELETE(iter); } DEBUG_LOG("####Step2: Cost %lluus", get_current_epoch_micros() - start_time); uint32 outrange = 0; LocationDeque::iterator lit = fetch_options.locs.begin(); DataArray::iterator vit = fetch_options.results.begin(); while (vit != fetch_options.results.end()) { Location& loc = *lit; GeoPoint point; point.x = loc.x; point.y = loc.y; /* * distance accuracy is 0.2m */ if (GeoHashHelper::GetDistanceSquareIfInRadius(GEO_MERCATOR_TYPE, x, y, point.x, point.y, options.radius, point.distance, 0.2)) { vit->GetDecodeString(point.value); /* * filter by exclude/include */ if (!options.includes.empty() || !options.excludes.empty()) { Data subst; subst.SetString(point.value, false); bool matched = options.includes.empty() ? true : false; if (!options.includes.empty()) { StringStringMap::const_iterator sit = options.includes.begin(); while (sit != options.includes.end()) { Data mv; if (0 != MatchValueByPattern(ctx, sit->first, sit->second, subst, mv)) { matched = false; break; } else { matched = true; } sit++; } } if (matched && !options.excludes.empty()) { StringStringMap::const_iterator sit = options.excludes.begin(); while (sit != options.excludes.end()) { Data mv; if (0 == MatchValueByPattern(ctx, sit->first, sit->second, subst, mv)) { matched = false; break; } else { matched = true; } sit++; } } if (matched) { points.push_back(point); } } else { points.push_back(point); } } else { outrange++; } vit++; lit++; } DEBUG_LOG("###Result size:%d, outrange:%d", points.size(), outrange); DEBUG_LOG("####Step3: Cost %lluus", get_current_epoch_micros() - start_time); if (!options.nosort) { std::sort(points.begin(), points.end(), options.asc ? less_by_distance : great_by_distance); } DEBUG_LOG("####Step3.5: Cost %lluus", get_current_epoch_micros() - start_time); if (options.offset > 0) { if ((uint32) options.offset > points.size()) { points.clear(); } else { GeoPointArray::iterator start = points.begin() + options.offset; points.erase(points.begin(), start); } } if (options.limit > 0) { if ((uint32) options.limit < points.size()) { GeoPointArray::iterator end = points.begin() + options.limit; points.erase(end, points.end()); } } DEBUG_LOG("####Step4: Cost %lluus", get_current_epoch_micros() - start_time); ValueObjectMap meta_cache; GeoPointArray::iterator pit = points.begin(); while (pit != points.end()) { RedisReply& r = ctx.reply.AddMember(); fill_str_reply(r, pit->value); GeoGetOptionArray::const_iterator ait = options.get_patterns.begin(); while (ait != options.get_patterns.end()) { if (ait->get_distances) { RedisReply& rr = ctx.reply.AddMember(); rr.type = REDIS_REPLY_STRING; char dbuf[128]; int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", sqrt(pit->distance)); rr.str.assign(dbuf, dlen); } else if (ait->get_coodinates) { if (options.coord_type == GEO_WGS84_TYPE) { pit->x = GeoHashHelper::GetWGS84X(pit->x); pit->y = GeoHashHelper::GetWGS84Y(pit->y); } RedisReply& rr1 = ctx.reply.AddMember(); RedisReply& rr2 = ctx.reply.AddMember(); if (options.coord_type == GEO_WGS84_TYPE) { fill_double_reply(rr1, pit->x); fill_double_reply(rr2, pit->y); } else { char dbuf[128]; int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->x); rr1.type = REDIS_REPLY_STRING; rr1.str.assign(dbuf, dlen); dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->y); rr2.type = REDIS_REPLY_STRING; rr2.str.assign(dbuf, dlen); } } else if (ait->hgetall) { std::string keystr(ait->get_pattern.data(), ait->get_pattern.size()); string_replace(keystr, "*", pit->value); RedisReply& rr = ctx.reply.AddMember(); rr.type = REDIS_REPLY_ARRAY; HashGetAll(ctx, keystr, rr); } else { Data v, attr; v.SetString(pit->value, false); GetValueByPattern(ctx, ait->get_pattern, v, attr, &meta_cache); RedisReply& rr = ctx.reply.AddMember(); fill_value_reply(rr, attr); } ait++; } pit++; } DEBUG_LOG("####Step5: Cost %lluus", get_current_epoch_micros() - start_time); uint64 end_time = get_current_epoch_micros(); DEBUG_LOG("Cost %llu microseconds to search.", end_time - start_time); return points.size(); }
int Ardb::SortCommand(Context& ctx, const Slice& key, SortOptions& options, DataArray& values) { values.clear(); KeyType keytype = KEY_END; GetType(ctx, key, keytype); switch (keytype) { case LIST_META: { ListRange(ctx, key, 0, -1); break; } case SET_META: { SetMembers(ctx, key); break; } case ZSET_META: { ZSetRange(ctx, key, 0, -1, false, false, OP_GET); if (NULL == options.by) { options.nosort = true; } break; } default: { return ERR_INVALID_TYPE; } } DataArray sortvals; if (ctx.reply.MemberSize() > 0) { for (uint32 i = 0; i < ctx.reply.MemberSize(); i++) { Data v; v.SetString(ctx.reply.MemberAt(i).str, true); sortvals.push_back(v); } } if (sortvals.empty()) { return 0; } if (options.with_limit) { if (options.limit_offset < 0) { options.limit_offset = 0; } if ((uint32) options.limit_offset > sortvals.size()) { values.clear(); return 0; } if (options.limit_count < 0) { options.limit_count = sortvals.size(); } } std::vector<SortValue> sortvec; if (!options.nosort) { if (NULL != options.by) { sortvec.reserve(sortvals.size()); } for (uint32 i = 0; i < sortvals.size(); i++) { if (NULL != options.by) { sortvec.push_back(SortValue(&sortvals[i])); if (GetValueByPattern(ctx, options.by, sortvals[i], sortvec[i].cmp) < 0) { DEBUG_LOG("Failed to get value by pattern:%s", options.by); sortvec[i].cmp.Clear(); continue; } } if (options.with_alpha) { if (NULL != options.by) { sortvec[i].cmp.ToString(); } else { sortvals[i].ToString(); } } } if (NULL != options.by) { if (!options.is_desc) { std::sort(sortvec.begin(), sortvec.end(), less_value<SortValue>); } else { std::sort(sortvec.begin(), sortvec.end(), greater_value<SortValue>); } } else { if (!options.is_desc) { std::sort(sortvals.begin(), sortvals.end(), less_value<Data>); } else { std::sort(sortvals.begin(), sortvals.end(), greater_value<Data>); } } } if (!options.with_limit) { options.limit_offset = 0; options.limit_count = sortvals.size(); } uint32 count = 0; for (uint32 i = options.limit_offset; i < sortvals.size() && count < (uint32) options.limit_count; i++, count++) { Data* patternObj = NULL; if (NULL != options.by) { patternObj = sortvec[i].value; } else { patternObj = &(sortvals[i]); } if (options.get_patterns.empty()) { values.push_back(*patternObj); } else { for (uint32 j = 0; j < options.get_patterns.size(); j++) { Data vo; if (GetValueByPattern(ctx, options.get_patterns[j], *patternObj, vo) < 0) { DEBUG_LOG("Failed to get value by pattern for:%s", options.get_patterns[j]); vo.Clear(); } values.push_back(vo); } } } uint32 step = options.get_patterns.empty() ? 1 : options.get_patterns.size(); switch (options.aggregate) { case AGGREGATE_SUM: case AGGREGATE_AVG: { DataArray result; result.resize(step); for (uint32 i = 0; i < result.size(); i++) { for (uint32 j = i; j < values.size(); j += step) { result[i].IncrBy(values[j]); } } if (options.aggregate == AGGREGATE_AVG) { size_t count = values.size() / step; for (uint32 i = 0; i < result.size(); i++) { result[i].SetDouble(result[i].NumberValue() / count); } } values.assign(result.begin(), result.end()); break; } case AGGREGATE_MAX: case AGGREGATE_MIN: { DataArray result; result.resize(step); for (uint32 i = 0; i < result.size(); i++) { for (uint32 j = i; j < values.size(); j += step) { if (result[i].IsNil()) { result[i] = values[j]; } else { if (options.aggregate == AGGREGATE_MIN) { if (values[j] < result[i]) { result[i] = values[j]; } } else { if (values[j] > result[i]) { result[i] = values[j]; } } } } } values.assign(result.begin(), result.end()); break; } case AGGREGATE_COUNT: { size_t size = values.size() / step; values.clear(); Data v; v.SetInt64(size); values.push_back(v); break; } default: { break; } } if (options.store_dst != NULL && !values.empty()) { DeleteKey(ctx, options.store_dst); ValueObject list_meta; list_meta.key.key = options.store_dst; list_meta.key.type = KEY_META; list_meta.key.db = ctx.currentDB; list_meta.type = LIST_META; list_meta.meta.SetEncoding(COLLECTION_ECODING_ZIPLIST); BatchWriteGuard guard(GetKeyValueEngine()); DataArray::iterator it = values.begin(); while (it != values.end()) { if (!it->IsNil()) { std::string tmp; it->GetDecodeString(tmp); ListInsert(ctx, list_meta, NULL, tmp, false, false); } it++; } SetKeyValue(ctx, list_meta); } return 0; }