Ejemplo n.º 1
0
    int Ardb::ListInsert(Context& ctx, ValueObject& meta, const std::string* match, const std::string& value, bool head,
            bool abort_nonexist)
    {
        if (WakeBlockList(ctx, meta.key.key, value))
        {
            fill_int_reply(ctx.reply, 1);
            return 0;
        }
        if (NULL != match)
        {
            if (meta.meta.Encoding() == COLLECTION_ENCODING_ZIPLIST)
            {
                Data element;
                element.SetString(value, true);
                DataArray::iterator zit = meta.meta.ziplist.begin();
                while (zit != meta.meta.ziplist.end())
                {
                    std::string tmp;
                    zit->GetDecodeString(tmp);
                    if (tmp == *match)
                    {
                        break;
                    }
                    zit++;
                }
                if (zit == meta.meta.ziplist.end())
                {
                    fill_int_reply(ctx.reply, 0);
                    return 0;
                }
                if (head)
                {
                    meta.meta.ziplist.insert(zit, element);
                }
                else
                {
                    zit++;
                    if (zit != meta.meta.ziplist.end())
                    {
                        meta.meta.ziplist.insert(zit, element);
                    }
                    else
                    {
                        meta.meta.ziplist.push_back(element);
                    }
                }
                if (meta.meta.Length() > 1
                        && (meta.meta.Length() >= m_cfg.list_max_ziplist_entries
                                || element.StringLength() >= m_cfg.list_max_ziplist_value))
                {
                    //convert to non ziplist
                    ZipListConvert(ctx, meta);
                }
            }
            else
            {
                ListIterator iter;
                ListIter(ctx, meta, iter, false);
                std::string tmp;
                Data prev, next;
                Data current;
                bool matched = false;
                while (iter.Valid())
                {
                    if (iter.Element()->GetDecodeString(tmp) == (*match))
                    {
                        current = *(iter.Score());
                        matched = true;
                        if (head)
                        {
                            break;
                        }
                    }
                    if (head)
                    {
                        prev = *(iter.Score());
                        iter.Next();
                    }
                    else
                    {
                        if (matched)
                        {
                            next = *(iter.Score());
                            break;
                        }
                    }
                    iter.Next();
                }
                if (!matched)
                {
                    fill_int_reply(ctx.reply, 0);
                    return 0;
                }
                Data score;
                if (head)
                {
                    if (prev.IsNil())
                    {
                        score = current.IncrBy(-1);
                    }
                    else
                    {
                        score.SetDouble((prev.NumberValue() + current.NumberValue()) / 2);
                        meta.meta.SetFlag(COLLECTION_FLAG_NORMAL);
                    }
                }
                else
                {
                    if (next.IsNil())
                    {
                        score = current.IncrBy(1);
                    }
                    else
                    {
                        score.SetDouble((next.NumberValue() + current.NumberValue()) / 2);
                        meta.meta.SetFlag(COLLECTION_FLAG_NORMAL);
                    }
                }

                meta.meta.len++;
                ValueObject v;
                v.type = LIST_ELEMENT;
                v.element.SetString(value, true);
                v.key.db = meta.key.db;
                v.key.key = meta.key.key;
                v.key.type = LIST_ELEMENT;
                v.key.score = score;
                SetKeyValue(ctx, v);
            }
            fill_int_reply(ctx.reply, meta.meta.Length());
            return 0;
        }
        else
        {
            if (meta.meta.Encoding() == COLLECTION_ENCODING_ZIPLIST)
            {
                Data element;
                element.SetString(value, true);
                if (head)
                {
                    meta.meta.ziplist.push_front(element);
                }
                else
                {
                    meta.meta.ziplist.push_back(element);
                }
                if (meta.meta.Length() >= m_cfg.list_max_ziplist_entries
                        || element.StringLength() >= m_cfg.list_max_ziplist_value)
                {
                    //convert to non ziplist
                    ZipListConvert(ctx, meta);
                }
            }
            else
            {
                meta.meta.len++;
                ValueObject v;
                v.type = LIST_ELEMENT;
                v.element.SetString(value, true);
                v.key.db = meta.key.db;
                v.key.key = meta.key.key;
                v.key.type = LIST_ELEMENT;

                if (head)
                {
                    v.key.score = meta.meta.min_index.IncrBy(-1);
                }
                else
                {
                    v.key.score = meta.meta.max_index.IncrBy(1);
                }
                SetKeyValue(ctx, v);
            }
            fill_int_reply(ctx.reply, meta.meta.Length());
            return 0;
        }
    }
Ejemplo n.º 2
0
Archivo: geo.cpp Proyecto: boreys/ardb
    int Ardb::GeoSearchByOptions(Context& ctx, ValueObject& meta, GeoSearchOptions& options)
    {
        uint64 start_time = get_current_epoch_micros();
        int ret = 0;
        double x = options.x, y = options.y;
        if (options.by_member)
        {
            Data element, score;
            element.SetString(options.member, true);
            ret = ZSetScore(ctx, meta, element, score);
            if (0 != ret || score.IsNil())
            {
                return -1;
            }
            GeoHashHelper::GetMercatorXYByHash(score.value.iv, x, y);
            //GeoHashHelper::GetXYByHash(score.value.iv, x, y);
        }
        else
        {
            if (options.coord_type != GEO_MERCATOR_TYPE)
            {
                x = GeoHashHelper::GetMercatorX(options.x);
                y = GeoHashHelper::GetMercatorY(options.y);
            }
        }
        DEBUG_LOG("####Step1: Cost %lluus", get_current_epoch_micros() - start_time);
        GeoPointArray points;
        ZSetRangeByScoreOptions fetch_options;
        fetch_options.withscores = false;
        fetch_options.op = OP_GET;
        fetch_options.fill_reply = false;
        fetch_options.fetch_geo_location = true;
        if (options.in_members)
        {
            StringSet::iterator it = options.submembers.begin();
            while (it != options.submembers.end())
            {
                Data element, score;
                element.SetString(*it, true);
                Location loc;
                ret = ZSetScore(ctx, meta, element, score, &loc);
                if (0 == ret)
                {
                    fetch_options.results.push_back(element);
                    fetch_options.locs.push_back(loc);
                }
                it++;
            }
        }
        else
        {
            GeoHashBitsSet ress;
            GeoHashHelper::GetAreasByRadiusV2(GEO_MERCATOR_TYPE, y, x, options.radius, ress);
            /*
             * Merge areas if possible to avoid disk search
             */
            std::vector<ZRangeSpec> range_array;
            GeoHashBitsSet::iterator rit = ress.begin();
            typedef TreeMap<uint64, uint64>::Type HashRangeMap;
            HashRangeMap tmp;
            while (rit != ress.end())
            {
                GeoHashBits& hash = *rit;
                GeoHashBits next = hash;
                next.bits++;
                tmp[GeoHashHelper::Allign60Bits(hash)] = GeoHashHelper::Allign60Bits(next);
                rit++;
            }
            HashRangeMap::iterator tit = tmp.begin();
            HashRangeMap::iterator nit = tmp.begin();
            nit++;
            while (tit != tmp.end())
            {
                ZRangeSpec range;
                range.contain_min = true;
                range.contain_max = true;
                range.min.SetInt64(tit->first);
                range.max.SetInt64(tit->second);
                while (nit != tmp.end() && nit->first == range.max.value.iv)
                {
                    range.max.SetInt64(nit->second);
                    nit++;
                    tit++;
                }
                range_array.push_back(range);
                nit++;
                tit++;
            }

            DEBUG_LOG("After areas merging, reduce searching area size from %u to %u", ress.size(), range_array.size());
            std::vector<ZRangeSpec>::iterator hit = range_array.begin();
            ZSetIterator* iter = NULL;
            while (hit != range_array.end())
            {
                ZRangeSpec& range = *hit;
                uint64 t1 = get_current_epoch_millis();
                ZSetRangeByScore(ctx, meta, range, fetch_options, iter);
                uint64 t2 = get_current_epoch_millis();
                DEBUG_LOG("####Cost %llums to range fetch", t2 - t1);
                hit++;
            }
            DELETE(iter);
        }
        DEBUG_LOG("####Step2: Cost %lluus", get_current_epoch_micros() - start_time);
        uint32 outrange = 0;
        LocationDeque::iterator lit = fetch_options.locs.begin();
        DataArray::iterator vit = fetch_options.results.begin();
        while (vit != fetch_options.results.end())
        {
            Location& loc = *lit;
            GeoPoint point;
            point.x = loc.x;
            point.y = loc.y;
            /*
             * distance accuracy is 0.2m
             */
            if (GeoHashHelper::GetDistanceSquareIfInRadius(GEO_MERCATOR_TYPE, x, y, point.x, point.y, options.radius,
                    point.distance, 0.2))
            {
                vit->GetDecodeString(point.value);
                /*
                 * filter by exclude/include
                 */
                if (!options.includes.empty() || !options.excludes.empty())
                {
                    Data subst;
                    subst.SetString(point.value, false);
                    bool matched = options.includes.empty() ? true : false;
                    if (!options.includes.empty())
                    {
                        StringStringMap::const_iterator sit = options.includes.begin();
                        while (sit != options.includes.end())
                        {
                            Data mv;
                            if (0 != MatchValueByPattern(ctx, sit->first, sit->second, subst, mv))
                            {
                                matched = false;
                                break;
                            }
                            else
                            {
                                matched = true;
                            }
                            sit++;
                        }
                    }
                    if (matched && !options.excludes.empty())
                    {
                        StringStringMap::const_iterator sit = options.excludes.begin();
                        while (sit != options.excludes.end())
                        {
                            Data mv;
                            if (0 == MatchValueByPattern(ctx, sit->first, sit->second, subst, mv))
                            {
                                matched = false;
                                break;
                            }
                            else
                            {
                                matched = true;
                            }
                            sit++;
                        }
                    }
                    if (matched)
                    {
                        points.push_back(point);
                    }
                }
                else
                {
                    points.push_back(point);
                }
            }
            else
            {
                outrange++;
            }
            vit++;
            lit++;
        }
        DEBUG_LOG("###Result size:%d,  outrange:%d", points.size(), outrange);
        DEBUG_LOG("####Step3: Cost %lluus", get_current_epoch_micros() - start_time);
        if (!options.nosort)
        {
            std::sort(points.begin(), points.end(), options.asc ? less_by_distance : great_by_distance);
        }
        DEBUG_LOG("####Step3.5: Cost %lluus", get_current_epoch_micros() - start_time);
        if (options.offset > 0)
        {
            if ((uint32) options.offset > points.size())
            {
                points.clear();
            }
            else
            {
                GeoPointArray::iterator start = points.begin() + options.offset;
                points.erase(points.begin(), start);
            }
        }
        if (options.limit > 0)
        {
            if ((uint32) options.limit < points.size())
            {
                GeoPointArray::iterator end = points.begin() + options.limit;
                points.erase(end, points.end());
            }
        }
        DEBUG_LOG("####Step4: Cost %lluus", get_current_epoch_micros() - start_time);
        ValueObjectMap meta_cache;
        GeoPointArray::iterator pit = points.begin();
        while (pit != points.end())
        {
            RedisReply& r = ctx.reply.AddMember();
            fill_str_reply(r, pit->value);
            GeoGetOptionArray::const_iterator ait = options.get_patterns.begin();
            while (ait != options.get_patterns.end())
            {
                if (ait->get_distances)
                {
                    RedisReply& rr = ctx.reply.AddMember();
                    rr.type = REDIS_REPLY_STRING;
                    char dbuf[128];
                    int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", sqrt(pit->distance));
                    rr.str.assign(dbuf, dlen);
                }
                else if (ait->get_coodinates)
                {
                    if (options.coord_type == GEO_WGS84_TYPE)
                    {
                        pit->x = GeoHashHelper::GetWGS84X(pit->x);
                        pit->y = GeoHashHelper::GetWGS84Y(pit->y);
                    }
                    RedisReply& rr1 = ctx.reply.AddMember();
                    RedisReply& rr2 = ctx.reply.AddMember();
                    if (options.coord_type == GEO_WGS84_TYPE)
                    {
                        fill_double_reply(rr1, pit->x);
                        fill_double_reply(rr2, pit->y);
                    }
                    else
                    {
                        char dbuf[128];
                        int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->x);
                        rr1.type = REDIS_REPLY_STRING;
                        rr1.str.assign(dbuf, dlen);
                        dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->y);
                        rr2.type = REDIS_REPLY_STRING;
                        rr2.str.assign(dbuf, dlen);
                    }
                }
                else if (ait->hgetall)
                {
                    std::string keystr(ait->get_pattern.data(), ait->get_pattern.size());
                    string_replace(keystr, "*", pit->value);
                    RedisReply& rr = ctx.reply.AddMember();
                    rr.type = REDIS_REPLY_ARRAY;
                    HashGetAll(ctx, keystr, rr);
                }
                else
                {
                    Data v, attr;
                    v.SetString(pit->value, false);
                    GetValueByPattern(ctx, ait->get_pattern, v, attr, &meta_cache);
                    RedisReply& rr = ctx.reply.AddMember();
                    fill_value_reply(rr, attr);
                }
                ait++;
            }
            pit++;
        }
        DEBUG_LOG("####Step5: Cost %lluus", get_current_epoch_micros() - start_time);
        uint64 end_time = get_current_epoch_micros();
        DEBUG_LOG("Cost %llu microseconds to search.", end_time - start_time);
        return points.size();
    }
Ejemplo n.º 3
0
    int Ardb::SortCommand(Context& ctx, const Slice& key, SortOptions& options, DataArray& values)
    {
        values.clear();

        KeyType keytype = KEY_END;
        GetType(ctx, key, keytype);

        switch (keytype)
        {
            case LIST_META:
            {
                ListRange(ctx, key, 0, -1);
                break;
            }
            case SET_META:
            {
                SetMembers(ctx, key);
                break;
            }
            case ZSET_META:
            {
                ZSetRange(ctx, key, 0, -1, false, false, OP_GET);
                if (NULL == options.by)
                {
                    options.nosort = true;
                }
                break;
            }
            default:
            {
                return ERR_INVALID_TYPE;
            }
        }
        DataArray sortvals;
        if (ctx.reply.MemberSize() > 0)
        {
            for (uint32 i = 0; i < ctx.reply.MemberSize(); i++)
            {
                Data v;
                v.SetString(ctx.reply.MemberAt(i).str, true);
                sortvals.push_back(v);
            }
        }
        if (sortvals.empty())
        {
            return 0;
        }
        if (options.with_limit)
        {
            if (options.limit_offset < 0)
            {
                options.limit_offset = 0;
            }
            if ((uint32) options.limit_offset > sortvals.size())
            {
                values.clear();
                return 0;
            }
            if (options.limit_count < 0)
            {
                options.limit_count = sortvals.size();
            }
        }

        std::vector<SortValue> sortvec;
        if (!options.nosort)
        {
            if (NULL != options.by)
            {
                sortvec.reserve(sortvals.size());
            }
            for (uint32 i = 0; i < sortvals.size(); i++)
            {
                if (NULL != options.by)
                {
                    sortvec.push_back(SortValue(&sortvals[i]));
                    if (GetValueByPattern(ctx, options.by, sortvals[i], sortvec[i].cmp) < 0)
                    {
                        DEBUG_LOG("Failed to get value by pattern:%s", options.by);
                        sortvec[i].cmp.Clear();
                        continue;
                    }
                }
                if (options.with_alpha)
                {
                    if (NULL != options.by)
                    {
                        sortvec[i].cmp.ToString();
                    }
                    else
                    {
                        sortvals[i].ToString();
                    }
                }
            }
            if (NULL != options.by)
            {
                if (!options.is_desc)
                {
                    std::sort(sortvec.begin(), sortvec.end(), less_value<SortValue>);
                }
                else
                {
                    std::sort(sortvec.begin(), sortvec.end(), greater_value<SortValue>);
                }

            }
            else
            {
                if (!options.is_desc)
                {
                    std::sort(sortvals.begin(), sortvals.end(), less_value<Data>);
                }
                else
                {
                    std::sort(sortvals.begin(), sortvals.end(), greater_value<Data>);
                }
            }
        }

        if (!options.with_limit)
        {
            options.limit_offset = 0;
            options.limit_count = sortvals.size();
        }

        uint32 count = 0;
        for (uint32 i = options.limit_offset; i < sortvals.size() && count < (uint32) options.limit_count; i++, count++)
        {
            Data* patternObj = NULL;
            if (NULL != options.by)
            {
                patternObj = sortvec[i].value;
            }
            else
            {
                patternObj = &(sortvals[i]);
            }
            if (options.get_patterns.empty())
            {
                values.push_back(*patternObj);
            }
            else
            {
                for (uint32 j = 0; j < options.get_patterns.size(); j++)
                {
                    Data vo;
                    if (GetValueByPattern(ctx, options.get_patterns[j], *patternObj, vo) < 0)
                    {
                        DEBUG_LOG("Failed to get value by pattern for:%s", options.get_patterns[j]);
                        vo.Clear();
                    }
                    values.push_back(vo);
                }
            }
        }

        uint32 step = options.get_patterns.empty() ? 1 : options.get_patterns.size();
        switch (options.aggregate)
        {
            case AGGREGATE_SUM:
            case AGGREGATE_AVG:
            {
                DataArray result;
                result.resize(step);

                for (uint32 i = 0; i < result.size(); i++)
                {
                    for (uint32 j = i; j < values.size(); j += step)
                    {
                        result[i].IncrBy(values[j]);
                    }
                }
                if (options.aggregate == AGGREGATE_AVG)
                {
                    size_t count = values.size() / step;
                    for (uint32 i = 0; i < result.size(); i++)
                    {
                        result[i].SetDouble(result[i].NumberValue() / count);
                    }
                }
                values.assign(result.begin(), result.end());
                break;
            }
            case AGGREGATE_MAX:
            case AGGREGATE_MIN:
            {
                DataArray result;
                result.resize(step);
                for (uint32 i = 0; i < result.size(); i++)
                {
                    for (uint32 j = i; j < values.size(); j += step)
                    {
                        if (result[i].IsNil())
                        {
                            result[i] = values[j];
                        }
                        else
                        {
                            if (options.aggregate == AGGREGATE_MIN)
                            {
                                if (values[j] < result[i])
                                {
                                    result[i] = values[j];
                                }
                            }
                            else
                            {
                                if (values[j] > result[i])
                                {
                                    result[i] = values[j];
                                }
                            }
                        }
                    }
                }
                values.assign(result.begin(), result.end());
                break;
            }
            case AGGREGATE_COUNT:
            {
                size_t size = values.size() / step;
                values.clear();
                Data v;
                v.SetInt64(size);
                values.push_back(v);
                break;
            }
            default:
            {
                break;
            }
        }

        if (options.store_dst != NULL && !values.empty())
        {
            DeleteKey(ctx, options.store_dst);

            ValueObject list_meta;
            list_meta.key.key = options.store_dst;
            list_meta.key.type = KEY_META;
            list_meta.key.db = ctx.currentDB;
            list_meta.type = LIST_META;
            list_meta.meta.SetEncoding(COLLECTION_ECODING_ZIPLIST);

            BatchWriteGuard guard(GetKeyValueEngine());
            DataArray::iterator it = values.begin();
            while (it != values.end())
            {
                if (!it->IsNil())
                {
                    std::string tmp;
                    it->GetDecodeString(tmp);
                    ListInsert(ctx, list_meta, NULL, tmp, false, false);
                }
                it++;
            }
            SetKeyValue(ctx, list_meta);
        }
        return 0;
    }