Example #1
0
 int PerconaFTEngine::ListNameSpaces(Context& ctx, DataArray& nss)
 {
     RWLockGuard<SpinRWLock> guard(m_lock, true);
     FTDBTable::iterator it = m_dbs.begin();
     while (it != m_dbs.end())
     {
         nss.push_back(it->first);
         it++;
     }
     return 0;
 }
Example #2
0
/**
 * 準備
 */
static void setup(){
	std::vector<std::string>::iterator it = input_images.begin();
	for(; it != input_images.end(); it++){
		cv::Mat img = cv::imread(*it, -1);
		if(img.empty()){
			std::cerr << "[warrning] could not read file \"" << *it << "\"" << std::endl;
			continue;
		}
		std::cout << "[info] file:\"" << *it << "\" w:" << img.size().width << " h:" << img.size().height << " ch:" << img.channels() << std::endl;
		inputs.push_back(Data(*it, img.size().width, img.size().height));
		if(img.channels() == 4)
			with_alpha = true;
	}
}
Example #3
0
    int PerconaFTEngine::Init(const std::string& dir, const std::string& options)
    {
        Properties props;
        parse_conf_content(options, props);
        //parse config
        conf_get_int64(props, "cache_size", g_perconaft_config.cache_size);
        conf_get_uint32(props, "checkpoint_pool_threads", g_perconaft_config.checkpoint_pool_threads);
        conf_get_uint32(props, "checkpoint_period", g_perconaft_config.checkpoint_period);
        conf_get_uint32(props, "cleaner_period", g_perconaft_config.cleaner_period);
        conf_get_uint32(props, "cleaner_iterations", g_perconaft_config.cleaner_iterations);
        conf_get_bool(props, "evictor_enable_partial_eviction", g_perconaft_config.evictor_enable_partial_eviction);
        std::string compression;
        conf_get_string(props, "compression", compression);
        if (compression == "none")
        {
            g_perconaft_config.compression = TOKU_NO_COMPRESSION;
        }
        else if (compression == "snappy" || compression.empty())
        {
            g_perconaft_config.compression = TOKU_SNAPPY_METHOD;
        }
        else if (compression == "zlib")
        {
            g_perconaft_config.compression = TOKU_ZLIB_METHOD;
        }
        else
        {
            ERROR_LOG("Invalid compression config:%s for PercanoFT.", compression.c_str());
            return -1;
        }

        uint32 env_open_flags = DB_CREATE | DB_PRIVATE | DB_THREAD | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG | DB_RECOVER;
        int env_open_mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
        db_env_create(&m_env, 0);
        m_env->set_default_bt_compare(m_env, ardb_perconaft_compare);
        m_env->set_errcall(m_env, _err_callback);

        /*
         * set env config
         */
        uint32 cache_gsize = g_perconaft_config.cache_size >> 30;
        uint32 cache_bytes = g_perconaft_config.cache_size % (1024 * 1024 * 1024);
        m_env->set_cachesize(m_env, cache_gsize, cache_bytes, 1);
        m_env->set_cachetable_pool_threads(m_env, g_perconaft_config.checkpoint_pool_threads);
        m_env->checkpointing_set_period(m_env, g_perconaft_config.checkpoint_period);
        m_env->cleaner_set_period(m_env, g_perconaft_config.cleaner_period);
        m_env->cleaner_set_iterations(m_env, g_perconaft_config.cleaner_iterations);

        m_env->evictor_set_enable_partial_eviction(m_env, g_perconaft_config.evictor_enable_partial_eviction);

        int r = m_env->open(m_env, dir.c_str(), env_open_flags, env_open_mode);
        CHECK_EXPR(r);
        DataArray nss;
        if (0 == r)
        {
            g_toku_env = m_env;
            DB* db = m_env->get_db_for_directory(m_env);
            if (NULL != db)
            {
                PerconaFTLocalContext& local_ctx = g_local_ctx.GetValue();
                DB_TXN* txn = local_ctx.transc.Get();
                DBC *c = NULL;
                CHECK_EXPR(r = db->cursor(db, txn, &c, 0));
                if (0 == r)
                {
                    r = c->c_getf_first(c, 0, nil_callback, NULL);
                }
                while (0 == r)
                {
                    DBT raw_key;
                    DBT raw_val;
                    memset(&raw_key, 0, sizeof(raw_key));
                    memset(&raw_val, 0, sizeof(raw_key));
                    if (0 == c->c_get(c, &raw_key, &raw_val, DB_CURRENT))
                    {
                        //std::string ns_str
                        Data ns;
                        ns.SetString((const char*) raw_key.data, false);
                        INFO_LOG("TokuFT directory db %s:%s", (const char* ) raw_key.data, (const char* ) raw_val.data);
                        nss.push_back(ns);
                    }
                    r = c->c_getf_next(c, 0, nil_callback, NULL);
                }
                if (NULL != c)
                {
                    c->c_close(c);
                }
                local_ctx.transc.Release(true);
                r = 0;
            }
        }
        for (size_t i = 0; i < nss.size(); i++)
        {
            Context tmp;
            GetFTDB(tmp, nss[i], true);
        }
        return ENGINE_ERR(r);
    }
Example #4
0
    int Ardb::LTrim(Context& ctx, RedisCommandFrame& cmd)
    {
        int64 start, end;
        if (!GetInt64Value(ctx, cmd.GetArguments()[1], start) || !GetInt64Value(ctx, cmd.GetArguments()[2], end))
        {
            return 0;
        }
        ValueObject meta;
        int err = GetMetaValue(ctx, cmd.GetArguments()[0], LIST_META, meta);
        CHECK_ARDB_RETURN_VALUE(ctx.reply, err);
        if (0 != err)
        {
            fill_status_reply(ctx.reply, "OK");
            return 0;
        }
        /* convert negative indexes */
        if (start < 0)
            start = meta.meta.Length() + start;
        if (end < 0)
            end = meta.meta.Length() + end;
        if (start < 0)
            start = 0;
        if (end >= meta.meta.Length())
            end = meta.meta.Length() - 1;

        /* Invariant: start >= 0, so this test will be true when end < 0.
         * The range is empty when start > end or start >= length. */
        if (start > end || start >= meta.meta.Length())
        {
            /* Out of range start or start > end result in empty list */
            DeleteKey(ctx, cmd.GetArguments()[0]);
            return 0;
        }
        if (meta.meta.Encoding() == COLLECTION_ENCODING_ZIPLIST)
        {
            DataArray newzip;
            for (int64 i = start; i <= end; i++)
            {
                newzip.push_back(meta.meta.ziplist[i]);
            }
            meta.meta.ziplist = newzip;
            SetKeyValue(ctx, meta);
        }
        else
        {
            BatchWriteGuard guard(GetKeyValueEngine());
            if (meta.meta.IsSequentialList())
            {
                int64 listlen = meta.meta.Length();
                for (int64 s = 0; s < listlen; s++)
                {
                    if (s == start)
                    {
                        s = end;
                        continue;
                    }
                    KeyObject lk;
                    lk.db = meta.key.db;
                    lk.key = meta.key.key;
                    lk.type = LIST_ELEMENT;
                    lk.score = meta.meta.min_index.IncrBy(s);
                    meta.meta.len--;
                    DelKeyValue(ctx, lk);
                }
                meta.meta.max_index = meta.meta.min_index;
                meta.meta.min_index.IncrBy(start);
                meta.meta.max_index.IncrBy(end);
            }
            else
            {
                ListIterator iter;
                ListIter(ctx, meta, iter, false);
                int64 cursor = 0;
                while (iter.Valid())
                {
                    if (cursor < start || cursor > end)
                    {
                        DelRaw(ctx, iter.CurrentRawKey());
                        meta.meta.len--;
                    }
                    if (cursor == start)
                    {
                        meta.meta.min_index = *(iter.Element());
                    }
                    else if (cursor == end)
                    {
                        meta.meta.max_index = *(iter.Element());
                    }
                    cursor++;
                    iter.Next();
                }
            }
            SetKeyValue(ctx, meta);
        }
        return 0;
    }
Example #5
0
 int Ardb::LRem(Context& ctx, RedisCommandFrame& cmd)
 {
     int64 count;
     if (!GetInt64Value(ctx, cmd.GetArguments()[1], count))
     {
         return 0;
     }
     int64 toremove = std::abs(count);
     ValueObject meta;
     int err = GetMetaValue(ctx, cmd.GetArguments()[0], LIST_META, meta);
     CHECK_ARDB_RETURN_VALUE(ctx.reply, err);
     if (0 != err)
     {
         fill_int_reply(ctx.reply, 0);
         return 0;
     }
     Data element;
     element.SetString(cmd.GetArguments()[2], true);
     KeyLockerGuard lock(m_key_lock, ctx.currentDB, cmd.GetArguments()[0]);
     if (meta.meta.Encoding() == COLLECTION_ENCODING_ZIPLIST)
     {
         uint32 oldlen = meta.meta.ziplist.size();
         int64 removed = 0;
         DataArray newzip;
         if (count >= 0)
         {
             for (uint32 i = 0; i < oldlen; i++)
             {
                 if (meta.meta.ziplist[i] == element)
                 {
                     if (toremove == 0 || removed < toremove)
                     {
                         removed++;
                         continue;
                     }
                 }
                 newzip.push_back(meta.meta.ziplist[i]);
             }
         }
         else
         {
             for (uint32 i = 0; i < oldlen; i++)
             {
                 if (meta.meta.ziplist[oldlen - 1 - i] == element)
                 {
                     if (toremove == 0 || removed < toremove)
                     {
                         removed++;
                         continue;
                     }
                 }
                 newzip.push_front(meta.meta.ziplist[i]);
             }
         }
         if (removed > 0)
         {
             meta.meta.ziplist = newzip;
             SetKeyValue(ctx, meta);
         }
         fill_int_reply(ctx.reply, removed);
         return 0;
     }
     BatchWriteGuard guard(GetKeyValueEngine());
     ListIterator iter;
     ListIter(ctx, meta, iter, count < 0);
     int64 remove = 0;
     while (iter.Valid())
     {
         if (iter.Element()->Compare(element) == 0)
         {
             meta.meta.len--;
             meta.meta.SetFlag(COLLECTION_FLAG_NORMAL);
             DelRaw(ctx, iter.CurrentRawKey());
             //DelKeyValue(ctx, k);
             remove++;
             if (remove == toremove)
             {
                 break;
             }
         }
         if (count < 0)
         {
             iter.Prev();
         }
         else
         {
             iter.Next();
         }
     }
     if (remove > 0)
     {
         SetKeyValue(ctx, meta);
     }
     fill_int_reply(ctx.reply, remove);
     return 0;
 }
Example #6
0
    int Ardb::SortCommand(Context& ctx, const Slice& key, SortOptions& options, DataArray& values)
    {
        values.clear();

        KeyType keytype = KEY_END;
        GetType(ctx, key, keytype);

        switch (keytype)
        {
            case LIST_META:
            {
                ListRange(ctx, key, 0, -1);
                break;
            }
            case SET_META:
            {
                SetMembers(ctx, key);
                break;
            }
            case ZSET_META:
            {
                ZSetRange(ctx, key, 0, -1, false, false, OP_GET);
                if (NULL == options.by)
                {
                    options.nosort = true;
                }
                break;
            }
            default:
            {
                return ERR_INVALID_TYPE;
            }
        }
        DataArray sortvals;
        if (ctx.reply.MemberSize() > 0)
        {
            for (uint32 i = 0; i < ctx.reply.MemberSize(); i++)
            {
                Data v;
                v.SetString(ctx.reply.MemberAt(i).str, true);
                sortvals.push_back(v);
            }
        }
        if (sortvals.empty())
        {
            return 0;
        }
        if (options.with_limit)
        {
            if (options.limit_offset < 0)
            {
                options.limit_offset = 0;
            }
            if ((uint32) options.limit_offset > sortvals.size())
            {
                values.clear();
                return 0;
            }
            if (options.limit_count < 0)
            {
                options.limit_count = sortvals.size();
            }
        }

        std::vector<SortValue> sortvec;
        if (!options.nosort)
        {
            if (NULL != options.by)
            {
                sortvec.reserve(sortvals.size());
            }
            for (uint32 i = 0; i < sortvals.size(); i++)
            {
                if (NULL != options.by)
                {
                    sortvec.push_back(SortValue(&sortvals[i]));
                    if (GetValueByPattern(ctx, options.by, sortvals[i], sortvec[i].cmp) < 0)
                    {
                        DEBUG_LOG("Failed to get value by pattern:%s", options.by);
                        sortvec[i].cmp.Clear();
                        continue;
                    }
                }
                if (options.with_alpha)
                {
                    if (NULL != options.by)
                    {
                        sortvec[i].cmp.ToString();
                    }
                    else
                    {
                        sortvals[i].ToString();
                    }
                }
            }
            if (NULL != options.by)
            {
                if (!options.is_desc)
                {
                    std::sort(sortvec.begin(), sortvec.end(), less_value<SortValue>);
                }
                else
                {
                    std::sort(sortvec.begin(), sortvec.end(), greater_value<SortValue>);
                }

            }
            else
            {
                if (!options.is_desc)
                {
                    std::sort(sortvals.begin(), sortvals.end(), less_value<Data>);
                }
                else
                {
                    std::sort(sortvals.begin(), sortvals.end(), greater_value<Data>);
                }
            }
        }

        if (!options.with_limit)
        {
            options.limit_offset = 0;
            options.limit_count = sortvals.size();
        }

        uint32 count = 0;
        for (uint32 i = options.limit_offset; i < sortvals.size() && count < (uint32) options.limit_count; i++, count++)
        {
            Data* patternObj = NULL;
            if (NULL != options.by)
            {
                patternObj = sortvec[i].value;
            }
            else
            {
                patternObj = &(sortvals[i]);
            }
            if (options.get_patterns.empty())
            {
                values.push_back(*patternObj);
            }
            else
            {
                for (uint32 j = 0; j < options.get_patterns.size(); j++)
                {
                    Data vo;
                    if (GetValueByPattern(ctx, options.get_patterns[j], *patternObj, vo) < 0)
                    {
                        DEBUG_LOG("Failed to get value by pattern for:%s", options.get_patterns[j]);
                        vo.Clear();
                    }
                    values.push_back(vo);
                }
            }
        }

        uint32 step = options.get_patterns.empty() ? 1 : options.get_patterns.size();
        switch (options.aggregate)
        {
            case AGGREGATE_SUM:
            case AGGREGATE_AVG:
            {
                DataArray result;
                result.resize(step);

                for (uint32 i = 0; i < result.size(); i++)
                {
                    for (uint32 j = i; j < values.size(); j += step)
                    {
                        result[i].IncrBy(values[j]);
                    }
                }
                if (options.aggregate == AGGREGATE_AVG)
                {
                    size_t count = values.size() / step;
                    for (uint32 i = 0; i < result.size(); i++)
                    {
                        result[i].SetDouble(result[i].NumberValue() / count);
                    }
                }
                values.assign(result.begin(), result.end());
                break;
            }
            case AGGREGATE_MAX:
            case AGGREGATE_MIN:
            {
                DataArray result;
                result.resize(step);
                for (uint32 i = 0; i < result.size(); i++)
                {
                    for (uint32 j = i; j < values.size(); j += step)
                    {
                        if (result[i].IsNil())
                        {
                            result[i] = values[j];
                        }
                        else
                        {
                            if (options.aggregate == AGGREGATE_MIN)
                            {
                                if (values[j] < result[i])
                                {
                                    result[i] = values[j];
                                }
                            }
                            else
                            {
                                if (values[j] > result[i])
                                {
                                    result[i] = values[j];
                                }
                            }
                        }
                    }
                }
                values.assign(result.begin(), result.end());
                break;
            }
            case AGGREGATE_COUNT:
            {
                size_t size = values.size() / step;
                values.clear();
                Data v;
                v.SetInt64(size);
                values.push_back(v);
                break;
            }
            default:
            {
                break;
            }
        }

        if (options.store_dst != NULL && !values.empty())
        {
            DeleteKey(ctx, options.store_dst);

            ValueObject list_meta;
            list_meta.key.key = options.store_dst;
            list_meta.key.type = KEY_META;
            list_meta.key.db = ctx.currentDB;
            list_meta.type = LIST_META;
            list_meta.meta.SetEncoding(COLLECTION_ECODING_ZIPLIST);

            BatchWriteGuard guard(GetKeyValueEngine());
            DataArray::iterator it = values.begin();
            while (it != values.end())
            {
                if (!it->IsNil())
                {
                    std::string tmp;
                    it->GetDecodeString(tmp);
                    ListInsert(ctx, list_meta, NULL, tmp, false, false);
                }
                it++;
            }
            SetKeyValue(ctx, list_meta);
        }
        return 0;
    }