int PerconaFTEngine::ListNameSpaces(Context& ctx, DataArray& nss)
{
RWLockGuard<SpinRWLock> guard(m_lock, true);
FTDBTable::iterator it = m_dbs.begin();
while (it != m_dbs.end())
{
nss.push_back(it->first);
it++;
}
return 0;
}
/**
* 準備
*/
static void setup(){
std::vector<std::string>::iterator it = input_images.begin();
for(; it != input_images.end(); it++){
cv::Mat img = cv::imread(*it, -1);
if(img.empty()){
std::cerr << "[warrning] could not read file \"" << *it << "\"" << std::endl;
continue;
}
std::cout << "[info] file:\"" << *it << "\" w:" << img.size().width << " h:" << img.size().height << " ch:" << img.channels() << std::endl;
inputs.push_back(Data(*it, img.size().width, img.size().height));
if(img.channels() == 4)
with_alpha = true;
}
}
int PerconaFTEngine::Init(const std::string& dir, const std::string& options)
{
Properties props;
parse_conf_content(options, props);
//parse config
conf_get_int64(props, "cache_size", g_perconaft_config.cache_size);
conf_get_uint32(props, "checkpoint_pool_threads", g_perconaft_config.checkpoint_pool_threads);
conf_get_uint32(props, "checkpoint_period", g_perconaft_config.checkpoint_period);
conf_get_uint32(props, "cleaner_period", g_perconaft_config.cleaner_period);
conf_get_uint32(props, "cleaner_iterations", g_perconaft_config.cleaner_iterations);
conf_get_bool(props, "evictor_enable_partial_eviction", g_perconaft_config.evictor_enable_partial_eviction);
std::string compression;
conf_get_string(props, "compression", compression);
if (compression == "none")
{
g_perconaft_config.compression = TOKU_NO_COMPRESSION;
}
else if (compression == "snappy" || compression.empty())
{
g_perconaft_config.compression = TOKU_SNAPPY_METHOD;
}
else if (compression == "zlib")
{
g_perconaft_config.compression = TOKU_ZLIB_METHOD;
}
else
{
ERROR_LOG("Invalid compression config:%s for PercanoFT.", compression.c_str());
return -1;
}
uint32 env_open_flags = DB_CREATE | DB_PRIVATE | DB_THREAD | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG | DB_RECOVER;
int env_open_mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
db_env_create(&m_env, 0);
m_env->set_default_bt_compare(m_env, ardb_perconaft_compare);
m_env->set_errcall(m_env, _err_callback);
/*
* set env config
*/
uint32 cache_gsize = g_perconaft_config.cache_size >> 30;
uint32 cache_bytes = g_perconaft_config.cache_size % (1024 * 1024 * 1024);
m_env->set_cachesize(m_env, cache_gsize, cache_bytes, 1);
m_env->set_cachetable_pool_threads(m_env, g_perconaft_config.checkpoint_pool_threads);
m_env->checkpointing_set_period(m_env, g_perconaft_config.checkpoint_period);
m_env->cleaner_set_period(m_env, g_perconaft_config.cleaner_period);
m_env->cleaner_set_iterations(m_env, g_perconaft_config.cleaner_iterations);
m_env->evictor_set_enable_partial_eviction(m_env, g_perconaft_config.evictor_enable_partial_eviction);
int r = m_env->open(m_env, dir.c_str(), env_open_flags, env_open_mode);
CHECK_EXPR(r);
DataArray nss;
if (0 == r)
{
g_toku_env = m_env;
DB* db = m_env->get_db_for_directory(m_env);
if (NULL != db)
{
PerconaFTLocalContext& local_ctx = g_local_ctx.GetValue();
DB_TXN* txn = local_ctx.transc.Get();
DBC *c = NULL;
CHECK_EXPR(r = db->cursor(db, txn, &c, 0));
if (0 == r)
{
r = c->c_getf_first(c, 0, nil_callback, NULL);
}
while (0 == r)
{
DBT raw_key;
DBT raw_val;
memset(&raw_key, 0, sizeof(raw_key));
memset(&raw_val, 0, sizeof(raw_key));
if (0 == c->c_get(c, &raw_key, &raw_val, DB_CURRENT))
{
//std::string ns_str
Data ns;
ns.SetString((const char*) raw_key.data, false);
INFO_LOG("TokuFT directory db %s:%s", (const char* ) raw_key.data, (const char* ) raw_val.data);
nss.push_back(ns);
}
r = c->c_getf_next(c, 0, nil_callback, NULL);
}
if (NULL != c)
{
c->c_close(c);
}
local_ctx.transc.Release(true);
r = 0;
}
}
for (size_t i = 0; i < nss.size(); i++)
{
Context tmp;
GetFTDB(tmp, nss[i], true);
}
return ENGINE_ERR(r);
}
int Ardb::LTrim(Context& ctx, RedisCommandFrame& cmd)
{
int64 start, end;
if (!GetInt64Value(ctx, cmd.GetArguments()[1], start) || !GetInt64Value(ctx, cmd.GetArguments()[2], end))
{
return 0;
}
ValueObject meta;
int err = GetMetaValue(ctx, cmd.GetArguments()[0], LIST_META, meta);
CHECK_ARDB_RETURN_VALUE(ctx.reply, err);
if (0 != err)
{
fill_status_reply(ctx.reply, "OK");
return 0;
}
/* convert negative indexes */
if (start < 0)
start = meta.meta.Length() + start;
if (end < 0)
end = meta.meta.Length() + end;
if (start < 0)
start = 0;
if (end >= meta.meta.Length())
end = meta.meta.Length() - 1;
/* Invariant: start >= 0, so this test will be true when end < 0.
* The range is empty when start > end or start >= length. */
if (start > end || start >= meta.meta.Length())
{
/* Out of range start or start > end result in empty list */
DeleteKey(ctx, cmd.GetArguments()[0]);
return 0;
}
if (meta.meta.Encoding() == COLLECTION_ENCODING_ZIPLIST)
{
DataArray newzip;
for (int64 i = start; i <= end; i++)
{
newzip.push_back(meta.meta.ziplist[i]);
}
meta.meta.ziplist = newzip;
SetKeyValue(ctx, meta);
}
else
{
BatchWriteGuard guard(GetKeyValueEngine());
if (meta.meta.IsSequentialList())
{
int64 listlen = meta.meta.Length();
for (int64 s = 0; s < listlen; s++)
{
if (s == start)
{
s = end;
continue;
}
KeyObject lk;
lk.db = meta.key.db;
lk.key = meta.key.key;
lk.type = LIST_ELEMENT;
lk.score = meta.meta.min_index.IncrBy(s);
meta.meta.len--;
DelKeyValue(ctx, lk);
}
meta.meta.max_index = meta.meta.min_index;
meta.meta.min_index.IncrBy(start);
meta.meta.max_index.IncrBy(end);
}
else
{
ListIterator iter;
ListIter(ctx, meta, iter, false);
int64 cursor = 0;
while (iter.Valid())
{
if (cursor < start || cursor > end)
{
DelRaw(ctx, iter.CurrentRawKey());
meta.meta.len--;
}
if (cursor == start)
{
meta.meta.min_index = *(iter.Element());
}
else if (cursor == end)
{
meta.meta.max_index = *(iter.Element());
}
cursor++;
iter.Next();
}
}
SetKeyValue(ctx, meta);
}
return 0;
}
int Ardb::LRem(Context& ctx, RedisCommandFrame& cmd)
{
int64 count;
if (!GetInt64Value(ctx, cmd.GetArguments()[1], count))
{
return 0;
}
int64 toremove = std::abs(count);
ValueObject meta;
int err = GetMetaValue(ctx, cmd.GetArguments()[0], LIST_META, meta);
CHECK_ARDB_RETURN_VALUE(ctx.reply, err);
if (0 != err)
{
fill_int_reply(ctx.reply, 0);
return 0;
}
Data element;
element.SetString(cmd.GetArguments()[2], true);
KeyLockerGuard lock(m_key_lock, ctx.currentDB, cmd.GetArguments()[0]);
if (meta.meta.Encoding() == COLLECTION_ENCODING_ZIPLIST)
{
uint32 oldlen = meta.meta.ziplist.size();
int64 removed = 0;
DataArray newzip;
if (count >= 0)
{
for (uint32 i = 0; i < oldlen; i++)
{
if (meta.meta.ziplist[i] == element)
{
if (toremove == 0 || removed < toremove)
{
removed++;
continue;
}
}
newzip.push_back(meta.meta.ziplist[i]);
}
}
else
{
for (uint32 i = 0; i < oldlen; i++)
{
if (meta.meta.ziplist[oldlen - 1 - i] == element)
{
if (toremove == 0 || removed < toremove)
{
removed++;
continue;
}
}
newzip.push_front(meta.meta.ziplist[i]);
}
}
if (removed > 0)
{
meta.meta.ziplist = newzip;
SetKeyValue(ctx, meta);
}
fill_int_reply(ctx.reply, removed);
return 0;
}
BatchWriteGuard guard(GetKeyValueEngine());
ListIterator iter;
ListIter(ctx, meta, iter, count < 0);
int64 remove = 0;
while (iter.Valid())
{
if (iter.Element()->Compare(element) == 0)
{
meta.meta.len--;
meta.meta.SetFlag(COLLECTION_FLAG_NORMAL);
DelRaw(ctx, iter.CurrentRawKey());
//DelKeyValue(ctx, k);
remove++;
if (remove == toremove)
{
break;
}
}
if (count < 0)
{
iter.Prev();
}
else
{
iter.Next();
}
}
if (remove > 0)
{
SetKeyValue(ctx, meta);
}
fill_int_reply(ctx.reply, remove);
return 0;
}
int Ardb::SortCommand(Context& ctx, const Slice& key, SortOptions& options, DataArray& values)
{
values.clear();
KeyType keytype = KEY_END;
GetType(ctx, key, keytype);
switch (keytype)
{
case LIST_META:
{
ListRange(ctx, key, 0, -1);
break;
}
case SET_META:
{
SetMembers(ctx, key);
break;
}
case ZSET_META:
{
ZSetRange(ctx, key, 0, -1, false, false, OP_GET);
if (NULL == options.by)
{
options.nosort = true;
}
break;
}
default:
{
return ERR_INVALID_TYPE;
}
}
DataArray sortvals;
if (ctx.reply.MemberSize() > 0)
{
for (uint32 i = 0; i < ctx.reply.MemberSize(); i++)
{
Data v;
v.SetString(ctx.reply.MemberAt(i).str, true);
sortvals.push_back(v);
}
}
if (sortvals.empty())
{
return 0;
}
if (options.with_limit)
{
if (options.limit_offset < 0)
{
options.limit_offset = 0;
}
if ((uint32) options.limit_offset > sortvals.size())
{
values.clear();
return 0;
}
if (options.limit_count < 0)
{
options.limit_count = sortvals.size();
}
}
std::vector<SortValue> sortvec;
if (!options.nosort)
{
if (NULL != options.by)
{
sortvec.reserve(sortvals.size());
}
for (uint32 i = 0; i < sortvals.size(); i++)
{
if (NULL != options.by)
{
sortvec.push_back(SortValue(&sortvals[i]));
if (GetValueByPattern(ctx, options.by, sortvals[i], sortvec[i].cmp) < 0)
{
DEBUG_LOG("Failed to get value by pattern:%s", options.by);
sortvec[i].cmp.Clear();
continue;
}
}
if (options.with_alpha)
{
if (NULL != options.by)
{
sortvec[i].cmp.ToString();
}
else
{
sortvals[i].ToString();
}
}
}
if (NULL != options.by)
{
if (!options.is_desc)
{
std::sort(sortvec.begin(), sortvec.end(), less_value<SortValue>);
}
else
{
std::sort(sortvec.begin(), sortvec.end(), greater_value<SortValue>);
}
}
else
{
if (!options.is_desc)
{
std::sort(sortvals.begin(), sortvals.end(), less_value<Data>);
}
else
{
std::sort(sortvals.begin(), sortvals.end(), greater_value<Data>);
}
}
}
if (!options.with_limit)
{
options.limit_offset = 0;
options.limit_count = sortvals.size();
}
uint32 count = 0;
for (uint32 i = options.limit_offset; i < sortvals.size() && count < (uint32) options.limit_count; i++, count++)
{
Data* patternObj = NULL;
if (NULL != options.by)
{
patternObj = sortvec[i].value;
}
else
{
patternObj = &(sortvals[i]);
}
if (options.get_patterns.empty())
{
values.push_back(*patternObj);
}
else
{
for (uint32 j = 0; j < options.get_patterns.size(); j++)
{
Data vo;
if (GetValueByPattern(ctx, options.get_patterns[j], *patternObj, vo) < 0)
{
DEBUG_LOG("Failed to get value by pattern for:%s", options.get_patterns[j]);
vo.Clear();
}
values.push_back(vo);
}
}
}
uint32 step = options.get_patterns.empty() ? 1 : options.get_patterns.size();
switch (options.aggregate)
{
case AGGREGATE_SUM:
case AGGREGATE_AVG:
{
DataArray result;
result.resize(step);
for (uint32 i = 0; i < result.size(); i++)
{
for (uint32 j = i; j < values.size(); j += step)
{
result[i].IncrBy(values[j]);
}
}
if (options.aggregate == AGGREGATE_AVG)
{
size_t count = values.size() / step;
for (uint32 i = 0; i < result.size(); i++)
{
result[i].SetDouble(result[i].NumberValue() / count);
}
}
values.assign(result.begin(), result.end());
break;
}
case AGGREGATE_MAX:
case AGGREGATE_MIN:
{
DataArray result;
result.resize(step);
for (uint32 i = 0; i < result.size(); i++)
{
for (uint32 j = i; j < values.size(); j += step)
{
if (result[i].IsNil())
{
result[i] = values[j];
}
else
{
if (options.aggregate == AGGREGATE_MIN)
{
if (values[j] < result[i])
{
result[i] = values[j];
}
}
else
{
if (values[j] > result[i])
{
result[i] = values[j];
}
}
}
}
}
values.assign(result.begin(), result.end());
break;
}
case AGGREGATE_COUNT:
{
size_t size = values.size() / step;
values.clear();
Data v;
v.SetInt64(size);
values.push_back(v);
break;
}
default:
{
break;
}
}
if (options.store_dst != NULL && !values.empty())
{
DeleteKey(ctx, options.store_dst);
ValueObject list_meta;
list_meta.key.key = options.store_dst;
list_meta.key.type = KEY_META;
list_meta.key.db = ctx.currentDB;
list_meta.type = LIST_META;
list_meta.meta.SetEncoding(COLLECTION_ECODING_ZIPLIST);
BatchWriteGuard guard(GetKeyValueEngine());
DataArray::iterator it = values.begin();
while (it != values.end())
{
if (!it->IsNil())
{
std::string tmp;
it->GetDecodeString(tmp);
ListInsert(ctx, list_meta, NULL, tmp, false, false);
}
it++;
}
SetKeyValue(ctx, list_meta);
}
return 0;
}