void ExpireCheck::Run()
{
uint64 start = get_current_epoch_millis();
while (m_checking_db < ARDB_GLOBAL_DB)
{
uint64 now = get_current_epoch_millis();
if (now - start >= 500)
{
return;
}
DBID nexdb = 0;
if (!m_db->DBExist(m_checking_db, nexdb))
{
if (nexdb == m_checking_db || nexdb == ARDB_GLOBAL_DB)
{
m_checking_db = 0;
return;
}
m_checking_db = nexdb;
}
m_db->CheckExpireKey(m_checking_db);
m_checking_db++;
}
m_checking_db = 0;
}
void ExpireCheck::Run()
{
if (!m_server->GetConfig().master_host.empty())
{
/*
* no expire on slave, master would sync the expire/del operations to slaves.
*/
return;
}
Ardb& db = m_server->GetDB();
uint64 start = get_current_epoch_millis();
while (m_checking_db < ARDB_GLOBAL_DB)
{
uint64 now = get_current_epoch_millis();
if (now - start >= 500)
{
return;
}
DBID nexdb = 0;
if (!db.DBExist(m_checking_db, nexdb))
{
if (nexdb == m_checking_db || nexdb == ARDB_GLOBAL_DB)
{
m_checking_db = 0;
return;
}
m_checking_db = nexdb;
}
db.CheckExpireKey(m_checking_db, 50, 10000, ExpireCheckCallback, this);
m_checking_db++;
}
m_checking_db = 0;
}
void Run()
{
uint64 start = get_current_epoch_millis();
for (int i = 0; i < 1000000; i++)
{
WriteLockGuard<SpinRWLock> guard(*lock);
(*mm)[i] = 1;
}
uint64 end = get_current_epoch_millis();
INFO_LOG("Cost %llums to write", (end - start));
}
void Run()
{
uint64 start = get_current_epoch_millis();
for (int i = 0; i < 1000000; i++)
{
ReadLockGuard<SpinRWLock> guard(*lock);
mm->find(i);
}
uint64 end = get_current_epoch_millis();
INFO_LOG("Cost %llums to read", (end - start));
}
void ZSetCache::GetRange(const ZRangeSpec& range, bool with_scores, bool with_attrs, ValueVisitCallback* cb,
void* cbdata)
{
uint64 start = get_current_epoch_millis();
CacheReadLockGuard guard(m_lock);
ZSetCaheElement min_ele(range.min.NumberValue(), "");
ZSetCaheElement max_ele(range.max.NumberValue(), "");
ZSetCacheElementSet::iterator min_it = m_cache.lower_bound(min_ele);
ZSetCacheElementSet::iterator max_it = m_cache.lower_bound(max_ele);
int cursor = 0;
if (min_it != m_cache.end())
{
while (!range.contain_min && min_it != m_cache.end() && min_it->score == range.min.NumberValue())
{
min_it++;
}
while (range.contain_max && max_it != m_cache.end() && max_it->score == range.max.NumberValue())
{
max_it++;
}
while (min_it != max_it && min_it != m_cache.end())
{
ValueData v;
Buffer buf(const_cast<char*>(min_it->value.data()), 0, min_it->value.size());
v.Decode(buf);
cb(v, cursor++, cbdata);
if (with_scores)
{
ValueData score;
score.SetDoubleValue(min_it->score);
cb(score, cursor++, cbdata);
}
if (with_attrs)
{
ValueData attr_value;
Buffer attr_buf(const_cast<char*>(min_it->attr.data()), 0, min_it->attr.size());
attr_value.Decode(attr_buf);
cb(attr_value, cursor++, cbdata);
}
min_it++;
}
}
uint64 end = get_current_epoch_millis();
if (end - start > 10)
{
DEBUG_LOG("Cost %llums to get %d elements in between range [%.2f, %.2f]", end - start, cursor,
range.min.NumberValue(), range.max.NumberValue());
}
}
void test_pthread_create_performance()
{
uint64 start = get_current_epoch_millis();
uint32 loop = 1000000;
for (uint32 i = 0; i < loop; i++)
{
pthread_mutex_t mutex;
pthread_mutexattr_t attr;
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT);
pthread_mutex_init(&mutex, &attr);
pthread_mutexattr_destroy(&attr);
pthread_mutex_destroy(&mutex);
}
uint64 end = get_current_epoch_millis();
INFO_LOG("Cost %llums to create/destory %u mutexes.", (end - start), loop);
}
void Statistics::TrackOperationsPerSecond()
{
uint64 t = get_current_epoch_millis() - m_ops_sec_last_sample_time;
ServerStatisticsTable::iterator it = m_server_stats.begin();
while (it != m_server_stats.end())
{
ServerStatistics& st = it->second;
long long ops = st.stat_numcommands - st.ops_sec_last_sample_ops;
long long ops_sec;
ops_sec = t > 0 ? (ops * 1000 / t) : 0;
st.ops_sec_samples[st.ops_sec_idx] = ops_sec;
st.ops_sec_idx = (st.ops_sec_idx + 1) % COMMS_OPS_SEC_SAMPLES;
st.ops_sec_last_sample_ops = st.stat_numcommands;
st.instantaneous_ops = 0;
it++;
}
m_ops_sec_last_sample_time = get_current_epoch_millis();
}
int Ardb::TTL(Context& ctx, RedisCommandFrame& cmd)
{
uint64 ms;
int err = GenericTTL(ctx, cmd.GetArguments()[0], ms);
if (0 == err)
{
fill_int_reply(ctx.reply, (ms - get_current_epoch_millis()) / 1000);
}
return 0;
}
int Ardb::Expire(Context& ctx, RedisCommandFrame& cmd)
{
uint32 v = 0;
if (!check_uint32_arg(ctx.reply, cmd.GetArguments()[1], v))
{
return 0;
}
int err = GenericExpire(ctx, cmd.GetArguments()[0], v * 1000 + get_current_epoch_millis());
fill_int_reply(ctx.reply, err == 0 ? 1 : 0);
return 0;
}
int CoroChannel::SyncWrite(Buffer& buffer, int timeout)
{
uint64 start = timeout > 0 ? get_current_epoch_millis() : 0;
m_ch->GetOutputBuffer().Write(&buffer, buffer.ReadableBytes());
m_ch->EnableWriting();
CreateTimeoutTask(timeout);
while (m_ch->GetOutputBuffer().ReadableBytes() > 0)
{
if (timeout > 0)
{
if (get_current_epoch_millis() - start >= timeout)
{
return -1;
}
}
if (0 != WaitCoro())
{
return -1;
}
}
return 0;
}
int Ardb::PExpire(Context& ctx, RedisCommandFrame& cmd)
{
uint32 v = 0;
if (!check_uint32_arg(ctx.reply, cmd.GetArguments()[1], v))
{
return 0;
}
ValueObject meta;
int err = GenericExpire(ctx, cmd.GetArguments()[0], v + get_current_epoch_millis());
CHECK_ARDB_RETURN_VALUE(ctx.reply, err);
fill_int_reply(ctx.reply, err == 0 ? 1 : 0);
return 0;
}
void Run()
{
IdleConn conn;
uint64 now = get_current_epoch_millis();
while (lru.PeekFront(conn) && (now - conn.ts >= maxIdleTime))
{
Channel* ch = serv->GetChannel(conn.conn_id);
if (NULL != ch)
{
DEBUG_LOG("Closed timeout connection:%d.", ch->GetID());
ch->Close();
}
lru.PopFront();
}
}
int Ardb::GenericExpire(Context& ctx, const Slice& key, uint64 ms)
{
if (ms > 0 && get_current_epoch_millis() >= ms)
{
return 0;
}
if (m_cfg.slave_ignore_expire && ctx.IsSlave())
{
/*
* ignore expire setting, but issue data change event for replication
*/
ctx.data_change = true;
return 0;
}
ValueObject meta;
int err = GetMetaValue(ctx, key, KEY_END, meta);
if (0 != err)
{
return err;
}
BatchWriteGuard guard(GetKeyValueEngine());
if (meta.meta.expireat != ms && meta.meta.expireat > 0)
{
KeyObject expire;
expire.key = key;
expire.type = KEY_EXPIRATION_ELEMENT;
expire.db = ctx.currentDB;
expire.score.SetInt64(meta.meta.expireat);
DelKeyValue(ctx, expire);
}
meta.meta.expireat = ms;
SetKeyValue(ctx, meta);
if (meta.meta.expireat > 0)
{
ValueObject vv(KEY_EXPIRATION_ELEMENT);
vv.key.key = key;
vv.key.type = KEY_EXPIRATION_ELEMENT;
vv.key.db = ctx.currentDB;
vv.key.score.SetInt64(meta.meta.expireat);
SetKeyValue(ctx, vv);
}
return 0;
}
void ArdbServer::TouchIdleConn(Channel* ch)
{
static ThreadLocal<IdleConnManager> kLocalIdleConns;
uint64 now = get_current_epoch_millis();
IdleConn conn;
conn.conn_id = ch->GetID();
conn.ts = now;
IdleConnManager& m = kLocalIdleConns.GetValue();
if (m.timer_id == -1)
{
m.serv = &(ch->GetService());
m.maxIdleTime = m_cfg.timeout * 1000;
m.lru.SetMaxCacheSize(m_cfg.max_clients);
m.timer_id = ch->GetService().GetTimer().Schedule(&m, 1, 5,
SECONDS);
}
IdleConn tmp;
m.lru.Insert(conn.conn_id, conn, tmp);
}
OP_NAMESPACE_BEGIN
Statistics::Statistics() :
m_ops_sec_last_sample_time(get_current_epoch_millis()),stat_expiredkeys(0)
{
}
static void default_loghandler(LogLevel level, const char* filename, const char* function, int line,
const char* format, ...)
{
const char* levelstr = 0;
uint64_t timestamp = get_current_epoch_millis();
if (level > 0 && level < ALL_LOG_LEVEL)
{
levelstr = kLogLevelNames[level - 1];
}
else
{
levelstr = "???";
}
size_t log_line_size = k_default_log_line_buf_size;
va_list args;
std::string record;
va_start(args, format);
for (;;)
{
//char content[log_line_size + 1];
char* content = new char[log_line_size + 1];
#ifndef va_copy
#define va_copy(dst, src) memcpy(&(dst), &(src), sizeof(va_list))
#endif
va_list aq;
va_copy(aq, args);
int sz = vsnprintf(content, log_line_size, format, aq);
va_end(aq);
if (sz < 0)
{
DELETE_A(content);
return;
}
if ((size_t) sz < log_line_size)
{
record = content;
DELETE_A(content);
break;
}
log_line_size <<= 1;
DELETE_A(content);
}
uint32 mills = timestamp % 1000;
char timetag[256];
struct tm& tm = get_current_tm();
sprintf(timetag, "%02u-%02u %02u:%02u:%02u", tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
LockGuard<ThreadMutex> guard(kLogMutex);
fprintf(kLogFile, "[%u] %s,%03u %s %s\n", getpid(), timetag, mills, levelstr, record.c_str());
fflush(kLogFile);
if (!kLogFilePath.empty() && kLogFile != stdout)
{
long file_size = ftell(kLogFile);
if (file_size < 0)
{
reopen_default_logfile();
}
else if ((uint32) file_size >= k_max_file_size)
{
rollover_default_logfile();
reopen_default_logfile();
}
}
}
int Ardb::GeoSearch(Context& ctx, RedisCommandFrame& cmd)
{
GeoSearchOptions options;
std::string err;
if (0 != options.Parse(cmd.GetArguments(), err, 1))
{
fill_error_reply(ctx.reply, "%s", err.c_str());
return 0;
}
ValueObject meta;
int ret = GetMetaValue(ctx, cmd.GetArguments()[0], ZSET_META, meta);
CHECK_ARDB_RETURN_VALUE(ctx.reply, ret);
ctx.reply.type = REDIS_REPLY_ARRAY;
if (0 != ret)
{
return 0;
}
uint64 t1 = get_current_epoch_millis();
uint32 min_radius = 75; //magic number
if (options.asc && options.limit > 0 && options.offset == 0 && options.radius > min_radius)
{
uint32 old_radius = options.radius;
options.radius = min_radius;
do
{
ctx.reply.Clear();
int point_num = GeoSearchByOptions(ctx, meta, options);
if(point_num < 0)
{
break;
}
if (options.radius == old_radius)
{
break;
}
if (point_num > 0)
{
if (point_num >= options.limit)
{
break;
}
options.radius *= sqrt((options.limit / point_num) + 1);
}
else
{
options.radius *= 8;
}
if (options.radius > old_radius)
{
options.radius = old_radius;
}
} while (options.radius <= old_radius);
}
else
{
GeoSearchByOptions(ctx, meta, options);
}
ctx.reply.type = REDIS_REPLY_ARRAY;
uint64 t2 = get_current_epoch_millis();
DEBUG_LOG("####Cost %llums to range geosearch", t2 - t1);
return 0;
}
int Ardb::GeoSearchByOptions(Context& ctx, ValueObject& meta, GeoSearchOptions& options)
{
uint64 start_time = get_current_epoch_micros();
int ret = 0;
double x = options.x, y = options.y;
if (options.by_member)
{
Data element, score;
element.SetString(options.member, true);
ret = ZSetScore(ctx, meta, element, score);
if (0 != ret || score.IsNil())
{
return -1;
}
GeoHashHelper::GetMercatorXYByHash(score.value.iv, x, y);
//GeoHashHelper::GetXYByHash(score.value.iv, x, y);
}
else
{
if (options.coord_type != GEO_MERCATOR_TYPE)
{
x = GeoHashHelper::GetMercatorX(options.x);
y = GeoHashHelper::GetMercatorY(options.y);
}
}
DEBUG_LOG("####Step1: Cost %lluus", get_current_epoch_micros() - start_time);
GeoPointArray points;
ZSetRangeByScoreOptions fetch_options;
fetch_options.withscores = false;
fetch_options.op = OP_GET;
fetch_options.fill_reply = false;
fetch_options.fetch_geo_location = true;
if (options.in_members)
{
StringSet::iterator it = options.submembers.begin();
while (it != options.submembers.end())
{
Data element, score;
element.SetString(*it, true);
Location loc;
ret = ZSetScore(ctx, meta, element, score, &loc);
if (0 == ret)
{
fetch_options.results.push_back(element);
fetch_options.locs.push_back(loc);
}
it++;
}
}
else
{
GeoHashBitsSet ress;
GeoHashHelper::GetAreasByRadiusV2(GEO_MERCATOR_TYPE, y, x, options.radius, ress);
/*
* Merge areas if possible to avoid disk search
*/
std::vector<ZRangeSpec> range_array;
GeoHashBitsSet::iterator rit = ress.begin();
typedef TreeMap<uint64, uint64>::Type HashRangeMap;
HashRangeMap tmp;
while (rit != ress.end())
{
GeoHashBits& hash = *rit;
GeoHashBits next = hash;
next.bits++;
tmp[GeoHashHelper::Allign60Bits(hash)] = GeoHashHelper::Allign60Bits(next);
rit++;
}
HashRangeMap::iterator tit = tmp.begin();
HashRangeMap::iterator nit = tmp.begin();
nit++;
while (tit != tmp.end())
{
ZRangeSpec range;
range.contain_min = true;
range.contain_max = true;
range.min.SetInt64(tit->first);
range.max.SetInt64(tit->second);
while (nit != tmp.end() && nit->first == range.max.value.iv)
{
range.max.SetInt64(nit->second);
nit++;
tit++;
}
range_array.push_back(range);
nit++;
tit++;
}
DEBUG_LOG("After areas merging, reduce searching area size from %u to %u", ress.size(), range_array.size());
std::vector<ZRangeSpec>::iterator hit = range_array.begin();
ZSetIterator* iter = NULL;
while (hit != range_array.end())
{
ZRangeSpec& range = *hit;
uint64 t1 = get_current_epoch_millis();
ZSetRangeByScore(ctx, meta, range, fetch_options, iter);
uint64 t2 = get_current_epoch_millis();
DEBUG_LOG("####Cost %llums to range fetch", t2 - t1);
hit++;
}
DELETE(iter);
}
DEBUG_LOG("####Step2: Cost %lluus", get_current_epoch_micros() - start_time);
uint32 outrange = 0;
LocationDeque::iterator lit = fetch_options.locs.begin();
DataArray::iterator vit = fetch_options.results.begin();
while (vit != fetch_options.results.end())
{
Location& loc = *lit;
GeoPoint point;
point.x = loc.x;
point.y = loc.y;
/*
* distance accuracy is 0.2m
*/
if (GeoHashHelper::GetDistanceSquareIfInRadius(GEO_MERCATOR_TYPE, x, y, point.x, point.y, options.radius,
point.distance, 0.2))
{
vit->GetDecodeString(point.value);
/*
* filter by exclude/include
*/
if (!options.includes.empty() || !options.excludes.empty())
{
Data subst;
subst.SetString(point.value, false);
bool matched = options.includes.empty() ? true : false;
if (!options.includes.empty())
{
StringStringMap::const_iterator sit = options.includes.begin();
while (sit != options.includes.end())
{
Data mv;
if (0 != MatchValueByPattern(ctx, sit->first, sit->second, subst, mv))
{
matched = false;
break;
}
else
{
matched = true;
}
sit++;
}
}
if (matched && !options.excludes.empty())
{
StringStringMap::const_iterator sit = options.excludes.begin();
while (sit != options.excludes.end())
{
Data mv;
if (0 == MatchValueByPattern(ctx, sit->first, sit->second, subst, mv))
{
matched = false;
break;
}
else
{
matched = true;
}
sit++;
}
}
if (matched)
{
points.push_back(point);
}
}
else
{
points.push_back(point);
}
}
else
{
outrange++;
}
vit++;
lit++;
}
DEBUG_LOG("###Result size:%d, outrange:%d", points.size(), outrange);
DEBUG_LOG("####Step3: Cost %lluus", get_current_epoch_micros() - start_time);
if (!options.nosort)
{
std::sort(points.begin(), points.end(), options.asc ? less_by_distance : great_by_distance);
}
DEBUG_LOG("####Step3.5: Cost %lluus", get_current_epoch_micros() - start_time);
if (options.offset > 0)
{
if ((uint32) options.offset > points.size())
{
points.clear();
}
else
{
GeoPointArray::iterator start = points.begin() + options.offset;
points.erase(points.begin(), start);
}
}
if (options.limit > 0)
{
if ((uint32) options.limit < points.size())
{
GeoPointArray::iterator end = points.begin() + options.limit;
points.erase(end, points.end());
}
}
DEBUG_LOG("####Step4: Cost %lluus", get_current_epoch_micros() - start_time);
ValueObjectMap meta_cache;
GeoPointArray::iterator pit = points.begin();
while (pit != points.end())
{
RedisReply& r = ctx.reply.AddMember();
fill_str_reply(r, pit->value);
GeoGetOptionArray::const_iterator ait = options.get_patterns.begin();
while (ait != options.get_patterns.end())
{
if (ait->get_distances)
{
RedisReply& rr = ctx.reply.AddMember();
rr.type = REDIS_REPLY_STRING;
char dbuf[128];
int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", sqrt(pit->distance));
rr.str.assign(dbuf, dlen);
}
else if (ait->get_coodinates)
{
if (options.coord_type == GEO_WGS84_TYPE)
{
pit->x = GeoHashHelper::GetWGS84X(pit->x);
pit->y = GeoHashHelper::GetWGS84Y(pit->y);
}
RedisReply& rr1 = ctx.reply.AddMember();
RedisReply& rr2 = ctx.reply.AddMember();
if (options.coord_type == GEO_WGS84_TYPE)
{
fill_double_reply(rr1, pit->x);
fill_double_reply(rr2, pit->y);
}
else
{
char dbuf[128];
int dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->x);
rr1.type = REDIS_REPLY_STRING;
rr1.str.assign(dbuf, dlen);
dlen = snprintf(dbuf, sizeof(dbuf), "%.2f", pit->y);
rr2.type = REDIS_REPLY_STRING;
rr2.str.assign(dbuf, dlen);
}
}
else if (ait->hgetall)
{
std::string keystr(ait->get_pattern.data(), ait->get_pattern.size());
string_replace(keystr, "*", pit->value);
RedisReply& rr = ctx.reply.AddMember();
rr.type = REDIS_REPLY_ARRAY;
HashGetAll(ctx, keystr, rr);
}
else
{
Data v, attr;
v.SetString(pit->value, false);
GetValueByPattern(ctx, ait->get_pattern, v, attr, &meta_cache);
RedisReply& rr = ctx.reply.AddMember();
fill_value_reply(rr, attr);
}
ait++;
}
pit++;
}
DEBUG_LOG("####Step5: Cost %lluus", get_current_epoch_micros() - start_time);
uint64 end_time = get_current_epoch_micros();
DEBUG_LOG("Cost %llu microseconds to search.", end_time - start_time);
return points.size();
}
