void LMDBEngineFactory::ParseConfig(const Properties& props, LMDBConfig& cfg)
{
cfg.path = ".";
conf_get_string(props, "data-dir", cfg.path);
conf_get_int64(props, "lmdb.database_max_size", cfg.max_db_size);
conf_get_int64(props, "lmdb.batch_commit_watermark", cfg.batch_commit_watermark);
conf_get_bool(props, "lmdb.readahead", cfg.readahead);
}
void WiredTigerEngineFactory::ParseConfig(const Properties& props, WiredTigerConfig& cfg)
{
cfg.path = ".";
conf_get_string(props, "data-dir", cfg.path);
conf_get_int64(props, "wiredtiger.batch_commit_watermark", cfg.batch_commit_watermark);
conf_get_string(props, "wiredtiger.init_options", cfg.init_options);
conf_get_string(props, "wiredtiger.init_table_options", cfg.init_table_options);
//conf_get_bool(props, "leveldb.logenable", cfg.logenable);
}
void LevelDBEngineFactory::ParseConfig(const Properties& props, LevelDBConfig& cfg)
{
cfg.path = ".";
conf_get_string(props, "data-dir", cfg.path);
conf_get_int64(props, "leveldb.block_cache_size", cfg.block_cache_size);
conf_get_int64(props, "leveldb.write_buffer_size", cfg.write_buffer_size);
conf_get_int64(props, "leveldb.max_open_files", cfg.max_open_files);
conf_get_int64(props, "leveldb.block_size", cfg.block_size);
conf_get_int64(props, "leveldb.block_restart_interval", cfg.block_restart_interval);
conf_get_int64(props, "leveldb.bloom_bits", cfg.bloom_bits);
conf_get_int64(props, "leveldb.batch_commit_watermark", cfg.batch_commit_watermark);
}
bool ArdbConfig::Parse(const Properties& props)
{
conf_props = props;
conf_get_string(props, "home", home);
if (home.empty())
{
home = "../ardb";
}
make_dir(home);
int err = real_path(home, home);
if (0 != err)
{
ERROR_LOG("Invalid 'home' config:%s for reason:%s", home.c_str(), strerror(err));
return false;
}
err = access(home.c_str(), R_OK | W_OK);
if (0 != err)
{
err = errno;
ERROR_LOG("Invalid 'home' config:%s for reason:%s", home.c_str(), strerror(err));
return false;
}
setenv("ARDB_HOME", home.c_str(), 1);
replace_env_var(const_cast<Properties&>(props));
conf_get_string(props, "pidfile", pidfile);
conf_get_int64(props, "thread-pool-size", thread_pool_size);
if (thread_pool_size <= 0)
{
thread_pool_size = available_processors();
}
conf_get_int64(props, "hz", hz);
if (hz < CONFIG_MIN_HZ)
hz = CONFIG_MIN_HZ;
if (hz > CONFIG_MAX_HZ)
hz = CONFIG_MAX_HZ;
conf_get_int64(props, "tcp-keepalive", tcp_keepalive);
conf_get_int64(props, "timeout", timeout);
//conf_get_int64(props, "unixsocketperm", unixsocketperm);
conf_get_int64(props, "slowlog-log-slower-than", slowlog_log_slower_than);
conf_get_int64(props, "slowlog-max-len", slowlog_max_len);
conf_get_int64(props, "maxclients", max_clients);
if(max_clients <= 0)
{
max_clients = 10000;
}
for (int i = 0;; i++)
{
char config_key[256];
sprintf(config_key, "server[%d].listen", i);
ListenPoint lp;
std::string address;
if (!conf_get_string(props, config_key, address))
{
break;
}
if (address.find(":") == std::string::npos)
{
lp.host = address;
}
else
{
std::vector<std::string> ss = split_string(address, ":");
uint32 port;
if (ss.size() < 2 || !string_touint32(ss[ss.size() - 1], port) || port > 65535)
{
ERROR_LOG("Invalid listen address %s", address.c_str());
return false;
}
lp.host = address.substr(0, address.size() - ss[ss.size() - 1].size() - 1);
lp.port = port;
}
sprintf(config_key, "server[%d].qps-limit", i);
conf_get_int64(props, config_key, lp.qps_limit);
sprintf(config_key, "server[%d].unixsocketperm", i);
conf_get_int64(props, config_key, lp.qps_limit);
servers.push_back(lp);
}
if (servers.empty())
{
ListenPoint lp;
lp.host = "0.0.0.0";
lp.port = 16379;
servers.push_back(lp);
}
if (strcasecmp(engine.c_str(), "rocksdb") == 0)
{
conf_get_string(props, "rocksdb.compaction", rocksdb_compaction);
conf_get_bool(props, "rocksdb.disableWAL", rocksdb_disablewal);
conf_get_bool(props, "rocksdb.scan-total-order", rocksdb_scan_total_order);
}
conf_get_string(props, "engine", engine);
conf_get_string(props, "data-dir", data_base_path);
conf_get_string(props, "backup-dir", backup_dir);
conf_get_string(props, "repl-dir", repl_data_dir);
make_dir(repl_data_dir);
make_dir(backup_dir);
err = real_path(repl_data_dir, repl_data_dir);
if (0 != err)
{
ERROR_LOG("Invalid 'repl-dir' config:%s for reason:%s", repl_data_dir.c_str(), strerror(err));
return false;
}
std::string backup_file_format;
conf_get_string(props, "backup-file-format", backup_file_format);
if (!strcasecmp(backup_file_format.c_str(), "redis"))
{
backup_redis_format = true;
}
conf_get_string(props, "zookeeper-servers", zookeeper_servers);
conf_get_string(props, "zk-clientid-file", zk_clientid_file);
conf_get_string(props, "loglevel", loglevel);
conf_get_string(props, "logfile", logfile);
conf_get_bool(props, "daemonize", daemonize);
conf_get_int64(props, "repl-backlog-size", repl_backlog_size);
conf_get_int64(props, "repl-backlog-cache-size", repl_backlog_cache_size);
conf_get_int64(props, "repl-ping-slave-period", repl_ping_slave_period);
conf_get_int64(props, "repl-timeout", repl_timeout);
conf_get_int64(props, "repl-backlog-sync-period", repl_backlog_sync_period);
conf_get_int64(props, "repl-backlog-ttl", repl_backlog_time_limit);
conf_get_int64(props, "min-slaves-to-write", repl_min_slaves_to_write);
conf_get_int64(props, "min-slaves-max-lag", repl_min_slaves_max_lag);
conf_get_bool(props, "slave-serve-stale-data", repl_serve_stale_data);
conf_get_int64(props, "max-slave-worker-queue", max_slave_worker_queue);
if(max_slave_worker_queue <= 0)
{
max_slave_worker_queue = 1024;
}
conf_get_bool(props, "repl-disable-tcp-nodelay", repl_disable_tcp_nodelay);
conf_get_int64(props, "lua-time-limit", lua_time_limit);
conf_get_int64(props, "snapshot-max-lag-offset", snapshot_max_lag_offset);
conf_get_int64(props, "maxsnapshots", maxsnapshots);
if(maxsnapshots == 0)
{
maxsnapshots = 1;
}
if (snapshot_max_lag_offset > repl_backlog_size / 2)
{
snapshot_max_lag_offset = repl_backlog_size / 2;
}
conf_get_int64(props, "hll-sparse-max-bytes", hll_sparse_max_bytes);
conf_get_bool(props, "slave-read-only", slave_readonly);
conf_get_bool(props, "slave-serve-stale-data", slave_serve_stale_data);
conf_get_int64(props, "slave-priority", slave_priority);
conf_get_bool(props, "slave-ignore-expire", slave_ignore_expire);
conf_get_bool(props, "slave-ignore-del", slave_ignore_del);
conf_get_bool(props, "slave-cleardb-before-fullresync", slave_cleardb_before_fullresync);
conf_get_int64(props, "statistics-log-period", statistics_log_period);
if (statistics_log_period <= 0)
{
statistics_log_period = DEFAULT_STAT_LOG_PERIOD_SECS;
}
std::string slaveof;
if (conf_get_string(props, "slaveof", slaveof))
{
std::vector<std::string> ss = split_string(slaveof, ":");
if (ss.size() == 2)
{
master_host = ss[0];
if (!string_touint32(ss[1], master_port))
{
master_host = "";
WARN_LOG("Invalid 'slaveof' config.");
}
}
else
{
WARN_LOG("Invalid 'slaveof' config.");
}
}
conf_get_string(props, "masterauth", masterauth);
if (data_base_path.empty())
{
data_base_path = ".";
}
make_dir(data_base_path);
err = real_path(data_base_path, data_base_path);
if (0 != err)
{
ERROR_LOG("Invalid 'data-dir' config:%s for reason:%s", data_base_path.c_str(), strerror(err));
return false;
}
conf_get_string(props, "requirepass", requirepass);
Properties::const_iterator fit = props.find("rename-command");
if (fit != props.end())
{
rename_commands.clear();
const ConfItemsArray& cs = fit->second;
ConfItemsArray::const_iterator cit = cs.begin();
while (cit != cs.end())
{
if (cit->size() != 2)
{
ERROR_LOG("Invalid 'rename-command' config with %u args.", cit->size());
}
else
{
rename_commands[cit->at(0)] = cit->at(1);
}
cit++;
}
}
conf_get_int64(props, "reply-pool-size", reply_pool_size);
conf_get_int64(props, "slave-client-output-buffer-limit", slave_client_output_buffer_limit);
conf_get_int64(props, "pubsub-client-output-buffer-limit", pubsub_client_output_buffer_limit);
conf_get_string(props, "redis-compatible-version", redis_compatible_version);
conf_get_bool(props, "redis-compatible-mode", redis_compatible);
conf_get_bool(props, "compact-after-snapshot-load", compact_after_snapshot_load);
conf_get_int64(props, "qps-limit-per-host", qps_limit_per_host);
conf_get_int64(props, "qps-limit-per-connection", qps_limit_per_connection);
conf_get_int64(props, "range-delete-min-size", range_delete_min_size);
conf_get_int64(props, "stream-lru-cache-size", stream_lru_cache_size);
//trusted_ip.clear();
Properties::const_iterator ip_it = props.find("trusted-ip");
if (ip_it != props.end())
{
const ConfItemsArray& cs = ip_it->second;
for (uint32 i = 0; i < cs.size(); i++)
{
//trusted_ip.insert(cs[i][0]);
}
}
if (!verify_config(*this))
{
return false;
}
ArdbLogger::SetLogLevel(loglevel);
return true;
}
bool ArdbConfig::Parse(const Properties& props)
{
conf_props = props;
conf_get_string(props, "home", home);
if (home.empty())
{
home = "../ardb";
}
make_dir(home);
int err = real_path(home, home);
if (0 != err)
{
ERROR_LOG("Invalid 'home' config:%s for reason:%s", home.c_str(), strerror(err));
return false;
}
err = access(home.c_str(), R_OK | W_OK);
if (0 != err)
{
err = errno;
ERROR_LOG("Invalid 'home' config:%s for reason:%s", home.c_str(), strerror(err));
return false;
}
setenv("ARDB_HOME", home.c_str(), 1);
replace_env_var(const_cast<Properties&>(props));
conf_get_string(props, "pidfile", pidfile);
conf_get_int64(props, "tcp-keepalive", tcp_keepalive);
conf_get_int64(props, "timeout", timeout);
conf_get_int64(props, "unixsocketperm", unixsocketperm);
conf_get_int64(props, "slowlog-log-slower-than", slowlog_log_slower_than);
conf_get_int64(props, "slowlog-max-len", slowlog_max_len);
conf_get_int64(props, "maxclients", max_clients);
Properties::const_iterator listen_it = props.find("listen");
if (listen_it != props.end())
{
const ConfItemsArray& cs = listen_it->second;
for (uint32 i = 0; i < cs.size(); i++)
{
if (cs[i].size() != 1)
{
WARN_LOG("Invalid config 'listen'");
}
else
{
const std::string& str = cs[i][0];
listen_addresses.push_back(str);
}
}
}
if (listen_addresses.empty())
{
listen_addresses.push_back("0.0.0.0:16379");
}
Properties::const_iterator tp_it = props.find("thread-pool-size");
if (tp_it != props.end())
{
const ConfItemsArray& cs = tp_it->second;
for (uint32 i = 0; i < cs.size(); i++)
{
uint32 size = 0;
if (cs[i].size() != 1 || !string_touint32(cs[i][0], size))
{
WARN_LOG("Invalid config 'thread-pool-size'");
}
else
{
thread_pool_sizes.push_back((int64) size);
}
}
}
Properties::const_iterator qp_it = props.find("qps-limit");
if (qp_it != props.end())
{
const ConfItemsArray& cs = qp_it->second;
for (uint32 i = 0; i < cs.size(); i++)
{
uint32 limit = 0;
if (cs[i].size() != 1 || !string_touint32(cs[i][0], limit))
{
WARN_LOG("Invalid config 'qps-limit'");
}
else
{
qps_limits.push_back((int64) limit);
}
}
}
thread_pool_sizes.resize(listen_addresses.size());
qps_limits.resize(listen_addresses.size());
conf_get_string(props, "data-dir", data_base_path);
conf_get_string(props, "backup-dir", backup_dir);
conf_get_string(props, "repl-dir", repl_data_dir);
make_dir(repl_data_dir);
make_dir(backup_dir);
err = real_path(repl_data_dir, repl_data_dir);
if (0 != err)
{
ERROR_LOG("Invalid 'repl-dir' config:%s for reason:%s", repl_data_dir.c_str(), strerror(err));
return false;
}
std::string backup_file_format;
conf_get_string(props, "backup-file-format", backup_file_format);
if (!strcasecmp(backup_file_format.c_str(), "redis"))
{
backup_redis_format = true;
}
conf_get_string(props, "zookeeper-servers", zookeeper_servers);
conf_get_string(props, "loglevel", loglevel);
conf_get_string(props, "logfile", logfile);
conf_get_bool(props, "daemonize", daemonize);
conf_get_int64(props, "repl-backlog-size", repl_backlog_size);
conf_get_int64(props, "repl-ping-slave-period", repl_ping_slave_period);
conf_get_int64(props, "repl-timeout", repl_timeout);
conf_get_int64(props, "repl-state-persist-period", repl_state_persist_period);
conf_get_int64(props, "repl-backlog-ttl", repl_backlog_time_limit);
conf_get_bool(props, "repl-disable-tcp-nodelay", repl_disable_tcp_nodelay);
conf_get_int64(props, "lua-time-limit", lua_time_limit);
conf_get_int64(props, "hash-max-ziplist-entries", hash_max_ziplist_entries);
conf_get_int64(props, "hash_max-ziplist-value", hash_max_ziplist_value);
conf_get_int64(props, "set-max-ziplist-entries", set_max_ziplist_entries);
conf_get_int64(props, "set-max-ziplist-value", set_max_ziplist_value);
conf_get_int64(props, "list-max-ziplist-entries", list_max_ziplist_entries);
conf_get_int64(props, "list-max-ziplist-value", list_max_ziplist_value);
conf_get_int64(props, "zset-max-ziplist-entries", zset_max_ziplist_entries);
conf_get_int64(props, "zset_max_ziplist_value", zset_max_ziplist_value);
conf_get_int64(props, "L1-zset-max-cache-size", L1_zset_max_cache_size);
conf_get_int64(props, "L1-set-max-cache-size", L1_set_max_cache_size);
conf_get_int64(props, "L1-hash-max-cache-size", L1_hash_max_cache_size);
conf_get_int64(props, "L1-list-max-cache-size", L1_list_max_cache_size);
conf_get_int64(props, "L1-string-max-cache-size", L1_string_max_cache_size);
conf_get_bool(props, "L1-zset-read-fill-cache", L1_zset_read_fill_cache);
conf_get_bool(props, "L1-zset-seek-load-cache", L1_zset_seek_load_cache);
conf_get_bool(props, "L1-set-read-fill-cache", L1_set_read_fill_cache);
conf_get_bool(props, "L1-set-seek-load-cache", L1_set_seek_load_cache);
conf_get_bool(props, "L1-hash-read-fill-cache", L1_hash_read_fill_cache);
conf_get_bool(props, "L1-hash-seek-load-cache", L1_hash_seek_load_cache);
conf_get_bool(props, "L1-list-read-fill-cache", L1_list_read_fill_cache);
conf_get_bool(props, "L1-list-seek-load-cache", L1_list_seek_load_cache);
conf_get_bool(props, "L1-string-read-fill-cache", L1_string_read_fill_cache);
conf_get_int64(props, "hll-sparse-max-bytes", hll_sparse_max_bytes);
conf_get_bool(props, "slave-read-only", slave_readonly);
conf_get_bool(props, "slave-serve-stale-data", slave_serve_stale_data);
conf_get_int64(props, "slave-priority", slave_priority);
conf_get_bool(props, "slave-ignore-expire", slave_ignore_expire);
conf_get_bool(props, "slave-ignore-del", slave_ignore_del);
std::string slaveof;
if (conf_get_string(props, "slaveof", slaveof))
{
std::vector<std::string> ss = split_string(slaveof, ":");
if (ss.size() == 2)
{
master_host = ss[0];
if (!string_touint32(ss[1], master_port))
{
master_host = "";
WARN_LOG("Invalid 'slaveof' config.");
}
}
else
{
WARN_LOG("Invalid 'slaveof' config.");
}
}
std::string include_dbs, exclude_dbs;
repl_includes.clear();
repl_excludes.clear();
conf_get_string(props, "replicate-include-db", include_dbs);
conf_get_string(props, "replicate-exclude-db", exclude_dbs);
if (0 != split_uint32_array(include_dbs, "|", repl_includes))
{
ERROR_LOG("Invalid 'replicate-include-db' config.");
repl_includes.clear();
}
if (0 != split_uint32_array(exclude_dbs, "|", repl_excludes))
{
ERROR_LOG("Invalid 'replicate-exclude-db' config.");
repl_excludes.clear();
}
if (data_base_path.empty())
{
data_base_path = ".";
}
make_dir(data_base_path);
err = real_path(data_base_path, data_base_path);
if (0 != err)
{
ERROR_LOG("Invalid 'data-dir' config:%s for reason:%s", data_base_path.c_str(), strerror(err));
return false;
}
conf_get_string(props, "additional-misc-info", additional_misc_info);
conf_get_string(props, "requirepass", requirepass);
Properties::const_iterator fit = props.find("rename-command");
if (fit != props.end())
{
rename_commands.clear();
StringSet newcmdset;
const ConfItemsArray& cs = fit->second;
ConfItemsArray::const_iterator cit = cs.begin();
while (cit != cs.end())
{
if (cit->size() != 2 || newcmdset.count(cit->at(1)) > 0)
{
ERROR_LOG("Invalid 'rename-command' config.");
}
else
{
rename_commands[cit->at(0)] = cit->at(1);
newcmdset.insert(cit->at(1));
}
cit++;
}
}
conf_get_int64(props, "compact-min-interval", compact_min_interval);
conf_get_int64(props, "compact-max-interval", compact_max_interval);
conf_get_int64(props, "compact-after-write", compact_trigger_write_count);
conf_get_bool(props, "compact-enable", compact_enable);
conf_get_int64(props, "reply-pool-size", reply_pool_size);
conf_get_bool(props, "replace-all-for-multi-sadd", replace_for_multi_sadd);
conf_get_bool(props, "replace-all-for-hmset", replace_for_hmset);
conf_get_int64(props, "slave-client-output-buffer-limit", slave_client_output_buffer_limit);
conf_get_int64(props, "pubsub-client-output-buffer-limit", pubsub_client_output_buffer_limit);
conf_get_bool(props, "scan-redis-compatible", scan_redis_compatible);
conf_get_int64(props, "scan-cursor-expire-after", scan_cursor_expire_after);
conf_get_int64(props, "max-string-bitset-value", max_string_bitset_value);
conf_get_int64(props, "databases", maxdb);
conf_get_bool(props, "lua-exec-atomic", lua_exec_atomic);
trusted_ip.clear();
Properties::const_iterator ip_it = props.find("trusted-ip");
if (ip_it != props.end())
{
const ConfItemsArray& cs = ip_it->second;
for (uint32 i = 0; i < cs.size(); i++)
{
trusted_ip.insert(cs[i][0]);
}
}
if (!verify_config(*this))
{
return false;
}
ArdbLogger::SetLogLevel(loglevel);
return true;
}
int PerconaFTEngine::Init(const std::string& dir, const std::string& options)
{
Properties props;
parse_conf_content(options, props);
//parse config
conf_get_int64(props, "cache_size", g_perconaft_config.cache_size);
conf_get_uint32(props, "checkpoint_pool_threads", g_perconaft_config.checkpoint_pool_threads);
conf_get_uint32(props, "checkpoint_period", g_perconaft_config.checkpoint_period);
conf_get_uint32(props, "cleaner_period", g_perconaft_config.cleaner_period);
conf_get_uint32(props, "cleaner_iterations", g_perconaft_config.cleaner_iterations);
conf_get_bool(props, "evictor_enable_partial_eviction", g_perconaft_config.evictor_enable_partial_eviction);
std::string compression;
conf_get_string(props, "compression", compression);
if (compression == "none")
{
g_perconaft_config.compression = TOKU_NO_COMPRESSION;
}
else if (compression == "snappy" || compression.empty())
{
g_perconaft_config.compression = TOKU_SNAPPY_METHOD;
}
else if (compression == "zlib")
{
g_perconaft_config.compression = TOKU_ZLIB_METHOD;
}
else
{
ERROR_LOG("Invalid compression config:%s for PercanoFT.", compression.c_str());
return -1;
}
uint32 env_open_flags = DB_CREATE | DB_PRIVATE | DB_THREAD | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG | DB_RECOVER;
int env_open_mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
db_env_create(&m_env, 0);
m_env->set_default_bt_compare(m_env, ardb_perconaft_compare);
m_env->set_errcall(m_env, _err_callback);
/*
* set env config
*/
uint32 cache_gsize = g_perconaft_config.cache_size >> 30;
uint32 cache_bytes = g_perconaft_config.cache_size % (1024 * 1024 * 1024);
m_env->set_cachesize(m_env, cache_gsize, cache_bytes, 1);
m_env->set_cachetable_pool_threads(m_env, g_perconaft_config.checkpoint_pool_threads);
m_env->checkpointing_set_period(m_env, g_perconaft_config.checkpoint_period);
m_env->cleaner_set_period(m_env, g_perconaft_config.cleaner_period);
m_env->cleaner_set_iterations(m_env, g_perconaft_config.cleaner_iterations);
m_env->evictor_set_enable_partial_eviction(m_env, g_perconaft_config.evictor_enable_partial_eviction);
int r = m_env->open(m_env, dir.c_str(), env_open_flags, env_open_mode);
CHECK_EXPR(r);
DataArray nss;
if (0 == r)
{
g_toku_env = m_env;
DB* db = m_env->get_db_for_directory(m_env);
if (NULL != db)
{
PerconaFTLocalContext& local_ctx = g_local_ctx.GetValue();
DB_TXN* txn = local_ctx.transc.Get();
DBC *c = NULL;
CHECK_EXPR(r = db->cursor(db, txn, &c, 0));
if (0 == r)
{
r = c->c_getf_first(c, 0, nil_callback, NULL);
}
while (0 == r)
{
DBT raw_key;
DBT raw_val;
memset(&raw_key, 0, sizeof(raw_key));
memset(&raw_val, 0, sizeof(raw_key));
if (0 == c->c_get(c, &raw_key, &raw_val, DB_CURRENT))
{
//std::string ns_str
Data ns;
ns.SetString((const char*) raw_key.data, false);
INFO_LOG("TokuFT directory db %s:%s", (const char* ) raw_key.data, (const char* ) raw_val.data);
nss.push_back(ns);
}
r = c->c_getf_next(c, 0, nil_callback, NULL);
}
if (NULL != c)
{
c->c_close(c);
}
local_ctx.transc.Release(true);
r = 0;
}
}
for (size_t i = 0; i < nss.size(); i++)
{
Context tmp;
GetFTDB(tmp, nss[i], true);
}
return ENGINE_ERR(r);
}
