bool IndexCursor::fetchMoreRows() {
// We're going to get more rows, so get rid of what's there.
_buffer.empty();
int r;
const int rows_to_fetch = getf_fetch_count();
struct cursor_getf_extra extra(&_buffer, rows_to_fetch);
DBC *cursor = _cursor.dbc();
if ( forward() ) {
r = cursor->c_getf_next(cursor, getf_flags(), cursor_getf, &extra);
} else {
r = cursor->c_getf_prev(cursor, getf_flags(), cursor_getf, &extra);
}
if ( extra.ex != NULL ) {
throw *extra.ex;
}
if (r == TOKUDB_INTERRUPTED) {
_interrupt_extra.throwException();
}
if ( r != 0 && r != DB_NOTFOUND ) {
extra.throwException();
storage::handle_ydb_error(r);
}
_getf_iteration++;
return extra.rows_fetched > 0 ? true : false;
}
int ftfs_bstore_scan_pages(DB *data_db, struct ftfs_meta_key *meta_key,
DB_TXN *txn, struct ftio *ftio)
{
int ret, r;
uint64_t block_num;
struct ftfs_scan_pages_cb_info info;
struct ftfs_data_key *data_key, *prefetch_data_key;
DBT key_dbt, prefetch_key_dbt;
DBC *cursor;
if (ftio_job_done(ftio))
return 0;
block_num = ftio_current_page(ftio)->index;
data_key = alloc_data_key_from_meta_key(meta_key, block_num);
if (!data_key)
return -ENOMEM;
block_num = ftio_last_page(ftio)->index;
prefetch_data_key = alloc_data_key_from_meta_key(meta_key, block_num);
if (!prefetch_data_key) {
ret = -ENOMEM;
goto out;
}
dbt_init(&key_dbt, data_key, SIZEOF_KEY(*data_key));
dbt_init(&prefetch_key_dbt, prefetch_data_key, SIZEOF_KEY(*prefetch_data_key));
ret = data_db->cursor(data_db, txn, &cursor, DB_CURSOR_FLAGS);
if (ret)
goto free_out;
ret = cursor->c_set_bounds(cursor, &key_dbt, &prefetch_key_dbt, true, 0);
if (ret)
goto close_cursor;
info.meta_key = meta_key;
info.ftio = ftio;
r = cursor->c_getf_set_range(cursor, 0, &key_dbt, ftfs_scan_pages_cb, &info);
while (info.do_continue && !r)
r = cursor->c_getf_next(cursor, 0, ftfs_scan_pages_cb, &info);
if (r && r != DB_NOTFOUND)
ret = r;
if (!ret)
ftfs_bstore_fill_rest_page(ftio);
close_cursor:
r = cursor->c_close(cursor);
BUG_ON(r);
free_out:
data_key_free(prefetch_data_key);
out:
data_key_free(data_key);
return ret;
}
static void scanscan_range (void) {
int r;
double texperiments[n_experiments];
u_int64_t k = 0;
char kv[8];
DBT key, val;
int counter;
for (counter = 0; counter < n_experiments; counter++) {
if (1) { //if ((counter&1) == 0) {
makekey:
// generate a random key in the key range
k = (start_range + (random() % (end_range - start_range))) * (1<<6);
int i;
for (i = 0; i < 8; i++)
kv[i] = k >> (56-8*i);
}
memset(&key, 0, sizeof key); key.data = &kv, key.size = sizeof kv;
memset(&val, 0, sizeof val);
double tstart = gettime();
DBC *dbc;
r = db->cursor(db, tid, &dbc, 0); assert(r==0);
// set the cursor to the random key
r = dbc->c_get(dbc, &key, &val, DB_SET_RANGE+lock_flag);
if (r != 0) {
assert(r == DB_NOTFOUND);
printf("%s:%d %" PRIu64 "\n", __FUNCTION__, __LINE__, k);
goto makekey;
}
#ifdef TOKUDB
// do the range scan
long rowcounter = 0;
struct extra_count e = {0,0};
while (limitcount > 0 && rowcounter < limitcount) {
r = dbc->c_getf_next(dbc, prelockflag ? lock_flag : 0, counttotalbytes, &e);
if (r != 0)
break;
rowcounter++;
}
#endif
r = dbc->c_close(dbc);
assert(r==0);
texperiments[counter] = gettime() - tstart;
printf("%" PRIu64 " %f\n", k, texperiments[counter]); fflush(stdout);
}
static void scanscan_lwc (void) {
int r;
int counter=0;
for (counter=0; counter<n_experiments; counter++) {
struct extra_count e = {0,0};
double prevtime = gettime();
DBC *dbc;
r = db->cursor(db, tid, &dbc, 0); assert(r==0);
u_int32_t f_flags = 0;
if (prelockflag && (counter || prelock)) {
f_flags |= lock_flag;
}
long rowcounter=0;
while (0 == (r = dbc->c_getf_next(dbc, f_flags, counttotalbytes, &e))) {
rowcounter++;
if (limitcount>0 && rowcounter>=limitcount) break;
}
r = dbc->c_close(dbc); assert(r==0);
double thistime = gettime();
double tdiff = thistime-prevtime;
printf("LWC Scan %lld bytes (%d rows) in %9.6fs at %9fMB/s\n", e.totalbytes, e.rowcounter, tdiff, 1e-6*e.totalbytes/tdiff);
}
}
int PerconaFTEngine::Init(const std::string& dir, const std::string& options)
{
Properties props;
parse_conf_content(options, props);
//parse config
conf_get_int64(props, "cache_size", g_perconaft_config.cache_size);
conf_get_uint32(props, "checkpoint_pool_threads", g_perconaft_config.checkpoint_pool_threads);
conf_get_uint32(props, "checkpoint_period", g_perconaft_config.checkpoint_period);
conf_get_uint32(props, "cleaner_period", g_perconaft_config.cleaner_period);
conf_get_uint32(props, "cleaner_iterations", g_perconaft_config.cleaner_iterations);
conf_get_bool(props, "evictor_enable_partial_eviction", g_perconaft_config.evictor_enable_partial_eviction);
std::string compression;
conf_get_string(props, "compression", compression);
if (compression == "none")
{
g_perconaft_config.compression = TOKU_NO_COMPRESSION;
}
else if (compression == "snappy" || compression.empty())
{
g_perconaft_config.compression = TOKU_SNAPPY_METHOD;
}
else if (compression == "zlib")
{
g_perconaft_config.compression = TOKU_ZLIB_METHOD;
}
else
{
ERROR_LOG("Invalid compression config:%s for PercanoFT.", compression.c_str());
return -1;
}
uint32 env_open_flags = DB_CREATE | DB_PRIVATE | DB_THREAD | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG | DB_RECOVER;
int env_open_mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
db_env_create(&m_env, 0);
m_env->set_default_bt_compare(m_env, ardb_perconaft_compare);
m_env->set_errcall(m_env, _err_callback);
/*
* set env config
*/
uint32 cache_gsize = g_perconaft_config.cache_size >> 30;
uint32 cache_bytes = g_perconaft_config.cache_size % (1024 * 1024 * 1024);
m_env->set_cachesize(m_env, cache_gsize, cache_bytes, 1);
m_env->set_cachetable_pool_threads(m_env, g_perconaft_config.checkpoint_pool_threads);
m_env->checkpointing_set_period(m_env, g_perconaft_config.checkpoint_period);
m_env->cleaner_set_period(m_env, g_perconaft_config.cleaner_period);
m_env->cleaner_set_iterations(m_env, g_perconaft_config.cleaner_iterations);
m_env->evictor_set_enable_partial_eviction(m_env, g_perconaft_config.evictor_enable_partial_eviction);
int r = m_env->open(m_env, dir.c_str(), env_open_flags, env_open_mode);
CHECK_EXPR(r);
DataArray nss;
if (0 == r)
{
g_toku_env = m_env;
DB* db = m_env->get_db_for_directory(m_env);
if (NULL != db)
{
PerconaFTLocalContext& local_ctx = g_local_ctx.GetValue();
DB_TXN* txn = local_ctx.transc.Get();
DBC *c = NULL;
CHECK_EXPR(r = db->cursor(db, txn, &c, 0));
if (0 == r)
{
r = c->c_getf_first(c, 0, nil_callback, NULL);
}
while (0 == r)
{
DBT raw_key;
DBT raw_val;
memset(&raw_key, 0, sizeof(raw_key));
memset(&raw_val, 0, sizeof(raw_key));
if (0 == c->c_get(c, &raw_key, &raw_val, DB_CURRENT))
{
//std::string ns_str
Data ns;
ns.SetString((const char*) raw_key.data, false);
INFO_LOG("TokuFT directory db %s:%s", (const char* ) raw_key.data, (const char* ) raw_val.data);
nss.push_back(ns);
}
r = c->c_getf_next(c, 0, nil_callback, NULL);
}
if (NULL != c)
{
c->c_close(c);
}
local_ctx.transc.Release(true);
r = 0;
}
}
for (size_t i = 0; i < nss.size(); i++)
{
Context tmp;
GetFTDB(tmp, nss[i], true);
}
return ENGINE_ERR(r);
}