impl_t(inner_cursor cur,
KeyFn keySelector,
Compare comp = Compare())
: keySelector(keySelector)
, groupIndex(comp)
{
// TODO: make lazy
insert_all(std::move(cur));
}
void map::explore(point const* start, std::vector<point const*> goals)
{
std::set<std::pair<point const*, float>, priority_compare> unvisited;
std::unordered_map<point const*, float> visited;
std::unordered_map<point const*, point const*> previous;
unvisited.emplace(start, 0);
visited.emplace(start, 0);
while (!unvisited.empty())
{
// pop the smallest one
auto current = unvisited.begin();
for (auto link : current->first->get_links())
{
//TODO: check if memoized first
if (visited.find(link) != visited.end())
continue;
auto tentative = current->second + current->first->distance(link);
auto it = std::find_if(unvisited.begin(), unvisited.end(),
[&](std::pair<point const*, float> kvp) {
return kvp.first == link;
});
// if the node is in the 'unvisited' map
if (it != unvisited.end())
{
// use the shorter path of the previous and the current
if (tentative < it->second)
{
unvisited.emplace(link, tentative);
previous[link] = current->first;
}
}
else
{
// if it's not in the map, add it
unvisited.emplace(link, tentative);
previous[link] = current->first;
}
visited[link] = current->second;
// if this is a goal node
auto goals_it = std::find(goals.begin(), goals.end(), link);
if (goals_it != goals.end())
{
// remove it as a goal
goals.erase(goals_it);
// if it was the last goal
if (goals.size() == 0)
{
// we done yo
insert_all(visited, previous);
return;
}
}
}
unvisited.erase(current);
}
}
static void insert_row(DB_ENV *db_env, struct table *t, DB_TXN *txn, long a, long b, long c, long d) {
int r;
// generate the primary key
char key_buffer[8];
a = htonl64(a);
memcpy(key_buffer, &a, sizeof a);
// generate the primary value
char val_buffer[3*8];
b = htonl64(b);
memcpy(val_buffer+0, &b, sizeof b);
c = htonl64(c);
memcpy(val_buffer+8, &c, sizeof c);
d = htonl64(d);
memcpy(val_buffer+16, &d, sizeof d);
DBT key = { .data = key_buffer, .size = sizeof key_buffer };
DBT value = { .data = val_buffer, .size = sizeof val_buffer };
#if defined(TOKUDB)
if (!force_multiple && t->ndbs == 1) {
r = t->dbs[0]->put(t->dbs[0], txn, &key, &value, t->mult_flags[0]); assert(r == 0);
} else {
r = db_env->put_multiple(db_env, t->dbs[0], txn, &key, &value, t->ndbs, &t->dbs[0], t->mult_keys, t->mult_vals, t->mult_flags); assert(r == 0);
}
#else
assert(db_env);
r = t->dbs[0]->put(t->dbs[0], txn, &key, &value, 0); assert(r == 0);
#endif
}
static inline float tdiff (struct timeval *a, struct timeval *b) {
return (a->tv_sec - b->tv_sec) +1e-6*(a->tv_usec - b->tv_usec);
}
static void insert_all(DB_ENV *db_env, struct table *t, long nrows, long max_rows_per_txn, long key_range, long rows_per_report, bool do_txn) {
int r;
struct timeval tstart;
r = gettimeofday(&tstart, NULL); assert(r == 0);
struct timeval tlast = tstart;
DB_TXN *txn = NULL;
if (do_txn) {
r = db_env->txn_begin(db_env, NULL, &txn, 0); assert(r == 0);
}
long n_rows_per_txn = 0;
long rowi;
for (rowi = 0; rowi < nrows; rowi++) {
long a = rowi;
long b = random64() % key_range;
long c = random64() % key_range;
long d = random64() % key_range;
insert_row(db_env, t, txn, a, b, c, d);
n_rows_per_txn++;
// maybe commit
if (do_txn && n_rows_per_txn == max_rows_per_txn) {
r = txn->commit(txn, 0); assert(r == 0);
r = db_env->txn_begin(db_env, NULL, &txn, 0); assert(r == 0);
n_rows_per_txn = 0;
}
// maybe report performance
if (((rowi + 1) % rows_per_report) == 0) {
struct timeval tnow;
r = gettimeofday(&tnow, NULL); assert(r == 0);
float last_time = tdiff(&tnow, &tlast);
float total_time = tdiff(&tnow, &tstart);
printf("%ld %.3f %.0f/s %.0f/s\n", rowi + 1, last_time, rows_per_report/last_time, rowi/total_time); fflush(stdout);
tlast = tnow;
}
}
if (do_txn) {
r = txn->commit(txn, 0); assert(r == 0);
}
struct timeval tnow;
r = gettimeofday(&tnow, NULL); assert(r == 0);
printf("total %ld %.3f %.0f/s\n", nrows, tdiff(&tnow, &tstart), nrows/tdiff(&tnow, &tstart)); fflush(stdout);
}
int main(int argc, char *argv[]) {
#if defined(TOKDUB)
char *db_env_dir = "insertm.env.tokudb";
#else
char *db_env_dir = "insertm.env.bdb";
#endif
int db_env_open_flags = DB_CREATE | DB_PRIVATE | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG;
long rows = 100000000;
long rows_per_txn = 1000;
long rows_per_report = 100000;
long key_range = 100000;
bool do_txn = true;
u_int32_t pagesize = 0;
u_int64_t cachesize = 1000000000;
int ndbs = 4;
#if defined(TOKUDB)
u_int32_t checkpoint_period = 60;
#endif
int i;
for (i = 1; i < argc; i++) {
char *arg = argv[i];
if (strcmp(arg, "--verbose") == 0) {
verbose++;
continue;
}
if (strcmp(arg, "--ndbs") == 0 && i+1 < argc) {
ndbs = atoi(argv[++i]);
continue;
}
if (strcmp(arg, "--rows") == 0 && i+1 < argc) {
rows = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--rows_per_txn") == 0 && i+1 < argc) {
rows_per_txn = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--rows_per_report") == 0 && i+1 < argc) {
rows_per_report = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--key_range") == 0 && i+1 < argc) {
key_range = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--txn") == 0 && i+1 < argc) {
do_txn = atoi(argv[++i]);
continue;
}
if (strcmp(arg, "--pagesize") == 0 && i+1 < argc) {
pagesize = atoi(argv[++i]);
continue;
}
if (strcmp(arg, "--cachesize") == 0 && i+1 < argc) {
cachesize = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--force_multiple") == 0 && i+1 < argc) {
force_multiple = atoi(argv[++i]);
continue;
}
#if defined(TOKUDB)
if (strcmp(arg, "--checkpoint_period") == 0 && i+1 < argc) {
checkpoint_period = atoi(argv[++i]);
continue;
}
#endif
assert(0);
}
int r;
char rm_cmd[strlen(db_env_dir) + strlen("rm -rf ") + 1];
snprintf(rm_cmd, sizeof(rm_cmd), "rm -rf %s", db_env_dir);
r = system(rm_cmd); assert(r == 0);
r = mkdir(db_env_dir, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); assert(r == 0);
// create and open the env
DB_ENV *db_env = NULL;
r = db_env_create(&db_env, 0); assert(r == 0);
if (!do_txn)
db_env_open_flags &= ~(DB_INIT_TXN | DB_INIT_LOG);
if (cachesize) {
const u_int64_t gig = 1 << 30;
r = db_env->set_cachesize(db_env, cachesize / gig, cachesize % gig, 1); assert(r == 0);
}
#if defined(TOKUDB)
r = db_env->set_generate_row_callback_for_put(db_env, my_generate_row_for_put); assert(r == 0);
#endif
r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
#if defined(TOKUDB)
if (checkpoint_period) {
r = db_env->checkpointing_set_period(db_env, checkpoint_period); assert(r == 0);
u_int32_t period;
r = db_env->checkpointing_get_period(db_env, &period); assert(r == 0 && period == checkpoint_period);
}
#endif
// create the db
DB *dbs[ndbs];
for (i = 0; i < ndbs; i++) {
DB *db = NULL;
r = db_create(&db, db_env, 0); assert(r == 0);
DB_TXN *create_txn = NULL;
if (do_txn) {
r = db_env->txn_begin(db_env, NULL, &create_txn, 0); assert(r == 0);
}
if (pagesize) {
r = db->set_pagesize(db, pagesize); assert(r == 0);
}
char db_filename[32]; sprintf(db_filename, "test%d", i);
r = db->open(db, create_txn, db_filename, NULL, DB_BTREE, DB_CREATE, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
#if defined(TOKUDB)
DESCRIPTOR_S new_descriptor;
int index_num = htonl(i);
new_descriptor.dbt.data = &index_num;
new_descriptor.dbt.size = sizeof i;
r = db->change_descriptor(db, create_txn, &new_descriptor.dbt, 0); assert(r == 0);
#else
db->app_private = (void *) (intptr_t) i;
if (i > 0) {
r = dbs[0]->associate(dbs[0], create_txn, db, my_secondary_key, 0); assert(r == 0);
}
#endif
if (do_txn) {
r = create_txn->commit(create_txn, 0); assert(r == 0);
}
dbs[i] = db;
}
// insert all rows
struct table table;
table_init(&table, ndbs, dbs, 4 * 8, 4 * 8);
insert_all(db_env, &table, rows, rows_per_txn, key_range, rows_per_report, do_txn);
table_destroy(&table);
// shutdown
for (i = 0; i < ndbs; i++) {
DB *db = dbs[i];
r = db->close(db, 0); assert(r == 0); db = NULL;
}
r = db_env->close(db_env, 0); assert(r == 0); db_env = NULL;
return 0;
}
