int
main(int argc, char **argv)
{
MdbHandle *mdb;
int print_mdbver = 0;
int opt;
/* setlocale (LC_ALL, ""); */
bindtextdomain (PACKAGE, LOCALEDIR);
textdomain (PACKAGE);
while ((opt=getopt(argc, argv, "M"))!=-1) {
switch (opt) {
case 'M':
print_mdbver = 1;
break;
default:
break;
}
}
if (print_mdbver) {
fprintf(stdout,"%s\n", MDB_FULL_VERSION);
if (argc-optind < 1) exit(0);
}
/*
** optind is now the position of the first non-option arg,
** see getopt(3)
*/
if (argc-optind < 1) {
fprintf(stderr,_("Usage: %s [-M] <file>\n"),argv[0]);
exit(1);
}
if (!(mdb = mdb_open(argv[optind], MDB_NOFLAGS))) {
fprintf(stderr,_("Error: unable to open file %s\n"),argv[optind]);
exit(1);
}
switch(mdb->f->jet_version) {
case MDB_VER_JET3:
printf("JET3\n");
break;
case MDB_VER_JET4:
printf("JET4\n");
break;
case MDB_VER_ACCDB_2007:
printf("ACE12\n");
break;
case MDB_VER_ACCDB_2010:
printf("ACE14\n");
break;
default:
printf(_("unknown database version\n"));
break;
}
mdb_close(mdb);
return 0;
}
main(int argc, char **argv)
{
int rows;
int i;
unsigned char buf[2048];
MdbHandle *mdb;
MdbCatalogEntry entry;
#if 0
if (argc<2) {
fprintf(stderr,"Usage: %s <file> <table>\n",argv[0]);
exit(1);
}
mdb_init();
mdb = mdb_open(argv[1]);
mdb_read_pg(mdb, MDB_CATALOG_PG);
rows = mdb_catalog_rows(mdb);
for (i=0;i<rows;i++) {
if (mdb_read_catalog_entry(mdb, i, &entry)) {
if (!strcmp(entry.object_name,argv[2])) {
mdb_kkd_dump(&entry);
}
}
}
mdb_free_handle(mdb);
mdb_exit();
#endif
exit(0);
}
DB(const char* db_path, bool use_leveldb_) : use_leveldb(use_leveldb_), count(0), batch(NULL) {
// Open db
if (use_leveldb) { // leveldb
LOG(INFO) << "Opening leveldb " << db_path;
leveldb::Options options;
options.error_if_exists = true;
options.create_if_missing = true;
options.write_buffer_size = 268435456;
leveldb::Status status = leveldb::DB::Open(options, db_path, &db);
CHECK(status.ok()) << "Failed to open leveldb " << db_path
<< ". Is it already existing?";
batch = new leveldb::WriteBatch();
} else { // lmdb
LOG(INFO) << "Opening lmdb " << db_path;
CHECK_EQ(mkdir(db_path, 0744), 0)
<< "mkdir " << db_path << "failed";
CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed";
CHECK_EQ(mdb_env_set_mapsize(mdb_env, 1099511627776), MDB_SUCCESS) // 1TB
<< "mdb_env_set_mapsize failed";
CHECK_EQ(mdb_env_open(mdb_env, db_path, 0, 0664), MDB_SUCCESS)
<< "mdb_env_open failed";
CHECK_EQ(mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn), MDB_SUCCESS)
<< "mdb_txn_begin failed";
CHECK_EQ(mdb_open(mdb_txn, NULL, 0, &mdb_dbi), MDB_SUCCESS)
<< "mdb_open failed. Does the lmdb already exist? ";
}
}
int CDbxMdb::Load(bool bSkipInit)
{
if (!Remap())
return EGROKPRF_CANTREAD;
if (!bSkipInit) {
txn_ptr trnlck(m_pMdbEnv);
mdb_open(trnlck, "global", MDB_CREATE | MDB_INTEGERKEY, &m_dbGlobal);
mdb_open(trnlck, "contacts", MDB_CREATE | MDB_INTEGERKEY, &m_dbContacts);
mdb_open(trnlck, "modules", MDB_CREATE | MDB_INTEGERKEY, &m_dbModules);
mdb_open(trnlck, "events", MDB_CREATE | MDB_INTEGERKEY, &m_dbEvents);
mdb_open(trnlck, "eventsrt", MDB_CREATE | MDB_INTEGERKEY, &m_dbEventsSort);
mdb_open(trnlck, "settings", MDB_CREATE, &m_dbSettings);
DWORD keyVal = 1;
MDB_val key = { sizeof(DWORD), &keyVal }, data;
if (mdb_get(trnlck, m_dbGlobal, &key, &data) == MDB_SUCCESS) {
DBHeader *hdr = (DBHeader*)data.mv_data;
if (hdr->dwSignature != DBHEADER_SIGNATURE)
DatabaseCorruption(NULL);
memcpy(&m_header, data.mv_data, sizeof(m_header));
}
else {
m_header.dwSignature = DBHEADER_SIGNATURE;
m_header.dwVersion = 1;
data.mv_data = &m_header; data.mv_size = sizeof(m_header);
mdb_put(trnlck, m_dbGlobal, &key, &data, 0);
keyVal = 0;
DBContact dbc = { DBCONTACT_SIGNATURE, 0, 0, 0 };
data.mv_data = &dbc; data.mv_size = sizeof(dbc);
mdb_put(trnlck, m_dbContacts, &key, &data, 0);
}
trnlck.commit();
if (InitModuleNames()) return EGROKPRF_CANTREAD;
if (InitCrypt()) return EGROKPRF_CANTREAD;
// everything is ok, go on
if (!m_bReadOnly) {
// we don't need events in the service mode
if (ServiceExists(MS_DB_SETSAFETYMODE)) {
hContactDeletedEvent = CreateHookableEvent(ME_DB_CONTACT_DELETED);
hContactAddedEvent = CreateHookableEvent(ME_DB_CONTACT_ADDED);
hSettingChangeEvent = CreateHookableEvent(ME_DB_CONTACT_SETTINGCHANGED);
hEventMarkedRead = CreateHookableEvent(ME_DB_EVENT_MARKED_READ);
hEventAddedEvent = CreateHookableEvent(ME_DB_EVENT_ADDED);
hEventDeletedEvent = CreateHookableEvent(ME_DB_EVENT_DELETED);
hEventFilterAddedEvent = CreateHookableEvent(ME_DB_EVENT_FILTER_ADD);
}
}
FillContacts();
}
return ERROR_SUCCESS;
}
int LMDBStore::open(const char *path, bool durable)
{
int rc;
rc = mdb_env_create(&env);
rc = mdb_env_set_mapsize(env, size_t(1048576000));
rc = mdb_env_set_maxdbs(env, 1024);
rc = mdb_env_open(env, path, MDB_NOSYNC | MDB_WRITEMAP, 0664);
MDB_txn *txn;
rc = mdb_txn_begin(env, NULL, 0, &txn);
rc = mdb_open(txn, NULL, 0, &dbi);
rc = mdb_txn_commit(txn);
return HCAT_SUCCESS;
}
static int dbase_open(struct cache *cache, const char *handle)
{
long page_size = sysconf(_SC_PAGESIZE);
if (page_size <= 0) {
return KNOT_EINVAL;
}
int ret = mdb_env_create(&cache->env);
if (ret != 0) {
return ret;
}
size_t map_size = (LMDB_MAPSIZE / page_size) * page_size;
ret = mdb_env_set_mapsize(cache->env, map_size);
if (ret != 0) {
mdb_env_close(cache->env);
return ret;
}
ret = mdb_env_open(cache->env, handle, 0, 0644);
if (ret != 0) {
mdb_env_close(cache->env);
return ret;
}
MDB_txn *txn = NULL;
ret = mdb_txn_begin(cache->env, NULL, 0, &txn);
if (ret != 0) {
mdb_env_close(cache->env);
return ret;
}
ret = mdb_open(txn, NULL, MDB_DUPSORT, &cache->dbi);
if (ret != 0) {
mdb_txn_abort(txn);
mdb_env_close(cache->env);
return ret;
}
ret = mdb_txn_commit(txn);
if (ret != 0) {
mdb_env_close(cache->env);
return ret;
}
return 0;
}
int main(int argc,char * argv[])
{
int rc;
MDB_env *env;
MDB_txn *txn;
MDB_dbi dbi;
MDB_stat mst;
char *envname = argv[1];
char *subname = NULL;
rc = mdb_env_create(&env);
if (argc > 2) {
mdb_env_set_maxdbs(env, 4);
subname = argv[2];
}
rc = mdb_env_open(env, envname, MDB_RDONLY, 0);
if (rc) {
printf("mdb_env_open failed, error %d\n", rc);
exit(1);
}
rc = mdb_txn_begin(env, NULL, 1, &txn);
if (rc) {
printf("mdb_txn_begin failed, error %d\n", rc);
exit(1);
}
rc = mdb_open(txn, subname, 0, &dbi);
if (rc) {
printf("mdb_open failed, error %d\n", rc);
exit(1);
}
rc = mdb_stat(txn, dbi, &mst);
printf("Page size: %u\n", mst.ms_psize);
printf("Tree depth: %u\n", mst.ms_depth);
printf("Branch pages: %zu\n", mst.ms_branch_pages);
printf("Leaf pages: %zu\n", mst.ms_leaf_pages);
printf("Overflow pages: %zu\n", mst.ms_overflow_pages);
printf("Entries: %zu\n", mst.ms_entries);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}
int LMDBEngine::Init(const LMDBConfig& cfg, MDB_env *env,
const std::string& name)
{
m_env = env;
MDB_txn *txn;
int rc = mdb_txn_begin(env, NULL, 0, &txn);
rc = mdb_open(txn, NULL, MDB_CREATE, &m_dbi);
if (rc != 0)
{
ERROR_LOG(
"Failed to open mdb:%s for reason:%s\n", name.c_str(), mdb_strerror(rc));
return -1;
}
mdb_set_compare(txn, m_dbi, LMDBCompareFunc);
mdb_txn_commit(txn);
return 0;
}
int
main(int argc, char **argv)
{
MdbHandle *mdb;
MdbTableDef *table;
char *colname, *tabname;
char *colval;
int colnum;
char *sargname = NULL;
char *updstr = NULL;
char data[255];
int len;
if (argc<4) {
fprintf(stderr,"Usage: %s <file> <table> <sargs> <updstr>\n",argv[0]);
exit(1);
}
mdb = mdb_open(argv[1], MDB_WRITABLE);
tabname = argv[2];
sargname = argv[3];
updstr = g_strdup(argv[4]);
table = mdb_read_table_by_name(mdb, tabname, MDB_TABLE);
if (table) {
mdb_read_columns(table);
mdb_read_indices(table);
printf("updstr %s\n",updstr);
colname = strtok(updstr,"=");
colval = strtok(NULL,"=");
colnum = mdb_bind_column_by_name(table, colname, data, &len);
printf("column %d\n", colnum);
read_to_row(table, sargname);
printf("current value of %s is %s, changing to %s\n", colname, data, colval);
len = strlen(colval);
strcpy(data,colval);
mdb_update_row(table);
mdb_free_tabledef(table);
}
mdb_close(mdb);
return 0;
}
int
main(int argc, char **argv)
{
MdbHandle *mdb;
if (argc<2 || argc>3) {
fprintf(stderr, "Usage: %s <file> [<objtype>]\n", argv[0]);
fprintf(stderr, "Use objtype -1 for all\n");
exit(1);
}
mdb = mdb_open(argv[1], MDB_NOFLAGS);
mdb_dump_catalog(mdb,(argc > 2) ? atoi(argv[2]) : MDB_TABLE);
mdb_close(mdb);
return 0;
}
int main(int argc,char * argv[])
{
int rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_cursor *cursor;
char sval[32];
rc = mdb_env_create(&env);
rc = mdb_env_open(env, "./testdb", 0, 0664);
rc = mdb_txn_begin(env, NULL, 0, &txn);
rc = mdb_open(txn, NULL, 0, &dbi);
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
sprintf(sval, "%03x %d foo bar", 32, 3141592);
rc = mdb_put(txn, dbi, &key, &data, 0);
rc = mdb_txn_commit(txn);
if (rc) {
fprintf(stderr, "mdb_txn_commit: (%d) %s\n", rc, mdb_strerror(rc));
goto leave;
}
rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
rc = mdb_cursor_open(txn, dbi, &cursor);
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
leave:
mdb_close(env, dbi);
mdb_env_close(env);
return 0;
}
int
main(int argc, char **argv)
{
MdbHandle *mdb;
MdbTableDef *table;
char *dot, *colname, *tabname;
char *sargname = NULL;
if (argc<2) {
fprintf(stderr,"Usage: %s <file> <table.column> [sargs]\n",argv[0]);
exit(1);
}
mdb_init();
mdb = mdb_open(argv[1], MDB_NOFLAGS);
dot = strchr(argv[2],'.');
if (argc>3) sargname = argv[3];
if (!dot) {
fprintf(stderr,"Usage: %s <file> <table.column> [sarg]\n",argv[0]);
exit(1);
}
tabname = argv[2];
*dot='\0';
colname = ++dot;
table = mdb_read_table_by_name(mdb, tabname, MDB_TABLE);
if (table) {
mdb_read_columns(table);
dump_ole(table, colname, sargname);
mdb_free_tabledef(table);
}
mdb_close(mdb);
mdb_exit();
exit(0);
}
main(int argc, char **argv)
{
MdbHandle *mdb;
MdbTableDef *table;
mdb_init();
if (!(mdb = mdb_open(MDB_FILE, MDB_NOFLAGS))) {
exit(1);
}
table = mdb_read_table_by_name(mdb, TABLE_NAME, MDB_TABLE);
if (table) {
print_table(table);
mdb_free_tabledef(table);
}
mdb_close(mdb);
mdb_exit();
exit(0);
}
kvdb_t *kvdb_lmdb_open(kvenv_t *kvenv, const char *dbname)
{
MDB_txn *txn;
kvdb_lmdb_t *lmdb = (kvdb_lmdb_t *)zmalloc(sizeof(struct kvdb_lmdb_t));
memset(lmdb, 0, sizeof(kvdb_lmdb_t));
lmdb->kvdb.kvenv = kvenv;
lmdb->kvdb.dbclass = "lmdb";
lmdb->kvdb.db_methods = &lmdb_methods;
kvenv_lmdb_t *kvenv_lmdb = (kvenv_lmdb_t*)kvenv;
int rc = mdb_txn_begin(kvenv_lmdb->env, NULL, 0, &txn);
if ( rc != 0 ) {
zfree(lmdb);
error_log("mdb_txn_begin() failed.");
return NULL;
}
rc = mdb_open(txn, dbname, MDB_CREATE, &lmdb->dbi);
if ( rc != 0 ) {
zfree(lmdb);
error_log("mdb_open() failed. dbname:%s", dbname);
return NULL;
}
rc = mdb_txn_commit(txn);
if ( rc != 0 ) {
zfree(lmdb);
error_log("mdb_txn_commit() failed.");
return NULL;
}
return (kvdb_t*)lmdb;
}
int
mdb_exec_open(
PMDB_INSPECT_CMD pCmd,
PMDB_INSPECT_CONTEXT pCtxt
)
{
int error = 0;
if(!pCmd || !pCtxt)
{
error = EINVAL;
bail_on_error(error);
}
error = mdb_open(pCmd->extra_argv[0], pCtxt);
bail_on_error(error);
fprintf(stdout, "opened file: %s\n", pCmd->extra_argv[0]);
cleanup:
return error;
error:
goto cleanup;
}
main(int argc, char **argv)
{
int rows;
int i;
unsigned char buf[2048];
MdbHandle *mdb;
MdbCatalogEntry *entry;
MdbTableDef *table;
GList *l;
if (argc<2) {
fprintf(stderr,"Usage: %s <file> <table>\n",argv[0]);
exit(1);
}
mdb_init();
mdb = mdb_open(argv[1]);
mdb_read_catalog(mdb, MDB_TABLE);
for (i=0;i<mdb->num_catalog;i++) {
entry = g_ptr_array_index(mdb->catalog,i);
if (entry->object_type == MDB_TABLE &&
!strcmp(entry->object_name,argv[2])) {
table = mdb_read_table(entry);
mdb_read_columns(table);
mdb_data_dump(table);
}
}
mdb_free_handle(mdb);
mdb_exit();
exit(0);
}
int
main(int argc, char **argv)
{
unsigned int i;
MdbHandle *mdb;
MdbCatalogEntry *entry;
int found = 0;
if (argc<3) {
fprintf(stderr,"Usage: %s <file> <table>\n",argv[0]);
exit(1);
}
mdb_init();
mdb = mdb_open(argv[1], MDB_NOFLAGS);
mdb_read_catalog(mdb, MDB_TABLE);
for (i=0;i<mdb->num_catalog;i++) {
entry = g_ptr_array_index(mdb->catalog,i);
if (entry->object_type == MDB_TABLE &&
!strcmp(entry->object_name,argv[2])) {
mdb_table_dump(entry);
found++;
}
}
if (!found) {
fprintf(stderr,"No table named %s found.\n", argv[2]);
}
mdb_close(mdb);
mdb_exit();
exit(0);
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[8];
char kval[sizeof(int)];
memset(sval, 0, sizeof(sval));
count = 510;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = i*5;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
strcpy(kval, "001");
for (i=0;i<count;i++) {
sprintf(sval, "%07x", values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
/* there should be one full page of dups now.
*/
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
/* test all 3 branches of split code:
* 1: new key in lower half
* 2: new key at split point
* 3: new key in upper half
*/
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
sprintf(sval, "%07x", values[3]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
mdb_txn_abort(txn);
sprintf(sval, "%07x", values[255]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
mdb_txn_abort(txn);
sprintf(sval, "%07x", values[500]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
E(mdb_txn_commit(txn));
/* Try MDB_NEXT_MULTIPLE */
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_MULTIPLE)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
for (i= count - 1; i > -1; i-= (rand()%3)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%07x", values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}
int
main (int argc, char **argv)
{
unsigned int i, j, k;
unsigned int unsupported = 0;
MdbHandle *mdb;
MdbCatalogEntry *entry;
MdbTableDef *table;
MdbColumn *col;
FILE *typesfile;
FILE *headerfile;
FILE *cfile;
if (argc < 2) {
fprintf (stderr, "Usage: %s <file>\n",argv[0]);
exit (1);
}
mdb_init();
/* open the database */
mdb = mdb_open (argv[1], MDB_NOFLAGS);
if (!mdb) {
mdb_exit();
exit(1);
}
typesfile = fopen ("types.h", "w");
headerfile = fopen ("dumptypes.h", "w");
cfile = fopen ("dumptypes.c", "w");
copy_header (typesfile);
copy_header (headerfile);
fprintf (headerfile, "#include \"types.h\"\n");
copy_header (cfile);
fprintf (cfile, "#include <stdio.h>\n");
fprintf (cfile, "#include \"dumptypes.h\"\n");
/* read the catalog */
mdb_read_catalog (mdb, MDB_TABLE);
/* loop over each entry in the catalog */
for (i=0; i < mdb->num_catalog; i++)
{
entry = g_ptr_array_index (mdb->catalog, i);
if (!mdb_is_user_table(entry))
continue;
fprintf (typesfile, "typedef struct _%s\n", entry->object_name);
fprintf (typesfile, "{\n");
fprintf (headerfile, "void dump_%s (%s x);\n",
entry->object_name, entry->object_name);
fprintf (cfile, "void dump_%s (%s x)\n{\n",
entry->object_name, entry->object_name);
fprintf (cfile, "\tfprintf (stdout, \"**************** %s ****************\\n\");\n", entry->object_name);
table = mdb_read_table (entry);
/* get the columns */
mdb_read_columns (table);
/* loop over the columns, dumping the names and types */
for (k = 0; k < table->num_cols; k++)
{
col = g_ptr_array_index (table->columns, k);
fprintf (cfile, "\tfprintf (stdout, \"x.");
for (j = 0; j < strlen (col->name); j++)
{
fprintf (cfile, "%c", tolower (col->name [j]));
}
fprintf (cfile, " = \");\n");
switch (col->col_type)
{
case MDB_INT:
fprintf (typesfile, "\tint\t");
fprintf (cfile, "\tdump_int (x.");
break;
case MDB_LONGINT:
fprintf (typesfile, "\tlong\t");
fprintf (cfile, "\tdump_long (x.");
break;
case MDB_TEXT:
case MDB_MEMO:
fprintf (typesfile, "\tchar *\t");
fprintf (cfile, "\tdump_string (x.");
break;
default:
unsupported = 1;
break;
}
for (j = 0; j < strlen (col->name); j++)
{
fprintf (typesfile, "%c", tolower (col->name [j]));
fprintf (cfile, "%c", tolower (col->name [j]));
}
fprintf (typesfile, ";\n");
fprintf (cfile, ");\n");
}
fprintf (typesfile, "\n} %s ;\n", entry->object_name);
fprintf (typesfile, "\n");
fprintf (cfile, "}\n\n");
mdb_free_tabledef(table);
}
fclose (headerfile);
fclose (typesfile);
fclose (cfile);
mdb_close (mdb);
mdb_exit();
if (unsupported)
fputs("ERROR: unsupported type.\n", stderr);
exit(unsupported);
}
int main(int argc, char *argv[])
{
int i, rc;
MDB_env *env;
MDB_txn *txn;
MDB_cursor *mc;
MDB_dbi dbi;
char *envname;
int envflags = 0, putflags = 0;
int dohdr = 0;
prog = argv[0];
if (argc < 2) {
usage();
}
/* -f: load file instead of stdin
* -n: use NOSUBDIR flag on env_open
* -s: load into named subDB
* -N: use NOOVERWRITE on puts
* -T: read plaintext
* -V: print version and exit
*/
while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
switch(i) {
case 'V':
printf("%s\n", MDB_VERSION_STRING);
exit(0);
break;
case 'f':
if (freopen(optarg, "r", stdin) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n",
prog, optarg, strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 's':
subname = strdup(optarg);
break;
case 'N':
putflags = MDB_NOOVERWRITE|MDB_NODUPDATA;
break;
case 'T':
mode |= NOHDR | PRINT;
break;
default:
usage();
}
}
if (optind != argc - 1)
usage();
dbuf.mv_size = 4096;
dbuf.mv_data = malloc(dbuf.mv_size);
if (!(mode & NOHDR))
readhdr();
envname = argv[optind];
rc = mdb_env_create(&env);
if (rc) {
fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
return EXIT_FAILURE;
}
mdb_env_set_maxdbs(env, 2);
if (info.me_maxreaders)
mdb_env_set_maxreaders(env, info.me_maxreaders);
if (info.me_mapsize)
mdb_env_set_mapsize(env, info.me_mapsize);
if (info.me_mapaddr)
envflags |= MDB_FIXEDMAP;
rc = mdb_env_open(env, envname, envflags, 0664);
if (rc) {
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
kbuf.mv_data = malloc(kbuf.mv_size);
while(!Eof) {
MDB_val key, data;
int batch = 0;
flags = 0;
if (!dohdr) {
dohdr = 1;
} else if (!(mode & NOHDR))
readhdr();
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi);
if (rc) {
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
while(1) {
rc = readline(&key, &kbuf);
if (rc) /* rc == EOF */
break;
rc = readline(&data, &dbuf);
if (rc) {
fprintf(stderr, "%s: line %" Z "d: failed to read key value\n", prog, lineno);
goto txn_abort;
}
rc = mdb_cursor_put(mc, &key, &data, putflags);
if (rc == MDB_KEYEXIST && putflags)
continue;
if (rc) {
fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
batch++;
if (batch == 100) {
rc = mdb_txn_commit(txn);
if (rc) {
fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
prog, lineno, mdb_strerror(rc));
goto env_close;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
batch = 0;
}
}
rc = mdb_txn_commit(txn);
txn = NULL;
if (rc) {
fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
prog, lineno, mdb_strerror(rc));
goto env_close;
}
mdb_dbi_close(env, dbi);
}
txn_abort:
mdb_txn_abort(txn);
env_close:
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}
int main(int argc, char *argv[])
{
int i, rc;
MDB_env *env;
MDB_txn *txn;
MDB_dbi dbi;
MDB_stat mst;
MDB_envinfo mei;
char *prog = argv[0];
char *envname;
char *subname = NULL;
int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0;
if (argc < 2) {
usage(prog);
}
/* -a: print stat of main DB and all subDBs
* -s: print stat of only the named subDB
* -e: print env info
* -f: print freelist info
* -r: print reader info
* -n: use NOSUBDIR flag on env_open
* -V: print version and exit
* (default) print stat of only the main DB
*/
while ((i = getopt(argc, argv, "Vaefnrs:")) != EOF) {
switch(i) {
case 'V':
printf("%s\n", MDB_VERSION_STRING);
exit(0);
break;
case 'a':
if (subname)
usage(prog);
alldbs++;
break;
case 'e':
envinfo++;
break;
case 'f':
freinfo++;
break;
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 'r':
rdrinfo++;
break;
case 's':
if (alldbs)
usage(prog);
subname = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
envname = argv[optind];
rc = mdb_env_create(&env);
if (rc) {
fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
return EXIT_FAILURE;
}
if (alldbs || subname) {
mdb_env_set_maxdbs(env, 4);
}
rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664);
if (rc) {
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
if (envinfo) {
(void)mdb_env_stat(env, &mst);
(void)mdb_env_info(env, &mei);
printf("Environment Info\n");
printf(" Map address: %p\n", mei.me_mapaddr);
printf(" Map size: %"Z"u\n", mei.me_mapsize);
printf(" Page size: %u\n", mst.ms_psize);
printf(" Max pages: %"Z"u\n", mei.me_mapsize / mst.ms_psize);
printf(" Number of pages used: %"Z"u\n", mei.me_last_pgno+1);
printf(" Last transaction ID: %"Z"u\n", mei.me_last_txnid);
printf(" Max readers: %u\n", mei.me_maxreaders);
printf(" Number of readers used: %u\n", mei.me_numreaders);
}
if (rdrinfo) {
printf("Reader Table Status\n");
rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout);
if (rdrinfo > 1) {
int dead;
mdb_reader_check(env, &dead);
printf(" %d stale readers cleared.\n", dead);
rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout);
}
if (!(subname || alldbs || freinfo))
goto env_close;
}
rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
if (freinfo) {
MDB_cursor *cursor;
MDB_val key, data;
size_t pages = 0, *iptr;
printf("Freelist Status\n");
dbi = 0;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_stat(txn, dbi, &mst);
if (rc) {
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
prstat(&mst);
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
iptr = data.mv_data;
pages += *iptr;
if (freinfo > 1) {
char *bad = "";
size_t pg, prev;
ssize_t i, j, span = 0;
j = *iptr++;
for (i = j, prev = 1; --i >= 0; ) {
pg = iptr[i];
if (pg <= prev)
bad = " [bad sequence]";
prev = pg;
pg += span;
for (; i >= span && iptr[i-span] == pg; span++, pg++) ;
}
printf(" Transaction %"Z"u, %"Z"d pages, maxspan %"Z"d%s\n",
*(size_t *)key.mv_data, j, span, bad);
if (freinfo > 2) {
for (--j; j >= 0; ) {
pg = iptr[j];
for (span=1; --j >= 0 && iptr[j] == pg+span; span++) ;
printf(span>1 ? " %9"Z"u[%"Z"d]\n" : " %9"Z"u\n",
pg, span);
}
}
}
}
mdb_cursor_close(cursor);
printf(" Free pages: %"Z"u\n", pages);
}
rc = mdb_open(txn, subname, 0, &dbi);
if (rc) {
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_stat(txn, dbi, &mst);
if (rc) {
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
printf("Status of %s\n", subname ? subname : "Main DB");
prstat(&mst);
if (alldbs) {
MDB_cursor *cursor;
MDB_val key;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) {
char *str;
MDB_dbi db2;
if (memchr(key.mv_data, '\0', key.mv_size))
continue;
str = malloc(key.mv_size+1);
memcpy(str, key.mv_data, key.mv_size);
str[key.mv_size] = '\0';
rc = mdb_open(txn, str, 0, &db2);
if (rc == MDB_SUCCESS)
printf("Status of %s\n", str);
free(str);
if (rc) continue;
rc = mdb_stat(txn, db2, &mst);
if (rc) {
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
prstat(&mst);
mdb_close(env, db2);
}
mdb_cursor_close(cursor);
}
if (rc == MDB_NOTFOUND)
rc = MDB_SUCCESS;
mdb_close(env, dbi);
txn_abort:
mdb_txn_abort(txn);
env_close:
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
long kval;
char *sval;
srand(time(NULL));
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
E(mdb_stat(txn, dbi, &mst));
sval = calloc(1, mst.ms_psize / 4);
key.mv_size = sizeof(long);
key.mv_data = &kval;
data.mv_size = mst.ms_psize / 4 - 30;
data.mv_data = sval;
printf("Adding 12 values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5;
sprintf(sval, "%08x", kval);
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+4;
sprintf(sval, "%08x", kval);
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+1;
sprintf(sval, "%08x", kval);
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST));
do {
printf("key: %p %s, data: %p %.*s\n",
key.mv_data, mdb_dkey(&key, dkbuf),
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
} while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0);
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_commit(txn);
#if 0
j=0;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(kval, "%03x", values[i & ~0x0f]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
#endif
mdb_env_close(env);
return 0;
}
int
main (int argc, char **argv)
{
unsigned int j;
MdbHandle *mdb;
MdbTableDef *table;
MdbColumn *col;
/* doesn't handle tables > 256 columns. Can that happen? */
char *bound_values [256];
char delimiter [] = ", ";
char quote_text = 1;
int count = 0;
int started;
if (argc < 3)
{
fprintf (stderr, "Usage: %s <file> <table>\n", argv [0]);
exit (1);
}
mdb_init();
mdb = mdb_open (argv [1], MDB_NOFLAGS);
table = mdb_read_table_by_name (mdb, argv[2], MDB_TABLE);
if (table)
{
mdb_read_columns (table);
mdb_rewind_table (table);
for (j = 0; j < table->num_cols; j++)
{
bound_values [j] = (char *) g_malloc (MDB_BIND_SIZE);
bound_values [j] [0] = '\0';
mdb_bind_column (table, j+1, bound_values[j], NULL);
}
fprintf (stdout, "/******************************************************************/\n");
fprintf (stdout, "/* THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT IT!!!!!! */\n");
fprintf (stdout, "/******************************************************************/\n");
fprintf (stdout, "\n");
fprintf (stdout, "#include <stdio.h>\n");
fprintf (stdout, "#include \"types.h\"\n");
fprintf (stdout, "#include \"dump.h\"\n");
fprintf (stdout, "\n");
fprintf (stdout, "const %s %s_array [] = {\n", argv [2], argv [2]);
count = 0;
started = 0;
while (mdb_fetch_row (table))
{
if (started != 0)
{
fprintf (stdout, ",\n");
}
started = 1;
fprintf (stdout, "{\t\t\t\t/* %6d */\n\t", count);
for (j = 0; j < table->num_cols; j++)
{
fprintf (stdout, "\t");
col = g_ptr_array_index (table->columns, j);
if (quote_text &&
(col->col_type == MDB_TEXT ||
col->col_type == MDB_MEMO))
{
fprintf (stdout, "\"%s\"", bound_values [j]);
}
else
{
fprintf (stdout, "%s", bound_values [j]);
}
if (j != table->num_cols - 1)
{
fprintf (stdout, "%s\n", delimiter);
}
else
{
fprintf (stdout, "\n");
}
}
fprintf (stdout, "}");
count++;
}
fprintf (stdout, "\n};\n\n");
for (j = 0; j < table->num_cols; j++)
{
g_free (bound_values [j]);
}
mdb_free_tabledef(table);
}
mdb_close (mdb);
mdb_exit();
fprintf (stdout, "const int %s_array_length = %d;\n",
argv [2],
count);
exit(0);
}
int main(int argc, char *argv[])
{
int size;
char string[LINE_SIZE+1];
char s_key[LINE_SIZE+1];
char s_value[LINE_SIZE+1];
char* token;
char* save;
int cnt_line;
int rc;
MDB_env* env;
MDB_txn* txn;
MDB_cursor* mc;
MDB_dbi dbi;
MDB_val key, data;
char* envname;
int envflags=0;
char* subname;
char* prog = argv[0];
size_t map_size = (SIZE_MAX / (1024*1024*1024) / 4)*6; // 4giga * 6
struct timeval tv1, tv2;
if(argc != 3) {
fprintf(stderr,"%s <envname> <subname>\n",prog);
exit(1);
}
envflags = MDB_NOSUBDIR | MDB_NOLOCK;
envname = argv[1];
rc = mdb_env_create(&env);
if(rc) {
fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
return EXIT_FAILURE;
}
mdb_env_set_maxdbs(env, 2);
mdb_env_set_mapsize(env, map_size);
rc = mdb_env_open(env, envname, envflags, 0664);
if(rc) {
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if(rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
subname = argv[2];
rc = mdb_open(txn, subname, MDB_CREATE, &dbi);
if (rc) {
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
gettimeofday(&tv1, NULL);
cnt_line = 0;
while(fgets(string, LINE_SIZE, stdin) != NULL) {
size = strlen(string);
if(string[size-1] == '\n'){
string[size-1] = '\0';
--size;
}
if(size > 1 && string[size-1] == '\r'){
string[size-1] = '\0';
--size;
}
if(string[0] == '\0')
continue;
if(cnt_line % 10000 == 0)
fprintf(stderr,"[linecount]\t%d\n",cnt_line);
/*
token = strtok_r(string, "\t", &save);
if(token != NULL) {
strcpy(s_key, token);
token = strtok_r(NULL, "\t", &save);
if(token != NULL) {
strcpy(s_value, token);
} else continue;
} else continue;
*/
token = strtok_r(string, "\t", &save);
if(token != NULL) {
strcpy(s_key, token);
} else continue;
key.mv_data = s_key;
key.mv_size = strlen(s_key) + 1;
rc = mdb_get(txn, dbi, &key, &data);
if(!rc) {
fprintf(stdout, "%s\t%s\n", s_key, (char*)data.mv_data);
}
cnt_line++;
}
gettimeofday(&tv2, NULL);
fprintf(stderr, "<-end > : t2.sec = %d t2.usec = %d\n",(int)tv2.tv_sec,(int)tv2.tv_usec);
fprintf(stderr, "<+time> : sec = %d usec = %d\n",(int)(tv2.tv_sec-tv1.tv_sec),(int)(tv2.tv_usec-tv1.tv_usec));
rc = mdb_txn_commit(txn);
txn = NULL;
if(rc) {
fprintf(stderr, "%s: txn_commit fail: %s\n", prog, mdb_strerror(rc));
goto env_close;
}
mdb_dbi_close(env, dbi);
txn_abort:
mdb_txn_abort(txn);
env_close:
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}
void DataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
vector<Blob<Dtype>*>* top) {
// Initialize DB
switch (this->layer_param_.data_param().backend()) {
case DataParameter_DB_LEVELDB:
{
leveldb::DB* db_temp;
leveldb::Options options = GetLevelDBOptions();
options.create_if_missing = false;
LOG(INFO) << "Opening leveldb " << this->layer_param_.data_param().source();
leveldb::Status status = leveldb::DB::Open(
options, this->layer_param_.data_param().source(), &db_temp);
CHECK(status.ok()) << "Failed to open leveldb "
<< this->layer_param_.data_param().source() << std::endl
<< status.ToString();
db_.reset(db_temp);
iter_.reset(db_->NewIterator(leveldb::ReadOptions()));
iter_->SeekToFirst();
}
break;
case DataParameter_DB_LMDB:
CHECK_EQ(mdb_env_create(&mdb_env_), MDB_SUCCESS) << "mdb_env_create failed";
CHECK_EQ(mdb_env_set_mapsize(mdb_env_, 1099511627776), MDB_SUCCESS); // 1TB
CHECK_EQ(mdb_env_open(mdb_env_,
this->layer_param_.data_param().source().c_str(),
MDB_RDONLY|MDB_NOTLS, 0664), MDB_SUCCESS) << "mdb_env_open failed";
CHECK_EQ(mdb_txn_begin(mdb_env_, NULL, MDB_RDONLY, &mdb_txn_), MDB_SUCCESS)
<< "mdb_txn_begin failed";
CHECK_EQ(mdb_open(mdb_txn_, NULL, 0, &mdb_dbi_), MDB_SUCCESS)
<< "mdb_open failed";
CHECK_EQ(mdb_cursor_open(mdb_txn_, mdb_dbi_, &mdb_cursor_), MDB_SUCCESS)
<< "mdb_cursor_open failed";
LOG(INFO) << "Opening lmdb " << this->layer_param_.data_param().source();
CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_FIRST),
MDB_SUCCESS) << "mdb_cursor_get failed";
break;
default:
LOG(FATAL) << "Unknown database backend";
}
// Check if we would need to randomly skip a few data points
if (this->layer_param_.data_param().rand_skip()) {
unsigned int skip = caffe_rng_rand() %
this->layer_param_.data_param().rand_skip();
LOG(INFO) << "Skipping first " << skip << " data points.";
while (skip-- > 0) {
switch (this->layer_param_.data_param().backend()) {
case DataParameter_DB_LEVELDB:
iter_->Next();
if (!iter_->Valid()) {
iter_->SeekToFirst();
}
break;
case DataParameter_DB_LMDB:
if (mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT)
!= MDB_SUCCESS) {
CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_,
MDB_FIRST), MDB_SUCCESS);
}
break;
default:
LOG(FATAL) << "Unknown database backend";
}
}
}
// Read a data point, and use it to initialize the top blob.
Datum datum;
switch (this->layer_param_.data_param().backend()) {
case DataParameter_DB_LEVELDB:
datum.ParseFromString(iter_->value().ToString());
break;
case DataParameter_DB_LMDB:
datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size);
break;
default:
LOG(FATAL) << "Unknown database backend";
}
// image
int crop_size = this->layer_param_.transform_param().crop_size();
if (crop_size > 0) {
(*top)[0]->Reshape(this->layer_param_.data_param().batch_size(),
datum.channels(), crop_size, crop_size);
this->prefetch_data_.Reshape(this->layer_param_.data_param().batch_size(),
datum.channels(), crop_size, crop_size);
} else {
(*top)[0]->Reshape(
this->layer_param_.data_param().batch_size(), datum.channels(),
datum.height(), datum.width());
this->prefetch_data_.Reshape(this->layer_param_.data_param().batch_size(),
datum.channels(), datum.height(), datum.width());
}
LOG(INFO) << "output data size: " << (*top)[0]->num() << ","
<< (*top)[0]->channels() << "," << (*top)[0]->height() << ","
<< (*top)[0]->width();
// label
if (this->output_labels_) {
// liu
(*top)[1]->Reshape(this->layer_param_.data_param().batch_size(), 4, 1, 1);
this->prefetch_label_.Reshape(this->layer_param_.data_param().batch_size(),
4, 1, 1);
/*
(*top)[1]->Reshape(this->layer_param_.data_param().batch_size(), 1, 1, 1);
this->prefetch_label_.Reshape(this->layer_param_.data_param().batch_size(),
1, 1, 1);
*/
}
// datum size
this->datum_channels_ = datum.channels();
this->datum_height_ = datum.height();
this->datum_width_ = datum.width();
this->datum_size_ = datum.channels() * datum.height() * datum.width();
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor, *cur2;
MDB_cursor_op op;
int count;
int *values;
char sval[32] = "";
srandom(time(NULL));
count = (random()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = random()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP /*|MDB_NOSYNC*/, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, NULL, 0, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) {
j++;
data.mv_size = sizeof(sval);
data.mv_data = sval;
}
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
key.mv_data = sval;
for (i= count - 1; i > -1; i-= (random()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%03x ", values[i]);
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor last\n");
E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor last/prev\n");
E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
E(mdb_cursor_get(cursor, &key, &data, MDB_PREV));
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
mdb_txn_abort(txn);
printf("Deleting with cursor\n");
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_cursor_open(txn, dbi, &cur2));
for (i=0; i<50; i++) {
if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, MDB_NEXT)))
break;
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
E(mdb_del(txn, dbi, &key, NULL));
}
printf("Restarting cursor in txn\n");
for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, op)))
break;
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
mdb_cursor_close(cur2);
E(mdb_txn_commit(txn));
printf("Restarting cursor outside txn\n");
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
if (RES(MDB_NOTFOUND, mdb_cursor_get(cursor, &key, &data, op)))
break;
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}
int main(int argc,char * argv[])
{
int rc;
MDB_env *env;
MDB_txn *txn;
MDB_dbi db;
rc = mdb_env_create(&env);
if (rc != 0) {
return rc;
}
rc = mdb_env_set_mapsize(env, 107374182400); //100G
if (rc != 0) {
return rc;
}
rc = mdb_env_set_maxdbs(env, 27);
if (rc != 0) {
return rc;
}
rc = mdb_env_open(env, "/home/pieter/Downloads/thundergraph/src/main/native/testdb", MDB_NOSYNC, 0664);
if (rc != 0) {
return rc;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_open(txn, "testdb", MDB_CREATE | MDB_INTEGERKEY, &db);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_set_compare(txn, db, compareTestDbId);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_txn_commit(txn);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc != 0) {
mdb_txn_abort(txn);
return rc;
}
int i= 0;
for (i = 0; i < 1000; i++) {
MDB_cursor *cursor;
rc = mdb_cursor_open(txn, db, &cursor);
if (rc != 0) {
printf("open cursor failure = %i!\n", rc);
goto fail;
}
//Create a core vertex
MDB_val key, data;
TestDbId testDbId;
testDbId.testId = i;
testDbId.coreOrPropertyEnum = 0;
testDbId.labelId = -1;
testDbId.propertykeyId = -1;
testDbId.edgeId = -1LL;
key.mv_size = sizeof(TestDbId);
key.mv_data = &testDbId;
data.mv_size = 50;
char *value = malloc(5);
char v[] = "12345";
memcpy(value, v, 5);
data.mv_data = &v;
mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE);
free(value);
//Create a vertex property
MDB_val propertyKey, propertyData;
TestDbId propertyTestDbId;
propertyTestDbId.testId = i;
propertyTestDbId.coreOrPropertyEnum = 1;
propertyTestDbId.labelId = -1;
propertyTestDbId.propertykeyId = -1;
propertyTestDbId.edgeId = -1LL;
propertyKey.mv_size = sizeof(TestDbId);
propertyKey.mv_data = &propertyTestDbId;
data.mv_size = 50;
char *propertyValue = malloc(5);
char propertyV[] = "12345";
memcpy(propertyValue, propertyV, 5);
data.mv_data = &propertyV;
mdb_cursor_put(cursor, &propertyKey, &propertyData, MDB_NOOVERWRITE);
free(propertyValue);
//Create a vertex in edge
MDB_val inEdgeKey, inEdgeData;
TestDbId inEdgeTestDbId;
inEdgeTestDbId.testId = i;
inEdgeTestDbId.coreOrPropertyEnum = 2;
inEdgeTestDbId.labelId = -1;
inEdgeTestDbId.propertykeyId = -1;
inEdgeTestDbId.edgeId = -1LL;
inEdgeKey.mv_size = sizeof(TestDbId);
inEdgeKey.mv_data = &inEdgeTestDbId;
data.mv_size = 50;
char *inEdgeValue = malloc(5);
char inEdgeV[] = "12345";
memcpy(inEdgeValue, inEdgeV, 5);
data.mv_data = &inEdgeV;
mdb_cursor_put(cursor, &inEdgeKey, &inEdgeData, MDB_NOOVERWRITE);
mdb_cursor_close(cursor);
free(inEdgeValue);
}
MDB_cursor *cursor;
mdb_cursor_close(cursor);
rc = mdb_cursor_open(txn, db, &cursor);
if (rc != 0) {
printf("open cursor failure = %i!\n", rc);
goto fail;
}
MDB_val key, data;
for (i = 0; i < 1000; i++) {
TestDbId testDbId;
testDbId.testId = i;
testDbId.coreOrPropertyEnum = 0;
testDbId.labelId = -1;
testDbId.propertykeyId = -1;
testDbId.edgeId = -1LL;
key.mv_size = sizeof(TestDbId);
key.mv_data = &testDbId;
rc = mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY);
printf("mdb_cursor_get index = %i result = %i\n", i, rc);
MDB_val keyDelete, dataDelete;
TestDbId deleteTestDbId;
deleteTestDbId.testId = i;
deleteTestDbId.coreOrPropertyEnum = 0;
deleteTestDbId.labelId = -1;
deleteTestDbId.propertykeyId = -1;
deleteTestDbId.edgeId = -1LL;
keyDelete.mv_size = sizeof(TestDbId);
keyDelete.mv_data = &deleteTestDbId;
rc = mdb_del(txn, db, &keyDelete, &dataDelete);
printf("mdb_del = %i\n", rc);
MDB_val propertyKeyDelete, propertyDataDelete;
TestDbId propertyDeleteTestDbId;
propertyDeleteTestDbId.testId = i;
propertyDeleteTestDbId.coreOrPropertyEnum = 1;
propertyDeleteTestDbId.labelId = -1;
propertyDeleteTestDbId.propertykeyId = -1;
propertyDeleteTestDbId.edgeId = -1LL;
propertyKeyDelete.mv_size = sizeof(TestDbId);
propertyKeyDelete.mv_data = &propertyDeleteTestDbId;
rc = mdb_del(txn, db, &propertyKeyDelete, &propertyDataDelete);
printf("mdb_del = %i\n", rc);
MDB_val inEdgeKeyDelete, inEdgeDataDelete;
TestDbId inEdgeDeleteTestDbId;
inEdgeDeleteTestDbId.testId = i;
inEdgeDeleteTestDbId.coreOrPropertyEnum = 2;
inEdgeDeleteTestDbId.labelId = -1;
inEdgeDeleteTestDbId.propertykeyId = -1;
inEdgeDeleteTestDbId.edgeId = -1LL;
inEdgeKeyDelete.mv_size = sizeof(TestDbId);
inEdgeKeyDelete.mv_data = &inEdgeDeleteTestDbId;
rc = mdb_del(txn, db, &inEdgeKeyDelete, &inEdgeDataDelete);
printf("mdb_del = %i\n", rc);
}
mdb_cursor_close(cursor);
rc = mdb_cursor_open(txn, db, &cursor);
if (rc != 0) {
printf("open cursor failure = %i!\n", rc);
goto fail;
}
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
TestDbId testDbId = *((TestDbId *) (key.mv_data));
printf("keyId = %llu\n", testDbId.testId);
}
mdb_cursor_close(cursor);
fail:
mdb_close(env, db);
mdb_env_close(env);
printf("closing graph!\n");
return 0;
}
int
main(int argc, char **argv)
{
MdbHandle *mdb;
MdbTableDef *table;
gchar name[256];
gchar *propColName;
void *buf;
int col_num;
int found = 0;
if (argc < 3) {
fprintf(stderr,"Usage: %s <file> <object name> [<prop col>]\n",
argv[0]);
return 1;
}
if (argc < 4)
propColName = "LvProp";
else
propColName = argv[3];
mdb_init();
mdb = mdb_open(argv[1], MDB_NOFLAGS);
if (!mdb) {
mdb_exit();
return 1;
}
table = mdb_read_table_by_name(mdb, "MSysObjects", MDB_ANY);
if (!table) {
mdb_close(mdb);
mdb_exit();
return 1;
}
mdb_read_columns(table);
mdb_rewind_table(table);
mdb_bind_column_by_name(table, "Name", name, NULL);
buf = g_malloc(MDB_BIND_SIZE);
col_num = mdb_bind_column_by_name(table, propColName, buf, NULL);
if (col_num < 1) {
g_free(buf);
mdb_free_tabledef(table);
mdb_close(mdb);
mdb_exit();
printf("Column %s not found in MSysObjects!\n", argv[3]);
return 1;
}
while(mdb_fetch_row(table)) {
if (!strcmp(name, argv[2])) {
found = 1;
break;
}
}
if (found) {
MdbColumn *col = g_ptr_array_index(table->columns, col_num-1);
size_t size;
void *kkd = mdb_ole_read_full(mdb, col, &size);
dump_kkd(mdb, kkd, size);
free(kkd);
}
g_free(buf);
mdb_free_tabledef(table);
mdb_close(mdb);
mdb_exit();
return 0;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32];
char kval[sizeof(int)];
srand(time(NULL));
memset(sval, 0, sizeof(sval));
count = (rand()%384) + 64;
values = (int *)je_malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "/tmp/testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
if (!(i & 0x0f))
sprintf(kval, "%03x", values[i]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(kval, "%03x", values[i & ~0x0f]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
je_free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}
int
main(int argc, char **argv)
{
MdbHandle *mdb;
int print_mdbver = 0;
/* setlocale (LC_ALL, ""); */
bindtextdomain (PACKAGE, LOCALEDIR);
textdomain (PACKAGE);
GOptionEntry entries[] = {
{ "mdbtools", 'M', 0, G_OPTION_ARG_NONE, &print_mdbver, "Show MDBtools version", NULL},
{ NULL },
};
GError *error = NULL;
GOptionContext *opt_context;
opt_context = g_option_context_new("<file> - display MDB file version");
g_option_context_add_main_entries(opt_context, entries, NULL /*i18n*/);
// g_option_context_set_strict_posix(opt_context, TRUE); /* options first, requires glib 2.44 */
if (!g_option_context_parse (opt_context, &argc, &argv, &error))
{
fprintf(stderr, "option parsing failed: %s\n", error->message);
fputs(g_option_context_get_help(opt_context, TRUE, NULL), stderr);
exit (1);
}
if (print_mdbver) {
fprintf(stdout,"%s\n", MDB_FULL_VERSION);
if (argc == 1)
exit(0);
}
if (argc != 2) {
fputs("Wrong number of arguments.\n\n", stderr);
fputs(g_option_context_get_help(opt_context, TRUE, NULL), stderr);
exit(1);
}
if (!(mdb = mdb_open(argv[1], MDB_NOFLAGS))) {
fprintf(stderr,_("Error: unable to open file %s\n"), argv[1]);
exit(1);
}
switch(mdb->f->jet_version) {
case MDB_VER_JET3:
printf("JET3\n");
break;
case MDB_VER_JET4:
printf("JET4\n");
break;
case MDB_VER_ACCDB_2007:
printf("ACE12\n");
break;
case MDB_VER_ACCDB_2010:
printf("ACE14\n");
break;
default:
printf(_("unknown database version\n"));
break;
}
mdb_close(mdb);
g_option_context_free(opt_context);
return 0;
}
