示例#1
0
int db_open()
{
    char *error = NULL;
    char *filename = option_get_str("db_file");
    
    ldb_options = leveldb_options_create();
    ldb_cache = leveldb_cache_create_lru(option_get_int("cache_size"));
    
    leveldb_options_set_create_if_missing(ldb_options, option_get_int("create_db_if_missing"));
    leveldb_options_set_error_if_exists(ldb_options, option_get_int("error_if_db_exists"));
    leveldb_options_set_paranoid_checks(ldb_options, option_get_int("paranoid_checks"));
    leveldb_options_set_write_buffer_size(ldb_options, option_get_int("write_buffer_size"));
    leveldb_options_set_block_size(ldb_options, option_get_int("block_size"));
    leveldb_options_set_cache(ldb_options, ldb_cache);
    leveldb_options_set_max_open_files(ldb_options, option_get_int("leveldb_max_open_files"));
    leveldb_options_set_block_restart_interval(ldb_options, 8);
    leveldb_options_set_compression(ldb_options, option_get_int("compression"));
    
    // leveldb_options_set_env(options, self->_env);
    leveldb_options_set_info_log(ldb_options, NULL);
    
    ldb = leveldb_open(ldb_options, filename, &error);
    if (error) {
        fprintf(stderr, "ERROR opening db:%s\n", error);
        return 0;
    }
    return 1;
}
示例#2
0
void db_leveldb_init()
{
    char *errstr = NULL;
    leveldb_options = leveldb_options_create();
    /* create if missing */
    leveldb_options_set_create_if_missing(leveldb_options, 1);
    /* lru cache */
    leveldb_cache = leveldb_cache_create_lru(conf->leveldb_cache_size);
    leveldb_options_set_cache(leveldb_options, leveldb_cache);
    /* block size */
    leveldb_options_set_block_size(leveldb_options, conf->leveldb_block_size);
    /* write buffer size */
    leveldb_options_set_write_buffer_size(leveldb_options, conf->leveldb_write_buffer_size);
    /* filter policy */
    leveldb_filterpolicy = leveldb_filterpolicy_create_bloom(10);
    leveldb_options_set_filter_policy(leveldb_options, leveldb_filterpolicy);
    /* open db */
    leveldb_db = leveldb_open(leveldb_options, conf->db, &errstr);

    if (errstr) {
        terrx(1, "unable to open db at %s: %s", conf->db, errstr);
    }

    leveldb_roptions = leveldb_readoptions_create();
    leveldb_woptions = leveldb_writeoptions_create();
}
示例#3
0
文件: ldb.c 项目: hellocjfeii/TSDB
struct _leveldb_stuff *ldb_initialize(char *path)
{

    struct _leveldb_stuff *ldbs = NULL;
    leveldb_cache_t *cache;
    leveldb_filterpolicy_t *policy;
    char* err = NULL;

    ldbs = (struct _leveldb_stuff *) malloc(sizeof(struct _leveldb_stuff));
    memset(ldbs, 0, sizeof(struct _leveldb_stuff));

    ldbs->options = leveldb_options_create();
    snprintf(ldbs->dbname, sizeof(ldbs->dbname), "%s%s%s", LDB_WORK_PATH, "/",
            path);

    cache = leveldb_cache_create_lru(LDB_CACHE_LRU_SIZE);
    policy = leveldb_filterpolicy_create_bloom(LDB_BLOOM_KEY_SIZE);

    leveldb_options_set_filter_policy(ldbs->options, policy);
    leveldb_options_set_cache(ldbs->options, cache);
    leveldb_options_set_block_size(ldbs->options, LDB_BLOCK_SIZE);
    leveldb_options_set_write_buffer_size(ldbs->options, LDB_WRITE_BUFFER_SIZE);
#if defined(OPEN_COMPRESSION)
    leveldb_options_set_compression(ldbs->options, leveldb_snappy_compression);
#else
    leveldb_options_set_compression(ldbs->options, leveldb_no_compression);
#endif
    /* Read options */
    ldbs->roptions = leveldb_readoptions_create();
    leveldb_readoptions_set_verify_checksums(ldbs->roptions, 1);
    leveldb_readoptions_set_fill_cache(ldbs->roptions, 1);/* set 1 is faster */

    /* W  options */
    ldbs->woptions = leveldb_writeoptions_create();
#ifdef SYNC_PUT
    leveldb_writeoptions_set_sync(ldbs->woptions, 1);
#else
    leveldb_writeoptions_set_sync(ldbs->woptions, 0);
#endif

    /* Batch write */
    ldbs->wbatch = leveldb_writebatch_create();

    leveldb_options_set_create_if_missing(ldbs->options, 1);
    ldbs->db = leveldb_open(ldbs->options, ldbs->dbname, &err);

    if (err) {
        fprintf(stderr, "%s", err);
        leveldb_free(err);
        err = NULL;
        free(ldbs);
        return NULL;
    } else {
        return ldbs;
    }

}
示例#4
0
/*
 * Open the specified database, if no exists, then create it.
 * Args:
 *      1. name: db name
 *      2. block_size: block size
 *      3. wb_size: write buffer size
 *      4. lru_size: lru cache size
 *      5. bloom_size: bloom key size
 * Return:
 *      _leveldb_stuff: leveldb handler.
 */
struct _leveldb_stuff *ldb_initialize(const char *name, size_t block_size, size_t wb_size, size_t lru_size, short bloom_size)
{

    struct _leveldb_stuff *ldbs = NULL;
    leveldb_cache_t *cache;
    leveldb_filterpolicy_t *policy;
    char* err = NULL;

    ldbs = malloc(sizeof(struct _leveldb_stuff));
    memset(ldbs, 0, sizeof(struct _leveldb_stuff));

    ldbs->options = leveldb_options_create();

    snprintf(ldbs->dbname, sizeof(ldbs->dbname), "%s", name);

    cache = leveldb_cache_create_lru(lru_size);
    policy = leveldb_filterpolicy_create_bloom(bloom_size);

    leveldb_options_set_filter_policy(ldbs->options, policy);
    leveldb_options_set_cache(ldbs->options, cache);
    leveldb_options_set_block_size(ldbs->options, block_size);
    leveldb_options_set_write_buffer_size(ldbs->options, wb_size);
#if defined(OPEN_COMPRESSION)
    leveldb_options_set_compression(ldbs->options, leveldb_snappy_compression);
#else
    leveldb_options_set_compression(ldbs->options, leveldb_no_compression);
#endif
    /* R */
    ldbs->roptions = leveldb_readoptions_create();
    leveldb_readoptions_set_verify_checksums(ldbs->roptions, 1);
    leveldb_readoptions_set_fill_cache(ldbs->roptions, 1);/* set 1 is faster */

    /* W */
    ldbs->woptions = leveldb_writeoptions_create();
#ifdef SYNC_PUT
    leveldb_writeoptions_set_sync(ldbs->woptions, 1);
#else
    leveldb_writeoptions_set_sync(ldbs->woptions, 0);
#endif

    /* B */
    ldbs->wbatch = leveldb_writebatch_create();

    leveldb_options_set_create_if_missing(ldbs->options, 1);
    ldbs->db = leveldb_open(ldbs->options, ldbs->dbname, &err);

    if (err) {
        fprintf(stderr, "%s", err);
        leveldb_free(err);
        err = NULL;
        free(ldbs);
        return NULL ;
    }

    return ldbs;
}
示例#5
0
void ds_init() {
    char *err = NULL;

    server.ds_cache = leveldb_cache_create_lru(server.ds_lru_cache * 1048576);
    server.ds_options = leveldb_options_create();

    server.policy = leveldb_filterpolicy_create_bloom(10);


    //leveldb_options_set_comparator(server.ds_options, cmp);
    leveldb_options_set_filter_policy(server.ds_options, server.policy);
    leveldb_options_set_create_if_missing(server.ds_options, server.ds_create_if_missing);
    leveldb_options_set_error_if_exists(server.ds_options, server.ds_error_if_exists);
    leveldb_options_set_cache(server.ds_options, server.ds_cache);
    leveldb_options_set_info_log(server.ds_options, NULL);
    leveldb_options_set_write_buffer_size(server.ds_options, server.ds_write_buffer_size * 1048576);
    leveldb_options_set_paranoid_checks(server.ds_options, server.ds_paranoid_checks);
    leveldb_options_set_max_open_files(server.ds_options, server.ds_max_open_files);
    leveldb_options_set_block_size(server.ds_options, server.ds_block_size * 1024);
    leveldb_options_set_block_restart_interval(server.ds_options, server.ds_block_restart_interval);
    leveldb_options_set_compression(server.ds_options, leveldb_snappy_compression);

    server.ds_db = leveldb_open(server.ds_options, server.ds_path, &err);
    if (err != NULL) {
        fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, err);
        leveldb_free(err);
        exit(1);
    }

    server.woptions = leveldb_writeoptions_create();
    server.roptions = leveldb_readoptions_create();
    leveldb_readoptions_set_verify_checksums(server.roptions, 0);
    leveldb_readoptions_set_fill_cache(server.roptions, 1);

    leveldb_writeoptions_set_sync(server.woptions, 0);
}
示例#6
0
文件: c_test.c 项目: Aleda/tera
int main(int argc, char** argv) {
  leveldb_t* db;
  leveldb_comparator_t* cmp;
  leveldb_cache_t* cache;
  leveldb_env_t* env;
  leveldb_options_t* options;
  leveldb_readoptions_t* roptions;
  leveldb_writeoptions_t* woptions;
  char* err = NULL;
  int run = -1;

  CheckCondition(leveldb_major_version() >= 1);
  CheckCondition(leveldb_minor_version() >= 1);

  snprintf(dbname, sizeof(dbname),
           "%s/leveldb_c_test-%di/meta/0",
           GetTempDir(),
           ((int) geteuid()));

  StartPhase("create_objects");
  cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
  env = leveldb_create_default_env();
  cache = leveldb_cache_create_lru(100000);

  options = leveldb_options_create();
  leveldb_options_set_comparator(options, cmp);
  leveldb_options_set_error_if_exists(options, 1);
  leveldb_options_set_cache(options, cache);
  leveldb_options_set_env(options, env);
  leveldb_options_set_info_log(options, NULL);
  leveldb_options_set_write_buffer_size(options, 100000);
  leveldb_options_set_paranoid_checks(options, 1);
  leveldb_options_set_max_open_files(options, 10);
  leveldb_options_set_block_size(options, 1024);
  leveldb_options_set_block_restart_interval(options, 8);
  leveldb_options_set_compression(options, leveldb_no_compression);

  roptions = leveldb_readoptions_create();
  leveldb_readoptions_set_verify_checksums(roptions, 1);
  leveldb_readoptions_set_fill_cache(roptions, 0);

  woptions = leveldb_writeoptions_create();
  leveldb_writeoptions_set_sync(woptions, 1);

  StartPhase("destroy");
  leveldb_destroy_db(options, dbname, &err);
  Free(&err);

  StartPhase("open");
  db = leveldb_open(options, dbname, &err);
  CheckNoError(err);
  CheckGet(db, roptions, "foo", NULL);

  StartPhase("put");
  leveldb_put(db, woptions, "foo", 3, "hello", 5, &err);
  CheckNoError(err);
  CheckGet(db, roptions, "foo", "hello");

  StartPhase("compactall");
  leveldb_compact_range(db, NULL, 0, NULL, 0);
  CheckGet(db, roptions, "foo", "hello");

  StartPhase("compactrange");
  leveldb_compact_range(db, "a", 1, "z", 1);
  CheckGet(db, roptions, "foo", "hello");

  StartPhase("writebatch");
  {
    leveldb_writebatch_t* wb = leveldb_writebatch_create();
    leveldb_writebatch_put(wb, "foo", 3, "a", 1);
    leveldb_writebatch_clear(wb);
    leveldb_writebatch_put(wb, "bar", 3, "b", 1);
    leveldb_writebatch_put(wb, "box", 3, "c", 1);
    leveldb_writebatch_delete(wb, "bar", 3);
    leveldb_write(db, woptions, wb, &err);
    CheckNoError(err);
    CheckGet(db, roptions, "foo", "hello");
    CheckGet(db, roptions, "bar", NULL);
    CheckGet(db, roptions, "box", "c");
    int pos = 0;
    leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
    CheckCondition(pos == 3);
    leveldb_writebatch_destroy(wb);
  }

  StartPhase("iter");
  {
    leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
    CheckCondition(!leveldb_iter_valid(iter));
    leveldb_iter_seek_to_first(iter);
    CheckCondition(leveldb_iter_valid(iter));
    CheckIter(iter, "box", "c");
    leveldb_iter_next(iter);
    CheckIter(iter, "foo", "hello");
    leveldb_iter_prev(iter);
    CheckIter(iter, "box", "c");
    leveldb_iter_prev(iter);
    CheckCondition(!leveldb_iter_valid(iter));
    leveldb_iter_seek_to_last(iter);
    CheckIter(iter, "foo", "hello");
    leveldb_iter_seek(iter, "b", 1);
    CheckIter(iter, "box", "c");
    leveldb_iter_get_error(iter, &err);
    CheckNoError(err);
    leveldb_iter_destroy(iter);
  }

  StartPhase("approximate_sizes");
  {
    int i;
    int n = 20000;
    char keybuf[100];
    char valbuf[100];
    uint64_t sizes[2];
    const char* start[2] = { "a", "k00000000000000010000" };
    size_t start_len[2] = { 1, 21 };
    const char* limit[2] = { "k00000000000000010000", "z" };
    size_t limit_len[2] = { 21, 1 };
    leveldb_writeoptions_set_sync(woptions, 0);
    for (i = 0; i < n; i++) {
      snprintf(keybuf, sizeof(keybuf), "k%020d", i);
      snprintf(valbuf, sizeof(valbuf), "v%020d", i);
      leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
                  &err);
      CheckNoError(err);
    }
    leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
    CheckCondition(sizes[0] > 0);
    CheckCondition(sizes[1] > 0);
  }

  StartPhase("property");
  {
    char* prop = leveldb_property_value(db, "nosuchprop");
/*    CheckCondition(prop == NULL);*/
    prop = leveldb_property_value(db, "leveldb.stats");
    CheckCondition(prop != NULL);
    Free(&prop);
  }

  StartPhase("snapshot");
  {
    const leveldb_snapshot_t* snap;
    snap = leveldb_create_snapshot(db);
    leveldb_delete(db, woptions, "foo", 3, &err);
    CheckNoError(err);
    leveldb_readoptions_set_snapshot(roptions, snap);
    CheckGet(db, roptions, "foo", "hello");
    leveldb_readoptions_set_snapshot(roptions, NULL);
    CheckGet(db, roptions, "foo", NULL);
    leveldb_release_snapshot(db, snap);
  }

  StartPhase("repair");
  {
    leveldb_close(db);
    leveldb_options_set_error_if_exists(options, 0);
    leveldb_repair_db(options, dbname, &err);
    CheckNoError(err);
    db = leveldb_open(options, dbname, &err);
    CheckNoError(err);
    CheckGet(db, roptions, "foo", NULL);
    CheckGet(db, roptions, "bar", NULL);
    CheckGet(db, roptions, "box", "c");
    leveldb_options_set_error_if_exists(options, 1);
  }

  StartPhase("filter");
  for (run = 0; run < 2; run++) {
    // First run uses custom filter, second run uses bloom filter
    CheckNoError(err);
    leveldb_filterpolicy_t* policy;
    if (run == 0) {
      policy = leveldb_filterpolicy_create(
          NULL, FilterDestroy, FilterCreate, FilterKeyMatch, FilterName);
    } else {
      policy = leveldb_filterpolicy_create_bloom(10);
    }

    // Create new database
    leveldb_close(db);
    leveldb_destroy_db(options, dbname, &err);
    leveldb_options_set_filter_policy(options, policy);
    db = leveldb_open(options, dbname, &err);
    CheckNoError(err);
    leveldb_put(db, woptions, "foo", 3, "foovalue", 8, &err);
    CheckNoError(err);
    leveldb_put(db, woptions, "bar", 3, "barvalue", 8, &err);
    CheckNoError(err);
    leveldb_compact_range(db, NULL, 0, NULL, 0);

    fake_filter_result = 1;
    CheckGet(db, roptions, "foo", "foovalue");
    CheckGet(db, roptions, "bar", "barvalue");
    if (phase == 0) {
      // Must not find value when custom filter returns false
      fake_filter_result = 0;
      CheckGet(db, roptions, "foo", NULL);
      CheckGet(db, roptions, "bar", NULL);
      fake_filter_result = 1;

      CheckGet(db, roptions, "foo", "foovalue");
      CheckGet(db, roptions, "bar", "barvalue");
    }
    leveldb_options_set_filter_policy(options, NULL);
    leveldb_filterpolicy_destroy(policy);
  }

  StartPhase("cleanup");
  leveldb_close(db);
  leveldb_options_destroy(options);
  leveldb_readoptions_destroy(roptions);
  leveldb_writeoptions_destroy(woptions);
  leveldb_cache_destroy(cache);
  leveldb_comparator_destroy(cmp);
  leveldb_env_destroy(env);

  fprintf(stderr, "PASS\n");
  return 0;
}
示例#7
0
int main(int argc, char** argv) {
  leveldb_t* db;
  leveldb_comparator_t* cmp;
  leveldb_cache_t* cache;
  leveldb_env_t* env;
  leveldb_options_t* options;
  leveldb_readoptions_t* roptions;
  leveldb_writeoptions_t* woptions;
  char* err = NULL;

#if defined(LEVELDB_PLATFORM_WINDOWS)
  snprintf(dbname, sizeof(dbname), "tmp\\leveldb_c_test");
#else
  snprintf(dbname, sizeof(dbname), "/tmp/leveldb_c_test-%d",
           ((int) geteuid()));
#endif

  StartPhase("create_objects");
  cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
  env = leveldb_create_default_env();
  cache = leveldb_cache_create_lru(100000);

  options = leveldb_options_create();
  leveldb_options_set_comparator(options, cmp);
  leveldb_options_set_error_if_exists(options, 1);
  leveldb_options_set_cache(options, cache);
  leveldb_options_set_env(options, env);
  leveldb_options_set_info_log(options, NULL);
  leveldb_options_set_write_buffer_size(options, 100000);
  leveldb_options_set_paranoid_checks(options, 1);
  leveldb_options_set_max_open_files(options, 10);
  leveldb_options_set_block_size(options, 1024);
  leveldb_options_set_block_restart_interval(options, 8);
  leveldb_options_set_compression(options, leveldb_no_compression);

  roptions = leveldb_readoptions_create();
  leveldb_readoptions_set_verify_checksums(roptions, 1);
  leveldb_readoptions_set_fill_cache(roptions, 0);

  woptions = leveldb_writeoptions_create();
  leveldb_writeoptions_set_sync(woptions, 1);

  StartPhase("destroy");
  leveldb_destroy_db(options, dbname, &err);
  leveldb_free(&err);

  StartPhase("open_error");
  db = leveldb_open(options, dbname, &err);
  CheckCondition(err != NULL);
  leveldb_free(&err);

  StartPhase("open");
  leveldb_options_set_create_if_missing(options, 1);
  db = leveldb_open(options, dbname, &err);
  CheckNoError(err);
  CheckGet(db, roptions, "foo", NULL);

  StartPhase("put");
  leveldb_put(db, woptions, "foo", 3, "hello", 5, &err);
  CheckNoError(err);
  CheckGet(db, roptions, "foo", "hello");

  StartPhase("writebatch");
  {
    int pos = 0;
    leveldb_writebatch_t* wb = leveldb_writebatch_create();
    leveldb_writebatch_put(wb, "foo", 3, "a", 1);
    leveldb_writebatch_clear(wb);
    leveldb_writebatch_put(wb, "bar", 3, "b", 1);
    leveldb_writebatch_put(wb, "box", 3, "c", 1);
    leveldb_writebatch_delete(wb, "bar", 3);
    leveldb_write(db, woptions, wb, &err);
    CheckNoError(err);
    CheckGet(db, roptions, "foo", "hello");
    CheckGet(db, roptions, "bar", NULL);
    CheckGet(db, roptions, "box", "c");
    leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
    CheckCondition(pos == 3);
    leveldb_writebatch_destroy(wb);
  }

  StartPhase("iter");
  {
    leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
    CheckCondition(!leveldb_iter_valid(iter));
    leveldb_iter_seek_to_first(iter);
    CheckCondition(leveldb_iter_valid(iter));
    CheckIter(iter, "box", "c");
    leveldb_iter_next(iter);
    CheckIter(iter, "foo", "hello");
    leveldb_iter_prev(iter);
    CheckIter(iter, "box", "c");
    leveldb_iter_prev(iter);
    CheckCondition(!leveldb_iter_valid(iter));
    leveldb_iter_seek_to_last(iter);
    CheckIter(iter, "foo", "hello");
    leveldb_iter_seek(iter, "b", 1);
    CheckIter(iter, "box", "c");
    leveldb_iter_get_error(iter, &err);
    CheckNoError(err);
    leveldb_iter_destroy(iter);
  }

  StartPhase("approximate_sizes");
  {
    int i;
    int n = 20000;
    char keybuf[100];
    char valbuf[100];
    uint64_t sizes[2];
    const char* start[2] = { "a", "k00000000000000010000" };
    size_t start_len[2] = { 1, 21 };
    const char* limit[2] = { "k00000000000000010000", "z" };
    size_t limit_len[2] = { 21, 1 };
    leveldb_writeoptions_set_sync(woptions, 0);
    for (i = 0; i < n; i++) {
      snprintf(keybuf, sizeof(keybuf), "k%020d", i);
      snprintf(valbuf, sizeof(valbuf), "v%020d", i);
      leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
                  &err);
      CheckNoError(err);
    }
    leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
    CheckCondition(sizes[0] > 0);
    CheckCondition(sizes[1] > 0);
  }

  StartPhase("property");
  {
    char* prop = leveldb_property_value(db, "nosuchprop");
    CheckCondition(prop == NULL);
    prop = leveldb_property_value(db, "leveldb.stats");
    CheckCondition(prop != NULL);
    leveldb_free(&prop);
  }

  StartPhase("snapshot");
  {
    const leveldb_snapshot_t* snap;
    snap = leveldb_create_snapshot(db);
    leveldb_delete(db, woptions, "foo", 3, &err);
    CheckNoError(err);
    leveldb_readoptions_set_snapshot(roptions, snap);
    CheckGet(db, roptions, "foo", "hello");
    leveldb_readoptions_set_snapshot(roptions, NULL);
    CheckGet(db, roptions, "foo", NULL);
    leveldb_release_snapshot(db, snap);
  }

  StartPhase("repair");
  {
    leveldb_close(db);
    leveldb_options_set_create_if_missing(options, 0);
    leveldb_options_set_error_if_exists(options, 0);
    leveldb_repair_db(options, dbname, &err);
    CheckNoError(err);
    db = leveldb_open(options, dbname, &err);
    CheckNoError(err);
    CheckGet(db, roptions, "foo", NULL);
    CheckGet(db, roptions, "bar", NULL);
    CheckGet(db, roptions, "box", "c");
  }

  StartPhase("cleanup");
  leveldb_close(db);
  leveldb_options_destroy(options);
  leveldb_readoptions_destroy(roptions);
  leveldb_writeoptions_destroy(woptions);
  leveldb_cache_destroy(cache);
  leveldb_comparator_destroy(cmp);
  leveldb_env_destroy(env);

  fprintf(stderr, "PASS\n");
  return 0;
}
示例#8
0
文件: db.c 项目: WuGaohang/ktsdb
void db_options_set_cache(db_options_t *o, size_t capacity) {
	leveldb_cache_t *c = leveldb_cache_create_lru(capacity);

	leveldb_options_set_cache(o, c);
}
示例#9
0
int
main(int argc, char *argv[]) {
	if (argc != 5) {
		fprintf(stderr, "Usage: ./leveldb_index [leveldb_directory] [sample_input] [index_output] [index_time_file]\n");
		return -1;
	}
	if ((file = fopen(argv[2], "r")) == NULL) {
		fprintf(stderr, "Failed to open sample file \"%s\" for reading\n", argv[1]);
		return -1;
	}
	for (i = 0; fscanf(file, "%"PRId64"\t", &cnt) != EOF; ++i) {
		fgets(buffer, BUFFER_SIZE, file);
		for (len = strlen(buffer) - 1; len >= 0 && (buffer[len] == '\r' || buffer[len] == '\n'); buffer[len--] = 0);
		++len;
		strcpy(request[i].key, buffer);
	}
	fclose(file);

	if ((file = fopen(argv[4], "w")) == NULL) {
		fprintf(stderr,  "Failed to open time information file \"%s\" for reading\n", argv[4]);
		return -1;
	}
	fprintf(stdout, "Start indexing\n");
	srand((unsigned)time(NULL));
	options = leveldb_options_create();
	leveldb_options_set_create_if_missing(options, 1);
	// default: write_buffer_size: 4MB, block_cache: 8MB, block_size ~ 4KB
	leveldb_options_set_write_buffer_size(options, BLK_SIZE * 1024);
	leveldb_options_set_cache(options, cache = leveldb_cache_create_lru(BLK_SIZE * 2048));
	leveldb_options_set_block_size(options, BLK_SIZE);
	leveldb_options_set_compression(options, 0);
	db = leveldb_open(options, argv[1], &err);
	if (err != NULL) {
		fprintf(stderr, "Failed to open leveldb.\n");
		return -1;
	}
	leveldb_free(err);
	roptions = leveldb_readoptions_create();
	gettimeofday(&start, NULL);
	for (i = seekn = tot = 0; i < DISK_INDEX_SIZE; ++i) {
		val = leveldb_get(db, roptions, request[i].key, strlen(request[i].key), &read_len, &err);
		if (err != NULL) {
			fprintf(stderr, "Failed to read from leveldb.\n");
			return -1;
		}
		leveldb_free(err);
		if (read_len == 0)
			request[i].value = 0;
		else {
			val[read_len] = 0;
			sscanf(val, "%"PRId64"", &request[i].value);
		}
		leveldb_free(val);
		if ((i + 1) % 1000 == 0) {
			gettimeofday(&end, NULL);
			tot += (int64_t)(end.tv_sec - start.tv_sec) * 1000000 + end.tv_usec - start.tv_usec;
			fprintf(stdout, "\rIndexed %"PRId64" items in %"PRId64"us", i + 1, tot);
			fflush(stdout);
			fprintf(file, "Indexed %"PRId64" items in %"PRId64"us\n", i + 1, tot);
			fflush(file);
			gettimeofday(&start, NULL);
		}
	}
	fprintf(stdout, "\rIndexed %"PRId64" items in %"PRId64"us\n", i, tot);
	fprintf(file, "Indexed %"PRId64" items in %"PRId64"us\n", i, tot);
	fflush(file);
	fclose(file);
	leveldb_cache_destroy(cache);
	leveldb_readoptions_destroy(roptions);
	leveldb_options_destroy(options);
	leveldb_close(db);
	fprintf(stdout, "Used %"PRId64" us and seeked %"PRId64" times to index %"PRId64" keys\n", tot, seekn, i);
	if ((file = fopen(argv[3], "w")) == NULL) {
		fprintf(stdout, "Failed to open file \"%s\" for storing indexing results\n", argv[3]);
		return -1;
	}
	for (i = 0; i < DISK_INDEX_SIZE; ++i)
		fprintf(file, "%"PRId64"\t%s\n", request[i].value, request[i].key);
	fflush(file);
	fclose(file);
	return 0;
}
示例#10
0
//TODO: add "bool create_if_missing" to this API!
int leveldb_client_start(leveldb **db, const char *name,
		size_t write_buffer_size, bool use_compression, bool sync_writes)
{
	int namelen;
	char *db_name;
	char *err = NULL;
	leveldb_t* db_t;
	leveldb_comparator_t *cmp;
	leveldb_cache_t *cache;
	leveldb_env_t *env;
	leveldb_options_t *options;
	leveldb_readoptions_t *roptions;
	leveldb_writeoptions_t *woptions;
	
	*db = (leveldb *)malloc(sizeof(leveldb));
	if (*db == NULL) {
		kp_error("malloc(leveldb) failed!\n");
		return -1;
	}

	/* Check arguments: */
	if (strlen(name) < 1) {
		kp_error("name has no length!\n");
		return -1;
	}
	if (write_buffer_size == 0) {
		write_buffer_size = WRITE_BUFFER_DEFAULT;
	}
	if (write_buffer_size < WRITE_BUFFER_MIN) {
		kp_error("write_buffer_size=%zu too small; min=%u\n",
				write_buffer_size, WRITE_BUFFER_MIN);
		return -1;
	}
	kp_debug("got args: name=%s, write_buffer_size=%zu, use_compression=%s, "
			"sync_writes=%s\n",
			name, write_buffer_size, use_compression ? "true" : "false",
			sync_writes ? "true" : "false");

	/* Format of db name: LEVELDB_DIR + / + name */
	namelen = strlen(name);
	db_name = malloc(namelen + 1);
	if (!db_name) {
		kp_error("malloc(db_name) failed!\n");
		return -1;
	}
	strncpy(db_name, name, namelen+1);          //copy null-zero
	(*db)->name = db_name;
	kp_debug("constructed db->name=[%s]\n", (*db)->name);

	/* Set up the database's comparator, environment, cache, etc. According
	 * to leveldb/include/leveldb/c.h, all functions that can raise an error
	 * must be passed a "char **errptr" (set to NULL!) as the last argument;
	 * I guess functions that don't take this pointer can't return errors.
	 */
	cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_compare,
			cmp_name);  //first arg is "state"
	env = leveldb_create_default_env();
	cache = leveldb_cache_create_lru(CACHE_CAPACITY);
	//	cache = NULL;
	(*db)->cmp = cmp;
	(*db)->env = env;
	(*db)->cache = cache;

	/* Set up the database's various options. Many of these will affect the
	 * database's performance! (see leveldb/include/leveldb/options.h).
	 */
	options = leveldb_options_create();
	leveldb_options_set_create_if_missing(options, 1);
	leveldb_options_set_comparator(options, cmp);
	leveldb_options_set_error_if_exists(options, 1);  //raise error if db already exists
	leveldb_options_set_cache(options, cache);
	//	leveldb_options_set_cache(options, NULL); //disable cache??
	leveldb_options_set_env(options, env);
	leveldb_options_set_info_log(options, NULL);  //NULL: write info to file in db's dir
	leveldb_options_set_write_buffer_size(options, write_buffer_size);
	  /* Amount of data to build up in memory (backed by an unsorted log
	   * on disk) before converting to a sorted on-disk file.
	   * Larger values increase performance, especially during bulk loads.
	   * Up to two write buffers may be held in memory at the same time,
	   * so you may wish to adjust this parameter to control memory usage.
	   * Also, a larger write buffer will result in a longer recovery time
	   * the next time the database is opened.
	   * Default: 4MB  (test file uses 100000 bytes)
	   */
	leveldb_options_set_paranoid_checks(options, 0);  //default false; test file uses true
	leveldb_options_set_max_open_files(options, MAX_OPEN_FILES);
	leveldb_options_set_block_size(options, BLOCK_SIZE);
	leveldb_options_set_block_restart_interval(options, BLOCK_RESTART_INTERVAL);
	leveldb_options_set_compression(options,
			use_compression ? leveldb_snappy_compression : leveldb_no_compression);
	(*db)->options = options;

	roptions = leveldb_readoptions_create();
	leveldb_readoptions_set_verify_checksums(roptions, 0);
	  /* If true, all data read from underlying storage will be
	   * verified against corresponding checksums.
	   * Default false; test file uses true.
	   */
	leveldb_readoptions_set_fill_cache(roptions, true);
	  /* Should the data read for this iteration be cached in memory?
	   * Callers may wish to set this field to false for bulk scans.
	   * Default true; test file uses false.
	   */
	(*db)->roptions = roptions;

	woptions = leveldb_writeoptions_create();
	leveldb_writeoptions_set_sync(woptions, sync_writes ? 1 : 0);
	  /* If true, the write will be flushed from the operating system
	   * buffer cache (by calling WritableFile::Sync()) before the write
	   * is considered complete.  If this flag is true, writes will be
	   * slower.
	   * If this flag is false, and the machine crashes, some recent
	   * writes may be lost.  Note that if it is just the process that
	   * crashes (i.e., the machine does not reboot), no writes will be
	   * lost even if sync==false.
	   * In other words, a DB write with sync==false has similar
	   * crash semantics as the "write()" system call.  A DB write
	   * with sync==true has similar crash semantics to a "write()"
	   * system call followed by "fsync()".
	   */
	(*db)->woptions = woptions;

	kp_debug("destroying previous copy of database, if it exists\n");
	leveldb_destroy_db((*db)->options, (*db)->name, &err);
	free_err(&err);

	kp_debug("opening/creating database [%s]\n", (*db)->name);
	db_t = leveldb_open((*db)->options, (*db)->name, &err);
	if (err) {
		kp_error("opening db returned error: %s\n", err);
		return -1;
	}
	free_err(&err);
	(*db)->db = db_t;

	kp_debug("successfully started leveldb [%s]\n", (*db)->name);
	return 0;
}
示例#11
0
int mdhim_leveldb_open(void **dbh, void **dbs, char *path, int flags, 
		       struct mdhim_store_opts_t *mstore_opts) {
	leveldb_t *db;
	leveldb_options_t *options;
	leveldb_options_t *stat_options;
	char *err = NULL;
	leveldb_comparator_t* cmp = NULL;
	char stats_path[PATH_MAX];
	leveldb_filterpolicy_t *main_filter;
	leveldb_filterpolicy_t *stats_filter;
	leveldb_cache_t *main_cache;
	leveldb_cache_t *stats_cache;
	leveldb_env_t *main_env;
	leveldb_env_t *stats_env;

	//Create the options for the main database
	options = leveldb_options_create();
	leveldb_options_set_create_if_missing(options, 1);
	//leveldb_options_set_compression(options, 0);
	main_filter = leveldb_filterpolicy_create_bloom(256);
	main_cache = leveldb_cache_create_lru(5242880);
	main_env = leveldb_create_default_env();
	leveldb_options_set_cache(options, main_cache);
	leveldb_options_set_filter_policy(options, main_filter);
	leveldb_options_set_max_open_files(options, 10000);
	leveldb_options_set_write_buffer_size(options, 5242880);
	leveldb_options_set_env(options, main_env);

	//Create the options for the stat database
	stat_options = leveldb_options_create();
	leveldb_options_set_create_if_missing(stat_options, 1);
	//leveldb_options_set_compression(stat_options, 0);
	stats_filter = leveldb_filterpolicy_create_bloom(256);       
	stats_cache = leveldb_cache_create_lru(1024);
	stats_env = leveldb_create_default_env();
	leveldb_options_set_cache(stat_options, stats_cache);
	leveldb_options_set_filter_policy(stat_options, stats_filter);
	leveldb_options_set_write_buffer_size(stat_options, 1024);
	leveldb_options_set_env(stat_options, stats_env);

	switch(mstore_opts->key_type) {
	case MDHIM_INT_KEY:
		cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_int_compare, cmp_name);
		break;
	case MDHIM_LONG_INT_KEY:
		cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_lint_compare, cmp_name);
		break;
	case MDHIM_FLOAT_KEY:
		cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_float_compare, cmp_name);
		break;
	case MDHIM_DOUBLE_KEY:
		cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_double_compare, cmp_name);
		break;
	case MDHIM_STRING_KEY:
		cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_string_compare, cmp_name);
		break;
	default:
		cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_byte_compare, cmp_name);
		break;
	}
	
	leveldb_options_set_comparator(options, cmp);
	//Check to see if the given path + "_stat" and the null char will be more than the max
	if (strlen(path) + 6 > PATH_MAX) {
		mlog(MDHIM_SERVER_CRIT, "Error opening leveldb database - path provided is too long");
		return MDHIM_DB_ERROR;
	}
	sprintf(stats_path, "%s_stats", path);
	//Open the main database
	db = leveldb_open(options, path, &err);
	//Set the output handle
	*((leveldb_t **) dbh) = db;
	if (err != NULL) {
		mlog(MDHIM_SERVER_CRIT, "Error opening leveldb database");
		return MDHIM_DB_ERROR;
	}

	//Open the stats database
	cmp = leveldb_comparator_create(NULL, cmp_destroy, cmp_int_compare, cmp_name);
	leveldb_options_set_comparator(stat_options, cmp);
	db = leveldb_open(stat_options, stats_path, &err);
	*((leveldb_t **) dbs) = db;
	if (err != NULL) {
		mlog(MDHIM_SERVER_CRIT, "Error opening leveldb database");
		return MDHIM_DB_ERROR;
	}

	//Set the output comparator
	mstore_opts->db_ptr1 = cmp;
	//Set the generic pointers to hold the options for the mdhim db
	mstore_opts->db_ptr2 = options;
	mstore_opts->db_ptr3 = leveldb_readoptions_create();
	mstore_opts->db_ptr4 = leveldb_writeoptions_create();
	//Set the generic pointers to hold the options for the mdhim stats db
	mstore_opts->db_ptr5 = stat_options;
	mstore_opts->db_ptr6 = leveldb_readoptions_create();
	mstore_opts->db_ptr7 = leveldb_writeoptions_create();

	//Set the rest of the options that aren't used until close
	mstore_opts->db_ptr8 = main_filter;
	mstore_opts->db_ptr9 = stats_filter;
	mstore_opts->db_ptr10 = main_cache;
	mstore_opts->db_ptr11 = stats_cache;
	mstore_opts->db_ptr12 = main_env;
	mstore_opts->db_ptr13 = stats_env;

	return MDHIM_SUCCESS;
}