int main(int argc, const char **argv)
{
rados_t cluster;
rados_ioctx_t io_ctx;
rbd_image_t image;
srand(time(0));
assert(rados_create(&cluster, NULL) == 0);
assert(rados_conf_parse_argv(cluster, argc, argv) == 0);
assert(rados_conf_read_file(cluster, NULL) == 0);
assert(rados_connect(cluster) == 0);
if (rados_pool_lookup(cluster, TEST_POOL) != -ENOENT) {
int r = rados_pool_delete(cluster, TEST_POOL);
printf("rados_pool_delete returned %d\n", r);
}
int r = rados_pool_create(cluster, TEST_POOL);
printf("rados_pool_create returned %d\n", r);
assert(rados_ioctx_create(cluster, TEST_POOL, &io_ctx) == 0);
test_ls(io_ctx, 0);
test_create_and_stat(io_ctx, TEST_IMAGE, MB_BYTES(1));
assert(rbd_open(io_ctx, TEST_IMAGE, &image, NULL) == 0);
test_ls(io_ctx, 1, TEST_IMAGE);
test_ls_snaps(image, 0);
test_create_snap(image, TEST_SNAP);
test_ls_snaps(image, 1, TEST_SNAP, MB_BYTES(1));
test_resize_and_stat(image, MB_BYTES(2));
test_io(io_ctx, image);
test_create_snap(image, TEST_SNAP "1");
test_ls_snaps(image, 2, TEST_SNAP, MB_BYTES(1), TEST_SNAP "1", MB_BYTES(2));
test_delete_snap(image, TEST_SNAP);
test_ls_snaps(image, 1, TEST_SNAP "1", MB_BYTES(2));
test_delete_snap(image, TEST_SNAP "1");
test_ls_snaps(image, 0);
test_io_to_snapshot(io_ctx, image, MB_BYTES(2));
assert(rbd_close(image) == 0);
test_create_and_stat(io_ctx, TEST_IMAGE "1", MB_BYTES(2));
test_ls(io_ctx, 2, TEST_IMAGE, TEST_IMAGE "1");
test_delete(io_ctx, TEST_IMAGE);
test_ls(io_ctx, 1, TEST_IMAGE "1");
test_delete(io_ctx, TEST_IMAGE "1");
test_ls(io_ctx, 0);
rados_ioctx_destroy(io_ctx);
rados_shutdown(cluster);
return 0;
}
int
create_image()
{
int r;
int order = 0;
r = rados_create(&cluster, NULL);
if (r < 0) {
simple_err("Could not create cluster handle", r);
return r;
}
rados_conf_parse_env(cluster, NULL);
r = rados_conf_read_file(cluster, NULL);
if (r < 0) {
simple_err("Error reading ceph config file", r);
goto failed_shutdown;
}
r = rados_connect(cluster);
if (r < 0) {
simple_err("Error connecting to cluster", r);
goto failed_shutdown;
}
r = krbd_create_from_context(rados_cct(cluster), &krbd);
if (r < 0) {
simple_err("Could not create libkrbd handle", r);
goto failed_shutdown;
}
r = rados_pool_create(cluster, pool);
if (r < 0 && r != -EEXIST) {
simple_err("Error creating pool", r);
goto failed_krbd;
}
r = rados_ioctx_create(cluster, pool, &ioctx);
if (r < 0) {
simple_err("Error creating ioctx", r);
goto failed_krbd;
}
if (clone_calls) {
r = rbd_create2(ioctx, iname, 0, RBD_FEATURE_LAYERING, &order);
} else {
r = rbd_create(ioctx, iname, 0, &order);
}
if (r < 0) {
simple_err("Error creating image", r);
goto failed_open;
}
return 0;
failed_open:
rados_ioctx_destroy(ioctx);
failed_krbd:
krbd_destroy(krbd);
failed_shutdown:
rados_shutdown(cluster);
return r;
}
static void qemu_rbd_close(BlockDriverState *bs)
{
BDRVRBDState *s = bs->opaque;
rbd_close(s->image);
rados_ioctx_destroy(s->io_ctx);
g_free(s->snap);
rados_shutdown(s->cluster);
}
static void bs_rbd_close(struct scsi_lu *lu)
{
struct active_rbd *rbd = RBDP(lu->fd);
if (rbd->rbd_image) {
rbd_close(rbd->rbd_image);
rados_ioctx_destroy(rbd->ioctx);
rbd->rbd_image = rbd->ioctx = NULL;
}
}
int main(int argc, char **argv)
{
rados_t cluster;
char cluster_name[] = "ceph";
char user_name[] = "client.admin";
APPNAME = &argv[0][2];
uint64_t flags;
int err;
/* Initialize the cluster handle */
err = rados_create2(&cluster, cluster_name, user_name, flags);
// err = rados_create(&cluster, NULL);
error_print(err, "Couldn't create the cluster handle!", "Created a cluster handle.");
/* Read a Ceph configuration file to configure the cluster handle. */
err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
error_print(err, "Cannot read config file!", "Read the config file.");
/* Connect to the cluster */
err = rados_connect(cluster);
error_print(err, "Cannot connect to cluster!", "Connected to the cluster.");
/* List pools. */
char buf[1024];
err = rados_pool_list(cluster, buf, 1024);
split_str(buf, 1024);
error_print(err, "Cannot list pools!", buf);
/* List pool stats */
char *poolname = strtok(buf, ",");
while(poolname != NULL)
{
rados_ioctx_t io;
struct rados_pool_stat_t stats;
err = rados_ioctx_create(cluster, poolname, &io);
error_print(err, "Cannot create IO context!", "IO context created.");
err = rados_ioctx_pool_stat(io, &stats);
error_print(err, "Cannot list pool stats!", "Pool stats listed.");
printf("[%s][%s]: Number of read: %"PRIu64"\n", APPNAME, poolname, stats.num_rd);
printf("[%s][%s]: Number of read in kb: %"PRIu64"\n", APPNAME, poolname, stats.num_rd_kb);
printf("[%s][%s]: Number of write: %"PRIu64"\n", APPNAME, poolname, stats.num_wr);
printf("[%s][%s]: Number of write in kb: %"PRIu64"\n", APPNAME, poolname, stats.num_wr_kb);
rados_ioctx_destroy(io);
poolname = strtok(NULL, ",");
}
/* Shut down the cluster */
rados_shutdown(cluster);
printf("[%s]: Cluster shut down.\n", APPNAME);
}
static int rados_close_storage(struct storage_backend * store) {
struct rados_ctx * ctx = (struct rados_ctx *)store->storage_ctx;
rados_ioctx_destroy(ctx->io);
rados_shutdown(ctx->cluster);
log_message(STORE_LOGLVL_DEBUG,"rados_close_storage: Closed rados backend");
free(ctx->metadata_cache.data);
free(ctx->pool);
free(ctx);
return 0;
}
int main(int argc, char* argv[])
{
int err;
rados_t cluster;
rados_ioctx_t io;
rbd_image_t image;
char *poolname = "pool100";
char buf[IMAGE_BUF_SIZE] = {0};
err = rados_create(&cluster, NULL);
if (err < 0) {
fprintf(stderr, "%s: cannot create a cluster handle: %s\n", argv[0], strerror(-err));
exit(1);
}
err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
if (err < 0) {
fprintf(stderr, "%s: cannot read config file: %s\n", argv[0], strerror(-err));
exit(1);
}
err = rados_connect(cluster);
if (err < 0) {
fprintf(stderr, "%s: cannot connect to cluster: %s\n", argv[0], strerror(-err));
exit(1);
}
err = rados_ioctx_create(cluster, poolname, &io);
if (err < 0) {
fprintf(stderr, "%s: cannot open rados pool %s: %s\n", argv[0], poolname, strerror(-err));
rados_shutdown(cluster);
exit(1);
}
err = rbd_open(io, "user1_image2", &image, NULL);
if (err < 0){
fprintf(stderr, "open image failed: %s\n", strerror(-err));
goto out;
}
err = rbd_read(image, IMAGE_BUF_SIZE*2, IMAGE_BUF_SIZE, buf);
if (err < 0) {
fprintf(stderr, "%s: cannot read image: %s\n", poolname, strerror(-err));
}else{
fprintf(stderr, "read image return :%d\n", err);
}
out:
rados_ioctx_destroy(io);
rados_shutdown(cluster);
return 0;
}
static VALUE rb_rados_cluster_pool_objects_each(VALUE self, VALUE pool_name) {
GET_CLUSTER(self);
int err, i;
Check_Type(pool_name, T_STRING);
char *cpool_name = StringValuePtr(pool_name);
// Seems like assuming malloc succeeded is all the rage
struct nogvl_objects_list_args *args = xmalloc(sizeof(struct nogvl_objects_list_args));
err = rados_ioctx_create(*wrapper->cluster, cpool_name, &args->ioctx);
if (err < 0) {
xfree(args);
rb_raise(rb_const_get(mRados, rb_intern("Error")), "error creating context for pool '%s': %s", cpool_name, strerror(-err));
}
err = rados_objects_list_open(args->ioctx, &args->lctx);
if (err < 0) {
rados_ioctx_destroy(args->ioctx);
xfree(args);
rb_raise(rb_const_get(mRados, rb_intern("PoolError")), "error listing objects for pool '%s': %s", cpool_name, strerror(-err));
}
for (;;) {
err = (int)rb_thread_blocking_region(nogvl_objects_list_next_batch, args, NULL, NULL);
for (i = 0; ; i++) {
if (args->oids[i] == NULL)
break;
rb_yield(rb_str_new2(args->oids[i]));
}
if (err == -ENOENT)
break;
if (err < 0) {
rados_ioctx_destroy(args->ioctx);
rados_objects_list_close(args->lctx);
xfree(args);
rb_raise(rb_const_get(mRados, rb_intern("PoolError")), "error listing objects for pool '%s': %s", cpool_name, strerror(-err));
}
}
rados_objects_list_close(args->lctx);
rados_ioctx_destroy(args->ioctx);
xfree(args);
return Qtrue;
}
static void qemu_rbd_close(BlockDriverState *bs)
{
BDRVRBDState *s = bs->opaque;
close(s->fds[0]);
close(s->fds[1]);
qemu_aio_set_fd_handler(s->fds[RBD_FD_READ], NULL, NULL, NULL, NULL);
rbd_close(s->image);
rados_ioctx_destroy(s->io_ctx);
g_free(s->snap);
rados_shutdown(s->cluster);
}
/* FIXME Deprecate and remove keypairs or make it available in QMP. */
static int qemu_rbd_do_create(BlockdevCreateOptions *options,
const char *keypairs, const char *password_secret,
Error **errp)
{
BlockdevCreateOptionsRbd *opts = &options->u.rbd;
rados_t cluster;
rados_ioctx_t io_ctx;
int obj_order = 0;
int ret;
assert(options->driver == BLOCKDEV_DRIVER_RBD);
if (opts->location->has_snapshot) {
error_setg(errp, "Can't use snapshot name for image creation");
return -EINVAL;
}
if (opts->has_cluster_size) {
int64_t objsize = opts->cluster_size;
if ((objsize - 1) & objsize) { /* not a power of 2? */
error_setg(errp, "obj size needs to be power of 2");
return -EINVAL;
}
if (objsize < 4096) {
error_setg(errp, "obj size too small");
return -EINVAL;
}
obj_order = ctz32(objsize);
}
ret = qemu_rbd_connect(&cluster, &io_ctx, opts->location, false, keypairs,
password_secret, errp);
if (ret < 0) {
return ret;
}
ret = rbd_create(io_ctx, opts->location->image, opts->size, &obj_order);
if (ret < 0) {
error_setg_errno(errp, -ret, "error rbd create");
goto out;
}
ret = 0;
out:
rados_ioctx_destroy(io_ctx);
rados_shutdown(cluster);
return ret;
}
int
create_image()
{
int r;
rados_t cluster;
int order = 0;
r = rados_create(&cluster, NULL);
if (r < 0) {
simple_err("Could not create cluster handle", r);
return r;
}
rados_conf_parse_env(cluster, NULL);
r = rados_conf_read_file(cluster, NULL);
if (r < 0) {
simple_err("Error reading ceph config file", r);
goto failed_shutdown;
}
r = rados_connect(cluster);
if (r < 0) {
simple_err("Error connecting to cluster", r);
goto failed_shutdown;
}
r = rados_pool_create(cluster, pool);
if (r < 0 && r != -EEXIST) {
simple_err("Error creating pool", r);
goto failed_shutdown;
}
r = rados_ioctx_create(cluster, pool, &ioctx);
if (r < 0) {
simple_err("Error creating ioctx", r);
goto failed_shutdown;
}
r = rbd_create(ioctx, iname, 0, &order);
if (r < 0) {
simple_err("Error creating image", r);
goto failed_open;
}
return 0;
failed_open:
rados_ioctx_destroy(ioctx);
failed_shutdown:
rados_shutdown(cluster);
return r;
}
static int virStorageBackendRBDCloseRADOSConn(virStorageBackendRBDStatePtr ptr)
{
int ret = 0;
if (ptr.ioctx != NULL) {
VIR_DEBUG("Closing RADOS IoCTX");
rados_ioctx_destroy(ptr.ioctx);
ret = -1;
}
ptr.ioctx = NULL;
if (ptr.cluster != NULL) {
VIR_DEBUG("Closing RADOS connection");
rados_shutdown(ptr.cluster);
ret = -2;
}
ptr.cluster = NULL;
time_t runtime = time(0) - ptr.starttime;
VIR_DEBUG("RADOS connection existed for %ld seconds", runtime);
return ret;
}
static int testrados(void)
{
char tmp[32];
int i, r;
rados_t cl;
if (rados_create(&cl, NULL) < 0) {
printf("error initializing\n");
return 1;
}
if (rados_conf_read_file(cl, NULL)) {
printf("error reading configuration file\n");
return 1;
}
// Try to set a configuration option that doesn't exist.
// This should fail.
if (!rados_conf_set(cl, "config option that doesn't exist",
"some random value")) {
printf("error: succeeded in setting nonexistent config option\n");
return 1;
}
if (rados_conf_get(cl, "log to stderr", tmp, sizeof(tmp))) {
printf("error: failed to read log_to_stderr from config\n");
return 1;
}
// Can we change it?
if (rados_conf_set(cl, "log to stderr", "2")) {
printf("error: error setting log_to_stderr\n");
return 1;
}
if (rados_conf_get(cl, "log to stderr", tmp, sizeof(tmp))) {
printf("error: failed to read log_to_stderr from config\n");
return 1;
}
if (tmp[0] != '2') {
printf("error: new setting for log_to_stderr failed to take effect.\n");
return 1;
}
if (rados_connect(cl)) {
printf("error connecting\n");
return 1;
}
if (rados_connect(cl) == 0) {
printf("second connect attempt didn't return an error\n");
return 1;
}
/* create an io_ctx */
r = rados_pool_create(cl, "foo");
printf("rados_pool_create = %d\n", r);
rados_ioctx_t io_ctx;
r = rados_ioctx_create(cl, "foo", &io_ctx);
printf("rados_ioctx_create = %d, io_ctx = %p\n", r, io_ctx);
/* list all pools */
{
int buf_sz = rados_pool_list(cl, NULL, 0);
printf("need buffer size of %d\n", buf_sz);
char buf[buf_sz];
int r = rados_pool_list(cl, buf, buf_sz);
if (r != buf_sz) {
printf("buffer size mismatch: got %d the first time, but %d "
"the second.\n", buf_sz, r);
return 1;
}
const char *b = buf;
printf("begin pools.\n");
while (1) {
if (b[0] == '\0')
break;
printf(" pool: '%s'\n", b);
b += strlen(b) + 1;
};
printf("end pools.\n");
}
/* stat */
struct rados_pool_stat_t st;
r = rados_ioctx_pool_stat(io_ctx, &st);
printf("rados_ioctx_pool_stat = %d, %lld KB, %lld objects\n", r, (long long)st.num_kb, (long long)st.num_objects);
/* snapshots */
r = rados_ioctx_snap_create(io_ctx, "snap1");
printf("rados_ioctx_snap_create snap1 = %d\n", r);
rados_snap_t snaps[10];
r = rados_ioctx_snap_list(io_ctx, snaps, 10);
for (i=0; i<r; i++) {
char name[100];
rados_ioctx_snap_get_name(io_ctx, snaps[i], name, sizeof(name));
printf("rados_ioctx_snap_list got snap %lld %s\n", (long long)snaps[i], name);
}
rados_snap_t snapid;
r = rados_ioctx_snap_lookup(io_ctx, "snap1", &snapid);
printf("rados_ioctx_snap_lookup snap1 got %lld, result %d\n", (long long)snapid, r);
r = rados_ioctx_snap_remove(io_ctx, "snap1");
printf("rados_ioctx_snap_remove snap1 = %d\n", r);
/* sync io */
time_t tm;
char buf[128], buf2[128];
time(&tm);
snprintf(buf, 128, "%s", ctime(&tm));
const char *oid = "foo_object";
r = rados_write(io_ctx, oid, buf, strlen(buf) + 1, 0);
printf("rados_write = %d\n", r);
r = rados_read(io_ctx, oid, buf2, sizeof(buf2), 0);
printf("rados_read = %d\n", r);
if (memcmp(buf, buf2, r))
printf("*** content mismatch ***\n");
/* attrs */
if (do_rados_setxattr(io_ctx, oid, "b", "2"))
return 1;
if (do_rados_setxattr(io_ctx, oid, "a", "1"))
return 1;
if (do_rados_setxattr(io_ctx, oid, "c", "3"))
return 1;
if (do_rados_getxattr(io_ctx, oid, "a", "1"))
return 1;
if (do_rados_getxattr(io_ctx, oid, "b", "2"))
return 1;
if (do_rados_getxattr(io_ctx, oid, "c", "3"))
return 1;
const char *exkeys[] = { "a", "b", "c", NULL };
const char *exvals[] = { "1", "2", "3", NULL };
if (do_rados_getxattrs(io_ctx, oid, exkeys, exvals))
return 1;
uint64_t size;
time_t mtime;
r = rados_stat(io_ctx, oid, &size, &mtime);
printf("rados_stat size = %lld mtime = %d = %d\n", (long long)size, (int)mtime, r);
r = rados_stat(io_ctx, "does_not_exist", NULL, NULL);
printf("rados_stat(does_not_exist) = %d\n", r);
/* exec */
rados_exec(io_ctx, oid, "crypto", "md5", buf, strlen(buf) + 1, buf, 128);
printf("exec result=%s\n", buf);
r = rados_read(io_ctx, oid, buf2, 128, 0);
printf("read result=%s\n", buf2);
printf("size=%d\n", r);
/* aio */
rados_completion_t a, b;
rados_aio_create_completion(0, 0, 0, &a);
rados_aio_create_completion(0, 0, 0, &b);
rados_aio_write(io_ctx, "a", a, buf, 100, 0);
rados_aio_write(io_ctx, "../b/bb_bb_bb\\foo\\bar", b, buf, 100, 0);
rados_aio_wait_for_safe(a);
printf("a safe\n");
rados_aio_wait_for_safe(b);
printf("b safe\n");
rados_aio_release(a);
rados_aio_release(b);
/* test flush */
printf("testing aio flush\n");
rados_completion_t c;
rados_aio_create_completion(0, 0, 0, &c);
rados_aio_write(io_ctx, "c", c, buf, 100, 0);
int safe = rados_aio_is_safe(c);
printf("a should not yet be safe and ... %s\n", safe ? "is":"is not");
assert(!safe);
rados_aio_flush(io_ctx);
safe = rados_aio_is_safe(c);
printf("a should be safe and ... %s\n", safe ? "is":"is not");
assert(safe);
rados_aio_release(c);
rados_read(io_ctx, "../b/bb_bb_bb\\foo\\bar", buf2, 128, 0);
/* list objects */
rados_list_ctx_t h;
r = rados_objects_list_open(io_ctx, &h);
printf("rados_list_objects_open = %d, h = %p\n", r, h);
const char *poolname;
while (rados_objects_list_next(h, &poolname) == 0)
printf("rados_list_objects_next got object '%s'\n", poolname);
rados_objects_list_close(h);
/* stat */
r = rados_ioctx_pool_stat(io_ctx, &st);
printf("rados_stat_pool = %d, %lld KB, %lld objects\n", r, (long long)st.num_kb, (long long)st.num_objects);
/* delete a pool */
printf("rados_delete_pool = %d\n", r);
rados_ioctx_destroy(io_ctx);
r = rados_pool_delete(cl, "foo");
printf("rados_ioctx_pool_delete = %d\n", r);
rados_shutdown(cl);
return 0;
}
static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
Error **errp)
{
BDRVRBDState *s = bs->opaque;
const char *pool, *snap, *conf, *user, *image_name, *keypairs;
const char *secretid, *filename;
QemuOpts *opts;
Error *local_err = NULL;
char *mon_host = NULL;
int r;
/* If we are given a filename, parse the filename, with precedence given to
* filename encoded options */
filename = qdict_get_try_str(options, "filename");
if (filename) {
warn_report("'filename' option specified. "
"This is an unsupported option, and may be deprecated "
"in the future");
qemu_rbd_parse_filename(filename, options, &local_err);
if (local_err) {
r = -EINVAL;
error_propagate(errp, local_err);
goto exit;
}
}
opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
qemu_opts_absorb_qdict(opts, options, &local_err);
if (local_err) {
error_propagate(errp, local_err);
r = -EINVAL;
goto failed_opts;
}
mon_host = qemu_rbd_mon_host(options, &local_err);
if (local_err) {
error_propagate(errp, local_err);
r = -EINVAL;
goto failed_opts;
}
secretid = qemu_opt_get(opts, "password-secret");
pool = qemu_opt_get(opts, "pool");
conf = qemu_opt_get(opts, "conf");
snap = qemu_opt_get(opts, "snapshot");
user = qemu_opt_get(opts, "user");
image_name = qemu_opt_get(opts, "image");
keypairs = qemu_opt_get(opts, "=keyvalue-pairs");
if (!pool || !image_name) {
error_setg(errp, "Parameters 'pool' and 'image' are required");
r = -EINVAL;
goto failed_opts;
}
r = rados_create(&s->cluster, user);
if (r < 0) {
error_setg_errno(errp, -r, "error initializing");
goto failed_opts;
}
s->snap = g_strdup(snap);
s->image_name = g_strdup(image_name);
/* try default location when conf=NULL, but ignore failure */
r = rados_conf_read_file(s->cluster, conf);
if (conf && r < 0) {
error_setg_errno(errp, -r, "error reading conf file %s", conf);
goto failed_shutdown;
}
r = qemu_rbd_set_keypairs(s->cluster, keypairs, errp);
if (r < 0) {
goto failed_shutdown;
}
if (mon_host) {
r = rados_conf_set(s->cluster, "mon_host", mon_host);
if (r < 0) {
goto failed_shutdown;
}
}
if (qemu_rbd_set_auth(s->cluster, secretid, errp) < 0) {
r = -EIO;
goto failed_shutdown;
}
/*
* Fallback to more conservative semantics if setting cache
* options fails. Ignore errors from setting rbd_cache because the
* only possible error is that the option does not exist, and
* librbd defaults to no caching. If write through caching cannot
* be set up, fall back to no caching.
*/
if (flags & BDRV_O_NOCACHE) {
rados_conf_set(s->cluster, "rbd_cache", "false");
} else {
rados_conf_set(s->cluster, "rbd_cache", "true");
}
r = rados_connect(s->cluster);
if (r < 0) {
error_setg_errno(errp, -r, "error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
if (r < 0) {
error_setg_errno(errp, -r, "error opening pool %s", pool);
goto failed_shutdown;
}
/* rbd_open is always r/w */
r = rbd_open(s->io_ctx, s->image_name, &s->image, s->snap);
if (r < 0) {
error_setg_errno(errp, -r, "error reading header from %s",
s->image_name);
goto failed_open;
}
/* If we are using an rbd snapshot, we must be r/o, otherwise
* leave as-is */
if (s->snap != NULL) {
if (!bdrv_is_read_only(bs)) {
error_report("Opening rbd snapshots without an explicit "
"read-only=on option is deprecated. Future versions "
"will refuse to open the image instead of "
"automatically marking the image read-only.");
r = bdrv_set_read_only(bs, true, &local_err);
if (r < 0) {
error_propagate(errp, local_err);
goto failed_open;
}
}
}
qemu_opts_del(opts);
return 0;
failed_open:
rados_ioctx_destroy(s->io_ctx);
failed_shutdown:
rados_shutdown(s->cluster);
g_free(s->snap);
g_free(s->image_name);
failed_opts:
qemu_opts_del(opts);
g_free(mon_host);
exit:
return r;
}
static int coroutine_fn qemu_rbd_co_create_opts(const char *filename,
QemuOpts *opts,
Error **errp)
{
Error *local_err = NULL;
int64_t bytes = 0;
int64_t objsize;
int obj_order = 0;
const char *pool, *image_name, *conf, *user, *keypairs;
const char *secretid;
rados_t cluster;
rados_ioctx_t io_ctx;
QDict *options = NULL;
int ret = 0;
secretid = qemu_opt_get(opts, "password-secret");
/* Read out options */
bytes = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
BDRV_SECTOR_SIZE);
objsize = qemu_opt_get_size_del(opts, BLOCK_OPT_CLUSTER_SIZE, 0);
if (objsize) {
if ((objsize - 1) & objsize) { /* not a power of 2? */
error_setg(errp, "obj size needs to be power of 2");
ret = -EINVAL;
goto exit;
}
if (objsize < 4096) {
error_setg(errp, "obj size too small");
ret = -EINVAL;
goto exit;
}
obj_order = ctz32(objsize);
}
options = qdict_new();
qemu_rbd_parse_filename(filename, options, &local_err);
if (local_err) {
ret = -EINVAL;
error_propagate(errp, local_err);
goto exit;
}
/*
* Caution: while qdict_get_try_str() is fine, getting non-string
* types would require more care. When @options come from -blockdev
* or blockdev_add, its members are typed according to the QAPI
* schema, but when they come from -drive, they're all QString.
*/
pool = qdict_get_try_str(options, "pool");
conf = qdict_get_try_str(options, "conf");
user = qdict_get_try_str(options, "user");
image_name = qdict_get_try_str(options, "image");
keypairs = qdict_get_try_str(options, "=keyvalue-pairs");
ret = rados_create(&cluster, user);
if (ret < 0) {
error_setg_errno(errp, -ret, "error initializing");
goto exit;
}
/* try default location when conf=NULL, but ignore failure */
ret = rados_conf_read_file(cluster, conf);
if (conf && ret < 0) {
error_setg_errno(errp, -ret, "error reading conf file %s", conf);
ret = -EIO;
goto shutdown;
}
ret = qemu_rbd_set_keypairs(cluster, keypairs, errp);
if (ret < 0) {
ret = -EIO;
goto shutdown;
}
if (qemu_rbd_set_auth(cluster, secretid, errp) < 0) {
ret = -EIO;
goto shutdown;
}
ret = rados_connect(cluster);
if (ret < 0) {
error_setg_errno(errp, -ret, "error connecting");
goto shutdown;
}
ret = rados_ioctx_create(cluster, pool, &io_ctx);
if (ret < 0) {
error_setg_errno(errp, -ret, "error opening pool %s", pool);
goto shutdown;
}
ret = rbd_create(io_ctx, image_name, bytes, &obj_order);
if (ret < 0) {
error_setg_errno(errp, -ret, "error rbd create");
}
rados_ioctx_destroy(io_ctx);
shutdown:
rados_shutdown(cluster);
exit:
QDECREF(options);
return ret;
}
static int app_rbd_open()
{
int ret;
/* poolname imagename username password monitor */
fprintf(stdout,
"open rados as following setting:\n"
"poolname: %s\n"
"imagename: %s\n"
"username: %s\n"
"password: %s\n"
"monitor: %s\n",
poolname, imagename, username, password, monitor);
/**
* 创建一个 rados_t 句柄, 该句柄暴扣了rados 客户端的数据结构, 用来和
* rados 通信, 第二个参数是连接 rados 的客户端 ID, 这里我们使用 admin
*/
ret = rados_create(&cluster, username);
if (ret < 0) {
fprintf(stderr, "cannot create a cluster handle: %s\n", strerror(-ret));
goto failed_create;
}
/* 载入配置文件 */
ret = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
if (ret < 0) {
fprintf(stderr, "cannot read config file: %s\n", strerror(-ret));
goto failed_shutdown;
}
/**
* 调用 rados_conf_set() 设置 rados 参数, 包括认证信息, monitor 地址等,
* 如果设置了 cephx 认证, 那么之前创建 rados 句柄的时候, 必须设置客户端
* ID, 并且必须设置 key 的密码.
*/
if (conf_set("key", password) < 0) {
goto failed_shutdown;
}
if (conf_set("auth_supported", "cephx") < 0) {
goto failed_shutdown;
}
if (conf_set("mon_host", monitor) < 0) {
goto failed_shutdown;
}
/* 完成了上面的设置之后, 就可以用我们的 rados 句柄连接 rados 服务器了 */
ret = rados_connect(cluster);
if (ret < 0) {
fprintf(stderr, "cannot connect to cluster: %s\n", strerror(-ret));
goto failed_shutdown;
}
/**
* 成功连接上 rados 服务器之后, 就可以用 rados_ioctx_create() 打开 rados
* 上的 pool 了, 该函数需要传递一个 rados_ioctx_t, 用来对打开的 pool 进行
* 操作.
*/
ret = rados_ioctx_create(cluster, poolname, &io_ctx);
if (ret < 0) {
fprintf(stderr, "cannot open rados pool %s: %s\n",
poolname, strerror(-ret));
goto failed_shutdown;
}
/**
* 上面说过, 我们已经能够有一个 rados_ioctx_t 的指针了, 该指针用来对关联的
* pool 进行 I/O 操作, 比如打开池中的 image 等, 这里我们直接对 rbd 操作.
*/
ret = rbd_open(io_ctx, imagename, &image, NULL);
if (ret < 0) {
fprintf(stderr, "error reading header from image %s\n", imagename);
goto failed_open;
}
return 0;
failed_open:
rados_ioctx_destroy(io_ctx);
failed_shutdown:
rados_shutdown(cluster);
failed_create:
return ret;
}
static int qemu_rbd_create(const char *filename, QemuOpts *opts, Error **errp)
{
Error *local_err = NULL;
int64_t bytes = 0;
int64_t objsize;
int obj_order = 0;
const char *pool, *name, *conf, *clientname, *keypairs;
const char *secretid;
rados_t cluster;
rados_ioctx_t io_ctx;
QDict *options = NULL;
int ret = 0;
secretid = qemu_opt_get(opts, "password-secret");
/* Read out options */
bytes = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
BDRV_SECTOR_SIZE);
objsize = qemu_opt_get_size_del(opts, BLOCK_OPT_CLUSTER_SIZE, 0);
if (objsize) {
if ((objsize - 1) & objsize) { /* not a power of 2? */
error_setg(errp, "obj size needs to be power of 2");
ret = -EINVAL;
goto exit;
}
if (objsize < 4096) {
error_setg(errp, "obj size too small");
ret = -EINVAL;
goto exit;
}
obj_order = ctz32(objsize);
}
options = qdict_new();
qemu_rbd_parse_filename(filename, options, &local_err);
if (local_err) {
ret = -EINVAL;
error_propagate(errp, local_err);
goto exit;
}
pool = qdict_get_try_str(options, "pool");
conf = qdict_get_try_str(options, "conf");
clientname = qdict_get_try_str(options, "user");
name = qdict_get_try_str(options, "image");
keypairs = qdict_get_try_str(options, "=keyvalue-pairs");
ret = rados_create(&cluster, clientname);
if (ret < 0) {
error_setg_errno(errp, -ret, "error initializing");
goto exit;
}
/* try default location when conf=NULL, but ignore failure */
ret = rados_conf_read_file(cluster, conf);
if (conf && ret < 0) {
error_setg_errno(errp, -ret, "error reading conf file %s", conf);
ret = -EIO;
goto shutdown;
}
ret = qemu_rbd_set_keypairs(cluster, keypairs, errp);
if (ret < 0) {
ret = -EIO;
goto shutdown;
}
if (qemu_rbd_set_auth(cluster, secretid, errp) < 0) {
ret = -EIO;
goto shutdown;
}
ret = rados_connect(cluster);
if (ret < 0) {
error_setg_errno(errp, -ret, "error connecting");
goto shutdown;
}
ret = rados_ioctx_create(cluster, pool, &io_ctx);
if (ret < 0) {
error_setg_errno(errp, -ret, "error opening pool %s", pool);
goto shutdown;
}
ret = rbd_create(io_ctx, name, bytes, &obj_order);
if (ret < 0) {
error_setg_errno(errp, -ret, "error rbd create");
}
rados_ioctx_destroy(io_ctx);
shutdown:
rados_shutdown(cluster);
exit:
QDECREF(options);
return ret;
}
static int qemu_rbd_open(BlockDriverState *bs, const char *filename, int flags)
{
BDRVRBDState *s = bs->opaque;
char pool[RBD_MAX_POOL_NAME_SIZE];
char snap_buf[RBD_MAX_SNAP_NAME_SIZE];
char conf[RBD_MAX_CONF_SIZE];
char clientname_buf[RBD_MAX_CONF_SIZE];
char *clientname;
int r;
if (qemu_rbd_parsename(filename, pool, sizeof(pool),
snap_buf, sizeof(snap_buf),
s->name, sizeof(s->name),
conf, sizeof(conf)) < 0) {
return -EINVAL;
}
clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
r = rados_create(&s->cluster, clientname);
if (r < 0) {
error_report("error initializing");
return r;
}
s->snap = NULL;
if (snap_buf[0] != '\0') {
s->snap = g_strdup(snap_buf);
}
/*
* Fallback to more conservative semantics if setting cache
* options fails. Ignore errors from setting rbd_cache because the
* only possible error is that the option does not exist, and
* librbd defaults to no caching. If write through caching cannot
* be set up, fall back to no caching.
*/
if (flags & BDRV_O_NOCACHE) {
rados_conf_set(s->cluster, "rbd_cache", "false");
} else {
rados_conf_set(s->cluster, "rbd_cache", "true");
}
if (strstr(conf, "conf=") == NULL) {
/* try default location, but ignore failure */
rados_conf_read_file(s->cluster, NULL);
}
if (conf[0] != '\0') {
r = qemu_rbd_set_conf(s->cluster, conf);
if (r < 0) {
error_report("error setting config options");
goto failed_shutdown;
}
}
r = rados_connect(s->cluster);
if (r < 0) {
error_report("error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
if (r < 0) {
error_report("error opening pool %s", pool);
goto failed_shutdown;
}
r = rbd_open(s->io_ctx, s->name, &s->image, s->snap);
if (r < 0) {
error_report("error reading header from %s", s->name);
goto failed_open;
}
bs->read_only = (s->snap != NULL);
s->event_reader_pos = 0;
r = qemu_pipe(s->fds);
if (r < 0) {
error_report("error opening eventfd");
goto failed;
}
fcntl(s->fds[0], F_SETFL, O_NONBLOCK);
fcntl(s->fds[1], F_SETFL, O_NONBLOCK);
qemu_aio_set_fd_handler(s->fds[RBD_FD_READ], qemu_rbd_aio_event_reader,
NULL, qemu_rbd_aio_flush_cb, s);
return 0;
failed:
rbd_close(s->image);
failed_open:
rados_ioctx_destroy(s->io_ctx);
failed_shutdown:
rados_shutdown(s->cluster);
g_free(s->snap);
return r;
}
static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
Error **errp)
{
BDRVRBDState *s = bs->opaque;
BlockdevOptionsRbd *opts = NULL;
Visitor *v;
QObject *crumpled = NULL;
const QDictEntry *e;
Error *local_err = NULL;
const char *filename;
char *keypairs, *secretid;
int r;
/* If we are given a filename, parse the filename, with precedence given to
* filename encoded options */
filename = qdict_get_try_str(options, "filename");
if (filename) {
warn_report("'filename' option specified. "
"This is an unsupported option, and may be deprecated "
"in the future");
qemu_rbd_parse_filename(filename, options, &local_err);
qdict_del(options, "filename");
if (local_err) {
error_propagate(errp, local_err);
return -EINVAL;
}
}
keypairs = g_strdup(qdict_get_try_str(options, "=keyvalue-pairs"));
if (keypairs) {
qdict_del(options, "=keyvalue-pairs");
}
secretid = g_strdup(qdict_get_try_str(options, "password-secret"));
if (secretid) {
qdict_del(options, "password-secret");
}
/* Convert the remaining options into a QAPI object */
crumpled = qdict_crumple(options, errp);
if (crumpled == NULL) {
r = -EINVAL;
goto out;
}
v = qobject_input_visitor_new_keyval(crumpled);
visit_type_BlockdevOptionsRbd(v, NULL, &opts, &local_err);
visit_free(v);
qobject_unref(crumpled);
if (local_err) {
error_propagate(errp, local_err);
r = -EINVAL;
goto out;
}
/* Remove the processed options from the QDict (the visitor processes
* _all_ options in the QDict) */
while ((e = qdict_first(options))) {
qdict_del(options, e->key);
}
r = qemu_rbd_connect(&s->cluster, &s->io_ctx, opts,
!(flags & BDRV_O_NOCACHE), keypairs, secretid, errp);
if (r < 0) {
goto out;
}
s->snap = g_strdup(opts->snapshot);
s->image_name = g_strdup(opts->image);
/* rbd_open is always r/w */
r = rbd_open(s->io_ctx, s->image_name, &s->image, s->snap);
if (r < 0) {
error_setg_errno(errp, -r, "error reading header from %s",
s->image_name);
goto failed_open;
}
/* If we are using an rbd snapshot, we must be r/o, otherwise
* leave as-is */
if (s->snap != NULL) {
if (!bdrv_is_read_only(bs)) {
error_report("Opening rbd snapshots without an explicit "
"read-only=on option is deprecated. Future versions "
"will refuse to open the image instead of "
"automatically marking the image read-only.");
r = bdrv_set_read_only(bs, true, &local_err);
if (r < 0) {
error_propagate(errp, local_err);
goto failed_open;
}
}
}
r = 0;
goto out;
failed_open:
rados_ioctx_destroy(s->io_ctx);
g_free(s->snap);
g_free(s->image_name);
rados_shutdown(s->cluster);
out:
qapi_free_BlockdevOptionsRbd(opts);
g_free(keypairs);
g_free(secretid);
return r;
}
static int qemu_rbd_create(const char *filename, QemuOpts *opts, Error **errp)
{
Error *local_err = NULL;
int64_t bytes = 0;
int64_t objsize;
int obj_order = 0;
char pool[RBD_MAX_POOL_NAME_SIZE];
char name[RBD_MAX_IMAGE_NAME_SIZE];
char snap_buf[RBD_MAX_SNAP_NAME_SIZE];
char conf[RBD_MAX_CONF_SIZE];
char clientname_buf[RBD_MAX_CONF_SIZE];
char *clientname;
rados_t cluster;
rados_ioctx_t io_ctx;
int ret;
if (qemu_rbd_parsename(filename, pool, sizeof(pool),
snap_buf, sizeof(snap_buf),
name, sizeof(name),
conf, sizeof(conf), &local_err) < 0) {
error_propagate(errp, local_err);
return -EINVAL;
}
/* Read out options */
bytes = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);
objsize = qemu_opt_get_size_del(opts, BLOCK_OPT_CLUSTER_SIZE, 0);
if (objsize) {
if ((objsize - 1) & objsize) { /* not a power of 2? */
error_setg(errp, "obj size needs to be power of 2");
return -EINVAL;
}
if (objsize < 4096) {
error_setg(errp, "obj size too small");
return -EINVAL;
}
obj_order = ffs(objsize) - 1;
}
clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
if (rados_create(&cluster, clientname) < 0) {
error_setg(errp, "error initializing");
return -EIO;
}
if (strstr(conf, "conf=") == NULL) {
/* try default location, but ignore failure */
rados_conf_read_file(cluster, NULL);
}
if (conf[0] != '\0' &&
qemu_rbd_set_conf(cluster, conf, &local_err) < 0) {
rados_shutdown(cluster);
error_propagate(errp, local_err);
return -EIO;
}
if (rados_connect(cluster) < 0) {
error_setg(errp, "error connecting");
rados_shutdown(cluster);
return -EIO;
}
if (rados_ioctx_create(cluster, pool, &io_ctx) < 0) {
error_setg(errp, "error opening pool %s", pool);
rados_shutdown(cluster);
return -EIO;
}
ret = rbd_create(io_ctx, name, bytes, &obj_order);
rados_ioctx_destroy(io_ctx);
rados_shutdown(cluster);
return ret;
}
static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
Error **errp)
{
BDRVRBDState *s = bs->opaque;
const char *pool, *snap, *conf, *clientname, *name, *keypairs;
const char *secretid;
QemuOpts *opts;
Error *local_err = NULL;
char *mon_host = NULL;
int r;
opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
qemu_opts_absorb_qdict(opts, options, &local_err);
if (local_err) {
error_propagate(errp, local_err);
r = -EINVAL;
goto failed_opts;
}
mon_host = qemu_rbd_mon_host(options, &local_err);
if (local_err) {
error_propagate(errp, local_err);
r = -EINVAL;
goto failed_opts;
}
secretid = qemu_opt_get(opts, "password-secret");
pool = qemu_opt_get(opts, "pool");
conf = qemu_opt_get(opts, "conf");
snap = qemu_opt_get(opts, "snapshot");
clientname = qemu_opt_get(opts, "user");
name = qemu_opt_get(opts, "image");
keypairs = qemu_opt_get(opts, "=keyvalue-pairs");
if (!pool || !name) {
error_setg(errp, "Parameters 'pool' and 'image' are required");
r = -EINVAL;
goto failed_opts;
}
r = rados_create(&s->cluster, clientname);
if (r < 0) {
error_setg_errno(errp, -r, "error initializing");
goto failed_opts;
}
s->snap = g_strdup(snap);
s->name = g_strdup(name);
/* try default location when conf=NULL, but ignore failure */
r = rados_conf_read_file(s->cluster, conf);
if (conf && r < 0) {
error_setg_errno(errp, -r, "error reading conf file %s", conf);
goto failed_shutdown;
}
r = qemu_rbd_set_keypairs(s->cluster, keypairs, errp);
if (r < 0) {
goto failed_shutdown;
}
if (mon_host) {
r = rados_conf_set(s->cluster, "mon_host", mon_host);
if (r < 0) {
goto failed_shutdown;
}
}
if (qemu_rbd_set_auth(s->cluster, secretid, errp) < 0) {
r = -EIO;
goto failed_shutdown;
}
/*
* Fallback to more conservative semantics if setting cache
* options fails. Ignore errors from setting rbd_cache because the
* only possible error is that the option does not exist, and
* librbd defaults to no caching. If write through caching cannot
* be set up, fall back to no caching.
*/
if (flags & BDRV_O_NOCACHE) {
rados_conf_set(s->cluster, "rbd_cache", "false");
} else {
rados_conf_set(s->cluster, "rbd_cache", "true");
}
r = rados_connect(s->cluster);
if (r < 0) {
error_setg_errno(errp, -r, "error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
if (r < 0) {
error_setg_errno(errp, -r, "error opening pool %s", pool);
goto failed_shutdown;
}
r = rbd_open(s->io_ctx, s->name, &s->image, s->snap);
if (r < 0) {
error_setg_errno(errp, -r, "error reading header from %s", s->name);
goto failed_open;
}
bs->read_only = (s->snap != NULL);
qemu_opts_del(opts);
return 0;
failed_open:
rados_ioctx_destroy(s->io_ctx);
failed_shutdown:
rados_shutdown(s->cluster);
g_free(s->snap);
g_free(s->name);
failed_opts:
qemu_opts_del(opts);
g_free(mon_host);
return r;
}
static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
Error **errp)
{
BDRVRBDState *s = bs->opaque;
BlockdevOptionsRbd *opts = NULL;
const QDictEntry *e;
Error *local_err = NULL;
char *keypairs, *secretid;
int r;
keypairs = g_strdup(qdict_get_try_str(options, "=keyvalue-pairs"));
if (keypairs) {
qdict_del(options, "=keyvalue-pairs");
}
secretid = g_strdup(qdict_get_try_str(options, "password-secret"));
if (secretid) {
qdict_del(options, "password-secret");
}
r = qemu_rbd_convert_options(options, &opts, &local_err);
if (local_err) {
/* If keypairs are present, that means some options are present in
* the modern option format. Don't attempt to parse legacy option
* formats, as we won't support mixed usage. */
if (keypairs) {
error_propagate(errp, local_err);
goto out;
}
/* If the initial attempt to convert and process the options failed,
* we may be attempting to open an image file that has the rbd options
* specified in the older format consisting of all key/value pairs
* encoded in the filename. Go ahead and attempt to parse the
* filename, and see if we can pull out the required options. */
r = qemu_rbd_attempt_legacy_options(options, &opts, &keypairs);
if (r < 0) {
/* Propagate the original error, not the legacy parsing fallback
* error, as the latter was just a best-effort attempt. */
error_propagate(errp, local_err);
goto out;
}
/* Take care whenever deciding to actually deprecate; once this ability
* is removed, we will not be able to open any images with legacy-styled
* backing image strings. */
warn_report("RBD options encoded in the filename as keyvalue pairs "
"is deprecated");
}
/* Remove the processed options from the QDict (the visitor processes
* _all_ options in the QDict) */
while ((e = qdict_first(options))) {
qdict_del(options, e->key);
}
r = qemu_rbd_connect(&s->cluster, &s->io_ctx, opts,
!(flags & BDRV_O_NOCACHE), keypairs, secretid, errp);
if (r < 0) {
goto out;
}
s->snap = g_strdup(opts->snapshot);
s->image_name = g_strdup(opts->image);
/* rbd_open is always r/w */
r = rbd_open(s->io_ctx, s->image_name, &s->image, s->snap);
if (r < 0) {
error_setg_errno(errp, -r, "error reading header from %s",
s->image_name);
goto failed_open;
}
/* If we are using an rbd snapshot, we must be r/o, otherwise
* leave as-is */
if (s->snap != NULL) {
r = bdrv_apply_auto_read_only(bs, "rbd snapshots are read-only", errp);
if (r < 0) {
rbd_close(s->image);
goto failed_open;
}
}
r = 0;
goto out;
failed_open:
rados_ioctx_destroy(s->io_ctx);
g_free(s->snap);
g_free(s->image_name);
rados_shutdown(s->cluster);
out:
qapi_free_BlockdevOptionsRbd(opts);
g_free(keypairs);
g_free(secretid);
return r;
}
int main(int argc, const char **argv)
{
int ret = 0;
// we will use all of these below
const char *pool_name = "hello_world_pool";
const char* hello = "hello world!";
const char* object_name = "hello_object";
rados_ioctx_t io_ctx = NULL;
int pool_created = 0;
// first, we create a Rados object and initialize it
rados_t rados = NULL;
{
ret = rados_create(&rados, "admin"); // just use the client.admin keyring
if (ret < 0) { // let's handle any error that might have come back
printf("couldn't initialize rados! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we just set up a rados cluster object\n");
}
}
/*
* Now we need to get the rados object its config info. It can
* parse argv for us to find the id, monitors, etc, so let's just
* use that.
*/
{
ret = rados_conf_parse_argv(rados, argc, argv);
if (ret < 0) {
// This really can't happen, but we need to check to be a good citizen.
printf("failed to parse config options! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we just parsed our config options\n");
// We also want to apply the config file if the user specified
// one, and conf_parse_argv won't do that for us.
int i;
for (i = 0; i < argc; ++i) {
if ((strcmp(argv[i], "-c") == 0) || (strcmp(argv[i], "--conf") == 0)) {
ret = rados_conf_read_file(rados, argv[i+1]);
if (ret < 0) {
// This could fail if the config file is malformed, but it'd be hard.
printf("failed to parse config file %s! error %d\n", argv[i+1], ret);
ret = EXIT_FAILURE;
goto out;
}
break;
}
}
}
}
/*
* next, we actually connect to the cluster
*/
{
ret = rados_connect(rados);
if (ret < 0) {
printf("couldn't connect to cluster! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we just connected to the rados cluster\n");
}
}
/*
* let's create our own pool instead of scribbling over real data.
* Note that this command creates pools with default PG counts specified
* by the monitors, which may not be appropriate for real use -- it's fine
* for testing, though.
*/
{
ret = rados_pool_create(rados, pool_name);
if (ret < 0) {
printf("couldn't create pool! error %d\n", ret);
return EXIT_FAILURE;
} else {
printf("we just created a new pool named %s\n", pool_name);
}
pool_created = 1;
}
/*
* create an "IoCtx" which is used to do IO to a pool
*/
{
ret = rados_ioctx_create(rados, pool_name, &io_ctx);
if (ret < 0) {
printf("couldn't set up ioctx! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we just created an ioctx for our pool\n");
}
}
/*
* now let's do some IO to the pool! We'll write "hello world!" to a
* new object.
*/
{
/*
* now that we have the data to write, let's send it to an object.
* We'll use the synchronous interface for simplicity.
*/
ret = rados_write_full(io_ctx, object_name, hello, strlen(hello));
if (ret < 0) {
printf("couldn't write object! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we just wrote new object %s, with contents '%s'\n", object_name, hello);
}
}
/*
* now let's read that object back! Just for fun, we'll do it using
* async IO instead of synchronous. (This would be more useful if we
* wanted to send off multiple reads at once; see
* http://ceph.com/docs/master/rados/api/librados/#asychronous-io )
*/
{
int read_len = 4194304; // this is way more than we need
char* read_buf = malloc(read_len + 1); // add one for the terminating 0 we'll add later
if (!read_buf) {
printf("couldn't allocate read buffer\n");
ret = EXIT_FAILURE;
goto out;
}
// allocate the completion from librados
rados_completion_t read_completion;
ret = rados_aio_create_completion(NULL, NULL, NULL, &read_completion);
if (ret < 0) {
printf("couldn't create completion! error %d\n", ret);
ret = EXIT_FAILURE;
free(read_buf);
goto out;
} else {
printf("we just created a new completion\n");
}
// send off the request.
ret = rados_aio_read(io_ctx, object_name, read_completion, read_buf, read_len, 0);
if (ret < 0) {
printf("couldn't start read object! error %d\n", ret);
ret = EXIT_FAILURE;
free(read_buf);
rados_aio_release(read_completion);
goto out;
}
// wait for the request to complete, and check that it succeeded.
rados_aio_wait_for_complete(read_completion);
ret = rados_aio_get_return_value(read_completion);
if (ret < 0) {
printf("couldn't read object! error %d\n", ret);
ret = EXIT_FAILURE;
free(read_buf);
rados_aio_release(read_completion);
goto out;
} else {
read_buf[ret] = 0; // null-terminate the string
printf("we read our object %s, and got back %d bytes with contents\n%s\n", object_name, ret, read_buf);
}
free(read_buf);
rados_aio_release(read_completion);
}
/*
* We can also use xattrs that go alongside the object.
*/
{
const char* version = "1";
ret = rados_setxattr(io_ctx, object_name, "version", version, strlen(version));
if (ret < 0) {
printf("failed to set xattr version entry! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we set the xattr 'version' on our object!\n");
}
}
/*
* And if we want to be really cool, we can do multiple things in a single
* atomic operation. For instance, we can update the contents of our object
* and set the version at the same time.
*/
{
const char* content = "v2";
rados_write_op_t write_op = rados_create_write_op();
if (!write_op) {
printf("failed to allocate write op\n");
ret = EXIT_FAILURE;
goto out;
}
rados_write_op_write_full(write_op, content, strlen(content));
const char* version = "2";
rados_write_op_setxattr(write_op, "version", version, strlen(version));
ret = rados_write_op_operate(write_op, io_ctx, object_name, NULL, 0);
if (ret < 0) {
printf("failed to do compound write! error %d\n", ret);
ret = EXIT_FAILURE;
rados_release_write_op(write_op);
goto out;
} else {
printf("we overwrote our object %s with contents\n%s\n", object_name, content);
}
rados_release_write_op(write_op);
}
/*
* And to be even cooler, we can make sure that the object looks the
* way we expect before doing the write! Notice how this attempt fails
* because the xattr differs.
*/
{
rados_write_op_t failed_write_op = rados_create_write_op();
if (!failed_write_op) {
printf("failed to allocate write op\n");
ret = EXIT_FAILURE;
goto out;
}
const char* content = "v2";
const char* version = "2";
const char* old_version = "1";
rados_write_op_cmpxattr(failed_write_op, "version", LIBRADOS_CMPXATTR_OP_EQ, old_version, strlen(old_version));
rados_write_op_write_full(failed_write_op, content, strlen(content));
rados_write_op_setxattr(failed_write_op, "version", version, strlen(version));
ret = rados_write_op_operate(failed_write_op, io_ctx, object_name, NULL, 0);
if (ret < 0) {
printf("we just failed a write because the xattr wasn't as specified\n");
} else {
printf("we succeeded on writing despite an xattr comparison mismatch!\n");
ret = EXIT_FAILURE;
rados_release_write_op(failed_write_op);
goto out;
}
rados_release_write_op(failed_write_op);
/*
* Now let's do the update with the correct xattr values so it
* actually goes through
*/
content = "v3";
old_version = "2";
version = "3";
rados_write_op_t update_op = rados_create_write_op();
if (!failed_write_op) {
printf("failed to allocate write op\n");
ret = EXIT_FAILURE;
goto out;
}
rados_write_op_cmpxattr(update_op, "version", LIBRADOS_CMPXATTR_OP_EQ, old_version, strlen(old_version));
rados_write_op_write_full(update_op, content, strlen(content));
rados_write_op_setxattr(update_op, "version", version, strlen(version));
ret = rados_write_op_operate(update_op, io_ctx, object_name, NULL, 0);
if (ret < 0) {
printf("failed to do a compound write update! error %d\n", ret);
ret = EXIT_FAILURE;
rados_release_write_op(update_op);
goto out;
} else {
printf("we overwrote our object %s following an xattr test with contents\n%s\n", object_name, content);
}
rados_release_write_op(update_op);
}
ret = EXIT_SUCCESS;
out:
if (io_ctx) {
rados_ioctx_destroy(io_ctx);
}
if (pool_created) {
/*
* And now we're done, so let's remove our pool and then
* shut down the connection gracefully.
*/
int delete_ret = rados_pool_delete(rados, pool_name);
if (delete_ret < 0) {
// be careful not to
printf("We failed to delete our test pool!\n");
ret = EXIT_FAILURE;
}
}
rados_shutdown(rados);
return ret;
}
static int qemu_rbd_open(BlockDriverState *bs, const char *filename, int flags)
{
BDRVRBDState *s = bs->opaque;
char pool[RBD_MAX_POOL_NAME_SIZE];
char snap_buf[RBD_MAX_SNAP_NAME_SIZE];
char conf[RBD_MAX_CONF_SIZE];
char clientname_buf[RBD_MAX_CONF_SIZE];
char *clientname;
int r;
if (qemu_rbd_parsename(filename, pool, sizeof(pool),
snap_buf, sizeof(snap_buf),
s->name, sizeof(s->name),
conf, sizeof(conf)) < 0) {
return -EINVAL;
}
clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
r = rados_create(&s->cluster, clientname);
if (r < 0) {
error_report("error initializing");
return r;
}
s->snap = NULL;
if (snap_buf[0] != '\0') {
s->snap = g_strdup(snap_buf);
}
if (strstr(conf, "conf=") == NULL) {
/* try default location, but ignore failure */
rados_conf_read_file(s->cluster, NULL);
}
if (conf[0] != '\0') {
r = qemu_rbd_set_conf(s->cluster, conf);
if (r < 0) {
error_report("error setting config options");
goto failed_shutdown;
}
}
r = rados_connect(s->cluster);
if (r < 0) {
error_report("error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
if (r < 0) {
error_report("error opening pool %s", pool);
goto failed_shutdown;
}
r = rbd_open(s->io_ctx, s->name, &s->image, s->snap);
if (r < 0) {
error_report("error reading header from %s", s->name);
goto failed_open;
}
bs->read_only = (s->snap != NULL);
s->event_reader_pos = 0;
r = qemu_pipe(s->fds);
if (r < 0) {
error_report("error opening eventfd");
goto failed;
}
fcntl(s->fds[0], F_SETFL, O_NONBLOCK);
fcntl(s->fds[1], F_SETFL, O_NONBLOCK);
qemu_aio_set_fd_handler(s->fds[RBD_FD_READ], qemu_rbd_aio_event_reader,
NULL, qemu_rbd_aio_flush_cb, NULL, s);
return 0;
failed:
rbd_close(s->image);
failed_open:
rados_ioctx_destroy(s->io_ctx);
failed_shutdown:
rados_shutdown(s->cluster);
g_free(s->snap);
return r;
}
static void rb_rados_ioctx_free(void * ptr) {
rados_ioctx_wrapper *wrapper = (rados_ioctx_wrapper *)ptr;
rados_ioctx_destroy(wrapper->ioctx);
// rados_ioctx_destroy apparently handles freeing :/
// xfree(ptr);
}
static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
Error **errp)
{
BDRVRBDState *s = bs->opaque;
char pool[RBD_MAX_POOL_NAME_SIZE];
char snap_buf[RBD_MAX_SNAP_NAME_SIZE];
char conf[RBD_MAX_CONF_SIZE];
char clientname_buf[RBD_MAX_CONF_SIZE];
char *clientname;
QemuOpts *opts;
Error *local_err = NULL;
const char *filename;
int r;
opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
qemu_opts_absorb_qdict(opts, options, &local_err);
if (local_err) {
qerror_report_err(local_err);
error_free(local_err);
qemu_opts_del(opts);
return -EINVAL;
}
filename = qemu_opt_get(opts, "filename");
if (qemu_rbd_parsename(filename, pool, sizeof(pool),
snap_buf, sizeof(snap_buf),
s->name, sizeof(s->name),
conf, sizeof(conf)) < 0) {
r = -EINVAL;
goto failed_opts;
}
clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
r = rados_create(&s->cluster, clientname);
if (r < 0) {
error_report("error initializing");
goto failed_opts;
}
s->snap = NULL;
if (snap_buf[0] != '\0') {
s->snap = g_strdup(snap_buf);
}
/*
* Fallback to more conservative semantics if setting cache
* options fails. Ignore errors from setting rbd_cache because the
* only possible error is that the option does not exist, and
* librbd defaults to no caching. If write through caching cannot
* be set up, fall back to no caching.
*/
if (flags & BDRV_O_NOCACHE) {
rados_conf_set(s->cluster, "rbd_cache", "false");
} else {
rados_conf_set(s->cluster, "rbd_cache", "true");
}
if (strstr(conf, "conf=") == NULL) {
/* try default location, but ignore failure */
rados_conf_read_file(s->cluster, NULL);
}
if (conf[0] != '\0') {
r = qemu_rbd_set_conf(s->cluster, conf);
if (r < 0) {
error_report("error setting config options");
goto failed_shutdown;
}
}
r = rados_connect(s->cluster);
if (r < 0) {
error_report("error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
if (r < 0) {
error_report("error opening pool %s", pool);
goto failed_shutdown;
}
r = rbd_open(s->io_ctx, s->name, &s->image, s->snap);
if (r < 0) {
error_report("error reading header from %s", s->name);
goto failed_open;
}
bs->read_only = (s->snap != NULL);
qemu_opts_del(opts);
return 0;
failed_open:
rados_ioctx_destroy(s->io_ctx);
failed_shutdown:
rados_shutdown(s->cluster);
g_free(s->snap);
failed_opts:
qemu_opts_del(opts);
return r;
}
static int qemu_rbd_create(const char *filename, QEMUOptionParameter *options)
{
int64_t bytes = 0;
int64_t objsize;
int obj_order = 0;
char pool[RBD_MAX_POOL_NAME_SIZE];
char name[RBD_MAX_IMAGE_NAME_SIZE];
char snap_buf[RBD_MAX_SNAP_NAME_SIZE];
char conf[RBD_MAX_CONF_SIZE];
char clientname_buf[RBD_MAX_CONF_SIZE];
char *clientname;
rados_t cluster;
rados_ioctx_t io_ctx;
int ret;
if (qemu_rbd_parsename(filename, pool, sizeof(pool),
snap_buf, sizeof(snap_buf),
name, sizeof(name),
conf, sizeof(conf)) < 0) {
return -EINVAL;
}
/* Read out options */
while (options && options->name) {
if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
bytes = options->value.n;
} else if (!strcmp(options->name, BLOCK_OPT_CLUSTER_SIZE)) {
if (options->value.n) {
objsize = options->value.n;
if ((objsize - 1) & objsize) { /* not a power of 2? */
error_report("obj size needs to be power of 2");
return -EINVAL;
}
if (objsize < 4096) {
error_report("obj size too small");
return -EINVAL;
}
obj_order = ffs(objsize) - 1;
}
}
options++;
}
clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
if (rados_create(&cluster, clientname) < 0) {
error_report("error initializing");
return -EIO;
}
if (strstr(conf, "conf=") == NULL) {
/* try default location, but ignore failure */
rados_conf_read_file(cluster, NULL);
}
if (conf[0] != '\0' &&
qemu_rbd_set_conf(cluster, conf) < 0) {
error_report("error setting config options");
rados_shutdown(cluster);
return -EIO;
}
if (rados_connect(cluster) < 0) {
error_report("error connecting");
rados_shutdown(cluster);
return -EIO;
}
if (rados_ioctx_create(cluster, pool, &io_ctx) < 0) {
error_report("error opening pool %s", pool);
rados_shutdown(cluster);
return -EIO;
}
ret = rbd_create(io_ctx, name, bytes, &obj_order);
rados_ioctx_destroy(io_ctx);
rados_shutdown(cluster);
return ret;
}
int main()
{
int ret,len = 0;
rados_t cluster;
char *buf, *out_buf = NULL;
char *pool = "rbd";
char *rbd_name = "vol1";
int fd = -1;
int i = 0;
rados_completion_t read_comp, write_comp;
rbd_image_t rbd_image;
time_t start_t, end_t, total_t = 0;
unsigned char *cipher_key = (unsigned char *) "7190c8bc27ac4a1bbe1ab1cf55cf3b097190c8bc27ac4a1bbe1ab1cf55cf3b09";
unsigned char *iv = (unsigned char *) "dcbfdd41e40f74a2";
ERR_load_crypto_strings();
OpenSSL_add_all_algorithms();
ret = rados_create(&cluster, NULL);
if (ret) {
printf("error creating rados_t object\n");
return -1;
}
ret = rados_conf_read_file(cluster,"/home/shishir/repos/ceph/src/ceph.conf");
if (ret) {
printf("Error reading conf files \n");
return -1;
}
ret = rados_connect(cluster);
if (ret) {
printf("Could not connect to cluster\n");
return -1;
}
rados_ioctx_t io;
ret = rados_ioctx_create(cluster, pool, &io);
if (ret) {
printf("Could not open connection to cluster\n");
return -1;
}
ret = rbd_open(io, rbd_name, &rbd_image, NULL);
if (ret) {
printf("Failed to open image %s\n",rbd_name);
return -1;
}
posix_memalign((void**)&buf, 4096, 65536);
if (!buf) {
printf("Failed to malloc\n");
return -1;
}
fd = open("/tmp/input_file", O_RDONLY);
if (fd > 0) {
read(fd, buf, 65536);
close(fd);
}
posix_memalign((void**)&out_buf, 4096, 65536);
if (!out_buf) {
printf("Failed to malloc\n");
return -1;
}
len = strlen((char*)buf);
//printf("plain text length %lld\n",strlen((char*)buf));
//printf("===========plain text========\n");
//BIO_dump_fp(stdout,(const char*)buf,len);
//printf("=================\n");
start_t = clock();
len = encrypt (buf, strlen((char*)buf), cipher_key, iv, out_buf);
end_t = clock();
total_t = (double)(end_t - start_t)/ CLOCKS_PER_SEC;
//printf("encrypted len is %d timetaken is %fsec\n",len,total_t);
//printf("===========encrypted text========\n");
//BIO_dump_fp(stdout,(const char*)out_buf,len);
//printf("=================\n");
fd = open("/tmp/encrypted_file", O_CREAT|O_RDWR);
if (fd > 0) {
write(fd, out_buf, 65536);
close(fd);
}
ret = rbd_aio_create_completion(NULL, NULL, &write_comp);
if (ret) {
printf ("Failed to create aio completion\n");
return -1;
}
ret = rbd_aio_write(rbd_image, 0, 65536, out_buf, write_comp);
if (ret < 0) {
printf ("Failed to write aio completion\n");
return -1;
}
rbd_aio_wait_for_complete(write_comp);
rbd_aio_release(write_comp);
ret = rbd_aio_create_completion(NULL, NULL, &read_comp);
if (ret) {
printf ("Failed to create aio completion\n");
return -1;
}
memset(buf, 0, 65536);
memset(out_buf, 0, 65536);
ret = rbd_aio_read(rbd_image, 0, 65536, buf, read_comp);
if (ret < 0) {
printf ("Failed to read aio completion\n");
return -1;
}
rbd_aio_wait_for_complete(read_comp);
rbd_aio_release(read_comp);
len=65536;
start_t = clock();
len = decrypt(buf, len, cipher_key, iv, out_buf);
end_t = clock();
total_t = (double)(end_t - start_t)/ CLOCKS_PER_SEC;
//printf("decrypted len is %d time taken: %fs\n",len,total_t);
//printf("===========decrypted text========\n");
//BIO_dump_fp(stdout,(const char*)out_buf,len);
//printf("=================\n");
fd = open("/tmp/decrypted_file", O_CREAT|O_RDWR);
if (fd > 0) {
write(fd, out_buf, 65536);
close(fd);
}
rados_ioctx_destroy(io);
rados_shutdown(cluster);
free(buf);
free(out_buf);
return 0;
}
static void app_rbd_close()
{
rados_ioctx_destroy(io_ctx);
rados_shutdown(cluster);
}
static void ctdb_mutex_rados_ctx_destroy(rados_t ceph_cluster,
rados_ioctx_t ioctx)
{
rados_ioctx_destroy(ioctx);
rados_shutdown(ceph_cluster);
}
