int main(int argc, const char **argv)
{
rados_t cluster;
rados_ioctx_t io_ctx;
rbd_image_t image;
srand(time(0));
assert(rados_create(&cluster, NULL) == 0);
assert(rados_conf_parse_argv(cluster, argc, argv) == 0);
assert(rados_conf_read_file(cluster, NULL) == 0);
assert(rados_connect(cluster) == 0);
if (rados_pool_lookup(cluster, TEST_POOL) != -ENOENT) {
int r = rados_pool_delete(cluster, TEST_POOL);
printf("rados_pool_delete returned %d\n", r);
}
int r = rados_pool_create(cluster, TEST_POOL);
printf("rados_pool_create returned %d\n", r);
assert(rados_ioctx_create(cluster, TEST_POOL, &io_ctx) == 0);
test_ls(io_ctx, 0);
test_create_and_stat(io_ctx, TEST_IMAGE, MB_BYTES(1));
assert(rbd_open(io_ctx, TEST_IMAGE, &image, NULL) == 0);
test_ls(io_ctx, 1, TEST_IMAGE);
test_ls_snaps(image, 0);
test_create_snap(image, TEST_SNAP);
test_ls_snaps(image, 1, TEST_SNAP, MB_BYTES(1));
test_resize_and_stat(image, MB_BYTES(2));
test_io(io_ctx, image);
test_create_snap(image, TEST_SNAP "1");
test_ls_snaps(image, 2, TEST_SNAP, MB_BYTES(1), TEST_SNAP "1", MB_BYTES(2));
test_delete_snap(image, TEST_SNAP);
test_ls_snaps(image, 1, TEST_SNAP "1", MB_BYTES(2));
test_delete_snap(image, TEST_SNAP "1");
test_ls_snaps(image, 0);
test_io_to_snapshot(io_ctx, image, MB_BYTES(2));
assert(rbd_close(image) == 0);
test_create_and_stat(io_ctx, TEST_IMAGE "1", MB_BYTES(2));
test_ls(io_ctx, 2, TEST_IMAGE, TEST_IMAGE "1");
test_delete(io_ctx, TEST_IMAGE);
test_ls(io_ctx, 1, TEST_IMAGE "1");
test_delete(io_ctx, TEST_IMAGE "1");
test_ls(io_ctx, 0);
rados_ioctx_destroy(io_ctx);
rados_shutdown(cluster);
return 0;
}
static int virStorageBackendRBDRefreshVol(virConnectPtr conn,
virStoragePoolObjPtr pool ATTRIBUTE_UNUSED,
virStorageVolDefPtr vol)
{
virStorageBackendRBDStatePtr ptr;
ptr.cluster = NULL;
ptr.ioctx = NULL;
int ret = -1;
if (virStorageBackendRBDOpenRADOSConn(&ptr, conn, pool) < 0) {
goto cleanup;
}
if (rados_ioctx_create(ptr.cluster,
pool->def->source.name, &ptr.ioctx) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to create the RBD IoCTX. Does the pool '%s' exist?"),
pool->def->source.name);
goto cleanup;
}
if (volStorageBackendRBDRefreshVolInfo(vol, pool, ptr) < 0) {
goto cleanup;
}
ret = 0;
cleanup:
virStorageBackendRBDCloseRADOSConn(ptr);
return ret;
}
void *
rbdfs_init(struct fuse_conn_info *conn)
{
int ret;
// init cannot fail, so if we fail here, gotrados remains at 0,
// causing other operations to fail immediately with ENXIO
ret = connect_to_cluster(&cluster);
if (ret < 0)
exit(90);
pool_name = rbd_options.pool_name;
mount_image_name = rbd_options.image_name;
ret = rados_ioctx_create(cluster, pool_name, &ioctx);
if (ret < 0)
exit(91);
#if FUSE_VERSION >= FUSE_MAKE_VERSION(2, 8)
conn->want |= FUSE_CAP_BIG_WRITES;
#endif
gotrados = 1;
// init's return value shows up in fuse_context.private_data,
// also to void (*destroy)(void *); useful?
return NULL;
}
static void uwsgi_rados_add_mountpoint(char *arg, size_t arg_len) {
char *rad_mountpoint = NULL;
char *rad_config = NULL;
char *rad_poolname = NULL;
if (uwsgi_kvlist_parse(arg, arg_len, ',', '=',
"mountpoint", &rad_mountpoint,
"config", &rad_config,
"pool", &rad_poolname,
NULL)) {
uwsgi_log("unable to parse rados mountpoint definition\n");
exit(1);
}
if (!rad_mountpoint|| !rad_poolname) {
uwsgi_log("[rados] mount requires a mountpoint, and a pool name.\n");
exit(1);
}
time_t now = uwsgi_now();
uwsgi_log("[rados] mounting %s ...\n", rad_mountpoint);
rados_t cluster;
if (rados_create(&cluster, NULL) < 0) {
uwsgi_error("Can't create Ceph cluster handle");
exit(1);
}
if (rad_config)
uwsgi_log("Using Ceph conf:%s\n", rad_config);
else
uwsgi_log("Using default Ceph conf.\n");
if (rados_conf_read_file(cluster, rad_config) < 0) {
uwsgi_error("Can't configure Ceph cluster handle");
exit(1);
}
if (rados_connect(cluster) < 0) {
uwsgi_error("Can't connect with Ceph cluster");
exit(1);
}
rados_ioctx_t ctx;
uwsgi_log("Ceph pool: %s\n", rad_poolname);
if (rados_ioctx_create(cluster, rad_poolname, &ctx) < 0) {
uwsgi_error("Can't open rados pool")
rados_shutdown(cluster);
exit(1);
}
int id = uwsgi_apps_cnt;
struct uwsgi_app *ua = uwsgi_add_app(id, rados_plugin.modifier1, rad_mountpoint, strlen(rad_mountpoint), NULL, NULL);
if (!ua) {
uwsgi_log("[rados] unable to mount %s\n", rad_mountpoint);
rados_shutdown(cluster);
exit(1);
}
ua->responder0 = cluster;
ua->responder1 = ctx;
ua->started_at = now;
ua->startup_time = uwsgi_now() - now;
uwsgi_log("Rados app/mountpoint %d (%s) loaded in %d seconds at %p\n", id, rad_mountpoint, (int) ua->startup_time, ctx);
}
int
create_image()
{
int r;
int order = 0;
r = rados_create(&cluster, NULL);
if (r < 0) {
simple_err("Could not create cluster handle", r);
return r;
}
rados_conf_parse_env(cluster, NULL);
r = rados_conf_read_file(cluster, NULL);
if (r < 0) {
simple_err("Error reading ceph config file", r);
goto failed_shutdown;
}
r = rados_connect(cluster);
if (r < 0) {
simple_err("Error connecting to cluster", r);
goto failed_shutdown;
}
r = krbd_create_from_context(rados_cct(cluster), &krbd);
if (r < 0) {
simple_err("Could not create libkrbd handle", r);
goto failed_shutdown;
}
r = rados_pool_create(cluster, pool);
if (r < 0 && r != -EEXIST) {
simple_err("Error creating pool", r);
goto failed_krbd;
}
r = rados_ioctx_create(cluster, pool, &ioctx);
if (r < 0) {
simple_err("Error creating ioctx", r);
goto failed_krbd;
}
if (clone_calls) {
r = rbd_create2(ioctx, iname, 0, RBD_FEATURE_LAYERING, &order);
} else {
r = rbd_create(ioctx, iname, 0, &order);
}
if (r < 0) {
simple_err("Error creating image", r);
goto failed_open;
}
return 0;
failed_open:
rados_ioctx_destroy(ioctx);
failed_krbd:
krbd_destroy(krbd);
failed_shutdown:
rados_shutdown(cluster);
return r;
}
static int virStorageBackendRBDOpenIoCTX(virStorageBackendRBDStatePtr ptr, virStoragePoolObjPtr pool)
{
int r = rados_ioctx_create(ptr->cluster, pool->def->source.name, &ptr->ioctx);
if (r < 0) {
virReportSystemError(-r, _("failed to create the RBD IoCTX. Does the pool '%s' exist?"),
pool->def->source.name);
}
return r;
}
/*
* Connect via RADOS protocol to a CEPH cluster.
*/
static bRC connect_to_rados(bpContext *ctx)
{
int status;
plugin_ctx *p_ctx = (plugin_ctx *)ctx->pContext;
/*
* See if we need to initialize the cluster connection.
*/
if (!p_ctx->cluster_initialized) {
status = rados_create(&p_ctx->cluster, NULL);
if (status < 0) {
berrno be;
Jmsg(ctx, M_ERROR, "rados_create() failed: %s\n", be.bstrerror(-status));
return bRC_Error;
}
status = rados_conf_read_file(p_ctx->cluster, p_ctx->rados_conffile);
if (status < 0) {
berrno be;
Jmsg(ctx, M_ERROR, "rados_conf_read_file(%s) failed: %s\n",
p_ctx->rados_conffile, be.bstrerror(-status));
return bRC_Error;
}
status = rados_connect(p_ctx->cluster);
if (status < 0) {
berrno be;
Jmsg(ctx, M_ERROR, "rados_connect() failed: %s\n", be.bstrerror(-status));
rados_shutdown(p_ctx->cluster);
return bRC_Error;
}
p_ctx->cluster_initialized = true;
}
/*
* See if we need to initialize the IO context.
*/
if (!p_ctx->ioctx) {
status = rados_ioctx_create(p_ctx->cluster, p_ctx->rados_poolname, &p_ctx->ioctx);
if (status < 0) {
berrno be;
Jmsg(ctx, M_ERROR, "rados_ioctx_create(%s) failed: %s\n",
p_ctx->rados_poolname, be.bstrerror(-status));
rados_shutdown(p_ctx->cluster);
p_ctx->cluster_initialized = false;
return bRC_Error;
}
}
return bRC_OK;
}
int main(int argc, char **argv)
{
rados_t cluster;
char cluster_name[] = "ceph";
char user_name[] = "client.admin";
APPNAME = &argv[0][2];
uint64_t flags;
int err;
/* Initialize the cluster handle */
err = rados_create2(&cluster, cluster_name, user_name, flags);
// err = rados_create(&cluster, NULL);
error_print(err, "Couldn't create the cluster handle!", "Created a cluster handle.");
/* Read a Ceph configuration file to configure the cluster handle. */
err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
error_print(err, "Cannot read config file!", "Read the config file.");
/* Connect to the cluster */
err = rados_connect(cluster);
error_print(err, "Cannot connect to cluster!", "Connected to the cluster.");
/* List pools. */
char buf[1024];
err = rados_pool_list(cluster, buf, 1024);
split_str(buf, 1024);
error_print(err, "Cannot list pools!", buf);
/* List pool stats */
char *poolname = strtok(buf, ",");
while(poolname != NULL)
{
rados_ioctx_t io;
struct rados_pool_stat_t stats;
err = rados_ioctx_create(cluster, poolname, &io);
error_print(err, "Cannot create IO context!", "IO context created.");
err = rados_ioctx_pool_stat(io, &stats);
error_print(err, "Cannot list pool stats!", "Pool stats listed.");
printf("[%s][%s]: Number of read: %"PRIu64"\n", APPNAME, poolname, stats.num_rd);
printf("[%s][%s]: Number of read in kb: %"PRIu64"\n", APPNAME, poolname, stats.num_rd_kb);
printf("[%s][%s]: Number of write: %"PRIu64"\n", APPNAME, poolname, stats.num_wr);
printf("[%s][%s]: Number of write in kb: %"PRIu64"\n", APPNAME, poolname, stats.num_wr_kb);
rados_ioctx_destroy(io);
poolname = strtok(NULL, ",");
}
/* Shut down the cluster */
rados_shutdown(cluster);
printf("[%s]: Cluster shut down.\n", APPNAME);
}
int main(int argc, char* argv[])
{
int err;
rados_t cluster;
rados_ioctx_t io;
rbd_image_t image;
char *poolname = "pool100";
char buf[IMAGE_BUF_SIZE] = {0};
err = rados_create(&cluster, NULL);
if (err < 0) {
fprintf(stderr, "%s: cannot create a cluster handle: %s\n", argv[0], strerror(-err));
exit(1);
}
err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
if (err < 0) {
fprintf(stderr, "%s: cannot read config file: %s\n", argv[0], strerror(-err));
exit(1);
}
err = rados_connect(cluster);
if (err < 0) {
fprintf(stderr, "%s: cannot connect to cluster: %s\n", argv[0], strerror(-err));
exit(1);
}
err = rados_ioctx_create(cluster, poolname, &io);
if (err < 0) {
fprintf(stderr, "%s: cannot open rados pool %s: %s\n", argv[0], poolname, strerror(-err));
rados_shutdown(cluster);
exit(1);
}
err = rbd_open(io, "user1_image2", &image, NULL);
if (err < 0){
fprintf(stderr, "open image failed: %s\n", strerror(-err));
goto out;
}
err = rbd_read(image, IMAGE_BUF_SIZE*2, IMAGE_BUF_SIZE, buf);
if (err < 0) {
fprintf(stderr, "%s: cannot read image: %s\n", poolname, strerror(-err));
}else{
fprintf(stderr, "read image return :%d\n", err);
}
out:
rados_ioctx_destroy(io);
rados_shutdown(cluster);
return 0;
}
static int bs_rbd_open(struct scsi_lu *lu, char *path, int *fd, uint64_t *size)
{
uint32_t blksize = 0;
int ret;
rbd_image_info_t inf;
char *poolname;
char *imagename;
char *snapname;
struct active_rbd *rbd = NULL;
int lfd;
parse_imagepath(path, &poolname, &imagename, &snapname);
for (lfd = 0; lfd < MAX_IMAGES; lfd++) {
if (active_rbds[lfd].rbd_image == NULL) {
rbd = &active_rbds[lfd];
*fd = lfd;
break;
}
}
if (!rbd) {
*fd = -1;
return -EMFILE;
}
rbd->poolname = poolname;
rbd->imagename = imagename;
rbd->snapname = snapname;
eprintf("bs_rbd_open: pool: %s image: %s snap: %s\n",
poolname, imagename, snapname);
if ((ret == rados_ioctx_create(cluster, poolname, &rbd->ioctx)) < 0) {
eprintf("bs_rbd_open: rados_ioctx_create: %d\n", ret);
return -EIO;
}
/* null snap name */
ret = rbd_open(rbd->ioctx, imagename, &rbd->rbd_image, snapname);
if (ret < 0) {
eprintf("bs_rbd_open: rbd_open: %d\n", ret);
return ret;
}
if (rbd_stat(rbd->rbd_image, &inf, sizeof(inf)) < 0) {
eprintf("bs_rbd_open: rbd_stat: %d\n", ret);
return ret;
}
*size = inf.size;
blksize = inf.obj_size;
if (!lu->attrs.no_auto_lbppbe)
update_lbppbe(lu, blksize);
return 0;
}
static int virStorageBackendRBDCreateVol(virConnectPtr conn,
virStoragePoolObjPtr pool,
virStorageVolDefPtr vol)
{
virStorageBackendRBDStatePtr ptr;
ptr.cluster = NULL;
ptr.ioctx = NULL;
int order = 0;
int ret = -1;
VIR_DEBUG("Creating RBD image %s/%s with size %llu",
pool->def->source.name,
vol->name, vol->capacity);
if (virStorageBackendRBDOpenRADOSConn(&ptr, conn, pool) < 0) {
goto cleanup;
}
if (rados_ioctx_create(ptr.cluster,
pool->def->source.name,&ptr.ioctx) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to create the RBD IoCTX. Does the pool '%s' exist?"),
pool->def->source.name);
goto cleanup;
}
if (vol->target.encryption != NULL) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("storage pool does not support encrypted volumes"));
goto cleanup;
}
if (rbd_create(ptr.ioctx, vol->name, vol->capacity, &order) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to create volume '%s/%s'"),
pool->def->source.name,
vol->name);
goto cleanup;
}
if (volStorageBackendRBDRefreshVolInfo(vol, pool, ptr) < 0) {
goto cleanup;
}
ret = 0;
cleanup:
virStorageBackendRBDCloseRADOSConn(ptr);
return ret;
}
static int ctdb_mutex_rados_ctx_create(const char *ceph_cluster_name,
const char *ceph_auth_name,
const char *pool_name,
rados_t *_ceph_cluster,
rados_ioctx_t *_ioctx)
{
rados_t ceph_cluster = NULL;
rados_ioctx_t ioctx = NULL;
int ret;
ret = rados_create2(&ceph_cluster, ceph_cluster_name, ceph_auth_name, 0);
if (ret < 0) {
fprintf(stderr, "%s: failed to initialise Ceph cluster %s as %s"
" - (%s)\n", progname, ceph_cluster_name, ceph_auth_name,
strerror(-ret));
return ret;
}
/* path=NULL tells librados to use default locations */
ret = rados_conf_read_file(ceph_cluster, NULL);
if (ret < 0) {
fprintf(stderr, "%s: failed to parse Ceph cluster config"
" - (%s)\n", progname, strerror(-ret));
rados_shutdown(ceph_cluster);
return ret;
}
ret = rados_connect(ceph_cluster);
if (ret < 0) {
fprintf(stderr, "%s: failed to connect to Ceph cluster %s as %s"
" - (%s)\n", progname, ceph_cluster_name, ceph_auth_name,
strerror(-ret));
rados_shutdown(ceph_cluster);
return ret;
}
ret = rados_ioctx_create(ceph_cluster, pool_name, &ioctx);
if (ret < 0) {
fprintf(stderr, "%s: failed to create Ceph ioctx for pool %s"
" - (%s)\n", progname, pool_name, strerror(-ret));
rados_shutdown(ceph_cluster);
return ret;
}
*_ceph_cluster = ceph_cluster;
*_ioctx = ioctx;
return 0;
}
static int virStorageBackendRBDResizeVol(virConnectPtr conn ATTRIBUTE_UNUSED,
virStoragePoolObjPtr pool ATTRIBUTE_UNUSED,
virStorageVolDefPtr vol,
unsigned long long capacity,
unsigned int flags)
{
virStorageBackendRBDStatePtr ptr;
ptr.cluster = NULL;
ptr.ioctx = NULL;
rbd_image_t image = NULL;
int ret = -1;
virCheckFlags(0, -1);
if (virStorageBackendRBDOpenRADOSConn(&ptr, conn, pool) < 0) {
goto cleanup;
}
if (rados_ioctx_create(ptr.cluster,
pool->def->source.name, &ptr.ioctx) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to create the RBD IoCTX. Does the pool '%s' exist?"),
pool->def->source.name);
goto cleanup;
}
if (rbd_open(ptr.ioctx, vol->name, &image, NULL) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to open the RBD image '%s'"),
vol->name);
goto cleanup;
}
if (rbd_resize(image, capacity) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to resize the RBD image '%s'"),
vol->name);
goto cleanup;
}
ret = 0;
cleanup:
if (image != NULL)
rbd_close(image);
virStorageBackendRBDCloseRADOSConn(ptr);
return ret;
}
VALUE rb_rados_ioctx_initialize(VALUE self, VALUE cluster, VALUE pool_name) {
GET_IOCTX(self);
int err;
rados_cluster_wrapper *cwrapper;
Check_Type(pool_name, T_STRING);
char *cpool_name = StringValuePtr(pool_name);
// FIXME: Check type of cluster
wrapper->cluster = cluster;
Data_Get_Struct(cluster, rados_cluster_wrapper, cwrapper);
// FIXME: Check cluster is initialized/connected?
err = rados_ioctx_create(*cwrapper->cluster, cpool_name, wrapper->ioctx);
if (err < 0) {
rb_raise(cRadosError, "error creating IoContext for pool '%s': %s", cpool_name, strerror(-err));
}
return Qtrue;
}
int
create_image()
{
int r;
rados_t cluster;
int order = 0;
r = rados_create(&cluster, NULL);
if (r < 0) {
simple_err("Could not create cluster handle", r);
return r;
}
rados_conf_parse_env(cluster, NULL);
r = rados_conf_read_file(cluster, NULL);
if (r < 0) {
simple_err("Error reading ceph config file", r);
goto failed_shutdown;
}
r = rados_connect(cluster);
if (r < 0) {
simple_err("Error connecting to cluster", r);
goto failed_shutdown;
}
r = rados_pool_create(cluster, pool);
if (r < 0 && r != -EEXIST) {
simple_err("Error creating pool", r);
goto failed_shutdown;
}
r = rados_ioctx_create(cluster, pool, &ioctx);
if (r < 0) {
simple_err("Error creating ioctx", r);
goto failed_shutdown;
}
r = rbd_create(ioctx, iname, 0, &order);
if (r < 0) {
simple_err("Error creating image", r);
goto failed_open;
}
return 0;
failed_open:
rados_ioctx_destroy(ioctx);
failed_shutdown:
rados_shutdown(cluster);
return r;
}
static int virStorageBackendRBDDeleteVol(virConnectPtr conn,
virStoragePoolObjPtr pool,
virStorageVolDefPtr vol,
unsigned int flags)
{
int ret = -1;
virStorageBackendRBDStatePtr ptr;
ptr.cluster = NULL;
ptr.ioctx = NULL;
VIR_DEBUG("Removing RBD image %s/%s", pool->def->source.name, vol->name);
if (flags & VIR_STORAGE_VOL_DELETE_ZEROED) {
VIR_WARN("%s", _("This storage backend does not supported zeroed removal of volumes"));
}
if (virStorageBackendRBDOpenRADOSConn(&ptr, conn, pool) < 0) {
goto cleanup;
}
if (rados_ioctx_create(ptr.cluster,
pool->def->source.name, &ptr.ioctx) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to create the RBD IoCTX. Does the pool '%s' exist?"),
pool->def->source.name);
goto cleanup;
}
if (rbd_remove(ptr.ioctx, vol->name) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to remove volume '%s/%s'"),
pool->def->source.name,
vol->name);
goto cleanup;
}
ret = 0;
cleanup:
virStorageBackendRBDCloseRADOSConn(ptr);
return ret;
}
static VALUE rb_rados_cluster_pool_objects_each(VALUE self, VALUE pool_name) {
GET_CLUSTER(self);
int err, i;
Check_Type(pool_name, T_STRING);
char *cpool_name = StringValuePtr(pool_name);
// Seems like assuming malloc succeeded is all the rage
struct nogvl_objects_list_args *args = xmalloc(sizeof(struct nogvl_objects_list_args));
err = rados_ioctx_create(*wrapper->cluster, cpool_name, &args->ioctx);
if (err < 0) {
xfree(args);
rb_raise(rb_const_get(mRados, rb_intern("Error")), "error creating context for pool '%s': %s", cpool_name, strerror(-err));
}
err = rados_objects_list_open(args->ioctx, &args->lctx);
if (err < 0) {
rados_ioctx_destroy(args->ioctx);
xfree(args);
rb_raise(rb_const_get(mRados, rb_intern("PoolError")), "error listing objects for pool '%s': %s", cpool_name, strerror(-err));
}
for (;;) {
err = (int)rb_thread_blocking_region(nogvl_objects_list_next_batch, args, NULL, NULL);
for (i = 0; ; i++) {
if (args->oids[i] == NULL)
break;
rb_yield(rb_str_new2(args->oids[i]));
}
if (err == -ENOENT)
break;
if (err < 0) {
rados_ioctx_destroy(args->ioctx);
rados_objects_list_close(args->lctx);
xfree(args);
rb_raise(rb_const_get(mRados, rb_intern("PoolError")), "error listing objects for pool '%s': %s", cpool_name, strerror(-err));
}
}
rados_objects_list_close(args->lctx);
rados_ioctx_destroy(args->ioctx);
xfree(args);
return Qtrue;
}
static int virStorageBackendRBDRefreshPool(virConnectPtr conn ATTRIBUTE_UNUSED,
virStoragePoolObjPtr pool)
{
size_t max_size = 1024;
int ret = -1;
int len = -1;
int i;
char *name, *names = NULL;
virStorageBackendRBDStatePtr ptr;
ptr.cluster = NULL;
ptr.ioctx = NULL;
if (virStorageBackendRBDOpenRADOSConn(&ptr, conn, pool) < 0) {
goto cleanup;
}
if (rados_ioctx_create(ptr.cluster,
pool->def->source.name, &ptr.ioctx) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to create the RBD IoCTX. Does the pool '%s' exist?"),
pool->def->source.name);
goto cleanup;
}
struct rados_cluster_stat_t stat;
if (rados_cluster_stat(ptr.cluster, &stat) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("failed to stat the RADOS cluster"));
goto cleanup;
}
struct rados_pool_stat_t poolstat;
if (rados_ioctx_pool_stat(ptr.ioctx, &poolstat) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to stat the RADOS pool '%s'"),
pool->def->source.name);
goto cleanup;
}
pool->def->capacity = stat.kb * 1024;
pool->def->available = stat.kb_avail * 1024;
pool->def->allocation = poolstat.num_bytes;
VIR_DEBUG("Utilization of RBD pool %s: (kb: %llu kb_avail: %llu num_bytes: %llu)",
pool->def->source.name, (unsigned long long)stat.kb,
(unsigned long long)stat.kb_avail,
(unsigned long long)poolstat.num_bytes);
while (true) {
if (VIR_ALLOC_N(names, max_size) < 0)
goto out_of_memory;
len = rbd_list(ptr.ioctx, names, &max_size);
if (len >= 0)
break;
if (len != -ERANGE) {
VIR_WARN("%s", _("A problem occurred while listing RBD images"));
goto cleanup;
}
VIR_FREE(names);
}
for (i = 0, name = names; name < names + max_size; i++) {
virStorageVolDefPtr vol;
if (VIR_REALLOC_N(pool->volumes.objs, pool->volumes.count + 1) < 0) {
virStoragePoolObjClearVols(pool);
goto out_of_memory;
}
if (STREQ(name, ""))
break;
if (VIR_ALLOC(vol) < 0)
goto out_of_memory;
vol->name = strdup(name);
if (vol->name == NULL) {
VIR_FREE(vol);
goto out_of_memory;
}
name += strlen(name) + 1;
if (volStorageBackendRBDRefreshVolInfo(vol, pool, ptr) < 0) {
virStorageVolDefFree(vol);
goto cleanup;
}
pool->volumes.objs[pool->volumes.count++] = vol;
}
VIR_DEBUG("Found %d images in RBD pool %s",
pool->volumes.count, pool->def->source.name);
ret = 0;
cleanup:
VIR_FREE(names);
virStorageBackendRBDCloseRADOSConn(ptr);
return ret;
out_of_memory:
virReportOOMError();
goto cleanup;
}
static int testrados(void)
{
char tmp[32];
int i, r;
rados_t cl;
if (rados_create(&cl, NULL) < 0) {
printf("error initializing\n");
return 1;
}
if (rados_conf_read_file(cl, NULL)) {
printf("error reading configuration file\n");
return 1;
}
// Try to set a configuration option that doesn't exist.
// This should fail.
if (!rados_conf_set(cl, "config option that doesn't exist",
"some random value")) {
printf("error: succeeded in setting nonexistent config option\n");
return 1;
}
if (rados_conf_get(cl, "log to stderr", tmp, sizeof(tmp))) {
printf("error: failed to read log_to_stderr from config\n");
return 1;
}
// Can we change it?
if (rados_conf_set(cl, "log to stderr", "2")) {
printf("error: error setting log_to_stderr\n");
return 1;
}
if (rados_conf_get(cl, "log to stderr", tmp, sizeof(tmp))) {
printf("error: failed to read log_to_stderr from config\n");
return 1;
}
if (tmp[0] != '2') {
printf("error: new setting for log_to_stderr failed to take effect.\n");
return 1;
}
if (rados_connect(cl)) {
printf("error connecting\n");
return 1;
}
if (rados_connect(cl) == 0) {
printf("second connect attempt didn't return an error\n");
return 1;
}
/* create an io_ctx */
r = rados_pool_create(cl, "foo");
printf("rados_pool_create = %d\n", r);
rados_ioctx_t io_ctx;
r = rados_ioctx_create(cl, "foo", &io_ctx);
printf("rados_ioctx_create = %d, io_ctx = %p\n", r, io_ctx);
/* list all pools */
{
int buf_sz = rados_pool_list(cl, NULL, 0);
printf("need buffer size of %d\n", buf_sz);
char buf[buf_sz];
int r = rados_pool_list(cl, buf, buf_sz);
if (r != buf_sz) {
printf("buffer size mismatch: got %d the first time, but %d "
"the second.\n", buf_sz, r);
return 1;
}
const char *b = buf;
printf("begin pools.\n");
while (1) {
if (b[0] == '\0')
break;
printf(" pool: '%s'\n", b);
b += strlen(b) + 1;
};
printf("end pools.\n");
}
/* stat */
struct rados_pool_stat_t st;
r = rados_ioctx_pool_stat(io_ctx, &st);
printf("rados_ioctx_pool_stat = %d, %lld KB, %lld objects\n", r, (long long)st.num_kb, (long long)st.num_objects);
/* snapshots */
r = rados_ioctx_snap_create(io_ctx, "snap1");
printf("rados_ioctx_snap_create snap1 = %d\n", r);
rados_snap_t snaps[10];
r = rados_ioctx_snap_list(io_ctx, snaps, 10);
for (i=0; i<r; i++) {
char name[100];
rados_ioctx_snap_get_name(io_ctx, snaps[i], name, sizeof(name));
printf("rados_ioctx_snap_list got snap %lld %s\n", (long long)snaps[i], name);
}
rados_snap_t snapid;
r = rados_ioctx_snap_lookup(io_ctx, "snap1", &snapid);
printf("rados_ioctx_snap_lookup snap1 got %lld, result %d\n", (long long)snapid, r);
r = rados_ioctx_snap_remove(io_ctx, "snap1");
printf("rados_ioctx_snap_remove snap1 = %d\n", r);
/* sync io */
time_t tm;
char buf[128], buf2[128];
time(&tm);
snprintf(buf, 128, "%s", ctime(&tm));
const char *oid = "foo_object";
r = rados_write(io_ctx, oid, buf, strlen(buf) + 1, 0);
printf("rados_write = %d\n", r);
r = rados_read(io_ctx, oid, buf2, sizeof(buf2), 0);
printf("rados_read = %d\n", r);
if (memcmp(buf, buf2, r))
printf("*** content mismatch ***\n");
/* attrs */
if (do_rados_setxattr(io_ctx, oid, "b", "2"))
return 1;
if (do_rados_setxattr(io_ctx, oid, "a", "1"))
return 1;
if (do_rados_setxattr(io_ctx, oid, "c", "3"))
return 1;
if (do_rados_getxattr(io_ctx, oid, "a", "1"))
return 1;
if (do_rados_getxattr(io_ctx, oid, "b", "2"))
return 1;
if (do_rados_getxattr(io_ctx, oid, "c", "3"))
return 1;
const char *exkeys[] = { "a", "b", "c", NULL };
const char *exvals[] = { "1", "2", "3", NULL };
if (do_rados_getxattrs(io_ctx, oid, exkeys, exvals))
return 1;
uint64_t size;
time_t mtime;
r = rados_stat(io_ctx, oid, &size, &mtime);
printf("rados_stat size = %lld mtime = %d = %d\n", (long long)size, (int)mtime, r);
r = rados_stat(io_ctx, "does_not_exist", NULL, NULL);
printf("rados_stat(does_not_exist) = %d\n", r);
/* exec */
rados_exec(io_ctx, oid, "crypto", "md5", buf, strlen(buf) + 1, buf, 128);
printf("exec result=%s\n", buf);
r = rados_read(io_ctx, oid, buf2, 128, 0);
printf("read result=%s\n", buf2);
printf("size=%d\n", r);
/* aio */
rados_completion_t a, b;
rados_aio_create_completion(0, 0, 0, &a);
rados_aio_create_completion(0, 0, 0, &b);
rados_aio_write(io_ctx, "a", a, buf, 100, 0);
rados_aio_write(io_ctx, "../b/bb_bb_bb\\foo\\bar", b, buf, 100, 0);
rados_aio_wait_for_safe(a);
printf("a safe\n");
rados_aio_wait_for_safe(b);
printf("b safe\n");
rados_aio_release(a);
rados_aio_release(b);
/* test flush */
printf("testing aio flush\n");
rados_completion_t c;
rados_aio_create_completion(0, 0, 0, &c);
rados_aio_write(io_ctx, "c", c, buf, 100, 0);
int safe = rados_aio_is_safe(c);
printf("a should not yet be safe and ... %s\n", safe ? "is":"is not");
assert(!safe);
rados_aio_flush(io_ctx);
safe = rados_aio_is_safe(c);
printf("a should be safe and ... %s\n", safe ? "is":"is not");
assert(safe);
rados_aio_release(c);
rados_read(io_ctx, "../b/bb_bb_bb\\foo\\bar", buf2, 128, 0);
/* list objects */
rados_list_ctx_t h;
r = rados_objects_list_open(io_ctx, &h);
printf("rados_list_objects_open = %d, h = %p\n", r, h);
const char *poolname;
while (rados_objects_list_next(h, &poolname) == 0)
printf("rados_list_objects_next got object '%s'\n", poolname);
rados_objects_list_close(h);
/* stat */
r = rados_ioctx_pool_stat(io_ctx, &st);
printf("rados_stat_pool = %d, %lld KB, %lld objects\n", r, (long long)st.num_kb, (long long)st.num_objects);
/* delete a pool */
printf("rados_delete_pool = %d\n", r);
rados_ioctx_destroy(io_ctx);
r = rados_pool_delete(cl, "foo");
printf("rados_ioctx_pool_delete = %d\n", r);
rados_shutdown(cl);
return 0;
}
int main(int argc, const char **argv)
{
int ret = 0;
// we will use all of these below
const char *pool_name = "hello_world_pool";
const char* hello = "hello world!";
const char* object_name = "hello_object";
rados_ioctx_t io_ctx = NULL;
int pool_created = 0;
// first, we create a Rados object and initialize it
rados_t rados = NULL;
{
ret = rados_create(&rados, "admin"); // just use the client.admin keyring
if (ret < 0) { // let's handle any error that might have come back
printf("couldn't initialize rados! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we just set up a rados cluster object\n");
}
}
/*
* Now we need to get the rados object its config info. It can
* parse argv for us to find the id, monitors, etc, so let's just
* use that.
*/
{
ret = rados_conf_parse_argv(rados, argc, argv);
if (ret < 0) {
// This really can't happen, but we need to check to be a good citizen.
printf("failed to parse config options! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we just parsed our config options\n");
// We also want to apply the config file if the user specified
// one, and conf_parse_argv won't do that for us.
int i;
for (i = 0; i < argc; ++i) {
if ((strcmp(argv[i], "-c") == 0) || (strcmp(argv[i], "--conf") == 0)) {
ret = rados_conf_read_file(rados, argv[i+1]);
if (ret < 0) {
// This could fail if the config file is malformed, but it'd be hard.
printf("failed to parse config file %s! error %d\n", argv[i+1], ret);
ret = EXIT_FAILURE;
goto out;
}
break;
}
}
}
}
/*
* next, we actually connect to the cluster
*/
{
ret = rados_connect(rados);
if (ret < 0) {
printf("couldn't connect to cluster! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we just connected to the rados cluster\n");
}
}
/*
* let's create our own pool instead of scribbling over real data.
* Note that this command creates pools with default PG counts specified
* by the monitors, which may not be appropriate for real use -- it's fine
* for testing, though.
*/
{
ret = rados_pool_create(rados, pool_name);
if (ret < 0) {
printf("couldn't create pool! error %d\n", ret);
return EXIT_FAILURE;
} else {
printf("we just created a new pool named %s\n", pool_name);
}
pool_created = 1;
}
/*
* create an "IoCtx" which is used to do IO to a pool
*/
{
ret = rados_ioctx_create(rados, pool_name, &io_ctx);
if (ret < 0) {
printf("couldn't set up ioctx! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we just created an ioctx for our pool\n");
}
}
/*
* now let's do some IO to the pool! We'll write "hello world!" to a
* new object.
*/
{
/*
* now that we have the data to write, let's send it to an object.
* We'll use the synchronous interface for simplicity.
*/
ret = rados_write_full(io_ctx, object_name, hello, strlen(hello));
if (ret < 0) {
printf("couldn't write object! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we just wrote new object %s, with contents '%s'\n", object_name, hello);
}
}
/*
* now let's read that object back! Just for fun, we'll do it using
* async IO instead of synchronous. (This would be more useful if we
* wanted to send off multiple reads at once; see
* http://ceph.com/docs/master/rados/api/librados/#asychronous-io )
*/
{
int read_len = 4194304; // this is way more than we need
char* read_buf = malloc(read_len + 1); // add one for the terminating 0 we'll add later
if (!read_buf) {
printf("couldn't allocate read buffer\n");
ret = EXIT_FAILURE;
goto out;
}
// allocate the completion from librados
rados_completion_t read_completion;
ret = rados_aio_create_completion(NULL, NULL, NULL, &read_completion);
if (ret < 0) {
printf("couldn't create completion! error %d\n", ret);
ret = EXIT_FAILURE;
free(read_buf);
goto out;
} else {
printf("we just created a new completion\n");
}
// send off the request.
ret = rados_aio_read(io_ctx, object_name, read_completion, read_buf, read_len, 0);
if (ret < 0) {
printf("couldn't start read object! error %d\n", ret);
ret = EXIT_FAILURE;
free(read_buf);
rados_aio_release(read_completion);
goto out;
}
// wait for the request to complete, and check that it succeeded.
rados_aio_wait_for_complete(read_completion);
ret = rados_aio_get_return_value(read_completion);
if (ret < 0) {
printf("couldn't read object! error %d\n", ret);
ret = EXIT_FAILURE;
free(read_buf);
rados_aio_release(read_completion);
goto out;
} else {
read_buf[ret] = 0; // null-terminate the string
printf("we read our object %s, and got back %d bytes with contents\n%s\n", object_name, ret, read_buf);
}
free(read_buf);
rados_aio_release(read_completion);
}
/*
* We can also use xattrs that go alongside the object.
*/
{
const char* version = "1";
ret = rados_setxattr(io_ctx, object_name, "version", version, strlen(version));
if (ret < 0) {
printf("failed to set xattr version entry! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we set the xattr 'version' on our object!\n");
}
}
/*
* And if we want to be really cool, we can do multiple things in a single
* atomic operation. For instance, we can update the contents of our object
* and set the version at the same time.
*/
{
const char* content = "v2";
rados_write_op_t write_op = rados_create_write_op();
if (!write_op) {
printf("failed to allocate write op\n");
ret = EXIT_FAILURE;
goto out;
}
rados_write_op_write_full(write_op, content, strlen(content));
const char* version = "2";
rados_write_op_setxattr(write_op, "version", version, strlen(version));
ret = rados_write_op_operate(write_op, io_ctx, object_name, NULL, 0);
if (ret < 0) {
printf("failed to do compound write! error %d\n", ret);
ret = EXIT_FAILURE;
rados_release_write_op(write_op);
goto out;
} else {
printf("we overwrote our object %s with contents\n%s\n", object_name, content);
}
rados_release_write_op(write_op);
}
/*
* And to be even cooler, we can make sure that the object looks the
* way we expect before doing the write! Notice how this attempt fails
* because the xattr differs.
*/
{
rados_write_op_t failed_write_op = rados_create_write_op();
if (!failed_write_op) {
printf("failed to allocate write op\n");
ret = EXIT_FAILURE;
goto out;
}
const char* content = "v2";
const char* version = "2";
const char* old_version = "1";
rados_write_op_cmpxattr(failed_write_op, "version", LIBRADOS_CMPXATTR_OP_EQ, old_version, strlen(old_version));
rados_write_op_write_full(failed_write_op, content, strlen(content));
rados_write_op_setxattr(failed_write_op, "version", version, strlen(version));
ret = rados_write_op_operate(failed_write_op, io_ctx, object_name, NULL, 0);
if (ret < 0) {
printf("we just failed a write because the xattr wasn't as specified\n");
} else {
printf("we succeeded on writing despite an xattr comparison mismatch!\n");
ret = EXIT_FAILURE;
rados_release_write_op(failed_write_op);
goto out;
}
rados_release_write_op(failed_write_op);
/*
* Now let's do the update with the correct xattr values so it
* actually goes through
*/
content = "v3";
old_version = "2";
version = "3";
rados_write_op_t update_op = rados_create_write_op();
if (!failed_write_op) {
printf("failed to allocate write op\n");
ret = EXIT_FAILURE;
goto out;
}
rados_write_op_cmpxattr(update_op, "version", LIBRADOS_CMPXATTR_OP_EQ, old_version, strlen(old_version));
rados_write_op_write_full(update_op, content, strlen(content));
rados_write_op_setxattr(update_op, "version", version, strlen(version));
ret = rados_write_op_operate(update_op, io_ctx, object_name, NULL, 0);
if (ret < 0) {
printf("failed to do a compound write update! error %d\n", ret);
ret = EXIT_FAILURE;
rados_release_write_op(update_op);
goto out;
} else {
printf("we overwrote our object %s following an xattr test with contents\n%s\n", object_name, content);
}
rados_release_write_op(update_op);
}
ret = EXIT_SUCCESS;
out:
if (io_ctx) {
rados_ioctx_destroy(io_ctx);
}
if (pool_created) {
/*
* And now we're done, so let's remove our pool and then
* shut down the connection gracefully.
*/
int delete_ret = rados_pool_delete(rados, pool_name);
if (delete_ret < 0) {
// be careful not to
printf("We failed to delete our test pool!\n");
ret = EXIT_FAILURE;
}
}
rados_shutdown(rados);
return ret;
}
static int qemu_rbd_open(BlockDriverState *bs, const char *filename, int flags)
{
BDRVRBDState *s = bs->opaque;
char pool[RBD_MAX_POOL_NAME_SIZE];
char snap_buf[RBD_MAX_SNAP_NAME_SIZE];
char conf[RBD_MAX_CONF_SIZE];
char clientname_buf[RBD_MAX_CONF_SIZE];
char *clientname;
int r;
if (qemu_rbd_parsename(filename, pool, sizeof(pool),
snap_buf, sizeof(snap_buf),
s->name, sizeof(s->name),
conf, sizeof(conf)) < 0) {
return -EINVAL;
}
clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
r = rados_create(&s->cluster, clientname);
if (r < 0) {
error_report("error initializing");
return r;
}
s->snap = NULL;
if (snap_buf[0] != '\0') {
s->snap = g_strdup(snap_buf);
}
if (strstr(conf, "conf=") == NULL) {
/* try default location, but ignore failure */
rados_conf_read_file(s->cluster, NULL);
}
if (conf[0] != '\0') {
r = qemu_rbd_set_conf(s->cluster, conf);
if (r < 0) {
error_report("error setting config options");
goto failed_shutdown;
}
}
r = rados_connect(s->cluster);
if (r < 0) {
error_report("error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
if (r < 0) {
error_report("error opening pool %s", pool);
goto failed_shutdown;
}
r = rbd_open(s->io_ctx, s->name, &s->image, s->snap);
if (r < 0) {
error_report("error reading header from %s", s->name);
goto failed_open;
}
bs->read_only = (s->snap != NULL);
s->event_reader_pos = 0;
r = qemu_pipe(s->fds);
if (r < 0) {
error_report("error opening eventfd");
goto failed;
}
fcntl(s->fds[0], F_SETFL, O_NONBLOCK);
fcntl(s->fds[1], F_SETFL, O_NONBLOCK);
qemu_aio_set_fd_handler(s->fds[RBD_FD_READ], qemu_rbd_aio_event_reader,
NULL, qemu_rbd_aio_flush_cb, NULL, s);
return 0;
failed:
rbd_close(s->image);
failed_open:
rados_ioctx_destroy(s->io_ctx);
failed_shutdown:
rados_shutdown(s->cluster);
g_free(s->snap);
return r;
}
static int qemu_rbd_create(const char *filename, QEMUOptionParameter *options)
{
int64_t bytes = 0;
int64_t objsize;
int obj_order = 0;
char pool[RBD_MAX_POOL_NAME_SIZE];
char name[RBD_MAX_IMAGE_NAME_SIZE];
char snap_buf[RBD_MAX_SNAP_NAME_SIZE];
char conf[RBD_MAX_CONF_SIZE];
char clientname_buf[RBD_MAX_CONF_SIZE];
char *clientname;
rados_t cluster;
rados_ioctx_t io_ctx;
int ret;
if (qemu_rbd_parsename(filename, pool, sizeof(pool),
snap_buf, sizeof(snap_buf),
name, sizeof(name),
conf, sizeof(conf)) < 0) {
return -EINVAL;
}
/* Read out options */
while (options && options->name) {
if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
bytes = options->value.n;
} else if (!strcmp(options->name, BLOCK_OPT_CLUSTER_SIZE)) {
if (options->value.n) {
objsize = options->value.n;
if ((objsize - 1) & objsize) { /* not a power of 2? */
error_report("obj size needs to be power of 2");
return -EINVAL;
}
if (objsize < 4096) {
error_report("obj size too small");
return -EINVAL;
}
obj_order = ffs(objsize) - 1;
}
}
options++;
}
clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
if (rados_create(&cluster, clientname) < 0) {
error_report("error initializing");
return -EIO;
}
if (strstr(conf, "conf=") == NULL) {
/* try default location, but ignore failure */
rados_conf_read_file(cluster, NULL);
}
if (conf[0] != '\0' &&
qemu_rbd_set_conf(cluster, conf) < 0) {
error_report("error setting config options");
rados_shutdown(cluster);
return -EIO;
}
if (rados_connect(cluster) < 0) {
error_report("error connecting");
rados_shutdown(cluster);
return -EIO;
}
if (rados_ioctx_create(cluster, pool, &io_ctx) < 0) {
error_report("error opening pool %s", pool);
rados_shutdown(cluster);
return -EIO;
}
ret = rbd_create(io_ctx, name, bytes, &obj_order);
rados_ioctx_destroy(io_ctx);
rados_shutdown(cluster);
return ret;
}
static int qemu_rbd_create(const char *filename, QemuOpts *opts, Error **errp)
{
Error *local_err = NULL;
int64_t bytes = 0;
int64_t objsize;
int obj_order = 0;
char pool[RBD_MAX_POOL_NAME_SIZE];
char name[RBD_MAX_IMAGE_NAME_SIZE];
char snap_buf[RBD_MAX_SNAP_NAME_SIZE];
char conf[RBD_MAX_CONF_SIZE];
char clientname_buf[RBD_MAX_CONF_SIZE];
char *clientname;
rados_t cluster;
rados_ioctx_t io_ctx;
int ret;
if (qemu_rbd_parsename(filename, pool, sizeof(pool),
snap_buf, sizeof(snap_buf),
name, sizeof(name),
conf, sizeof(conf), &local_err) < 0) {
error_propagate(errp, local_err);
return -EINVAL;
}
/* Read out options */
bytes = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);
objsize = qemu_opt_get_size_del(opts, BLOCK_OPT_CLUSTER_SIZE, 0);
if (objsize) {
if ((objsize - 1) & objsize) { /* not a power of 2? */
error_setg(errp, "obj size needs to be power of 2");
return -EINVAL;
}
if (objsize < 4096) {
error_setg(errp, "obj size too small");
return -EINVAL;
}
obj_order = ffs(objsize) - 1;
}
clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
if (rados_create(&cluster, clientname) < 0) {
error_setg(errp, "error initializing");
return -EIO;
}
if (strstr(conf, "conf=") == NULL) {
/* try default location, but ignore failure */
rados_conf_read_file(cluster, NULL);
}
if (conf[0] != '\0' &&
qemu_rbd_set_conf(cluster, conf, &local_err) < 0) {
rados_shutdown(cluster);
error_propagate(errp, local_err);
return -EIO;
}
if (rados_connect(cluster) < 0) {
error_setg(errp, "error connecting");
rados_shutdown(cluster);
return -EIO;
}
if (rados_ioctx_create(cluster, pool, &io_ctx) < 0) {
error_setg(errp, "error opening pool %s", pool);
rados_shutdown(cluster);
return -EIO;
}
ret = rbd_create(io_ctx, name, bytes, &obj_order);
rados_ioctx_destroy(io_ctx);
rados_shutdown(cluster);
return ret;
}
static int qemu_rbd_connect(rados_t *cluster, rados_ioctx_t *io_ctx,
BlockdevOptionsRbd *opts, bool cache,
const char *keypairs, const char *secretid,
Error **errp)
{
char *mon_host = NULL;
Error *local_err = NULL;
int r;
if (secretid) {
if (opts->key_secret) {
error_setg(errp,
"Legacy 'password-secret' clashes with 'key-secret'");
return -EINVAL;
}
opts->key_secret = g_strdup(secretid);
opts->has_key_secret = true;
}
mon_host = qemu_rbd_mon_host(opts, &local_err);
if (local_err) {
error_propagate(errp, local_err);
r = -EINVAL;
goto failed_opts;
}
r = rados_create(cluster, opts->user);
if (r < 0) {
error_setg_errno(errp, -r, "error initializing");
goto failed_opts;
}
/* try default location when conf=NULL, but ignore failure */
r = rados_conf_read_file(*cluster, opts->conf);
if (opts->has_conf && r < 0) {
error_setg_errno(errp, -r, "error reading conf file %s", opts->conf);
goto failed_shutdown;
}
r = qemu_rbd_set_keypairs(*cluster, keypairs, errp);
if (r < 0) {
goto failed_shutdown;
}
if (mon_host) {
r = rados_conf_set(*cluster, "mon_host", mon_host);
if (r < 0) {
goto failed_shutdown;
}
}
r = qemu_rbd_set_auth(*cluster, opts, errp);
if (r < 0) {
goto failed_shutdown;
}
/*
* Fallback to more conservative semantics if setting cache
* options fails. Ignore errors from setting rbd_cache because the
* only possible error is that the option does not exist, and
* librbd defaults to no caching. If write through caching cannot
* be set up, fall back to no caching.
*/
if (cache) {
rados_conf_set(*cluster, "rbd_cache", "true");
} else {
rados_conf_set(*cluster, "rbd_cache", "false");
}
r = rados_connect(*cluster);
if (r < 0) {
error_setg_errno(errp, -r, "error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(*cluster, opts->pool, io_ctx);
if (r < 0) {
error_setg_errno(errp, -r, "error opening pool %s", opts->pool);
goto failed_shutdown;
}
return 0;
failed_shutdown:
rados_shutdown(*cluster);
failed_opts:
g_free(mon_host);
return r;
}
static int coroutine_fn qemu_rbd_co_create_opts(const char *filename,
QemuOpts *opts,
Error **errp)
{
Error *local_err = NULL;
int64_t bytes = 0;
int64_t objsize;
int obj_order = 0;
const char *pool, *image_name, *conf, *user, *keypairs;
const char *secretid;
rados_t cluster;
rados_ioctx_t io_ctx;
QDict *options = NULL;
int ret = 0;
secretid = qemu_opt_get(opts, "password-secret");
/* Read out options */
bytes = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
BDRV_SECTOR_SIZE);
objsize = qemu_opt_get_size_del(opts, BLOCK_OPT_CLUSTER_SIZE, 0);
if (objsize) {
if ((objsize - 1) & objsize) { /* not a power of 2? */
error_setg(errp, "obj size needs to be power of 2");
ret = -EINVAL;
goto exit;
}
if (objsize < 4096) {
error_setg(errp, "obj size too small");
ret = -EINVAL;
goto exit;
}
obj_order = ctz32(objsize);
}
options = qdict_new();
qemu_rbd_parse_filename(filename, options, &local_err);
if (local_err) {
ret = -EINVAL;
error_propagate(errp, local_err);
goto exit;
}
/*
* Caution: while qdict_get_try_str() is fine, getting non-string
* types would require more care. When @options come from -blockdev
* or blockdev_add, its members are typed according to the QAPI
* schema, but when they come from -drive, they're all QString.
*/
pool = qdict_get_try_str(options, "pool");
conf = qdict_get_try_str(options, "conf");
user = qdict_get_try_str(options, "user");
image_name = qdict_get_try_str(options, "image");
keypairs = qdict_get_try_str(options, "=keyvalue-pairs");
ret = rados_create(&cluster, user);
if (ret < 0) {
error_setg_errno(errp, -ret, "error initializing");
goto exit;
}
/* try default location when conf=NULL, but ignore failure */
ret = rados_conf_read_file(cluster, conf);
if (conf && ret < 0) {
error_setg_errno(errp, -ret, "error reading conf file %s", conf);
ret = -EIO;
goto shutdown;
}
ret = qemu_rbd_set_keypairs(cluster, keypairs, errp);
if (ret < 0) {
ret = -EIO;
goto shutdown;
}
if (qemu_rbd_set_auth(cluster, secretid, errp) < 0) {
ret = -EIO;
goto shutdown;
}
ret = rados_connect(cluster);
if (ret < 0) {
error_setg_errno(errp, -ret, "error connecting");
goto shutdown;
}
ret = rados_ioctx_create(cluster, pool, &io_ctx);
if (ret < 0) {
error_setg_errno(errp, -ret, "error opening pool %s", pool);
goto shutdown;
}
ret = rbd_create(io_ctx, image_name, bytes, &obj_order);
if (ret < 0) {
error_setg_errno(errp, -ret, "error rbd create");
}
rados_ioctx_destroy(io_ctx);
shutdown:
rados_shutdown(cluster);
exit:
QDECREF(options);
return ret;
}
static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
Error **errp)
{
BDRVRBDState *s = bs->opaque;
const char *pool, *snap, *conf, *user, *image_name, *keypairs;
const char *secretid, *filename;
QemuOpts *opts;
Error *local_err = NULL;
char *mon_host = NULL;
int r;
/* If we are given a filename, parse the filename, with precedence given to
* filename encoded options */
filename = qdict_get_try_str(options, "filename");
if (filename) {
warn_report("'filename' option specified. "
"This is an unsupported option, and may be deprecated "
"in the future");
qemu_rbd_parse_filename(filename, options, &local_err);
if (local_err) {
r = -EINVAL;
error_propagate(errp, local_err);
goto exit;
}
}
opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
qemu_opts_absorb_qdict(opts, options, &local_err);
if (local_err) {
error_propagate(errp, local_err);
r = -EINVAL;
goto failed_opts;
}
mon_host = qemu_rbd_mon_host(options, &local_err);
if (local_err) {
error_propagate(errp, local_err);
r = -EINVAL;
goto failed_opts;
}
secretid = qemu_opt_get(opts, "password-secret");
pool = qemu_opt_get(opts, "pool");
conf = qemu_opt_get(opts, "conf");
snap = qemu_opt_get(opts, "snapshot");
user = qemu_opt_get(opts, "user");
image_name = qemu_opt_get(opts, "image");
keypairs = qemu_opt_get(opts, "=keyvalue-pairs");
if (!pool || !image_name) {
error_setg(errp, "Parameters 'pool' and 'image' are required");
r = -EINVAL;
goto failed_opts;
}
r = rados_create(&s->cluster, user);
if (r < 0) {
error_setg_errno(errp, -r, "error initializing");
goto failed_opts;
}
s->snap = g_strdup(snap);
s->image_name = g_strdup(image_name);
/* try default location when conf=NULL, but ignore failure */
r = rados_conf_read_file(s->cluster, conf);
if (conf && r < 0) {
error_setg_errno(errp, -r, "error reading conf file %s", conf);
goto failed_shutdown;
}
r = qemu_rbd_set_keypairs(s->cluster, keypairs, errp);
if (r < 0) {
goto failed_shutdown;
}
if (mon_host) {
r = rados_conf_set(s->cluster, "mon_host", mon_host);
if (r < 0) {
goto failed_shutdown;
}
}
if (qemu_rbd_set_auth(s->cluster, secretid, errp) < 0) {
r = -EIO;
goto failed_shutdown;
}
/*
* Fallback to more conservative semantics if setting cache
* options fails. Ignore errors from setting rbd_cache because the
* only possible error is that the option does not exist, and
* librbd defaults to no caching. If write through caching cannot
* be set up, fall back to no caching.
*/
if (flags & BDRV_O_NOCACHE) {
rados_conf_set(s->cluster, "rbd_cache", "false");
} else {
rados_conf_set(s->cluster, "rbd_cache", "true");
}
r = rados_connect(s->cluster);
if (r < 0) {
error_setg_errno(errp, -r, "error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
if (r < 0) {
error_setg_errno(errp, -r, "error opening pool %s", pool);
goto failed_shutdown;
}
/* rbd_open is always r/w */
r = rbd_open(s->io_ctx, s->image_name, &s->image, s->snap);
if (r < 0) {
error_setg_errno(errp, -r, "error reading header from %s",
s->image_name);
goto failed_open;
}
/* If we are using an rbd snapshot, we must be r/o, otherwise
* leave as-is */
if (s->snap != NULL) {
if (!bdrv_is_read_only(bs)) {
error_report("Opening rbd snapshots without an explicit "
"read-only=on option is deprecated. Future versions "
"will refuse to open the image instead of "
"automatically marking the image read-only.");
r = bdrv_set_read_only(bs, true, &local_err);
if (r < 0) {
error_propagate(errp, local_err);
goto failed_open;
}
}
}
qemu_opts_del(opts);
return 0;
failed_open:
rados_ioctx_destroy(s->io_ctx);
failed_shutdown:
rados_shutdown(s->cluster);
g_free(s->snap);
g_free(s->image_name);
failed_opts:
qemu_opts_del(opts);
g_free(mon_host);
exit:
return r;
}
static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
Error **errp)
{
BDRVRBDState *s = bs->opaque;
char pool[RBD_MAX_POOL_NAME_SIZE];
char snap_buf[RBD_MAX_SNAP_NAME_SIZE];
char conf[RBD_MAX_CONF_SIZE];
char clientname_buf[RBD_MAX_CONF_SIZE];
char *clientname;
QemuOpts *opts;
Error *local_err = NULL;
const char *filename;
int r;
opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
qemu_opts_absorb_qdict(opts, options, &local_err);
if (local_err) {
qerror_report_err(local_err);
error_free(local_err);
qemu_opts_del(opts);
return -EINVAL;
}
filename = qemu_opt_get(opts, "filename");
if (qemu_rbd_parsename(filename, pool, sizeof(pool),
snap_buf, sizeof(snap_buf),
s->name, sizeof(s->name),
conf, sizeof(conf)) < 0) {
r = -EINVAL;
goto failed_opts;
}
clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
r = rados_create(&s->cluster, clientname);
if (r < 0) {
error_report("error initializing");
goto failed_opts;
}
s->snap = NULL;
if (snap_buf[0] != '\0') {
s->snap = g_strdup(snap_buf);
}
/*
* Fallback to more conservative semantics if setting cache
* options fails. Ignore errors from setting rbd_cache because the
* only possible error is that the option does not exist, and
* librbd defaults to no caching. If write through caching cannot
* be set up, fall back to no caching.
*/
if (flags & BDRV_O_NOCACHE) {
rados_conf_set(s->cluster, "rbd_cache", "false");
} else {
rados_conf_set(s->cluster, "rbd_cache", "true");
}
if (strstr(conf, "conf=") == NULL) {
/* try default location, but ignore failure */
rados_conf_read_file(s->cluster, NULL);
}
if (conf[0] != '\0') {
r = qemu_rbd_set_conf(s->cluster, conf);
if (r < 0) {
error_report("error setting config options");
goto failed_shutdown;
}
}
r = rados_connect(s->cluster);
if (r < 0) {
error_report("error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
if (r < 0) {
error_report("error opening pool %s", pool);
goto failed_shutdown;
}
r = rbd_open(s->io_ctx, s->name, &s->image, s->snap);
if (r < 0) {
error_report("error reading header from %s", s->name);
goto failed_open;
}
bs->read_only = (s->snap != NULL);
qemu_opts_del(opts);
return 0;
failed_open:
rados_ioctx_destroy(s->io_ctx);
failed_shutdown:
rados_shutdown(s->cluster);
g_free(s->snap);
failed_opts:
qemu_opts_del(opts);
return r;
}
static void uwsgi_rados_add_mountpoint(char *arg, size_t arg_len) {
struct uwsgi_rados_mountpoint *urmp = uwsgi_calloc(sizeof(struct uwsgi_rados_mountpoint));
if (uwsgi_kvlist_parse(arg, arg_len, ',', '=',
"mountpoint", &urmp->mountpoint,
"config", &urmp->config,
"pool", &urmp->pool,
"timeout", &urmp->str_timeout,
"allow_put", &urmp->allow_put,
"allow_delete", &urmp->allow_delete,
"allow_mkcol", &urmp->allow_mkcol,
"allow_propfind", &urmp->allow_propfind,
"username", &urmp->username,
"buffer_size", &urmp->str_buffer_size,
"put_buffer_size", &urmp->str_put_buffer_size,
NULL)) {
uwsgi_log("unable to parse rados mountpoint definition\n");
exit(1);
}
if (!urmp->mountpoint|| !urmp->pool) {
uwsgi_log("[rados] mount requires a mountpoint, and a pool name.\n");
exit(1);
}
if (urmp->str_timeout) {
urmp->timeout = atoi(urmp->str_timeout);
}
if (urmp->str_buffer_size) {
urmp->buffer_size = atoi(urmp->str_buffer_size);
if (urmp->buffer_size > MAX_BUF_SIZE) {
urmp->buffer_size = MAX_BUF_SIZE;
}
else if (urmp->buffer_size < MIN_BUF_SIZE) {
urmp->buffer_size = MIN_BUF_SIZE;
}
}
else {
urmp->buffer_size = DEF_BUF_SIZE;
}
if (urmp->str_put_buffer_size) {
urmp->put_buffer_size = atoi(urmp->str_put_buffer_size);
if (urmp->put_buffer_size > MAX_BUF_SIZE) {
urmp->put_buffer_size = MAX_BUF_SIZE;
}
else if (urmp->put_buffer_size < MIN_BUF_SIZE) {
urmp->put_buffer_size = MIN_BUF_SIZE;
}
}
else {
urmp->put_buffer_size = urmp->buffer_size;
}
time_t now = uwsgi_now();
uwsgi_log("[rados] mounting %s ...\n", urmp->mountpoint);
rados_t cluster;
if (rados_create(&cluster, urmp->username) < 0) {
uwsgi_error("can't create Ceph cluster handle");
exit(1);
}
urmp->cluster = cluster;
if (urmp->config)
uwsgi_log("using Ceph conf:%s\n", urmp->config);
else
uwsgi_log("using default Ceph conf.\n");
if (rados_conf_read_file(cluster, urmp->config) < 0) {
uwsgi_error("can't configure Ceph cluster handle");
exit(1);
}
int timeout = urmp->timeout ? urmp->timeout : urados.timeout;
char *timeout_str = uwsgi_num2str(timeout);
rados_conf_set(cluster, "client_mount_timeout", timeout_str);
rados_conf_set(cluster, "rados_mon_op_timeout", timeout_str);
rados_conf_set(cluster, "rados_osd_op_timeout", timeout_str);
free(timeout_str);
if (rados_connect(cluster) < 0) {
uwsgi_error("can't connect with Ceph cluster");
exit(1);
}
void *ctx_ptr;
if (uwsgi.threads > 1) {
int i;
rados_ioctx_t *ctxes = uwsgi_calloc(sizeof(rados_ioctx_t) * uwsgi.threads);
for(i=0;i<uwsgi.threads;i++) {
if (rados_ioctx_create(cluster, urmp->pool, &ctxes[i]) < 0) {
uwsgi_error("can't open rados pool")
rados_shutdown(cluster);
exit(1);
}
}
ctx_ptr = ctxes;
}
else {
rados_ioctx_t ctx;
if (rados_ioctx_create(cluster, urmp->pool, &ctx) < 0) {
uwsgi_error("can't open rados pool")
rados_shutdown(cluster);
exit(1);
}
ctx_ptr = ctx;
}
char fsid[37];
rados_cluster_fsid(cluster, fsid, 37);
uwsgi_log("connected to Ceph pool: %s on cluster %.*s\n", urmp->pool, 37, fsid);
int id = uwsgi_apps_cnt;
struct uwsgi_app *ua = uwsgi_add_app(id, rados_plugin.modifier1, urmp->mountpoint, strlen(urmp->mountpoint), NULL, (void*)1);
if (!ua) {
uwsgi_log("[rados] unable to mount %s\n", urmp->mountpoint);
rados_shutdown(cluster);
exit(1);
}
ua->responder0 = ctx_ptr;
ua->responder1 = urmp;
ua->started_at = now;
ua->startup_time = uwsgi_now() - now;
uwsgi_log("Rados app/mountpoint %d (%s) loaded in %d seconds at %p\n", id, urmp->mountpoint, (int) ua->startup_time, ctx_ptr);
}
static int qemu_rbd_open(BlockDriverState *bs, const char *filename, int flags)
{
BDRVRBDState *s = bs->opaque;
char pool[RBD_MAX_POOL_NAME_SIZE];
char snap_buf[RBD_MAX_SNAP_NAME_SIZE];
char conf[RBD_MAX_CONF_SIZE];
char clientname_buf[RBD_MAX_CONF_SIZE];
char *clientname;
int r;
if (qemu_rbd_parsename(filename, pool, sizeof(pool),
snap_buf, sizeof(snap_buf),
s->name, sizeof(s->name),
conf, sizeof(conf)) < 0) {
return -EINVAL;
}
clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
r = rados_create(&s->cluster, clientname);
if (r < 0) {
error_report("error initializing");
return r;
}
s->snap = NULL;
if (snap_buf[0] != '\0') {
s->snap = g_strdup(snap_buf);
}
/*
* Fallback to more conservative semantics if setting cache
* options fails. Ignore errors from setting rbd_cache because the
* only possible error is that the option does not exist, and
* librbd defaults to no caching. If write through caching cannot
* be set up, fall back to no caching.
*/
if (flags & BDRV_O_NOCACHE) {
rados_conf_set(s->cluster, "rbd_cache", "false");
} else {
rados_conf_set(s->cluster, "rbd_cache", "true");
}
if (strstr(conf, "conf=") == NULL) {
/* try default location, but ignore failure */
rados_conf_read_file(s->cluster, NULL);
}
if (conf[0] != '\0') {
r = qemu_rbd_set_conf(s->cluster, conf);
if (r < 0) {
error_report("error setting config options");
goto failed_shutdown;
}
}
r = rados_connect(s->cluster);
if (r < 0) {
error_report("error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
if (r < 0) {
error_report("error opening pool %s", pool);
goto failed_shutdown;
}
r = rbd_open(s->io_ctx, s->name, &s->image, s->snap);
if (r < 0) {
error_report("error reading header from %s", s->name);
goto failed_open;
}
bs->read_only = (s->snap != NULL);
s->event_reader_pos = 0;
r = qemu_pipe(s->fds);
if (r < 0) {
error_report("error opening eventfd");
goto failed;
}
fcntl(s->fds[0], F_SETFL, O_NONBLOCK);
fcntl(s->fds[1], F_SETFL, O_NONBLOCK);
qemu_aio_set_fd_handler(s->fds[RBD_FD_READ], qemu_rbd_aio_event_reader,
NULL, qemu_rbd_aio_flush_cb, s);
return 0;
failed:
rbd_close(s->image);
failed_open:
rados_ioctx_destroy(s->io_ctx);
failed_shutdown:
rados_shutdown(s->cluster);
g_free(s->snap);
return r;
}
static int app_rbd_open()
{
int ret;
/* poolname imagename username password monitor */
fprintf(stdout,
"open rados as following setting:\n"
"poolname: %s\n"
"imagename: %s\n"
"username: %s\n"
"password: %s\n"
"monitor: %s\n",
poolname, imagename, username, password, monitor);
/**
* 创建一个 rados_t 句柄, 该句柄暴扣了rados 客户端的数据结构, 用来和
* rados 通信, 第二个参数是连接 rados 的客户端 ID, 这里我们使用 admin
*/
ret = rados_create(&cluster, username);
if (ret < 0) {
fprintf(stderr, "cannot create a cluster handle: %s\n", strerror(-ret));
goto failed_create;
}
/* 载入配置文件 */
ret = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
if (ret < 0) {
fprintf(stderr, "cannot read config file: %s\n", strerror(-ret));
goto failed_shutdown;
}
/**
* 调用 rados_conf_set() 设置 rados 参数, 包括认证信息, monitor 地址等,
* 如果设置了 cephx 认证, 那么之前创建 rados 句柄的时候, 必须设置客户端
* ID, 并且必须设置 key 的密码.
*/
if (conf_set("key", password) < 0) {
goto failed_shutdown;
}
if (conf_set("auth_supported", "cephx") < 0) {
goto failed_shutdown;
}
if (conf_set("mon_host", monitor) < 0) {
goto failed_shutdown;
}
/* 完成了上面的设置之后, 就可以用我们的 rados 句柄连接 rados 服务器了 */
ret = rados_connect(cluster);
if (ret < 0) {
fprintf(stderr, "cannot connect to cluster: %s\n", strerror(-ret));
goto failed_shutdown;
}
/**
* 成功连接上 rados 服务器之后, 就可以用 rados_ioctx_create() 打开 rados
* 上的 pool 了, 该函数需要传递一个 rados_ioctx_t, 用来对打开的 pool 进行
* 操作.
*/
ret = rados_ioctx_create(cluster, poolname, &io_ctx);
if (ret < 0) {
fprintf(stderr, "cannot open rados pool %s: %s\n",
poolname, strerror(-ret));
goto failed_shutdown;
}
/**
* 上面说过, 我们已经能够有一个 rados_ioctx_t 的指针了, 该指针用来对关联的
* pool 进行 I/O 操作, 比如打开池中的 image 等, 这里我们直接对 rbd 操作.
*/
ret = rbd_open(io_ctx, imagename, &image, NULL);
if (ret < 0) {
fprintf(stderr, "error reading header from image %s\n", imagename);
goto failed_open;
}
return 0;
failed_open:
rados_ioctx_destroy(io_ctx);
failed_shutdown:
rados_shutdown(cluster);
failed_create:
return ret;
}
