int
connect_to_cluster(rados_t *pcluster)
{
int r;
r = rados_create(pcluster, NULL);
if (r < 0) {
simple_err("Could not create cluster handle", r);
return r;
}
rados_conf_parse_env(*pcluster, NULL);
r = rados_conf_read_file(*pcluster, rbd_options.ceph_config);
if (r < 0) {
simple_err("Error reading Ceph config file", r);
goto failed_shutdown;
}
r = rados_connect(*pcluster);
if (r < 0) {
simple_err("Error connecting to cluster", r);
goto failed_shutdown;
}
return 0;
failed_shutdown:
rados_shutdown(*pcluster);
return r;
}
static void uwsgi_rados_add_mountpoint(char *arg, size_t arg_len) {
char *rad_mountpoint = NULL;
char *rad_config = NULL;
char *rad_poolname = NULL;
if (uwsgi_kvlist_parse(arg, arg_len, ',', '=',
"mountpoint", &rad_mountpoint,
"config", &rad_config,
"pool", &rad_poolname,
NULL)) {
uwsgi_log("unable to parse rados mountpoint definition\n");
exit(1);
}
if (!rad_mountpoint|| !rad_poolname) {
uwsgi_log("[rados] mount requires a mountpoint, and a pool name.\n");
exit(1);
}
time_t now = uwsgi_now();
uwsgi_log("[rados] mounting %s ...\n", rad_mountpoint);
rados_t cluster;
if (rados_create(&cluster, NULL) < 0) {
uwsgi_error("Can't create Ceph cluster handle");
exit(1);
}
if (rad_config)
uwsgi_log("Using Ceph conf:%s\n", rad_config);
else
uwsgi_log("Using default Ceph conf.\n");
if (rados_conf_read_file(cluster, rad_config) < 0) {
uwsgi_error("Can't configure Ceph cluster handle");
exit(1);
}
if (rados_connect(cluster) < 0) {
uwsgi_error("Can't connect with Ceph cluster");
exit(1);
}
rados_ioctx_t ctx;
uwsgi_log("Ceph pool: %s\n", rad_poolname);
if (rados_ioctx_create(cluster, rad_poolname, &ctx) < 0) {
uwsgi_error("Can't open rados pool")
rados_shutdown(cluster);
exit(1);
}
int id = uwsgi_apps_cnt;
struct uwsgi_app *ua = uwsgi_add_app(id, rados_plugin.modifier1, rad_mountpoint, strlen(rad_mountpoint), NULL, NULL);
if (!ua) {
uwsgi_log("[rados] unable to mount %s\n", rad_mountpoint);
rados_shutdown(cluster);
exit(1);
}
ua->responder0 = cluster;
ua->responder1 = ctx;
ua->started_at = now;
ua->startup_time = uwsgi_now() - now;
uwsgi_log("Rados app/mountpoint %d (%s) loaded in %d seconds at %p\n", id, rad_mountpoint, (int) ua->startup_time, ctx);
}
static VALUE initialize_ext(VALUE self) {
int err;
GET_CLUSTER(self);
// FIXME: can specify user id here for auth!
err = rados_create(wrapper->cluster, NULL);
if (err < 0) {
rb_raise(cRadosError, "cannot create a cluster handle: %s", strerror(-err));
}
// FIXME: Allow specifying config filename
err = rados_conf_read_file(*wrapper->cluster, NULL);
if (err < 0) {
rb_raise(cRadosError, "cannot read config file: %s", strerror(-err));
}
// FIXME: should release global lock
err = rados_connect(*wrapper->cluster);
switch (err) {
case 0:
break;
case -110:
rb_raise(rb_const_get(mRados, rb_intern("ConnectionTimeout")), "%s", strerror(-err));
default:
rb_raise(cRadosError, "cannot connect: %s", strerror(-err));
}
wrapper->connected = 1;
return self;
}
int main(int argc, const char **argv)
{
rados_t cluster;
rados_ioctx_t io_ctx;
rbd_image_t image;
srand(time(0));
assert(rados_create(&cluster, NULL) == 0);
assert(rados_conf_parse_argv(cluster, argc, argv) == 0);
assert(rados_conf_read_file(cluster, NULL) == 0);
assert(rados_connect(cluster) == 0);
if (rados_pool_lookup(cluster, TEST_POOL) != -ENOENT) {
int r = rados_pool_delete(cluster, TEST_POOL);
printf("rados_pool_delete returned %d\n", r);
}
int r = rados_pool_create(cluster, TEST_POOL);
printf("rados_pool_create returned %d\n", r);
assert(rados_ioctx_create(cluster, TEST_POOL, &io_ctx) == 0);
test_ls(io_ctx, 0);
test_create_and_stat(io_ctx, TEST_IMAGE, MB_BYTES(1));
assert(rbd_open(io_ctx, TEST_IMAGE, &image, NULL) == 0);
test_ls(io_ctx, 1, TEST_IMAGE);
test_ls_snaps(image, 0);
test_create_snap(image, TEST_SNAP);
test_ls_snaps(image, 1, TEST_SNAP, MB_BYTES(1));
test_resize_and_stat(image, MB_BYTES(2));
test_io(io_ctx, image);
test_create_snap(image, TEST_SNAP "1");
test_ls_snaps(image, 2, TEST_SNAP, MB_BYTES(1), TEST_SNAP "1", MB_BYTES(2));
test_delete_snap(image, TEST_SNAP);
test_ls_snaps(image, 1, TEST_SNAP "1", MB_BYTES(2));
test_delete_snap(image, TEST_SNAP "1");
test_ls_snaps(image, 0);
test_io_to_snapshot(io_ctx, image, MB_BYTES(2));
assert(rbd_close(image) == 0);
test_create_and_stat(io_ctx, TEST_IMAGE "1", MB_BYTES(2));
test_ls(io_ctx, 2, TEST_IMAGE, TEST_IMAGE "1");
test_delete(io_ctx, TEST_IMAGE);
test_ls(io_ctx, 1, TEST_IMAGE "1");
test_delete(io_ctx, TEST_IMAGE "1");
test_ls(io_ctx, 0);
rados_ioctx_destroy(io_ctx);
rados_shutdown(cluster);
return 0;
}
static tgtadm_err bs_rbd_init(struct scsi_lu *lu)
{
tgtadm_err ret = TGTADM_UNKNOWN_ERR;
int rados_ret;
struct bs_thread_info *info = BS_THREAD_I(lu);
rados_ret = rados_create(&cluster, NULL);
if (rados_ret < 0) {
eprintf("bs_rbd_init: rados_create: %d\n", rados_ret);
return ret;
}
/* read config from environment and then default files */
rados_ret = rados_conf_parse_env(cluster, NULL);
if (rados_ret < 0) {
eprintf("bs_rbd_init: rados_conf_parse_env: %d\n", rados_ret);
goto fail;
}
rados_ret = rados_conf_read_file(cluster, NULL);
if (rados_ret < 0) {
eprintf("bs_rbd_init: rados_conf_read_file: %d\n", rados_ret);
goto fail;
}
rados_ret = rados_connect(cluster);
if (rados_ret < 0) {
eprintf("bs_rbd_init: rados_connect: %d\n", rados_ret);
goto fail;
}
ret = bs_thread_open(info, bs_rbd_request, nr_iothreads);
if (ret == TGTADM_SUCCESS)
return ret;
fail:
rados_shutdown(&cluster);
return ret;
}
ERL_NIF_TERM x_connect(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
{
const char * func_name = "x_connect()";
logger.debug(MOD_NAME, func_name, "Entered");
uint64_t id;
if (!enif_get_uint64(env, argv[0], &id))
{
logger.error(MOD_NAME, func_name, "enif get params failed");
return enif_make_badarg(env);
}
logger.debug(MOD_NAME, func_name, "cluster : %ld", id);
rados_t cluster = map_cluster_get(id);
if (cluster == NULL)
{
logger.error(MOD_NAME, func_name, "cluster non-existing : %ld", id);
return enif_make_badarg(env);
}
int err = rados_connect(cluster);
if (err < 0)
{
logger.error(MOD_NAME, func_name, "failed to connect to cluster %ld: %s", id, strerror(-err));
return make_error_tuple(env, -err);
}
return enif_make_atom(env, "ok");
}
int
create_image()
{
int r;
int order = 0;
r = rados_create(&cluster, NULL);
if (r < 0) {
simple_err("Could not create cluster handle", r);
return r;
}
rados_conf_parse_env(cluster, NULL);
r = rados_conf_read_file(cluster, NULL);
if (r < 0) {
simple_err("Error reading ceph config file", r);
goto failed_shutdown;
}
r = rados_connect(cluster);
if (r < 0) {
simple_err("Error connecting to cluster", r);
goto failed_shutdown;
}
r = krbd_create_from_context(rados_cct(cluster), &krbd);
if (r < 0) {
simple_err("Could not create libkrbd handle", r);
goto failed_shutdown;
}
r = rados_pool_create(cluster, pool);
if (r < 0 && r != -EEXIST) {
simple_err("Error creating pool", r);
goto failed_krbd;
}
r = rados_ioctx_create(cluster, pool, &ioctx);
if (r < 0) {
simple_err("Error creating ioctx", r);
goto failed_krbd;
}
if (clone_calls) {
r = rbd_create2(ioctx, iname, 0, RBD_FEATURE_LAYERING, &order);
} else {
r = rbd_create(ioctx, iname, 0, &order);
}
if (r < 0) {
simple_err("Error creating image", r);
goto failed_open;
}
return 0;
failed_open:
rados_ioctx_destroy(ioctx);
failed_krbd:
krbd_destroy(krbd);
failed_shutdown:
rados_shutdown(cluster);
return r;
}
/*
* Connect via RADOS protocol to a CEPH cluster.
*/
static bRC connect_to_rados(bpContext *ctx)
{
int status;
plugin_ctx *p_ctx = (plugin_ctx *)ctx->pContext;
/*
* See if we need to initialize the cluster connection.
*/
if (!p_ctx->cluster_initialized) {
status = rados_create(&p_ctx->cluster, NULL);
if (status < 0) {
berrno be;
Jmsg(ctx, M_ERROR, "rados_create() failed: %s\n", be.bstrerror(-status));
return bRC_Error;
}
status = rados_conf_read_file(p_ctx->cluster, p_ctx->rados_conffile);
if (status < 0) {
berrno be;
Jmsg(ctx, M_ERROR, "rados_conf_read_file(%s) failed: %s\n",
p_ctx->rados_conffile, be.bstrerror(-status));
return bRC_Error;
}
status = rados_connect(p_ctx->cluster);
if (status < 0) {
berrno be;
Jmsg(ctx, M_ERROR, "rados_connect() failed: %s\n", be.bstrerror(-status));
rados_shutdown(p_ctx->cluster);
return bRC_Error;
}
p_ctx->cluster_initialized = true;
}
/*
* See if we need to initialize the IO context.
*/
if (!p_ctx->ioctx) {
status = rados_ioctx_create(p_ctx->cluster, p_ctx->rados_poolname, &p_ctx->ioctx);
if (status < 0) {
berrno be;
Jmsg(ctx, M_ERROR, "rados_ioctx_create(%s) failed: %s\n",
p_ctx->rados_poolname, be.bstrerror(-status));
rados_shutdown(p_ctx->cluster);
p_ctx->cluster_initialized = false;
return bRC_Error;
}
}
return bRC_OK;
}
int main(int argc, char **argv)
{
rados_t cluster;
char cluster_name[] = "ceph";
char user_name[] = "client.admin";
APPNAME = &argv[0][2];
uint64_t flags;
int err;
/* Initialize the cluster handle */
err = rados_create2(&cluster, cluster_name, user_name, flags);
// err = rados_create(&cluster, NULL);
error_print(err, "Couldn't create the cluster handle!", "Created a cluster handle.");
/* Read a Ceph configuration file to configure the cluster handle. */
err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
error_print(err, "Cannot read config file!", "Read the config file.");
/* Connect to the cluster */
err = rados_connect(cluster);
error_print(err, "Cannot connect to cluster!", "Connected to the cluster.");
/* List pools. */
char buf[1024];
err = rados_pool_list(cluster, buf, 1024);
split_str(buf, 1024);
error_print(err, "Cannot list pools!", buf);
/* List pool stats */
char *poolname = strtok(buf, ",");
while(poolname != NULL)
{
rados_ioctx_t io;
struct rados_pool_stat_t stats;
err = rados_ioctx_create(cluster, poolname, &io);
error_print(err, "Cannot create IO context!", "IO context created.");
err = rados_ioctx_pool_stat(io, &stats);
error_print(err, "Cannot list pool stats!", "Pool stats listed.");
printf("[%s][%s]: Number of read: %"PRIu64"\n", APPNAME, poolname, stats.num_rd);
printf("[%s][%s]: Number of read in kb: %"PRIu64"\n", APPNAME, poolname, stats.num_rd_kb);
printf("[%s][%s]: Number of write: %"PRIu64"\n", APPNAME, poolname, stats.num_wr);
printf("[%s][%s]: Number of write in kb: %"PRIu64"\n", APPNAME, poolname, stats.num_wr_kb);
rados_ioctx_destroy(io);
poolname = strtok(NULL, ",");
}
/* Shut down the cluster */
rados_shutdown(cluster);
printf("[%s]: Cluster shut down.\n", APPNAME);
}
int main(int argc, char* argv[])
{
int err;
rados_t cluster;
rados_ioctx_t io;
rbd_image_t image;
char *poolname = "pool100";
char buf[IMAGE_BUF_SIZE] = {0};
err = rados_create(&cluster, NULL);
if (err < 0) {
fprintf(stderr, "%s: cannot create a cluster handle: %s\n", argv[0], strerror(-err));
exit(1);
}
err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
if (err < 0) {
fprintf(stderr, "%s: cannot read config file: %s\n", argv[0], strerror(-err));
exit(1);
}
err = rados_connect(cluster);
if (err < 0) {
fprintf(stderr, "%s: cannot connect to cluster: %s\n", argv[0], strerror(-err));
exit(1);
}
err = rados_ioctx_create(cluster, poolname, &io);
if (err < 0) {
fprintf(stderr, "%s: cannot open rados pool %s: %s\n", argv[0], poolname, strerror(-err));
rados_shutdown(cluster);
exit(1);
}
err = rbd_open(io, "user1_image2", &image, NULL);
if (err < 0){
fprintf(stderr, "open image failed: %s\n", strerror(-err));
goto out;
}
err = rbd_read(image, IMAGE_BUF_SIZE*2, IMAGE_BUF_SIZE, buf);
if (err < 0) {
fprintf(stderr, "%s: cannot read image: %s\n", poolname, strerror(-err));
}else{
fprintf(stderr, "read image return :%d\n", err);
}
out:
rados_ioctx_destroy(io);
rados_shutdown(cluster);
return 0;
}
int
main(int argc, const char **argv)
{
/* Declare the cluster handle and required arguments. */
rados_t cluster;
char cluster_name[] = "ceph";
char user_name[] = "client.admin";
uint64_t flags;
/* Initialize the cluster handle with the "ceph" cluster name and the "client.admin" user */
int err;
err = rados_create2(&cluster, cluster_name, user_name, flags);
//err = rados_create(&cluster, cluster_name);
//err = rados_create(&cluster, "admin");
if (err < 0) {
fprintf(stderr, "%s: Couldn't create the cluster handle! %s\n", argv[0], strerror(-err));
exit(-1);
} else {
printf("\nCreated a cluster handle.\n");
}
/* Read a Ceph configuration file to configure the cluster handle. */
err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
if (err < 0) {
fprintf(stderr, "%s: cannot read config file: %s\n", argv[0], strerror(-err));
exit(-1);
} else {
printf("\nRead the config file.\n");
}
#if 0
/* Read command line arguments */
err = rados_conf_parse_argv(cluster, argc, argv);
if (err < 0) {
fprintf(stderr, "%s: cannot parse command line arguments: %s\n", argv[0], strerror(-err));
exit(EXIT_FAILURE);
} else {
printf("\nRead the command line arguments.\n");
}
#endif
/* Connect to the cluster */
err = rados_connect(cluster);
if (err < 0) {
fprintf(stderr, "%s: cannot connect to cluster: %s\n", argv[0], strerror(-err));
exit(EXIT_FAILURE);
} else {
printf("\nConnected to the cluster.\n");
}
}
static int ctdb_mutex_rados_ctx_create(const char *ceph_cluster_name,
const char *ceph_auth_name,
const char *pool_name,
rados_t *_ceph_cluster,
rados_ioctx_t *_ioctx)
{
rados_t ceph_cluster = NULL;
rados_ioctx_t ioctx = NULL;
int ret;
ret = rados_create2(&ceph_cluster, ceph_cluster_name, ceph_auth_name, 0);
if (ret < 0) {
fprintf(stderr, "%s: failed to initialise Ceph cluster %s as %s"
" - (%s)\n", progname, ceph_cluster_name, ceph_auth_name,
strerror(-ret));
return ret;
}
/* path=NULL tells librados to use default locations */
ret = rados_conf_read_file(ceph_cluster, NULL);
if (ret < 0) {
fprintf(stderr, "%s: failed to parse Ceph cluster config"
" - (%s)\n", progname, strerror(-ret));
rados_shutdown(ceph_cluster);
return ret;
}
ret = rados_connect(ceph_cluster);
if (ret < 0) {
fprintf(stderr, "%s: failed to connect to Ceph cluster %s as %s"
" - (%s)\n", progname, ceph_cluster_name, ceph_auth_name,
strerror(-ret));
rados_shutdown(ceph_cluster);
return ret;
}
ret = rados_ioctx_create(ceph_cluster, pool_name, &ioctx);
if (ret < 0) {
fprintf(stderr, "%s: failed to create Ceph ioctx for pool %s"
" - (%s)\n", progname, pool_name, strerror(-ret));
rados_shutdown(ceph_cluster);
return ret;
}
*_ceph_cluster = ceph_cluster;
*_ioctx = ioctx;
return 0;
}
int
create_image()
{
int r;
rados_t cluster;
int order = 0;
r = rados_create(&cluster, NULL);
if (r < 0) {
simple_err("Could not create cluster handle", r);
return r;
}
rados_conf_parse_env(cluster, NULL);
r = rados_conf_read_file(cluster, NULL);
if (r < 0) {
simple_err("Error reading ceph config file", r);
goto failed_shutdown;
}
r = rados_connect(cluster);
if (r < 0) {
simple_err("Error connecting to cluster", r);
goto failed_shutdown;
}
r = rados_pool_create(cluster, pool);
if (r < 0 && r != -EEXIST) {
simple_err("Error creating pool", r);
goto failed_shutdown;
}
r = rados_ioctx_create(cluster, pool, &ioctx);
if (r < 0) {
simple_err("Error creating ioctx", r);
goto failed_shutdown;
}
r = rbd_create(ioctx, iname, 0, &order);
if (r < 0) {
simple_err("Error creating image", r);
goto failed_open;
}
return 0;
failed_open:
rados_ioctx_destroy(ioctx);
failed_shutdown:
rados_shutdown(cluster);
return r;
}
static int qemu_rbd_open(BlockDriverState *bs, const char *filename, int flags)
{
BDRVRBDState *s = bs->opaque;
char pool[RBD_MAX_POOL_NAME_SIZE];
char snap_buf[RBD_MAX_SNAP_NAME_SIZE];
char conf[RBD_MAX_CONF_SIZE];
char clientname_buf[RBD_MAX_CONF_SIZE];
char *clientname;
int r;
if (qemu_rbd_parsename(filename, pool, sizeof(pool),
snap_buf, sizeof(snap_buf),
s->name, sizeof(s->name),
conf, sizeof(conf)) < 0) {
return -EINVAL;
}
clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
r = rados_create(&s->cluster, clientname);
if (r < 0) {
error_report("error initializing");
return r;
}
s->snap = NULL;
if (snap_buf[0] != '\0') {
s->snap = g_strdup(snap_buf);
}
if (strstr(conf, "conf=") == NULL) {
/* try default location, but ignore failure */
rados_conf_read_file(s->cluster, NULL);
}
if (conf[0] != '\0') {
r = qemu_rbd_set_conf(s->cluster, conf);
if (r < 0) {
error_report("error setting config options");
goto failed_shutdown;
}
}
r = rados_connect(s->cluster);
if (r < 0) {
error_report("error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
if (r < 0) {
error_report("error opening pool %s", pool);
goto failed_shutdown;
}
r = rbd_open(s->io_ctx, s->name, &s->image, s->snap);
if (r < 0) {
error_report("error reading header from %s", s->name);
goto failed_open;
}
bs->read_only = (s->snap != NULL);
s->event_reader_pos = 0;
r = qemu_pipe(s->fds);
if (r < 0) {
error_report("error opening eventfd");
goto failed;
}
fcntl(s->fds[0], F_SETFL, O_NONBLOCK);
fcntl(s->fds[1], F_SETFL, O_NONBLOCK);
qemu_aio_set_fd_handler(s->fds[RBD_FD_READ], qemu_rbd_aio_event_reader,
NULL, qemu_rbd_aio_flush_cb, NULL, s);
return 0;
failed:
rbd_close(s->image);
failed_open:
rados_ioctx_destroy(s->io_ctx);
failed_shutdown:
rados_shutdown(s->cluster);
g_free(s->snap);
return r;
}
static int virStorageBackendRBDOpenRADOSConn(virStorageBackendRBDStatePtr *ptr,
virConnectPtr conn,
virStoragePoolObjPtr pool)
{
int ret = -1;
unsigned char *secret_value = NULL;
size_t secret_value_size;
char *rados_key = NULL;
virBuffer mon_host = VIR_BUFFER_INITIALIZER;
virSecretPtr secret = NULL;
char secretUuid[VIR_UUID_STRING_BUFLEN];
int i;
char *mon_buff = NULL;
VIR_DEBUG("Found Cephx username: %s",
pool->def->source.auth.cephx.username);
if (pool->def->source.auth.cephx.username != NULL) {
VIR_DEBUG("Using cephx authorization");
if (rados_create(&ptr->cluster,
pool->def->source.auth.cephx.username) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("failed to initialize RADOS"));
goto cleanup;
}
if (pool->def->source.auth.cephx.secret.uuidUsable) {
virUUIDFormat(pool->def->source.auth.cephx.secret.uuid, secretUuid);
VIR_DEBUG("Looking up secret by UUID: %s", secretUuid);
secret = virSecretLookupByUUIDString(conn, secretUuid);
} else if (pool->def->source.auth.cephx.secret.usage != NULL) {
VIR_DEBUG("Looking up secret by usage: %s",
pool->def->source.auth.cephx.secret.usage);
secret = virSecretLookupByUsage(conn, VIR_SECRET_USAGE_TYPE_CEPH,
pool->def->source.auth.cephx.secret.usage);
}
if (secret == NULL) {
virReportError(VIR_ERR_NO_SECRET, "%s",
_("failed to find the secret"));
goto cleanup;
}
secret_value = virSecretGetValue(secret, &secret_value_size, 0);
base64_encode_alloc((char *)secret_value,
secret_value_size, &rados_key);
memset(secret_value, 0, secret_value_size);
if (rados_key == NULL) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("failed to decode the RADOS key"));
goto cleanup;
}
VIR_DEBUG("Found cephx key: %s", rados_key);
if (rados_conf_set(ptr->cluster, "key", rados_key) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to set RADOS option: %s"),
"rados_key");
goto cleanup;
}
memset(rados_key, 0, strlen(rados_key));
if (rados_conf_set(ptr->cluster, "auth_supported", "cephx") < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to set RADOS option: %s"),
"auth_supported");
goto cleanup;
}
} else {
VIR_DEBUG("Not using cephx authorization");
if (rados_create(&ptr->cluster, NULL) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("failed to create the RADOS cluster"));
goto cleanup;
}
if (rados_conf_set(ptr->cluster, "auth_supported", "none") < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to set RADOS option: %s"),
"auth_supported");
goto cleanup;
}
}
VIR_DEBUG("Found %zu RADOS cluster monitors in the pool configuration",
pool->def->source.nhost);
for (i = 0; i < pool->def->source.nhost; i++) {
if (pool->def->source.hosts[i].name != NULL &&
!pool->def->source.hosts[i].port) {
virBufferAsprintf(&mon_host, "%s:6789,",
pool->def->source.hosts[i].name);
} else if (pool->def->source.hosts[i].name != NULL &&
pool->def->source.hosts[i].port) {
virBufferAsprintf(&mon_host, "%s:%d,",
pool->def->source.hosts[i].name,
pool->def->source.hosts[i].port);
} else {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("received malformed monitor, check the XML definition"));
}
}
if (virBufferError(&mon_host)) {
virReportOOMError();
goto cleanup;
}
mon_buff = virBufferContentAndReset(&mon_host);
VIR_DEBUG("RADOS mon_host has been set to: %s", mon_buff);
if (rados_conf_set(ptr->cluster, "mon_host", mon_buff) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to set RADOS option: %s"),
"mon_host");
goto cleanup;
}
ptr->starttime = time(0);
if (rados_connect(ptr->cluster) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to connect to the RADOS monitor on: %s"),
mon_buff);
goto cleanup;
}
ret = 0;
cleanup:
VIR_FREE(secret_value);
VIR_FREE(rados_key);
virSecretFree(secret);
virBufferFreeAndReset(&mon_host);
VIR_FREE(mon_buff);
return ret;
}
static int testrados(void)
{
char tmp[32];
int i, r;
rados_t cl;
if (rados_create(&cl, NULL) < 0) {
printf("error initializing\n");
return 1;
}
if (rados_conf_read_file(cl, NULL)) {
printf("error reading configuration file\n");
return 1;
}
// Try to set a configuration option that doesn't exist.
// This should fail.
if (!rados_conf_set(cl, "config option that doesn't exist",
"some random value")) {
printf("error: succeeded in setting nonexistent config option\n");
return 1;
}
if (rados_conf_get(cl, "log to stderr", tmp, sizeof(tmp))) {
printf("error: failed to read log_to_stderr from config\n");
return 1;
}
// Can we change it?
if (rados_conf_set(cl, "log to stderr", "2")) {
printf("error: error setting log_to_stderr\n");
return 1;
}
if (rados_conf_get(cl, "log to stderr", tmp, sizeof(tmp))) {
printf("error: failed to read log_to_stderr from config\n");
return 1;
}
if (tmp[0] != '2') {
printf("error: new setting for log_to_stderr failed to take effect.\n");
return 1;
}
if (rados_connect(cl)) {
printf("error connecting\n");
return 1;
}
if (rados_connect(cl) == 0) {
printf("second connect attempt didn't return an error\n");
return 1;
}
/* create an io_ctx */
r = rados_pool_create(cl, "foo");
printf("rados_pool_create = %d\n", r);
rados_ioctx_t io_ctx;
r = rados_ioctx_create(cl, "foo", &io_ctx);
printf("rados_ioctx_create = %d, io_ctx = %p\n", r, io_ctx);
/* list all pools */
{
int buf_sz = rados_pool_list(cl, NULL, 0);
printf("need buffer size of %d\n", buf_sz);
char buf[buf_sz];
int r = rados_pool_list(cl, buf, buf_sz);
if (r != buf_sz) {
printf("buffer size mismatch: got %d the first time, but %d "
"the second.\n", buf_sz, r);
return 1;
}
const char *b = buf;
printf("begin pools.\n");
while (1) {
if (b[0] == '\0')
break;
printf(" pool: '%s'\n", b);
b += strlen(b) + 1;
};
printf("end pools.\n");
}
/* stat */
struct rados_pool_stat_t st;
r = rados_ioctx_pool_stat(io_ctx, &st);
printf("rados_ioctx_pool_stat = %d, %lld KB, %lld objects\n", r, (long long)st.num_kb, (long long)st.num_objects);
/* snapshots */
r = rados_ioctx_snap_create(io_ctx, "snap1");
printf("rados_ioctx_snap_create snap1 = %d\n", r);
rados_snap_t snaps[10];
r = rados_ioctx_snap_list(io_ctx, snaps, 10);
for (i=0; i<r; i++) {
char name[100];
rados_ioctx_snap_get_name(io_ctx, snaps[i], name, sizeof(name));
printf("rados_ioctx_snap_list got snap %lld %s\n", (long long)snaps[i], name);
}
rados_snap_t snapid;
r = rados_ioctx_snap_lookup(io_ctx, "snap1", &snapid);
printf("rados_ioctx_snap_lookup snap1 got %lld, result %d\n", (long long)snapid, r);
r = rados_ioctx_snap_remove(io_ctx, "snap1");
printf("rados_ioctx_snap_remove snap1 = %d\n", r);
/* sync io */
time_t tm;
char buf[128], buf2[128];
time(&tm);
snprintf(buf, 128, "%s", ctime(&tm));
const char *oid = "foo_object";
r = rados_write(io_ctx, oid, buf, strlen(buf) + 1, 0);
printf("rados_write = %d\n", r);
r = rados_read(io_ctx, oid, buf2, sizeof(buf2), 0);
printf("rados_read = %d\n", r);
if (memcmp(buf, buf2, r))
printf("*** content mismatch ***\n");
/* attrs */
if (do_rados_setxattr(io_ctx, oid, "b", "2"))
return 1;
if (do_rados_setxattr(io_ctx, oid, "a", "1"))
return 1;
if (do_rados_setxattr(io_ctx, oid, "c", "3"))
return 1;
if (do_rados_getxattr(io_ctx, oid, "a", "1"))
return 1;
if (do_rados_getxattr(io_ctx, oid, "b", "2"))
return 1;
if (do_rados_getxattr(io_ctx, oid, "c", "3"))
return 1;
const char *exkeys[] = { "a", "b", "c", NULL };
const char *exvals[] = { "1", "2", "3", NULL };
if (do_rados_getxattrs(io_ctx, oid, exkeys, exvals))
return 1;
uint64_t size;
time_t mtime;
r = rados_stat(io_ctx, oid, &size, &mtime);
printf("rados_stat size = %lld mtime = %d = %d\n", (long long)size, (int)mtime, r);
r = rados_stat(io_ctx, "does_not_exist", NULL, NULL);
printf("rados_stat(does_not_exist) = %d\n", r);
/* exec */
rados_exec(io_ctx, oid, "crypto", "md5", buf, strlen(buf) + 1, buf, 128);
printf("exec result=%s\n", buf);
r = rados_read(io_ctx, oid, buf2, 128, 0);
printf("read result=%s\n", buf2);
printf("size=%d\n", r);
/* aio */
rados_completion_t a, b;
rados_aio_create_completion(0, 0, 0, &a);
rados_aio_create_completion(0, 0, 0, &b);
rados_aio_write(io_ctx, "a", a, buf, 100, 0);
rados_aio_write(io_ctx, "../b/bb_bb_bb\\foo\\bar", b, buf, 100, 0);
rados_aio_wait_for_safe(a);
printf("a safe\n");
rados_aio_wait_for_safe(b);
printf("b safe\n");
rados_aio_release(a);
rados_aio_release(b);
/* test flush */
printf("testing aio flush\n");
rados_completion_t c;
rados_aio_create_completion(0, 0, 0, &c);
rados_aio_write(io_ctx, "c", c, buf, 100, 0);
int safe = rados_aio_is_safe(c);
printf("a should not yet be safe and ... %s\n", safe ? "is":"is not");
assert(!safe);
rados_aio_flush(io_ctx);
safe = rados_aio_is_safe(c);
printf("a should be safe and ... %s\n", safe ? "is":"is not");
assert(safe);
rados_aio_release(c);
rados_read(io_ctx, "../b/bb_bb_bb\\foo\\bar", buf2, 128, 0);
/* list objects */
rados_list_ctx_t h;
r = rados_objects_list_open(io_ctx, &h);
printf("rados_list_objects_open = %d, h = %p\n", r, h);
const char *poolname;
while (rados_objects_list_next(h, &poolname) == 0)
printf("rados_list_objects_next got object '%s'\n", poolname);
rados_objects_list_close(h);
/* stat */
r = rados_ioctx_pool_stat(io_ctx, &st);
printf("rados_stat_pool = %d, %lld KB, %lld objects\n", r, (long long)st.num_kb, (long long)st.num_objects);
/* delete a pool */
printf("rados_delete_pool = %d\n", r);
rados_ioctx_destroy(io_ctx);
r = rados_pool_delete(cl, "foo");
printf("rados_ioctx_pool_delete = %d\n", r);
rados_shutdown(cl);
return 0;
}
static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
Error **errp)
{
BDRVRBDState *s = bs->opaque;
const char *pool, *snap, *conf, *user, *image_name, *keypairs;
const char *secretid, *filename;
QemuOpts *opts;
Error *local_err = NULL;
char *mon_host = NULL;
int r;
/* If we are given a filename, parse the filename, with precedence given to
* filename encoded options */
filename = qdict_get_try_str(options, "filename");
if (filename) {
warn_report("'filename' option specified. "
"This is an unsupported option, and may be deprecated "
"in the future");
qemu_rbd_parse_filename(filename, options, &local_err);
if (local_err) {
r = -EINVAL;
error_propagate(errp, local_err);
goto exit;
}
}
opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
qemu_opts_absorb_qdict(opts, options, &local_err);
if (local_err) {
error_propagate(errp, local_err);
r = -EINVAL;
goto failed_opts;
}
mon_host = qemu_rbd_mon_host(options, &local_err);
if (local_err) {
error_propagate(errp, local_err);
r = -EINVAL;
goto failed_opts;
}
secretid = qemu_opt_get(opts, "password-secret");
pool = qemu_opt_get(opts, "pool");
conf = qemu_opt_get(opts, "conf");
snap = qemu_opt_get(opts, "snapshot");
user = qemu_opt_get(opts, "user");
image_name = qemu_opt_get(opts, "image");
keypairs = qemu_opt_get(opts, "=keyvalue-pairs");
if (!pool || !image_name) {
error_setg(errp, "Parameters 'pool' and 'image' are required");
r = -EINVAL;
goto failed_opts;
}
r = rados_create(&s->cluster, user);
if (r < 0) {
error_setg_errno(errp, -r, "error initializing");
goto failed_opts;
}
s->snap = g_strdup(snap);
s->image_name = g_strdup(image_name);
/* try default location when conf=NULL, but ignore failure */
r = rados_conf_read_file(s->cluster, conf);
if (conf && r < 0) {
error_setg_errno(errp, -r, "error reading conf file %s", conf);
goto failed_shutdown;
}
r = qemu_rbd_set_keypairs(s->cluster, keypairs, errp);
if (r < 0) {
goto failed_shutdown;
}
if (mon_host) {
r = rados_conf_set(s->cluster, "mon_host", mon_host);
if (r < 0) {
goto failed_shutdown;
}
}
if (qemu_rbd_set_auth(s->cluster, secretid, errp) < 0) {
r = -EIO;
goto failed_shutdown;
}
/*
* Fallback to more conservative semantics if setting cache
* options fails. Ignore errors from setting rbd_cache because the
* only possible error is that the option does not exist, and
* librbd defaults to no caching. If write through caching cannot
* be set up, fall back to no caching.
*/
if (flags & BDRV_O_NOCACHE) {
rados_conf_set(s->cluster, "rbd_cache", "false");
} else {
rados_conf_set(s->cluster, "rbd_cache", "true");
}
r = rados_connect(s->cluster);
if (r < 0) {
error_setg_errno(errp, -r, "error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
if (r < 0) {
error_setg_errno(errp, -r, "error opening pool %s", pool);
goto failed_shutdown;
}
/* rbd_open is always r/w */
r = rbd_open(s->io_ctx, s->image_name, &s->image, s->snap);
if (r < 0) {
error_setg_errno(errp, -r, "error reading header from %s",
s->image_name);
goto failed_open;
}
/* If we are using an rbd snapshot, we must be r/o, otherwise
* leave as-is */
if (s->snap != NULL) {
if (!bdrv_is_read_only(bs)) {
error_report("Opening rbd snapshots without an explicit "
"read-only=on option is deprecated. Future versions "
"will refuse to open the image instead of "
"automatically marking the image read-only.");
r = bdrv_set_read_only(bs, true, &local_err);
if (r < 0) {
error_propagate(errp, local_err);
goto failed_open;
}
}
}
qemu_opts_del(opts);
return 0;
failed_open:
rados_ioctx_destroy(s->io_ctx);
failed_shutdown:
rados_shutdown(s->cluster);
g_free(s->snap);
g_free(s->image_name);
failed_opts:
qemu_opts_del(opts);
g_free(mon_host);
exit:
return r;
}
int main()
{
int ret,len = 0;
rados_t cluster;
char *buf, *out_buf = NULL;
char *pool = "rbd";
char *rbd_name = "vol1";
int fd = -1;
int i = 0;
rados_completion_t read_comp, write_comp;
rbd_image_t rbd_image;
time_t start_t, end_t, total_t = 0;
unsigned char *cipher_key = (unsigned char *) "7190c8bc27ac4a1bbe1ab1cf55cf3b097190c8bc27ac4a1bbe1ab1cf55cf3b09";
unsigned char *iv = (unsigned char *) "dcbfdd41e40f74a2";
ERR_load_crypto_strings();
OpenSSL_add_all_algorithms();
ret = rados_create(&cluster, NULL);
if (ret) {
printf("error creating rados_t object\n");
return -1;
}
ret = rados_conf_read_file(cluster,"/home/shishir/repos/ceph/src/ceph.conf");
if (ret) {
printf("Error reading conf files \n");
return -1;
}
ret = rados_connect(cluster);
if (ret) {
printf("Could not connect to cluster\n");
return -1;
}
rados_ioctx_t io;
ret = rados_ioctx_create(cluster, pool, &io);
if (ret) {
printf("Could not open connection to cluster\n");
return -1;
}
ret = rbd_open(io, rbd_name, &rbd_image, NULL);
if (ret) {
printf("Failed to open image %s\n",rbd_name);
return -1;
}
posix_memalign((void**)&buf, 4096, 65536);
if (!buf) {
printf("Failed to malloc\n");
return -1;
}
fd = open("/tmp/input_file", O_RDONLY);
if (fd > 0) {
read(fd, buf, 65536);
close(fd);
}
posix_memalign((void**)&out_buf, 4096, 65536);
if (!out_buf) {
printf("Failed to malloc\n");
return -1;
}
len = strlen((char*)buf);
//printf("plain text length %lld\n",strlen((char*)buf));
//printf("===========plain text========\n");
//BIO_dump_fp(stdout,(const char*)buf,len);
//printf("=================\n");
start_t = clock();
len = encrypt (buf, strlen((char*)buf), cipher_key, iv, out_buf);
end_t = clock();
total_t = (double)(end_t - start_t)/ CLOCKS_PER_SEC;
//printf("encrypted len is %d timetaken is %fsec\n",len,total_t);
//printf("===========encrypted text========\n");
//BIO_dump_fp(stdout,(const char*)out_buf,len);
//printf("=================\n");
fd = open("/tmp/encrypted_file", O_CREAT|O_RDWR);
if (fd > 0) {
write(fd, out_buf, 65536);
close(fd);
}
ret = rbd_aio_create_completion(NULL, NULL, &write_comp);
if (ret) {
printf ("Failed to create aio completion\n");
return -1;
}
ret = rbd_aio_write(rbd_image, 0, 65536, out_buf, write_comp);
if (ret < 0) {
printf ("Failed to write aio completion\n");
return -1;
}
rbd_aio_wait_for_complete(write_comp);
rbd_aio_release(write_comp);
ret = rbd_aio_create_completion(NULL, NULL, &read_comp);
if (ret) {
printf ("Failed to create aio completion\n");
return -1;
}
memset(buf, 0, 65536);
memset(out_buf, 0, 65536);
ret = rbd_aio_read(rbd_image, 0, 65536, buf, read_comp);
if (ret < 0) {
printf ("Failed to read aio completion\n");
return -1;
}
rbd_aio_wait_for_complete(read_comp);
rbd_aio_release(read_comp);
len=65536;
start_t = clock();
len = decrypt(buf, len, cipher_key, iv, out_buf);
end_t = clock();
total_t = (double)(end_t - start_t)/ CLOCKS_PER_SEC;
//printf("decrypted len is %d time taken: %fs\n",len,total_t);
//printf("===========decrypted text========\n");
//BIO_dump_fp(stdout,(const char*)out_buf,len);
//printf("=================\n");
fd = open("/tmp/decrypted_file", O_CREAT|O_RDWR);
if (fd > 0) {
write(fd, out_buf, 65536);
close(fd);
}
rados_ioctx_destroy(io);
rados_shutdown(cluster);
free(buf);
free(out_buf);
return 0;
}
static int coroutine_fn qemu_rbd_co_create_opts(const char *filename,
QemuOpts *opts,
Error **errp)
{
Error *local_err = NULL;
int64_t bytes = 0;
int64_t objsize;
int obj_order = 0;
const char *pool, *image_name, *conf, *user, *keypairs;
const char *secretid;
rados_t cluster;
rados_ioctx_t io_ctx;
QDict *options = NULL;
int ret = 0;
secretid = qemu_opt_get(opts, "password-secret");
/* Read out options */
bytes = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
BDRV_SECTOR_SIZE);
objsize = qemu_opt_get_size_del(opts, BLOCK_OPT_CLUSTER_SIZE, 0);
if (objsize) {
if ((objsize - 1) & objsize) { /* not a power of 2? */
error_setg(errp, "obj size needs to be power of 2");
ret = -EINVAL;
goto exit;
}
if (objsize < 4096) {
error_setg(errp, "obj size too small");
ret = -EINVAL;
goto exit;
}
obj_order = ctz32(objsize);
}
options = qdict_new();
qemu_rbd_parse_filename(filename, options, &local_err);
if (local_err) {
ret = -EINVAL;
error_propagate(errp, local_err);
goto exit;
}
/*
* Caution: while qdict_get_try_str() is fine, getting non-string
* types would require more care. When @options come from -blockdev
* or blockdev_add, its members are typed according to the QAPI
* schema, but when they come from -drive, they're all QString.
*/
pool = qdict_get_try_str(options, "pool");
conf = qdict_get_try_str(options, "conf");
user = qdict_get_try_str(options, "user");
image_name = qdict_get_try_str(options, "image");
keypairs = qdict_get_try_str(options, "=keyvalue-pairs");
ret = rados_create(&cluster, user);
if (ret < 0) {
error_setg_errno(errp, -ret, "error initializing");
goto exit;
}
/* try default location when conf=NULL, but ignore failure */
ret = rados_conf_read_file(cluster, conf);
if (conf && ret < 0) {
error_setg_errno(errp, -ret, "error reading conf file %s", conf);
ret = -EIO;
goto shutdown;
}
ret = qemu_rbd_set_keypairs(cluster, keypairs, errp);
if (ret < 0) {
ret = -EIO;
goto shutdown;
}
if (qemu_rbd_set_auth(cluster, secretid, errp) < 0) {
ret = -EIO;
goto shutdown;
}
ret = rados_connect(cluster);
if (ret < 0) {
error_setg_errno(errp, -ret, "error connecting");
goto shutdown;
}
ret = rados_ioctx_create(cluster, pool, &io_ctx);
if (ret < 0) {
error_setg_errno(errp, -ret, "error opening pool %s", pool);
goto shutdown;
}
ret = rbd_create(io_ctx, image_name, bytes, &obj_order);
if (ret < 0) {
error_setg_errno(errp, -ret, "error rbd create");
}
rados_ioctx_destroy(io_ctx);
shutdown:
rados_shutdown(cluster);
exit:
QDECREF(options);
return ret;
}
static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
Error **errp)
{
BDRVRBDState *s = bs->opaque;
char pool[RBD_MAX_POOL_NAME_SIZE];
char snap_buf[RBD_MAX_SNAP_NAME_SIZE];
char conf[RBD_MAX_CONF_SIZE];
char clientname_buf[RBD_MAX_CONF_SIZE];
char *clientname;
QemuOpts *opts;
Error *local_err = NULL;
const char *filename;
int r;
opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
qemu_opts_absorb_qdict(opts, options, &local_err);
if (local_err) {
qerror_report_err(local_err);
error_free(local_err);
qemu_opts_del(opts);
return -EINVAL;
}
filename = qemu_opt_get(opts, "filename");
if (qemu_rbd_parsename(filename, pool, sizeof(pool),
snap_buf, sizeof(snap_buf),
s->name, sizeof(s->name),
conf, sizeof(conf)) < 0) {
r = -EINVAL;
goto failed_opts;
}
clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
r = rados_create(&s->cluster, clientname);
if (r < 0) {
error_report("error initializing");
goto failed_opts;
}
s->snap = NULL;
if (snap_buf[0] != '\0') {
s->snap = g_strdup(snap_buf);
}
/*
* Fallback to more conservative semantics if setting cache
* options fails. Ignore errors from setting rbd_cache because the
* only possible error is that the option does not exist, and
* librbd defaults to no caching. If write through caching cannot
* be set up, fall back to no caching.
*/
if (flags & BDRV_O_NOCACHE) {
rados_conf_set(s->cluster, "rbd_cache", "false");
} else {
rados_conf_set(s->cluster, "rbd_cache", "true");
}
if (strstr(conf, "conf=") == NULL) {
/* try default location, but ignore failure */
rados_conf_read_file(s->cluster, NULL);
}
if (conf[0] != '\0') {
r = qemu_rbd_set_conf(s->cluster, conf);
if (r < 0) {
error_report("error setting config options");
goto failed_shutdown;
}
}
r = rados_connect(s->cluster);
if (r < 0) {
error_report("error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
if (r < 0) {
error_report("error opening pool %s", pool);
goto failed_shutdown;
}
r = rbd_open(s->io_ctx, s->name, &s->image, s->snap);
if (r < 0) {
error_report("error reading header from %s", s->name);
goto failed_open;
}
bs->read_only = (s->snap != NULL);
qemu_opts_del(opts);
return 0;
failed_open:
rados_ioctx_destroy(s->io_ctx);
failed_shutdown:
rados_shutdown(s->cluster);
g_free(s->snap);
failed_opts:
qemu_opts_del(opts);
return r;
}
static int app_rbd_open()
{
int ret;
/* poolname imagename username password monitor */
fprintf(stdout,
"open rados as following setting:\n"
"poolname: %s\n"
"imagename: %s\n"
"username: %s\n"
"password: %s\n"
"monitor: %s\n",
poolname, imagename, username, password, monitor);
/**
* 创建一个 rados_t 句柄, 该句柄暴扣了rados 客户端的数据结构, 用来和
* rados 通信, 第二个参数是连接 rados 的客户端 ID, 这里我们使用 admin
*/
ret = rados_create(&cluster, username);
if (ret < 0) {
fprintf(stderr, "cannot create a cluster handle: %s\n", strerror(-ret));
goto failed_create;
}
/* 载入配置文件 */
ret = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
if (ret < 0) {
fprintf(stderr, "cannot read config file: %s\n", strerror(-ret));
goto failed_shutdown;
}
/**
* 调用 rados_conf_set() 设置 rados 参数, 包括认证信息, monitor 地址等,
* 如果设置了 cephx 认证, 那么之前创建 rados 句柄的时候, 必须设置客户端
* ID, 并且必须设置 key 的密码.
*/
if (conf_set("key", password) < 0) {
goto failed_shutdown;
}
if (conf_set("auth_supported", "cephx") < 0) {
goto failed_shutdown;
}
if (conf_set("mon_host", monitor) < 0) {
goto failed_shutdown;
}
/* 完成了上面的设置之后, 就可以用我们的 rados 句柄连接 rados 服务器了 */
ret = rados_connect(cluster);
if (ret < 0) {
fprintf(stderr, "cannot connect to cluster: %s\n", strerror(-ret));
goto failed_shutdown;
}
/**
* 成功连接上 rados 服务器之后, 就可以用 rados_ioctx_create() 打开 rados
* 上的 pool 了, 该函数需要传递一个 rados_ioctx_t, 用来对打开的 pool 进行
* 操作.
*/
ret = rados_ioctx_create(cluster, poolname, &io_ctx);
if (ret < 0) {
fprintf(stderr, "cannot open rados pool %s: %s\n",
poolname, strerror(-ret));
goto failed_shutdown;
}
/**
* 上面说过, 我们已经能够有一个 rados_ioctx_t 的指针了, 该指针用来对关联的
* pool 进行 I/O 操作, 比如打开池中的 image 等, 这里我们直接对 rbd 操作.
*/
ret = rbd_open(io_ctx, imagename, &image, NULL);
if (ret < 0) {
fprintf(stderr, "error reading header from image %s\n", imagename);
goto failed_open;
}
return 0;
failed_open:
rados_ioctx_destroy(io_ctx);
failed_shutdown:
rados_shutdown(cluster);
failed_create:
return ret;
}
static int qemu_rbd_create(const char *filename, QemuOpts *opts, Error **errp)
{
Error *local_err = NULL;
int64_t bytes = 0;
int64_t objsize;
int obj_order = 0;
char pool[RBD_MAX_POOL_NAME_SIZE];
char name[RBD_MAX_IMAGE_NAME_SIZE];
char snap_buf[RBD_MAX_SNAP_NAME_SIZE];
char conf[RBD_MAX_CONF_SIZE];
char clientname_buf[RBD_MAX_CONF_SIZE];
char *clientname;
rados_t cluster;
rados_ioctx_t io_ctx;
int ret;
if (qemu_rbd_parsename(filename, pool, sizeof(pool),
snap_buf, sizeof(snap_buf),
name, sizeof(name),
conf, sizeof(conf), &local_err) < 0) {
error_propagate(errp, local_err);
return -EINVAL;
}
/* Read out options */
bytes = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);
objsize = qemu_opt_get_size_del(opts, BLOCK_OPT_CLUSTER_SIZE, 0);
if (objsize) {
if ((objsize - 1) & objsize) { /* not a power of 2? */
error_setg(errp, "obj size needs to be power of 2");
return -EINVAL;
}
if (objsize < 4096) {
error_setg(errp, "obj size too small");
return -EINVAL;
}
obj_order = ffs(objsize) - 1;
}
clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
if (rados_create(&cluster, clientname) < 0) {
error_setg(errp, "error initializing");
return -EIO;
}
if (strstr(conf, "conf=") == NULL) {
/* try default location, but ignore failure */
rados_conf_read_file(cluster, NULL);
}
if (conf[0] != '\0' &&
qemu_rbd_set_conf(cluster, conf, &local_err) < 0) {
rados_shutdown(cluster);
error_propagate(errp, local_err);
return -EIO;
}
if (rados_connect(cluster) < 0) {
error_setg(errp, "error connecting");
rados_shutdown(cluster);
return -EIO;
}
if (rados_ioctx_create(cluster, pool, &io_ctx) < 0) {
error_setg(errp, "error opening pool %s", pool);
rados_shutdown(cluster);
return -EIO;
}
ret = rbd_create(io_ctx, name, bytes, &obj_order);
rados_ioctx_destroy(io_ctx);
rados_shutdown(cluster);
return ret;
}
int main(int argc, const char **argv)
{
int ret = 0;
// we will use all of these below
const char *pool_name = "hello_world_pool";
const char* hello = "hello world!";
const char* object_name = "hello_object";
rados_ioctx_t io_ctx = NULL;
int pool_created = 0;
// first, we create a Rados object and initialize it
rados_t rados = NULL;
{
ret = rados_create(&rados, "admin"); // just use the client.admin keyring
if (ret < 0) { // let's handle any error that might have come back
printf("couldn't initialize rados! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we just set up a rados cluster object\n");
}
}
/*
* Now we need to get the rados object its config info. It can
* parse argv for us to find the id, monitors, etc, so let's just
* use that.
*/
{
ret = rados_conf_parse_argv(rados, argc, argv);
if (ret < 0) {
// This really can't happen, but we need to check to be a good citizen.
printf("failed to parse config options! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we just parsed our config options\n");
// We also want to apply the config file if the user specified
// one, and conf_parse_argv won't do that for us.
int i;
for (i = 0; i < argc; ++i) {
if ((strcmp(argv[i], "-c") == 0) || (strcmp(argv[i], "--conf") == 0)) {
ret = rados_conf_read_file(rados, argv[i+1]);
if (ret < 0) {
// This could fail if the config file is malformed, but it'd be hard.
printf("failed to parse config file %s! error %d\n", argv[i+1], ret);
ret = EXIT_FAILURE;
goto out;
}
break;
}
}
}
}
/*
* next, we actually connect to the cluster
*/
{
ret = rados_connect(rados);
if (ret < 0) {
printf("couldn't connect to cluster! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we just connected to the rados cluster\n");
}
}
/*
* let's create our own pool instead of scribbling over real data.
* Note that this command creates pools with default PG counts specified
* by the monitors, which may not be appropriate for real use -- it's fine
* for testing, though.
*/
{
ret = rados_pool_create(rados, pool_name);
if (ret < 0) {
printf("couldn't create pool! error %d\n", ret);
return EXIT_FAILURE;
} else {
printf("we just created a new pool named %s\n", pool_name);
}
pool_created = 1;
}
/*
* create an "IoCtx" which is used to do IO to a pool
*/
{
ret = rados_ioctx_create(rados, pool_name, &io_ctx);
if (ret < 0) {
printf("couldn't set up ioctx! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we just created an ioctx for our pool\n");
}
}
/*
* now let's do some IO to the pool! We'll write "hello world!" to a
* new object.
*/
{
/*
* now that we have the data to write, let's send it to an object.
* We'll use the synchronous interface for simplicity.
*/
ret = rados_write_full(io_ctx, object_name, hello, strlen(hello));
if (ret < 0) {
printf("couldn't write object! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we just wrote new object %s, with contents '%s'\n", object_name, hello);
}
}
/*
* now let's read that object back! Just for fun, we'll do it using
* async IO instead of synchronous. (This would be more useful if we
* wanted to send off multiple reads at once; see
* http://ceph.com/docs/master/rados/api/librados/#asychronous-io )
*/
{
int read_len = 4194304; // this is way more than we need
char* read_buf = malloc(read_len + 1); // add one for the terminating 0 we'll add later
if (!read_buf) {
printf("couldn't allocate read buffer\n");
ret = EXIT_FAILURE;
goto out;
}
// allocate the completion from librados
rados_completion_t read_completion;
ret = rados_aio_create_completion(NULL, NULL, NULL, &read_completion);
if (ret < 0) {
printf("couldn't create completion! error %d\n", ret);
ret = EXIT_FAILURE;
free(read_buf);
goto out;
} else {
printf("we just created a new completion\n");
}
// send off the request.
ret = rados_aio_read(io_ctx, object_name, read_completion, read_buf, read_len, 0);
if (ret < 0) {
printf("couldn't start read object! error %d\n", ret);
ret = EXIT_FAILURE;
free(read_buf);
rados_aio_release(read_completion);
goto out;
}
// wait for the request to complete, and check that it succeeded.
rados_aio_wait_for_complete(read_completion);
ret = rados_aio_get_return_value(read_completion);
if (ret < 0) {
printf("couldn't read object! error %d\n", ret);
ret = EXIT_FAILURE;
free(read_buf);
rados_aio_release(read_completion);
goto out;
} else {
read_buf[ret] = 0; // null-terminate the string
printf("we read our object %s, and got back %d bytes with contents\n%s\n", object_name, ret, read_buf);
}
free(read_buf);
rados_aio_release(read_completion);
}
/*
* We can also use xattrs that go alongside the object.
*/
{
const char* version = "1";
ret = rados_setxattr(io_ctx, object_name, "version", version, strlen(version));
if (ret < 0) {
printf("failed to set xattr version entry! error %d\n", ret);
ret = EXIT_FAILURE;
goto out;
} else {
printf("we set the xattr 'version' on our object!\n");
}
}
/*
* And if we want to be really cool, we can do multiple things in a single
* atomic operation. For instance, we can update the contents of our object
* and set the version at the same time.
*/
{
const char* content = "v2";
rados_write_op_t write_op = rados_create_write_op();
if (!write_op) {
printf("failed to allocate write op\n");
ret = EXIT_FAILURE;
goto out;
}
rados_write_op_write_full(write_op, content, strlen(content));
const char* version = "2";
rados_write_op_setxattr(write_op, "version", version, strlen(version));
ret = rados_write_op_operate(write_op, io_ctx, object_name, NULL, 0);
if (ret < 0) {
printf("failed to do compound write! error %d\n", ret);
ret = EXIT_FAILURE;
rados_release_write_op(write_op);
goto out;
} else {
printf("we overwrote our object %s with contents\n%s\n", object_name, content);
}
rados_release_write_op(write_op);
}
/*
* And to be even cooler, we can make sure that the object looks the
* way we expect before doing the write! Notice how this attempt fails
* because the xattr differs.
*/
{
rados_write_op_t failed_write_op = rados_create_write_op();
if (!failed_write_op) {
printf("failed to allocate write op\n");
ret = EXIT_FAILURE;
goto out;
}
const char* content = "v2";
const char* version = "2";
const char* old_version = "1";
rados_write_op_cmpxattr(failed_write_op, "version", LIBRADOS_CMPXATTR_OP_EQ, old_version, strlen(old_version));
rados_write_op_write_full(failed_write_op, content, strlen(content));
rados_write_op_setxattr(failed_write_op, "version", version, strlen(version));
ret = rados_write_op_operate(failed_write_op, io_ctx, object_name, NULL, 0);
if (ret < 0) {
printf("we just failed a write because the xattr wasn't as specified\n");
} else {
printf("we succeeded on writing despite an xattr comparison mismatch!\n");
ret = EXIT_FAILURE;
rados_release_write_op(failed_write_op);
goto out;
}
rados_release_write_op(failed_write_op);
/*
* Now let's do the update with the correct xattr values so it
* actually goes through
*/
content = "v3";
old_version = "2";
version = "3";
rados_write_op_t update_op = rados_create_write_op();
if (!failed_write_op) {
printf("failed to allocate write op\n");
ret = EXIT_FAILURE;
goto out;
}
rados_write_op_cmpxattr(update_op, "version", LIBRADOS_CMPXATTR_OP_EQ, old_version, strlen(old_version));
rados_write_op_write_full(update_op, content, strlen(content));
rados_write_op_setxattr(update_op, "version", version, strlen(version));
ret = rados_write_op_operate(update_op, io_ctx, object_name, NULL, 0);
if (ret < 0) {
printf("failed to do a compound write update! error %d\n", ret);
ret = EXIT_FAILURE;
rados_release_write_op(update_op);
goto out;
} else {
printf("we overwrote our object %s following an xattr test with contents\n%s\n", object_name, content);
}
rados_release_write_op(update_op);
}
ret = EXIT_SUCCESS;
out:
if (io_ctx) {
rados_ioctx_destroy(io_ctx);
}
if (pool_created) {
/*
* And now we're done, so let's remove our pool and then
* shut down the connection gracefully.
*/
int delete_ret = rados_pool_delete(rados, pool_name);
if (delete_ret < 0) {
// be careful not to
printf("We failed to delete our test pool!\n");
ret = EXIT_FAILURE;
}
}
rados_shutdown(rados);
return ret;
}
static int qemu_rbd_create(const char *filename, QemuOpts *opts, Error **errp)
{
Error *local_err = NULL;
int64_t bytes = 0;
int64_t objsize;
int obj_order = 0;
const char *pool, *name, *conf, *clientname, *keypairs;
const char *secretid;
rados_t cluster;
rados_ioctx_t io_ctx;
QDict *options = NULL;
int ret = 0;
secretid = qemu_opt_get(opts, "password-secret");
/* Read out options */
bytes = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
BDRV_SECTOR_SIZE);
objsize = qemu_opt_get_size_del(opts, BLOCK_OPT_CLUSTER_SIZE, 0);
if (objsize) {
if ((objsize - 1) & objsize) { /* not a power of 2? */
error_setg(errp, "obj size needs to be power of 2");
ret = -EINVAL;
goto exit;
}
if (objsize < 4096) {
error_setg(errp, "obj size too small");
ret = -EINVAL;
goto exit;
}
obj_order = ctz32(objsize);
}
options = qdict_new();
qemu_rbd_parse_filename(filename, options, &local_err);
if (local_err) {
ret = -EINVAL;
error_propagate(errp, local_err);
goto exit;
}
pool = qdict_get_try_str(options, "pool");
conf = qdict_get_try_str(options, "conf");
clientname = qdict_get_try_str(options, "user");
name = qdict_get_try_str(options, "image");
keypairs = qdict_get_try_str(options, "=keyvalue-pairs");
ret = rados_create(&cluster, clientname);
if (ret < 0) {
error_setg_errno(errp, -ret, "error initializing");
goto exit;
}
/* try default location when conf=NULL, but ignore failure */
ret = rados_conf_read_file(cluster, conf);
if (conf && ret < 0) {
error_setg_errno(errp, -ret, "error reading conf file %s", conf);
ret = -EIO;
goto shutdown;
}
ret = qemu_rbd_set_keypairs(cluster, keypairs, errp);
if (ret < 0) {
ret = -EIO;
goto shutdown;
}
if (qemu_rbd_set_auth(cluster, secretid, errp) < 0) {
ret = -EIO;
goto shutdown;
}
ret = rados_connect(cluster);
if (ret < 0) {
error_setg_errno(errp, -ret, "error connecting");
goto shutdown;
}
ret = rados_ioctx_create(cluster, pool, &io_ctx);
if (ret < 0) {
error_setg_errno(errp, -ret, "error opening pool %s", pool);
goto shutdown;
}
ret = rbd_create(io_ctx, name, bytes, &obj_order);
if (ret < 0) {
error_setg_errno(errp, -ret, "error rbd create");
}
rados_ioctx_destroy(io_ctx);
shutdown:
rados_shutdown(cluster);
exit:
QDECREF(options);
return ret;
}
static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
Error **errp)
{
BDRVRBDState *s = bs->opaque;
const char *pool, *snap, *conf, *clientname, *name, *keypairs;
const char *secretid;
QemuOpts *opts;
Error *local_err = NULL;
char *mon_host = NULL;
int r;
opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
qemu_opts_absorb_qdict(opts, options, &local_err);
if (local_err) {
error_propagate(errp, local_err);
r = -EINVAL;
goto failed_opts;
}
mon_host = qemu_rbd_mon_host(options, &local_err);
if (local_err) {
error_propagate(errp, local_err);
r = -EINVAL;
goto failed_opts;
}
secretid = qemu_opt_get(opts, "password-secret");
pool = qemu_opt_get(opts, "pool");
conf = qemu_opt_get(opts, "conf");
snap = qemu_opt_get(opts, "snapshot");
clientname = qemu_opt_get(opts, "user");
name = qemu_opt_get(opts, "image");
keypairs = qemu_opt_get(opts, "=keyvalue-pairs");
if (!pool || !name) {
error_setg(errp, "Parameters 'pool' and 'image' are required");
r = -EINVAL;
goto failed_opts;
}
r = rados_create(&s->cluster, clientname);
if (r < 0) {
error_setg_errno(errp, -r, "error initializing");
goto failed_opts;
}
s->snap = g_strdup(snap);
s->name = g_strdup(name);
/* try default location when conf=NULL, but ignore failure */
r = rados_conf_read_file(s->cluster, conf);
if (conf && r < 0) {
error_setg_errno(errp, -r, "error reading conf file %s", conf);
goto failed_shutdown;
}
r = qemu_rbd_set_keypairs(s->cluster, keypairs, errp);
if (r < 0) {
goto failed_shutdown;
}
if (mon_host) {
r = rados_conf_set(s->cluster, "mon_host", mon_host);
if (r < 0) {
goto failed_shutdown;
}
}
if (qemu_rbd_set_auth(s->cluster, secretid, errp) < 0) {
r = -EIO;
goto failed_shutdown;
}
/*
* Fallback to more conservative semantics if setting cache
* options fails. Ignore errors from setting rbd_cache because the
* only possible error is that the option does not exist, and
* librbd defaults to no caching. If write through caching cannot
* be set up, fall back to no caching.
*/
if (flags & BDRV_O_NOCACHE) {
rados_conf_set(s->cluster, "rbd_cache", "false");
} else {
rados_conf_set(s->cluster, "rbd_cache", "true");
}
r = rados_connect(s->cluster);
if (r < 0) {
error_setg_errno(errp, -r, "error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
if (r < 0) {
error_setg_errno(errp, -r, "error opening pool %s", pool);
goto failed_shutdown;
}
r = rbd_open(s->io_ctx, s->name, &s->image, s->snap);
if (r < 0) {
error_setg_errno(errp, -r, "error reading header from %s", s->name);
goto failed_open;
}
bs->read_only = (s->snap != NULL);
qemu_opts_del(opts);
return 0;
failed_open:
rados_ioctx_destroy(s->io_ctx);
failed_shutdown:
rados_shutdown(s->cluster);
g_free(s->snap);
g_free(s->name);
failed_opts:
qemu_opts_del(opts);
g_free(mon_host);
return r;
}
static int qemu_rbd_create(const char *filename, QEMUOptionParameter *options)
{
int64_t bytes = 0;
int64_t objsize;
int obj_order = 0;
char pool[RBD_MAX_POOL_NAME_SIZE];
char name[RBD_MAX_IMAGE_NAME_SIZE];
char snap_buf[RBD_MAX_SNAP_NAME_SIZE];
char conf[RBD_MAX_CONF_SIZE];
char clientname_buf[RBD_MAX_CONF_SIZE];
char *clientname;
rados_t cluster;
rados_ioctx_t io_ctx;
int ret;
if (qemu_rbd_parsename(filename, pool, sizeof(pool),
snap_buf, sizeof(snap_buf),
name, sizeof(name),
conf, sizeof(conf)) < 0) {
return -EINVAL;
}
/* Read out options */
while (options && options->name) {
if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
bytes = options->value.n;
} else if (!strcmp(options->name, BLOCK_OPT_CLUSTER_SIZE)) {
if (options->value.n) {
objsize = options->value.n;
if ((objsize - 1) & objsize) { /* not a power of 2? */
error_report("obj size needs to be power of 2");
return -EINVAL;
}
if (objsize < 4096) {
error_report("obj size too small");
return -EINVAL;
}
obj_order = ffs(objsize) - 1;
}
}
options++;
}
clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
if (rados_create(&cluster, clientname) < 0) {
error_report("error initializing");
return -EIO;
}
if (strstr(conf, "conf=") == NULL) {
/* try default location, but ignore failure */
rados_conf_read_file(cluster, NULL);
}
if (conf[0] != '\0' &&
qemu_rbd_set_conf(cluster, conf) < 0) {
error_report("error setting config options");
rados_shutdown(cluster);
return -EIO;
}
if (rados_connect(cluster) < 0) {
error_report("error connecting");
rados_shutdown(cluster);
return -EIO;
}
if (rados_ioctx_create(cluster, pool, &io_ctx) < 0) {
error_report("error opening pool %s", pool);
rados_shutdown(cluster);
return -EIO;
}
ret = rbd_create(io_ctx, name, bytes, &obj_order);
rados_ioctx_destroy(io_ctx);
rados_shutdown(cluster);
return ret;
}
static int
virStorageBackendRBDOpenRADOSConn(virStorageBackendRBDStatePtr ptr,
virConnectPtr conn,
virStoragePoolSourcePtr source)
{
int ret = -1;
int r = 0;
virStorageAuthDefPtr authdef = source->auth;
unsigned char *secret_value = NULL;
size_t secret_value_size = 0;
char *rados_key = NULL;
virBuffer mon_host = VIR_BUFFER_INITIALIZER;
size_t i;
char *mon_buff = NULL;
const char *client_mount_timeout = "30";
const char *mon_op_timeout = "30";
const char *osd_op_timeout = "30";
const char *rbd_default_format = "2";
if (authdef) {
VIR_DEBUG("Using cephx authorization, username: %s", authdef->username);
if ((r = rados_create(&ptr->cluster, authdef->username)) < 0) {
virReportSystemError(-r, "%s", _("failed to initialize RADOS"));
goto cleanup;
}
if (!conn) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("'ceph' authentication not supported "
"for autostarted pools"));
return -1;
}
if (virSecretGetSecretString(conn, &authdef->seclookupdef,
VIR_SECRET_USAGE_TYPE_CEPH,
&secret_value, &secret_value_size) < 0)
goto cleanup;
if (!(rados_key = virStringEncodeBase64(secret_value, secret_value_size)))
goto cleanup;
VIR_DEBUG("Found cephx key: %s", rados_key);
if (rados_conf_set(ptr->cluster, "key", rados_key) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to set RADOS option: %s"),
"rados_key");
goto cleanup;
}
if (rados_conf_set(ptr->cluster, "auth_supported", "cephx") < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to set RADOS option: %s"),
"auth_supported");
goto cleanup;
}
} else {
VIR_DEBUG("Not using cephx authorization");
if (rados_create(&ptr->cluster, NULL) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("failed to create the RADOS cluster"));
goto cleanup;
}
if (rados_conf_set(ptr->cluster, "auth_supported", "none") < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to set RADOS option: %s"),
"auth_supported");
goto cleanup;
}
}
VIR_DEBUG("Found %zu RADOS cluster monitors in the pool configuration",
source->nhost);
for (i = 0; i < source->nhost; i++) {
if (source->hosts[i].name != NULL &&
!source->hosts[i].port) {
virBufferAsprintf(&mon_host, "%s,",
source->hosts[i].name);
} else if (source->hosts[i].name != NULL &&
source->hosts[i].port) {
virBufferAsprintf(&mon_host, "%s:%d,",
source->hosts[i].name,
source->hosts[i].port);
} else {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("received malformed monitor, check the XML definition"));
}
}
if (virBufferCheckError(&mon_host) < 0)
goto cleanup;
mon_buff = virBufferContentAndReset(&mon_host);
VIR_DEBUG("RADOS mon_host has been set to: %s", mon_buff);
if (rados_conf_set(ptr->cluster, "mon_host", mon_buff) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("failed to set RADOS option: %s"),
"mon_host");
goto cleanup;
}
/*
* Set timeout options for librados.
* In case the Ceph cluster is down libvirt won't block forever.
* Operations in librados will return -ETIMEDOUT when the timeout is reached.
*/
VIR_DEBUG("Setting RADOS option client_mount_timeout to %s", client_mount_timeout);
rados_conf_set(ptr->cluster, "client_mount_timeout", client_mount_timeout);
VIR_DEBUG("Setting RADOS option rados_mon_op_timeout to %s", mon_op_timeout);
rados_conf_set(ptr->cluster, "rados_mon_op_timeout", mon_op_timeout);
VIR_DEBUG("Setting RADOS option rados_osd_op_timeout to %s", osd_op_timeout);
rados_conf_set(ptr->cluster, "rados_osd_op_timeout", osd_op_timeout);
/*
* Librbd supports creating RBD format 2 images. We no longer have to invoke
* rbd_create3(), we can tell librbd to default to format 2.
* This leaves us to simply use rbd_create() and use the default behavior of librbd
*/
VIR_DEBUG("Setting RADOS option rbd_default_format to %s", rbd_default_format);
rados_conf_set(ptr->cluster, "rbd_default_format", rbd_default_format);
ptr->starttime = time(0);
if ((r = rados_connect(ptr->cluster)) < 0) {
virReportSystemError(-r, _("failed to connect to the RADOS monitor on: %s"),
mon_buff);
goto cleanup;
}
ret = 0;
cleanup:
VIR_DISPOSE_N(secret_value, secret_value_size);
VIR_DISPOSE_STRING(rados_key);
virBufferFreeAndReset(&mon_host);
VIR_FREE(mon_buff);
return ret;
}
static void uwsgi_rados_add_mountpoint(char *arg, size_t arg_len) {
struct uwsgi_rados_mountpoint *urmp = uwsgi_calloc(sizeof(struct uwsgi_rados_mountpoint));
if (uwsgi_kvlist_parse(arg, arg_len, ',', '=',
"mountpoint", &urmp->mountpoint,
"config", &urmp->config,
"pool", &urmp->pool,
"timeout", &urmp->str_timeout,
"allow_put", &urmp->allow_put,
"allow_delete", &urmp->allow_delete,
"allow_mkcol", &urmp->allow_mkcol,
"allow_propfind", &urmp->allow_propfind,
"username", &urmp->username,
"buffer_size", &urmp->str_buffer_size,
"put_buffer_size", &urmp->str_put_buffer_size,
NULL)) {
uwsgi_log("unable to parse rados mountpoint definition\n");
exit(1);
}
if (!urmp->mountpoint|| !urmp->pool) {
uwsgi_log("[rados] mount requires a mountpoint, and a pool name.\n");
exit(1);
}
if (urmp->str_timeout) {
urmp->timeout = atoi(urmp->str_timeout);
}
if (urmp->str_buffer_size) {
urmp->buffer_size = atoi(urmp->str_buffer_size);
if (urmp->buffer_size > MAX_BUF_SIZE) {
urmp->buffer_size = MAX_BUF_SIZE;
}
else if (urmp->buffer_size < MIN_BUF_SIZE) {
urmp->buffer_size = MIN_BUF_SIZE;
}
}
else {
urmp->buffer_size = DEF_BUF_SIZE;
}
if (urmp->str_put_buffer_size) {
urmp->put_buffer_size = atoi(urmp->str_put_buffer_size);
if (urmp->put_buffer_size > MAX_BUF_SIZE) {
urmp->put_buffer_size = MAX_BUF_SIZE;
}
else if (urmp->put_buffer_size < MIN_BUF_SIZE) {
urmp->put_buffer_size = MIN_BUF_SIZE;
}
}
else {
urmp->put_buffer_size = urmp->buffer_size;
}
time_t now = uwsgi_now();
uwsgi_log("[rados] mounting %s ...\n", urmp->mountpoint);
rados_t cluster;
if (rados_create(&cluster, urmp->username) < 0) {
uwsgi_error("can't create Ceph cluster handle");
exit(1);
}
urmp->cluster = cluster;
if (urmp->config)
uwsgi_log("using Ceph conf:%s\n", urmp->config);
else
uwsgi_log("using default Ceph conf.\n");
if (rados_conf_read_file(cluster, urmp->config) < 0) {
uwsgi_error("can't configure Ceph cluster handle");
exit(1);
}
int timeout = urmp->timeout ? urmp->timeout : urados.timeout;
char *timeout_str = uwsgi_num2str(timeout);
rados_conf_set(cluster, "client_mount_timeout", timeout_str);
rados_conf_set(cluster, "rados_mon_op_timeout", timeout_str);
rados_conf_set(cluster, "rados_osd_op_timeout", timeout_str);
free(timeout_str);
if (rados_connect(cluster) < 0) {
uwsgi_error("can't connect with Ceph cluster");
exit(1);
}
void *ctx_ptr;
if (uwsgi.threads > 1) {
int i;
rados_ioctx_t *ctxes = uwsgi_calloc(sizeof(rados_ioctx_t) * uwsgi.threads);
for(i=0;i<uwsgi.threads;i++) {
if (rados_ioctx_create(cluster, urmp->pool, &ctxes[i]) < 0) {
uwsgi_error("can't open rados pool")
rados_shutdown(cluster);
exit(1);
}
}
ctx_ptr = ctxes;
}
else {
rados_ioctx_t ctx;
if (rados_ioctx_create(cluster, urmp->pool, &ctx) < 0) {
uwsgi_error("can't open rados pool")
rados_shutdown(cluster);
exit(1);
}
ctx_ptr = ctx;
}
char fsid[37];
rados_cluster_fsid(cluster, fsid, 37);
uwsgi_log("connected to Ceph pool: %s on cluster %.*s\n", urmp->pool, 37, fsid);
int id = uwsgi_apps_cnt;
struct uwsgi_app *ua = uwsgi_add_app(id, rados_plugin.modifier1, urmp->mountpoint, strlen(urmp->mountpoint), NULL, (void*)1);
if (!ua) {
uwsgi_log("[rados] unable to mount %s\n", urmp->mountpoint);
rados_shutdown(cluster);
exit(1);
}
ua->responder0 = ctx_ptr;
ua->responder1 = urmp;
ua->started_at = now;
ua->startup_time = uwsgi_now() - now;
uwsgi_log("Rados app/mountpoint %d (%s) loaded in %d seconds at %p\n", id, urmp->mountpoint, (int) ua->startup_time, ctx_ptr);
}
static int qemu_rbd_open(BlockDriverState *bs, const char *filename, int flags)
{
BDRVRBDState *s = bs->opaque;
char pool[RBD_MAX_POOL_NAME_SIZE];
char snap_buf[RBD_MAX_SNAP_NAME_SIZE];
char conf[RBD_MAX_CONF_SIZE];
char clientname_buf[RBD_MAX_CONF_SIZE];
char *clientname;
int r;
if (qemu_rbd_parsename(filename, pool, sizeof(pool),
snap_buf, sizeof(snap_buf),
s->name, sizeof(s->name),
conf, sizeof(conf)) < 0) {
return -EINVAL;
}
clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
r = rados_create(&s->cluster, clientname);
if (r < 0) {
error_report("error initializing");
return r;
}
s->snap = NULL;
if (snap_buf[0] != '\0') {
s->snap = g_strdup(snap_buf);
}
/*
* Fallback to more conservative semantics if setting cache
* options fails. Ignore errors from setting rbd_cache because the
* only possible error is that the option does not exist, and
* librbd defaults to no caching. If write through caching cannot
* be set up, fall back to no caching.
*/
if (flags & BDRV_O_NOCACHE) {
rados_conf_set(s->cluster, "rbd_cache", "false");
} else {
rados_conf_set(s->cluster, "rbd_cache", "true");
}
if (strstr(conf, "conf=") == NULL) {
/* try default location, but ignore failure */
rados_conf_read_file(s->cluster, NULL);
}
if (conf[0] != '\0') {
r = qemu_rbd_set_conf(s->cluster, conf);
if (r < 0) {
error_report("error setting config options");
goto failed_shutdown;
}
}
r = rados_connect(s->cluster);
if (r < 0) {
error_report("error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
if (r < 0) {
error_report("error opening pool %s", pool);
goto failed_shutdown;
}
r = rbd_open(s->io_ctx, s->name, &s->image, s->snap);
if (r < 0) {
error_report("error reading header from %s", s->name);
goto failed_open;
}
bs->read_only = (s->snap != NULL);
s->event_reader_pos = 0;
r = qemu_pipe(s->fds);
if (r < 0) {
error_report("error opening eventfd");
goto failed;
}
fcntl(s->fds[0], F_SETFL, O_NONBLOCK);
fcntl(s->fds[1], F_SETFL, O_NONBLOCK);
qemu_aio_set_fd_handler(s->fds[RBD_FD_READ], qemu_rbd_aio_event_reader,
NULL, qemu_rbd_aio_flush_cb, s);
return 0;
failed:
rbd_close(s->image);
failed_open:
rados_ioctx_destroy(s->io_ctx);
failed_shutdown:
rados_shutdown(s->cluster);
g_free(s->snap);
return r;
}
static int qemu_rbd_connect(rados_t *cluster, rados_ioctx_t *io_ctx,
BlockdevOptionsRbd *opts, bool cache,
const char *keypairs, const char *secretid,
Error **errp)
{
char *mon_host = NULL;
Error *local_err = NULL;
int r;
if (secretid) {
if (opts->key_secret) {
error_setg(errp,
"Legacy 'password-secret' clashes with 'key-secret'");
return -EINVAL;
}
opts->key_secret = g_strdup(secretid);
opts->has_key_secret = true;
}
mon_host = qemu_rbd_mon_host(opts, &local_err);
if (local_err) {
error_propagate(errp, local_err);
r = -EINVAL;
goto failed_opts;
}
r = rados_create(cluster, opts->user);
if (r < 0) {
error_setg_errno(errp, -r, "error initializing");
goto failed_opts;
}
/* try default location when conf=NULL, but ignore failure */
r = rados_conf_read_file(*cluster, opts->conf);
if (opts->has_conf && r < 0) {
error_setg_errno(errp, -r, "error reading conf file %s", opts->conf);
goto failed_shutdown;
}
r = qemu_rbd_set_keypairs(*cluster, keypairs, errp);
if (r < 0) {
goto failed_shutdown;
}
if (mon_host) {
r = rados_conf_set(*cluster, "mon_host", mon_host);
if (r < 0) {
goto failed_shutdown;
}
}
r = qemu_rbd_set_auth(*cluster, opts, errp);
if (r < 0) {
goto failed_shutdown;
}
/*
* Fallback to more conservative semantics if setting cache
* options fails. Ignore errors from setting rbd_cache because the
* only possible error is that the option does not exist, and
* librbd defaults to no caching. If write through caching cannot
* be set up, fall back to no caching.
*/
if (cache) {
rados_conf_set(*cluster, "rbd_cache", "true");
} else {
rados_conf_set(*cluster, "rbd_cache", "false");
}
r = rados_connect(*cluster);
if (r < 0) {
error_setg_errno(errp, -r, "error connecting");
goto failed_shutdown;
}
r = rados_ioctx_create(*cluster, opts->pool, io_ctx);
if (r < 0) {
error_setg_errno(errp, -r, "error opening pool %s", opts->pool);
goto failed_shutdown;
}
return 0;
failed_shutdown:
rados_shutdown(*cluster);
failed_opts:
g_free(mon_host);
return r;
}
