static int uwsgi_rados_read_async(struct wsgi_request *wsgi_req, rados_ioctx_t ctx, const char *key, uint64_t off, uint64_t remains, size_t bufsize, int timeout) { int ret = -1; char *buf = uwsgi_malloc(UMIN(remains, bufsize)); struct uwsgi_rados_io *urio = &urados.urio[wsgi_req->async_id]; // increase request counter pthread_mutex_lock(&urio->mutex); urio->rid++; pthread_mutex_unlock(&urio->mutex); while(remains > 0) { struct uwsgi_rados_cb *urcb = uwsgi_malloc(sizeof(struct uwsgi_rados_cb)); // map the current request id to the callback urcb->rid = urio->rid; // map urio to the callback urcb->urio = urio; rados_completion_t comp; if (rados_aio_create_completion(urcb, uwsgi_rados_read_async_cb, NULL, &comp) < 0) { free(urcb); break; } // trigger an async read if (rados_aio_read(ctx, key, comp, buf, UMIN(remains, bufsize), off) < 0) { free(urcb); rados_aio_release(comp); break; } // wait for the callback to be executed if (uwsgi.wait_read_hook(urio->fds[0], timeout) <= 0) { rados_aio_release(comp); break; } char ack = 1; if (read(urio->fds[0], &ack, 1) != 1) { rados_aio_release(comp); uwsgi_error("uwsgi_rados_read_async()/read()"); break; } int rlen = -1; if (rados_aio_is_complete_and_cb(comp)) { rlen = rados_aio_get_return_value(comp); } rados_aio_release(comp); if (rlen <= 0) break; if (uwsgi_response_write_body_do(wsgi_req, buf, rlen)) break; remains -= rlen; off += rlen; } free(buf); if (remains == 0) ret = 0; pthread_mutex_lock(&urio->mutex); // increase the counter again urio->rid++; pthread_mutex_unlock(&urio->mutex); return ret; }
static int uwsgi_rados_read_async(struct wsgi_request *wsgi_req, rados_ioctx_t *ctx, const char *key, size_t remains) { uint64_t off = 0; char buf[8192]; struct uwsgi_rados_async_io aio; int ret = -1; if (pipe(aio.fd)) { uwsgi_error("uwsgi_rados_read_async()/pipe()"); return -1; } aio.rlen = -1; rados_completion_t comp; if (rados_aio_create_completion(&aio, uwsgi_rados_read_async_cb, NULL, &comp) < 0) goto end; while(remains > 0) { // trigger an async read if (rados_aio_read(ctx, key, comp, buf, 8192, off) < 0) goto end; // wait for the callback to be executed if (uwsgi.wait_read_hook(aio.fd[0], urados.timeout) <= 0) goto end; if (aio.rlen <= 0) goto end; if (uwsgi_response_write_body_do(wsgi_req, buf, aio.rlen)) goto end; remains -= aio.rlen; } ret = 0; end: rados_aio_release(&comp); close(aio.fd[0]); close(aio.fd[1]); return ret; }
static int uwsgi_rados_delete(struct wsgi_request *wsgi_req, rados_ioctx_t ctx, char *key, int timeout) { if (uwsgi.async < 1) { return rados_remove(ctx, key); } struct uwsgi_rados_io *urio = &urados.urio[wsgi_req->async_id]; int ret = -1; // increase request counter pthread_mutex_lock(&urio->mutex); urio->rid++; pthread_mutex_unlock(&urio->mutex); struct uwsgi_rados_cb *urcb = uwsgi_malloc(sizeof(struct uwsgi_rados_cb)); // map the current request id to the callback urcb->rid = urio->rid; // map urio to the callback urcb->urio = urio; rados_completion_t comp; // we use the safe cb here if (rados_aio_create_completion(urcb, NULL, uwsgi_rados_read_async_cb, &comp) < 0) { free(urcb); goto end; } if (rados_aio_remove(ctx, key, comp) < 0) { free(urcb); rados_aio_release(comp); goto end; } // wait for the callback to be executed if (uwsgi.wait_read_hook(urio->fds[0], timeout) <= 0) { rados_aio_release(comp); goto end; } char ack = 1; if (read(urio->fds[0], &ack, 1) != 1) { rados_aio_release(comp); uwsgi_error("uwsgi_rados_delete()/read()"); goto end; } if (rados_aio_is_safe_and_cb(comp)) { ret = rados_aio_get_return_value(comp); } rados_aio_release(comp); end: return ret; }
static int testrados(void) { char tmp[32]; int i, r; rados_t cl; if (rados_create(&cl, NULL) < 0) { printf("error initializing\n"); return 1; } if (rados_conf_read_file(cl, NULL)) { printf("error reading configuration file\n"); return 1; } // Try to set a configuration option that doesn't exist. // This should fail. if (!rados_conf_set(cl, "config option that doesn't exist", "some random value")) { printf("error: succeeded in setting nonexistent config option\n"); return 1; } if (rados_conf_get(cl, "log to stderr", tmp, sizeof(tmp))) { printf("error: failed to read log_to_stderr from config\n"); return 1; } // Can we change it? if (rados_conf_set(cl, "log to stderr", "2")) { printf("error: error setting log_to_stderr\n"); return 1; } if (rados_conf_get(cl, "log to stderr", tmp, sizeof(tmp))) { printf("error: failed to read log_to_stderr from config\n"); return 1; } if (tmp[0] != '2') { printf("error: new setting for log_to_stderr failed to take effect.\n"); return 1; } if (rados_connect(cl)) { printf("error connecting\n"); return 1; } if (rados_connect(cl) == 0) { printf("second connect attempt didn't return an error\n"); return 1; } /* create an io_ctx */ r = rados_pool_create(cl, "foo"); printf("rados_pool_create = %d\n", r); rados_ioctx_t io_ctx; r = rados_ioctx_create(cl, "foo", &io_ctx); printf("rados_ioctx_create = %d, io_ctx = %p\n", r, io_ctx); /* list all pools */ { int buf_sz = rados_pool_list(cl, NULL, 0); printf("need buffer size of %d\n", buf_sz); char buf[buf_sz]; int r = rados_pool_list(cl, buf, buf_sz); if (r != buf_sz) { printf("buffer size mismatch: got %d the first time, but %d " "the second.\n", buf_sz, r); return 1; } const char *b = buf; printf("begin pools.\n"); while (1) { if (b[0] == '\0') break; printf(" pool: '%s'\n", b); b += strlen(b) + 1; }; printf("end pools.\n"); } /* stat */ struct rados_pool_stat_t st; r = rados_ioctx_pool_stat(io_ctx, &st); printf("rados_ioctx_pool_stat = %d, %lld KB, %lld objects\n", r, (long long)st.num_kb, (long long)st.num_objects); /* snapshots */ r = rados_ioctx_snap_create(io_ctx, "snap1"); printf("rados_ioctx_snap_create snap1 = %d\n", r); rados_snap_t snaps[10]; r = rados_ioctx_snap_list(io_ctx, snaps, 10); for (i=0; i<r; i++) { char name[100]; rados_ioctx_snap_get_name(io_ctx, snaps[i], name, sizeof(name)); printf("rados_ioctx_snap_list got snap %lld %s\n", (long long)snaps[i], name); } rados_snap_t snapid; r = rados_ioctx_snap_lookup(io_ctx, "snap1", &snapid); printf("rados_ioctx_snap_lookup snap1 got %lld, result %d\n", (long long)snapid, r); r = rados_ioctx_snap_remove(io_ctx, "snap1"); printf("rados_ioctx_snap_remove snap1 = %d\n", r); /* sync io */ time_t tm; char buf[128], buf2[128]; time(&tm); snprintf(buf, 128, "%s", ctime(&tm)); const char *oid = "foo_object"; r = rados_write(io_ctx, oid, buf, strlen(buf) + 1, 0); printf("rados_write = %d\n", r); r = rados_read(io_ctx, oid, buf2, sizeof(buf2), 0); printf("rados_read = %d\n", r); if (memcmp(buf, buf2, r)) printf("*** content mismatch ***\n"); /* attrs */ if (do_rados_setxattr(io_ctx, oid, "b", "2")) return 1; if (do_rados_setxattr(io_ctx, oid, "a", "1")) return 1; if (do_rados_setxattr(io_ctx, oid, "c", "3")) return 1; if (do_rados_getxattr(io_ctx, oid, "a", "1")) return 1; if (do_rados_getxattr(io_ctx, oid, "b", "2")) return 1; if (do_rados_getxattr(io_ctx, oid, "c", "3")) return 1; const char *exkeys[] = { "a", "b", "c", NULL }; const char *exvals[] = { "1", "2", "3", NULL }; if (do_rados_getxattrs(io_ctx, oid, exkeys, exvals)) return 1; uint64_t size; time_t mtime; r = rados_stat(io_ctx, oid, &size, &mtime); printf("rados_stat size = %lld mtime = %d = %d\n", (long long)size, (int)mtime, r); r = rados_stat(io_ctx, "does_not_exist", NULL, NULL); printf("rados_stat(does_not_exist) = %d\n", r); /* exec */ rados_exec(io_ctx, oid, "crypto", "md5", buf, strlen(buf) + 1, buf, 128); printf("exec result=%s\n", buf); r = rados_read(io_ctx, oid, buf2, 128, 0); printf("read result=%s\n", buf2); printf("size=%d\n", r); /* aio */ rados_completion_t a, b; rados_aio_create_completion(0, 0, 0, &a); rados_aio_create_completion(0, 0, 0, &b); rados_aio_write(io_ctx, "a", a, buf, 100, 0); rados_aio_write(io_ctx, "../b/bb_bb_bb\\foo\\bar", b, buf, 100, 0); rados_aio_wait_for_safe(a); printf("a safe\n"); rados_aio_wait_for_safe(b); printf("b safe\n"); rados_aio_release(a); rados_aio_release(b); /* test flush */ printf("testing aio flush\n"); rados_completion_t c; rados_aio_create_completion(0, 0, 0, &c); rados_aio_write(io_ctx, "c", c, buf, 100, 0); int safe = rados_aio_is_safe(c); printf("a should not yet be safe and ... %s\n", safe ? "is":"is not"); assert(!safe); rados_aio_flush(io_ctx); safe = rados_aio_is_safe(c); printf("a should be safe and ... %s\n", safe ? "is":"is not"); assert(safe); rados_aio_release(c); rados_read(io_ctx, "../b/bb_bb_bb\\foo\\bar", buf2, 128, 0); /* list objects */ rados_list_ctx_t h; r = rados_objects_list_open(io_ctx, &h); printf("rados_list_objects_open = %d, h = %p\n", r, h); const char *poolname; while (rados_objects_list_next(h, &poolname) == 0) printf("rados_list_objects_next got object '%s'\n", poolname); rados_objects_list_close(h); /* stat */ r = rados_ioctx_pool_stat(io_ctx, &st); printf("rados_stat_pool = %d, %lld KB, %lld objects\n", r, (long long)st.num_kb, (long long)st.num_objects); /* delete a pool */ printf("rados_delete_pool = %d\n", r); rados_ioctx_destroy(io_ctx); r = rados_pool_delete(cl, "foo"); printf("rados_ioctx_pool_delete = %d\n", r); rados_shutdown(cl); return 0; }
static int uwsgi_rados_put(struct wsgi_request *wsgi_req, rados_ioctx_t ctx, char *key, size_t buffer_size, int timeout) { struct uwsgi_rados_io *urio = &urados.urio[wsgi_req->async_id]; size_t remains = wsgi_req->post_cl; uint64_t off = 0; int ret; const char* method; int truncate = remains == 0; #ifdef HAS_RADOS_POOL_REQUIRES_ALIGNMENT2 if (!truncate && !rados_ioctx_pool_requires_alignment2(ctx, &ret) && ret) { uint64_t alignment; if (rados_ioctx_pool_required_alignment2(ctx, &alignment)) { /* ignore error here */ } else #else if (!truncate && rados_ioctx_pool_requires_alignment(ctx)) { uint64_t alignment = rados_ioctx_pool_required_alignment(ctx); #endif if (buffer_size <= alignment) { buffer_size = alignment; } else if (buffer_size <= alignment * 2) { buffer_size = alignment * 2; } else if (alignment) { buffer_size -= buffer_size % alignment; } } while(truncate || remains > 0) { ssize_t body_len = 0; char *body = "\x00"; if (!truncate) { body = uwsgi_request_body_read(wsgi_req, UMIN(remains, buffer_size) , &body_len); if (!body || body == uwsgi.empty) goto error; } else { truncate = 0; } if (uwsgi.async < 1) { if (off == 0) { ret = rados_write_full(ctx, key, body, body_len); method = "rados_write_full()"; } else { ret = rados_write(ctx, key, body, body_len, off); method = "rados_write()"; } if (ret < 0) { uwsgi_log("uwsgi_rados_put():%s() %s\n", method, strerror(-ret)); return -1; } } else { // increase request counter pthread_mutex_lock(&urio->mutex); urio->rid++; pthread_mutex_unlock(&urio->mutex); struct uwsgi_rados_cb *urcb = uwsgi_malloc(sizeof(struct uwsgi_rados_cb)); // map the current request id to the callback urcb->rid = urio->rid; // map urio to the callback urcb->urio = urio; rados_completion_t comp; // use safe for write if (rados_aio_create_completion(urcb, NULL, uwsgi_rados_read_async_cb, &comp) < 0) { free(urcb); goto error; } if (off == 0) { ret = rados_aio_write_full(ctx, key, comp, body, body_len); method = "rados_aio_write_full"; } else { ret = rados_aio_write(ctx, key, comp, body, body_len, off); method = "rados_aio_write"; } if (ret < 0) { uwsgi_log("uwsgi_rados_put():%s() %s\n", method, strerror(-ret)); free(urcb); rados_aio_release(comp); goto error; } // wait for the callback to be executed if (uwsgi.wait_read_hook(urio->fds[0], timeout) <= 0) { rados_aio_release(comp); goto error; } char ack = 1; if (read(urio->fds[0], &ack, 1) != 1) { rados_aio_release(comp); uwsgi_error("uwsgi_rados_read_async()/read()"); goto error; } if (rados_aio_is_safe_and_cb(comp)) { ret = rados_aio_get_return_value(comp); if (ret < 0) { uwsgi_log("uwsgi_rados_put():%s() %s\n", method, strerror(-ret)); rados_aio_release(comp); goto error; } } rados_aio_release(comp); } remains -= body_len; off += body_len; } return 0; error: return -1; } // async stat static int uwsgi_rados_async_stat(struct uwsgi_rados_io *urio, rados_ioctx_t ctx, const char *key, uint64_t *stat_size, time_t *stat_mtime, int timeout) { int ret = -1; // increase request counter pthread_mutex_lock(&urio->mutex); urio->rid++; pthread_mutex_unlock(&urio->mutex); struct uwsgi_rados_cb *urcb = uwsgi_malloc(sizeof(struct uwsgi_rados_cb)); // map the current request id to the callback urcb->rid = urio->rid; // map urio to the callback urcb->urio = urio; rados_completion_t comp; if (rados_aio_create_completion(urcb, uwsgi_rados_read_async_cb, NULL, &comp) < 0) { free(urcb); goto end; } if (rados_aio_stat(ctx, key, comp, stat_size, stat_mtime) < 0) { free(urcb); rados_aio_release(comp); goto end; } // wait for the callback to be executed if (uwsgi.wait_read_hook(urio->fds[0], timeout) <= 0) { rados_aio_release(comp); goto end; } char ack = 1; if (read(urio->fds[0], &ack, 1) != 1) { rados_aio_release(comp); uwsgi_error("uwsgi_rados_read_async()/read()"); goto end; } if (rados_aio_is_complete_and_cb(comp)) { ret = rados_aio_get_return_value(comp); } rados_aio_release(comp); end: return ret; }
int main(int argc, const char **argv) { int ret = 0; // we will use all of these below const char *pool_name = "hello_world_pool"; const char* hello = "hello world!"; const char* object_name = "hello_object"; rados_ioctx_t io_ctx = NULL; int pool_created = 0; // first, we create a Rados object and initialize it rados_t rados = NULL; { ret = rados_create(&rados, "admin"); // just use the client.admin keyring if (ret < 0) { // let's handle any error that might have come back printf("couldn't initialize rados! error %d\n", ret); ret = EXIT_FAILURE; goto out; } else { printf("we just set up a rados cluster object\n"); } } /* * Now we need to get the rados object its config info. It can * parse argv for us to find the id, monitors, etc, so let's just * use that. */ { ret = rados_conf_parse_argv(rados, argc, argv); if (ret < 0) { // This really can't happen, but we need to check to be a good citizen. printf("failed to parse config options! error %d\n", ret); ret = EXIT_FAILURE; goto out; } else { printf("we just parsed our config options\n"); // We also want to apply the config file if the user specified // one, and conf_parse_argv won't do that for us. int i; for (i = 0; i < argc; ++i) { if ((strcmp(argv[i], "-c") == 0) || (strcmp(argv[i], "--conf") == 0)) { ret = rados_conf_read_file(rados, argv[i+1]); if (ret < 0) { // This could fail if the config file is malformed, but it'd be hard. printf("failed to parse config file %s! error %d\n", argv[i+1], ret); ret = EXIT_FAILURE; goto out; } break; } } } } /* * next, we actually connect to the cluster */ { ret = rados_connect(rados); if (ret < 0) { printf("couldn't connect to cluster! error %d\n", ret); ret = EXIT_FAILURE; goto out; } else { printf("we just connected to the rados cluster\n"); } } /* * let's create our own pool instead of scribbling over real data. * Note that this command creates pools with default PG counts specified * by the monitors, which may not be appropriate for real use -- it's fine * for testing, though. */ { ret = rados_pool_create(rados, pool_name); if (ret < 0) { printf("couldn't create pool! error %d\n", ret); return EXIT_FAILURE; } else { printf("we just created a new pool named %s\n", pool_name); } pool_created = 1; } /* * create an "IoCtx" which is used to do IO to a pool */ { ret = rados_ioctx_create(rados, pool_name, &io_ctx); if (ret < 0) { printf("couldn't set up ioctx! error %d\n", ret); ret = EXIT_FAILURE; goto out; } else { printf("we just created an ioctx for our pool\n"); } } /* * now let's do some IO to the pool! We'll write "hello world!" to a * new object. */ { /* * now that we have the data to write, let's send it to an object. * We'll use the synchronous interface for simplicity. */ ret = rados_write_full(io_ctx, object_name, hello, strlen(hello)); if (ret < 0) { printf("couldn't write object! error %d\n", ret); ret = EXIT_FAILURE; goto out; } else { printf("we just wrote new object %s, with contents '%s'\n", object_name, hello); } } /* * now let's read that object back! Just for fun, we'll do it using * async IO instead of synchronous. (This would be more useful if we * wanted to send off multiple reads at once; see * http://ceph.com/docs/master/rados/api/librados/#asychronous-io ) */ { int read_len = 4194304; // this is way more than we need char* read_buf = malloc(read_len + 1); // add one for the terminating 0 we'll add later if (!read_buf) { printf("couldn't allocate read buffer\n"); ret = EXIT_FAILURE; goto out; } // allocate the completion from librados rados_completion_t read_completion; ret = rados_aio_create_completion(NULL, NULL, NULL, &read_completion); if (ret < 0) { printf("couldn't create completion! error %d\n", ret); ret = EXIT_FAILURE; free(read_buf); goto out; } else { printf("we just created a new completion\n"); } // send off the request. ret = rados_aio_read(io_ctx, object_name, read_completion, read_buf, read_len, 0); if (ret < 0) { printf("couldn't start read object! error %d\n", ret); ret = EXIT_FAILURE; free(read_buf); rados_aio_release(read_completion); goto out; } // wait for the request to complete, and check that it succeeded. rados_aio_wait_for_complete(read_completion); ret = rados_aio_get_return_value(read_completion); if (ret < 0) { printf("couldn't read object! error %d\n", ret); ret = EXIT_FAILURE; free(read_buf); rados_aio_release(read_completion); goto out; } else { read_buf[ret] = 0; // null-terminate the string printf("we read our object %s, and got back %d bytes with contents\n%s\n", object_name, ret, read_buf); } free(read_buf); rados_aio_release(read_completion); } /* * We can also use xattrs that go alongside the object. */ { const char* version = "1"; ret = rados_setxattr(io_ctx, object_name, "version", version, strlen(version)); if (ret < 0) { printf("failed to set xattr version entry! error %d\n", ret); ret = EXIT_FAILURE; goto out; } else { printf("we set the xattr 'version' on our object!\n"); } } /* * And if we want to be really cool, we can do multiple things in a single * atomic operation. For instance, we can update the contents of our object * and set the version at the same time. */ { const char* content = "v2"; rados_write_op_t write_op = rados_create_write_op(); if (!write_op) { printf("failed to allocate write op\n"); ret = EXIT_FAILURE; goto out; } rados_write_op_write_full(write_op, content, strlen(content)); const char* version = "2"; rados_write_op_setxattr(write_op, "version", version, strlen(version)); ret = rados_write_op_operate(write_op, io_ctx, object_name, NULL, 0); if (ret < 0) { printf("failed to do compound write! error %d\n", ret); ret = EXIT_FAILURE; rados_release_write_op(write_op); goto out; } else { printf("we overwrote our object %s with contents\n%s\n", object_name, content); } rados_release_write_op(write_op); } /* * And to be even cooler, we can make sure that the object looks the * way we expect before doing the write! Notice how this attempt fails * because the xattr differs. */ { rados_write_op_t failed_write_op = rados_create_write_op(); if (!failed_write_op) { printf("failed to allocate write op\n"); ret = EXIT_FAILURE; goto out; } const char* content = "v2"; const char* version = "2"; const char* old_version = "1"; rados_write_op_cmpxattr(failed_write_op, "version", LIBRADOS_CMPXATTR_OP_EQ, old_version, strlen(old_version)); rados_write_op_write_full(failed_write_op, content, strlen(content)); rados_write_op_setxattr(failed_write_op, "version", version, strlen(version)); ret = rados_write_op_operate(failed_write_op, io_ctx, object_name, NULL, 0); if (ret < 0) { printf("we just failed a write because the xattr wasn't as specified\n"); } else { printf("we succeeded on writing despite an xattr comparison mismatch!\n"); ret = EXIT_FAILURE; rados_release_write_op(failed_write_op); goto out; } rados_release_write_op(failed_write_op); /* * Now let's do the update with the correct xattr values so it * actually goes through */ content = "v3"; old_version = "2"; version = "3"; rados_write_op_t update_op = rados_create_write_op(); if (!failed_write_op) { printf("failed to allocate write op\n"); ret = EXIT_FAILURE; goto out; } rados_write_op_cmpxattr(update_op, "version", LIBRADOS_CMPXATTR_OP_EQ, old_version, strlen(old_version)); rados_write_op_write_full(update_op, content, strlen(content)); rados_write_op_setxattr(update_op, "version", version, strlen(version)); ret = rados_write_op_operate(update_op, io_ctx, object_name, NULL, 0); if (ret < 0) { printf("failed to do a compound write update! error %d\n", ret); ret = EXIT_FAILURE; rados_release_write_op(update_op); goto out; } else { printf("we overwrote our object %s following an xattr test with contents\n%s\n", object_name, content); } rados_release_write_op(update_op); } ret = EXIT_SUCCESS; out: if (io_ctx) { rados_ioctx_destroy(io_ctx); } if (pool_created) { /* * And now we're done, so let's remove our pool and then * shut down the connection gracefully. */ int delete_ret = rados_pool_delete(rados, pool_name); if (delete_ret < 0) { // be careful not to printf("We failed to delete our test pool!\n"); ret = EXIT_FAILURE; } } rados_shutdown(rados); return ret; }