Ejemplo n.º 1
0
static int internal_content_load (optparse_t *p, int ac, char *av[])
{
    int n;
    const char *blobref;
    uint8_t *data;
    int size;
    flux_t *h;
    flux_rpc_t *rpc;
    const char *topic;

    n = optparse_optind (p);
    if (n != ac - 1) {
        optparse_print_usage (p);
        exit (1);
    }
    blobref = av[n];
    if (!(h = builtin_get_flux_handle (p)))
        log_err_exit ("flux_open");
    if (optparse_hasopt (p, "bypass-cache"))
        topic = "content-backing.load";
    else
        topic = "content.load";
    if (!(rpc = flux_rpc_raw (h, topic, blobref, strlen (blobref) + 1, 0, 0)))
        log_err_exit ("%s", topic);
    if (flux_rpc_get_raw (rpc, &data, &size) < 0)
        log_err_exit ("%s", topic);
    if (write_all (STDOUT_FILENO, data, size) < 0)
        log_err_exit ("write");
    flux_rpc_destroy (rpc);
    flux_close (h);
    return (0);
}
Ejemplo n.º 2
0
static int internal_content_store (optparse_t *p, int ac, char *av[])
{
    const uint32_t blob_size_limit = 1048576; /* RFC 10 */
    uint8_t *data;
    int size;
    flux_t *h;
    flux_rpc_t *rpc;
    const char *topic;

    if (optparse_optind (p)  != ac) {
        optparse_print_usage (p);
        exit (1);
    }
    if ((size = read_all (STDIN_FILENO, &data)) < 0)
        log_err_exit ("read");
    if (!(h = builtin_get_flux_handle (p)))
        log_err_exit ("flux_open");
    if (optparse_hasopt (p, "dry-run")) {
        int flags;
        const char *hashfun;

        if (size > blob_size_limit)
            log_errn_exit (EFBIG, "content-store");
        if (!(hashfun = flux_attr_get (h, "content-hash", &flags)))
            log_err_exit ("flux_attr_get content-hash");
        if (!strcmp (hashfun, "sha1")) {
            uint8_t hash[SHA1_DIGEST_SIZE];
            char hashstr[SHA1_STRING_SIZE];
            SHA1_CTX sha1_ctx;

            SHA1_Init (&sha1_ctx);
            SHA1_Update (&sha1_ctx, (uint8_t *)data, size);
            SHA1_Final (&sha1_ctx, hash);
            sha1_hashtostr (hash, hashstr);
            printf ("%s\n", hashstr);
        } else
            log_msg_exit ("content-store: unsupported hash function: %s", hashfun);
    } else {
        const char *blobref;
        int blobref_size;
        if (optparse_hasopt (p, "bypass-cache"))
            topic = "content-backing.store";
        else
            topic = "content.store";
        if (!(rpc = flux_rpc_raw (h, topic, data, size, 0, 0)))
            log_err_exit ("%s", topic);
        if (flux_rpc_get_raw (rpc, &blobref, &blobref_size) < 0)
            log_err_exit ("%s", topic);
        if (!blobref || blobref[blobref_size - 1] != '\0')
            log_msg_exit ("%s: protocol error", topic);
        printf ("%s\n", blobref);
        flux_rpc_destroy (rpc);
    }
    flux_close (h);
    free (data);
    return (0);
}
Ejemplo n.º 3
0
static void store_completion (flux_rpc_t *rpc, void *arg)
{
    flux_reactor_t *r = arg;
    const char *blobref;
    int blobref_size;

    if (flux_rpc_get_raw (rpc, &blobref, &blobref_size) < 0)
        log_err_exit ("store");
    if (!blobref || blobref[blobref_size - 1] != '\0')
        log_msg_exit ("store: protocol error");
    //printf ("%s\n", blobref);
    flux_rpc_destroy (rpc);
    if (--spam_cur_inflight < spam_max_inflight/2)
        flux_reactor_stop (r);
}
Ejemplo n.º 4
0
/* Manage shutdown of this module.
 * Tell content cache to disable backing store,
 * then write everything back to it before exiting.
 */
void shutdown_cb (flux_t *h, flux_msg_handler_t *mh,
                  const flux_msg_t *msg, void *arg)
{
    sqlite_ctx_t *ctx = arg;
    flux_future_t *f;
    int count = 0;
    int old_state;

    flux_log (h, LOG_DEBUG, "shutdown: begin");
    if (register_backing_store (h, false, "content-sqlite") < 0) {
        flux_log_error (h, "shutdown: unregistering backing store");
        goto done;
    }
    if (ctx->broker_shutdown) {
        flux_log (h, LOG_DEBUG, "shutdown: instance is terminating, don't reload to cache");
        goto done;
    }
    //delay cancellation to ensure lock-correctness in sqlite
    pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state);
    while (sqlite3_step (ctx->dump_stmt) == SQLITE_ROW) {
        const char *blobref;
        int blobref_size;
        const void *data = NULL;
        int uncompressed_size;
        int size = sqlite3_column_bytes (ctx->dump_stmt, 0);
        if (sqlite3_column_type (ctx->dump_stmt, 0) != SQLITE_BLOB
                                                            && size > 0) {
            flux_log (h, LOG_ERR, "shutdown: encountered non-blob value");
            continue;
        }
        data = sqlite3_column_blob (ctx->dump_stmt, 0);
        if (sqlite3_column_type (ctx->dump_stmt, 1) != SQLITE_INTEGER) {
            flux_log (h, LOG_ERR, "shutdown: selected value is not an integer");
            errno = EINVAL;
            goto done;
        }
        uncompressed_size = sqlite3_column_int (ctx->dump_stmt, 1);
        if (uncompressed_size != -1) {
            if (ctx->lzo_bufsize < uncompressed_size
                            && grow_lzo_buf (ctx, uncompressed_size) < 0)
                goto done;
            int r = LZ4_decompress_safe (data, ctx->lzo_buf, size, uncompressed_size);
            if (r < 0) {
                errno = EINVAL;
                goto done;
            }
            if (r != uncompressed_size) {
                flux_log (h, LOG_ERR, "shutdown: blob size mismatch");
                errno = EINVAL;
                goto done;
            }
            data = ctx->lzo_buf;
            size = uncompressed_size;
        }
        if (!(f = flux_rpc_raw (h, "content.store", data, size,
                                                        FLUX_NODEID_ANY, 0))) {
            flux_log_error (h, "shutdown: store");
            continue;
        }
        if (flux_rpc_get_raw (f, (const void **)&blobref, &blobref_size) < 0) {
            flux_log_error (h, "shutdown: store");
            flux_future_destroy (f);
            continue;
        }
        if (!blobref || blobref[blobref_size - 1] != '\0') {
            flux_log (h, LOG_ERR, "shutdown: store returned malformed blobref");
            flux_future_destroy (f);
            continue;
        }
        flux_future_destroy (f);
        count++;
    }
    (void )sqlite3_reset (ctx->dump_stmt);
    flux_log (h, LOG_DEBUG, "shutdown: %d entries returned to cache", count);
done:
    pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
    flux_reactor_stop (flux_get_reactor (h));
}
Ejemplo n.º 5
0
void rpctest_begin_cb (flux_t *h, flux_msg_handler_t *w,
                       const flux_msg_t *msg, void *arg)
{
    const char *json_str;
    flux_rpc_t *r;

    errno = 0;
    ok (!(r = flux_rpc (h, NULL, NULL, FLUX_NODEID_ANY, 0))
        && errno == EINVAL,
        "flux_rpc with NULL topic fails with EINVAL");

    /* working no-payload RPC */
    ok ((r = flux_rpc (h, "rpctest.hello", NULL, FLUX_NODEID_ANY, 0)) != NULL,
        "flux_rpc with no payload when none is expected works");
    ok (flux_rpc_check (r) == false,
        "flux_rpc_check says get would block");
    ok (flux_rpc_get (r, NULL) == 0,
        "flux_rpc_get works");
    flux_rpc_destroy (r);

    /* cause remote EPROTO (unexpected payload) - will be picked up in _get() */
    ok ((r = flux_rpc (h, "rpctest.hello", "{}", FLUX_NODEID_ANY, 0)) != NULL,
        "flux_rpc with payload when none is expected works, at first");
    ok (flux_rpc_check (r) == false,
        "flux_rpc_check says get would block");
    errno = 0;
    ok (flux_rpc_get (r, NULL) < 0
        && errno == EPROTO,
        "flux_rpc_get fails with EPROTO");
    flux_rpc_destroy (r);

    /* cause remote EPROTO (missing payload) - will be picked up in _get() */
    errno = 0;
    ok ((r = flux_rpc (h, "rpctest.echo", NULL, FLUX_NODEID_ANY, 0)) != NULL,
        "flux_rpc with no payload when payload is expected works, at first");
    ok (flux_rpc_check (r) == false,
        "flux_rpc_check says get would block");
    errno = 0;
    ok (flux_rpc_get (r, NULL) < 0
        && errno == EPROTO,
        "flux_rpc_get fails with EPROTO");
    flux_rpc_destroy (r);

    /* working with-payload RPC */
    ok ((r = flux_rpc (h, "rpctest.echo", "{}", FLUX_NODEID_ANY, 0)) != NULL,
        "flux_rpc with payload when payload is expected works");
    ok (flux_rpc_check (r) == false,
        "flux_rpc_check says get would block");
    json_str = NULL;
    ok (flux_rpc_get (r, &json_str) == 0
        && json_str && !strcmp (json_str, "{}"),
        "flux_rpc_get works and returned expected payload");
    flux_rpc_destroy (r);

    /* working with-payload RPC (raw) */
    char *d, data[] = "aaaaaaaaaaaaaaaaaaaa";
    int l, len = strlen (data);
    ok ((r = flux_rpc_raw (h, "rpctest.rawecho", data, len,
                          FLUX_NODEID_ANY, 0)) != NULL,
        "flux_rpc_raw with payload when payload is expected works");
    ok (flux_rpc_check (r) == false,
        "flux_rpc_check says get would block");
    json_str = NULL;
    ok (flux_rpc_get_raw (r, &d, &l) == 0
        && d != NULL && l == len && memcmp (data, d, len) == 0,
        "flux_rpc_get_raw works and returned expected payload");
    flux_rpc_destroy (r);

    /* use newish pack/unpack payload interfaces */
    int i = 0;
    ok ((r = flux_rpcf (h, "rpctest.incr", FLUX_NODEID_ANY, 0,
                        "{s:i}", "n", 107)) != NULL,
        "flux_rpcf works");
    ok (flux_rpc_getf (r, "{s:i}", "n", &i) == 0,
        "flux_rpc_getf works");
    ok (i == 108,
        "and service returned incremented value");
    flux_rpc_destroy (r);

    flux_reactor_stop (flux_get_reactor (h));
}