int kvs_conf_save (flux_t h, flux_conf_t cf)
{
flux_conf_itr_t itr = flux_conf_itr_create (cf);
const char *key;
int rc = -1;
if (kvs_unlink (h, kvs_conf_root) < 0)
goto done;
if (kvs_commit (h) < 0)
goto done;
while ((key = flux_conf_next (itr))) {
char *nkey = xasprintf ("%s.%s", kvs_conf_root, key);
const char *val = flux_conf_get (cf, key);
int n = kvs_put_string (h, nkey, val);
free (nkey);
if (n < 0)
goto done;
}
if (kvs_commit (h) < 0)
goto done;
rc = 0;
done:
flux_conf_itr_destroy (itr);
return rc;
}
static void _store_hosts (flux_t h)
{
char *key;
long cores = sysconf(_SC_NPROCESSORS_ONLN);
long pagesize = sysconf(_SC_PAGE_SIZE);
long pages = sysconf(_SC_PHYS_PAGES);
long memMB = pages * pagesize / 1024 / 1024;
if ((asprintf (&key, "resrc.rank.%d.cores", flux_rank (h)) < 0) ||
kvs_put_int64 (h, key, cores)) {
err ("resrc: kvs_put_int64 %d %lu failed", flux_rank (h), cores);
}
free (key);
if ((asprintf (&key, "resrc.rank.%d.alloc.cores", flux_rank (h)) < 0) ||
kvs_put_int64 (h, key, 0)) {
err ("resrc: kvs_put_int64 %d %d failed", flux_rank (h), 0);
}
free (key);
if ((asprintf (&key, "resrc.rank.%d.mem", flux_rank (h)) < 0) ||
kvs_put_int64 (h, key, memMB)) {
err ("resrc: kvs_put_int64 %d %lu failed", flux_rank (h), memMB);
}
free (key);
kvs_commit(h);
}
static int update_pdesc (flux_t h, int64_t j, JSON o)
{
int i = 0;
int rc = -1;
int64_t size = 0;
JSON h_arr = NULL, e_arr = NULL, pd_arr = NULL, pde = NULL;
if (!Jget_int64 (o, JSC_PDESC_SIZE, &size)) return -1;
if (!Jget_obj (o, JSC_PDESC_PDARRAY, &pd_arr)) return -1;
if (!Jget_obj (o, JSC_PDESC_HOSTNAMES, &h_arr)) return -1;
if (!Jget_obj (o, JSC_PDESC_EXECS, &e_arr)) return -1;
for (i=0; i < (int) size; ++i) {
if (!Jget_ar_obj (pd_arr, i, &pde))
goto done;
if ( (rc = update_1pdesc (h, i, j, pde, h_arr, e_arr)) < 0)
goto done;
}
if (kvs_commit (h) < 0) {
flux_log (h, LOG_ERR, "update_pdesc commit failed");
goto done;
}
rc = 0;
done:
return rc;
}
void cmd_copy_tokvs (flux_t h, int argc, char **argv)
{
char *file, *key;
int fd, len;
uint8_t *buf;
JSON o;
if (argc != 2)
msg_exit ("copy-tokvs: specify key and filename");
key = argv[0];
file = argv[1];
if (!strcmp (file, "-")) {
if ((len = read_all (STDIN_FILENO, &buf)) < 0)
err_exit ("stdin");
} else {
if ((fd = open (file, O_RDONLY)) < 0)
err_exit ("%s", file);
if ((len = read_all (fd, &buf)) < 0)
err_exit ("%s", file);
(void)close (fd);
}
o = Jnew ();
util_json_object_add_data (o, "data", buf, len);
if (kvs_put (h, key, Jtostr (o)) < 0)
err_exit ("%s", key);
if (kvs_commit (h) < 0)
err_exit ("kvs_commit");
Jput (o);
free (buf);
}
static int update_rdesc (flux_t h, int64_t j, JSON o)
{
int rc = -1;
int64_t nnodes = 0;
int64_t ntasks = 0;
char key1[20] = {'\0'};
char key2[20] = {'\0'};
if (!Jget_int64 (o, JSC_RDESC_NNODES, &nnodes)) return -1;
if (!Jget_int64 (o, JSC_RDESC_NTASKS, &ntasks)) return -1;
if ((nnodes < 0) || (ntasks < 0)) return -1;
snprintf (key1, 20, "lwj.%ld.nnodes", j);
snprintf (key2, 20, "lwj.%ld.ntasks", j);
if (kvs_put_int64 (h, key1, nnodes) < 0)
flux_log (h, LOG_ERR, "update %s: %s", key1, strerror (errno));
else if (kvs_put_int64 (h, key2, ntasks) < 0)
flux_log (h, LOG_ERR, "update %s: %s", key2, strerror (errno));
else if (kvs_commit (h) < 0)
flux_log (h, LOG_ERR, "commit failed");
else {
flux_log (h, LOG_DEBUG, "job (%ld) assigned new resources.", j);
rc = 0;
}
return rc;
}
static void config_hwloc_paths (flux_t h, const char *dirpath)
{
uint32_t size, rank;
const char *key_prefix = "config.resource.hwloc.xml";
char key[64];
char path[PATH_MAX];
int n;
if (flux_get_size (h, &size) < 0)
log_err_exit ("flux_get_size");
for (rank = 0; rank < size; rank++) {
n = snprintf (key, sizeof (key), "%s.%"PRIu32, key_prefix, rank);
assert (n < sizeof (key));
if (dirpath == NULL) {
/* Remove any per rank xml and reload default xml */
if (kvs_unlink (h, key) < 0)
log_err_exit ("kvs_unlink");
continue;
}
n = snprintf (path, sizeof (path), "%s/%"PRIu32".xml", dirpath, rank);
assert (n < sizeof (path));
if (access (path, R_OK) < 0)
log_err_exit ("%s", path);
if (kvs_put_string (h, key, path) < 0)
log_err_exit ("kvs_put_string");
}
if (kvs_commit (h) < 0)
log_err_exit ("kvs_commit");
}
int kz_flush (kz_t *kz)
{
int rc = 0;
if ((kz->flags & KZ_FLAGS_WRITE))
rc = kvs_commit (kz->h);
return rc;
}
void test_selfmod (int argc, char **argv)
{
flux_t *h;
char *key;
if (argc != 1) {
fprintf (stderr, "Usage: selfmod key\n");
exit (1);
}
key = argv[0];
if (!(h = flux_open (NULL, 0)))
log_err_exit ("flux_open");
if (kvs_put_int (h, key, -1) < 0)
log_err_exit ("kvs_put_int");
if (kvs_commit (h) < 0)
log_err_exit ("kvs_commit");
if (kvs_watch_int (h, key, selfmod_watch_cb, h) < 0)
log_err_exit ("kvs_watch_int");
log_msg ("reactor: start");
flux_reactor_run (flux_get_reactor (h), 0);
log_msg ("reactor: end");
flux_close (h);
}
static int update_state (flux_t h, int64_t j, JSON o)
{
int rc = -1;
int64_t st = 0;
char *key;
if (!Jget_int64 (o, JSC_STATE_PAIR_NSTATE, &st)) return -1;
if ((st >= J_FOR_RENT) || (st < J_NULL)) return -1;
key = xasprintf ("lwj.%"PRId64".state", j);
if (kvs_put_string (h, key, jsc_job_num2state ((job_state_t)st)) < 0)
flux_log_error (h, "update %s", key);
else if (kvs_commit (h) < 0)
flux_log_error (h, "commit %s", key);
else {
flux_log (h, LOG_DEBUG, "job (%"PRId64") assigned new state: %s", j,
jsc_job_num2state ((job_state_t)st));
rc = 0;
}
free (key);
if (send_state_event (h, st, j) < 0)
flux_log_error (h, "send state event");
return rc;
}
int kz_close (kz_t *kz)
{
int rc = -1;
char *json_str = NULL;
char *key = NULL;
if ((kz->flags & KZ_FLAGS_WRITE)) {
if (!(kz->flags & KZ_FLAGS_RAW)) {
if (asprintf (&key, "%s.%.6d", kz->name, kz->seq++) < 0)
oom ();
if (!(json_str = zio_json_encode (NULL, 0, true))) { /* EOF */
errno = EPROTO;
goto done;
}
if (kvs_put (kz->h, key, json_str) < 0)
goto done;
}
if (!(kz->flags & KZ_FLAGS_NOCOMMIT_CLOSE)) {
if (kvs_commit (kz->h) < 0)
goto done;
}
if (kz->nprocs > 0 && kz->grpname) {
if (kz_fence (kz) < 0)
goto done;
}
}
rc = 0;
done:
if (json_str)
free (json_str);
if (key)
free (key);
kz_destroy (kz);
return rc;
}
void cmd_link (flux_t h, int argc, char **argv)
{
if (argc != 2)
msg_exit ("link: specify target and link_name");
if (kvs_symlink (h, argv[1], argv[0]) < 0)
err_exit ("%s", argv[1]);
if (kvs_commit (h) < 0)
err_exit ("kvs_commit");
}
/*
* Set a new value for lwj.next-id and commit
*/
static int set_next_jobid (flux_t h, unsigned long jobid)
{
int rc;
if ((rc = kvs_put_int64 (h, "lwj.next-id", jobid)) < 0) {
err ("kvs_put: %s", strerror (errno));
return -1;
}
return kvs_commit (h);
}
static int selfmod_watch_cb (const char *key, int val, void *arg, int errnum)
{
log_msg ("%s: value = %d errnum = %d", __FUNCTION__, val, errnum);
flux_t *h = arg;
if (kvs_put_int (h, key, val + 1) < 0)
log_err_exit ("%s: kvs_put_int", __FUNCTION__);
if (kvs_commit (h) < 0)
log_err_exit ("%s: kvs_commit", __FUNCTION__);
return (val == 0 ? -1 : 0);
}
void cmd_mkdir (flux_t h, int argc, char **argv)
{
int i;
if (argc == 0)
msg_exit ("mkdir: specify one or more directories");
for (i = 0; i < argc; i++) {
if (kvs_mkdir (h, argv[i]) < 0)
err_exit ("%s", argv[i]);
}
if (kvs_commit (h) < 0)
err_exit ("kvs_commit");
}
static int unwatch_timer_cb (flux_t h, void *arg)
{
static int count = 0;
const char *key = arg;
if (kvs_put_int (h, key, count++) < 0)
err_exit ("%s: kvs_put_int", __FUNCTION__);
if (kvs_commit (h) < 0)
err_exit ("%s: kvs_commit", __FUNCTION__);
if (count == 10) {
if (kvs_unwatch (h, key) < 0)
err_exit ("%s: kvs_unwatch", __FUNCTION__);
} else if (count == 20)
flux_reactor_stop (flux_get_reactor (h));
return 0;
}
static int kvs_job_new (flux_t h, unsigned long jobid)
{
int rc;
char *key;
if (asprintf (&key, "lwj.%lu.state", jobid) < 0)
return (-1);
flux_log (h, LOG_INFO, "Setting job %ld to reserved", jobid);
rc = kvs_put_string (h, key, "reserved");
kvs_commit (h);
free (key);
return rc;
}
void cmd_unlink (flux_t h, int argc, char **argv)
{
int i;
if (argc == 0)
msg_exit ("unlink: specify one or more keys");
for (i = 0; i < argc; i++) {
/* FIXME: unlink nonexistent silently fails */
/* FIXME: unlink directory silently succedes */
if (kvs_unlink (h, argv[i]) < 0)
err_exit ("%s", argv[i]);
}
if (kvs_commit (h) < 0)
err_exit ("kvs_commit");
}
static int update_rdl (flux_t h, int64_t j, const char *rs)
{
int rc = -1;
char *key = xasprintf ("lwj.%"PRId64".rdl", j);
if (kvs_put_string (h, key, rs) < 0)
flux_log_error (h, "update %s", key);
else if (kvs_commit (h) < 0)
flux_log_error (h, "commit failed");
else {
flux_log (h, LOG_DEBUG, "job (%"PRId64") assigned new rdl.", j);
rc = 0;
}
free (key);
return rc;
}
static void unwatch_timer_cb (flux_reactor_t *r, flux_watcher_t *w,
int revents, void *arg)
{
struct timer_ctx *ctx = arg;
static int count = 0;
log_msg ("%s", __FUNCTION__);
if (kvs_put_int (ctx->h, ctx->key, count++) < 0)
log_err_exit ("%s: kvs_put_int", __FUNCTION__);
if (kvs_commit (ctx->h) < 0)
log_err_exit ("%s: kvs_commit", __FUNCTION__);
if (count == 10) {
if (kvs_unwatch (ctx->h, ctx->key) < 0)
log_err_exit ("%s: kvs_unwatch", __FUNCTION__);
} else if (count == 20)
flux_reactor_stop (r);
}
static int update_rdl (flux_t h, int64_t j, const char *rs)
{
int rc = -1;
char key[20] = {'\0'};
snprintf (key, 20, "lwj.%ld.rdl", j);
if (kvs_put_string (h, key, rs) < 0)
flux_log (h, LOG_ERR, "update %s: %s", key, strerror (errno));
else if (kvs_commit (h) < 0)
flux_log (h, LOG_ERR, "commit failed");
else {
flux_log (h, LOG_DEBUG, "job (%ld) assigned new rdl.", j);
rc = 0;
}
return rc;
}
static int update_rdl_alloc (flux_t h, int64_t j, JSON o)
{
int i = 0;
int rc = -1;
int size = 0;
JSON ra_e = NULL;
const char *key = NULL;
zhash_t *rtab = NULL;
int64_t *ncores = NULL;
if (!(rtab = zhash_new ()))
oom ();
if (!Jget_ar_len (o, &size))
goto done;
for (i=0; i < (int) size; ++i) {
if (!Jget_ar_obj (o, i, &ra_e))
goto done;
/* 'o' represents an array of per-node core count to use.
* However, becasue the same rank can appear multiple times
* in this array in emulation mode, update_hash_1ra is
* used to determine the total core count per rank.
*/
if ( (rc = update_hash_1ra (h, j, ra_e, rtab)) < 0)
goto done;
}
FOREACH_ZHASH (rtab, key, ncores) {
if ( (rc = kvs_put_int64 (h, key, *ncores)) < 0) {
flux_log_error (h, "put %s", key);
goto done;
}
}
if (kvs_commit (h) < 0) {
flux_log (h, LOG_ERR, "update_pdesc commit failed");
goto done;
}
rc = 0;
done:
zhash_destroy (&rtab);
return rc;
}
void cmd_put (flux_t h, int argc, char **argv)
{
int i;
if (argc == 0)
msg_exit ("put: specify one or more key=value pairs");
for (i = 0; i < argc; i++) {
char *key = xstrdup (argv[i]);
char *val = strchr (key, '=');
if (!val)
msg_exit ("put: you must specify a value as key=value");
*val++ = '\0';
if (kvs_put (h, key, val) < 0) {
if (errno != EINVAL || kvs_put_string (h, key, val) < 0)
err_exit ("%s", key);
}
free (key);
}
if (kvs_commit (h) < 0)
err_exit ("kvs_commit");
}
/*
* wiredtiger_extension_terminate --
* Shutdown the KVS connector code.
*/
int
wiredtiger_extension_terminate(WT_CONNECTION *connection)
{
DATA_SOURCE *p;
int ret, tret;
(void)connection; /* Unused parameters */
ret = writelock(NULL, &global_lock);
/* Start a flush on any open objects. */
for (p = data_source_head; p != NULL; p = p->next)
if ((tret = kvs_commit(p->kvs)) != 0)
ESET(NULL, WT_ERROR,
"kvs_commit: %s: %s", p->uri, kvs_strerror(tret));
/* Complain if any of the objects are in use. */
for (p = data_source_head; p != NULL; p = p->next)
if (p->open_cursors != 0)
ESET(NULL, WT_ERROR,
"%s: has open cursors during close", p->uri);
/* Close and discard the remaining objects. */
while ((p = data_source_head) != NULL) {
if ((tret = kvs_close(p->kvs)) != 0)
ESET(NULL, WT_ERROR,
"kvs_close: %s: %s", p->uri, kvs_strerror(tret));
data_source_head = p->next;
free(p->uri);
ETRET(lock_destroy(NULL, &p->lock));
free(p);
}
ETRET(unlock(NULL, &global_lock));
ETRET(lock_destroy(NULL, &global_lock));
wt_ext = NULL;
return (ret);
}
static int update_state (flux_t h, int64_t j, JSON o)
{
int rc = -1;
int64_t st = 0;
char key[20] = {'\0'};
if (!Jget_int64 (o, JSC_STATE_PAIR_NSTATE, &st)) return -1;
if ((st >= J_FOR_RENT) || (st < J_NULL)) return -1;
snprintf (key, 20, "lwj.%ld.state", j);
if (kvs_put_string (h, key, jsc_job_num2state ((job_state_t)st)) < 0)
flux_log (h, LOG_ERR, "update %s: %s", key, strerror (errno));
else if (kvs_commit (h) < 0)
flux_log (h, LOG_ERR, "commit %s: %s", key, strerror (errno));
else {
flux_log (h, LOG_DEBUG, "job (%ld) assigned new state: %s", j,
jsc_job_num2state ((job_state_t)st));
rc = 0;
}
return rc;
}
static int kvs_job_set_state (flux_t h, unsigned long jobid, const char *state)
{
int rc;
char *key = NULL;
char *link = NULL;
char *target = NULL;
/* Create lwj entry in lwj-active dir at first:
*/
if ((asprintf (&key, "lwj-active.%lu.state", jobid) < 0)
|| (asprintf (&link, "lwj.%lu", jobid) < 0)
|| (asprintf (&target, "lwj-active.%lu", jobid) < 0)) {
flux_log_error (h, "kvs_job_set_state: asprintf");
return (-1);
}
flux_log (h, LOG_INFO, "Setting job %ld to %s", jobid, state);
if ((rc = kvs_put_string (h, key, state)) < 0) {
flux_log_error (h, "kvs_put_string (%s)", key);
goto out;
}
/*
* Create link from lwj.<id> to lwj-active.<id>
*/
if ((rc = kvs_symlink (h, link, target)) < 0) {
flux_log_error (h, "kvs_symlink (%s, %s)", link, key);
goto out;
}
if ((rc = kvs_commit (h)) < 0)
flux_log_error (h, "kvs_job_set_state: kvs_commit");
out:
free (key);
free (link);
free (target);
return rc;
}
static int putnext (kz_t *kz, const char *json_str)
{
char *key = NULL;
int rc = -1;
if (!(kz->flags & KZ_FLAGS_WRITE)) {
errno = EINVAL;
goto done;
}
if (asprintf (&key, "%s.%.6d", kz->name, kz->seq++) < 0)
oom ();
if (kvs_put (kz->h, key, json_str) < 0)
goto done;
if (!(kz->flags & KZ_FLAGS_NOCOMMIT_PUT)) {
if (kvs_commit (kz->h) < 0)
goto done;
}
rc = 0;
done:
if (key)
free (key);
return rc;
}
kz_t *kz_open (flux_t *h, const char *name, int flags)
{
kz_t *kz = xzmalloc (sizeof (*kz));
kz->flags = flags;
kz->name = xstrdup (name);
if ((kz->stream = strchr (kz->name, '.')))
kz->stream++;
else
kz->stream = kz->name;
kz->h = h;
if ((flags & KZ_FLAGS_WRITE)) {
if (key_exists (h, name)) {
if (!(flags & KZ_FLAGS_TRUNC)) {
errno = EEXIST;
goto error;
} else if (kvs_unlink (h, name) < 0)
goto error;
}
if (kvs_mkdir (h, name) < 0) /* N.B. does not catch EEXIST */
goto error;
if (!(flags & KZ_FLAGS_NOCOMMIT_OPEN)) {
if (kvs_commit (h) < 0)
goto error;
}
} else if ((flags & KZ_FLAGS_READ)) {
if (!(flags & KZ_FLAGS_NOEXIST)) {
if (kvs_get_dir (h, &kz->dir, "%s", name) < 0)
goto error;
}
}
return kz;
error:
kz_destroy (kz);
return NULL;
}
static int update_rdesc (flux_t h, int64_t j, JSON o)
{
int rc = -1;
int64_t nnodes = 0;
int64_t ntasks = 0;
int64_t walltime = 0;
char *key1;
char *key2;
char *key3;
if (!Jget_int64 (o, JSC_RDESC_NNODES, &nnodes)) return -1;
if (!Jget_int64 (o, JSC_RDESC_NTASKS, &ntasks)) return -1;
if (!Jget_int64 (o, JSC_RDESC_WALLTIME, &walltime)) return -1;
if ((nnodes < 0) || (ntasks < 0) || (walltime < 0)) return -1;
key1 = xasprintf ("lwj.%"PRId64".nnodes", j);
key2 = xasprintf ("lwj.%"PRId64".ntasks", j);
key3 = xasprintf ("lwj.%"PRId64".walltime", j);
if (kvs_put_int64 (h, key1, nnodes) < 0)
flux_log_error (h, "update %s", key1);
else if (kvs_put_int64 (h, key2, ntasks) < 0)
flux_log_error (h, "update %s", key2);
else if (kvs_put_int64 (h, key3, walltime) < 0)
flux_log_error (h, "update %s", key3);
else if (kvs_commit (h) < 0)
flux_log_error (h, "commit failed");
else {
flux_log (h, LOG_DEBUG, "job (%"PRId64") assigned new resources.", j);
rc = 0;
}
free (key1);
free (key2);
return rc;
}
static int update_rdl_alloc (flux_t h, int64_t j, JSON o)
{
int i = 0;
int rc = -1;
int size = 0;
JSON ra_e = NULL;
if (!Jget_ar_len (o, &size)) return -1;
for (i=0; i < (int) size; ++i) {
if (!Jget_ar_obj (o, i, &ra_e))
goto done;
if ( (rc = update_1ra (h, i, j, ra_e)) < 0)
goto done;
}
if (kvs_commit (h) < 0) {
flux_log (h, LOG_ERR, "update_pdesc commit failed");
goto done;
}
rc = 0;
done:
return rc;
}
void test_mt (int argc, char **argv)
{
thd_t *thd;
int i, rc;
flux_t *h;
int errors = 0;
if (argc != 3) {
fprintf (stderr, "Usage: mt nthreads changes key\n");
exit (1);
}
nthreads = strtoul (argv[0], NULL, 10);
changes = strtoul (argv[1], NULL, 10);
key = argv[2];
thd = xzmalloc (sizeof (*thd) * nthreads);
if (!(h = flux_open (NULL, 0)))
log_err_exit ("flux_open");
/* Set initial value of 'key' to -1 */
if (kvs_put_int (h, key, -1) < 0)
log_err_exit ("kvs_put_int %s", key);
key_stable = xasprintf ("%s-stable", key);
if (kvs_put_int (h, key_stable, 0) < 0)
log_err_exit ("kvs_put_int %s", key);
if (kvs_commit (h) < 0)
log_err_exit ("kvs_commit");
for (i = 0; i < nthreads; i++) {
thd[i].n = i;
thd[i].last_val = -42;
if ((rc = pthread_attr_init (&thd[i].attr)))
log_errn (rc, "pthread_attr_init");
if ((rc = pthread_create (&thd[i].tid, &thd[i].attr, thread, &thd[i])))
log_errn (rc, "pthread_create");
}
wait_ready ();
for (i = 0; i < changes; i++) {
if (kvs_put_int (h, key, i) < 0)
log_err_exit ("kvs_put_int %s", key);
if (kvs_commit (h) < 0)
log_err_exit ("kvs_commit");
}
/* Verify that callbacks were called the correct number of times.
* The nil and stable callbacks will be called exactly once before the
* reactor is started, then should never be called again.
* Due to commit merging on the master, the changing callback may
* miss intervening values but it shouldn't be called extra times.
*/
for (i = 0; i < nthreads; i++) {
if ((rc = pthread_join (thd[i].tid, NULL)))
log_errn (rc, "pthread_join");
if (thd[i].nil_count != 1) {
log_msg ("%d: nil callback called %d times (expected one)",
i, thd[i].nil_count);
errors++;
}
if (thd[i].stable_count != 1) {
log_msg ("%d: stable callback called %d times (expected one)",
i, thd[i].stable_count);
errors++;
}
if (thd[i].change_count > changes + 1) {
log_msg ("%d: changing callback called %d times (expected <= %d)",
i, thd[i].change_count, changes + 1);
errors++;
}
}
if (errors > 0)
exit (1);
free (thd);
free (key_stable);
flux_close (h);
}
