int mod_main (flux_t *h, int argc, char **argv)
{
ctx_t *ctx = getctx (h);
uint32_t rank;
flux_msg_handler_t **handlers = NULL;
int rc = -1;
if (flux_get_rank (h, &rank) < 0)
return -1;
if (rank != 0) {
flux_log (h, LOG_ERR, "this module must only run on rank 0");
return -1;
}
flux_log (h, LOG_INFO, "module starting");
if (flux_event_subscribe (h, "sim.start") < 0) {
flux_log (h, LOG_ERR, "subscribing to event: %s", strerror (errno));
return -1;
}
if (flux_msg_handler_addvec (h, htab, ctx, &handlers) < 0) {
flux_log (h, LOG_ERR, "flux_msg_handler_add: %s", strerror (errno));
return -1;
}
send_alive_request (h, module_name);
if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
flux_log (h, LOG_ERR, "flux_reactor_run: %s", strerror (errno));
goto done_delvec;
}
rc = 0;
done_delvec:
flux_msg_handler_delvec (handlers);
return rc;
}
static ctx_t *getctx (flux_t h)
{
int saved_errno;
ctx_t *ctx = (ctx_t *)flux_aux_get (h, "req");
if (!ctx) {
ctx = xzmalloc (sizeof (*ctx));
ctx->h = h;
ctx->ping_requests = zhash_new ();
ctx->clog_requests = zlist_new ();
if (!ctx->clog_requests || !ctx->ping_requests) {
saved_errno = ENOMEM;
goto error;
}
if (flux_get_rank (h, &ctx->rank) < 0) {
saved_errno = errno;
goto error;
}
flux_aux_set (h, "req", ctx, freectx);
}
return ctx;
error:
freectx (ctx);
errno = saved_errno;
return NULL;
}
int heartbeat_start (heartbeat_t *hb)
{
uint32_t rank;
struct flux_match match = FLUX_MATCH_EVENT;
if (!hb->h) {
errno = EINVAL;
return -1;
}
if (flux_get_rank (hb->h, &rank) < 0)
return -1;
if (rank == 0) {
flux_reactor_t *r = flux_get_reactor (hb->h);
flux_reactor_now_update (r);
if (!(hb->timer = flux_timer_watcher_create (r, hb->rate, hb->rate,
timer_cb, hb)))
return -1;
flux_watcher_start (hb->timer);
}
match.topic_glob = "hb";
if (!(hb->mh = flux_msg_handler_create (hb->h, match, event_cb, hb)))
return -1;
flux_msg_handler_start (hb->mh);
return 0;
}
static int rpc_request_send (flux_rpc_t *rpc, int n, const char *topic,
const char *json_str, uint32_t nodeid)
{
flux_msg_t *msg;
int flags = 0;
int rc = -1;
if (!(msg = flux_request_encode (topic, json_str)))
goto done;
if (flux_msg_set_matchtag (msg, rpc->m.matchtag + n) < 0)
goto done;
if (nodeid == FLUX_NODEID_UPSTREAM) {
flags |= FLUX_MSGFLAG_UPSTREAM;
if (flux_get_rank (rpc->h, &nodeid) < 0)
goto done;
}
if (flux_msg_set_nodeid (msg, nodeid, flags) < 0)
goto done;
if (flux_send (rpc->h, msg, 0) < 0)
goto done;
rc = 0;
done:
if (msg)
flux_msg_destroy (msg);
return rc;
}
bool flux_treeroot (flux_t h)
{
uint32_t rank;
if (flux_get_rank (h, &rank) == 0 && rank == 0)
return true;
return false;
}
int hello_start (hello_t *hello)
{
int rc = -1;
uint32_t rank;
if (flux_get_rank (hello->h, &rank) < 0)
goto done;
if (rank == 0) {
monotime (&hello->start);
if (!(hello->timer = flux_timer_watcher_create (hello->reactor,
hello->timeout,
hello->timeout,
timer_cb, hello)))
goto done;
flux_watcher_start (hello->timer);
if (hello_add_rank (hello, 0) < 0)
goto done;
} else {
if (hello_sendmsg (hello, rank) < 0)
goto done;
}
rc = 0;
done:
return rc;
}
int mod_main (flux_t *h, int argc, char **argv)
{
int saved_errno;
flux_msg_handler_t **handlers = NULL;
if (argc == 1 && !strcmp (argv[0], "--init-failure")) {
flux_log (h, LOG_INFO, "aborting during init per test request");
errno = EIO;
goto error;
}
if (!(modules = zhash_new ())) {
errno = ENOMEM;
goto error;
}
if (flux_get_rank (h, &rank) < 0)
goto error;
if (flux_msg_handler_addvec (h, htab, NULL, &handlers) < 0)
goto error;
if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
flux_log_error (h, "flux_reactor_run");
goto error;
}
zhash_destroy (&modules);
return 0;
error:
saved_errno = errno;
flux_msg_handler_delvec (handlers);
zhash_destroy (&modules);
errno = saved_errno;
return -1;
}
static ctx_t *getctx (flux_t *h)
{
ctx_t *ctx = (ctx_t *)flux_aux_get (h, "flux::barrier");
if (!ctx) {
ctx = xzmalloc (sizeof (*ctx));
if (!(ctx->barriers = zhash_new ())) {
errno = ENOMEM;
goto error;
}
if (flux_get_rank (h, &ctx->rank) < 0) {
flux_log_error (h, "flux_get_rank");
goto error;
}
if (!(ctx->timer = flux_timer_watcher_create (flux_get_reactor (h),
barrier_reduction_timeout_sec, 0., timeout_cb, ctx) )) {
flux_log_error (h, "flux_timer_watacher_create");
goto error;
}
ctx->h = h;
flux_aux_set (h, "flux::barrier", ctx, freectx);
}
return ctx;
error:
freectx (ctx);
return NULL;
}
int flux_info (flux_t h, uint32_t *rank, uint32_t *size, int *arity)
{
if (rank && flux_get_rank (h, rank) < 0)
return -1;
if (size && flux_get_size (h, size) < 0)
return -1;
if (arity && flux_get_arity (h, arity) < 0)
return -1;
return 0;
}
void *thread (void *arg)
{
thd_t *t = arg;
char *key, *fence_name = NULL;
int i, flags = 0;
struct timespec t0;
uint32_t rank;
flux_future_t *f;
flux_kvs_txn_t *txn;
if (!(t->h = flux_open (NULL, 0))) {
log_err ("%d: flux_open", t->n);
goto done;
}
if (flux_get_rank (t->h, &rank) < 0) {
log_err ("%d: flux_get_rank", t->n);
goto done;
}
for (i = 0; i < count; i++) {
if (!(txn = flux_kvs_txn_create ()))
log_err_exit ("flux_kvs_txn_create");
key = xasprintf ("%s.%"PRIu32".%d.%d", prefix, rank, t->n, i);
if (fopt)
fence_name = xasprintf ("%s-%d", prefix, i);
if (sopt)
monotime (&t0);
if (flux_kvs_txn_pack (txn, 0, key, "i", 42) < 0)
log_err_exit ("%s", key);
if (nopt && (i % nopt_divisor) == 0)
flags |= FLUX_KVS_NO_MERGE;
else
flags = 0;
if (fopt) {
if (!(f = flux_kvs_fence (t->h, flags, fence_name,
fence_nprocs, txn))
|| flux_future_get (f, NULL) < 0)
log_err_exit ("flux_kvs_fence");
flux_future_destroy (f);
} else {
if (!(f = flux_kvs_commit (t->h, flags, txn))
|| flux_future_get (f, NULL) < 0)
log_err_exit ("flux_kvs_commit");
flux_future_destroy (f);
}
if (sopt && zlist_append (t->perf, ddup (monotime_since (t0))) < 0)
oom ();
free (key);
free (fence_name);
flux_kvs_txn_destroy (txn);
}
done:
if (t->h)
flux_close (t->h);
return NULL;
}
int flux_rank (flux_t h)
{
uint32_t rank;
if (flux_get_rank (h, &rank) < 0)
return -1;
if (rank >= INT_MAX) {
errno = ERANGE;
return -1;
}
return rank;
}
int mod_main (flux_t *h, int argc, char **argv)
{
zhash_t *args = zhash_fromargv (argc, argv);
ctx_t *ctx;
char *eoc_str;
bool exit_on_complete;
uint32_t rank;
if (flux_get_rank (h, &rank) < 0)
return -1;
if (rank != 0) {
flux_log (h, LOG_ERR, "sim module must only run on rank 0");
return -1;
}
flux_log (h, LOG_INFO, "sim comms module starting");
if (!(eoc_str = zhash_lookup (args, "exit-on-complete"))) {
flux_log (h,
LOG_ERR,
"exit-on-complete argument is not set, defaulting to false");
exit_on_complete = false;
} else {
exit_on_complete =
(!strcmp (eoc_str, "true") || !strcmp (eoc_str, "True"));
}
ctx = getctx (h, exit_on_complete);
if (flux_event_subscribe (h, "rdl.update") < 0) {
flux_log (h, LOG_ERR, "subscribing to event: %s", strerror (errno));
return -1;
}
if (flux_msg_handler_addvec (h, htab, ctx) < 0) {
flux_log (h, LOG_ERR, "flux_msg_handler_add: %s", strerror (errno));
return -1;
}
if (send_start_event (h) < 0) {
flux_log (h, LOG_ERR, "sim failed to send start event");
return -1;
}
flux_log (h, LOG_DEBUG, "sim sent start event");
if (flux_reactor_run (flux_get_reactor (h), 0) < 0) {
flux_log (h, LOG_ERR, "flux_reactor_run: %s", strerror (errno));
return -1;
}
return 0;
}
static ctx_t *getctx (flux_t h)
{
ctx_t *ctx = (ctx_t *)flux_aux_get (h, "barriersrv");
if (!ctx) {
ctx = xzmalloc (sizeof (*ctx));
if (!(ctx->barriers = zhash_new ()))
oom ();
if (flux_get_rank (h, &ctx->rank) < 0)
err_exit ("flux_get_rank");
ctx->h = h;
flux_aux_set (h, "barriersrv", ctx, freectx);
}
return ctx;
}
void shutdown_set_handle (shutdown_t *s, flux_t h)
{
struct flux_match match = FLUX_MATCH_EVENT;
s->h = h;
match.topic_glob = "shutdown";
if (!(s->shutdown = flux_msg_handler_create (s->h, match,
shutdown_handler, s)))
log_err_exit ("flux_msg_handler_create");
flux_msg_handler_start (s->shutdown);
if (flux_event_subscribe (s->h, "shutdown") < 0)
log_err_exit ("flux_event_subscribe");
if (flux_get_rank (s->h, &s->myrank) < 0)
log_err_exit ("flux_get_rank");
}
static ns_t *ns_guess (flux_t h)
{
ns_t *ns = xzmalloc (sizeof (*ns));
uint32_t size, rank;
if (flux_get_rank (h, &rank) < 0)
log_err_exit ("flux_get_rank");
if (flux_get_size (h, &size) < 0)
log_err_exit ("flux_get_size");
ns->ok = nodeset_create ();
ns->slow = nodeset_create ();
ns->fail = nodeset_create ();
ns->unknown = nodeset_create ();
if (!ns->ok || !ns->slow || !ns->fail || !ns->unknown)
oom ();
nodeset_add_range (ns->ok, rank, size - 1);
return ns;
}
void flux_vlog (flux_t *h, int level, const char *fmt, va_list ap)
{
logctx_t *ctx = getctx (h);
int saved_errno = errno;
uint32_t rank;
flux_rpc_t *rpc = NULL;
int n, len;
char timestamp[WALLCLOCK_MAXLEN];
char hostname[STDLOG_MAX_HOSTNAME + 1];
struct stdlog_header hdr;
stdlog_init (&hdr);
hdr.pri = STDLOG_PRI (level, LOG_USER);
if (wallclock_get_zulu (timestamp, sizeof (timestamp)) >= 0)
hdr.timestamp = timestamp;
if (flux_get_rank (h, &rank) == 0) {
snprintf (hostname, sizeof (hostname), "%" PRIu32, rank);
hdr.hostname = hostname;
}
hdr.appname = ctx->appname;
hdr.procid = ctx->procid;
len = stdlog_encode (ctx->buf, sizeof (ctx->buf), &hdr,
STDLOG_NILVALUE, "");
assert (len < sizeof (ctx->buf));
n = vsnprintf (ctx->buf + len, sizeof (ctx->buf) - len, fmt, ap);
if (n > sizeof (ctx->buf) - len) /* ignore truncation of message */
n = sizeof (ctx->buf) - len;
len += n;
if (ctx->cb) {
ctx->cb (ctx->buf, len, ctx->cb_arg);
} else {
if (!(rpc = flux_rpc_raw (h, "cmb.log", ctx->buf, len,
FLUX_NODEID_ANY, FLUX_RPC_NORESPONSE)))
goto done;
}
done:
flux_rpc_destroy (rpc);
errno = saved_errno;
}
// Send out a call to all modules that the simulation is starting
// and that they should join
int send_start_event (flux_t *h)
{
int rc = 0;
flux_msg_t *msg = NULL;
uint32_t rank;
if (flux_get_rank (h, &rank) < 0)
return -1;
if (!(msg = flux_event_pack ("sim.start", "{ s:s s:i s:i }",
"mod_name", "sim",
"rank", rank,
"sim_time", 0))
|| flux_send (h, msg, 0) < 0) {
rc = -1;
}
flux_msg_destroy (msg);
return rc;
}
int main (int argc, char **argv)
{
flux_t h;
uint32_t rank, n;
int k;
if (!(h = flux_open (NULL, 0)))
err_exit ("flux_open");
if (flux_get_rank (h, &rank) < 0)
err_exit ("flux_get_rank");
if (flux_get_size (h, &n) < 0)
err_exit ("flux_get_size");
if (flux_get_arity (h, &k) < 0)
err_exit ("flux_get_arity");
printf ("height of %d-ary tree of size %" PRIu32 ": %d\n",
k, n, tree_height (n, k));
printf ("height of %d-ary at rank %" PRIu32 ": %d\n",
k, rank, tree_height (rank + 1, k));
flux_close (h);
return (0);
}
static void ping_request_cb (flux_t *h, flux_msg_handler_t *mh,
const flux_msg_t *msg, void *arg)
{
const char *json_str;
char *route_str = NULL;
char *full_route_str = NULL;
char *resp_str = NULL;
uint32_t rank, userid, rolemask;
if (flux_request_decode (msg, NULL, &json_str) < 0)
goto error;
if (flux_msg_get_rolemask (msg, &rolemask) < 0)
goto error;
if (flux_msg_get_userid (msg, &userid) < 0)
goto error;
if (!(route_str = flux_msg_get_route_string (msg)))
goto error;
if (flux_get_rank (h, &rank) < 0)
goto error;
if (asprintf (&full_route_str, "%s!%u", route_str, rank) < 0) {
errno = ENOMEM;
goto error;
}
if (!(resp_str = make_json_response_payload (json_str, full_route_str,
userid, rolemask))) {
goto error;
}
if (flux_respond (h, msg, resp_str) < 0)
flux_log_error (h, "%s: flux_respond", __FUNCTION__);
free (route_str);
free (full_route_str);
free (resp_str);
return;
error:
if (flux_respond_error (h, msg, errno, NULL) < 0)
flux_log_error (h, "%s: flux_respond_error", __FUNCTION__);
free (route_str);
free (full_route_str);
free (resp_str);
}
static int rpc_request_prepare (flux_rpc_t *rpc, flux_msg_t *msg,
uint32_t nodeid)
{
int flags = 0;
int rc = -1;
uint32_t matchtag = rpc->m.matchtag & ~FLUX_MATCHTAG_GROUP_MASK;
uint32_t matchgrp = rpc->m.matchtag & FLUX_MATCHTAG_GROUP_MASK;
/* So that flux_rpc_get_nodeid() can get nodeid from a response:
* For group rpc, use the lower matchtag bits to stash the nodeid
* in the request, and it will come back in the response.
* For singleton rpc, stash destination nodeid in rpc->nodeid.
* TODO-scalability: If nodeids exceed 1<<20, we may need to use a seq#
* in lower matchtag bits and stash a mapping array inthe rpc object.
*/
if (matchgrp > 0) {
if ((nodeid & FLUX_MATCHTAG_GROUP_MASK) != 0) {
errno = ERANGE;
goto done;
}
matchtag = matchgrp | nodeid;
} else
rpc->nodeid = nodeid;
if (flux_msg_set_matchtag (msg, matchtag) < 0)
goto done;
if (nodeid == FLUX_NODEID_UPSTREAM) {
flags |= FLUX_MSGFLAG_UPSTREAM;
if (flux_get_rank (rpc->h, &nodeid) < 0)
goto done;
}
if (flux_msg_set_nodeid (msg, nodeid, flags) < 0)
goto done;
rc = 0;
done:
return rc;
}
int main (int argc, char *argv[])
{
int ch;
bool copt = false;
bool aopt = false;
char *key = NULL;
int blocksize = 4096;
int kzoutflags = KZ_FLAGS_WRITE;
flux_t *h;
uint32_t rank;
bool rawtty = false;
log_init ("kzutil");
while ((ch = getopt_long (argc, argv, OPTIONS, longopts, NULL)) != -1) {
switch (ch) {
case 'a': /* --attach NAME */
aopt = true;
key = xstrdup (optarg);
break;
case 'c': /* --copy */
copt = true;
break;
case 'k': /* --key NAME */
key = xstrdup (optarg);
break;
case 'r': /* --raw-tty */
rawtty = true;
break;
case 't': /* --trunc */
kzoutflags |= KZ_FLAGS_TRUNC;
break;
case 'd': /* --delay-commit */
kzoutflags |= KZ_FLAGS_DELAYCOMMIT;
break;
case 'b': /* --blocksize bytes */
blocksize = strtoul (optarg, NULL, 10);
break;
default: /* --help|? */
usage ();
break;
}
}
argc -= optind;
argv += optind;
if (!aopt && !copt)
usage ();
if (copt) {
if (argc != 2)
usage ();
} else {
if (argc != 0)
usage ();
}
if (!(h = flux_open (NULL, 0)))
log_err_exit ("flux_open");
if (flux_get_rank (h, &rank) < 0)
log_err_exit ("flux_get_rank");
if (aopt) {
attach (h, key, rawtty, kzoutflags, blocksize);
} else if (copt) {
copy (h, argv[0], argv[1], kzoutflags, blocksize);
}
flux_close (h);
free (key);
log_fini ();
return 0;
}
int main (int argc, char *argv[])
{
flux_t h;
int ch;
uint32_t rank = FLUX_NODEID_ANY; /* local */
char *cmd;
int e;
log_init ("flux-comms");
while ((ch = getopt_long (argc, argv, OPTIONS, longopts, NULL)) != -1) {
switch (ch) {
case 'h': /* --help */
usage ();
break;
case 'r': /* --rank N */
rank = strtoul (optarg, NULL, 10);
break;
default:
usage ();
break;
}
}
if (optind == argc)
usage ();
cmd = argv[optind++];
if (rank != FLUX_NODEID_ANY
&& (!strcmp (cmd, "recover-all") || !strcmp (cmd, "info")))
usage ();
if (!(h = flux_open (NULL, 0)))
log_err_exit ("flux_open");
if (!strcmp (cmd, "reparent")) {
if (optind != argc - 1)
usage ();
if (flux_reparent (h, rank, argv[optind]) < 0)
log_err_exit ("flux_reparent");
} else if (!strcmp (cmd, "idle")) {
if (optind != argc)
usage ();
char *peers;
if (!(peers = flux_lspeer (h, rank)))
log_err_exit ("flux_peer");
printf ("%s\n", peers);
free (peers);
} else if (!strcmp (cmd, "panic")) {
char *msg = NULL;
size_t len = 0;
if (optind < argc) {
if ((e = argz_create (argv + optind, &msg, &len)) != 0)
log_errn_exit (e, "argz_create");
argz_stringify (msg, len, ' ');
}
flux_panic (h, rank, msg);
if (msg)
free (msg);
} else if (!strcmp (cmd, "failover")) {
if (optind != argc)
usage ();
if (flux_failover (h, rank) < 0)
log_err_exit ("flux_failover");
} else if (!strcmp (cmd, "recover")) {
if (optind != argc)
usage ();
if (flux_recover (h, rank) < 0)
log_err_exit ("flux_recover");
} else if (!strcmp (cmd, "recover-all")) {
if (optind != argc)
usage ();
if (flux_recover_all (h) < 0)
log_err_exit ("flux_recover_all");
} else if (!strcmp (cmd, "info")) {
int arity;
uint32_t rank, size;
const char *s;
if (flux_get_rank (h, &rank) < 0 || flux_get_size (h, &size) < 0)
log_err_exit ("flux_get_rank/size");
if (!(s = flux_attr_get (h, "tbon.arity", NULL)))
log_err_exit ("flux_attr_get tbon.arity");
arity = strtoul (s, NULL, 10);
printf ("rank=%d\n", rank);
printf ("size=%d\n", size);
printf ("arity=%d\n", arity);
} else
usage ();
flux_close (h);
log_fini ();
return 0;
}
