int qthread_fork_remote(qthread_f f,
const void *arg,
aligned_t *ret,
int rank,
size_t arg_len)
{
qthread_debug(MULTINODE_CALLS, "[%d] begin f=0x%lx, arg=0x%lx, ret=0x%lx, rank=%d, arg_len=%ld)\n",
my_rank, (unsigned long)f, (unsigned long)arg,
(unsigned long)ret, rank, arg_len);
uint64_t const uid = (uint64_t)qt_hash_get(ptr_to_uid_hash, f);
if (qt_hash_get(uid_to_ptr_hash, (qt_key_t)(uintptr_t)uid) != f) {
fprintf(stderr, "action not registered at source\n");
abort();
}
if (NULL != ret) {
qthread_empty(ret);
}
if (arg_len <= FORK_MSG_PAYLOAD) {
struct fork_msg_t msg;
msg.uid = uid;
msg.return_addr = (uint64_t)ret;
msg.origin_node = my_rank;
msg.arg_len = arg_len;
memcpy(msg.args, arg, arg_len);
qthread_debug(MULTINODE_DETAILS, "[%d] remote fork %d %d 0x%lx %d\n",
my_rank, rank, msg.uid, msg.return_addr, msg.arg_len);
return qthread_internal_net_driver_send(rank, SHORT_MSG_TAG, &msg, sizeof(msg));
} else {
struct fork_long_msg_t * long_msg;
size_t long_msg_size = sizeof(struct fork_long_msg_t) + arg_len;
long_msg = malloc(long_msg_size);
assert(NULL != long_msg);
long_msg->uid = uid;
long_msg->return_addr = (uint64_t)ret;
long_msg->origin_node = my_rank;
long_msg->arg_len = arg_len;
memcpy(&long_msg->args, arg, arg_len);
qthread_debug(MULTINODE_DETAILS, "[%d] remote long fork rank=%d uid=%d return_addr=0x%lx arg_len=%d\n",
my_rank, rank, long_msg->uid, long_msg->return_addr, long_msg->arg_len);
int const rc = qthread_internal_net_driver_send(rank, LONG_MSG_TAG, long_msg, long_msg_size);
free(long_msg);
return rc;
}
}
static aligned_t fork_helper(void *info)
{
struct fork_msg_t *msg = (struct fork_msg_t *)info;
aligned_t ret;
qthread_f f;
qthread_debug(MULTINODE_FUNCTIONS, "[%d] begin fork_helper\n", my_rank);
f = qt_hash_get(uid_to_ptr_hash, (qt_key_t)(uintptr_t)msg->uid);
if (NULL != f) {
ret = f(msg->args);
if (0 != msg->return_addr) {
struct return_msg_t ret_msg;
ret_msg.return_addr = msg->return_addr;
ret_msg.return_val = ret;
qthread_debug(MULTINODE_DETAILS, "[%d] sending return msg 0x%lx, %ld\n",
my_rank, ret_msg.return_addr, ret_msg.return_val);
qthread_internal_net_driver_send(msg->origin_node, RETURN_MSG_TAG,
&ret_msg, sizeof(ret_msg));
}
} else {
fprintf(stderr, "action uid %d not registered at destination\n", msg->uid);
abort();
}
qthread_debug(MULTINODE_FUNCTIONS, "[%d] end fork_helper\n", my_rank);
return 0;
}
int qthread_fork_remote_sinc(qthread_f f,
const void *arg,
qt_sinc_t *ret,
int rank,
size_t arg_len)
{
struct fork_msg_t msg;
qthread_debug(MULTINODE_CALLS, "[%d] begin qthread_fork_remote_sinc(0x%lx, 0x%lx, 0x%lx, %d, %ld)\n",
my_rank, (unsigned long)f, (unsigned long)arg,
(unsigned long)ret, rank, arg_len);
if (arg_len <= sizeof(msg.args)) {
msg.uid = (uint64_t)qt_hash_get(ptr_to_uid_hash, f);
if (qt_hash_get(uid_to_ptr_hash, (qt_key_t)(uintptr_t)msg.uid) != f) {
fprintf(stderr, "action not registered at source\n");
abort();
}
msg.return_addr = (uint64_t)ret;
msg.origin_node = my_rank;
msg.arg_len = arg_len;
memcpy(msg.args, arg, arg_len);
qthread_debug(MULTINODE_DETAILS, "[%d] remote fork %d %d 0x%lx %d\n",
my_rank, rank, msg.uid, msg.return_addr, msg.arg_len);
return qthread_internal_net_driver_send(rank, SHORT_MSG_SINC_TAG, &msg, sizeof(msg));
}
fprintf(stderr, "long remote fork unsupported\n");
abort();
}
int qthread_multinode_run(void)
{
aligned_t val;
if (0 == initialized) { return 1; }
qthread_debug(MULTINODE_CALLS, "[%d] begin qthread_multinode_run\n", my_rank);
qthread_internal_net_driver_barrier();
if (0 != my_rank) {
struct die_msg_t msg;
qthread_readFE(&val, &time_to_die);
qthread_debug(MULTINODE_DETAILS, "[%d] time to die\n", my_rank);
msg.my_rank = my_rank;
qthread_internal_net_driver_send(0, DIE_MSG_TAG, &msg, sizeof(msg));
qthread_finalize();
exit(0);
}
qthread_debug(MULTINODE_CALLS, "[%d] end qthread_multinode_run\n", my_rank);
return QTHREAD_SUCCESS;
}
int qthread_multinode_multistop(void)
{
aligned_t val;
qthread_debug(MULTINODE_CALLS, "[%d] begin qthread_multinode_multistop\n", my_rank);
if (0 != my_rank) {
struct die_msg_t msg;
qthread_readFF(&val, &time_to_die);
qthread_debug(MULTINODE_DETAILS, "[%d] time to die\n", my_rank);
msg.my_rank = my_rank;
qthread_internal_net_driver_send(0, DIE_MSG_TAG, &msg, sizeof(msg));
exit(0); // triggers atexit(net_cleanup)
}
qthread_debug(MULTINODE_CALLS, "[%d] end qthread_multinode_multistop\n", my_rank);
return QTHREAD_SUCCESS;
}
static void net_cleanup(void)
{
qthread_debug(MULTINODE_FUNCTIONS, "[%d] begin net_cleanup\n", my_rank);
if (my_rank == 0) {
int i;
for (i = 1; i < world_size; ++i) {
struct die_msg_t msg;
msg.my_rank = my_rank;
qthread_debug(MULTINODE_DETAILS, "[%d] sending die message to %d\n", my_rank, i);
qthread_internal_net_driver_send(i, DIE_MSG_TAG, &msg, sizeof(msg));
}
while (num_ended != world_size - 1) SPINLOCK_BODY();
}
qthread_internal_net_driver_finalize();
qt_hash_destroy(uid_to_ptr_hash);
qt_hash_destroy(ptr_to_uid_hash);
qthread_debug(MULTINODE_FUNCTIONS, "[%d] end net_cleanup\n", my_rank);
}
