Connection *Connection_new(const char *in_addr)
{
DEBUG(("alloc Connection\n"));
Connection *connection = Alloc_alloc_T(Connection);
if(connection == NULL) {
Module_set_error(GET_MODULE(), "Out of memory while allocating Connection");
return NULL;
}
connection->state = CS_CLOSED;
connection->current_batch = NULL;
connection->current_executor = NULL;
connection->parser = ReplyParser_new();
if(connection->parser == NULL) {
Connection_free(connection);
return NULL;
}
//copy socket addr
if(ADDR_SIZE < snprintf(connection->addr, ADDR_SIZE, "%s", in_addr)) {
Module_set_error(GET_MODULE(), "Invalid address for Connection");
Connection_free(connection);
return NULL;
}
//parse addr
int port = DEFAULT_IP_PORT;
char *pport;
char addr[ADDR_SIZE];
DEBUG(("parsing connection addr: %s\n", connection->addr));
if(NULL == (pport = strchr(connection->addr, ':'))) {
port = DEFAULT_IP_PORT;
snprintf(addr, ADDR_SIZE, "%s", connection->addr);
}
else {
port = atoi(pport + 1);
snprintf(addr, (pport - connection->addr + 1), "%s", connection->addr);
}
connection->sa.sin_family = AF_INET;
connection->sa.sin_port = htons(port);
if(!inet_pton(AF_INET, addr, &connection->sa.sin_addr)) {
Module_set_error(GET_MODULE(), "Could not parse ip address");
Connection_free(connection);
return NULL;
}
DEBUG(("Connection ip: '%s', port: %d\n", addr, port));
connection->sockfd = 0;
return connection;
}
void ComponentClock::Initialize()
{
std::shared_ptr<GameObjectModule> gameObjectModule = GET_MODULE(GameObjectModule).lock();
if (gameObjectModule != nullptr)
{
std::shared_ptr<GameObject> newHoursLine = gameObjectModule->CloneObject(m_HoursLinePrefab).lock();
if (newHoursLine != nullptr)
{
newHoursLine->SetParent(GetOwner());
m_HoursLineTransformComponent = newHoursLine->GetComponent<ComponentTransform2D>();
}
std::shared_ptr<GameObject> newMinutesLine = gameObjectModule->CloneObject(m_MinutesLinePrefab).lock();
if (newMinutesLine != nullptr)
{
newMinutesLine->SetParent(GetOwner());
m_MinutesLineTransformComponent = newMinutesLine->GetComponent<ComponentTransform2D>();
}
std::shared_ptr<GameObject> newSecondsLine = gameObjectModule->CloneObject(m_SecondsLinePrefab).lock();
if (newSecondsLine != nullptr)
{
newSecondsLine->SetParent(GetOwner());
m_SecondsLineTransformComponent = newSecondsLine->GetComponent<ComponentTransform2D>();
}
}
SetClock();
}
int ompi_osc_rdma_raccumulate (const void *origin_addr, int origin_count,
struct ompi_datatype_t *origin_datatype, int target_rank,
OPAL_PTRDIFF_TYPE target_disp, int target_count,
struct ompi_datatype_t *target_datatype, struct ompi_op_t *op,
struct ompi_win_t *win, struct ompi_request_t **request)
{
ompi_osc_rdma_module_t *module = GET_MODULE(win);
ompi_osc_rdma_peer_t *peer;
ompi_osc_rdma_request_t *rdma_request;
ompi_osc_rdma_sync_t *sync;
int ret;
sync = ompi_osc_rdma_module_sync_lookup (module, target_rank, &peer);
if (OPAL_UNLIKELY(NULL == sync)) {
return OMPI_ERR_RMA_SYNC;
}
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "racc: 0x%lx, %d, %s, %d, 0x%lx, %d, %s, %s, %s",
(unsigned long) origin_addr, origin_count, origin_datatype->name, target_rank,
(unsigned long) target_disp, target_count, target_datatype->name, op->o_name, win->w_name));
OMPI_OSC_RDMA_REQUEST_ALLOC(module, peer, rdma_request);
ret = ompi_osc_rdma_rget_accumulate_internal (sync, origin_addr, origin_count, origin_datatype, NULL, 0,
NULL, peer, target_rank, target_disp, target_count, target_datatype,
op, rdma_request);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
OMPI_OSC_RDMA_REQUEST_RETURN(rdma_request);
return ret;
}
*request = &rdma_request->super;
return OMPI_SUCCESS;
}
int ompi_osc_rdma_flush (int target, struct ompi_win_t *win)
{
ompi_osc_rdma_module_t *module = GET_MODULE(win);
ompi_osc_rdma_sync_t *lock;
ompi_osc_rdma_peer_t *peer;
assert (0 <= target);
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "flush: %d, %s", target, win->w_name);
if (ompi_comm_rank (module->comm) == target) {
/* nothing to flush. call one round of progress */
ompi_osc_rdma_progress (module);
return OMPI_SUCCESS;
}
OPAL_THREAD_LOCK(&module->lock);
lock = ompi_osc_rdma_module_sync_lookup (module, target, &peer);
if (OPAL_UNLIKELY(NULL == lock || OMPI_OSC_RDMA_SYNC_TYPE_LOCK != lock->type)) {
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "flush: target %d is not locked in window %s",
target, win->w_name);
OPAL_THREAD_UNLOCK(&module->lock);
return OMPI_ERR_RMA_SYNC;
}
OPAL_THREAD_UNLOCK(&module->lock);
/* finish all outstanding fragments */
ompi_osc_rdma_sync_rdma_complete (lock);
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "flush on target %d complete", target);
return OMPI_SUCCESS;
}
int ompi_osc_rdma_get_accumulate (const void *origin_addr, int origin_count,
struct ompi_datatype_t *origin_datatype,
void *result_addr, int result_count,
struct ompi_datatype_t *result_datatype,
int target_rank, MPI_Aint target_disp,
int target_count, struct ompi_datatype_t *target_datatype,
struct ompi_op_t *op, struct ompi_win_t *win)
{
ompi_osc_rdma_module_t *module = GET_MODULE(win);
ompi_osc_rdma_peer_t *peer;
ompi_osc_rdma_sync_t *sync;
sync = ompi_osc_rdma_module_sync_lookup (module, target_rank, &peer);
if (OPAL_UNLIKELY(NULL == sync)) {
return OMPI_ERR_RMA_SYNC;
}
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
"get_acc: 0x%lx, %d, %s, 0x%lx, %d, %s, %d, 0x%lx, %d, %s, %s, %s",
(unsigned long) origin_addr, origin_count, origin_datatype->name,
(unsigned long) result_addr, result_count, result_datatype->name, target_rank,
(unsigned long) target_disp, target_count, target_datatype->name, op->o_name,
win->w_name));
return ompi_osc_rdma_rget_accumulate_internal (sync, origin_addr, origin_count, origin_datatype,
result_addr, result_count, result_datatype,
peer, target_rank, target_disp, target_count,
target_datatype, op, NULL);
}
int ompi_osc_rdma_flush_all (struct ompi_win_t *win)
{
ompi_osc_rdma_module_t *module = GET_MODULE(win);
ompi_osc_rdma_sync_t *lock;
int ret = OMPI_SUCCESS;
uint32_t key;
void *node;
/* flush is only allowed from within a passive target epoch */
if (!ompi_osc_rdma_in_passive_epoch (module)) {
return OMPI_ERR_RMA_SYNC;
}
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "flush_all: %s", win->w_name);
/* globally complete all outstanding rdma requests */
if (OMPI_OSC_RDMA_SYNC_TYPE_LOCK == module->all_sync.type) {
ompi_osc_rdma_sync_rdma_complete (&module->all_sync);
}
/* flush all locks */
ret = opal_hash_table_get_first_key_uint32 (&module->outstanding_locks, &key, (void **) &lock, &node);
while (OPAL_SUCCESS == ret) {
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "flushing lock %p", (void *) lock);
ompi_osc_rdma_sync_rdma_complete (lock);
ret = opal_hash_table_get_next_key_uint32 (&module->outstanding_locks, &key, (void **) &lock,
node, &node);
}
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "flush_all complete");
return OPAL_SUCCESS;
}
/**
* @since 2016-11-20
*/
Object tm_load_module(Object file, Object code, Object name) {
Object mod = module_new(file, name, code);
Object fnc = func_new(mod, NONE_OBJECT, NULL);
GET_FUNCTION(fnc)->code = (unsigned char*) GET_STR(code);
GET_FUNCTION(fnc)->name = string_new("#main");
call_function(fnc);
return GET_MODULE(mod)->globals;
}
Connection *Connection_new(const char *in_addr)
{
DEBUG(("alloc Connection\n"));
Connection *connection = Alloc_alloc_T(Connection);
if(connection == NULL) {
Module_set_error(GET_MODULE(), "Out of memory while allocating Connection");
return NULL;
}
connection->state = CS_CLOSED;
connection->current_batch = NULL;
connection->current_executor = NULL;
connection->parser = ReplyParser_new();
if(connection->parser == NULL) {
Connection_free(connection);
return NULL;
}
//copy address
int invalid_address = 0;
char *service;
if (NULL == (service = strchr(in_addr, ':'))) {
invalid_address = ADDR_SIZE < snprintf(connection->addr, ADDR_SIZE, "%s", in_addr)
|| SERV_SIZE < snprintf(connection->serv, SERV_SIZE, "%s", XSTR(DEFAULT_IP_PORT));
}
else {
int addr_length = service - in_addr + 1;
if (ADDR_SIZE < addr_length) {
invalid_address = 1;
}
else {
snprintf(connection->addr, addr_length, "%s", in_addr);
invalid_address = SERV_SIZE < snprintf(connection->serv, SERV_SIZE, "%s", service + 1);
}
}
if (invalid_address) {
Module_set_error(GET_MODULE(), "Invalid address for Connection");
Connection_free(connection);
return NULL;
}
DEBUG(("Connection address: '%s', service: '%s'\n", connection->addr, connection->serv));
connection->addrinfo = NULL;
connection->sockfd = 0;
return connection;
}
int ompi_osc_rdma_lock_atomic (int lock_type, int target, int assert, ompi_win_t *win)
{
ompi_osc_rdma_module_t *module = GET_MODULE(win);
ompi_osc_rdma_peer_t *peer = ompi_osc_rdma_module_peer (module, target);
ompi_osc_rdma_sync_t *lock;
int ret = OMPI_SUCCESS;
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "lock: %d, %d, %d, %s", lock_type, target, assert, win->w_name);
if (module->no_locks) {
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "attempted to lock with no_locks set");
return OMPI_ERR_RMA_SYNC;
}
if (module->all_sync.epoch_active && (OMPI_OSC_RDMA_SYNC_TYPE_LOCK != module->all_sync.type || MPI_LOCK_EXCLUSIVE == lock_type)) {
/* impossible to get an exclusive lock while holding a global shared lock or in a active
* target access epoch */
return OMPI_ERR_RMA_SYNC;
}
/* clear the global sync object (in case MPI_Win_fence was called) */
module->all_sync.type = OMPI_OSC_RDMA_SYNC_TYPE_NONE;
/* create lock item */
lock = ompi_osc_rdma_sync_allocate (module);
if (OPAL_UNLIKELY(NULL == lock)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
lock->type = OMPI_OSC_RDMA_SYNC_TYPE_LOCK;
lock->sync.lock.target = target;
lock->sync.lock.type = lock_type;
lock->sync.lock.assert = assert;
lock->peer_list.peer = peer;
lock->num_peers = 1;
OBJ_RETAIN(peer);
if (0 == (assert & MPI_MODE_NOCHECK)) {
ret = ompi_osc_rdma_lock_atomic_internal (module, peer, lock);
}
if (OPAL_LIKELY(OMPI_SUCCESS == ret)) {
++module->passive_target_access_epoch;
opal_atomic_wmb ();
OPAL_THREAD_SCOPED_LOCK(&module->lock, ompi_osc_rdma_module_lock_insert (module, lock));
} else {
OBJ_RELEASE(lock);
}
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "lock %d complete", target);
return ret;
}
Executor *Executor_new()
{
DEBUG(("alloc Executor\n"));
Executor *executor = Alloc_alloc_T(Executor);
if(executor == NULL) {
Module_set_error(GET_MODULE(), "Out of memory while allocating Executor");
return NULL;
}
executor->numpairs = 0;
executor->numevents = 0;
return executor;
}
int main(int argc, char *argv[])
{
const TCExportInfo *info = NULL;
char *amod = "null";
char *vmod = "null";
char *mmod = "null";
int ret = 0;
ret = tc_setup_export_profile(&argc, &argv);
if (ret < 0) {
/* error, so bail out */
return 1;
}
printf("parsed: %i profiles\n", ret);
info = tc_load_export_profile();
amod = GET_MODULE(info->audio.module);
vmod = GET_MODULE(info->video.module);
mmod = GET_MODULE(info->mplex.module);
tc_export_profile_to_vob(info, &vob);
PRINT(divxbitrate, "%i");
PRINT(video_max_bitrate, "%i");
PRINT(mp3bitrate, "%i");
PRINT(mp3frequency, "%i");
PRINT(divxkeyframes, "%i");
PRINT(encode_fields, "%i");
PRINT(ex_frc, "%i");
PRINT(ex_v_codec, "%x");
PRINT(ex_a_codec, "%x");
PRINT(zoom_width, "%i");
PRINT(zoom_height, "%i");
printf("video module=%s\n", vmod);
printf("audio module=%s\n", amod);
printf("mplex module=%s\n", mmod);
tc_cleanup_export_profile();
return 0;
}
int Executor_add(Executor *executor, Connection *connection, Batch *batch)
{
assert(executor != NULL);
assert(connection != NULL);
assert(batch != NULL);
if(executor->numpairs >= MAX_PAIRS) {
Module_set_error(GET_MODULE(), "Executor is full, max = %d", MAX_PAIRS);
return -1;
}
struct _Pair *pair = &executor->pairs[executor->numpairs];
pair->batch = batch;
pair->connection = connection;
executor->numpairs += 1;
DEBUG(("Executor add, total: %d\n", executor->numpairs));
return 0;
}
int ompi_osc_rdma_compare_and_swap (const void *origin_addr, const void *compare_addr, void *result_addr,
struct ompi_datatype_t *dt, int target_rank, OPAL_PTRDIFF_TYPE target_disp,
struct ompi_win_t *win)
{
ompi_osc_rdma_module_t *module = GET_MODULE(win);
ompi_osc_rdma_peer_t *peer;
mca_btl_base_registration_handle_t *target_handle;
ompi_osc_rdma_sync_t *sync;
uint64_t target_address;
int ret;
sync = ompi_osc_rdma_module_sync_lookup (module, target_rank, &peer);
if (OPAL_UNLIKELY(NULL == sync)) {
return OMPI_ERR_RMA_SYNC;
}
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "cswap: 0x%lx, 0x%lx, 0x%lx, %s, %d, %d, %s",
(unsigned long) origin_addr, (unsigned long) compare_addr, (unsigned long) result_addr,
dt->name, target_rank, (int) target_disp, win->w_name));
ret = osc_rdma_get_remote_segment (module, peer, target_disp, 8, &target_address, &target_handle);
if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
return ret;
}
#if 0
if (MCA_OSC_RDMA_SAME_OP <= module->accumulate_ops) {
/* the user has indicated that they will only use the same op (or same op and no op)
* for operations on overlapping memory ranges. that indicates it is safe to go ahead
* and use network atomic operations. */
ret = ompi_osc_rdma_cas_atomic (sync, origin_addr, compare_addr, result_addr, dt,
peer, target_address, target_handle);
if (OMPI_SUCCESS == ret) {
return OMPI_SUCCESS;
}
} else
#endif
if (ompi_osc_rdma_peer_local_base (peer)) {
return ompi_osc_rdma_cas_local (origin_addr, compare_addr, result_addr, dt,
peer, target_address, target_handle, module);
}
return cas_rdma (sync, origin_addr, compare_addr, result_addr, dt, peer, target_address,
target_handle);
}
int ompi_osc_rdma_fetch_and_op (const void *origin_addr, void *result_addr, struct ompi_datatype_t *dt, int target_rank,
OPAL_PTRDIFF_TYPE target_disp, struct ompi_op_t *op, struct ompi_win_t *win)
{
ompi_osc_rdma_module_t *module = GET_MODULE(win);
ompi_osc_rdma_peer_t *peer;
ompi_osc_rdma_sync_t *sync;
sync = ompi_osc_rdma_module_sync_lookup (module, target_rank, &peer);
if (OPAL_UNLIKELY(NULL == sync)) {
return OMPI_ERR_RMA_SYNC;
}
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "fop: %p, %s, %d, %lu, %s, %s",
result_addr, dt->name, target_rank, (unsigned long) target_disp, op->o_name, win->w_name));
return ompi_osc_rdma_rget_accumulate_internal (sync, origin_addr, 1, dt, result_addr, 1, dt, peer, target_rank,
target_disp, 1, dt, op, NULL);
}
int ompi_osc_rdma_unlock_atomic (int target, ompi_win_t *win)
{
ompi_osc_rdma_module_t *module = GET_MODULE(win);
ompi_osc_rdma_peer_t *peer;
ompi_osc_rdma_sync_t *lock;
int ret = OMPI_SUCCESS;
OPAL_THREAD_LOCK(&module->lock);
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "unlock: %d, %s", target, win->w_name);
lock = ompi_osc_rdma_module_lock_find (module, target, &peer);
if (OPAL_UNLIKELY(NULL == lock)) {
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "target %d is not locked in window %s",
target, win->w_name);
OPAL_THREAD_UNLOCK(&module->lock);
return OMPI_ERR_RMA_SYNC;
}
ompi_osc_rdma_module_lock_remove (module, lock);
/* finish all outstanding fragments */
ompi_osc_rdma_sync_rdma_complete (lock);
if (!(lock->sync.lock.assert & MPI_MODE_NOCHECK)) {
ret = ompi_osc_rdma_unlock_atomic_internal (module, peer, lock);
}
/* release our reference to this peer */
OBJ_RELEASE(peer);
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "unlock %d complete", target);
--module->passive_target_access_epoch;
opal_atomic_wmb ();
OPAL_THREAD_UNLOCK(&module->lock);
/* delete the lock */
ompi_osc_rdma_sync_return (lock);
return ret;
}
int
ompi_osc_rdma_module_accumulate(void *origin_addr, int origin_count,
struct ompi_datatype_t *origin_dt,
int target, int target_disp, int target_count,
struct ompi_datatype_t *target_dt,
struct ompi_op_t *op, ompi_win_t *win)
{
int ret;
ompi_osc_rdma_sendreq_t *sendreq;
ompi_osc_rdma_module_t *module = GET_MODULE(win);
if ((OMPI_WIN_STARTED & ompi_win_get_mode(win)) &&
(!module->m_sc_remote_active_ranks[target])) {
return MPI_ERR_RMA_SYNC;
}
if (OMPI_WIN_FENCE & ompi_win_get_mode(win)) {
/* well, we're definitely in an access epoch now */
ompi_win_set_mode(win, OMPI_WIN_FENCE | OMPI_WIN_ACCESS_EPOCH |
OMPI_WIN_EXPOSE_EPOCH);
}
/* shortcut 0 count case */
if (0 == origin_count || 0 == target_count) {
return OMPI_SUCCESS;
}
/* create sendreq */
ret = ompi_osc_rdma_sendreq_alloc_init(OMPI_OSC_RDMA_ACC,
origin_addr,
origin_count,
origin_dt,
target,
target_disp,
target_count,
target_dt,
module,
&sendreq);
MEMCHECKER(
memchecker_convertor_call(&opal_memchecker_base_mem_noaccess,
&sendreq->req_origin_convertor);
);
int ompi_osc_rdma_unlock_all_atomic (struct ompi_win_t *win)
{
ompi_osc_rdma_module_t *module = GET_MODULE(win);
ompi_osc_rdma_sync_t *lock;
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "unlock_all: %s", win->w_name);
OPAL_THREAD_LOCK(&module->lock);
lock = &module->all_sync;
if (OMPI_OSC_RDMA_SYNC_TYPE_LOCK != lock->type) {
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "not locked in window %s", win->w_name);
OPAL_THREAD_UNLOCK(&module->lock);
return OMPI_ERR_RMA_SYNC;
}
/* finish all outstanding fragments */
ompi_osc_rdma_sync_rdma_complete (lock);
if (0 != (lock->sync.lock.assert & MPI_MODE_NOCHECK)) {
/* decrement the master lock shared count */
(void) ompi_osc_rdma_lock_release_shared (module, module->leader, -0x0000000100000000UL, offsetof (ompi_osc_rdma_state_t, global_lock));
}
lock->type = OMPI_OSC_RDMA_SYNC_TYPE_NONE;
lock->num_peers = 0;
lock->epoch_active = false;
--module->passive_target_access_epoch;
opal_atomic_wmb ();
OPAL_THREAD_UNLOCK(&module->lock);
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "unlock_all complete");
return OMPI_SUCCESS;
}
int ompi_osc_rdma_sync (struct ompi_win_t *win)
{
ompi_osc_rdma_progress (GET_MODULE(win));
return OMPI_SUCCESS;
}
int ompi_osc_rdma_lock_all_atomic (int assert, struct ompi_win_t *win)
{
ompi_osc_rdma_module_t *module = GET_MODULE(win);
ompi_osc_rdma_sync_t *lock;
int ret = OMPI_SUCCESS;
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "lock_all: %d, %s", assert, win->w_name);
if (module->no_locks) {
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "attempted to lock with no_locks set");
return OMPI_ERR_RMA_SYNC;
}
OPAL_THREAD_LOCK(&module->lock);
if (module->all_sync.epoch_active) {
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "attempted lock_all when active target epoch is %s "
"and lock all epoch is %s",
(OMPI_OSC_RDMA_SYNC_TYPE_LOCK != module->all_sync.type && module->all_sync.epoch_active) ?
"active" : "inactive",
(OMPI_OSC_RDMA_SYNC_TYPE_LOCK == module->all_sync.type) ? "active" : "inactive");
OPAL_THREAD_UNLOCK(&module->lock);
return OMPI_ERR_RMA_SYNC;
}
/* set up lock */
lock = &module->all_sync;
lock->type = OMPI_OSC_RDMA_SYNC_TYPE_LOCK;
lock->sync.lock.target = -1;
lock->sync.lock.type = MPI_LOCK_SHARED;
lock->sync.lock.assert = assert;
lock->num_peers = ompi_comm_size (module->comm);
lock->epoch_active = true;
/* NTH: TODO -- like fence it might be a good idea to create an array to access all peers
* without having to access the hash table. Such a change would likely increase performance
* at the expense of memory usage. Ex. if a window has 1M peers then 8MB per process would
* be needed for this array. */
if (0 != (assert & MPI_MODE_NOCHECK)) {
/* increment the global shared lock */
ret = ompi_osc_rdma_lock_acquire_shared (module, module->leader, 0x0000000100000000UL,
offsetof(ompi_osc_rdma_state_t, global_lock),
0x00000000ffffffffUL);
}
if (OPAL_LIKELY(OMPI_SUCCESS != ret)) {
lock->type = OMPI_OSC_RDMA_SYNC_TYPE_NONE;
lock->num_peers = 0;
lock->epoch_active = false;
} else {
++module->passive_target_access_epoch;
}
opal_atomic_wmb ();
OPAL_THREAD_UNLOCK(&module->lock);
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "lock_all complete");
return ret;
}
int Executor_execute(Executor *executor, int timeout_ms)
{
DEBUG(("Executor execute start\n"));
//determine max endtime based on timeout
struct timespec tm;
clock_gettime(CLOCK_MONOTONIC, &tm);
DEBUG(("Executor start_tm_ms: %3.2f\n", TIMESPEC_TO_MS(tm)));
executor->end_tm_ms = TIMESPEC_TO_MS(tm) + ((float)timeout_ms);
DEBUG(("Executor end_tm_ms: %3.2f\n", executor->end_tm_ms));
executor->numevents = 0;
for(int i = 0; i < executor->numpairs; i++) {
struct _Pair *pair = &executor->pairs[i];
Connection_execute_start(pair->connection, executor, pair->batch, i);
}
int poll_result = 1;
//for as long there are outstanding events and no error or timeout occurred:
while(executor->numevents > 0 && poll_result > 0) {
//figure out how many ms left for this execution
int timeout;
if (Executor_current_timeout(executor, &timeout) == -1) {
//if no time is left, force a timeout
poll_result = 0;
} else {
//do the poll
DEBUG(("Executor start poll num_events: %d\n", executor->numevents));
poll_result = poll(executor->fds, executor->numpairs, timeout);
DEBUG(("Executor select res %d\n", poll_result));
}
for(int i = 0; i < executor->numpairs; i++) {
struct _Pair *pair = &executor->pairs[i];
Connection *connection = pair->connection;
Batch *batch = pair->batch;
struct pollfd *fd = &executor->fds[i];
EventType event = 0;
if(fd->revents & POLLIN) {
event |= EVENT_READ;
fd->events &= ~POLLIN;
fd->revents &= ~POLLIN;
executor->numevents -= 1;
}
if(fd->revents & POLLOUT) {
event |= EVENT_WRITE;
fd->events &= ~POLLOUT;
fd->revents &= ~POLLOUT;
executor->numevents -= 1;
}
if(poll_result == 0) {
event = EVENT_TIMEOUT;
}
else if(poll_result < 0) {
event = EVENT_ERROR;
}
if(event > 0 && Batch_has_command(batch)) {
//there is an event, and batch is not finished
Connection_handle_event(connection, event, i);
}
}
}
if(poll_result > 1) {
poll_result = 1;
}
if(poll_result < 0) {
Module_set_error(GET_MODULE(), "Execute select error, errno: [%d] %s", errno, strerror(errno));
}
else if(poll_result == 0) {
Module_set_error(GET_MODULE(), "Execute timeout");
}
DEBUG(("Executor execute done\n"));
return poll_result;
}
static inline
int ompi_osc_pt2pt_rget_accumulate_internal (const void *origin_addr, int origin_count,
struct ompi_datatype_t *origin_datatype,
void *result_addr, int result_count,
struct ompi_datatype_t *result_datatype,
int target_rank, MPI_Aint target_disp,
int target_count, struct ompi_datatype_t *target_datatype,
struct ompi_op_t *op, struct ompi_win_t *win,
bool release_req, struct ompi_request_t **request)
{
int ret;
ompi_osc_pt2pt_module_t *module = GET_MODULE(win);
ompi_proc_t *proc = ompi_comm_peer_lookup(module->comm, target_rank);
bool is_long_datatype = false;
bool is_long_msg = false;
ompi_osc_pt2pt_frag_t *frag;
ompi_osc_pt2pt_header_acc_t *header;
ompi_osc_pt2pt_sync_t *pt2pt_sync;
size_t ddt_len, payload_len, frag_len;
char *ptr;
const void *packed_ddt;
int tag;
ompi_osc_pt2pt_request_t *pt2pt_request;
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
"rget_acc: 0x%lx, %d, %s, 0x%lx, %d, %s, 0x%x, %d, %d, %s, %s, %s",
(unsigned long) origin_addr, origin_count, origin_datatype->name,
(unsigned long) result_addr, result_count, result_datatype->name,
target_rank, (int) target_disp, target_count, target_datatype->name,
op->o_name, win->w_name));
pt2pt_sync = ompi_osc_pt2pt_module_sync_lookup (module, target_rank, NULL);
if (OPAL_UNLIKELY(NULL == pt2pt_sync)) {
return OMPI_ERR_RMA_SYNC;
}
/* get_accumulates are always request based, so that we know where to land the data */
OMPI_OSC_PT2PT_REQUEST_ALLOC(win, pt2pt_request);
pt2pt_request->internal = release_req;
/* short-circuit case. note that origin_count may be 0 if op is MPI_NO_OP */
if (0 == result_count || 0 == target_count) {
ompi_osc_pt2pt_request_complete (pt2pt_request, MPI_SUCCESS);
*request = &pt2pt_request->super;
return OMPI_SUCCESS;
}
if (!release_req) {
/* wait for epoch to begin before starting operation */
ompi_osc_pt2pt_sync_wait_expected (pt2pt_sync);
}
/* optimize the self case. TODO: optimize the local case */
if (ompi_comm_rank (module->comm) == target_rank) {
*request = &pt2pt_request->super;
return ompi_osc_pt2pt_gacc_self (pt2pt_sync, origin_addr, origin_count, origin_datatype,
result_addr, result_count, result_datatype,
target_disp, target_count, target_datatype,
op, module, pt2pt_request);
}
pt2pt_request->type = OMPI_OSC_PT2PT_HDR_TYPE_GET_ACC;
pt2pt_request->origin_addr = origin_addr;
pt2pt_request->origin_count = origin_count;
OMPI_DATATYPE_RETAIN(origin_datatype);
pt2pt_request->origin_dt = origin_datatype;
/* Compute datatype and payload lengths. Note that the datatype description
* must fit in a single frag */
ddt_len = ompi_datatype_pack_description_length(target_datatype);
if (&ompi_mpi_op_no_op.op != op) {
payload_len = origin_datatype->super.size * origin_count;
} else {
payload_len = 0;
}
frag_len = sizeof(*header) + ddt_len + payload_len;
ret = ompi_osc_pt2pt_frag_alloc(module, target_rank, frag_len, &frag, &ptr, false, release_req);
if (OMPI_SUCCESS != ret) {
frag_len = sizeof(*header) + ddt_len;
ret = ompi_osc_pt2pt_frag_alloc(module, target_rank, frag_len, &frag, &ptr, true, release_req);
if (OMPI_SUCCESS != ret) {
/* allocate space for the header plus space to store ddt_len */
frag_len = sizeof(*header) + 8;
ret = ompi_osc_pt2pt_frag_alloc(module, target_rank, frag_len, &frag, &ptr, true, release_req);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
is_long_datatype = true;
}
is_long_msg = true;
}
tag = get_tag (module);
/* If this is a long message then we need two completions before the
* request is complete (1 for the send, 1 for the receive) */
pt2pt_request->outstanding_requests = 1 + is_long_msg;
/* increment the number of outgoing fragments */
ompi_osc_signal_outgoing (module, target_rank, pt2pt_request->outstanding_requests);
header = (ompi_osc_pt2pt_header_acc_t *) ptr;
header->base.flags = 0;
header->len = frag_len;
header->count = target_count;
header->displacement = target_disp;
header->op = op->o_f_to_c_index;
header->tag = tag;
ptr = (char *)(header + 1);
do {
ret = ompi_datatype_get_pack_description(target_datatype, &packed_ddt);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
break;
}
if (is_long_datatype) {
/* the datatype does not fit in an eager message. send it seperately */
header->base.flags |= OMPI_OSC_PT2PT_HDR_FLAG_LARGE_DATATYPE;
OMPI_DATATYPE_RETAIN(target_datatype);
ret = ompi_osc_pt2pt_isend_w_cb ((void *) packed_ddt, ddt_len, MPI_BYTE,
target_rank, tag_to_target(tag), module->comm,
ompi_osc_pt2pt_dt_send_complete, target_datatype);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
break;
}
*((uint64_t *) ptr) = ddt_len;
ptr += 8;
} else {
memcpy((unsigned char*) ptr, packed_ddt, ddt_len);
ptr += ddt_len;
}
ret = ompi_osc_pt2pt_irecv_w_cb (result_addr, result_count, result_datatype,
target_rank, tag_to_origin(tag), module->comm,
NULL, ompi_osc_pt2pt_req_comm_complete, pt2pt_request);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
break;
}
if (!is_long_msg) {
header->base.type = OMPI_OSC_PT2PT_HDR_TYPE_GET_ACC;
osc_pt2pt_hton(header, proc);
if (&ompi_mpi_op_no_op.op != op) {
osc_pt2pt_copy_for_send (ptr, payload_len, origin_addr, proc, origin_count,
origin_datatype);
}
} else {
header->base.type = OMPI_OSC_PT2PT_HDR_TYPE_GET_ACC_LONG;
osc_pt2pt_hton(header, proc);
ret = ompi_osc_pt2pt_isend_w_cb (origin_addr, origin_count, origin_datatype,
target_rank, tag_to_target(tag), module->comm,
ompi_osc_pt2pt_req_comm_complete, pt2pt_request);
}
} while (0);
if (OMPI_SUCCESS == ret) {
header->base.flags |= OMPI_OSC_PT2PT_HDR_FLAG_VALID;
*request = (ompi_request_t *) pt2pt_request;
}
return ompi_osc_pt2pt_frag_finish(module, frag);
}
static inline int ompi_osc_pt2pt_rget_internal (void *origin_addr, int origin_count,
struct ompi_datatype_t *origin_dt,
int target,
OPAL_PTRDIFF_TYPE target_disp,
int target_count,
struct ompi_datatype_t *target_dt,
struct ompi_win_t *win, bool release_req,
struct ompi_request_t **request)
{
int ret, tag;
ompi_osc_pt2pt_module_t *module = GET_MODULE(win);
bool is_long_datatype = false;
ompi_osc_pt2pt_frag_t *frag;
ompi_osc_pt2pt_header_get_t *header;
ompi_osc_pt2pt_sync_t *pt2pt_sync;
size_t ddt_len, frag_len;
char *ptr;
const void *packed_ddt;
ompi_osc_pt2pt_request_t *pt2pt_request;
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
"get: 0x%lx, %d, %s, %d, %d, %d, %s, %s",
(unsigned long) origin_addr, origin_count,
origin_dt->name, target, (int) target_disp,
target_count, target_dt->name, win->w_name));
pt2pt_sync = ompi_osc_pt2pt_module_sync_lookup (module, target, NULL);
if (OPAL_UNLIKELY(NULL == pt2pt_sync)) {
return OMPI_ERR_RMA_SYNC;
}
/* gets are always request based, so that we know where to land the data */
OMPI_OSC_PT2PT_REQUEST_ALLOC(win, pt2pt_request);
pt2pt_request->internal = release_req;
/* short-circuit case */
if (0 == origin_count || 0 == target_count) {
ompi_osc_pt2pt_request_complete (pt2pt_request, MPI_SUCCESS);
*request = &pt2pt_request->super;
return OMPI_SUCCESS;
}
/* optimize self communication. TODO: optimize local communication */
if (ompi_comm_rank (module->comm) == target) {
*request = &pt2pt_request->super;
return ompi_osc_pt2pt_get_self (pt2pt_sync, origin_addr, origin_count, origin_dt,
target_disp, target_count, target_dt,
module, pt2pt_request);
}
pt2pt_request->type = OMPI_OSC_PT2PT_HDR_TYPE_GET;
pt2pt_request->origin_addr = origin_addr;
pt2pt_request->origin_count = origin_count;
OMPI_DATATYPE_RETAIN(origin_dt);
pt2pt_request->origin_dt = origin_dt;
/* Compute datatype length. Note that the datatype description
* must fit in a single frag */
ddt_len = ompi_datatype_pack_description_length(target_dt);
frag_len = sizeof(ompi_osc_pt2pt_header_get_t) + ddt_len;
ret = ompi_osc_pt2pt_frag_alloc(module, target, frag_len, &frag, &ptr, false, release_req);
if (OMPI_SUCCESS != ret) {
/* allocate space for the header plus space to store ddt_len */
frag_len = sizeof(ompi_osc_pt2pt_header_get_t) + 8;
ret = ompi_osc_pt2pt_frag_alloc(module, target, frag_len, &frag, &ptr, false, release_req);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
is_long_datatype = true;
}
tag = get_tag (module);
/* for bookkeeping the get is "outgoing" */
ompi_osc_signal_outgoing (module, target, 1);
if (!release_req) {
/* wait for epoch to begin before starting rget operation */
ompi_osc_pt2pt_sync_wait_expected (pt2pt_sync);
}
header = (ompi_osc_pt2pt_header_get_t*) ptr;
header->base.type = OMPI_OSC_PT2PT_HDR_TYPE_GET;
header->base.flags = 0;
header->len = frag_len;
header->count = target_count;
header->displacement = target_disp;
header->tag = tag;
OSC_PT2PT_HTON(header, module, target);
ptr += sizeof(ompi_osc_pt2pt_header_get_t);
do {
ret = ompi_datatype_get_pack_description(target_dt, &packed_ddt);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
break;
}
if (is_long_datatype) {
/* the datatype does not fit in an eager message. send it seperately */
header->base.flags |= OMPI_OSC_PT2PT_HDR_FLAG_LARGE_DATATYPE;
OMPI_DATATYPE_RETAIN(target_dt);
ret = ompi_osc_pt2pt_isend_w_cb ((void *) packed_ddt, ddt_len, MPI_BYTE,
target, tag_to_target(tag), module->comm,
ompi_osc_pt2pt_dt_send_complete, target_dt);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
break;
}
*((uint64_t *) ptr) = ddt_len;
ptr += 8;
} else {
memcpy((unsigned char*) ptr, packed_ddt, ddt_len);
ptr += ddt_len;
}
/* TODO -- store the request somewhere so we can cancel it on error */
pt2pt_request->outstanding_requests = 1;
ret = ompi_osc_pt2pt_irecv_w_cb (origin_addr, origin_count, origin_dt,
target, tag_to_origin(tag), module->comm,
NULL, ompi_osc_pt2pt_req_comm_complete, pt2pt_request);
} while (0);
if (OMPI_SUCCESS == ret) {
header->base.flags |= OMPI_OSC_PT2PT_HDR_FLAG_VALID;
*request = &pt2pt_request->super;
}
return ompi_osc_pt2pt_frag_finish(module, frag);
}
int ompi_osc_pt2pt_compare_and_swap (const void *origin_addr, const void *compare_addr,
void *result_addr, struct ompi_datatype_t *dt,
int target, OPAL_PTRDIFF_TYPE target_disp,
struct ompi_win_t *win)
{
ompi_osc_pt2pt_module_t *module = GET_MODULE(win);
ompi_proc_t *proc = ompi_comm_peer_lookup(module->comm, target);
ompi_osc_pt2pt_frag_t *frag;
ompi_osc_pt2pt_header_cswap_t *header;
ompi_osc_pt2pt_sync_t *pt2pt_sync;
size_t ddt_len, payload_len, frag_len;
ompi_osc_pt2pt_request_t *request;
const void *packed_ddt;
int ret, tag;
char *ptr;
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
"cswap: 0x%lx, 0x%lx, 0x%lx, %s, %d, %d, %s",
(unsigned long) origin_addr, (unsigned long) compare_addr,
(unsigned long) result_addr, dt->name, target, (int) target_disp,
win->w_name));
pt2pt_sync = ompi_osc_pt2pt_module_sync_lookup (module, target, NULL);
if (OPAL_UNLIKELY(NULL == pt2pt_sync)) {
return OMPI_ERR_RMA_SYNC;
}
/* optimize self case. TODO: optimize local case */
if (ompi_comm_rank (module->comm) == target) {
return ompi_osc_pt2pt_cas_self (pt2pt_sync, origin_addr, compare_addr, result_addr, dt, target_disp,
module);
}
/* compare-and-swaps are always request based, so that we know where to land the data */
OMPI_OSC_PT2PT_REQUEST_ALLOC(win, request);
request->type = OMPI_OSC_PT2PT_HDR_TYPE_CSWAP;
request->origin_addr = origin_addr;
request->internal = true;
OMPI_DATATYPE_RETAIN(dt);
request->origin_dt = dt;
/* Compute datatype and payload lengths. Note that the datatype description
* must fit in a single frag. It should be small in this case. */
ddt_len = ompi_datatype_pack_description_length(dt);
/* we need to send both the origin and compare buffers */
payload_len = dt->super.size * 2;
ret = ompi_datatype_get_pack_description(dt, &packed_ddt);
if (OMPI_SUCCESS != ret) {
return ret;
}
frag_len = sizeof(ompi_osc_pt2pt_header_cswap_t) + ddt_len + payload_len;
ret = ompi_osc_pt2pt_frag_alloc(module, target, frag_len, &frag, &ptr, false, false);
if (OMPI_SUCCESS != ret) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
tag = get_tag (module);
ompi_osc_signal_outgoing (module, target, 1);
header = (ompi_osc_pt2pt_header_cswap_t *) ptr;
header->base.type = OMPI_OSC_PT2PT_HDR_TYPE_CSWAP;
header->base.flags = OMPI_OSC_PT2PT_HDR_FLAG_VALID;
header->len = frag_len;
header->displacement = target_disp;
header->tag = tag;
osc_pt2pt_hton(header, proc);
ptr += sizeof(ompi_osc_pt2pt_header_cswap_t);
memcpy((unsigned char*) ptr, packed_ddt, ddt_len);
ptr += ddt_len;
/* pack the origin and compare data */
osc_pt2pt_copy_for_send (ptr, dt->super.size, origin_addr, proc, 1, dt);
ptr += dt->super.size;
osc_pt2pt_copy_for_send (ptr, dt->super.size, compare_addr, proc, 1, dt);
request->outstanding_requests = 1;
ret = ompi_osc_pt2pt_irecv_w_cb (result_addr, 1, dt,
target, tag_to_origin(tag), module->comm,
NULL, ompi_osc_pt2pt_req_comm_complete, request);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
return ret;
}
return ompi_osc_pt2pt_frag_finish (module, frag);
}
static int
ompi_osc_pt2pt_accumulate_w_req (const void *origin_addr, int origin_count,
struct ompi_datatype_t *origin_dt,
int target, OPAL_PTRDIFF_TYPE target_disp,
int target_count,
struct ompi_datatype_t *target_dt,
struct ompi_op_t *op, ompi_win_t *win,
ompi_osc_pt2pt_request_t *request)
{
int ret;
ompi_osc_pt2pt_module_t *module = GET_MODULE(win);
ompi_proc_t *proc = ompi_comm_peer_lookup(module->comm, target);
bool is_long_datatype = false;
bool is_long_msg = false;
ompi_osc_pt2pt_frag_t *frag;
ompi_osc_pt2pt_header_acc_t *header;
ompi_osc_pt2pt_sync_t *pt2pt_sync;
size_t ddt_len, payload_len, frag_len;
char *ptr;
const void *packed_ddt;
int tag = -1;
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
"acc: 0x%lx, %d, %s, %d, %d, %d, %s, %s, %s",
(unsigned long) origin_addr, origin_count,
origin_dt->name, target, (int) target_disp,
target_count, target_dt->name, op->o_name,
win->w_name));
pt2pt_sync = ompi_osc_pt2pt_module_sync_lookup (module, target, NULL);
if (OPAL_UNLIKELY(NULL == pt2pt_sync)) {
return OMPI_ERR_RMA_SYNC;
}
/* short-circuit case */
if (0 == origin_count || 0 == target_count) {
if (request) {
ompi_osc_pt2pt_request_complete (request, MPI_SUCCESS);
}
return OMPI_SUCCESS;
}
/* optimize the self case. TODO: optimize the local case */
if (ompi_comm_rank (module->comm) == target) {
return ompi_osc_pt2pt_acc_self (pt2pt_sync, origin_addr, origin_count, origin_dt,
target_disp, target_count, target_dt,
op, module, request);
}
/* Compute datatype and payload lengths. Note that the datatype description
* must fit in a single frag */
ddt_len = ompi_datatype_pack_description_length(target_dt);
payload_len = origin_dt->super.size * origin_count;
frag_len = sizeof(*header) + ddt_len + payload_len;
ret = ompi_osc_pt2pt_frag_alloc(module, target, frag_len, &frag, &ptr);
if (OMPI_SUCCESS != ret) {
frag_len = sizeof(*header) + ddt_len;
ret = ompi_osc_pt2pt_frag_alloc(module, target, frag_len, &frag, &ptr);
if (OMPI_SUCCESS != ret) {
/* allocate space for the header plus space to store ddt_len */
frag_len = sizeof(*header) + 8;
ret = ompi_osc_pt2pt_frag_alloc(module, target, frag_len, &frag, &ptr);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
is_long_datatype = true;
}
is_long_msg = true;
tag = get_rtag (module);
}
/* flush will be called at the end of this function. make sure all post messages have
* arrived. */
if ((is_long_msg || request) && OMPI_OSC_PT2PT_SYNC_TYPE_PSCW == pt2pt_sync->type) {
ompi_osc_pt2pt_sync_wait (pt2pt_sync);
}
header = (ompi_osc_pt2pt_header_acc_t*) ptr;
header->base.flags = 0;
header->len = frag_len;
header->count = target_count;
header->displacement = target_disp;
header->op = op->o_f_to_c_index;
ptr += sizeof (*header);
do {
ret = ompi_datatype_get_pack_description(target_dt, &packed_ddt);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
break;
}
if (is_long_datatype) {
/* the datatype does not fit in an eager message. send it seperately */
header->base.flags |= OMPI_OSC_PT2PT_HDR_FLAG_LARGE_DATATYPE;
OBJ_RETAIN(target_dt);
ret = ompi_osc_pt2pt_isend_w_cb ((void *) packed_ddt, ddt_len, MPI_BYTE, target,
tag, module->comm, ompi_osc_pt2pt_dt_send_complete,
target_dt);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
break;
}
*((uint64_t *) ptr) = ddt_len;
ptr += 8;
} else {
memcpy((unsigned char*) ptr, packed_ddt, ddt_len);
ptr += ddt_len;
}
if (!is_long_msg) {
header->base.type = OMPI_OSC_PT2PT_HDR_TYPE_ACC;
osc_pt2pt_hton(header, proc);
osc_pt2pt_copy_for_send (ptr, payload_len, origin_addr, proc,
origin_count, origin_dt);
/* the user's buffer is no longer needed so mark the request as
* complete. */
if (request) {
ompi_osc_pt2pt_request_complete (request, MPI_SUCCESS);
}
} else {
header->base.type = OMPI_OSC_PT2PT_HDR_TYPE_ACC_LONG;
header->tag = tag;
osc_pt2pt_hton(header, proc);
OPAL_OUTPUT_VERBOSE((25, ompi_osc_base_framework.framework_output,
"acc: starting long accumulate with tag %d", tag));
/* increment the outgoing send count */
ompi_osc_signal_outgoing (module, target, 1);
if (request) {
request->outstanding_requests = 1;
ret = ompi_osc_pt2pt_isend_w_cb (origin_addr, origin_count, origin_dt,
target, tag, module->comm, ompi_osc_pt2pt_req_comm_complete,
request);
} else {
ret = ompi_osc_pt2pt_component_isend (module, origin_addr, origin_count, origin_dt, target, tag,
module->comm);
}
}
} while (0);
if (OMPI_SUCCESS != ret) {
OPAL_OUTPUT_VERBOSE((25, ompi_osc_base_framework.framework_output,
"acc: failed with eror %d", ret));
} else {
/* mark the fragment as valid */
header->base.flags |= OMPI_OSC_PT2PT_HDR_FLAG_VALID;
}
ret = ompi_osc_pt2pt_frag_finish(module, frag);
if (is_long_msg || request) {
/* need to flush now in case the caller decides to wait on the request */
ompi_osc_pt2pt_frag_flush_target (module, target);
}
return ret;
}
int
ompi_osc_pt2pt_free(ompi_win_t *win)
{
int ret = OMPI_SUCCESS;
ompi_osc_pt2pt_module_t *module = GET_MODULE(win);
if (NULL == module) {
return OMPI_SUCCESS;
}
if (NULL != module->comm) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"pt2pt component destroying window with id %d",
ompi_comm_get_cid(module->comm));
/* finish with a barrier */
if (ompi_group_size(win->w_group) > 1) {
ret = module->comm->c_coll.coll_barrier(module->comm,
module->comm->c_coll.coll_barrier_module);
}
/* remove from component information */
OPAL_THREAD_SCOPED_LOCK(&mca_osc_pt2pt_component.lock,
opal_hash_table_remove_value_uint32(&mca_osc_pt2pt_component.modules,
ompi_comm_get_cid(module->comm)));
}
win->w_osc_module = NULL;
OBJ_DESTRUCT(&module->outstanding_locks);
OBJ_DESTRUCT(&module->locks_pending);
OBJ_DESTRUCT(&module->locks_pending_lock);
OBJ_DESTRUCT(&module->acc_lock);
OBJ_DESTRUCT(&module->cond);
OBJ_DESTRUCT(&module->lock);
/* it is erroneous to close a window with active operations on it so we should
* probably produce an error here instead of cleaning up */
OPAL_LIST_DESTRUCT(&module->pending_acc);
OPAL_LIST_DESTRUCT(&module->pending_posts);
osc_pt2pt_gc_clean (module);
OPAL_LIST_DESTRUCT(&module->request_gc);
OPAL_LIST_DESTRUCT(&module->buffer_gc);
OBJ_DESTRUCT(&module->gc_lock);
if (NULL != module->peers) {
for (int i = 0 ; i < ompi_comm_size (module->comm) ; ++i) {
OBJ_DESTRUCT(module->peers + i);
}
free(module->peers);
}
if (NULL != module->epoch_outgoing_frag_count) free(module->epoch_outgoing_frag_count);
if (NULL != module->frag_request) {
module->frag_request->req_complete_cb = NULL;
ompi_request_cancel (module->frag_request);
ompi_request_free (&module->frag_request);
}
if (NULL != module->comm) {
ompi_comm_free(&module->comm);
}
if (NULL != module->incoming_buffer) free (module->incoming_buffer);
if (NULL != module->free_after) free(module->free_after);
free (module);
return ret;
}
int ompi_osc_rdma_detach (struct ompi_win_t *win, const void *base)
{
ompi_osc_rdma_module_t *module = GET_MODULE(win);
const int my_rank = ompi_comm_rank (module->comm);
ompi_osc_rdma_peer_dynamic_t *my_peer = (ompi_osc_rdma_peer_dynamic_t *) ompi_osc_rdma_module_peer (module, my_rank);
osc_rdma_counter_t region_count, region_id;
ompi_osc_rdma_region_t *region;
int region_index;
if (module->flavor != MPI_WIN_FLAVOR_DYNAMIC) {
return OMPI_ERR_WIN;
}
OPAL_THREAD_LOCK(&module->lock);
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "detach: %s, %p", win->w_name, base);
/* the upper 4 bytes of the region count are an instance counter */
region_count = module->state->region_count & 0xffffffffL;
region_id = module->state->region_count >> 32;
region = ompi_osc_rdma_find_region_containing ((ompi_osc_rdma_region_t *) module->state->regions, 0,
region_count - 1, (intptr_t) base, (intptr_t) base + 1,
module->region_size, ®ion_index);
if (NULL == region) {
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "could not find dynamic memory region starting at %p", base);
OPAL_THREAD_UNLOCK(&module->lock);
return OMPI_ERROR;
}
if (--module->dynamic_handles[region_index].refcnt > 0) {
OPAL_THREAD_UNLOCK(&module->lock);
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "detach complete");
return OMPI_SUCCESS;
}
/* lock the region so it can't change while a peer is reading it */
ompi_osc_rdma_lock_acquire_exclusive (module, &my_peer->super, offsetof (ompi_osc_rdma_state_t, regions_lock));
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "detaching dynamic memory region {%p, %p} from index %d",
base, (void *)((intptr_t) base + region->len), region_index);
if (module->selected_btl->btl_register_mem) {
ompi_osc_rdma_deregister (module, module->dynamic_handles[region_index].btl_handle);
if (region_index < region_count - 1) {
memmove (module->dynamic_handles + region_index, module->dynamic_handles + region_index + 1,
(region_count - region_index - 1) * sizeof (void *));
}
memset (module->dynamic_handles + region_count - 1, 0, sizeof (module->dynamic_handles[0]));
}
if (region_index < region_count - 1) {
memmove (region, (void *)((intptr_t) region + module->region_size),
(region_count - region_index - 1) * module->region_size);;
}
module->state->region_count = ((region_id + 1) << 32) | (region_count - 1);
ompi_osc_rdma_lock_release_exclusive (module, &my_peer->super, offsetof (ompi_osc_rdma_state_t, regions_lock));
OPAL_THREAD_UNLOCK(&module->lock);
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "detach complete");
return OMPI_SUCCESS;
}
int
ompi_osc_rdma_module_free(ompi_win_t *win)
{
int ret = OMPI_SUCCESS;
int tmp, i;
ompi_osc_rdma_module_t *module = GET_MODULE(win);
opal_output_verbose(1, ompi_osc_base_output,
"rdma component destroying window with id %d",
ompi_comm_get_cid(module->m_comm));
/* finish with a barrier */
if (ompi_group_size(win->w_group) > 1) {
ret = module->m_comm->c_coll.coll_barrier(module->m_comm,
module->m_comm->c_coll.coll_barrier_module);
}
/* remove from component information */
OPAL_THREAD_LOCK(&mca_osc_rdma_component.c_lock);
tmp = opal_hash_table_remove_value_uint32(&mca_osc_rdma_component.c_modules,
ompi_comm_get_cid(module->m_comm));
/* only take the output of hast_table_remove if there wasn't already an error */
ret = (ret != OMPI_SUCCESS) ? ret : tmp;
if (0 == opal_hash_table_get_size(&mca_osc_rdma_component.c_modules)) {
#if OPAL_ENABLE_PROGRESS_THREADS
void *foo;
mca_osc_rdma_component.c_thread_run = false;
opal_condition_broadcast(&ompi_request_cond);
opal_thread_join(&mca_osc_rdma_component.c_thread, &foo);
#else
opal_progress_unregister(ompi_osc_rdma_component_progress);
#endif
}
OPAL_THREAD_UNLOCK(&mca_osc_rdma_component.c_lock);
win->w_osc_module = NULL;
OBJ_DESTRUCT(&module->m_unlocks_pending);
OBJ_DESTRUCT(&module->m_locks_pending);
OBJ_DESTRUCT(&module->m_queued_sendreqs);
OBJ_DESTRUCT(&module->m_copy_pending_sendreqs);
OBJ_DESTRUCT(&module->m_pending_sendreqs);
OBJ_DESTRUCT(&module->m_acc_lock);
OBJ_DESTRUCT(&module->m_cond);
OBJ_DESTRUCT(&module->m_lock);
if (NULL != module->m_sc_remote_ranks) {
free(module->m_sc_remote_ranks);
}
if (NULL != module->m_sc_remote_active_ranks) {
free(module->m_sc_remote_active_ranks);
}
if (NULL != module->m_pending_buffers) {
free(module->m_pending_buffers);
}
if (NULL != module->m_fence_coll_counts) {
free(module->m_fence_coll_counts);
}
if (NULL != module->m_copy_num_pending_sendreqs) {
free(module->m_copy_num_pending_sendreqs);
}
if (NULL != module->m_num_pending_sendreqs) {
free(module->m_num_pending_sendreqs);
}
if (NULL != module->m_peer_info) {
for (i = 0 ; i < ompi_comm_size(module->m_comm) ; ++i) {
ompi_osc_rdma_peer_info_free(&module->m_peer_info[i]);
}
free(module->m_peer_info);
}
if (NULL != module->m_comm) ompi_comm_free(&module->m_comm);
if (NULL != module) free(module);
return ret;
}
int ompi_osc_rdma_attach (struct ompi_win_t *win, void *base, size_t len)
{
ompi_osc_rdma_module_t *module = GET_MODULE(win);
const int my_rank = ompi_comm_rank (module->comm);
ompi_osc_rdma_peer_t *my_peer = ompi_osc_rdma_module_peer (module, my_rank);
ompi_osc_rdma_region_t *region;
osc_rdma_counter_t region_count;
osc_rdma_counter_t region_id;
void *bound;
intptr_t page_size = getpagesize ();
int region_index;
int ret;
if (module->flavor != MPI_WIN_FLAVOR_DYNAMIC) {
return OMPI_ERR_RMA_FLAVOR;
}
if (0 == len) {
/* shot-circuit 0-byte case */
return OMPI_SUCCESS;
}
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "attach: %s, %p, %lu", win->w_name, base, (unsigned long) len);
OPAL_THREAD_LOCK(&module->lock);
region_count = module->state->region_count & 0xffffffffL;
region_id = module->state->region_count >> 32;
if (region_count == mca_osc_rdma_component.max_attach) {
OPAL_THREAD_UNLOCK(&module->lock);
return OMPI_ERR_RMA_ATTACH;
}
/* it is wasteful to register less than a page. this may allow the remote side to access more
* memory but the MPI standard covers this with calling the calling behavior erroneous */
bound = (void *)OPAL_ALIGN((intptr_t) base + len, page_size, intptr_t);
base = (void *)((intptr_t) base & ~(page_size - 1));
len = (size_t)((intptr_t) bound - (intptr_t) base);
/* see if a matching region already exists */
region = ompi_osc_rdma_find_region_containing ((ompi_osc_rdma_region_t *) module->state->regions, 0, region_count - 1, (intptr_t) base,
(intptr_t) bound, module->region_size, ®ion_index);
if (NULL != region) {
++module->dynamic_handles[region_index].refcnt;
OPAL_THREAD_UNLOCK(&module->lock);
/* no need to invalidate remote caches */
return OMPI_SUCCESS;
}
/* region is in flux */
module->state->region_count = -1;
opal_atomic_wmb ();
ompi_osc_rdma_lock_acquire_exclusive (module, my_peer, offsetof (ompi_osc_rdma_state_t, regions_lock));
/* do a binary seach for where the region should be inserted */
if (region_count) {
region = find_insertion_point ((ompi_osc_rdma_region_t *) module->state->regions, 0, region_count - 1, (intptr_t) base,
module->region_size, ®ion_index);
if (region_index < region_count) {
memmove ((void *) ((intptr_t) region + module->region_size), region, (region_count - region_index) * module->region_size);
if (module->selected_btl->btl_register_mem) {
memmove (module->dynamic_handles + region_index + 1, module->dynamic_handles + region_index,
(region_count - region_index) * sizeof (module->dynamic_handles[0]));
}
}
} else {
region_index = 0;
region = (ompi_osc_rdma_region_t *) module->state->regions;
}
region->base = (intptr_t) base;
region->len = len;
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "attaching dynamic memory region {%p, %p} at index %d",
base, (void *)((intptr_t) base + len), region_index);
if (module->selected_btl->btl_register_mem) {
mca_btl_base_registration_handle_t *handle;
ret = ompi_osc_rdma_register (module, MCA_BTL_ENDPOINT_ANY, (void *) region->base, region->len, MCA_BTL_REG_FLAG_ACCESS_ANY,
&handle);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
OPAL_THREAD_UNLOCK(&module->lock);
return OMPI_ERR_RMA_ATTACH;
}
memcpy (region->btl_handle_data, handle, module->selected_btl->btl_registration_handle_size);
module->dynamic_handles[region_index].btl_handle = handle;
} else {
module->dynamic_handles[region_index].btl_handle = NULL;
}
module->dynamic_handles[region_index].refcnt = 1;
#if OPAL_ENABLE_DEBUG
for (int i = 0 ; i < region_count + 1 ; ++i) {
region = (ompi_osc_rdma_region_t *) ((intptr_t) module->state->regions + i * module->region_size);
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, " dynamic region %d: {%p, %lu}", i,
(void *) region->base, (unsigned long) region->len);
}
#endif
opal_atomic_mb ();
/* the region state has changed */
module->state->region_count = ((region_id + 1) << 32) | (region_count + 1);
ompi_osc_rdma_lock_release_exclusive (module, my_peer, offsetof (ompi_osc_rdma_state_t, regions_lock));
OPAL_THREAD_UNLOCK(&module->lock);
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "attach complete");
return OMPI_SUCCESS;
}
static int
ompi_osc_pt2pt_accumulate_w_req (const void *origin_addr, int origin_count,
struct ompi_datatype_t *origin_dt,
int target, OPAL_PTRDIFF_TYPE target_disp,
int target_count,
struct ompi_datatype_t *target_dt,
struct ompi_op_t *op, ompi_win_t *win,
ompi_osc_pt2pt_request_t *request)
{
int ret;
ompi_osc_pt2pt_module_t *module = GET_MODULE(win);
ompi_proc_t *proc = ompi_comm_peer_lookup(module->comm, target);
bool is_long_datatype = false;
bool is_long_msg = false;
ompi_osc_pt2pt_frag_t *frag;
ompi_osc_pt2pt_header_acc_t *header;
ompi_osc_pt2pt_sync_t *pt2pt_sync;
size_t ddt_len, payload_len, frag_len;
char *ptr;
const void *packed_ddt;
int tag = -1;
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
"acc: 0x%lx, %d, %s, %d, %d, %d, %s, %s, %s",
(unsigned long) origin_addr, origin_count,
origin_dt->name, target, (int) target_disp,
target_count, target_dt->name, op->o_name,
win->w_name));
pt2pt_sync = ompi_osc_pt2pt_module_sync_lookup (module, target, NULL);
if (OPAL_UNLIKELY(NULL == pt2pt_sync)) {
return OMPI_ERR_RMA_SYNC;
}
/* short-circuit case */
if (0 == origin_count || 0 == target_count) {
if (request) {
ompi_osc_pt2pt_request_complete (request, MPI_SUCCESS);
}
return OMPI_SUCCESS;
}
/* optimize the self case. TODO: optimize the local case */
if (ompi_comm_rank (module->comm) == target) {
return ompi_osc_pt2pt_acc_self (pt2pt_sync, origin_addr, origin_count, origin_dt,
target_disp, target_count, target_dt,
op, module, request);
}
/* Compute datatype and payload lengths. Note that the datatype description
* must fit in a single frag */
ddt_len = ompi_datatype_pack_description_length(target_dt);
payload_len = origin_dt->super.size * origin_count;
frag_len = sizeof(*header) + ddt_len + payload_len;
ret = ompi_osc_pt2pt_frag_alloc(module, target, frag_len, &frag, &ptr, false, true);
if (OMPI_SUCCESS != ret) {
frag_len = sizeof(*header) + ddt_len;
ret = ompi_osc_pt2pt_frag_alloc(module, target, frag_len, &frag, &ptr, true, !request);
if (OMPI_SUCCESS != ret) {
/* allocate space for the header plus space to store ddt_len */
frag_len = sizeof(*header) + 8;
ret = ompi_osc_pt2pt_frag_alloc(module, target, frag_len, &frag, &ptr, true, !request);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
is_long_datatype = true;
}
is_long_msg = true;
tag = get_tag (module);
} else {
/* still need to set the tag for the active/passive logic on the target */
tag = !!(module->passive_target_access_epoch);
}
if (is_long_msg) {
/* wait for synchronization before posting a long message */
if (pt2pt_sync->type == OMPI_OSC_PT2PT_SYNC_TYPE_LOCK) {
OPAL_THREAD_LOCK(&pt2pt_sync->lock);
ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, target);
while (!(peer->flags & OMPI_OSC_PT2PT_PEER_FLAG_EAGER)) {
opal_condition_wait(&pt2pt_sync->cond, &pt2pt_sync->lock);
}
OPAL_THREAD_UNLOCK(&pt2pt_sync->lock);
} else {
ompi_osc_pt2pt_sync_wait_expected (pt2pt_sync);
}
}
header = (ompi_osc_pt2pt_header_acc_t*) ptr;
header->base.flags = 0;
header->len = frag_len;
header->count = target_count;
header->displacement = target_disp;
header->op = op->o_f_to_c_index;
header->tag = tag;
ptr += sizeof (*header);
do {
ret = ompi_datatype_get_pack_description(target_dt, &packed_ddt);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
break;
}
if (is_long_datatype) {
/* the datatype does not fit in an eager message. send it seperately */
header->base.flags |= OMPI_OSC_PT2PT_HDR_FLAG_LARGE_DATATYPE;
OMPI_DATATYPE_RETAIN(target_dt);
ret = ompi_osc_pt2pt_isend_w_cb ((void *) packed_ddt, ddt_len, MPI_BYTE,
target, tag_to_target(tag), module->comm,
ompi_osc_pt2pt_dt_send_complete, target_dt);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
break;
}
*((uint64_t *) ptr) = ddt_len;
ptr += 8;
} else {
memcpy((unsigned char*) ptr, packed_ddt, ddt_len);
ptr += ddt_len;
}
if (!is_long_msg) {
header->base.type = OMPI_OSC_PT2PT_HDR_TYPE_ACC;
osc_pt2pt_hton(header, proc);
osc_pt2pt_copy_for_send (ptr, payload_len, origin_addr, proc,
origin_count, origin_dt);
/* the user's buffer is no longer needed so mark the request as
* complete. */
if (request) {
ompi_osc_pt2pt_request_complete (request, MPI_SUCCESS);
}
} else {
header->base.type = OMPI_OSC_PT2PT_HDR_TYPE_ACC_LONG;
osc_pt2pt_hton(header, proc);
OPAL_OUTPUT_VERBOSE((25, ompi_osc_base_framework.framework_output,
"acc: starting long accumulate with tag %d", tag));
ret = ompi_osc_pt2pt_data_isend (module, origin_addr, origin_count, origin_dt,
target, tag_to_target(tag), request);
}
} while (0);
if (OMPI_SUCCESS != ret) {
OPAL_OUTPUT_VERBOSE((25, ompi_osc_base_framework.framework_output,
"acc: failed with eror %d", ret));
} else {
/* mark the fragment as valid */
header->base.flags |= OMPI_OSC_PT2PT_HDR_FLAG_VALID;
}
return ompi_osc_pt2pt_frag_finish(module, frag);
}
int Executor_execute(Executor *executor, int timeout_ms)
{
DEBUG(("Executor execute start\n"));
//determine max endtime based on timeout
struct timespec tm;
clock_gettime(CLOCK_MONOTONIC, &tm);
DEBUG(("Executor start_tm_ms: %3.2f\n", TIMESPEC_TO_MS(tm)));
executor->end_tm_ms = TIMESPEC_TO_MS(tm) + ((float)timeout_ms);
DEBUG(("Executor end_tm_ms: %3.2f\n", executor->end_tm_ms));
executor->numevents = 0;
for(int i = 0; i < executor->numpairs; i++) {
struct _Pair *pair = &executor->pairs[i];
Connection_execute_start(pair->connection, executor, pair->batch);
}
int select_result = 1;
//for as long there are outstanding events and no error or timeout occurred:
while(executor->numevents > 0 && select_result > 0) {
//figure out how many ms left for this execution
struct timeval tv;
Executor_current_timeout(executor, &tv);
//copy filedes. sets, because select is going to modify them
fd_set readfds;
fd_set writefds;
readfds = executor->readfds;
writefds = executor->writefds;
//do the select
DEBUG(("Executor start select max_fd %d, num_events: %d\n", executor->max_fd, executor->numevents));
select_result = select(executor->max_fd + 1, &readfds, &writefds, NULL, &tv);
DEBUG(("Executor select res %d\n", select_result));
for(int i = 0; i < executor->numpairs; i++) {
struct _Pair *pair = &executor->pairs[i];
Connection *connection = pair->connection;
Batch *batch = pair->batch;
EventType event = 0;
if(FD_ISSET(connection->sockfd, &readfds)) {
event |= EVENT_READ;
FD_CLR(connection->sockfd, &executor->readfds);
executor->numevents -= 1;
}
if(FD_ISSET(connection->sockfd, &writefds)) {
event |= EVENT_WRITE;
FD_CLR(connection->sockfd, &executor->writefds);
executor->numevents -= 1;
}
if(select_result == 0) {
event = EVENT_TIMEOUT;
}
else if(select_result < 0) {
event = EVENT_ERROR;
}
if(event > 0 && Batch_has_command(batch)) {
//there is an event, and batch is not finished
Connection_handle_event(connection, event);
}
}
}
if(select_result > 1) {
select_result = 1;
}
if(select_result < 0) {
Module_set_error(GET_MODULE(), "Execute select error, errno: [%d] %s", errno, strerror(errno));
}
else if(select_result == 0) {
Module_set_error(GET_MODULE(), "Execute timeout");
}
DEBUG(("Executor execute done\n"));
return select_result;
}
