/**
* \brief Register the DCMF ARMCI Extention fence operation.
*
* \param[in] connection_array Connection array
*
* \see DCMF_Control_register
*/
void ARMCIX_DCMF_Fence_register (ARMCIX_DCMF_Connection_t * connection_array)
{
DCMF_CriticalSection_enter (0);
DCMF_Send_Configuration_t send_configuration = {
DCMF_DEFAULT_SEND_PROTOCOL,
DCMF_DEFAULT_NETWORK,
ARMCIX_DCMF_ReceiveFenceRequest,
connection_array,
NULL,
NULL
};
DCMF_Send_register (&__fence_rts_protocol, &send_configuration);
DCMF_Control_Configuration_t configuration = {
DCMF_DEFAULT_CONTROL_PROTOCOL,
DCMF_DEFAULT_NETWORK,
ARMCIX_DCMF_ReceiveFenceAck,
connection_array
};
DCMF_Control_register (&__fence_ack_protocol, &configuration);
DCMF_CriticalSection_exit (0);
}
/**
* \brief Register the DCMF ARMCI Extention rmw operation.
*
* \see DCMF_Control_register
* \see DCMF_Send_register
*/
void ARMCIX_DCMF_Rmw_register ()
{
DCMF_CriticalSection_enter (0);
DCMF_Send_Configuration_t request_configuration = {
DCMF_DEFAULT_SEND_PROTOCOL,
DCMF_DEFAULT_NETWORK,
ARMCIX_DCMF_RecvRMWRequest,
NULL,
NULL,
NULL
};
DCMF_Send_register (&__rmw_request_protocol, &request_configuration);
DCMF_Control_Configuration_t response_configuration = {
DCMF_DEFAULT_CONTROL_PROTOCOL,
DCMF_DEFAULT_NETWORK,
ARMCIX_DCMF_ReceiveRMWResponse,
NULL
};
DCMF_Control_register (&__rmw_response_protocol, &response_configuration);
DCMF_CriticalSection_exit (0);
}
void ARMCIX_DCMF_Connection_initialize ()
{
DCMF_CriticalSection_enter(0);
__global_connection.peer = (unsigned) -1;
unsigned rank = DCMF_Messager_rank ();
unsigned size = DCMF_Messager_size ();
posix_memalign ((void **)&__connection, 16, sizeof(ARMCIX_DCMF_Connection_t) * size);
bzero ((void *)__connection, sizeof(ARMCIX_DCMF_Connection_t) * size);
void * base = NULL;
size_t bytes = (size_t) -1;
unsigned i;
for (i = 0; i < size; i++)
{
__connection[i].peer = i;
#warning fix memregion setup to handle non-global address space pinning.
//DCMF_Result result =
DCMF_Memregion_create (&__connection[i].local_mem_region,
&bytes, (size_t) -1, NULL, 0);
}
// Register a send protocol to exchange memory regions
DCMF_Protocol_t send_protocol;
DCMF_Send_Configuration_t send_configuration = {
DCMF_DEFAULT_SEND_PROTOCOL,
DCMF_DEFAULT_NETWORK,
ARMCIX_DCMF_RecvMemregion1,
__connection,
ARMCIX_DCMF_RecvMemregion2,
__connection
};
DCMF_Send_register (&send_protocol, &send_configuration);
DCMF_Request_t request;
volatile unsigned active;
DCMF_Callback_t cb_done = { ARMCIX_DCMF_cb_decrement, (void *) &active };
// Exchange the memory regions
__memregions_to_receive = size;
for (i = 0; i < size; i++)
{
unsigned peer = (rank+i)%size;
active = 1;
DCMF_Send (&send_protocol,
&request,
cb_done,
DCMF_SEQUENTIAL_CONSISTENCY,
peer,
sizeof(DCMF_Memregion_t),
(char *) &__connection[peer].local_mem_region,
(DCQuad *) NULL,
0);
while (active) DCMF_Messager_advance();
}
while (__memregions_to_receive) DCMF_Messager_advance();
DCMF_CriticalSection_exit(0);
}
