gaspi_return_t
pgaspi_write_list_notify (const gaspi_number_t num,
gaspi_segment_id_t * const segment_id_local,
gaspi_offset_t * const offset_local,
const gaspi_rank_t rank,
gaspi_segment_id_t * const segment_id_remote,
gaspi_offset_t * const offset_remote,
gaspi_size_t * const size,
const gaspi_segment_id_t segment_id_notification,
const gaspi_notification_id_t notification_id,
const gaspi_notification_t notification_value,
const gaspi_queue_id_t queue,
const gaspi_timeout_t timeout_ms)
{
if(num == 0)
return GASPI_ERR_INV_NUM;
if(notification_value == 0)
return GASPI_ERR_INV_NOTIF_VAL;
#ifdef DEBUG
gaspi_verify_init("gaspi_write_list_notify");
gaspi_verify_queue(queue);
gaspi_number_t n;
for(n = 0; n < num; n++)
{
gaspi_verify_local_off(offset_local[n], segment_id_local[n]);
gaspi_verify_remote_off(offset_remote[n], segment_id_remote[n], rank);
gaspi_verify_comm_size(size[n], segment_id_local[n], segment_id_remote[n], rank, GASPI_MAX_TSIZE_C);
}
#endif
gaspi_return_t eret = GASPI_ERROR;
if(lock_gaspi_tout (&glb_gaspi_ctx.lockC[queue], timeout_ms))
return GASPI_TIMEOUT;
if( GASPI_ENDPOINT_DISCONNECTED == glb_gaspi_ctx.ep_conn[rank].cstat )
{
eret = pgaspi_connect((gaspi_rank_t) rank, timeout_ms);
if ( eret != GASPI_SUCCESS)
{
goto endL;
}
}
eret = pgaspi_dev_write_list_notify(num,
segment_id_local, offset_local, rank,
segment_id_remote, offset_remote, (unsigned int *)size,
segment_id_notification, notification_id, notification_value,
queue);
glb_gaspi_ctx.ne_count_c[queue] += (int) (num + 1);
endL:
unlock_gaspi (&glb_gaspi_ctx.lockC[queue]);
return eret;
}
gaspi_return_t
pgaspi_passive_receive (const gaspi_segment_id_t segment_id_local,
const gaspi_offset_t offset_local,
gaspi_rank_t * const rem_rank,
const gaspi_size_t size,
const gaspi_timeout_t timeout_ms)
{
gaspi_verify_init("gaspi_passive_receive");
gaspi_verify_local_off(offset_local, segment_id_local, size);
gaspi_verify_comm_size(size, segment_id_local,
segment_id_local, glb_gaspi_ctx.rank, GASPI_MAX_TSIZE_P);
gaspi_return_t eret = GASPI_ERROR;
if( lock_gaspi_tout (&glb_gaspi_ctx.lockPR, timeout_ms) )
{
return GASPI_TIMEOUT;
}
eret = pgaspi_dev_passive_receive(segment_id_local, offset_local, rem_rank,
size, timeout_ms);
unlock_gaspi (&glb_gaspi_ctx.lockPR);
return eret;
}
gaspi_return_t
pgaspi_gpu_write_notify(const gaspi_segment_id_t segment_id_local,
const gaspi_offset_t offset_local,
const gaspi_rank_t rank,
const gaspi_segment_id_t segment_id_remote,
const gaspi_offset_t offset_remote,
const gaspi_size_t size,
const gaspi_notification_id_t notification_id,
const gaspi_notification_t notification_value,
const gaspi_queue_id_t queue,
const gaspi_timeout_t timeout_ms)
{
gaspi_verify_init("gaspi_gpu_write_notify");
gaspi_verify_local_off(offset_local, segment_id_local, size);
gaspi_verify_remote_off(offset_remote, segment_id_remote, rank, size);
gaspi_verify_queue(queue);
gaspi_verify_comm_size(size, segment_id_local, segment_id_remote, rank, GASPI_MAX_TSIZE_C);
if( notification_value == 0 )
{
gaspi_printf("Zero is not allowed as notification value.");
return GASPI_ERR_INV_NOTIF_VAL;
}
gaspi_return_t eret = GASPI_ERROR;
gaspi_context_t * const gctx = &glb_gaspi_ctx;
if(lock_gaspi_tout (&gctx->lockC[queue], timeout_ms))
return GASPI_TIMEOUT;
if( GASPI_ENDPOINT_DISCONNECTED == gctx->ep_conn[rank].cstat )
{
eret = pgaspi_connect((gaspi_rank_t) rank, timeout_ms);
if ( eret != GASPI_SUCCESS)
{
goto endL;
}
}
eret = pgaspi_dev_gpu_write_notify(segment_id_local, offset_local, rank,
segment_id_remote, offset_remote, size,
notification_id, notification_value,
queue, timeout_ms);
if( eret != GASPI_SUCCESS )
{
/* gctx->qp_state_vec[queue][rank] = GASPI_STATE_CORRUPT; */
goto endL;
}
/* GPI2_STATS_INC_COUNT(GASPI_STATS_COUNTER_NUM_WRITE_NOT, 1); */
/* GPI2_STATS_INC_COUNT(GASPI_STATS_COUNTER_BYTES_WRITE, size); */
endL:
unlock_gaspi (&gctx->lockC[queue]);
return eret;
}
gaspi_return_t
pgaspi_write_notify (const gaspi_segment_id_t segment_id_local,
const gaspi_offset_t offset_local,
const gaspi_rank_t rank,
const gaspi_segment_id_t segment_id_remote,
const gaspi_offset_t offset_remote,
const gaspi_size_t size,
const gaspi_notification_id_t notification_id,
const gaspi_notification_t notification_value,
const gaspi_queue_id_t queue,
const gaspi_timeout_t timeout_ms)
{
gaspi_verify_init("gaspi_write_notify");
gaspi_verify_local_off(offset_local, segment_id_local);
gaspi_verify_remote_off(offset_remote, segment_id_remote, rank);
gaspi_verify_queue(queue);
gaspi_verify_comm_size(size, segment_id_local, segment_id_remote, rank, GASPI_MAX_TSIZE_C);
if(notification_value == 0)
return GASPI_ERR_INV_NOTIF_VAL;
gaspi_return_t eret = GASPI_ERROR;
if(lock_gaspi_tout (&glb_gaspi_ctx.lockC[queue], timeout_ms))
return GASPI_TIMEOUT;
if( GASPI_ENDPOINT_DISCONNECTED == glb_gaspi_ctx.ep_conn[rank].cstat )
{
eret = pgaspi_connect((gaspi_rank_t) rank, timeout_ms);
if ( eret != GASPI_SUCCESS)
{
goto endL;
}
}
eret = pgaspi_dev_write_notify(segment_id_local, offset_local, rank,
segment_id_remote, offset_remote, size,
notification_id, notification_value,
queue);
glb_gaspi_ctx.ne_count_c[queue] += 2;
endL:
unlock_gaspi (&glb_gaspi_ctx.lockC[queue]);
return eret;
}
gaspi_return_t
pgaspi_passive_send (const gaspi_segment_id_t segment_id_local,
const gaspi_offset_t offset_local,
const gaspi_rank_t rank,
const gaspi_size_t size,
const gaspi_timeout_t timeout_ms)
{
gaspi_verify_init("gaspi_passive_send");
gaspi_verify_local_off(offset_local, segment_id_local, size);
gaspi_verify_comm_size(size, segment_id_local,
segment_id_local, glb_gaspi_ctx.rank, GASPI_MAX_TSIZE_P);
gaspi_verify_rank(rank);
gaspi_return_t eret = GASPI_ERROR;
if( lock_gaspi_tout (&glb_gaspi_ctx.lockPS, timeout_ms) )
{
return GASPI_TIMEOUT;
}
if( GASPI_ENDPOINT_DISCONNECTED == glb_gaspi_ctx.ep_conn[rank].cstat )
{
eret = pgaspi_connect((gaspi_rank_t) rank, timeout_ms);
if( eret != GASPI_SUCCESS )
{
goto endL;
}
}
eret = pgaspi_dev_passive_send(segment_id_local, offset_local, rank,
size, glb_gaspi_ctx.ne_count_p, timeout_ms);
if( eret == GASPI_ERROR )
{
glb_gaspi_ctx.qp_state_vec[GASPI_PASSIVE_QP][rank] = GASPI_STATE_CORRUPT;
}
endL:
unlock_gaspi (&glb_gaspi_ctx.lockPS);
return eret;
}
gaspi_return_t
pgaspi_read (const gaspi_segment_id_t segment_id_local,
const gaspi_offset_t offset_local,
const gaspi_rank_t rank,
const gaspi_segment_id_t segment_id_remote,
const gaspi_offset_t offset_remote,
const gaspi_size_t size,
const gaspi_queue_id_t queue,
const gaspi_timeout_t timeout_ms)
{
gaspi_verify_init("gaspi_read");
gaspi_verify_local_off(offset_local, segment_id_local);
gaspi_verify_remote_off(offset_remote, segment_id_remote, rank);
gaspi_verify_queue(queue);
gaspi_verify_comm_size(size, segment_id_local, segment_id_remote, rank, GASPI_MAX_TSIZE_C);
gaspi_return_t eret = GASPI_ERROR;
if(lock_gaspi_tout (&glb_gaspi_ctx.lockC[queue], timeout_ms))
return GASPI_TIMEOUT;
if( GASPI_ENDPOINT_DISCONNECTED == glb_gaspi_ctx.ep_conn[rank].cstat )
{
eret = pgaspi_connect((gaspi_rank_t) rank, timeout_ms);
if ( eret != GASPI_SUCCESS)
{
goto endL;
}
}
eret = pgaspi_dev_read(segment_id_local, offset_local, rank,
segment_id_remote,offset_remote, (unsigned int) size,
queue);
glb_gaspi_ctx.ne_count_c[queue]++;
endL:
unlock_gaspi (&glb_gaspi_ctx.lockC[queue]);
return eret;
}
