/* Check for incoming messages.
Input Parameter:
. is_blocking - true if this routine should block until a message is
available
Returns -1 if nonblocking and no messages pending. Otherwise
returns error code (MPI_SUCCESS == 0 for success)
This routine makes use of a single dispatch routine to handle all
incoming messages. This makes the code a little lengthy, but each
piece is relatively simple.
This is the message-passing version. The shared-memory version is
in chchkshdev.c .
*/
int MPID_MPX_Check_incoming(
MPID_Device *dev,
MPID_BLOCKING_TYPE is_blocking)
{
DSECTION("MPID_MPX_Check_incoming");
MPID_PKT_T pkt;
int from_grank;
MPIR_RHANDLE *rhandle;
int is_posted;
int err = MPI_SUCCESS;
MPID_MPX_Data_global_type *global_data;
DSECTENTRYPOINT;
MPID_MPX_Test_device(MPID_devset->active_dev, "Check_incoming");
/* get the pointer to the global data struct of this device entity: */
global_data = MPID_MPX_Get_global_data(MPID_devset->active_dev);
MPID_MPX_DEBUG_PRINT_MSG("Entering check_incoming");
/* If nonblocking and no headers available, exit */
if (is_blocking == MPID_NOTBLOCKING) {
if (!MPID_PKT_CHECK()) {
MPID_MPX_DEBUG_PRINT_MSG("Leaving check_incoming (no messages)");
DSECTLEAVE
return -1;
}
MPID_MPX_DEBUG_PRINT_MSG("Message is available!");
}
void *shmat(int shmid, void *shmaddr, int shmflg)
{
PtrHandleChain dummy = start;
DWORD dwDesiredAccess;
HANDLE hFile = NULL;
int res;
DSECTION("shmat");
DSECTENTRYPOINT;
if ((shmflg&SHM_RDONLY) == SHM_RDONLY)
dwDesiredAccess = FILE_MAP_READ;
else
dwDesiredAccess = FILE_MAP_WRITE;
while ((dummy != NULL) && (dummy->key != shmid))
dummy = dummy->next;
if(dummy == NULL) {
/* We have to open the mapping first.
Hopefully someone else alredy created it...*/
res = shmget(shmid,0,0);
if(res <0) {
DWARNING("shmget failed for key");
DSECTLEAVE
return (void *) -1;
}
smi_error_t SMI_Get (void *dst, int src_region_id, int offset, size_t size)
{
DSECTION("SMI_Get");
smi_memcpy_handle h = NULL;
smi_error_t error;
DSECTENTRYPOINT;
error = SMI_Iget(dst, src_region_id, offset,size, &h);
if (error == SMI_SUCCESS)
error = SMI_Memwait(h);
DSECTLEAVE; return error;
}
smi_error_t SMI_Put (int dest_region_id, int offset, void *src, size_t size)
{
DSECTION("SMI_Put");
smi_memcpy_handle h = NULL;
smi_error_t error;
DSECTENTRYPOINT;
error = SMI_Iput(dest_region_id, offset, src, size, &h);
if (error == SMI_SUCCESS)
error = SMI_Memwait(h);
DSECTLEAVE; return error;
}
smi_error_t _smi_copy_globally_distributed_to_local(int global_region_id,
int local_region_id)
{
DSECTION("_smi_copy_globally_distributed_to_local");
smi_error_t error;
size_t i, j, k; /* loop counters */
smi_rlayout_t* distributed_layout; /* description of the detailed memory */
smi_rlayout_t* local_layout; /* layout of the two regions */
void* own_global_segment_start = NULL; /* start address and size of a segment*/
size_t own_global_segment_size = 0; /* and it's size of the entire copy */
void* other_global_segment_start = NULL;/* proceedure; once for the own source*/
size_t other_global_segment_size = 0; /* segment, once for the remote */
/* source segment */
double* ptr; /* tmp var */
int other_machine_rank;
size_t no_iterations; /* specifies how many */
/* SMI_COPY_BUFFER_SIZE-byte blocks */
/* the largest segment of the distrib.*/
/* region contains */
int tmp_machine, tmp_proc;
size_t act_inner_iterations;
size_t remaining_bytes;
size_t tmp_it;
double* tmp_ptr;
size_t no_inner_iterations = SMI_COPY_BUFFER_SIZE/sizeof(double);
double* s;
double* d;
DSECTENTRYPOINT;
/* _smi_allocate buffer space, if not already done */
if (buffer__smi_allocated == false)
{
buffer__smi_allocated = true;
error = SMI_Init_PC(&progress_counter);
ASSERT_R((error==SMI_SUCCESS),"Could not init progress-counter",error);
ALLOCATE( buffer, double**, _smi_nbr_machines * sizeof(double*) );
for(i=0; i<(size_t)_smi_nbr_machines; i++)
{
tmp_proc = _smi_first_proc_on_node((int) i);
error = SMI_Cmalloc(SMI_COPY_BUFFER_SIZE, tmp_proc|INTERNAL,
(char**)(&(buffer[i])));
ASSERT_R((error==SMI_SUCCESS),"Could not allocate memory",error);
}
}
void _smi_sci_desc_finalize() {
DSECTION("smi_sci_desc_finalize");
smi_sci_desc_list_t* pTemp;
sci_error_t sci_error;
DSECTENTRYPOINT;
DNOTICE("clearing LocList");
while ((pTemp = sd_node_pop_front(&pLocListRoot)) != NULL) {
rs_SCIClose(pTemp->sd,0,&sci_error);
free(pTemp);
}
DNOTICE("clearing LocIntList");
while ((pTemp = sd_node_pop_front(&pLocIntListRoot)) != NULL) {
rs_SCIClose(pTemp->sd,0,&sci_error);
free(pTemp);
}
DNOTICE("clearing RmtList");
while ((pTemp = sd_node_pop_front(&pRmtListRoot)) != NULL) {
rs_SCIClose(pTemp->sd,0,&sci_error);
free(pTemp);
}
DNOTICE("clearing RmtIntList");
while ((pTemp = sd_node_pop_front(&pRmtIntListRoot)) != NULL) {
rs_SCIClose(pTemp->sd,0,&sci_error);
free(pTemp);
}
DNOTICE("clearing IntList");
while ((pTemp = sd_node_pop_front(&pIntListRoot)) != NULL) {
rs_SCIClose(pTemp->sd,0,&sci_error);
free(pTemp);
}
DNOTICE("clearing FullList");
while ((pTemp = sd_node_pop_front(&pFullListRoot)) != NULL) {
rs_SCIClose(pTemp->sd,0,&sci_error);
free(pTemp);
}
DNOTICE("destroy mutex");
SMI_DESTROY_LOCK(&sdMutex);
DSECTLEAVE;
}
smi_error_t SMI_Redirect_IO(int err, void* errparam,
int out, void* outparam,
int in, void* inparam)
{
DSECTION(" SMI_Redirect_IO");
char filename[256];
DSECTENTRYPOINT;
ASSERT_R((_smi_initialized==true),"SMI not initialized",SMI_ERR_NOINIT);
/* It is just allowed to call this function once. */
/* If it is called multiple times, we just return */
if (smi_already_redirected != 0) {
DNOTICE("Output is already redirected");
DSECTLEAVE
return(SMI_SUCCESS);
}
/* init and finalizing routines */
void _smi_sci_desc_init()
{
DSECTION("smi_sci_desc_init");
DSECTENTRYPOINT;
pLocListRoot = NULL;
pLocIntListRoot = NULL;
pRmtListRoot = NULL;
pRmtIntListRoot = NULL;
pIntListRoot = NULL;
pFullListRoot = NULL;
DNOTICE("init mutex");
SMI_INIT_LOCK(&sdMutex);
DSECTLEAVE;
}
/*
| Diese Funktion initialisiert die Speicherverwaltung fuer das |
| uebergegebene Speichersegment. Zunaechst werden die ersten |
| 4 Kbytes fuer die Speicherverwaltung reserviert. Das erste |
| Element enthaelt die Wurzel des Verwaltungsbaumes und das |
| zweite den Kopf der Liste mit den freien Verwaltungselemen- |
| ten. Erst danach wird der Verwaltungsbaum aufgebaut. Der |
| erste belegte Speicherplatz ist die Baumverwaltung selber. |
| Falls die Initialisierung des Speichersegmentes ohne Probleme |
| verlaufen ist, wird der Wert 0 uebergeben, ansonsten ein ent- |
| sprechender Fehlercode. |
*/
int _smi_dynmem_init (void *sgmt_adr, size_t sgmt_sz)
{
DSECTION ("_smi_dynmem_init");
smi_memmgr_t *base_node, *free_list;
size_t tmp;
base_node = (smi_memmgr_t *) sgmt_adr;
/* Fix, then check segment size. The used buddy-technique can only
handle power-of-2 sizes. */
tmp = 1;
while (2*tmp <= sgmt_sz)
tmp <<= 1;
sgmt_sz = tmp;
if (sgmt_sz <= 2*FREELIST_SZ) {
DPROBLEMI ("dynamic memory region is too small: min. size is", 2*FREELIST_SZ);
return 1;
}
/* Create free_list. */
free_list = base_node + 1;
free_list->b_size = 0;
free_list->sb_l = NULL;
free_list = _smi_free_list_init(base_node+2, free_list);
/* Create the base node of the management system. */
base_node->b_size = sgmt_sz;
base_node->b_addr = (char * )base_node;
base_node->sb_l = NULL;
base_node->sb_r = NULL;
base_node->sb_avail = sgmt_sz;
base_node->mem_avail = sgmt_sz;
/* Mark base_node and free_list as allocated. */
_smi_dynmem_memtree(base_node, free_list, FREELIST_SZ);
DNOTICEI("Set up MMU for a region; effective size is", base_node->b_size);
DNOTICEI("Available memory for MMU is", base_node->sb_avail);
return 0;
}
smi_error_t SMI_Get_node_name (char *nodename, size_t *namelen)
{
DSECTION("SMI_Get_node_name");
int iError;
char szHostName[MAXHOSTNAMELEN];
ASSERT_R((nodename != NULL), "illegal parameter for nodename", SMI_ERR_PARAM);
ASSERT_R((namelen != NULL), "illegal parameter for namelen", SMI_ERR_PARAM);
ASSERT_R(((*namelen) > 0), "illegal parameter for *namelen", SMI_ERR_PARAM);
iError = _smi_tcp_init();
ASSERT_R((iError == 0),"could not init tcp", SMI_ERR_OTHER);
iError = gethostname(szHostName, MAXHOSTNAMELEN);
ASSERT_R((iError == 0),"gethostname failed", SMI_ERR_OTHER);
if ( strlen(szHostName) >= (*namelen) ) {
DPROBLEM("insufficient space is provided for hostname");
DSECTLEAVE
return(SMI_ERR_OTHER);
}
int
MPID_USOCK_getopt(int nargc, char * const *nargv, const char *ostr)
{
DSECTION("MPID_USOCK_getopt");
register char *oli; /* option letter list index */
char *p;
DSECTENTRYPOINT;
if (!*place) { /* update scanning pointer */
if (_optind >= nargc || *(place = nargv[_optind]) != '-') {
place = EMSG;
DSECTLEAVE
return(EOF);
}
if (place[1] && *++place == '-') { /* found "--" */
++_optind;
place = EMSG;
DSECTLEAVE
return(EOF);
}
smi_error_t SMI_Iget (void *dst, int src_region_id, int offset, size_t size, smi_memcpy_handle* pHandle)
{
DSECTION("SMI_Iput");
memtype_t mtLocalRegion = _smi_detect_memtype(dst);
memtype_t mtRemoteRegion = _smi_detect_memtype_rdma(src_region_id);
smi_memcpy_handle h;
char *reg_addr;
size_t is_misaligned = 0;
smi_error_t error;
#ifndef DOLPHIN_SISCI
DPROBLEM("DMA only supported by Dolphin's SISCI");
DSECTLEAVE;
return SMI_ERR_NOTIMPL;
#else
if ((mtLocalRegion != mt_lsown) && (mtLocalRegion != mt_ls)) {
DPROBLEM("src must be an local segment");
DSECTLEAVE;
return(SMI_ERR_PARAM);
}
ASSERT_R((mtRemoteRegion == mt_rs),"dest must be an remote segment",SMI_ERR_PARAM);
/* Check alignments. If the operation is peformed on a mapped region (which thus has
an address, we can use SMI_Imemcpy() in case of a misalignemt. */
if ((is_misaligned = size % _smi_dma_size_alignment) != 0) {
DPROBLEMI("misaligned size:", size);
} else {
if ((is_misaligned = (size_t)dst % _smi_dma_offset_alignment) != 0) {
DPROBLEMP("misaligned destination address:", dst);
} else {
if ((is_misaligned = (size_t)offset % _smi_dma_offset_alignment) != 0) {
DPROBLEMI("misaligned source offset:", offset);
}
}
}
if (is_misaligned) {
reg_addr = _smi_get_region_address(src_region_id);
if (reg_addr == NULL) {
/* no way to transfer data */
DPROBLEM ("destination region is not mapped -> no way to work around misalignment");
DSECTLEAVE; return (SMI_ERR_BADALIGN);
} else {
DPROBLEM ("destination region is not mapped -> using SMI_Imemcpy to cope with misalignment");
DSECTLEAVE; return SMI_Imemcpy (dst, reg_addr + offset, size, 0, pHandle);
}
}
_smi_mc_handle_lock();
h = *pHandle;
if (_smi_mc_dma_enabled() == FALSE) {
DNOTICE("DMA-read not implemented by Dolphin`s SISCI");
if (!((h->dma_used==TRUE)&&(h->dma_entries>0)))
_smi_mc_destroy_handle(h);
_smi_mc_handle_unlock();
DSECTLEAVE; return SMI_ERR_NOTIMPL;
}
if ( (h!=NULL) && (h->otid == SMI_MEMCPY_HANDLE_OTID) ) {
if (*(h->pHandle) != *pHandle) {
DPROBLEM("Not a valid handle");
_smi_mc_handle_unlock();
DSECTLEAVE; return SMI_ERR_PARAM;
}
DNOTICE("The Handle already exists!");
if ( (h->dma_used == TRUE) && (h->dma_entries > 0)) {
DNOTICE("The handle contains an nonposted queue");
}
else {
DPROBLEM("The handle is in use");
_smi_mc_handle_unlock();
DSECTLEAVE; return SMI_ERR_PARAM;
}
}
else {
h = _smi_mc_create_handle(pHandle);
if (h == NULL) {
DPROBLEM ("Could not get memory for memcpy-handle");
_smi_mc_handle_unlock();
DSECTLEAVE; return SMI_ERR_NOMEM;
}
h->mc_flags = 0;
}
error = _smi_dma_transfer(src_region_id, offset, dst, size, SCI_FLAG_DMA_READ, h);
_smi_mc_handle_unlock();
DSECTLEAVE; return error;
#endif
}
