void DictionarySuite_SetupTestDictData( DictionarySuite_TestDictData* testDD ) { Index ii=0; Index iter=0; Dictionary_Entry_Value* testStruct; Dictionary_Entry_Value* testStruct2; Dictionary_Entry_Value* testList; testDD->testEntriesCount = 9; testDD->testKeys = Memory_Alloc_Array_Unnamed( char*, testDD->testEntriesCount ); testDD->testValues = Memory_Alloc_Array_Unnamed( Dictionary_Entry_Value*, testDD->testEntriesCount ); for ( ii=0; ii< testDD->testEntriesCount; ii++ ) { testDD->testKeys[ii] = NULL; testDD->testValues[ii] = NULL; } Stg_asprintf( &testDD->testString, "hello" ); Stg_asprintf( &testDD->testPlaceHolder, "test_double" ); testDD->testDouble=45.567; testDD->testUint = 5; testDD->testInt = -5; testDD->testUnsignedlong = 52342423; testDD->testBool = True; iter = 0; Stg_asprintf( &testDD->testKeys[iter], "test_cstring" ); testDD->testValues[iter] = Dictionary_Entry_Value_FromString( testDD->testString ); Stg_asprintf( &testDD->testKeys[++iter], "test_double" ); testDD->testValues[iter] = Dictionary_Entry_Value_FromDouble( testDD->testDouble ); Stg_asprintf( &testDD->testKeys[++iter], "test_uint" ); testDD->testValues[iter] = Dictionary_Entry_Value_FromUnsignedInt( testDD->testUint ); Stg_asprintf( &testDD->testKeys[++iter], "test_int" ); testDD->testValues[iter] = Dictionary_Entry_Value_FromInt( testDD->testInt ); Stg_asprintf( &testDD->testKeys[++iter], "test_unsignedlong" ); testDD->testValues[iter] = Dictionary_Entry_Value_FromUnsignedLong( testDD->testUnsignedlong ); Stg_asprintf( &testDD->testKeys[++iter], "test_bool" ); testDD->testValues[iter] = Dictionary_Entry_Value_FromUnsignedInt( testDD->testBool ); Stg_asprintf( &testDD->testKeys[++iter], "test_placeholder" ); testDD->testValues[iter] = Dictionary_Entry_Value_FromString( testDD->testPlaceHolder ); /* adding a list */ testDD->testListCount = 5; testDD->testList = Memory_Alloc_Array_Unnamed( double, testDD->testListCount ); for (ii=0; ii<testDD->testListCount; ii++ ) { testDD->testList[ii] = 10.0 * ii; } testList = Dictionary_Entry_Value_NewList(); Stg_asprintf( &testDD->testKeys[++iter], "test_list" ); testDD->testValues[iter] = testList; for (ii=0; ii < testDD->testListCount; ii++ ) { Dictionary_Entry_Value_AddElement( testList, Dictionary_Entry_Value_FromDouble(testDD->testList[ii]) ); } /* Adding members to a struct */ testDD->testStruct = Memory_Alloc_Unnamed( TestStruct ); testDD->testStruct->height = 37; testDD->testStruct->anisotropic = True; Stg_asprintf( &testDD->testStruct->person, "Patrick" ); testDD->testStruct->geom.startx = 45; testDD->testStruct->geom.starty = 60; testDD->testStruct->geom.startz = 70; testStruct = Dictionary_Entry_Value_NewStruct(); Stg_asprintf( &testDD->testKeys[++iter], "test_struct" ); testDD->testValues[iter] = testStruct; Dictionary_Entry_Value_AddMember( testStruct, (Dictionary_Entry_Key)"height", Dictionary_Entry_Value_FromDouble( testDD->testStruct->height ) ); Dictionary_Entry_Value_AddMember( testStruct, (Dictionary_Entry_Key)"anisotropic", Dictionary_Entry_Value_FromBool( testDD->testStruct->anisotropic ) ); Dictionary_Entry_Value_AddMember( testStruct, (Dictionary_Entry_Key)"person", Dictionary_Entry_Value_FromString( testDD->testStruct->person ) ); /* Adding a 2nd struct within the first struct */ testStruct2 = Dictionary_Entry_Value_NewStruct( ); Dictionary_Entry_Value_AddMember( testStruct, (Dictionary_Entry_Key)"geom", testStruct2 ); Dictionary_Entry_Value_AddMember( testStruct2, (Dictionary_Entry_Key)"startx", Dictionary_Entry_Value_FromUnsignedInt( testDD->testStruct->geom.startx ) ); Dictionary_Entry_Value_AddMember( testStruct2, (Dictionary_Entry_Key)"starty", Dictionary_Entry_Value_FromUnsignedInt( testDD->testStruct->geom.starty ) ); Dictionary_Entry_Value_AddMember( testStruct2, (Dictionary_Entry_Key)"startz", Dictionary_Entry_Value_FromUnsignedInt( testDD->testStruct->geom.startz ) ); }
void ElementCellLayout_BuildShadowInfo( ElementCellLayout* self ) { unsigned nDims; Comm* comm; int nIncProcs; int* incProcs; unsigned n_i; nDims = Mesh_GetDimSize( self->mesh ); comm = Mesh_GetCommTopology( self->mesh, nDims ); Comm_GetNeighbours( comm, &nIncProcs, &incProcs ); /* Extract neighbouring proc information. */ self->cellShadowInfo.procNbrInfo = Memory_Alloc_Unnamed( ProcNbrInfo ); self->cellShadowInfo.procNbrInfo->procNbrCnt = nIncProcs; self->cellShadowInfo.procNbrInfo->procNbrTbl = AllocArray( unsigned, nIncProcs ); memcpy( self->cellShadowInfo.procNbrInfo->procNbrTbl, incProcs, nIncProcs * sizeof(unsigned) ); /* Count shadow info. */ if( nIncProcs ) { self->cellShadowInfo.procShadowedCnt = AllocArray( unsigned, nIncProcs ); memset( self->cellShadowInfo.procShadowedCnt, 0, nIncProcs * sizeof(unsigned) ); self->cellShadowInfo.procShadowCnt = AllocArray( unsigned, nIncProcs ); memset( self->cellShadowInfo.procShadowCnt, 0, nIncProcs * sizeof(unsigned) ); } for( n_i = 0; n_i < Mesh_GetSharedSize( self->mesh, nDims ); n_i++ ) { unsigned nSharers; unsigned* sharers; unsigned s_i; Mesh_GetSharers( self->mesh, nDims, n_i, &nSharers, &sharers ); for( s_i = 0; s_i < nSharers; s_i++ ) self->cellShadowInfo.procShadowedCnt[sharers[s_i]]++; } for( n_i = 0; n_i < Mesh_GetRemoteSize( self->mesh, nDims ); n_i++ ) { unsigned owner; owner = Mesh_GetOwner( self->mesh, nDims, n_i ); self->cellShadowInfo.procShadowCnt[owner]++; } /* Build shadow info indices. */ if( nIncProcs ) { self->cellShadowInfo.procShadowedTbl = Memory_Alloc_2DComplex_Unnamed( unsigned, nIncProcs, self->cellShadowInfo.procShadowedCnt ); self->cellShadowInfo.procShadowTbl = Memory_Alloc_2DComplex_Unnamed( unsigned, nIncProcs, self->cellShadowInfo.procShadowCnt ); memset( self->cellShadowInfo.procShadowedCnt, 0, nIncProcs * sizeof(unsigned) ); memset( self->cellShadowInfo.procShadowCnt, 0, nIncProcs * sizeof(unsigned) ); } for( n_i = 0; n_i < Mesh_GetSharedSize( self->mesh, nDims ); n_i++ ) { unsigned local; unsigned curInd; unsigned nSharers; unsigned* sharers; unsigned s_i; local = Mesh_SharedToLocal( self->mesh, nDims, n_i ); Mesh_GetSharers( self->mesh, nDims, n_i, &nSharers, &sharers ); for( s_i = 0; s_i < nSharers; s_i++ ) { curInd = self->cellShadowInfo.procShadowedCnt[sharers[s_i]]++; self->cellShadowInfo.procShadowedTbl[sharers[s_i]][curInd] = local; } } for( n_i = 0; n_i < Mesh_GetRemoteSize( self->mesh, nDims ); n_i++ ) { unsigned domain; unsigned curInd; unsigned owner; domain = Mesh_GetLocalSize( self->mesh, nDims ) + n_i; owner = Mesh_GetOwner( self->mesh, nDims, n_i ); curInd = self->cellShadowInfo.procShadowCnt[owner]++; self->cellShadowInfo.procShadowTbl[owner][curInd] = domain; } }
void DictionarySuite_Setup( DictionarySuiteData* data ) { data->dict = Dictionary_New(); data->testDD = Memory_Alloc_Unnamed( DictionarySuite_TestDictData ); DictionarySuite_SetupTestDictData( data->testDD ); }
void _SolutionVector_ShareValuesNotStoredLocally( SolutionVector* self, Index* reqFromOthersCounts, RequestInfo** reqFromOthersInfos, Dof_EquationNumber** reqFromOthers, double* localSolnVecValues ) { FeVariable* feVar = self->feVariable; FeMesh* feMesh = feVar->feMesh; FeEquationNumber* eqNum = self->eqNum; Comm* comm; MPI_Comm mpiComm; Partition_Index nProc; Partition_Index myRank; Partition_Index proc_I; Index req_I; Index indexIntoLocalSolnVecValues; MPI_Status status; Index* reqFromMeCounts; Dof_EquationNumber** reqFromMe; double** reqValuesFromMe; MPI_Request** reqValuesFromMeHandles; MPI_Request** reqFromOthersHandles; double** reqValuesFromOthers; MPI_Request** reqValuesFromOthersHandles; Bool* reqValuesFromOthersReceived; Partition_Index reqValueSetsFromOthersNotYetReceivedCount; Dof_EquationNumber totalRequestedFromOthers = 0; Dof_EquationNumber totalRequestedFromMe = 0; int ierr; Journal_DPrintf( self->debug, "In %s - for \"%s\"\n", __func__, self->name ); Stream_IndentBranch( StgFEM_Debug ); comm = Mesh_GetCommTopology( feMesh, MT_VERTEX ); mpiComm = Comm_GetMPIComm( comm ); MPI_Comm_size( mpiComm, (int*)&nProc ); MPI_Comm_rank( mpiComm, (int*)&myRank ); reqFromMeCounts = Memory_Alloc_Array( Index, nProc, "reqFromMeCounts" ); reqFromOthersHandles = Memory_Alloc_Array_Unnamed( MPI_Request*, nProc ); reqValuesFromOthersHandles = Memory_Alloc_Array_Unnamed( MPI_Request*, nProc ); reqValuesFromMeHandles = Memory_Alloc_Array_Unnamed( MPI_Request*, nProc ); reqValuesFromOthers = Memory_Alloc_2DComplex( double, nProc, reqFromOthersCounts, "reqValuesFromOthers" ); reqValuesFromOthersReceived = Memory_Alloc_Array_Unnamed( Bool, nProc ); #if DEBUG if ( Stream_IsPrintableLevel( self->debug, 2 ) ) { Journal_DPrintf( self->debug, "Final list of vec values I need from other procs:\n" ); for ( proc_I=0; proc_I < nProc; proc_I++ ) { if ( proc_I == myRank ) continue; Journal_DPrintf( self->debug, "\t%d[0-%d]: ", proc_I, reqFromOthersCounts[proc_I] ); for ( req_I=0; req_I < reqFromOthersCounts[proc_I]; req_I++ ) { RequestInfo* reqInfo = &reqFromOthersInfos[proc_I][req_I]; Journal_DPrintf( self->debug, "(lnode %d, dof %d -> %d ), ", reqInfo->lNode_I, reqInfo->nodeLocalDof_I, reqFromOthers[proc_I][req_I] ); } Journal_DPrintf( self->debug, "\n" ); } } #endif /* send out my request counts, receive the req. counts others want from me */ MPI_Alltoall( reqFromOthersCounts, 1, MPI_UNSIGNED, reqFromMeCounts, 1, MPI_UNSIGNED, mpiComm ); Journal_DPrintf( self->debug, "After MPI_Alltoall- counts are:\n" ); totalRequestedFromOthers = 0; totalRequestedFromMe = 0; Stream_Indent( self->debug ); Journal_DPrintf( self->debug, "reqFromOthersCounts: " ); for ( proc_I=0; proc_I < nProc; proc_I++ ) { if ( proc_I == myRank ) continue; Journal_DPrintf( self->debug, "\tp%d:%d, ", proc_I, reqFromOthersCounts[proc_I] ); totalRequestedFromOthers += reqFromOthersCounts[proc_I]; } Journal_DPrintf( self->debug, "\n" ); Journal_DPrintf( self->debug, "reqFromMeCounts: " ); for ( proc_I=0; proc_I < nProc; proc_I++ ) { if ( proc_I == myRank ) continue; Journal_DPrintf( self->debug, "\tp%d:%d, ", proc_I, reqFromMeCounts[proc_I] ); totalRequestedFromMe += reqFromMeCounts[proc_I]; } Journal_DPrintf( self->debug, "\n" ); Stream_UnIndent( self->debug ); if ( ( totalRequestedFromOthers == 0) && (totalRequestedFromMe == 0) ) { Journal_DPrintf( self->debug, "No vector values either required from others or " "required by others from me, therefore cleaning up memory and returning.\n" ); Memory_Free( reqFromMeCounts ); Memory_Free( reqFromOthersHandles ); Memory_Free( reqValuesFromOthersHandles ); Memory_Free( reqValuesFromMeHandles ); Memory_Free( reqValuesFromOthers ); Memory_Free( reqValuesFromOthersReceived ); Stream_UnIndentBranch( StgFEM_Debug ); return; } Journal_DPrintfL( self->debug, 2, "Starting non-blocking sends of my lists of vector entry indices I want from others:\n" ); Stream_Indent( self->debug ); for( proc_I=0; proc_I < nProc; proc_I++) { if ( proc_I == myRank ) continue; /* Journal_Printf( Journal_Register( Info_Type, (Name)"mpi" ), "!!! line %d, proc_I %d: count = %u\n", __LINE__, proc_I, reqFromOthersCounts[proc_I] ); */ if ( reqFromOthersCounts[proc_I] > 0 ) { Journal_DPrintfL( self->debug, 2, "Sending to proc %d the list of %d vector entry indices I want from it:\n" "\t(tracking via reqFromOthersHandles[%d], tag %d)\n", proc_I, reqFromOthersCounts[proc_I], proc_I, VALUE_REQUEST_TAG ); reqFromOthersHandles[proc_I] = Memory_Alloc_Unnamed( MPI_Request ); ierr=MPI_Isend( reqFromOthers[proc_I], reqFromOthersCounts[proc_I], MPI_UNSIGNED, proc_I, VALUE_REQUEST_TAG, mpiComm, reqFromOthersHandles[proc_I] ); } } Stream_UnIndent( self->debug ); Journal_DPrintfL( self->debug, 2, "Starting non-blocking receive of the vector entries I want from others:\n" ); Stream_Indent( self->debug ); for( proc_I=0; proc_I < nProc; proc_I++) { if ( proc_I == myRank ) continue; if ( reqFromOthersCounts[proc_I] > 0 ) { Journal_DPrintfL( self->debug, 2, "Posting recv reqst from proc %d for the %d vector entries I want from it:\n" "\t(tracking via reqValuesFromOthersHandles[%d], tag %d)\n", proc_I, reqFromOthersCounts[proc_I], proc_I, VALUE_TAG ); reqValuesFromOthersHandles[proc_I] = Memory_Alloc_Unnamed( MPI_Request ); ierr=MPI_Irecv( reqValuesFromOthers[proc_I], reqFromOthersCounts[proc_I], MPI_DOUBLE, proc_I, VALUE_TAG, mpiComm, reqValuesFromOthersHandles[proc_I] ); } } Stream_UnIndent( self->debug ); Journal_DPrintfL( self->debug, 2, "Starting blocking receive of the lists of vector entry indices " "others want from me:\n" ); Stream_Indent( self->debug ); reqFromMe = Memory_Alloc_2DComplex( Dof_EquationNumber, nProc, reqFromMeCounts, "reqFromMe" ); reqValuesFromMe = Memory_Alloc_2DComplex( double, nProc, reqFromMeCounts, "reqValuesFromMe" ); for( proc_I=0; proc_I < nProc; proc_I++) { if ( proc_I == myRank ) continue; /* /Journal_Printf( Journal_Register( Info_Type, (Name)"mpi" ), "!!! line %d, proc_I %d: count = %u\n", __LINE__, proc_I, reqFromMeCounts[proc_I] ); */ if ( reqFromMeCounts[proc_I] > 0 ) { ierr=MPI_Recv( reqFromMe[proc_I], reqFromMeCounts[proc_I], MPI_UNSIGNED, proc_I, VALUE_REQUEST_TAG, mpiComm, &status ); Journal_DPrintfL( self->debug, 3, "Received a list of %u requested vector entry indices from proc %u, " "with tag %d\n", reqFromMeCounts[proc_I], proc_I, status.MPI_TAG ); } } Stream_UnIndent( self->debug ); #if DEBUG if ( Stream_IsPrintableLevel( self->debug, 2 ) ) { Journal_DPrintf( self->debug, "Final lists of vector entry indices other procs want from me are:\n" ); Stream_Indent( self->debug ); for ( proc_I=0; proc_I < nProc; proc_I++ ) { if ( proc_I == myRank ) continue; if ( reqFromMeCounts[proc_I] > 0 ) { Journal_DPrintf( self->debug, "%d[0-%d]: ", proc_I, reqFromMeCounts[proc_I] ); for ( req_I=0; req_I < reqFromMeCounts[proc_I]; req_I++ ) { Journal_DPrintf( self->debug, "(eqNum %d), ", reqFromMe[proc_I][req_I] ); } Journal_DPrintf( self->debug, "\n" ); } } Stream_UnIndent( self->debug ); } #endif /* for all those requested from me, non-blocking send out values */ Journal_DPrintfL( self->debug, 2, "Beginning non-blocking send out of vector entry lists requested by others:\n" ); Stream_Indent( self->debug ); for( proc_I=0; proc_I < nProc; proc_I++) { if ( proc_I == myRank ) continue; if ( reqFromMeCounts[proc_I] > 0 ) { Journal_DPrintfL( self->debug, 3, "list to proc %d is: ", proc_I ); for ( req_I=0; req_I < reqFromMeCounts[proc_I]; req_I++ ) { /* look up and fill in correct value in array */ indexIntoLocalSolnVecValues = *(int*)STreeMap_Map( eqNum->ownedMap, reqFromMe[proc_I] + req_I ); reqValuesFromMe[proc_I][req_I] = localSolnVecValues[indexIntoLocalSolnVecValues]; Journal_DPrintfL( self->debug, 3, "%d=%f, ", reqFromMe[proc_I][req_I], reqValuesFromMe[proc_I][req_I] ); } Journal_DPrintfL( self->debug, 3, "\n" ); /* Non-blocking send out the now-complete list to this processor */ reqValuesFromMeHandles[proc_I] = Memory_Alloc_Unnamed( MPI_Request ); Journal_DPrintfL( self->debug, 2, "Sending to proc %d the list of %d vector entries they want:\n" "\t(tracking via reqValuesFromMe[%d], tag %d)\n", proc_I, reqFromMeCounts[proc_I], proc_I, VALUE_TAG ); ierr=MPI_Isend( reqValuesFromMe[proc_I], reqFromMeCounts[proc_I], MPI_DOUBLE, proc_I, VALUE_TAG, mpiComm, reqValuesFromMeHandles[proc_I] ); } } Stream_UnIndent( self->debug ); Journal_DPrintfL( self->debug, 1, "Starting iterative-test receive of the vector entries I " "requested from others:\n" ); /* Set up an array for keeping track of who we've received things from * already */ reqValueSetsFromOthersNotYetReceivedCount = nProc-1; for( proc_I=0; proc_I < nProc; proc_I++) { if ( proc_I == myRank ) continue; reqValuesFromOthersReceived[proc_I] = False; if ( reqFromOthersCounts[proc_I] == 0 ) { reqValueSetsFromOthersNotYetReceivedCount--; } } #if DEBUG Journal_DPrintfL( self->debug, 2, "(Expecting %d receives from procs: ", reqValueSetsFromOthersNotYetReceivedCount ); for( proc_I=0; proc_I < nProc; proc_I++) { if ( proc_I == myRank ) continue; if ( reqFromOthersCounts[proc_I] > 0 ) { Journal_DPrintfL( self->debug, 2, "%d, ", proc_I ); } } Journal_DPrintfL( self->debug, 2, ")\n" ); #endif Stream_Indent( self->debug ); /* now update the values at nodes that I requested from others, as they come in */ while ( reqValueSetsFromOthersNotYetReceivedCount ) { int flag = 0; Journal_DPrintfL( self->debug, 3, "%d sets still to go...\n", reqValueSetsFromOthersNotYetReceivedCount ); for( proc_I=0; proc_I < nProc; proc_I++) { if ( proc_I == myRank ) continue; if ( (reqFromOthersCounts[proc_I] > 0) && (False == reqValuesFromOthersReceived[proc_I]) ) { MPI_Test( reqValuesFromOthersHandles[proc_I], &flag, &status ); if ( !flag ) { /* No results yet from this proc -> continue to next. */ continue; } else { RequestInfo* reqInfo; Journal_DPrintfL( self->debug, 2, "received some requested " "values (using reqValuesFromOthersHandles) from proc %d " "(with tag %d, exp %d):", proc_I, status.MPI_TAG, VALUE_TAG ); /* go through each value received from that proc & update onto node */ for ( req_I=0; req_I < reqFromOthersCounts[proc_I]; req_I++ ) { reqInfo = &reqFromOthersInfos[proc_I][req_I]; Journal_DPrintfL( self->debug, 3, "(lnode %d, dof %d -> %d )=%f, ", reqInfo->lNode_I, reqInfo->nodeLocalDof_I, reqFromOthers[proc_I][req_I], reqValuesFromOthers[proc_I][req_I] ); DofLayout_SetValueDouble( feVar->dofLayout, reqInfo->lNode_I, reqInfo->nodeLocalDof_I, reqValuesFromOthers[proc_I][req_I] ); } Journal_DPrintfL( self->debug, 2, "\n" ); reqValuesFromOthersReceived[proc_I] = True; reqValueSetsFromOthersNotYetReceivedCount--; Memory_Free( reqValuesFromOthersHandles[proc_I] ); } } } } Stream_UnIndent( self->debug ); /* MPI_Wait to be sure all sends to others have completed */ Journal_DPrintfL( self->debug, 2, "Making sure all comms of this function finished:...\n" ); Stream_Indent( self->debug ); Journal_DPrintfL( self->debug, 2, "Confirming completion of my sends of " "vector entry index lists I wanted from others were received:\n" ); Stream_Indent( self->debug ); for( proc_I=0; proc_I < nProc; proc_I++) { if ( proc_I == myRank ) continue; if ( reqFromOthersCounts[proc_I] > 0 ) { ierr=MPI_Wait( reqFromOthersHandles[proc_I], MPI_STATUS_IGNORE ); Journal_DPrintfL( self->debug, 2, "Confirmed wait on reqFromOthersHandles[%u]" "\n", proc_I ); Memory_Free( reqFromOthersHandles[proc_I] ); } } Stream_UnIndent( self->debug ); Journal_DPrintfL( self->debug, 2, "done.\n" ); Journal_DPrintfL( self->debug, 2, "Confirming completion of my sends of " "vector entry values requested by others were received:\n" ); Stream_Indent( self->debug ); for( proc_I=0; proc_I < nProc; proc_I++) { if ( proc_I == myRank ) continue; if ( reqFromMeCounts[proc_I] > 0 ) { ierr=MPI_Wait( reqValuesFromMeHandles[proc_I], MPI_STATUS_IGNORE ); Journal_DPrintfL( self->debug, 2, "Confirmed wait on reqValuesFromMeHandles[%u]" "\n", proc_I ); Memory_Free( reqValuesFromMeHandles[proc_I] ); } } Stream_UnIndent( self->debug ); Journal_DPrintfL( self->debug, 2, "done.\n" ); Stream_UnIndent( self->debug ); Journal_DPrintfL( self->debug, 2, "done.\n" ); Memory_Free( reqFromMeCounts ); Memory_Free( reqFromMe ); Memory_Free( reqValuesFromMe ); Memory_Free( reqValuesFromOthers ); Memory_Free( reqValuesFromOthersReceived ); Memory_Free( reqFromOthersHandles ); Memory_Free( reqValuesFromOthersHandles ); Memory_Free( reqValuesFromMeHandles ); Stream_UnIndentBranch( StgFEM_Debug ); return; }