void gcm::Geo2MeshLoader::loadMesh(TetrMeshSecondOrder* mesh, GCMDispatcher* dispatcher, string fileName, float tetrSize) { if (!isMshFileCreated(fileName)) createMshFile(fileName, tetrSize); //IBody* body = mesh->getBody(); Engine& engine = Engine::getInstance(); if( engine.getRank() == 0 ) { LOG_DEBUG("Worker 0 started generating second order mesh from first order msh file"); TetrMeshFirstOrder* foMesh = new TetrMeshFirstOrder(); // foMesh->setBody(body); TetrMeshSecondOrder* soMesh = new TetrMeshSecondOrder(); // soMesh->setBody(body); int sd, nn; AABB scene; GCMDispatcher* myDispatcher = new DummyDispatcher(); myDispatcher->setEngine(&Engine::getInstance()); preLoadMesh(&scene, sd, nn, fileName, tetrSize); myDispatcher->prepare(1, &scene); MshTetrFileReader* reader = new MshTetrFileReader(); reader->readFile(getMshFileName(fileName), foMesh, myDispatcher, engine.getRank(), true); soMesh->copyMesh(foMesh); soMesh->preProcess(); VTK2SnapshotWriter* sw = new VTK2SnapshotWriter(); sw->setFileName(getVtkFileName(fileName)); sw->dump(soMesh, -1); delete sw; delete reader; delete foMesh; delete soMesh; LOG_DEBUG("Worker 0 completed generating second order mesh"); } MPI::COMM_WORLD.Barrier(); LOG_DEBUG("Starting reading mesh"); Vtu2TetrFileReader* reader = new Vtu2TetrFileReader(); reader->readFile(getVtkFileName(fileName), mesh, dispatcher, engine.getRank()); delete reader; mesh->preProcess(); }
void Ani3D2MeshLoader::loadMesh(TetrMeshSecondOrder* mesh, GCMDispatcher* dispatcher, const string& fileName) { auto body = mesh->getBody(); auto& engine = Engine::getInstance(); if( engine.getRank() == 0 ) { LOG_DEBUG("Worker 0 started generating second order mesh"); TetrMeshFirstOrder* foMesh = new TetrMeshFirstOrder(); foMesh->setBody(body); TetrMeshSecondOrder* soMesh = new TetrMeshSecondOrder(); soMesh->setBody(body); int sd, nn; AABB scene; GCMDispatcher* myDispatcher = new DummyDispatcher(); preLoadMesh(&scene, sd, nn, fileName); myDispatcher->prepare(1, &scene); Ani3DTetrFileReader* reader = new Ani3DTetrFileReader(); reader->readFile(fileName, foMesh, myDispatcher, engine.getRank(), true); soMesh->copyMesh(foMesh); soMesh->preProcess(); VTK2SnapshotWriter* sw = new VTK2SnapshotWriter(); // FIXME are we really about to overwrite this file? sw->dump(soMesh, -1, getVtkFileName(fileName)); delete sw; delete reader; delete foMesh; delete soMesh; LOG_DEBUG("Worker 0 completed generating second order mesh"); } MPI::COMM_WORLD.Barrier(); LOG_DEBUG("Starting reading mesh"); Vtu2TetrFileReader* reader = new Vtu2TetrFileReader(); reader->readFile(getVtkFileName(fileName), mesh, dispatcher, engine.getRank()); delete reader; LOG_DEBUG("Deleting generated file: " << getVtkFileName(fileName)); remove( getVtkFileName(fileName).c_str() ); mesh->preProcess(); }
void DataBus::syncMissedNodes(Mesh* _mesh, float tau) { if( numberOfWorkers == 1 ) return; bool transferRequired = false; AABB **reqZones = new AABB*[numberOfWorkers]; AABB *reqZones_data = new AABB[numberOfWorkers*numberOfWorkers]; for ( int i = 0; i < numberOfWorkers; ++i ) { reqZones[i] = reqZones_data + (i*numberOfWorkers); } auto& engine = Engine::getInstance(); GCMDispatcher* dispatcher = engine.getDispatcher(); // FIXME@avasyukov - workaround for SphProxyDispatcher // But we still need this if( dispatcher->getOutline(0) == NULL ) { THROW_BAD_METHOD("We can't do this because it will cause all MPI routines to freeze"); return; } BARRIER("DataBus::syncMissedNodes#1"); //Body* body = engine.getBodyById( engine.getDispatcher()->getMyBodyId() ); //TetrMeshSecondOrder* mesh = (TetrMeshSecondOrder*)body->getMeshes(); TetrMeshSecondOrder* mesh = (TetrMeshSecondOrder*) _mesh; AABB* areaOfInterest = &(mesh->areaOfInterest); if( (mesh->syncedArea).includes( areaOfInterest ) ) { LOG_DEBUG("We need nothing"); } else { LOG_DEBUG("Yes, we need additional nodes"); for (int i = 0 ; i < numberOfWorkers; i++) { if( i != rank) { LOG_DEBUG("Our area of interest: " << *areaOfInterest << " Outline[" << i << "]: " << *(dispatcher->getOutline(i))); areaOfInterest->findIntersection(dispatcher->getOutline(i), &reqZones[rank][i]); } } } BARRIER("DataBus::syncMissedNodes#2"); MPI::COMM_WORLD.Allgather( MPI_IN_PLACE, numberOfWorkers, MPI_OUTLINE, reqZones_data, numberOfWorkers, MPI_OUTLINE ); BARRIER("DataBus::syncMissedNodes#3"); vector<AABB> *_reqZones = new vector<AABB>[numberOfWorkers]; for (int i = 0 ; i < numberOfWorkers; i++) for (int j = 0 ; j < numberOfWorkers; j++) { _reqZones[i].push_back(reqZones[i][j]); if( !isinf(reqZones[i][j].minX) ) { transferRequired = true; } } if(transferRequired) { transferNodes(mesh, _reqZones); BARRIER("DataBus::syncMissedNodes#3"); //LOG_DEBUG("Rebuilding data types"); //createDynamicTypes(); LOG_DEBUG("Processing mesh after the sync"); // Overhead for( int z = 0; z < engine.getNumberOfBodies(); z++ ) { TetrMeshSecondOrder* tmpMesh = (TetrMeshSecondOrder*) engine.getBody(z)->getMeshes(); tmpMesh->preProcess(); tmpMesh->checkTopology(tau); } //FIXME@avasyukov - rethink it for( int z = 0; z < 3; z++ ) { (mesh->syncedArea).min_coords[z] = (mesh->areaOfInterest).min_coords[z] - EQUALITY_TOLERANCE; (mesh->syncedArea).max_coords[z] = (mesh->areaOfInterest).max_coords[z] + EQUALITY_TOLERANCE; } } for (int i = 0 ; i < numberOfWorkers; i++) _reqZones[i].clear(); delete[] _reqZones; delete[] reqZones_data; delete[] reqZones; }
void DataBus::createDynamicTypes(int bodyNum) { LOG_DEBUG("Building dynamic MPI types for fast node sync"); auto& engine = Engine::getInstance(); GCMDispatcher* dispatcher = engine.getDispatcher(); Body* body = engine.getBody(bodyNum);//ById( engine.getDispatcher()->getMyBodyId() ); TetrMeshSecondOrder* mesh = (TetrMeshSecondOrder*)body->getMeshes(); // TODO add more cleanup code here to prevent memory leaks if (MPI_NODE_TYPES != NULL) { LOG_TRACE("Cleaning old types"); for (int i = 0; i < numberOfWorkers; i++) { for (int j = 0; j < numberOfWorkers; j++) { LOG_TRACE("Cleaning type " << i << " " << j ); LOG_TRACE("Size " << i << " " << j << " = " << local_numbers[i][j].size()); if (local_numbers[i][j].size() > 0) MPI_NODE_TYPES[i][j].Free(); } } delete[] MPI_NODE_TYPES; } if (local_numbers != NULL) { for (int i = 0; i < numberOfWorkers; i++) delete[] local_numbers[i]; delete[] local_numbers; } // FIXME - it's overhead local_numbers = new vector<int>*[numberOfWorkers]; vector<int> **remote_numbers = new vector<int>*[numberOfWorkers]; MPI_NODE_TYPES = new MPI::Datatype*[numberOfWorkers]; for (int i = 0; i < numberOfWorkers; i++) { local_numbers[i] = new vector<int>[numberOfWorkers]; remote_numbers[i] = new vector<int>[numberOfWorkers]; MPI_NODE_TYPES[i] = new MPI::Datatype[numberOfWorkers]; } BARRIER("DataBus::createDynamicTypes#0"); // find all remote nodes for (int j = 0; j < mesh->getNodesNumber(); j++) { CalcNode& node = mesh->getNodeByLocalIndex(j); if ( node.isRemote() ) { //LOG_DEBUG("N: " << j); //LOG_DEBUG("R1: " << j << " " << mesh->getBody()->getId()); int owner = dispatcher->getOwner(node.coords/*, mesh->getBody()->getId()*/); //LOG_DEBUG("R2: " << owner); assert_ne(owner, rank ); local_numbers[rank][owner].push_back( mesh->nodesMap[node.number] ); remote_numbers[rank][owner].push_back(node.number); } } BARRIER("DataBus::createDynamicTypes#1"); LOG_DEBUG("Requests prepared:"); for (int i = 0; i < numberOfWorkers; i++) for (int j = 0; j < numberOfWorkers; j++) LOG_DEBUG("Request size from #" << i << " to #" << j << ": " << local_numbers[i][j].size()); // sync types unsigned int max_len = 0; for (int i = 0; i < numberOfWorkers; i++) for (int j = 0; j < numberOfWorkers; j++) if (local_numbers[i][j].size() > max_len) max_len = local_numbers[i][j].size(); vector<int> lengths; for (unsigned int i = 0; i < max_len; i++) lengths.push_back(1); int info[3]; vector<MPI::Request> reqs; for (int i = 0; i < numberOfWorkers; i++) for (int j = 0; j < numberOfWorkers; j++) if (local_numbers[i][j].size() > 0) { info[0] = remote_numbers[i][j].size(); info[1] = i; info[2] = j; MPI_NODE_TYPES[i][j] = MPI_ELNODE.Create_indexed( local_numbers[i][j].size(), &lengths[0], &local_numbers[i][j][0] ); MPI_NODE_TYPES[i][j].Commit(); reqs.push_back( MPI::COMM_WORLD.Isend( &remote_numbers[i][j][0], remote_numbers[i][j].size(), MPI::INT, j, TAG_SYNC_NODE_TYPES ) ); reqs.push_back( MPI::COMM_WORLD.Isend( info, 3, MPI::INT, j, TAG_SYNC_NODE_TYPES_I ) ); } BARRIER("DataBus::createDynamicTypes#2"); MPI::Status status; while (MPI::COMM_WORLD.Iprobe(MPI::ANY_SOURCE, TAG_SYNC_NODE_TYPES_I, status)) { MPI::COMM_WORLD.Recv( info, 3, MPI::INT, status.Get_source(), TAG_SYNC_NODE_TYPES_I ); local_numbers[info[1]][info[2]].resize(info[0]); MPI::COMM_WORLD.Recv( &local_numbers[info[1]][info[2]][0], info[0], MPI::INT, status.Get_source(), TAG_SYNC_NODE_TYPES ); if (lengths.size() < (unsigned)info[0]) for (int i = lengths.size(); i < info[0]; i++) lengths.push_back(1); for(int i = 0; i < info[0]; i++) local_numbers[info[1]][info[2]][i] = mesh->nodesMap[ local_numbers[info[1]][info[2]][i] ]; MPI_NODE_TYPES[info[1]][info[2]] = MPI_ELNODE.Create_indexed( info[0], &lengths[0], &local_numbers[info[1]][info[2]][0] ); MPI_NODE_TYPES[info[1]][info[2]].Commit(); } MPI::Request::Waitall(reqs.size(), &reqs[0]); BARRIER("DataBus::createDynamicTypes#3"); for (int i = 0 ; i < numberOfWorkers; i++) delete[] remote_numbers[i]; delete[] remote_numbers; LOG_DEBUG("Building dynamic MPI types for fast node sync done"); }