void gcm::Geo2MeshLoader::loadMesh(TetrMeshSecondOrder* mesh, GCMDispatcher* dispatcher, string fileName, float tetrSize)
{
if (!isMshFileCreated(fileName))
createMshFile(fileName, tetrSize);
//IBody* body = mesh->getBody();
Engine& engine = Engine::getInstance();
if( engine.getRank() == 0 )
{
LOG_DEBUG("Worker 0 started generating second order mesh from first order msh file");
TetrMeshFirstOrder* foMesh = new TetrMeshFirstOrder();
// foMesh->setBody(body);
TetrMeshSecondOrder* soMesh = new TetrMeshSecondOrder();
// soMesh->setBody(body);
int sd, nn;
AABB scene;
GCMDispatcher* myDispatcher = new DummyDispatcher();
myDispatcher->setEngine(&Engine::getInstance());
preLoadMesh(&scene, sd, nn, fileName, tetrSize);
myDispatcher->prepare(1, &scene);
MshTetrFileReader* reader = new MshTetrFileReader();
reader->readFile(getMshFileName(fileName), foMesh, myDispatcher, engine.getRank(), true);
soMesh->copyMesh(foMesh);
soMesh->preProcess();
VTK2SnapshotWriter* sw = new VTK2SnapshotWriter();
sw->setFileName(getVtkFileName(fileName));
sw->dump(soMesh, -1);
delete sw;
delete reader;
delete foMesh;
delete soMesh;
LOG_DEBUG("Worker 0 completed generating second order mesh");
}
MPI::COMM_WORLD.Barrier();
LOG_DEBUG("Starting reading mesh");
Vtu2TetrFileReader* reader = new Vtu2TetrFileReader();
reader->readFile(getVtkFileName(fileName), mesh, dispatcher, engine.getRank());
delete reader;
mesh->preProcess();
}
void Ani3D2MeshLoader::loadMesh(TetrMeshSecondOrder* mesh, GCMDispatcher* dispatcher, const string& fileName)
{
auto body = mesh->getBody();
auto& engine = Engine::getInstance();
if( engine.getRank() == 0 )
{
LOG_DEBUG("Worker 0 started generating second order mesh");
TetrMeshFirstOrder* foMesh = new TetrMeshFirstOrder();
foMesh->setBody(body);
TetrMeshSecondOrder* soMesh = new TetrMeshSecondOrder();
soMesh->setBody(body);
int sd, nn;
AABB scene;
GCMDispatcher* myDispatcher = new DummyDispatcher();
preLoadMesh(&scene, sd, nn, fileName);
myDispatcher->prepare(1, &scene);
Ani3DTetrFileReader* reader = new Ani3DTetrFileReader();
reader->readFile(fileName,
foMesh, myDispatcher, engine.getRank(), true);
soMesh->copyMesh(foMesh);
soMesh->preProcess();
VTK2SnapshotWriter* sw = new VTK2SnapshotWriter();
// FIXME are we really about to overwrite this file?
sw->dump(soMesh, -1, getVtkFileName(fileName));
delete sw;
delete reader;
delete foMesh;
delete soMesh;
LOG_DEBUG("Worker 0 completed generating second order mesh");
}
MPI::COMM_WORLD.Barrier();
LOG_DEBUG("Starting reading mesh");
Vtu2TetrFileReader* reader = new Vtu2TetrFileReader();
reader->readFile(getVtkFileName(fileName), mesh, dispatcher, engine.getRank());
delete reader;
LOG_DEBUG("Deleting generated file: " << getVtkFileName(fileName));
remove( getVtkFileName(fileName).c_str() );
mesh->preProcess();
}
void DataBus::syncMissedNodes(Mesh* _mesh, float tau)
{
if( numberOfWorkers == 1 )
return;
bool transferRequired = false;
AABB **reqZones = new AABB*[numberOfWorkers];
AABB *reqZones_data = new AABB[numberOfWorkers*numberOfWorkers];
for ( int i = 0; i < numberOfWorkers; ++i ) {
reqZones[i] = reqZones_data + (i*numberOfWorkers);
}
auto& engine = Engine::getInstance();
GCMDispatcher* dispatcher = engine.getDispatcher();
// FIXME@avasyukov - workaround for SphProxyDispatcher
// But we still need this
if( dispatcher->getOutline(0) == NULL ) {
THROW_BAD_METHOD("We can't do this because it will cause all MPI routines to freeze");
return;
}
BARRIER("DataBus::syncMissedNodes#1");
//Body* body = engine.getBodyById( engine.getDispatcher()->getMyBodyId() );
//TetrMeshSecondOrder* mesh = (TetrMeshSecondOrder*)body->getMeshes();
TetrMeshSecondOrder* mesh = (TetrMeshSecondOrder*) _mesh;
AABB* areaOfInterest = &(mesh->areaOfInterest);
if( (mesh->syncedArea).includes( areaOfInterest ) )
{
LOG_DEBUG("We need nothing");
}
else
{
LOG_DEBUG("Yes, we need additional nodes");
for (int i = 0 ; i < numberOfWorkers; i++)
{
if( i != rank)
{
LOG_DEBUG("Our area of interest: " << *areaOfInterest << " Outline[" << i << "]: " << *(dispatcher->getOutline(i)));
areaOfInterest->findIntersection(dispatcher->getOutline(i), &reqZones[rank][i]);
}
}
}
BARRIER("DataBus::syncMissedNodes#2");
MPI::COMM_WORLD.Allgather(
MPI_IN_PLACE,
numberOfWorkers, MPI_OUTLINE,
reqZones_data,
numberOfWorkers, MPI_OUTLINE
);
BARRIER("DataBus::syncMissedNodes#3");
vector<AABB> *_reqZones = new vector<AABB>[numberOfWorkers];
for (int i = 0 ; i < numberOfWorkers; i++)
for (int j = 0 ; j < numberOfWorkers; j++)
{
_reqZones[i].push_back(reqZones[i][j]);
if( !isinf(reqZones[i][j].minX) )
{
transferRequired = true;
}
}
if(transferRequired)
{
transferNodes(mesh, _reqZones);
BARRIER("DataBus::syncMissedNodes#3");
//LOG_DEBUG("Rebuilding data types");
//createDynamicTypes();
LOG_DEBUG("Processing mesh after the sync");
// Overhead
for( int z = 0; z < engine.getNumberOfBodies(); z++ )
{
TetrMeshSecondOrder* tmpMesh = (TetrMeshSecondOrder*) engine.getBody(z)->getMeshes();
tmpMesh->preProcess();
tmpMesh->checkTopology(tau);
}
//FIXME@avasyukov - rethink it
for( int z = 0; z < 3; z++ )
{
(mesh->syncedArea).min_coords[z] = (mesh->areaOfInterest).min_coords[z] - EQUALITY_TOLERANCE;
(mesh->syncedArea).max_coords[z] = (mesh->areaOfInterest).max_coords[z] + EQUALITY_TOLERANCE;
}
}
for (int i = 0 ; i < numberOfWorkers; i++)
_reqZones[i].clear();
delete[] _reqZones;
delete[] reqZones_data;
delete[] reqZones;
}
void DataBus::createDynamicTypes(int bodyNum)
{
LOG_DEBUG("Building dynamic MPI types for fast node sync");
auto& engine = Engine::getInstance();
GCMDispatcher* dispatcher = engine.getDispatcher();
Body* body = engine.getBody(bodyNum);//ById( engine.getDispatcher()->getMyBodyId() );
TetrMeshSecondOrder* mesh = (TetrMeshSecondOrder*)body->getMeshes();
// TODO add more cleanup code here to prevent memory leaks
if (MPI_NODE_TYPES != NULL) {
LOG_TRACE("Cleaning old types");
for (int i = 0; i < numberOfWorkers; i++)
{
for (int j = 0; j < numberOfWorkers; j++)
{
LOG_TRACE("Cleaning type " << i << " " << j );
LOG_TRACE("Size " << i << " " << j << " = " << local_numbers[i][j].size());
if (local_numbers[i][j].size() > 0)
MPI_NODE_TYPES[i][j].Free();
}
}
delete[] MPI_NODE_TYPES;
}
if (local_numbers != NULL) {
for (int i = 0; i < numberOfWorkers; i++)
delete[] local_numbers[i];
delete[] local_numbers;
}
// FIXME - it's overhead
local_numbers = new vector<int>*[numberOfWorkers];
vector<int> **remote_numbers = new vector<int>*[numberOfWorkers];
MPI_NODE_TYPES = new MPI::Datatype*[numberOfWorkers];
for (int i = 0; i < numberOfWorkers; i++)
{
local_numbers[i] = new vector<int>[numberOfWorkers];
remote_numbers[i] = new vector<int>[numberOfWorkers];
MPI_NODE_TYPES[i] = new MPI::Datatype[numberOfWorkers];
}
BARRIER("DataBus::createDynamicTypes#0");
// find all remote nodes
for (int j = 0; j < mesh->getNodesNumber(); j++)
{
CalcNode& node = mesh->getNodeByLocalIndex(j);
if ( node.isRemote() )
{
//LOG_DEBUG("N: " << j);
//LOG_DEBUG("R1: " << j << " " << mesh->getBody()->getId());
int owner = dispatcher->getOwner(node.coords/*, mesh->getBody()->getId()*/);
//LOG_DEBUG("R2: " << owner);
assert_ne(owner, rank );
local_numbers[rank][owner].push_back( mesh->nodesMap[node.number] );
remote_numbers[rank][owner].push_back(node.number);
}
}
BARRIER("DataBus::createDynamicTypes#1");
LOG_DEBUG("Requests prepared:");
for (int i = 0; i < numberOfWorkers; i++)
for (int j = 0; j < numberOfWorkers; j++)
LOG_DEBUG("Request size from #" << i << " to #" << j << ": " << local_numbers[i][j].size());
// sync types
unsigned int max_len = 0;
for (int i = 0; i < numberOfWorkers; i++)
for (int j = 0; j < numberOfWorkers; j++)
if (local_numbers[i][j].size() > max_len)
max_len = local_numbers[i][j].size();
vector<int> lengths;
for (unsigned int i = 0; i < max_len; i++)
lengths.push_back(1);
int info[3];
vector<MPI::Request> reqs;
for (int i = 0; i < numberOfWorkers; i++)
for (int j = 0; j < numberOfWorkers; j++)
if (local_numbers[i][j].size() > 0)
{
info[0] = remote_numbers[i][j].size();
info[1] = i;
info[2] = j;
MPI_NODE_TYPES[i][j] = MPI_ELNODE.Create_indexed(
local_numbers[i][j].size(),
&lengths[0],
&local_numbers[i][j][0]
);
MPI_NODE_TYPES[i][j].Commit();
reqs.push_back(
MPI::COMM_WORLD.Isend(
&remote_numbers[i][j][0],
remote_numbers[i][j].size(),
MPI::INT,
j,
TAG_SYNC_NODE_TYPES
)
);
reqs.push_back(
MPI::COMM_WORLD.Isend(
info,
3,
MPI::INT,
j,
TAG_SYNC_NODE_TYPES_I
)
);
}
BARRIER("DataBus::createDynamicTypes#2");
MPI::Status status;
while (MPI::COMM_WORLD.Iprobe(MPI::ANY_SOURCE, TAG_SYNC_NODE_TYPES_I, status))
{
MPI::COMM_WORLD.Recv(
info,
3,
MPI::INT,
status.Get_source(),
TAG_SYNC_NODE_TYPES_I
);
local_numbers[info[1]][info[2]].resize(info[0]);
MPI::COMM_WORLD.Recv(
&local_numbers[info[1]][info[2]][0],
info[0],
MPI::INT,
status.Get_source(),
TAG_SYNC_NODE_TYPES
);
if (lengths.size() < (unsigned)info[0])
for (int i = lengths.size(); i < info[0]; i++)
lengths.push_back(1);
for(int i = 0; i < info[0]; i++)
local_numbers[info[1]][info[2]][i] = mesh->nodesMap[ local_numbers[info[1]][info[2]][i] ];
MPI_NODE_TYPES[info[1]][info[2]] = MPI_ELNODE.Create_indexed(
info[0],
&lengths[0],
&local_numbers[info[1]][info[2]][0]
);
MPI_NODE_TYPES[info[1]][info[2]].Commit();
}
MPI::Request::Waitall(reqs.size(), &reqs[0]);
BARRIER("DataBus::createDynamicTypes#3");
for (int i = 0 ; i < numberOfWorkers; i++)
delete[] remote_numbers[i];
delete[] remote_numbers;
LOG_DEBUG("Building dynamic MPI types for fast node sync done");
}