Esempio n. 1
0
void gcm::Geo2MeshLoader::loadMesh(TetrMeshSecondOrder* mesh, GCMDispatcher* dispatcher, string fileName, float tetrSize)
{
    if (!isMshFileCreated(fileName))
        createMshFile(fileName, tetrSize);

    //IBody* body = mesh->getBody();
    Engine& engine = Engine::getInstance();
    if( engine.getRank() == 0 )
    {
        LOG_DEBUG("Worker 0 started generating second order mesh from first order msh file");
        TetrMeshFirstOrder* foMesh = new TetrMeshFirstOrder();
//        foMesh->setBody(body);

        TetrMeshSecondOrder* soMesh = new TetrMeshSecondOrder();
//        soMesh->setBody(body);

        int sd, nn;
        AABB scene;
        GCMDispatcher* myDispatcher = new DummyDispatcher();
        myDispatcher->setEngine(&Engine::getInstance());
        preLoadMesh(&scene, sd, nn, fileName, tetrSize);
        myDispatcher->prepare(1, &scene);

        MshTetrFileReader* reader = new MshTetrFileReader();
        reader->readFile(getMshFileName(fileName), foMesh, myDispatcher, engine.getRank(), true);
        soMesh->copyMesh(foMesh);
        soMesh->preProcess();

        VTK2SnapshotWriter* sw = new VTK2SnapshotWriter();
        sw->setFileName(getVtkFileName(fileName));
        sw->dump(soMesh, -1);

        delete sw;
        delete reader;
        delete foMesh;
        delete soMesh;
        LOG_DEBUG("Worker 0 completed generating second order mesh");
    }

    MPI::COMM_WORLD.Barrier();

    LOG_DEBUG("Starting reading mesh");
    Vtu2TetrFileReader* reader = new Vtu2TetrFileReader();
    reader->readFile(getVtkFileName(fileName), mesh, dispatcher, engine.getRank());
    delete reader;

    mesh->preProcess();
}
void Ani3D2MeshLoader::loadMesh(TetrMeshSecondOrder* mesh, GCMDispatcher* dispatcher, const string& fileName)
{
    auto body = mesh->getBody();
    auto& engine = Engine::getInstance();
    if( engine.getRank() == 0 )
    {
        LOG_DEBUG("Worker 0 started generating second order mesh");
        TetrMeshFirstOrder* foMesh = new TetrMeshFirstOrder();
        foMesh->setBody(body);

        TetrMeshSecondOrder* soMesh = new TetrMeshSecondOrder();
        soMesh->setBody(body);

        int sd, nn;
        AABB scene;
        GCMDispatcher* myDispatcher = new DummyDispatcher();
        preLoadMesh(&scene, sd, nn, fileName);
        myDispatcher->prepare(1, &scene);

        Ani3DTetrFileReader* reader = new Ani3DTetrFileReader();
        reader->readFile(fileName,
                            foMesh, myDispatcher, engine.getRank(), true);
        soMesh->copyMesh(foMesh);
        soMesh->preProcess();

        VTK2SnapshotWriter* sw = new VTK2SnapshotWriter();
        // FIXME are we really about to overwrite this file?
        sw->dump(soMesh, -1, getVtkFileName(fileName));

        delete sw;
        delete reader;
        delete foMesh;
        delete soMesh;
        LOG_DEBUG("Worker 0 completed generating second order mesh");
    }

    MPI::COMM_WORLD.Barrier();

    LOG_DEBUG("Starting reading mesh");
    Vtu2TetrFileReader* reader = new Vtu2TetrFileReader();
    reader->readFile(getVtkFileName(fileName), mesh, dispatcher, engine.getRank());
    delete reader;
    LOG_DEBUG("Deleting generated file: " << getVtkFileName(fileName));
    remove( getVtkFileName(fileName).c_str() );


    mesh->preProcess();
}
Esempio n. 3
0
void DataBus::syncMissedNodes(Mesh* _mesh, float tau)
{
    if( numberOfWorkers == 1 )
        return;

    bool transferRequired = false;

    AABB **reqZones = new AABB*[numberOfWorkers];
    AABB *reqZones_data = new AABB[numberOfWorkers*numberOfWorkers];
    for ( int i = 0; i < numberOfWorkers; ++i ) {
        reqZones[i] = reqZones_data + (i*numberOfWorkers);
    }
    auto& engine = Engine::getInstance();
    GCMDispatcher* dispatcher = engine.getDispatcher();

    // FIXME@avasyukov - workaround for SphProxyDispatcher
    // But we still need this
    if( dispatcher->getOutline(0) == NULL ) {
        THROW_BAD_METHOD("We can't do this because it will cause all MPI routines to freeze");
        return;
    }

    BARRIER("DataBus::syncMissedNodes#1");

    //Body* body = engine.getBodyById( engine.getDispatcher()->getMyBodyId() );
    //TetrMeshSecondOrder* mesh = (TetrMeshSecondOrder*)body->getMeshes();
    TetrMeshSecondOrder* mesh = (TetrMeshSecondOrder*) _mesh;
    AABB* areaOfInterest = &(mesh->areaOfInterest);
    if( (mesh->syncedArea).includes( areaOfInterest ) )
    {
        LOG_DEBUG("We need nothing");
    }
    else
    {
        LOG_DEBUG("Yes, we need additional nodes");
        for (int i = 0 ; i < numberOfWorkers; i++)
        {
            if( i != rank)
            {
                LOG_DEBUG("Our area of interest: " << *areaOfInterest << " Outline[" << i << "]: " << *(dispatcher->getOutline(i)));
                areaOfInterest->findIntersection(dispatcher->getOutline(i), &reqZones[rank][i]);
            }
        }
    }

    BARRIER("DataBus::syncMissedNodes#2");

    MPI::COMM_WORLD.Allgather(
        MPI_IN_PLACE,
        numberOfWorkers, MPI_OUTLINE,
        reqZones_data,
        numberOfWorkers, MPI_OUTLINE
    );

    BARRIER("DataBus::syncMissedNodes#3");

    vector<AABB> *_reqZones = new vector<AABB>[numberOfWorkers];
    for (int i = 0 ; i < numberOfWorkers; i++)
        for (int j = 0 ; j < numberOfWorkers; j++)
        {
            _reqZones[i].push_back(reqZones[i][j]);
            if( !isinf(reqZones[i][j].minX) )
            {
                transferRequired = true;
            }
        }

    if(transferRequired)
    {
        transferNodes(mesh, _reqZones);
        BARRIER("DataBus::syncMissedNodes#3");
        //LOG_DEBUG("Rebuilding data types");
        //createDynamicTypes();
        LOG_DEBUG("Processing mesh after the sync");
        // Overhead
        for( int z = 0; z < engine.getNumberOfBodies(); z++ )
        {
            TetrMeshSecondOrder* tmpMesh = (TetrMeshSecondOrder*) engine.getBody(z)->getMeshes();
            tmpMesh->preProcess();
            tmpMesh->checkTopology(tau);
        }
        //FIXME@avasyukov - rethink it
        for( int z = 0; z < 3; z++ )
        {
            (mesh->syncedArea).min_coords[z] = (mesh->areaOfInterest).min_coords[z] - EQUALITY_TOLERANCE;
            (mesh->syncedArea).max_coords[z] = (mesh->areaOfInterest).max_coords[z] + EQUALITY_TOLERANCE;
        }
    }

    for (int i = 0 ; i < numberOfWorkers; i++)
        _reqZones[i].clear();
    delete[] _reqZones;

    delete[] reqZones_data;
    delete[] reqZones;
}
Esempio n. 4
0
void DataBus::createDynamicTypes(int bodyNum)
{
    LOG_DEBUG("Building dynamic MPI types for fast node sync");
    auto& engine = Engine::getInstance();
    GCMDispatcher* dispatcher = engine.getDispatcher();
    Body* body = engine.getBody(bodyNum);//ById( engine.getDispatcher()->getMyBodyId() );
    TetrMeshSecondOrder* mesh = (TetrMeshSecondOrder*)body->getMeshes();

    // TODO add more cleanup code here to prevent memory leaks
    if (MPI_NODE_TYPES != NULL) {
        LOG_TRACE("Cleaning old types");
        for (int i = 0; i < numberOfWorkers; i++)
        {
            for (int j = 0; j < numberOfWorkers; j++)
            {
                LOG_TRACE("Cleaning type " << i << " " << j );
                LOG_TRACE("Size " << i << " " << j << " = " << local_numbers[i][j].size());
                if (local_numbers[i][j].size() > 0)
                    MPI_NODE_TYPES[i][j].Free();
            }
        }
        delete[] MPI_NODE_TYPES;
    }

    if (local_numbers != NULL) {
        for (int i = 0; i < numberOfWorkers; i++)
            delete[] local_numbers[i];
        delete[] local_numbers;
    }

    // FIXME - it's overhead
    local_numbers = new vector<int>*[numberOfWorkers];
    vector<int> **remote_numbers = new vector<int>*[numberOfWorkers];
    MPI_NODE_TYPES = new MPI::Datatype*[numberOfWorkers];

    for (int i = 0; i < numberOfWorkers; i++)
    {
        local_numbers[i] = new vector<int>[numberOfWorkers];
        remote_numbers[i] = new vector<int>[numberOfWorkers];
        MPI_NODE_TYPES[i] = new MPI::Datatype[numberOfWorkers];
    }

    BARRIER("DataBus::createDynamicTypes#0");

    // find all remote nodes
    for (int j = 0; j < mesh->getNodesNumber(); j++)
    {
        CalcNode& node = mesh->getNodeByLocalIndex(j);
        if ( node.isRemote() )
        {
            //LOG_DEBUG("N: " << j);
            //LOG_DEBUG("R1: " << j << " " << mesh->getBody()->getId());
            int owner = dispatcher->getOwner(node.coords/*, mesh->getBody()->getId()*/);
            //LOG_DEBUG("R2: " << owner);
            assert_ne(owner, rank );
            local_numbers[rank][owner].push_back( mesh->nodesMap[node.number] );
            remote_numbers[rank][owner].push_back(node.number);
        }
    }

    BARRIER("DataBus::createDynamicTypes#1");

    LOG_DEBUG("Requests prepared:");
    for (int i = 0; i < numberOfWorkers; i++)
        for (int j = 0; j < numberOfWorkers; j++)
            LOG_DEBUG("Request size from #" << i << " to #" << j << ": " << local_numbers[i][j].size());

    // sync types
    unsigned int max_len = 0;
    for (int i = 0; i < numberOfWorkers; i++)
        for (int j = 0; j < numberOfWorkers; j++)
            if (local_numbers[i][j].size() > max_len)
                max_len = local_numbers[i][j].size();

    vector<int> lengths;
    for (unsigned int i = 0; i < max_len; i++)
        lengths.push_back(1);

    int info[3];

    vector<MPI::Request> reqs;

    for (int i = 0; i < numberOfWorkers; i++)
        for (int j = 0; j < numberOfWorkers; j++)
            if (local_numbers[i][j].size() > 0)
            {
                info[0] = remote_numbers[i][j].size();
                info[1] = i;
                info[2] = j;
                MPI_NODE_TYPES[i][j] =  MPI_ELNODE.Create_indexed(
                    local_numbers[i][j].size(),
                    &lengths[0],
                    &local_numbers[i][j][0]
                );
                MPI_NODE_TYPES[i][j].Commit();
                reqs.push_back(
                    MPI::COMM_WORLD.Isend(
                        &remote_numbers[i][j][0],
                        remote_numbers[i][j].size(),
                        MPI::INT,
                        j,
                        TAG_SYNC_NODE_TYPES
                    )
                );
                reqs.push_back(
                    MPI::COMM_WORLD.Isend(
                        info,
                        3,
                        MPI::INT,
                        j,
                        TAG_SYNC_NODE_TYPES_I
                    )
                );
            }

    BARRIER("DataBus::createDynamicTypes#2");

    MPI::Status status;

    while (MPI::COMM_WORLD.Iprobe(MPI::ANY_SOURCE, TAG_SYNC_NODE_TYPES_I, status))
    {
        MPI::COMM_WORLD.Recv(
            info,
            3,
            MPI::INT,
            status.Get_source(),
            TAG_SYNC_NODE_TYPES_I
        );
        local_numbers[info[1]][info[2]].resize(info[0]);
        MPI::COMM_WORLD.Recv(
            &local_numbers[info[1]][info[2]][0],
            info[0],
            MPI::INT,
            status.Get_source(),
            TAG_SYNC_NODE_TYPES
        );
        if (lengths.size() < (unsigned)info[0])
            for (int i = lengths.size(); i < info[0]; i++)
                lengths.push_back(1);
        for(int i = 0; i < info[0]; i++)
            local_numbers[info[1]][info[2]][i] = mesh->nodesMap[ local_numbers[info[1]][info[2]][i] ];
        MPI_NODE_TYPES[info[1]][info[2]] =  MPI_ELNODE.Create_indexed(
            info[0],
            &lengths[0],
            &local_numbers[info[1]][info[2]][0]
        );
        MPI_NODE_TYPES[info[1]][info[2]].Commit();
    }

    MPI::Request::Waitall(reqs.size(), &reqs[0]);
    BARRIER("DataBus::createDynamicTypes#3");

    for (int i = 0 ; i < numberOfWorkers; i++)
        delete[] remote_numbers[i];
    delete[] remote_numbers;
    LOG_DEBUG("Building dynamic MPI types for fast node sync done");
}