int main(int argc,char ** argv) { Solver::Initialize(&argc,&argv,""); // Initialize the solver and MPI activity #if defined(USE_PARTITIONER) Partitioner::Initialize(&argc,&argv); // Initialize the partitioner activity #endif if( argc > 1 ) { TagReal phi; TagReal tag_F; TagRealArray tag_K; TagRealArray tag_BC; TagReal phi_ref; Mesh * m = new Mesh(); // Create an empty mesh double ttt = Timer(); bool repartition = false; m->SetCommunicator(INMOST_MPI_COMM_WORLD); // Set the MPI communicator for the mesh if( m->GetProcessorRank() == 0 ) // If the current process is the master one std::cout << argv[0] << std::endl; if( m->isParallelFileFormat(argv[1]) ) { m->Load(argv[1]); // Load mesh from the parallel file format repartition = true; } else { if( m->GetProcessorRank() == 0 ) m->Load(argv[1]); // Load mesh from the serial file format } BARRIER; if( m->GetProcessorRank() == 0 ) std::cout << "Processors: " << m->GetProcessorsNumber() << std::endl; if( m->GetProcessorRank() == 0 ) std::cout << "Load(MPI_File): " << Timer()-ttt << std::endl; //~ double ttt2 = Timer(); //~ Mesh t; //~ t.SetCommunicator(INMOST_MPI_COMM_WORLD); //~ t.SetParallelFileStrategy(0); //~ t.Load(argv[1]); //~ BARRIER //~ if( m->GetProcessorRank() == 0 ) std::cout << "Load(MPI_Scatter): " << Timer()-ttt2 << std::endl; #if defined(USE_PARTITIONER) if (m->GetProcessorsNumber() > 1) { // currently only non-distributed meshes are supported by Inner_RCM partitioner ttt = Timer(); Partitioner * p = new Partitioner(m); p->SetMethod(Partitioner::INNER_KMEANS,Partitioner::Partition); // Specify the partitioner p->Evaluate(); // Compute the partitioner and store new processor ID in the mesh delete p; BARRIER; if( m->GetProcessorRank() == 0 ) std::cout << "Evaluate: " << Timer()-ttt << std::endl; ttt = Timer(); m->Redistribute(); // Redistribute the mesh data m->ReorderEmpty(CELL|FACE|EDGE|NODE); // Clean the data after reordring BARRIER; if( m->GetProcessorRank() == 0 ) std::cout << "Redistribute: " << Timer()-ttt << std::endl; } #endif ttt = Timer(); phi = m->CreateTag("Solution",DATA_REAL,CELL,NONE,1); // Create a new tag for the solution phi bool makerefsol = true; if( m->HaveTag("PERM" ) ) { tag_K = m->GetTag("PERM"); makerefsol = false; std::cout << "Permeability from grid" << std::endl; } else { std::cout << "Set perm" << std::endl; tag_K = m->CreateTag("PERM",DATA_REAL,CELL,NONE,1); // Create a new tag for K tensor for( Mesh::iteratorCell cell = m->BeginCell(); cell != m->EndCell(); ++cell ) // Loop over mesh cells tag_K[*cell][0] = 1.0; // Store the tensor K value into the tag } if( m->HaveTag("BOUNDARY_CONDITION") ) { tag_BC = m->GetTag("BOUNDARY_CONDITION"); makerefsol = false; std::cout << "Boundary conditions from grid" << std::endl; } else { std::cout << "Set boundary conditions" << std::endl; double x[3]; tag_BC = m->CreateTag("BOUNDARY_CONDITION",DATA_REAL,FACE,FACE,3); for( Mesh::iteratorFace face = m->BeginFace(); face != m->EndFace(); ++face ) if( face->Boundary() && !(face->GetStatus() == Element::Ghost) ) { face->Centroid(x); tag_BC[*face][0] = 1; //dirichlet tag_BC[*face][1] = 0; //neumann tag_BC[*face][2] = func(x,0);//face->Mean(func, 0); } } if( m->HaveTag("FORCE") ) { tag_F = m->GetTag("FORCE"); makerefsol = false; std::cout << "Force from grid" << std::endl; } else if( makerefsol ) { std::cout << "Set rhs" << std::endl; tag_F = m->CreateTag("FORCE",DATA_REAL,CELL,NONE,1); // Create a new tag for external force double x[3]; for( Mesh::iteratorCell cell = m->BeginCell(); cell != m->EndCell(); ++cell ) // Loop over mesh cells { cell->Centroid(x); tag_F[*cell] = -func_rhs(x,1); //tag_F[*cell] = -cell->Mean(func_rhs,1); } } if(m->HaveTag("REFERENCE_SOLUTION") ) phi_ref = m->GetTag("REFERENCE_SOLUTION"); else if( makerefsol ) { phi_ref = m->CreateTag("REFRENCE_SOLUTION",DATA_REAL,CELL,NONE,1); double x[3]; for( Mesh::iteratorCell cell = m->BeginCell(); cell != m->EndCell(); ++cell ) { cell->Centroid(x); phi_ref[*cell] = func(x,0);//cell->Mean(func, 0); } } ttt = Timer(); m->ExchangeGhost(1,FACE); m->ExchangeData(tag_K,CELL,0); // Exchange the tag_K data over processors BARRIER; if( m->GetProcessorRank() == 0 ) std::cout << "Exchange ghost: " << Timer()-ttt << std::endl; ttt = Timer(); Solver S("inner_ilu2"); // Specify the linear solver to ASM+ILU2+BiCGStab one S.SetParameter("absolute_tolerance", "1e-8"); S.SetParameter("schwartz_overlap", "2"); Residual R; // Residual vector Sparse::LockService Locks; Sparse::Vector Update; // Declare the solution and the right-hand side vectors { Mesh::GeomParam table; table[CENTROID] = CELL | FACE; table[NORMAL] = FACE; table[ORIENTATION] = FACE; table[MEASURE] = CELL | FACE; table[BARYCENTER] = CELL | FACE; m->PrepareGeometricData(table); } BARRIER if( m->GetProcessorRank() == 0 ) std::cout << "Prepare geometric data: " << Timer()-ttt << std::endl; { Automatizator aut; Automatizator::MakeCurrent(&aut); INMOST_DATA_ENUM_TYPE iphi = aut.RegisterTag(phi,CELL); aut.EnumerateEntries(); // Set the indeces intervals for the matrix and vectors R.SetInterval(aut.GetFirstIndex(),aut.GetLastIndex()); R.InitLocks(); Update.SetInterval(aut.GetFirstIndex(),aut.GetLastIndex()); dynamic_variable Phi(aut,iphi); // Solve \nabla \cdot \nabla phi = f equation //for( Mesh::iteratorFace face = m->BeginFace(); face != m->EndFace(); ++face ) #if defined(USE_OMP) #pragma omp parallel #endif { variable flux; //should be more efficient to define here to avoid multiple memory allocations if storage for variations should be expanded rMatrix x1(3,1), x2(3,1), xf(3,1), n(3,1); double d1, d2, k1, k2, area, T, a, b, c; #if defined(USE_OMP) #pragma omp for #endif for(Storage::integer iface = 0; iface < m->FaceLastLocalID(); ++iface ) if( m->isValidFace(iface) ) { Face face = Face(m,ComposeFaceHandle(iface)); Element::Status s1,s2; Cell r1 = face->BackCell(); Cell r2 = face->FrontCell(); if( ((!r1->isValid() || (s1 = r1->GetStatus()) == Element::Ghost)?0:1) + ((!r2->isValid() || (s2 = r2->GetStatus()) == Element::Ghost)?0:1) == 0) continue; area = face->Area(); // Get the face area face->UnitNormal(n.data()); // Get the face normal face->Centroid(xf.data()); // Get the barycenter of the face r1->Centroid(x1.data()); // Get the barycenter of the cell k1 = n.DotProduct(rMatrix::FromTensor(tag_K[r1].data(), tag_K[r1].size(),3)*n); d1 = fabs(n.DotProduct(xf-x1)); if( !r2->isValid() ) // boundary condition { // bnd_pnt is a projection of the cell center to the face // a*pb + bT(pb-p1) = c // F = T(pb-p1) // pb = (c + bTp1)/(a+bT) // F = T/(a+bT)(c - ap1) T = k1/d1; a = 0; b = 1; c = 0; if( tag_BC.isValid() && face.HaveData(tag_BC) ) { a = tag_BC[face][0]; b = tag_BC[face][1]; c = tag_BC[face][2]; //std::cout << "a " << a << " b " << b << " c " << c << std::endl; } R.Lock(Phi.Index(r1)); R[Phi.Index(r1)] -= T/(a + b*T) * area * (c - a*Phi(r1)); R.UnLock(Phi.Index(r1)); } else { r2->Centroid(x2.data()); k2 = n.DotProduct(rMatrix::FromTensor(tag_K[r2].data(), tag_K[r2].size(),3)*n); d2 = fabs(n.DotProduct(x2-xf)); T = 1.0/(d1/k1 + d2/k2); flux = T* area * (Phi(r2) - Phi(r1)); if( s1 != Element::Ghost ) { R.Lock(Phi.Index(r1)); R[Phi.Index(r1)] -= flux; R.UnLock(Phi.Index(r1)); } if( s2 != Element::Ghost ) { R.Lock(Phi.Index(r2)); R[Phi.Index(r2)] += flux; R.UnLock(Phi.Index(r2)); } } } } if( tag_F.isValid() ) { #if defined(USE_OMP) #pragma omp parallel for #endif for( Storage::integer icell = 0; icell < m->CellLastLocalID(); ++icell ) if( m->isValidCell(icell) ) { Cell cell = Cell(m,ComposeCellHandle(icell)); if( cell->GetStatus() != Element::Ghost ) R[Phi.Index(cell)] -= tag_F[cell] * cell->Volume(); } } BARRIER; if( m->GetProcessorRank() == 0 ) std::cout << "Matrix assemble: " << Timer()-ttt << std::endl; //m->RemoveGeometricData(table); // Clean the computed geometric data if( argc > 3 ) // Save the matrix and RHS if required { ttt = Timer(); R.GetJacobian().Save(std::string(argv[2])); // "A.mtx" R.GetResidual().Save(std::string(argv[3])); // "b.rhs" BARRIER; if( m->GetProcessorRank() == 0 ) std::cout << "Save matrix \"" << argv[2] << "\" and RHS \"" << argv[3] << "\": " << Timer()-ttt << std::endl; } ttt = Timer(); S.SetMatrix(R.GetJacobian()); // Compute the preconditioner for the original matrix S.Solve(R.GetResidual(),Update); // Solve the linear system with the previously computted preconditioner BARRIER; if( m->GetProcessorRank() == 0 ) { std::cout << S.Residual() << " " << S.Iterations() << " " << S.ReturnReason() << std::endl; std::cout << "Solve system: " << Timer()-ttt << std::endl; } ttt = Timer(); if( phi_ref.isValid() ) { Tag error = m->CreateTag("error",DATA_REAL,CELL,NONE,1); double err_C = 0.0, err_L2 = 0.0, vol = 0.0; #if defined(USE_OMP) #pragma omp parallel #endif { double local_err_C = 0; #if defined(USE_OMP) #pragma omp for reduction(+:err_L2) reduction(+:vol) #endif for( Storage::integer icell = 0; icell < m->CellLastLocalID(); ++icell ) if( m->isValidCell(icell) ) { Cell cell = Cell(m,ComposeCellHandle(icell)); if( cell->GetStatus() != Element::Ghost ) { double old = phi[cell]; double exact = phi_ref[cell]; double res = Update[Phi.Index(cell)]; double sol = old-res; double err = fabs (sol - exact); if (err > local_err_C) local_err_C = err; err_L2 += err * err * cell->Volume(); vol += cell->Volume(); cell->Real(error) = err; phi[cell] = sol; } } #if defined(USE_OMP) #pragma omp critical #endif { if( local_err_C > err_C ) err_C = local_err_C; } } err_C = m->AggregateMax(err_C); // Compute the maximal C norm for the error err_L2 = sqrt(m->Integrate(err_L2)/m->Integrate(vol)); // Compute the global L2 norm for the error if( m->GetProcessorRank() == 0 ) std::cout << "err_C = " << err_C << std::endl; if( m->GetProcessorRank() == 0 ) std::cout << "err_L2 = " << err_L2 << std::endl; } } BARRIER; if( m->GetProcessorRank() == 0 ) std::cout << "Compute true residual: " << Timer()-ttt << std::endl; ttt = Timer(); m->ExchangeData(phi,CELL,0); // Data exchange over processors BARRIER; if( m->GetProcessorRank() == 0 ) std::cout << "Exchange phi: " << Timer()-ttt << std::endl; std::string filename = "result"; if( m->GetProcessorsNumber() == 1 ) filename += ".vtk"; else filename += ".pvtk"; ttt = Timer(); m->Save(filename); m->Save("result.pmf"); BARRIER; if( m->GetProcessorRank() == 0 ) std::cout << "Save \"" << filename << "\": " << Timer()-ttt << std::endl; delete m; }