SparseTileLayoutData<Dim>::SparseTileLayoutData(const Domain_t &bbox,
const Partitioner &gpar,
const ContextMapper<Dim> &cmap)
: LayoutBaseData<Dim>(false,false,
GuardLayers_t(0),GuardLayers_t(0),
bbox,bbox),
Observable<SparseTileLayoutData>(*this)
{
this->blocks_m = Loc<Dim>();
// Figure out if we have guards to worry about.
if (gpar.hasInternalGuards() && gpar.maxSize() > 1)
{
this->hasInternalGuards_m = true;
this->internalGuards_m = gpar.internalGuards();
}
if (gpar.hasExternalGuards())
{
this->hasExternalGuards_m = true;
this->externalGuards_m = gpar.externalGuards();
GuardLayers<Dim>::addGuardLayers(this->domain_m,this->externalGuards_m);
}
initialize(bbox, gpar,cmap);
}
UniformGridLayoutData<Dim>::
UniformGridLayoutData(const Domain_t &gdom,
const Partitioner &gpar,
const ContextMapper<Dim> & cmap )
: LayoutBaseData<Dim>(false,
false,
GuardLayers_t(0),
GuardLayers_t(0),
gdom,
gdom),
Observable<UniformGridLayoutData>(*this)
{
// Figure out if we have guards to worry about.
if (gpar.hasInternalGuards() && gpar.maxSize() > 1)
{
this->hasInternalGuards_m = true;
this->internalGuards_m = gpar.internalGuards();
}
if (gpar.hasExternalGuards())
{
this->hasExternalGuards_m = true;
this->externalGuards_m = gpar.externalGuards();
GuardLayers<Dim>::addGuardLayers(this->domain_m,this->externalGuards_m);
}
// Do the partitioning.
// This initializes allDomain_m, firsti_m, etc.
partition(gpar,cmap);
}
bool EqualJoin::isHashOnLeftKey(const Partitioner& part,const Attribute& key)const{
if(part.getPartitionFashion()!=PartitionFunction::hash_f)
return false;
for(unsigned i=0;i<joinkey_pair_list_.size();i++){
}
return part.getPartitionKey()==key;
}
void SparseTileLayoutData<Dim>::initialize(const Domain_t &bbox,
const Partitioner &gpar,
const ContextMapper<Dim> &cmap)
{
this->blocks_m = Loc<Dim>();
initialize(bbox,gpar.internalGuards(),gpar.externalGuards());
gpar.partition(bbox,this->all_m,cmap);
syncPatch();
}
int
main(int argc, char *argv[])
{
// Parse command-line
int argno=1;
for (/*void*/; argno<argc && '-'==argv[argno][0]; ++argno) {
if (!strcmp(argv[argno], "--")) {
++argno;
break;
} else {
std::cerr <<argv[0] <<": unrecognized switch: " <<argv[argno] <<"\n";
exit(1);
}
}
if (argno+1!=argc) {
std::cerr <<"usage: " <<argv[0] <<" [SWITCHES] [--] SPECIMEN\n";
exit(1);
}
std::string specimen_name = argv[argno++];
// Open the file
rose_addr_t start_va = 0;
MemoryMap map;
size_t file_size = map.insertFile(specimen_name, start_va);
map.at(start_va).limit(file_size).changeAccess(MemoryMap::EXECUTABLE, 0);
// Try to disassemble every byte, and print the CALL/FARCALL targets
InstructionMap insns;
size_t nerrors=0;
Disassembler *disassembler = new DisassemblerX86(4);
for (rose_addr_t offset=0; offset<file_size; ++offset) {
try {
rose_addr_t insn_va = start_va + offset;
if (SgAsmX86Instruction *insn = isSgAsmX86Instruction(disassembler->disassembleOne(&map, insn_va)))
insns[insn_va] = insn;
} catch (const Disassembler::Exception &e) {
++nerrors;
}
}
// Partition those instructions into basic blocks and functions
Partitioner partitioner;
SgAsmBlock *gblock = partitioner.partition(NULL, insns, &map);
// Print addresses of functions
struct T1: AstSimpleProcessing {
void visit(SgNode *node) {
if (SgAsmFunction *func = isSgAsmFunction(node))
std::cout <<StringUtility::addrToString(func->get_entry_va()) <<"\n";
}
};
T1().traverse(gblock, preorder);
std::cerr <<specimen_name <<": " <<insns.size() <<" instructions; " <<nerrors <<" errors\n";
return 0;
}
DataflowPartitioningDescriptor::DataflowPartitioningDescriptor(const Partitioner& partitioner){
partition_function_=partitioner.getPartitionFunction();
partition_key_=partitioner.getPartitionKey();
for(unsigned i=0;i<partitioner.getNumberOfPartitions();i++){
DataflowPartition dataflow_partition;
dataflow_partition.datasize_=partitioner.getPartitionDataSize(i);
dataflow_partition.location_=partitioner.getPartitionLocation(i);
dataflow_partition.partition_id_=i;
partition_list_.push_back(dataflow_partition);
}
}
PlanPartitioner::PlanPartitioner(const Partitioner& partitioner) {
partition_func_ = partitioner.getPartitionFunction();
partition_key_ = partitioner.getPartitionKey();
for (unsigned i = 0; i < partitioner.getNumberOfPartitions(); i++) {
PlanPartitionInfo plan_partition_info;
plan_partition_info.cardinality_ = partitioner.getPartitionCardinality(i);
plan_partition_info.location_ = partitioner.getPartitionLocation(i);
plan_partition_info.partition_id_ = i;
partition_list_.push_back(plan_partition_info);
}
}
bool Partitioner::hasSamePartitionLocation(const Partitioner & target_partition )const{
if(mode_==OneToMany||target_partition.get_binding_mode_()==OneToMany){
/** in the current version, any the location detection in OneToMany mode is ommited.*/
return false;
}
if(getNumberOfPartitions()!=target_partition.getNumberOfPartitions())
return false;
for(unsigned i=0;i<getNumberOfPartitions();i++){
if(!partition_info_list[i]->is_colocated(*target_partition.partition_info_list[i])){
return false;
}
}
return true;
}
void doPrepareData(TaskConfigure& configure, char* rootDirectory)
{
remove_all(rootDirectory);
create_directory( rootDirectory );
configure.setDllName("MapReduceDll.dll");
configure.setMapperClass("MyMapper");
configure.setReducerClass("MyReducer");
configure.setMapperOutKeyClass("int");
configure.setMapperOutValueClass("char*");
string className;
className = configure.getIntermediateFileRecordWriterClass();
TS_ASSERT(className == "TKeyValueWriter<int,char*>");
className = configure.getHashableComparableClass();
TS_ASSERT(className == "THashableComparable<int>");
vector<string> outputFiles;
outputFiles.push_back(string(rootDirectory) + "0.par");
outputFiles.push_back(string(rootDirectory) + "1.par");
outputFiles.push_back(string(rootDirectory) + "2.par");
outputFiles.push_back(string(rootDirectory) + "3.par");
outputFiles.push_back(string(rootDirectory) + "4.par");
Partitioner* partitioner = new Partitioner(configure, outputFiles);
char *data[]={
"123456789abcdef",
"23456789abcdef1",
"3456789abcdef12",
"456789abcdef123",
"56789abcdef1234",
"6789abcdef12345",
"789abcdef123456",
"89abcdef1234567XXX"
};
int dataSize = sizeof(data)/sizeof(char*);
for(int i=0; i< dataSize; i++){
char** pData = &data[i];
partitioner->collect( &i, pData);
}
delete partitioner;
//check result
}
CfgEmitter::CfgEmitter(const Partitioner &partitioner)
: BaseEmitter<ControlFlowGraph>(partitioner.cfg()), partitioner_(partitioner), useFunctionSubgraphs_(true),
showReturnEdges_(true), showInstructions_(false), showInstructionAddresses_(true), showInstructionStackDeltas_(true),
showInNeighbors_(true), showOutNeighbors_(true),
funcEnterColor_(0.33, 1.0, 0.9), // light green
funcReturnColor_(0.67, 1.0, 0.9), // light blue
warningColor_(0, 1.0, 0.80) // light red
{}
jboolean mappingEquals(JNIEnv* env, Object* obj ,string super, jobject jobj) {
string pattern("Partitioner<string>");
if (kmp_search(pattern.c_str(), pattern.size(), super.c_str(), super.size(),
0) >= 0) {
Partitioner<string>* ptr = dynamic_cast<Partitioner<string>*>(obj);
assert(ptr != NULL);
return ptr->equals(convertFromJObject<string>(env, jobj));
}
pattern = "Partitioner<double>";
if (kmp_search(pattern.c_str(), pattern.size(), super.c_str(), super.size(),
0) >= 0) {
Partitioner<double>* ptr = dynamic_cast<Partitioner<double>*>(obj);
assert(ptr != NULL);
return ptr->equals(convertFromJObject<double>(env, jobj));
}
pattern = "Partitioner<vector<string>>";
if (kmp_search(pattern.c_str(), pattern.size(), super.c_str(), super.size(),
0) >= 0) {
Partitioner<vector<string> >* ptr = dynamic_cast<Partitioner<
vector<string> >*>(obj);
assert(ptr != NULL);
return ptr->equals(
convertFromJObject<vector<string> >(env, jobj));
}
pattern = "Partitioner<vector<double>>";
if (kmp_search(pattern.c_str(), pattern.size(), super.c_str(), super.size(),
0) >= 0) {
Partitioner<vector<double> >* ptr = dynamic_cast<Partitioner<
vector<double> >* >(obj);
assert(ptr != NULL);
return ptr->equals(convertFromJObject<vector<double> >(env, jobj));
}
}
int main(int argc,char ** argv)
{
Solver::Initialize(&argc,&argv,""); // Initialize the solver and MPI activity
#if defined(USE_PARTITIONER)
Partitioner::Initialize(&argc,&argv); // Initialize the partitioner activity
#endif
if( argc > 1 )
{
TagReal phi;
TagReal tag_F;
TagRealArray tag_K;
TagRealArray tag_BC;
TagReal phi_ref;
Mesh * m = new Mesh(); // Create an empty mesh
double ttt = Timer();
bool repartition = false;
m->SetCommunicator(INMOST_MPI_COMM_WORLD); // Set the MPI communicator for the mesh
if( m->GetProcessorRank() == 0 ) // If the current process is the master one
std::cout << argv[0] << std::endl;
if( m->isParallelFileFormat(argv[1]) )
{
m->Load(argv[1]); // Load mesh from the parallel file format
repartition = true;
}
else
{
if( m->GetProcessorRank() == 0 )
m->Load(argv[1]); // Load mesh from the serial file format
}
BARRIER;
if( m->GetProcessorRank() == 0 ) std::cout << "Processors: " << m->GetProcessorsNumber() << std::endl;
if( m->GetProcessorRank() == 0 ) std::cout << "Load(MPI_File): " << Timer()-ttt << std::endl;
//~ double ttt2 = Timer();
//~ Mesh t;
//~ t.SetCommunicator(INMOST_MPI_COMM_WORLD);
//~ t.SetParallelFileStrategy(0);
//~ t.Load(argv[1]);
//~ BARRIER
//~ if( m->GetProcessorRank() == 0 ) std::cout << "Load(MPI_Scatter): " << Timer()-ttt2 << std::endl;
#if defined(USE_PARTITIONER)
if (m->GetProcessorsNumber() > 1)
{ // currently only non-distributed meshes are supported by Inner_RCM partitioner
ttt = Timer();
Partitioner * p = new Partitioner(m);
p->SetMethod(Partitioner::INNER_KMEANS,Partitioner::Partition); // Specify the partitioner
p->Evaluate(); // Compute the partitioner and store new processor ID in the mesh
delete p;
BARRIER;
if( m->GetProcessorRank() == 0 ) std::cout << "Evaluate: " << Timer()-ttt << std::endl;
ttt = Timer();
m->Redistribute(); // Redistribute the mesh data
m->ReorderEmpty(CELL|FACE|EDGE|NODE); // Clean the data after reordring
BARRIER;
if( m->GetProcessorRank() == 0 ) std::cout << "Redistribute: " << Timer()-ttt << std::endl;
}
#endif
ttt = Timer();
phi = m->CreateTag("Solution",DATA_REAL,CELL,NONE,1); // Create a new tag for the solution phi
bool makerefsol = true;
if( m->HaveTag("PERM" ) )
{
tag_K = m->GetTag("PERM");
makerefsol = false;
std::cout << "Permeability from grid" << std::endl;
}
else
{
std::cout << "Set perm" << std::endl;
tag_K = m->CreateTag("PERM",DATA_REAL,CELL,NONE,1); // Create a new tag for K tensor
for( Mesh::iteratorCell cell = m->BeginCell(); cell != m->EndCell(); ++cell ) // Loop over mesh cells
tag_K[*cell][0] = 1.0; // Store the tensor K value into the tag
}
if( m->HaveTag("BOUNDARY_CONDITION") )
{
tag_BC = m->GetTag("BOUNDARY_CONDITION");
makerefsol = false;
std::cout << "Boundary conditions from grid" << std::endl;
}
else
{
std::cout << "Set boundary conditions" << std::endl;
double x[3];
tag_BC = m->CreateTag("BOUNDARY_CONDITION",DATA_REAL,FACE,FACE,3);
for( Mesh::iteratorFace face = m->BeginFace(); face != m->EndFace(); ++face )
if( face->Boundary() && !(face->GetStatus() == Element::Ghost) )
{
face->Centroid(x);
tag_BC[*face][0] = 1; //dirichlet
tag_BC[*face][1] = 0; //neumann
tag_BC[*face][2] = func(x,0);//face->Mean(func, 0);
}
}
if( m->HaveTag("FORCE") )
{
tag_F = m->GetTag("FORCE");
makerefsol = false;
std::cout << "Force from grid" << std::endl;
}
else if( makerefsol )
{
std::cout << "Set rhs" << std::endl;
tag_F = m->CreateTag("FORCE",DATA_REAL,CELL,NONE,1); // Create a new tag for external force
double x[3];
for( Mesh::iteratorCell cell = m->BeginCell(); cell != m->EndCell(); ++cell ) // Loop over mesh cells
{
cell->Centroid(x);
tag_F[*cell] = -func_rhs(x,1);
//tag_F[*cell] = -cell->Mean(func_rhs,1);
}
}
if(m->HaveTag("REFERENCE_SOLUTION") )
phi_ref = m->GetTag("REFERENCE_SOLUTION");
else if( makerefsol )
{
phi_ref = m->CreateTag("REFRENCE_SOLUTION",DATA_REAL,CELL,NONE,1);
double x[3];
for( Mesh::iteratorCell cell = m->BeginCell(); cell != m->EndCell(); ++cell )
{
cell->Centroid(x);
phi_ref[*cell] = func(x,0);//cell->Mean(func, 0);
}
}
ttt = Timer();
m->ExchangeGhost(1,FACE);
m->ExchangeData(tag_K,CELL,0); // Exchange the tag_K data over processors
BARRIER;
if( m->GetProcessorRank() == 0 ) std::cout << "Exchange ghost: " << Timer()-ttt << std::endl;
ttt = Timer();
Solver S("inner_ilu2"); // Specify the linear solver to ASM+ILU2+BiCGStab one
S.SetParameter("absolute_tolerance", "1e-8");
S.SetParameter("schwartz_overlap", "2");
Residual R; // Residual vector
Sparse::LockService Locks;
Sparse::Vector Update; // Declare the solution and the right-hand side vectors
{
Mesh::GeomParam table;
table[CENTROID] = CELL | FACE;
table[NORMAL] = FACE;
table[ORIENTATION] = FACE;
table[MEASURE] = CELL | FACE;
table[BARYCENTER] = CELL | FACE;
m->PrepareGeometricData(table);
}
BARRIER
if( m->GetProcessorRank() == 0 ) std::cout << "Prepare geometric data: " << Timer()-ttt << std::endl;
{
Automatizator aut;
Automatizator::MakeCurrent(&aut);
INMOST_DATA_ENUM_TYPE iphi = aut.RegisterTag(phi,CELL);
aut.EnumerateEntries();
// Set the indeces intervals for the matrix and vectors
R.SetInterval(aut.GetFirstIndex(),aut.GetLastIndex());
R.InitLocks();
Update.SetInterval(aut.GetFirstIndex(),aut.GetLastIndex());
dynamic_variable Phi(aut,iphi);
// Solve \nabla \cdot \nabla phi = f equation
//for( Mesh::iteratorFace face = m->BeginFace(); face != m->EndFace(); ++face )
#if defined(USE_OMP)
#pragma omp parallel
#endif
{
variable flux; //should be more efficient to define here to avoid multiple memory allocations if storage for variations should be expanded
rMatrix x1(3,1), x2(3,1), xf(3,1), n(3,1);
double d1, d2, k1, k2, area, T, a, b, c;
#if defined(USE_OMP)
#pragma omp for
#endif
for(Storage::integer iface = 0; iface < m->FaceLastLocalID(); ++iface ) if( m->isValidFace(iface) )
{
Face face = Face(m,ComposeFaceHandle(iface));
Element::Status s1,s2;
Cell r1 = face->BackCell();
Cell r2 = face->FrontCell();
if( ((!r1->isValid() || (s1 = r1->GetStatus()) == Element::Ghost)?0:1) +
((!r2->isValid() || (s2 = r2->GetStatus()) == Element::Ghost)?0:1) == 0) continue;
area = face->Area(); // Get the face area
face->UnitNormal(n.data()); // Get the face normal
face->Centroid(xf.data()); // Get the barycenter of the face
r1->Centroid(x1.data()); // Get the barycenter of the cell
k1 = n.DotProduct(rMatrix::FromTensor(tag_K[r1].data(),
tag_K[r1].size(),3)*n);
d1 = fabs(n.DotProduct(xf-x1));
if( !r2->isValid() ) // boundary condition
{
// bnd_pnt is a projection of the cell center to the face
// a*pb + bT(pb-p1) = c
// F = T(pb-p1)
// pb = (c + bTp1)/(a+bT)
// F = T/(a+bT)(c - ap1)
T = k1/d1;
a = 0;
b = 1;
c = 0;
if( tag_BC.isValid() && face.HaveData(tag_BC) )
{
a = tag_BC[face][0];
b = tag_BC[face][1];
c = tag_BC[face][2];
//std::cout << "a " << a << " b " << b << " c " << c << std::endl;
}
R.Lock(Phi.Index(r1));
R[Phi.Index(r1)] -= T/(a + b*T) * area * (c - a*Phi(r1));
R.UnLock(Phi.Index(r1));
}
else
{
r2->Centroid(x2.data());
k2 = n.DotProduct(rMatrix::FromTensor(tag_K[r2].data(),
tag_K[r2].size(),3)*n);
d2 = fabs(n.DotProduct(x2-xf));
T = 1.0/(d1/k1 + d2/k2);
flux = T* area * (Phi(r2) - Phi(r1));
if( s1 != Element::Ghost )
{
R.Lock(Phi.Index(r1));
R[Phi.Index(r1)] -= flux;
R.UnLock(Phi.Index(r1));
}
if( s2 != Element::Ghost )
{
R.Lock(Phi.Index(r2));
R[Phi.Index(r2)] += flux;
R.UnLock(Phi.Index(r2));
}
}
}
}
if( tag_F.isValid() )
{
#if defined(USE_OMP)
#pragma omp parallel for
#endif
for( Storage::integer icell = 0; icell < m->CellLastLocalID(); ++icell ) if( m->isValidCell(icell) )
{
Cell cell = Cell(m,ComposeCellHandle(icell));
if( cell->GetStatus() != Element::Ghost )
R[Phi.Index(cell)] -= tag_F[cell] * cell->Volume();
}
}
BARRIER;
if( m->GetProcessorRank() == 0 ) std::cout << "Matrix assemble: " << Timer()-ttt << std::endl;
//m->RemoveGeometricData(table); // Clean the computed geometric data
if( argc > 3 ) // Save the matrix and RHS if required
{
ttt = Timer();
R.GetJacobian().Save(std::string(argv[2])); // "A.mtx"
R.GetResidual().Save(std::string(argv[3])); // "b.rhs"
BARRIER;
if( m->GetProcessorRank() == 0 ) std::cout << "Save matrix \"" << argv[2] << "\" and RHS \"" << argv[3] << "\": " << Timer()-ttt << std::endl;
}
ttt = Timer();
S.SetMatrix(R.GetJacobian()); // Compute the preconditioner for the original matrix
S.Solve(R.GetResidual(),Update); // Solve the linear system with the previously computted preconditioner
BARRIER;
if( m->GetProcessorRank() == 0 )
{
std::cout << S.Residual() << " " << S.Iterations() << " " << S.ReturnReason() << std::endl;
std::cout << "Solve system: " << Timer()-ttt << std::endl;
}
ttt = Timer();
if( phi_ref.isValid() )
{
Tag error = m->CreateTag("error",DATA_REAL,CELL,NONE,1);
double err_C = 0.0, err_L2 = 0.0, vol = 0.0;
#if defined(USE_OMP)
#pragma omp parallel
#endif
{
double local_err_C = 0;
#if defined(USE_OMP)
#pragma omp for reduction(+:err_L2) reduction(+:vol)
#endif
for( Storage::integer icell = 0; icell < m->CellLastLocalID(); ++icell ) if( m->isValidCell(icell) )
{
Cell cell = Cell(m,ComposeCellHandle(icell));
if( cell->GetStatus() != Element::Ghost )
{
double old = phi[cell];
double exact = phi_ref[cell];
double res = Update[Phi.Index(cell)];
double sol = old-res;
double err = fabs (sol - exact);
if (err > local_err_C) local_err_C = err;
err_L2 += err * err * cell->Volume();
vol += cell->Volume();
cell->Real(error) = err;
phi[cell] = sol;
}
}
#if defined(USE_OMP)
#pragma omp critical
#endif
{
if( local_err_C > err_C ) err_C = local_err_C;
}
}
err_C = m->AggregateMax(err_C); // Compute the maximal C norm for the error
err_L2 = sqrt(m->Integrate(err_L2)/m->Integrate(vol)); // Compute the global L2 norm for the error
if( m->GetProcessorRank() == 0 ) std::cout << "err_C = " << err_C << std::endl;
if( m->GetProcessorRank() == 0 ) std::cout << "err_L2 = " << err_L2 << std::endl;
}
}
BARRIER;
if( m->GetProcessorRank() == 0 ) std::cout << "Compute true residual: " << Timer()-ttt << std::endl;
ttt = Timer();
m->ExchangeData(phi,CELL,0); // Data exchange over processors
BARRIER;
if( m->GetProcessorRank() == 0 ) std::cout << "Exchange phi: " << Timer()-ttt << std::endl;
std::string filename = "result";
if( m->GetProcessorsNumber() == 1 )
filename += ".vtk";
else
filename += ".pvtk";
ttt = Timer();
m->Save(filename);
m->Save("result.pmf");
BARRIER;
if( m->GetProcessorRank() == 0 ) std::cout << "Save \"" << filename << "\": " << Timer()-ttt << std::endl;
delete m;
}
void UniformGridLayoutData<Dim>::partition(const Partitioner &gpar,
const ContextMapper<Dim> &cmap)
{
int i;
// In spite of being templated, this only works with uniform-grid
// partitioners.
CTAssert(Partitioner::uniform);
// We must have something to partition, and the domain lists must be
// empty.
PAssert(this->domain_m.size() > 0);
PAssert(this->innerdomain_m.size() > 0);
PAssert(this->all_m.size() == 0);
PAssert(this->local_m.size() == 0);
PAssert(this->remote_m.size() == 0);
// Save the first and block size info from the current domain.
this->blocks_m = gpar.blocks();
// Note, for the purposes of partitioning, we pretend like we're
// only working with the inner domain. The total domain includes the
// external guards, and those do not affect the partitioning.
blockstride_m[0] = 1;
int blocks[Dim];
for (i = 0; i < Dim; ++i)
{
this->firsti_m[i] = this->innerdomain_m[i].first();
this->firste_m[i] = this->domain_m[i].first();
blocks[i] = gpar.blocks()[i].first();
allDomain_m[i] = Interval<1>(blocks[i]);
blocksizes_m[i] = this->innerdomain_m[i].length() / blocks[i];
if (i > 0)
blockstride_m[i] = blockstride_m[i-1] * blocks[i-1];
}
// Invoke the partitioner.
gpar.partition(this->innerdomain_m, this->all_m, cmap);
// fill local and remote lists
typename List_t::const_iterator start = this->all_m.begin();
typename List_t::const_iterator end = this->all_m.end();
for ( ; start!=end ; ++start)
{
if ( (*start)->context() == Pooma::context()
|| (*start)->context() == -1 )
{
(*start)->localID() = this->local_m.size();
this->local_m.push_back(*start);
}
else
this->remote_m.push_back(*start);
}
if (this->hasInternalGuards_m)
{
this->gcFillList_m.clear();
calcGCFillList();
}
}
