void Filler::treat_region(GRegion* gr){
int NumSmooth = CTX::instance()->mesh.smoothCrossField;
std::cout << "NumSmooth = " << NumSmooth << std::endl ;
if(NumSmooth && (gr->dim() == 3)){
double scale = gr->bounds().diag()*1e-2;
Frame_field::initRegion(gr,NumSmooth);
Frame_field::saveCrossField("cross0.pos",scale);
Frame_field::smoothRegion(gr,NumSmooth);
Frame_field::saveCrossField("cross1.pos",scale);
}
#if defined(HAVE_RTREE)
unsigned int i;
int j;
int count;
int limit;
bool ok2;
double x,y,z;
SPoint3 point;
Node *node,*individual,*parent;
MVertex* vertex;
MElement* element;
MElementOctree* octree;
deMeshGRegion deleter;
Wrapper wrapper;
GFace* gf;
std::queue<Node*> fifo;
std::vector<Node*> spawns;
std::vector<Node*> garbage;
std::vector<MVertex*> boundary_vertices;
std::set<MVertex*> temp;
std::list<GFace*> faces;
std::map<MVertex*,int> limits;
std::set<MVertex*>::iterator it;
std::list<GFace*>::iterator it2;
std::map<MVertex*,int>::iterator it3;
RTree<Node*,double,3,double> rtree;
Frame_field::init_region(gr);
Size_field::init_region(gr);
Size_field::solve(gr);
octree = new MElementOctree(gr->model());
garbage.clear();
boundary_vertices.clear();
temp.clear();
new_vertices.clear();
faces.clear();
limits.clear();
faces = gr->faces();
for(it2=faces.begin();it2!=faces.end();it2++){
gf = *it2;
limit = code(gf->tag());
for(i=0;i<gf->getNumMeshElements();i++){
element = gf->getMeshElement(i);
for(j=0;j<element->getNumVertices();j++){
vertex = element->getVertex(j);
temp.insert(vertex);
limits.insert(std::pair<MVertex*,int>(vertex,limit));
}
}
}
/*for(i=0;i<gr->getNumMeshElements();i++){
element = gr->getMeshElement(i);
for(j=0;j<element->getNumVertices();j++){
vertex = element->getVertex(j);
temp.insert(vertex);
}
}*/
for(it=temp.begin();it!=temp.end();it++){
if((*it)->onWhat()->dim()==0){
boundary_vertices.push_back(*it);
}
}
for(it=temp.begin();it!=temp.end();it++){
if((*it)->onWhat()->dim()==1){
boundary_vertices.push_back(*it);
}
}
for(it=temp.begin();it!=temp.end();it++){
if((*it)->onWhat()->dim()==2){
boundary_vertices.push_back(*it);
}
}
/*for(it=temp.begin();it!=temp.end();it++){
if((*it)->onWhat()->dim()<3){
boundary_vertices.push_back(*it);
}
}*/
//std::ofstream file("nodes.pos");
//file << "View \"test\" {\n";
for(i=0;i<boundary_vertices.size();i++){
x = boundary_vertices[i]->x();
y = boundary_vertices[i]->y();
z = boundary_vertices[i]->z();
node = new Node(SPoint3(x,y,z));
compute_parameters(node,gr);
node->set_layer(0);
it3 = limits.find(boundary_vertices[i]);
node->set_limit(it3->second);
rtree.Insert(node->min,node->max,node);
fifo.push(node);
//print_node(node,file);
}
count = 1;
while(!fifo.empty()){
parent = fifo.front();
fifo.pop();
garbage.push_back(parent);
if(parent->get_limit()!=-1 && parent->get_layer()>=parent->get_limit()){
continue;
}
spawns.clear();
spawns.resize(6);
for(i=0;i<6;i++){
spawns[i] = new Node();
}
create_spawns(gr,octree,parent,spawns);
for(i=0;i<6;i++){
ok2 = 0;
individual = spawns[i];
point = individual->get_point();
x = point.x();
y = point.y();
z = point.z();
if(inside_domain(octree,x,y,z)){
compute_parameters(individual,gr);
individual->set_layer(parent->get_layer()+1);
individual->set_limit(parent->get_limit());
if(far_from_boundary(octree,individual)){
wrapper.set_ok(1);
wrapper.set_individual(individual);
wrapper.set_parent(parent);
rtree.Search(individual->min,individual->max,rtree_callback,&wrapper);
if(wrapper.get_ok()){
fifo.push(individual);
rtree.Insert(individual->min,individual->max,individual);
vertex = new MVertex(x,y,z,gr,0);
new_vertices.push_back(vertex);
ok2 = 1;
//print_segment(individual->get_point(),parent->get_point(),file);
}
}
}
if(!ok2) delete individual;
}
if(count%100==0){
printf("%d\n",count);
}
count++;
}
//file << "};\n";
int option = CTX::instance()->mesh.algo3d;
CTX::instance()->mesh.algo3d = ALGO_3D_DELAUNAY;
deleter(gr);
std::vector<GRegion*> regions;
regions.push_back(gr);
meshGRegion mesher(regions); //?
mesher(gr); //?
MeshDelaunayVolume(regions);
CTX::instance()->mesh.algo3d = option;
for(i=0;i<garbage.size();i++) delete garbage[i];
for(i=0;i<new_vertices.size();i++) delete new_vertices[i];
new_vertices.clear();
delete octree;
rtree.RemoveAll();
Size_field::clear();
Frame_field::clear();
#endif
}
/* *************************** *
* Main computational kernel *
* *************************** */
int correlationKernel(int rank,
int size,
double* dataMatrixX,
double* dataMatrixY,
int columns,
int rows,
char *out_filename,
int distance_flag) {
int local_check = 0, global_check = 0;
int i = 0, j, taskNo;
int err, count = 0;
unsigned long long fair_chunk = 0, coeff_count = 0;
unsigned int init_and_cleanup_loop_iter=0;
unsigned long long cor_cur_size = 0;
double start_time, end_time;
// Variables needed by the Indexed Datatype
MPI_Datatype coeff_index_dt;
MPI_File fh;
int *blocklens, *indices;
MPI_Status stat;
MPI_Comm comm = MPI_COMM_WORLD;
// Master processor keeps track of tasks
if (rank == 0) {
// Make sure everything will work fine even if there are
// less genes than available workers (there are size-1 workers
// master does not count)
if ( (size-1) > rows )
init_and_cleanup_loop_iter = rows+1;
else
init_and_cleanup_loop_iter = size;
// Start timer
start_time = MPI_Wtime();
// Send out initial tasks (remember you have size-1 workers, master does not count)
for (i=1; i<init_and_cleanup_loop_iter; i++) {
taskNo = i-1;
err = MPI_Send(&taskNo, 1, MPI_INT, i, 0, comm);
}
// Terminate any processes that were not working due to the fact
// that the number of rows where less than the actual available workers
for(i=init_and_cleanup_loop_iter; i < size; i++) {
PROF(rank, "\nPROF_idle : Worker %d terminated due to insufficient work load", i);
err = -1;
err = MPI_Send(&err, 1, MPI_INT, i, 0, comm);
}
// Wait for workers to finish their work assignment and ask for more
for (i=init_and_cleanup_loop_iter-1; i<rows; i++) {
err = MPI_Recv(&taskNo, 1, MPI_INT, MPI_ANY_SOURCE, 0, comm, &stat);
// Check taskNo to make sure everything is ok. Negative means there is problem
// thus terminate gracefully all remaining working workers
if ( taskNo < 0 ) {
// Reduce by one because one worker is already terminated
init_and_cleanup_loop_iter--;
// Break and cleanup
break;
}
// The sending processor is ready to work:
// It's ID is in stat.MPI_SOURCE
// Send it the current task (i)
err = MPI_Send(&i, 1, MPI_INT, stat.MPI_SOURCE, 0, comm);
}
// Clean up processors
for (i=1; i<init_and_cleanup_loop_iter; i++) {
// All tasks complete - shutdown workers
err = MPI_Recv(&taskNo, 1, MPI_INT, MPI_ANY_SOURCE, 0, comm, &stat);
// If process failed then it will not be waiting to receive anything
// We have to ignore the send because it will deadlock
if ( taskNo < 0 )
continue;
err = -1;
err = MPI_Send(&err, 1, MPI_INT, stat.MPI_SOURCE, 0, comm);
}
// Master is *always* OK
local_check = 0;
MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
// Check failed, abort
if ( global_check != 0 ) {
return -1;
}
// Stop timer
end_time = MPI_Wtime();
PROF(rank, "\nPROF_comp (workers=%d) : Time taken by correlation coefficients computations : %g\n", size-1, end_time - start_time);
// Start timer
start_time = MPI_Wtime();
// Master process must call MPI_File_set_view as well, it's a collective call
// Open the file handler
MPI_File_open(comm, out_filename, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
// Create the file view
MPI_File_set_view(fh, 0, MPI_DOUBLE, MPI_DOUBLE, "native", MPI_INFO_NULL);
// Write data to disk
MPI_File_write_all(fh, &cor[0], 0, MPI_DOUBLE, &stat);
// Stop timer
end_time = MPI_Wtime();
PROF(rank, "\nPROF_write (workers=%d) : Time taken for global write-file : %g\n", size-1, end_time - start_time);
} else {
// Compute how many workers will share the work load
// Two scenarios exist:
// (1) more OR equal number of workers and rows exist
// (2) more rows than workers
if ( (size-1) > rows ) {
// For this scenario each worker will get exaclty one work asssignment.
// There is not going to be any other work so it only compute "rows" number
// of coefficients
fair_chunk = rows;
cor_cur_size = fair_chunk;
} else {
// For this scenario we are going to allocate space equal to a fair
// distribution of work assignments *plus* an extra amount of space to
// cover any load imbalancing. This amount is expressed as a percentage
// of the fair work distribution (see on top, 20% for now)
// Plus 1 to round it up or just add some extra space, both are fine
fair_chunk = (rows / (size-1)) + 1;
DEBUG("fair_chunk %d \n", fair_chunk);
// We can use "j" as temporary variable.
// Plus 1 to avoid getting 0 from the multiplication.
j = (fair_chunk * MEM_PERC) + 1;
cor_cur_size = (fair_chunk + j) * rows;
DEBUG("cor_cur_size %lld \n", cor_cur_size);
}
// Allocate memory
DEBUG("cor_cur_size %lld \n", cor_cur_size);
long long double_size = sizeof(double);
DEBUG("malloc size %lld \n", (double_size * cor_cur_size));
cor = (double *)malloc(double_size * cor_cur_size);
blocklens = (int *)malloc(sizeof(int) * rows);
indices = (int *)malloc(sizeof(int) * rows);
mean_value_vectorX = (double *)malloc(sizeof(double) * rows);
Sxx_vector = (double *)malloc(sizeof(double) * rows);
mean_value_vectorY = (double *)malloc(sizeof(double) * rows);
Syy_vector = (double *)malloc(sizeof(double) * rows);
// Check that all memory is successfully allocated
if ( ( cor == NULL ) || ( blocklens == NULL ) || ( indices == NULL ) ||
( mean_value_vectorX == NULL ) || ( Sxx_vector == NULL ) ||
( mean_value_vectorY == NULL ) || ( Syy_vector == NULL ) ) {
ERR("**ERROR** : Memory allocation failed on worker process %d. Aborting.\n", rank);
// Free allocated memory
free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector);
// Let the master process know its aborting in order to terminate
// the rest of the working workers
// We have to receive a work assignment first and then terminate
// otherwise the master will deadlock trying to give work to this worker
err = MPI_Recv(&taskNo, 1, MPI_INT, 0, 0, comm, &stat);
taskNo = -1;
err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm);
// This worker failed
local_check = 1;
MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
return -1;
}
// Compute necessary parameters for Pearson method
// (this will transform the values of the input array to more meaningful data
// and save us from a lot of redundant computations)
compute_parameters(dataMatrixX, dataMatrixY, rows, columns);
// Main loop for workers. They get work from master, compute coefficients,
// save them to their *local* vector and ask for more work
for(;;) {
// Get work
err = 0;
err = MPI_Recv(&taskNo, 1, MPI_INT, 0, 0, comm, &stat);
// If received task is -1, function is terminated
if ( taskNo == -1 ) break;
// Check if there is enough memory to store the new coefficients, if not reallocate
// the current memory and expand it by MEM_PERC of the approximated size
if ( cor_cur_size < (coeff_count + rows) ) {
PROF(0, "\n**WARNING** : Worker process %3d run out of memory and reallocates. Potential work imbalancing\n", rank);
DEBUG("\n**WARNING** : Worker process %3d run out of memory and reallocates. Potential work imbalancing\n", rank);
// Use j as temporary again. Add two (or any other value) to avoid 0.
// (two is just a random value, you can put any value really...)
j = (fair_chunk * MEM_PERC) + 2;
cor_cur_size += (j * rows);
// Reallocate and check
cor = (double *)realloc(cor, sizeof(double) * cor_cur_size);
if ( cor == NULL ) {
ERR("**ERROR** : Memory re-allocation failed on worker process %d. Aborting.\n", rank);
// Let the master process know its aborting in order to terminate
// the rest of the working workers
taskNo = -1;
err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm);
// This worker failed
local_check = 1;
MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
// Free all allocated memory
free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector);
return -1;
}
}
// Compute the correlation coefficients
if(dataMatrixY != NULL) {
for (j=0; j < rows; j++) {
cor[coeff_count] = pearson_XY(dataMatrixX, dataMatrixY, j, taskNo, columns);
coeff_count++;
}
} else {
for (j=0; j < rows; j++) {
// Set main diagonal to 1
if ( j == taskNo ) {
cor[coeff_count] = 1.0;
coeff_count++;
continue;
}
cor[coeff_count] = pearson(dataMatrixX, taskNo, j, columns);
coeff_count++;
}
}
// The value of blocklens[] represents the number of coefficients on each
// row of the corellation array
blocklens[count] = rows;
// The value of indices[] represents the offset of each row in the data file
indices[count] = (taskNo * rows);
count++;
// Give the master the taskID
err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm);
}
// There are two possibilities
// (a) everything went well and all workers finished ok
// (b) some processes finished ok but one or more of the remaining working workers failed
// To make sure all is well an all-reduce will be performed to sync all workers and guarantee success
// before moving on to write the output file
// This worker is OK
local_check = 0;
MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
// Check failed
if ( global_check != 0 ) {
// Free all allocated memory
free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector);
return -1;
}
PROF(0, "\nPROF_stats (thread %3d) : Fair chunk of work : %d \t\t Allocated : %d \t\t Computed : %d\n",
rank, fair_chunk, cor_cur_size, coeff_count);
// If the distance_flag is set, then transform all correlation coefficients to distances
if ( distance_flag == 1 ) {
for(j=0; j < coeff_count; j++) {
cor[j] = 1 - cor[j];
}
}
// Create and commit the Indexed datatype *ONLY* if there are data available
if ( coeff_count != 0 ) {
MPI_Type_indexed(count, blocklens, indices, MPI_DOUBLE, &coeff_index_dt);
MPI_Type_commit(&coeff_index_dt);
}
// Open the file handler
MPI_File_open(comm, out_filename, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
// Create the file view
if ( coeff_count != 0 ) {
MPI_File_set_view(fh, 0, MPI_DOUBLE, coeff_index_dt, "native", MPI_INFO_NULL);
} else {
MPI_File_set_view(fh, 0, MPI_DOUBLE, MPI_DOUBLE, "native", MPI_INFO_NULL);
}
// Write data to disk
// TODO coeff_count cannot be greater than max int (for use in the MPI_File_write_all call).
// A better fix should be possible, for now throw error.
DEBUG("\ncoeff_count is %lld\n", coeff_count);
DEBUG("\INT_MAX is %d\n", INT_MAX);
if(coeff_count>INT_MAX)
{
ERR("**ERROR** : Could not run as the chunks of data are too large. Try running again with more MPI processes.\n");
// Free allocated memory
free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector);
// Let the master process know its aborting in order to terminate
// the rest of the working workers
// We have to receive a work assignment first and then terminate
// otherwise the master will deadlock trying to give work to this worker
err = MPI_Recv(&taskNo, 1, MPI_INT, 0, 0, comm, &stat);
taskNo = -1;
err = MPI_Send(&taskNo, 1, MPI_INT, 0, 0, comm);
// This worker failed
local_check = 1;
MPI_Allreduce(&local_check, &global_check, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
return -1;
}
DEBUG("\nWriting %d to disk\n", coeff_count);
MPI_File_write_all(fh, &cor[0], coeff_count, MPI_DOUBLE, &stat);
if (coeff_count != 0 )
MPI_Type_free(&coeff_index_dt);
// Free all allocated memory
free_all(cor, blocklens, indices, mean_value_vectorX, Sxx_vector, mean_value_vectorY, Syy_vector);
}
DEBUG("\nAbout to write to disk %d\n", rank);
MPI_File_sync( fh ) ; // Causes all previous writes to be transferred to the storage device
DEBUG("\nWritten to disk %d\n",rank);
// MPI_Barrier( MPI_COMM_WORLD ) ; // Blocks until all processes in the communicator have reached this routine.
DEBUG("\nAfter barrier \n", rank);
// Close file handler
MPI_File_close(&fh);
DEBUG("\nAfter file closed /n");
// MPI_Barrier( MPI_COMM_WORLD ) ; // Blocks until all processes in the communicator have reached this routine.
DEBUG("\nAbout to return from kernel /n");
return 0;
}