double ogs_op(real *u, int op, const gs_data *data)
{
double t = 0;
local_condense(u,op,data->local_cm);
#ifdef MPI
t = nonlocal(u,op,data->nlinfo,data->comm);
#endif
local_uncondense(u,data->local_cm);
return t;
}
void
cljp_coarsening(Mat depends_on, IS *pCoarse) {
const int debug = 0;
//create a vector of the weights.
Vec w;
MatGetVecs(depends_on, PETSC_NULL, &w);
VecZeroEntries(w);
//Get my local matrix size
PetscInt start;
PetscInt end;
MatGetOwnershipRange(depends_on, &start, &end);
//TODO: replace with something that doesn't require re-creating the matrix structure.
//Initialize all the weights
{
Mat influences;
MatTranspose(depends_on, &influences);
{
RawGraph influences_raw(influences);
assert(influences_raw.local_nrows() == end-start);
//Initialize the weight vector with \norm{S^T_i} + \sigma(i)
PetscScalar *local_weights;
VecGetArray(w, &local_weights);
for (int local_row=0; local_row < influences_raw.local_nrows(); local_row++) {
local_weights[local_row] =
influences_raw.ia(local_row+1)-influences_raw.ia(local_row) + frand();
}
VecRestoreArray(w, &local_weights);
}
MatDestroy(influences);
}
//VecView(w, PETSC_VIEWER_STDOUT_WORLD);
//--------------------------------------------------------------
//Prepare the scatters needed for the independent set algorithm.
IS all_local_nodes;
describe_partition(depends_on, &all_local_nodes);
NonlocalCollection nonlocal(depends_on, all_local_nodes);
ISDestroy(all_local_nodes);
//while we are here, get the matrix + graph nodes that we need.
Mat extended_depend_mat;
get_matrix_rows(depends_on, nonlocal.nodes, &extended_depend_mat);
// Vec used only for display purposes
enum NodeType {UNKNOWN=-1, FINE, COARSE};
Vec node_type;
VecDuplicate(w, &node_type);
VecSet(node_type, UNKNOWN);
Vec w_nonlocal;
VecDuplicate(nonlocal.vec, &w_nonlocal);
Vec node_type_nonlocal;
VecDuplicate(w_nonlocal, &node_type_nonlocal);
VecSet(node_type_nonlocal, UNKNOWN);
Vec is_not_independent;
VecDuplicate(w, &is_not_independent);
Vec is_not_independent_nonlocal;
VecDuplicate(w_nonlocal, &is_not_independent_nonlocal);
VecScatterBegin(nonlocal.scatter, w, w_nonlocal, INSERT_VALUES, SCATTER_FORWARD);
VecScatterEnd(nonlocal.scatter, w, w_nonlocal, INSERT_VALUES, SCATTER_FORWARD);
Vec w_update_nonlocal;
VecDuplicate(w_nonlocal, &w_update_nonlocal);
//get ready to find all the coarse and fine points
typedef std::set<PetscInt> IntSet;
IntSet unknown;
//initialize the unknown set with all points that are local to this processor.
for (int ii=start; ii<end; ii++) {
unknown.insert(ii);
}
//we use MPI_INT here because we need to allreduce it with MPI_LAND
int all_points_partitioned=0;
int inc = 0;
{
RawGraph dep_nonlocal_raw(extended_depend_mat);
//while not done
while(!all_points_partitioned) {
//Start: non-local weights, non-local coarse points
if (debug) {
LTRACE();
char fname[] = "weightsXXX";
char selection_graph[] = "selectionXXX";
sprintf(fname, "weights%03d", inc);
sprintf(selection_graph, "selection%03d", inc);
inc++;
/*
PetscViewer view;
PetscViewerBinaryMatlabOpen(PETSC_COMM_WORLD, fname, &view);
PetscViewerBinaryMatlabOutputVecDA(view, "z", w, user->da);
PetscViewerBinaryMatlabDestroy(view);
PetscViewerBinaryMatlabOpen(PETSC_COMM_WORLD, selection_graph, &view);
PetscViewerBinaryMatlabOutputVecDA(view, "z", node_type, user->da);
PetscViewerBinaryMatlabDestroy(view);
//*/
}
//Pre: non-local weights, non-local coarse points
//find the independent set.
//By using ADD_VALUES in a scattter, we can perform
//a boolean OR across procesors.
//is_not_independent[*] = false
VecSet(is_not_independent_nonlocal, 0);
//for all unknown points P
{
RawVector node_type_nonlocal_raw(node_type_nonlocal);
RawVector w_nonlocal_raw(w_nonlocal);
RawVector is_not_independent_nonlocal_raw(is_not_independent_nonlocal);
FOREACH(P, unknown) {
//get weight(P)
PetscScalar weight_P = w_nonlocal_raw.at(nonlocal.map[*P]);
//for all dependencies K of P (K st P->K)
for (PetscInt ii=0; ii<dep_nonlocal_raw.nnz_in_row(nonlocal.map[*P]); ii++) {
PetscInt K = dep_nonlocal_raw.col(nonlocal.map[*P], ii);
//skip if K is fine/coarse
/*
Notice that we don't have to consider the
independent set we've been generating here. By
construction, if K is in the independent set, then P
cannot be in the independent set.
*/
if (node_type_nonlocal_raw.at(nonlocal.map[K]) != UNKNOWN) {
continue;
}
//skip if P->K is marked
if (dep_nonlocal_raw.is_marked(nonlocal.map[*P], ii)) {
continue;
}
//get weight(K)
PetscScalar weight_K = w_nonlocal_raw.at(nonlocal.map[K]);
if (weight_K <= weight_P) {
//is_not_independent(K) = true
is_not_independent_nonlocal_raw.at(nonlocal.map[K]) = 1;
} else { // (weight(P) < weight_K)
is_not_independent_nonlocal_raw.at(nonlocal.map[*P]) = 1;
}
}
}
}
if (debug) {LTRACE();}
//VecView(is_not_independent_nonlocal, PETSC_VIEWER_STDOUT_WORLD);
//reconstruct is_not_independent vector with a ADD_VALUES, which
//performs boolean OR
VecSet(is_not_independent, 0);
VecScatterBegin(nonlocal.scatter, is_not_independent_nonlocal, is_not_independent, ADD_VALUES, SCATTER_REVERSE);
VecScatterEnd(nonlocal.scatter, is_not_independent_nonlocal, is_not_independent, ADD_VALUES, SCATTER_REVERSE);
IntSet new_coarse_points;
{
RawVector is_not_independent_raw(is_not_independent);
//for all unknown points P
FOREACH(P, unknown) {
//if (!is_not_independent(P))
if (is_not_independent_raw.at(*P) == 0) {
new_coarse_points.insert(*P);
if (debug) {SHOWVAR(*P, d);}
}
}
}
//Post: new coarse points (independent set)
if (debug) {LTRACE();}
//Pre: independent set
{
RawVector node_type_raw(node_type);
// for each independent point
FOREACH(I, new_coarse_points) {
//mark that point as coarse
node_type_raw.at(*I) = COARSE;
unknown.erase(*I);
}
}
//Post: updated coarse local
if (debug) {LTRACE();}
//Pre: updated coarse local
//scatter changes to other processors
VecScatterBegin(nonlocal.scatter, node_type, node_type_nonlocal, INSERT_VALUES, SCATTER_FORWARD);
VecScatterEnd(nonlocal.scatter, node_type, node_type_nonlocal, INSERT_VALUES, SCATTER_FORWARD);
//Post: updated coarse non-local
if (debug) {LTRACE();}
//Pre: updated coarse non-local, new local coarse points
VecSet(w_update_nonlocal, 0);
{
RawVector node_type_nonlocal_raw(node_type_nonlocal);
RawVector w_update_nonlocal_raw(w_update_nonlocal);
//for all new coarse points C
FOREACH(C, new_coarse_points) {
//for all K st C->K
for(PetscInt ii=0; ii<dep_nonlocal_raw.nnz_in_row(nonlocal.map[*C]); ii++) {
//mark (C->K)
dep_nonlocal_raw.mark(nonlocal.map[*C], ii);
PetscInt K = dep_nonlocal_raw.col(nonlocal.map[*C], ii);
//if K is unknown
if (node_type_nonlocal_raw.at(nonlocal.map[K]) == UNKNOWN) {
//measure(K)--
w_update_nonlocal_raw.at(nonlocal.map[K]) -= 1;
}
}
}
//for all unknown points I
FOREACH(I, unknown) {
IntSet common_coarse;
//for all (J->K)
for (PetscInt kk=0; kk<dep_nonlocal_raw.nnz_in_row(nonlocal.map[*I]); kk++) {
if (!dep_nonlocal_raw.is_marked(nonlocal.map[*I], kk)) {
//if K is coarse
PetscInt K = dep_nonlocal_raw.col(nonlocal.map[*I], kk);
if (node_type_nonlocal_raw.at(nonlocal.map[K]) == COARSE) {
//mark K as common coarse
common_coarse.insert(K);
//mark (J->K) if unmarked
dep_nonlocal_raw.mark(nonlocal.map[*I], kk);
}
}
}
//for all unmarked (I->J)
for (PetscInt jj=0; jj<dep_nonlocal_raw.nnz_in_row(nonlocal.map[*I]); jj++) {
if (!dep_nonlocal_raw.is_marked(nonlocal.map[*I], jj)) {
//for all (J->K), marked or no
PetscInt J = dep_nonlocal_raw.col(nonlocal.map[*I], jj);
for(PetscInt kk=0; kk<dep_nonlocal_raw.nnz_in_row(nonlocal.map[J]); kk++) {
//if K is in layer or ghost layer and common-coarse
PetscInt K = dep_nonlocal_raw.col(nonlocal.map[J], kk);
if (is_member(K, common_coarse)) {
//mark (I->J)
dep_nonlocal_raw.mark(nonlocal.map[*I], jj);
//measure(J)--
w_update_nonlocal_raw.at(nonlocal.map[J]) -= 1;
}
}
}
}
}
}