void
find_influences
/////////////////////////////////////////////////////////////
/**
*/
(
Mat graph,
//graph of non-zero structure
IS wanted,
//nodes we are interested in. Contains different values on each processor.
IS *is_influences
//all the influences of the nodes we are interested in.
) {
//First, get all the matrix rows we are concerned with.
Mat subgraph_global;
get_matrix_rows(graph, wanted, &subgraph_global);
Mat subgraph;
MatGetLocalMat(subgraph_global, MAT_INITIAL_MATRIX, &subgraph);
MatDestroy(subgraph_global);
PetscInt n;
PetscInt *ia;
PetscInt *ja;
PetscTruth success = PETSC_FALSE;
MatGetRowIJ(subgraph, 0, PETSC_FALSE, PETSC_FALSE, &n, &ia, &ja, &success);
assert(success == PETSC_TRUE);
std::set<PetscInt> influences;
for(int ii=ia[0]; ii<ia[n]; ii++) {
influences.insert(ja[ii]);
}
success = PETSC_TRUE;
MatRestoreRowIJ(subgraph, 0, PETSC_FALSE, PETSC_FALSE, &n, &ia, &ja, &success);
MatDestroy(subgraph);
std::vector<PetscInt> all_influences;
std::copy(influences.begin(), influences.end(), std::back_inserter(all_influences));
ISCreateGeneral(PETSC_COMM_WORLD, all_influences.size(), &all_influences[0], is_influences);
}
int bp(char *matfile, char *vecfile, char *oldbase, char *newbase)
{
int i, j=0;
int m, n = 4; /* m and n are rows and columns of matrix A */
float sum, rms;
float **A, **Acp, **At;
float **U, **Ut;
float **V, **Vt;
float **S, **St, **Si;
float *Sv;
float *b, *bT, *db, *x, *dx, *x0;
float *Utb, *SiUtb, *VSiUtb;
float **UUt, **VVt, **VtV, **US, **SVt, **USVt;
float bperp, dbperp, bpar, dbpar, btemp;
FILE *fp, *fpNew, *fpOld;
/*
* Part I: Singular Value Decomposition of matrix A
*/
/* printf("Beginning singular value decomposition of matrix A...\n");*/
/* determine number of rows 'm' */
m = get_matrix_rows(matfile,vecfile);
/* establish matrix A and vector b */
b = alloc_vector(1, m);
db = alloc_vector(1, m);
bT = alloc_vector(1, m);
x = alloc_vector(1, n);
dx = alloc_vector(1, n);
x0 = alloc_vector(1, n);
A = matrix(1, m, 1, n);
Acp = matrix(1, m, 1, n);
At = matrix(1, n, 1, m);
/* establish decomposition matrices */
U = matrix(1, m, 1, n);
Ut = matrix(1, n, 1, m);
S = matrix(1, n, 1, n);
St = matrix(1, n, 1, n);
Si = matrix(1, n, 1, n);
V = matrix(1, n, 1, n);
Vt = matrix(1, n, 1, n);
/* establish product matrices */
UUt = matrix(1, n, 1, n);
VVt = matrix(1, n, 1, n);
VtV = matrix(1, n, 1, n);
US = matrix(1, m, 1, n);
SVt = matrix(1, n, 1, n);
/* establish SVD product matrices */
USVt = matrix(1, m, 1, n);
/* vector version of diagonal matrix S */
Sv = alloc_vector(1, m);
/* vector products */
Utb = alloc_vector(1, n);
SiUtb = alloc_vector(1, n);
VSiUtb = alloc_vector(1, n);
/* read matrix and vector from input files */
fp = FOPEN(matfile,"r");
for (i = 1; i <= m; i++) {
for (j = 1; j <= n; j++) {
fscanf(fp, "%f", &A[i][j]);
}
}
fclose(fp);
fp = FOPEN(vecfile, "r");
for (i = 1; i <= m; i++)
fscanf(fp, "%f", &b[i]);
fclose(fp);
/* copy A into Acp */
for (i = 1; i <= m; i++) {
for (j = 1; j <= n; j++) {
Acp[i][j] = A[i][j];
}
}
/* transpose A into At */
for (i = 1; i <= m; i++) {
for (j = 1; j <= n; j++) {
At[j][i] = A[i][j];
}
}
/* NR fn to decompose A = U x S x Vt, where U is written into A */
svdcmp(A, m, n, Sv, V);
/* copy Sv into the diagonal of S and St */
for (i = 1; i <= 4; i++)
St[i][i] = S[i][i] = Sv[i];
/* copy A into U where it belongs, copy Acp back into A */
for (i = 1; i <= m; i++) {
for (j = 1; j <= n; j++) {
U[i][j] = A[i][j];
A[i][j] = Acp[i][j];
}
}
/* establish Ut and Vt */
for (i = 1; i <= m; i++) {
for (j = 1; j <= n; j++) {
Ut[j][i] = U[i][j];
}
}
for (i = 1; i <= n; i++) {
for (j = 1; j <= n; j++) {
Vt[j][i] = V[i][j];
}
}
/* check that SVD of A == A */
matrix_multiply(U, S, US, m, n, n);
matrix_multiply(US, Vt, USVt, m, n, n);
for (i = 1; i <= m; i++) {
for (j = 1; j <= n; j++) {
if (fabs(A[i][j] - USVt[i][j]) > 1e-12) {
/* FIXME: This check needs to be examined and reintroduced */
/* Exit(" reconstruction of A from SVD failed"); */
}
}
}
/* invert S into Si, automatically fixing small singular values */
for (i = 1; i <= n; i++) {
if (fabs(S[i][i]) < 0.0) {
Exit("svdcmp() found a negative singular value");
}
if (S[i][i] < 1e-6) {
printf(" singular value %d = %f; auto-set inverse to zero\n", i, S[i][i]);
Si[i][i] = 0.0;
}
else {
Si[i][i] = 1.0 / S[i][i];
}
}
/* breathe sigh of relief having gotten through SVD */
/* printf("\nSVD of A is ok\n\n");*/
/*
* Part II: Solve for n-vector x0
*/
/* multiply matrix Ut x vector b = vector Utb */
for (i = 1; i <= n; i++) {
for (j = 1, sum = 0.0; j <= m; j++) {
sum += Ut[i][j] * b[j];
}
Utb[i] = sum;
}
/* multiply matrix Si x vector Utb = vector SiUtb */
for (i = 1; i <= n; i++) {
SiUtb[i] = Si[i][i] * Utb[i];
}
/* multiply matrix V x vector SiUtb = vector VSiUtb */
for (i = 1; i <= n; i++) {
for (j = 1, sum = 0.0; j <= n; j++) {
sum += V[i][j] * SiUtb[j];
}
VSiUtb[i] = sum;
}
/* copy VSiUtb into x0 */
for (i = 1; i <= n; i++) {
x0[i] = VSiUtb[i];
}
/* calculate A x x0 */
for (i = 1; i <= m; i++) {
for (j = 1, sum = 0.0; j <= n; j++) {
sum += A[i][j] * x0[j];
}
bT[i] = sum;
}
/*print_vector(bT, 1, m, "b check, compare with ...");
print_vector(b, 1, m, "b");
print_vector(x0, 1, n, "x0");*/
for (i = 1, sum = 0.0; i <= m; i++) {
sum += (bT[i] - b[i])*(bT[i] - b[i]);
}
rms = sqrt(sum/(float)(m));
if (!quietflag) printf(" RMS of b-reconstructed and b = %f\n\n", rms);
/* test for sign of deltas */
fpOld = FOPEN(oldbase,"r");
fscanf(fpOld, "%f %f %f %f %f", &bperp, &dbperp, &bpar, &dbpar, &btemp);
fclose(fpOld);
printf(" New Baseline: Normal: %f, delta: %f\n"
" Parallel: %f, delta: %f\n"
" Temporal: %f days\n\n", x0[1], x0[2], x0[3], x0[4], btemp);
if (logflag) {
sprintf(logbuf," New Baseline: Normal: %f, delta: %f\n"
" Parallel: %f, delta: %f\n"
" Temporal: %f days\n\n", x0[1], x0[2], x0[3], x0[4], btemp);
printLog(logbuf);
}
fpNew = FOPEN(newbase,"w");
fprintf(fpNew, "%14.7f %14.7f %14.7f %14.7f %14.7f\n",
x0[1], x0[2], x0[3], x0[4], btemp);
fclose(fpNew);
/* free memory */
free_vector(b,1,m);
free_vector(db,1,m);
free_vector(bT,1,m);
free_vector(x,1,m);
free_vector(dx,1,m);
free_vector(x0,1,m);
free_matrix(A,1,n,1,m);
free_matrix(Acp,1,n,1,m);
free_matrix(At,1,n,1,m);
free_matrix(U,1,m,1,n);
free_matrix(Ut,1,n,1,m);
free_matrix(S,1,n,1,n);
free_matrix(St,1,n,1,n);
free_matrix(Si,1,n,1,n);
free_matrix(V,1,n,1,n);
free_matrix(Vt,1,n,1,n);
free_matrix(UUt,1,n,1,n);
free_matrix(VVt,1,n,1,n);
free_matrix(VtV,1,n,1,n);
free_matrix(US,1,n,1,n);
free_matrix(SVt,1,n,1,n);
free_matrix(USVt,1,m,1,n);
free_vector(Sv,1,m);
free_vector(Utb,1,n);
free_vector(SiUtb,1,n);
free_vector(VSiUtb,1,n);
return(0);
}
void
cljp_coarsening(Mat depends_on, IS *pCoarse) {
const int debug = 0;
//create a vector of the weights.
Vec w;
MatGetVecs(depends_on, PETSC_NULL, &w);
VecZeroEntries(w);
//Get my local matrix size
PetscInt start;
PetscInt end;
MatGetOwnershipRange(depends_on, &start, &end);
//TODO: replace with something that doesn't require re-creating the matrix structure.
//Initialize all the weights
{
Mat influences;
MatTranspose(depends_on, &influences);
{
RawGraph influences_raw(influences);
assert(influences_raw.local_nrows() == end-start);
//Initialize the weight vector with \norm{S^T_i} + \sigma(i)
PetscScalar *local_weights;
VecGetArray(w, &local_weights);
for (int local_row=0; local_row < influences_raw.local_nrows(); local_row++) {
local_weights[local_row] =
influences_raw.ia(local_row+1)-influences_raw.ia(local_row) + frand();
}
VecRestoreArray(w, &local_weights);
}
MatDestroy(influences);
}
//VecView(w, PETSC_VIEWER_STDOUT_WORLD);
//--------------------------------------------------------------
//Prepare the scatters needed for the independent set algorithm.
IS all_local_nodes;
describe_partition(depends_on, &all_local_nodes);
NonlocalCollection nonlocal(depends_on, all_local_nodes);
ISDestroy(all_local_nodes);
//while we are here, get the matrix + graph nodes that we need.
Mat extended_depend_mat;
get_matrix_rows(depends_on, nonlocal.nodes, &extended_depend_mat);
// Vec used only for display purposes
enum NodeType {UNKNOWN=-1, FINE, COARSE};
Vec node_type;
VecDuplicate(w, &node_type);
VecSet(node_type, UNKNOWN);
Vec w_nonlocal;
VecDuplicate(nonlocal.vec, &w_nonlocal);
Vec node_type_nonlocal;
VecDuplicate(w_nonlocal, &node_type_nonlocal);
VecSet(node_type_nonlocal, UNKNOWN);
Vec is_not_independent;
VecDuplicate(w, &is_not_independent);
Vec is_not_independent_nonlocal;
VecDuplicate(w_nonlocal, &is_not_independent_nonlocal);
VecScatterBegin(nonlocal.scatter, w, w_nonlocal, INSERT_VALUES, SCATTER_FORWARD);
VecScatterEnd(nonlocal.scatter, w, w_nonlocal, INSERT_VALUES, SCATTER_FORWARD);
Vec w_update_nonlocal;
VecDuplicate(w_nonlocal, &w_update_nonlocal);
//get ready to find all the coarse and fine points
typedef std::set<PetscInt> IntSet;
IntSet unknown;
//initialize the unknown set with all points that are local to this processor.
for (int ii=start; ii<end; ii++) {
unknown.insert(ii);
}
//we use MPI_INT here because we need to allreduce it with MPI_LAND
int all_points_partitioned=0;
int inc = 0;
{
RawGraph dep_nonlocal_raw(extended_depend_mat);
//while not done
while(!all_points_partitioned) {
//Start: non-local weights, non-local coarse points
if (debug) {
LTRACE();
char fname[] = "weightsXXX";
char selection_graph[] = "selectionXXX";
sprintf(fname, "weights%03d", inc);
sprintf(selection_graph, "selection%03d", inc);
inc++;
/*
PetscViewer view;
PetscViewerBinaryMatlabOpen(PETSC_COMM_WORLD, fname, &view);
PetscViewerBinaryMatlabOutputVecDA(view, "z", w, user->da);
PetscViewerBinaryMatlabDestroy(view);
PetscViewerBinaryMatlabOpen(PETSC_COMM_WORLD, selection_graph, &view);
PetscViewerBinaryMatlabOutputVecDA(view, "z", node_type, user->da);
PetscViewerBinaryMatlabDestroy(view);
//*/
}
//Pre: non-local weights, non-local coarse points
//find the independent set.
//By using ADD_VALUES in a scattter, we can perform
//a boolean OR across procesors.
//is_not_independent[*] = false
VecSet(is_not_independent_nonlocal, 0);
//for all unknown points P
{
RawVector node_type_nonlocal_raw(node_type_nonlocal);
RawVector w_nonlocal_raw(w_nonlocal);
RawVector is_not_independent_nonlocal_raw(is_not_independent_nonlocal);
FOREACH(P, unknown) {
//get weight(P)
PetscScalar weight_P = w_nonlocal_raw.at(nonlocal.map[*P]);
//for all dependencies K of P (K st P->K)
for (PetscInt ii=0; ii<dep_nonlocal_raw.nnz_in_row(nonlocal.map[*P]); ii++) {
PetscInt K = dep_nonlocal_raw.col(nonlocal.map[*P], ii);
//skip if K is fine/coarse
/*
Notice that we don't have to consider the
independent set we've been generating here. By
construction, if K is in the independent set, then P
cannot be in the independent set.
*/
if (node_type_nonlocal_raw.at(nonlocal.map[K]) != UNKNOWN) {
continue;
}
//skip if P->K is marked
if (dep_nonlocal_raw.is_marked(nonlocal.map[*P], ii)) {
continue;
}
//get weight(K)
PetscScalar weight_K = w_nonlocal_raw.at(nonlocal.map[K]);
if (weight_K <= weight_P) {
//is_not_independent(K) = true
is_not_independent_nonlocal_raw.at(nonlocal.map[K]) = 1;
} else { // (weight(P) < weight_K)
is_not_independent_nonlocal_raw.at(nonlocal.map[*P]) = 1;
}
}
}
}
if (debug) {LTRACE();}
//VecView(is_not_independent_nonlocal, PETSC_VIEWER_STDOUT_WORLD);
//reconstruct is_not_independent vector with a ADD_VALUES, which
//performs boolean OR
VecSet(is_not_independent, 0);
VecScatterBegin(nonlocal.scatter, is_not_independent_nonlocal, is_not_independent, ADD_VALUES, SCATTER_REVERSE);
VecScatterEnd(nonlocal.scatter, is_not_independent_nonlocal, is_not_independent, ADD_VALUES, SCATTER_REVERSE);
IntSet new_coarse_points;
{
RawVector is_not_independent_raw(is_not_independent);
//for all unknown points P
FOREACH(P, unknown) {
//if (!is_not_independent(P))
if (is_not_independent_raw.at(*P) == 0) {
new_coarse_points.insert(*P);
if (debug) {SHOWVAR(*P, d);}
}
}
}
//Post: new coarse points (independent set)
if (debug) {LTRACE();}
//Pre: independent set
{
RawVector node_type_raw(node_type);
// for each independent point
FOREACH(I, new_coarse_points) {
//mark that point as coarse
node_type_raw.at(*I) = COARSE;
unknown.erase(*I);
}
}
//Post: updated coarse local
if (debug) {LTRACE();}
//Pre: updated coarse local
//scatter changes to other processors
VecScatterBegin(nonlocal.scatter, node_type, node_type_nonlocal, INSERT_VALUES, SCATTER_FORWARD);
VecScatterEnd(nonlocal.scatter, node_type, node_type_nonlocal, INSERT_VALUES, SCATTER_FORWARD);
//Post: updated coarse non-local
if (debug) {LTRACE();}
//Pre: updated coarse non-local, new local coarse points
VecSet(w_update_nonlocal, 0);
{
RawVector node_type_nonlocal_raw(node_type_nonlocal);
RawVector w_update_nonlocal_raw(w_update_nonlocal);
//for all new coarse points C
FOREACH(C, new_coarse_points) {
//for all K st C->K
for(PetscInt ii=0; ii<dep_nonlocal_raw.nnz_in_row(nonlocal.map[*C]); ii++) {
//mark (C->K)
dep_nonlocal_raw.mark(nonlocal.map[*C], ii);
PetscInt K = dep_nonlocal_raw.col(nonlocal.map[*C], ii);
//if K is unknown
if (node_type_nonlocal_raw.at(nonlocal.map[K]) == UNKNOWN) {
//measure(K)--
w_update_nonlocal_raw.at(nonlocal.map[K]) -= 1;
}
}
}
//for all unknown points I
FOREACH(I, unknown) {
IntSet common_coarse;
//for all (J->K)
for (PetscInt kk=0; kk<dep_nonlocal_raw.nnz_in_row(nonlocal.map[*I]); kk++) {
if (!dep_nonlocal_raw.is_marked(nonlocal.map[*I], kk)) {
//if K is coarse
PetscInt K = dep_nonlocal_raw.col(nonlocal.map[*I], kk);
if (node_type_nonlocal_raw.at(nonlocal.map[K]) == COARSE) {
//mark K as common coarse
common_coarse.insert(K);
//mark (J->K) if unmarked
dep_nonlocal_raw.mark(nonlocal.map[*I], kk);
}
}
}
//for all unmarked (I->J)
for (PetscInt jj=0; jj<dep_nonlocal_raw.nnz_in_row(nonlocal.map[*I]); jj++) {
if (!dep_nonlocal_raw.is_marked(nonlocal.map[*I], jj)) {
//for all (J->K), marked or no
PetscInt J = dep_nonlocal_raw.col(nonlocal.map[*I], jj);
for(PetscInt kk=0; kk<dep_nonlocal_raw.nnz_in_row(nonlocal.map[J]); kk++) {
//if K is in layer or ghost layer and common-coarse
PetscInt K = dep_nonlocal_raw.col(nonlocal.map[J], kk);
if (is_member(K, common_coarse)) {
//mark (I->J)
dep_nonlocal_raw.mark(nonlocal.map[*I], jj);
//measure(J)--
w_update_nonlocal_raw.at(nonlocal.map[J]) -= 1;
}
}
}
}
}
}
