static int run_test(Teuchos::RCP<Epetra_CrsMatrix> matrix, bool verbose, // display the graph before & after bool contract, // set global number of partitions to 1/2 num procs int partitioningType, // hypergraph or graph partitioning, or simple int vertexWeightType, // use vertex weights? int edgeWeightType, // use edge/hyperedge weights? int objectType) // use isorropia's CrsMatrix or CrsGraph { int rc=0, fail = 0; #ifdef HAVE_EPETRAEXT int localProc = 0; double balance1, balance2, cutn1, cutn2, cutl1, cutl2; double balance3, cutn3, cutl3; double cutWgt1, cutWgt2, cutWgt3; int numCuts1, numCuts2, numCuts3, valid; int numPartitions = 0; int keepDenseEdges = 0; int numProcs = 1; #ifdef HAVE_MPI const Epetra_MpiComm &Comm = dynamic_cast<const Epetra_MpiComm &>(matrix->Comm()); localProc = Comm.MyPID(); numProcs = Comm.NumProc(); #else const Epetra_SerialComm &Comm = dynamic_cast<const Epetra_SerialComm &>(matrix->Comm()); #endif int numRows = matrix->NumGlobalRows(); if (numRows < (numProcs * 100)){ // By default Zoltan throws out dense edges, defined as those // whose number of non-zeros exceeds 25% of the number of vertices. // // If dense edges are thrown out of a small matrix, there may be nothing left. keepDenseEdges = 1; } double myShareBefore = 1.0 / numProcs; double myShare = myShareBefore; if (contract){ numPartitions = numProcs / 2; if (numPartitions > numRows) numPartitions = numRows; if (numPartitions > 0){ if (localProc < numPartitions){ myShare = 1.0 / numPartitions; } else{ myShare = 0.0; } } else{ contract = 0; } } // If we want Zoltan's or Isorropia's default weights, then we don't // need to supply a CostDescriber object to createBalancedCopy, // so we get to test the API functions that don't take a CostDescriber. bool noCosts = ((vertexWeightType == NO_APPLICATION_SUPPLIED_WEIGHTS) && (edgeWeightType == NO_APPLICATION_SUPPLIED_WEIGHTS)); // Test the interface that has no parameters, if possible bool noParams = ((partitioningType == HYPERGRAPH_PARTITIONING) && // default, so requires no params (numPartitions == 0) && // >0 would require a parameter (keepDenseEdges == 0)); // >0 would require a parameter // Maps for original object const Epetra_Map &sourceRowMap = matrix->RowMap(); const Epetra_Map &sourceRangeMap = matrix->RangeMap(); // const Epetra_Map &sourceColMap = matrix->ColMap(); const Epetra_Map &sourceDomainMap = matrix->DomainMap(); int numCols = matrix->NumGlobalCols(); int nMyRows = sourceRowMap.NumMyElements(); int base = sourceRowMap.IndexBase(); // Compute vertex and edge weights Isorropia::Epetra::CostDescriber costs; Teuchos::RCP<Epetra_Vector> vptr; Teuchos::RCP<Epetra_CrsMatrix> eptr; Teuchos::RCP<Epetra_Vector> hyperEdgeWeights; if (edgeWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS){ if (partitioningType == GRAPH_PARTITIONING){ // Create graph edge weights. eptr = Teuchos::rcp(new Epetra_CrsMatrix(*matrix)); if (vertexWeightType == SUPPLY_EQUAL_WEIGHTS){ eptr->PutScalar(1.0); // set all nonzeros to 1.0 } else{ int maxRowSize = eptr->MaxNumEntries(); double *newVal = NULL; if (maxRowSize > 0){ newVal = new double [maxRowSize]; for (int j=0; j<maxRowSize; j++){ newVal[j] = localProc + 1 + j; } } int numEntries; int *idx; double *val; for (int i=0; i<nMyRows; i++){ rc = eptr->ExtractMyRowView(i, numEntries, val, idx); for (int j=0; j<numEntries; j++){ val[j] = newVal[j]; } } if (newVal) delete [] newVal; } eptr->FillComplete(sourceDomainMap, sourceRangeMap); costs.setGraphEdgeWeights(eptr); } else{ // Create hyperedge weights. (Note that the list of hyperedges that a // process provides weights for has no relation to the columns // that it has non-zeroes for, or the rows that is has. Hypergraphs // in general are not square. Also more than one process can provide // a weight for the same edge. Zoltan combines the weights according // to the value of the PHG_EDGE_WEIGHT_OPERATION parameter. The default // for this parameter is to use the maximum edge weight provided by any // process for a given hyperedge.) Epetra_Map hyperEdgeMap(numCols, base, Comm); hyperEdgeWeights = Teuchos::rcp(new Epetra_Vector(hyperEdgeMap)); int *edgeGIDs = NULL; double *weights = NULL; int numHEweights = hyperEdgeMap.NumMyElements(); if (numHEweights){ edgeGIDs = new int [numHEweights]; weights = new double [numHEweights]; if (edgeWeightType == SUPPLY_EQUAL_WEIGHTS){ for (int i=0; i<numHEweights; i++){ edgeGIDs[i] = hyperEdgeMap.GID(i); weights[i] = 1.0; } } else{ int hiVolumeStart = matrix->NumGlobalCols() / 3; int hiVolumeEnd = hiVolumeStart * 2; for (int i=0; i<numHEweights; i++){ edgeGIDs[i] = hyperEdgeMap.GID(i); if ((edgeGIDs[i] < hiVolumeStart) || (edgeGIDs[i] >= hiVolumeEnd)){ weights[i] = 1.0; } else{ weights[i] = 3.0; } } } hyperEdgeWeights->ReplaceGlobalValues(numHEweights, weights, edgeGIDs); } if (weights){ delete [] weights; delete [] edgeGIDs; } costs.setHypergraphEdgeWeights(hyperEdgeWeights); } } bool need_importer = false; if ((vertexWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS)){ need_importer = true; // to redistribute row weights double *val = NULL; if (nMyRows){ val = new double [nMyRows]; if (vertexWeightType == SUPPLY_EQUAL_WEIGHTS){ for (int i=0; i<nMyRows; i++){ val[i] = 1.0; } } else if (vertexWeightType == SUPPLY_UNEQUAL_WEIGHTS){ for (int i=0; i<nMyRows; i++){ val[i] = 1.0 + ((localProc+1) / 2); } } } vptr = Teuchos::rcp(new Epetra_Vector(Copy, sourceRowMap, val)); if (val) delete [] val; costs.setVertexWeights(vptr); } // Calculate partition quality metrics before calling Zoltan if (partitioningType == GRAPH_PARTITIONING){ rc = ispatest::compute_graph_metrics(matrix->Graph(), costs, myShare, balance1, numCuts1, cutWgt1, cutn1, cutl1); if (contract){ // balance wrt target of balancing weight over *all* procs rc = ispatest::compute_graph_metrics(matrix->Graph(), costs, myShareBefore, balance3, numCuts3, cutWgt3, cutn3, cutl3); } } else{ rc = ispatest::compute_hypergraph_metrics(matrix->Graph(), costs, myShare, balance1, cutn1, cutl1); if (contract){ // balance wrt target of balancing weight over *all* procs rc = ispatest::compute_hypergraph_metrics(matrix->Graph(), costs, myShareBefore, balance3, cutn3, cutl3); } } if (rc){ ERROREXIT((localProc==0), "Error in computing partitioning metrics") } Teuchos::ParameterList params; #ifdef HAVE_ISORROPIA_ZOLTAN if (!noParams){ // We're using Zoltan for partitioning and supplying // parameters, overriding defaults. Teuchos::ParameterList &sublist = params.sublist("Zoltan"); if (partitioningType == GRAPH_PARTITIONING){ params.set("PARTITIONING METHOD", "GRAPH"); sublist.set("GRAPH_PACKAGE", "PHG"); } else{ params.set("PARTITIONING METHOD", "HYPERGRAPH"); sublist.set("LB_APPROACH", "PARTITION"); sublist.set("PHG_CUT_OBJECTIVE", "CONNECTIVITY"); // "cutl" } if (keepDenseEdges){ // only throw out rows that have no zeroes, default is to // throw out if .25 or more of the columns are non-zero sublist.set("PHG_EDGE_SIZE_THRESHOLD", "1.0"); } if (numPartitions > 0){ // test #Partitions < #Processes std::ostringstream os; os << numPartitions; std::string s = os.str(); // sublist.set("NUM_GLOBAL_PARTS", s); params.set("NUM PARTS", s); } //sublist.set("DEBUG_LEVEL", "1"); // Zoltan will print out parameters //sublist.set("DEBUG_LEVEL", "5"); // proc 0 will trace Zoltan calls //sublist.set("DEBUG_MEMORY", "2"); // Zoltan will trace alloc & free } #else ERROREXIT((localProc==0), "Zoltan partitioning required but Zoltan not available.") #endif // Function scope values Teuchos::RCP<Epetra_Vector> newvwgts; Teuchos::RCP<Epetra_CrsMatrix> newewgts; // Function scope values required for LinearProblem Epetra_LinearProblem *problem = NULL; Epetra_Map *LHSmap = NULL; Epetra_MultiVector *RHS = NULL; Epetra_MultiVector *LHS = NULL; // Reference counted pointer to balanced object Epetra_CrsMatrix *matrixPtr=NULL; Epetra_CrsGraph *graphPtr=NULL; Epetra_RowMatrix *rowMatrixPtr=NULL; Epetra_LinearProblem *problemPtr=NULL; // Row map for balanced object const Epetra_BlockMap *targetBlockRowMap=NULL; // for input CrsGraph const Epetra_Map *targetRowMap=NULL; // for all other inputs // Column map for balanced object const Epetra_BlockMap *targetBlockColMap=NULL; // for input CrsGraph const Epetra_Map *targetColMap=NULL; // for all other inputs if (objectType == EPETRA_CRSMATRIX){ if (noParams && noCosts){ matrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix); } else if (noCosts){ matrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix, params); } targetRowMap = &(matrixPtr->RowMap()); targetColMap = &(matrixPtr->ColMap()); } else if (objectType == EPETRA_CRSGRAPH){ const Epetra_CrsGraph graph = matrix->Graph(); if (noParams && noCosts){ graphPtr = Isorropia::Epetra::createBalancedCopy(graph); } else if (noCosts){ graphPtr = Isorropia::Epetra::createBalancedCopy(graph, params); } targetBlockRowMap = &(graphPtr->RowMap()); targetBlockColMap = &(graphPtr->ColMap()); } else if (objectType == EPETRA_ROWMATRIX){ if (noParams && noCosts){ rowMatrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix); } else if (noCosts){ rowMatrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix, params); } targetRowMap = &(rowMatrixPtr->RowMatrixRowMap()); targetColMap = &(rowMatrixPtr->RowMatrixColMap()); } else if (objectType == EPETRA_LINEARPROBLEM){ // Create a linear problem with this matrix. LHSmap = new Epetra_Map(numCols, base, Comm); int myRHSsize = sourceRowMap.NumMyElements(); int myLHSsize = LHSmap->NumMyElements(); int valSize = ((myRHSsize > myLHSsize) ? myRHSsize : myLHSsize); double *vals = NULL; if (valSize){ vals = new double [valSize]; } if (valSize){ for (int i=0; i < valSize; i++){ // put my rank in my portion of LHS and my portion of RHS vals[i] = localProc; } } RHS = new Epetra_MultiVector(Copy, sourceRowMap, vals, 1, 1); LHS = new Epetra_MultiVector(Copy, *LHSmap, vals, 1, 1); if (valSize){ delete [] vals; } problem = new Epetra_LinearProblem(matrix.get(), LHS, RHS); Epetra_LinearProblem lp = *problem; if (lp.CheckInput()){ ERROREXIT((localProc==0), "Error creating a LinearProblem"); } if (noParams && noCosts){ problemPtr = Isorropia::Epetra::createBalancedCopy(lp); } else if (noCosts){ problemPtr = Isorropia::Epetra::createBalancedCopy(lp, params); } targetRowMap = &(problemPtr->GetMatrix()->RowMatrixRowMap()); targetColMap = &(problemPtr->GetMatrix()->RowMatrixColMap()); } // Redistribute the edge weights // Comment this out since we don't redistribute columns if (edgeWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS){ if (partitioningType == GRAPH_PARTITIONING){ Epetra_Import *importer = NULL; if (objectType == EPETRA_CRSGRAPH){ newewgts = Teuchos::rcp(new Epetra_CrsMatrix(Copy, *graphPtr)); targetRowMap = &(newewgts->RowMap()); targetColMap = &(newewgts->ColMap()); } else{ newewgts = Teuchos::rcp(new Epetra_CrsMatrix(Copy, *targetRowMap, *targetColMap, 0)); } importer = new Epetra_Import(*targetRowMap, sourceRowMap); newewgts->Import(*eptr, *importer, Insert); newewgts->FillComplete(*targetColMap, *targetRowMap); costs.setGraphEdgeWeights(newewgts); } } // Redistribute the vertex weights if ((vertexWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS)){ Epetra_Import *importer = NULL; if (objectType == EPETRA_CRSGRAPH){ newvwgts = Teuchos::rcp(new Epetra_Vector(*targetBlockRowMap)); importer = new Epetra_Import(*targetBlockRowMap, sourceRowMap); } else{ newvwgts = Teuchos::rcp(new Epetra_Vector(*targetRowMap)); importer = new Epetra_Import(*targetRowMap, sourceRowMap); } newvwgts->Import(*vptr, *importer, Insert); costs.setVertexWeights(newvwgts); } if (localProc == 0){ test_type(numPartitions, partitioningType, vertexWeightType, edgeWeightType, objectType); } if (verbose){ // Picture of problem before balancing if (objectType == EPETRA_LINEARPROBLEM){ ispatest::show_matrix("Before load balancing", *problem, Comm); } else{ ispatest::show_matrix("Before load balancing", matrix->Graph(), Comm); } // Picture of problem after balancing if (objectType == EPETRA_LINEARPROBLEM){ ispatest::show_matrix("After load balancing (x in Ax=b is not redistributed)", *problemPtr, Comm); } else if (objectType == EPETRA_ROWMATRIX){ ispatest::show_matrix("After load balancing", *rowMatrixPtr, Comm); } else if (objectType == EPETRA_CRSMATRIX){ ispatest::show_matrix("After load balancing", matrixPtr->Graph(), Comm); } else if (objectType == EPETRA_CRSGRAPH){ ispatest::show_matrix("After load balancing", *graphPtr, Comm); } } // After partitioning, recompute the metrics if (partitioningType == GRAPH_PARTITIONING){ if (objectType == EPETRA_LINEARPROBLEM){ rc = ispatest::compute_graph_metrics(*(problemPtr->GetMatrix()), costs, myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2); } else if (objectType == EPETRA_ROWMATRIX){ rc = ispatest::compute_graph_metrics(*rowMatrixPtr, costs, myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2); } else if (objectType == EPETRA_CRSMATRIX){ rc = ispatest::compute_graph_metrics(matrixPtr->Graph(), costs, myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2); } else { rc = ispatest::compute_graph_metrics(*graphPtr, costs, myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2); } } else{ if (objectType == EPETRA_LINEARPROBLEM){ rc = ispatest::compute_hypergraph_metrics(*(problemPtr->GetMatrix()), costs, myShare, balance2, cutn2, cutl2); } else if (objectType == EPETRA_ROWMATRIX){ rc = ispatest::compute_hypergraph_metrics(*rowMatrixPtr, costs, myShare, balance2, cutn2, cutl2); } else if (objectType == EPETRA_CRSMATRIX){ rc = ispatest::compute_hypergraph_metrics(matrixPtr->Graph(), costs, myShare, balance2, cutn2, cutl2); } else{ rc = ispatest::compute_hypergraph_metrics(*graphPtr, costs, myShare, balance2, cutn2, cutl2); } } if (rc){ ERROREXIT((localProc==0), "Error in computing partitioning metrics") } std::string why; if (partitioningType == GRAPH_PARTITIONING){ fail = (cutWgt2 > cutWgt1); why = "New weighted edge cuts are worse"; if (localProc == 0){ std::cout << "Before partitioning: Balance " << balance1 ; std::cout << " cutn " << cutn1 ; std::cout << " cutl " << cutl1 ; if (contract){ std::cout << " (wrt balancing over " << numPartitions << " partitions)" << std::endl; std::cout << "Before partitioning: Balance " << balance3 ; std::cout << " cutn " << cutn3 ; std::cout << " cutl " << cutl3 ; std::cout << " (wrt balancing over " << numProcs << " partitions)" ; } std::cout << std::endl; std::cout << " Total edge cuts: " << numCuts1; std::cout << " Total weighted edge cuts: " << cutWgt1 << std::endl; std::cout << "After partitioning: Balance " << balance2 ; std::cout << " cutn " << cutn2 ; std::cout << " cutl " << cutl2 << std::endl; std::cout << " Total edge cuts: " << numCuts2; std::cout << " Total weighted edge cuts: " << cutWgt2 << std::endl; } } else{ fail = (cutl2 > cutl1); why = "New cutl is worse"; if (localProc == 0){ std::cout << "Before partitioning: Balance " << balance1 ; std::cout << " cutn " << cutn1 ; std::cout << " cutl " << cutl1 ; if (contract){ std::cout << " (wrt balancing over " << numPartitions << " partitions)" << std::endl; std::cout << "Before partitioning: Balance " << balance3 ; std::cout << " cutn " << cutn3 ; std::cout << " cutl " << cutl3 ; std::cout << " (wrt balancing over " << numProcs << " partitions)" ; } std::cout << std::endl; std::cout << "After partitioning: Balance " << balance2 ; std::cout << " cutn " << cutn2 ; std::cout << " cutl " << cutl2 << std::endl; } } if (fail){ if (localProc == 0) std::cout << "ERROR: "+why << std::endl; } // Check that input matrix is valid. This test constructs an "x" // with the matrix->DomainMap() and a "y" with matrix->RangeMap() // and then calculates y = Ax. if (objectType == EPETRA_LINEARPROBLEM){ valid = ispatest::test_matrix_vector_multiply(*problemPtr); } else if (objectType == EPETRA_ROWMATRIX){ valid = ispatest::test_row_matrix_vector_multiply(*rowMatrixPtr); } else if (objectType == EPETRA_CRSMATRIX){ valid = ispatest::test_matrix_vector_multiply(*matrixPtr); } else{ valid = ispatest::test_matrix_vector_multiply(*graphPtr); } if (!valid){ if (localProc == 0) std::cout << "Rebalanced matrix is not a valid Epetra matrix" << std::endl; fail = 1; } else{ if (localProc == 0) std::cout << "Rebalanced matrix is a valid Epetra matrix" << std::endl; } if (localProc == 0) std::cout << std::endl; #else std::cout << "test_simple main: currently can only test " << "with Epetra and EpetraExt enabled." << std::endl; rc = -1; #endif return fail; }
int main(int argc, char* argv[]) { assert(mlockall(MCL_CURRENT | MCL_FUTURE) == 0); char split = 0; uint8_t total_shares = 0, shares_required = 0; char* files[P]; uint8_t files_count = 0; char *in_file = (void*)0, *out_file_param = (void*)0; int i; while((i = getopt(argc, argv, "scn:k:f:o:i:h?")) != -1) switch(i) { case 's': if ((split & 0x2) && !(split & 0x1)) ERROREXIT("-s (split) and -c (combine) are mutually exclusive\n") else split = (0x2 | 0x1); break; case 'c': if ((split & 0x2) && (split & 0x1)) ERROREXIT("-s (split) and -c (combine) are mutually exclusive\n") else split = 0x2; break; case 'n': { int t = atoi(optarg); if (t <= 0 || t >= P) ERROREXIT("n must be > 0 and < %u\n", P) else total_shares = t; break; } case 'k': { int t = atoi(optarg); if (t <= 0 || t >= P) ERROREXIT("n must be > 0 and < %u\n", P) else shares_required = t; break; } case 'i': in_file = optarg; break; case 'o': out_file_param = optarg; break; case 'f': if (files_count >= P-1) ERROREXIT("May only specify up to %u files\n", P-1) files[files_count++] = optarg; break; case 'h': case '?': printf("Split usage: -s -n <total shares> -k <shares required> -i <input file> -o <output file path base>\n"); printf("Combine usage: -c -k <shares provided == shares required> <-f <share>>*k -o <output file>\n"); exit(0); break; default: ERROREXIT("getopt failed?\n") } if (!(split & 0x2)) ERROREXIT("Must specify one of -c, -s or -?\n") split &= 0x1; if (argc != optind) ERROREXIT("Invalid argument\n") if (split) { if (!total_shares || !shares_required) ERROREXIT("n and k must be set.\n") if (shares_required > total_shares) ERROREXIT("k must be <= n\n") if (files_count != 0 || !in_file || !out_file_param) ERROREXIT("Must specify -i <input file> and -o <output file path base> but not -f in split mode.\n") FILE* random = fopen(RAND_SOURCE, "r"); assert(random); FILE* secret_file = fopen(in_file, "r"); if (!secret_file) ERROREXIT("Could not open %s for reading.\n", in_file) uint8_t secret[MAX_LENGTH]; size_t secret_length = fread(secret, 1, MAX_LENGTH*sizeof(uint8_t), secret_file); if (secret_length == 0) ERROREXIT("Error reading secret\n") if (fread(secret, 1, 1, secret_file) > 0) ERROREXIT("Secret may not be longer than %u\n", MAX_LENGTH) fclose(secret_file); printf("Using secret of length %lu\n", secret_length); uint8_t a[shares_required], x[total_shares], D[total_shares][secret_length]; // TODO: The following loop may take a long time and eat lots of /dev/random if total_shares is high for (uint32_t i = 0; i < total_shares; i++) { int32_t j = 0; do { assert(fread(&x[i], sizeof(uint8_t), 1, random) == 1); if (x[i] == 0) continue; for (j = 0; j < i; j++) if (x[j] == x[i]) break; } while (j < i); // Inner loop will get to j = i when x[j] != x[i] for all j if (i % 32 == 31) printf("Finished picking X coordinates for %u shares\n", i+1); } for (uint32_t i = 0; i < secret_length; i++) { a[0] = secret[i]; for (uint8_t j = 1; j < shares_required; j++) assert(fread(&a[j], sizeof(uint8_t), 1, random) == 1); for (uint8_t j = 0; j < total_shares; j++) D[j][i] = calculateQ(a, shares_required, x[j]); // Now, for paranoia's sake, we ensure that no matter which piece we are missing, we can derive no information about the secret check_possible_missing_part_derivations(total_shares, shares_required, &(D[0][0]), x, i, secret_length); if (i % 32 == 31) printf("Finished processing %u bytes.\n", i+1); } char out_file_name_buf[strlen(out_file_param) + 4]; strcpy(out_file_name_buf, out_file_param); for (uint8_t i = 0; i < total_shares; i++) { sprintf(((char*)out_file_name_buf) + strlen(out_file_param), "%u", i); FILE* out_file = fopen(out_file_name_buf, "w+"); if (!out_file) ERROREXIT("Could not open output file %s\n", out_file_name_buf) if (fwrite(&x[i], sizeof(uint8_t), 1, out_file) != 1) ERROREXIT("Could not write 1 byte to %s\n", out_file_name_buf) if (fwrite(D[i], 1, secret_length, out_file) != secret_length) ERROREXIT("Could not write %lu bytes to %s\n", secret_length, out_file_name_buf) fclose(out_file); } // Clear sensitive data (No, GCC 4.7.2 is currently not optimizing this out) memset(secret, 0, sizeof(uint8_t)*secret_length); memset(a, 0, sizeof(uint8_t)*shares_required); memset(x, 0, sizeof(uint8_t)*total_shares); memset(in_file, 0, strlen(in_file)); fclose(random); } else { if (!shares_required)