int readSymmMatrix(char* filename, int& n, double*& v)
{
int m;
// try opening the files
FILE *fp;
if ((fp = fopen(filename, "r")) == NULL) {
fprintf(stderr, "Error occurs while reading from file %s.\n", filename);
return 1;
}
// try reading the banner
MM_typecode type;
if (mm_read_banner(fp, &type) != 0) {
fprintf(stderr, "Could not process Matrix Market banner.\n");
return 2;
}
// check the type
if (!mm_is_matrix(type) || !mm_is_array(type) || !mm_is_real(type) || !mm_is_symmetric(type)) {
fprintf(stderr, "Sorry, this application does not support Market Market type: [%s]\n",
mm_typecode_to_str(type));
return 3;
}
// read the sizes of the vectors
if (mm_read_mtx_array_size(fp, &m, &n)) {
fprintf(stderr, "Could not read the size of the matrix.\n");
return 4;
}
// check if it is a square matrix
if (n != m) {
fprintf(stderr, "Needs to be square.\n");
return 5;
}
// allocate the memory
printf("reading %s:\n\ta %d x %d matrix...", filename, m, n);
v = new double[m * n];
for (int j = 0; j < n; ++j) {
for (int i = j; i < m; ++i) {
fscanf(fp, "%lf\n", &v[i * n + j]);
if (i != j) {
v[j * n + i] = v[i * n + j];
}
}
}
printf("done\n");
// close the file
fclose(fp);
return 0;
}
void ReadMatrixMarketVector(std::string filename, T* &rhs)
{
/* Open the file */
FILE *f = fopen(filename.c_str(), "r");
if (f == NULL)
{
std::cout<<"failed to open file "<<filename<<std::endl;
}
/* Read the banner */
MM_typecode matcode;
if (mm_read_banner(f, &matcode) != 0)
{
std::cout<<"failed to read banner in file "<<filename<<std::endl;
}
/* Determine the size */
int status, nrows, ncols;
if ((status = mm_read_mtx_array_size(f, &nrows, &ncols)) != 0)
{
std::cout<<"failed to read number of rows and columns in file "<<filename<<std::endl;
}
if (ncols != 1)
{
fclose(f);
std::cout<<"number of columns is not 1 in file "<<filename<<std::endl;
}
/* Read values into vector */
rhs = new T[nrows];
int nval;
double val;
for (unsigned int n = 0; n < (unsigned int)nrows; n++)
{
nval = fscanf(f, "%lg\n", &val);
if (nval < 1)
{
std::cout<<"failed to read dense line from file "<<filename<<std::endl;
}
rhs[n] = (T)val;
}
/* Close file and return */
fclose(f);
return;
}
/** Reads the dimensions of the matrix (n - number of rows, m - number
of columns, nnzs - number of non-zeros) from the given Matrix
Market file. If the given file contains an array rather than a
COO matrix, nnzs will be set to n;
*/
void read_mm_matrix_size(FILE *f, int *n, int *m, int *nnzs, MM_typecode* mcode) {
if (mm_read_banner(f, mcode) != 0) {
printf("Could not process Matrix Market banner.\n");
exit(1);
}
if (mm_is_array(*mcode)) {
mm_read_mtx_array_size(f, n, m);
*nnzs = *n;
} else {
mm_read_mtx_crd_size(f, n, m, nnzs);
}
}
/** load a matrix market file into a matrix */
void load_matrix_market_matrix(const std::string & filename, int offset, int D){
MM_typecode matcode;
uint i,I,J;
double val;
uint rows, cols;
size_t nnz;
FILE * f = open_file(filename.c_str() ,"r");
int rc = mm_read_banner(f, &matcode);
if (rc != 0)
logstream(LOG_FATAL)<<"Failed to load matrix market banner in file: " << filename << std::endl;
if (mm_is_sparse(matcode)){
int rc = mm_read_mtx_crd_size(f, &rows, &cols, &nnz);
if (rc != 0)
logstream(LOG_FATAL)<<"Failed to load matrix market banner in file: " << filename << std::endl;
}
else { //dense matrix
rc = mm_read_mtx_array_size(f, &rows, &cols);
if (rc != 0)
logstream(LOG_FATAL)<<"Failed to load matrix market banner in file: " << filename << std::endl;
nnz = rows * cols;
}
if (D != (int)cols)
logstream(LOG_FATAL)<<"Wrong matrix size detected, command line argument should be --D=" << D << " instead of : " << cols << std::endl;
for (i=0; i<nnz; i++){
if (mm_is_sparse(matcode)){
rc = fscanf(f, "%u %u %lg\n", &I, &J, &val);
if (rc != 3)
logstream(LOG_FATAL)<<"Error reading input line " << i << std::endl;
I--; J--;
assert(I >= 0 && I < rows);
assert(J >= 0 && J < cols);
//set_val(a, I, J, val);
latent_factors_inmem[I+offset].set_val(J,val);
}
else {
rc = fscanf(f, "%lg", &val);
if (rc != 1)
logstream(LOG_FATAL)<<"Error reading nnz " << i << std::endl;
I = i / cols;
J = i % cols;
latent_factors_inmem[I+offset].set_val(J, val);
}
}
logstream(LOG_INFO) << "Factors from file: loaded matrix of size " << rows << " x " << cols << " from file: " << filename << " total of " << nnz << " entries. "<< i << std::endl;
fclose(f);
}
int DenseMatrix_mm_read_strassen(DenseMatrix *m, char *file_name, int *nr_rows, int *nr_cols) {
int res;
MM_typecode matcode;
FILE *file;
file = fopen(file_name, "r");
CHECK_NULL_RETURN(file);
res = mm_read_banner(file, &matcode);
CHECK_ZERO_ERROR_RETURN(res, "Failed to read matrix code");
CHECK_ERROR_RETURN(!mm_is_matrix(matcode), "File is not a matrix", 1);
if (mm_is_sparse(matcode)) {
int nr_sparse_elements;
res = mm_read_mtx_crd_size(file, nr_rows, nr_cols, &nr_sparse_elements);
CHECK_ZERO_ERROR_RETURN(res, "Failed to read sparse mm dimensions");
int dim = pow2dim(*nr_rows, *nr_cols);
// Initialzie matrix to zero to fill in with sparse elements
res = DenseMatrix_init_zero(m, dim, dim);
CHECK_ZERO_ERROR_RETURN(res, "Failed to allocate memory for mm matrix");
res = DenseMatrix_parse_mm_sparse(m, file, nr_sparse_elements);
} else if (mm_is_dense(matcode)) {
res = mm_read_mtx_array_size(file, nr_rows, nr_cols);
CHECK_ZERO_ERROR_RETURN(res, "Failed to read dense mm dimensions");
int dim = pow2dim(*nr_rows, *nr_cols);
res = DenseMatrix_init_zero(m, dim, dim);
CHECK_ZERO_ERROR_RETURN(res, "Failed to allocate memory for mm matrix");
res = DenseMatrix_parse_mm_dense(m, file);
} else {
ERROR("mm matrix code is not supported. Only supports dense and sparse matrices");
}
CHECK_ZERO_ERROR_RETURN(res, "Failed to parse mm file");
return 0;
}
/* fread dense */
static mm_dense *
mm_real_fread_dense (FILE *fp, MM_typecode typecode)
{
int k;
int m, n;
int ret;
mm_dense *d;
if (mm_read_mtx_array_size (fp, &m, &n) != 0) return NULL;
d = mm_real_new (MM_REAL_DENSE, MM_REAL_GENERAL, m, n, m * n);
k = 0;
do {
ret = fscanf (fp, "%lf", &d->data[k]);
if (ret > 0 && ++k >= d->nnz) break;
} while (ret != EOF);
return d;
}
int get_mm_info(const char *file, int *m, int *n, int *nz)
{
FILE *fp;
MM_typecode matcode;
if ((fp = fopen(file, "r")) == NULL)
{
fprintf(stderr,"ERROR: Could not open file: %s\n",file);
exit(1);
}
if (mm_read_banner(fp, &matcode) != 0)
{
fprintf(stderr,"ERROR: Could not process Matrix Market banner.\n");
exit(1);
}
if (!(mm_is_real(matcode) || mm_is_integer(matcode)))
{
fprintf(stderr,"ERROR: Market Market type: [%s] not supported\n",
mm_typecode_to_str(matcode));
exit(1);
}
if (mm_is_sparse(matcode))
{
if (mm_read_mtx_crd_size(fp, m, n, nz) !=0)
{ /* find out size of sparse matrix */
exit(1);
}
}
else
{
if (mm_read_mtx_array_size(fp, m, n) !=0)
{ /* find out size of dense matrix */
exit(1);
}
*nz = -1;
}
fclose(fp);
return 0;
}
int MatrixMarketFileToRowMap(const char* filename,
const Epetra_Comm& comm,
Epetra_BlockMap*& rowmap)
{
FILE* infile = fopen(filename, "r");
MM_typecode matcode;
int err = mm_read_banner(infile, &matcode);
if (err != 0) return(err);
if (!mm_is_matrix(matcode) || !mm_is_coordinate(matcode) ||
!mm_is_real(matcode) || !mm_is_general(matcode)) {
return(-1);
}
int numrows, numcols;
err = mm_read_mtx_array_size(infile, &numrows, &numcols);
if (err != 0) return(err);
fclose(infile);
rowmap = new Epetra_BlockMap(numrows, 1, 0, comm);
return(0);
}
vec load_matrix_market_vector(const std::string & filename, bool optional_field, bool allow_zeros)
{
int ret_code;
MM_typecode matcode;
uint M, N;
size_t i,nz;
logstream(LOG_INFO) <<"Going to read matrix market vector from input file: " << filename << std::endl;
FILE * f = open_file(filename.c_str(), "r", optional_field);
//if optional file not found return
if (f== NULL && optional_field){
return zeros(1);
}
if (mm_read_banner(f, &matcode) != 0)
logstream(LOG_FATAL) << "Could not process Matrix Market banner." << std::endl;
/* This is how one can screen matrix types if their application */
/* only supports a subset of the Matrix Market data types. */
if (mm_is_complex(matcode) && mm_is_matrix(matcode) &&
mm_is_sparse(matcode) )
logstream(LOG_FATAL) << "sorry, this application does not support " << std::endl <<
"Market Market type: " << mm_typecode_to_str(matcode) << std::endl;
/* find out size of sparse matrix .... */
if (mm_is_sparse(matcode)){
if ((ret_code = mm_read_mtx_crd_size(f, &M, &N, &nz)) !=0)
logstream(LOG_FATAL) << "failed to read matrix market cardinality size " << std::endl;
}
else {
if ((ret_code = mm_read_mtx_array_size(f, &M, &N))!= 0)
logstream(LOG_FATAL) << "failed to read matrix market vector size " << std::endl;
if (N > M){ //if this is a row vector, transpose
int tmp = N;
N = M;
M = tmp;
}
nz = M*N;
}
vec ret = zeros(M);
uint row,col;
double val;
for (i=0; i<nz; i++)
{
if (mm_is_sparse(matcode)){
int rc = fscanf(f, "%u %u %lg\n", &row, &col, &val);
if (rc != 3){
logstream(LOG_FATAL) << "Failed reading input file: " << filename << "Problm at data row " << i << " (not including header and comment lines)" << std::endl;
}
row--; /* adjust from 1-based to 0-based */
col--;
}
else {
int rc = fscanf(f, "%lg\n", &val);
if (rc != 1){
logstream(LOG_FATAL) << "Failed reading input file: " << filename << "Problm at data row " << i << " (not including header and comment lines)" << std::endl;
}
row = i;
col = 0;
}
//some users have gibrish in text file - better check both I and J are >=0 as well
assert(row >=0 && row< M);
assert(col == 0);
if (val == 0 && !allow_zeros)
logstream(LOG_FATAL)<<"Zero entries are not allowed in a sparse matrix market vector. Use --zero=true to avoid this error"<<std::endl;
//set observation value
ret[row] = val;
}
fclose(f);
logstream(LOG_INFO)<<"Succesfully read a vector of size: " << M << " [ " << nz << "]" << std::endl;
return ret;
}
int main(int argc, char **argv) {
const int MAX_ITER = 20;
const double TOL = 1e-12;
int rank;
int size;
int P = 8; // number of blocks to update P <= size
/* -----------------------------------
mode controls the selection schemes,
mode =0, fixed P
mode =1, dynamic update P
----------------------------------*/
int mode=1; // number of processors used to update each time
double lambda = 0.1;
srand (time(NULL));
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank); // Determine current running process
MPI_Comm_size(MPI_COMM_WORLD, &size); // Total number of processes
// data directory (you need to change the path to your own data directory)
char* dataCenterDir = "../Data/Gaussian";
char* big_dir;
if(argc==2)
big_dir = argv[1];
else
big_dir = "big1";
/* Read in local data */
FILE *f, *test;
int m, n, j;
int row, col;
double entry, startTime, endTime;
double total_start_time, total_end_time;
/*
* Subsystem n will look for files called An.dat and bn.dat
* in the current directory; these are its local data and do not need to be
* visible to any other processes. Note that
* m and n here refer to the dimensions of the *local* coefficient matrix.
*/
/* ------------
Read in A
------------*/
if(rank ==0){
printf("=============================\n");
printf("| Start to load data! |\n");
printf("=============================\n");
}
char s[100];
sprintf(s, "%s/%s/A%d.dat",dataCenterDir,big_dir, rank + 1);
printf("[%d] reading %s\n", rank, s);
f = fopen(s, "r");
if (f == NULL) {
printf("[%d] ERROR: %s does not exist, exiting.\n", rank, s);
exit(EXIT_FAILURE);
}
mm_read_mtx_array_size(f, &m, &n);
gsl_matrix *A = gsl_matrix_calloc(m, n);
for (int i = 0; i < m*n; i++) {
row = i % m;
col = floor(i/m);
fscanf(f, "%lf", &entry);
gsl_matrix_set(A, row, col, entry);
}
fclose(f);
/* ------------
Read in b
-------------*/
sprintf(s, "%s/%s/b.dat", dataCenterDir, big_dir);
printf("[%d] reading %s\n", rank, s);
f = fopen(s, "r");
if (f == NULL) {
printf("[%d] ERROR: %s does not exist, exiting.\n", rank, s);
exit(EXIT_FAILURE);
}
mm_read_mtx_array_size(f, &m, &n);
gsl_vector *b = gsl_vector_calloc(m);
for (int i = 0; i < m; i++) {
fscanf(f, "%lf", &entry);
gsl_vector_set(b, i, entry);
}
fclose(f);
/* ------------
Read in xs
------------*/
sprintf(s, "%s/%s/xs%d.dat", dataCenterDir, big_dir, rank + 1);
printf("[%d] reading %s\n", rank, s);
f = fopen(s, "r");
if (f == NULL) {
printf("[%d] ERROR: %s does not exist, exiting.\n", rank, s);
exit(EXIT_FAILURE);
}
mm_read_mtx_array_size(f, &m, &n);
gsl_vector *xs = gsl_vector_calloc(m);
for (int i = 0; i < m; i++) {
fscanf(f, "%lf", &entry);
gsl_vector_set(xs, i, entry);
}
fclose(f);
m = A->size1;
n = A->size2;
MPI_Barrier(MPI_COMM_WORLD);
/*----------------------------------------
* These are all variables related to GRock
----------------------------------------*/
struct value table[size];
gsl_vector *x = gsl_vector_calloc(n);
gsl_vector *As = gsl_vector_calloc(n);
gsl_vector *invAs = gsl_vector_calloc(n);
gsl_vector *local_b = gsl_vector_calloc(m);
gsl_vector *beta = gsl_vector_calloc(n);
gsl_vector *tmp = gsl_vector_calloc(n);
gsl_vector *d = gsl_vector_calloc(n);
gsl_vector *absd = gsl_vector_calloc(n);
gsl_vector *oldx = gsl_vector_calloc(n);
gsl_vector *tmpx = gsl_vector_calloc(n);
gsl_vector *z = gsl_vector_calloc(m);
gsl_vector *tmpz = gsl_vector_calloc(m);
gsl_vector *Ax = gsl_vector_calloc(m);
gsl_vector *Atmpx = gsl_vector_calloc(m);
gsl_vector *xdiff = gsl_vector_calloc(n);
gsl_permutation *idx = gsl_permutation_calloc(n);
double send[1];
double recv[1];
double err;
int num_upd = (int)(n*0.08);
double sigma = 0.01;
double xs_local_nrm[1], xs_nrm[1];
double local_old_obj, global_old_obj, local_new_obj, global_new_obj;
//calculate the 2 norm of xs
xs_local_nrm[0] = gsl_blas_dnrm2(xs);
xs_local_nrm[0] *=xs_local_nrm[0];
MPI_Allreduce(xs_local_nrm, xs_nrm, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
xs_nrm[0] = sqrt(xs_nrm[0]);
// evaluate the two norm of the columns of A
for(j=0;j<n;j++){
gsl_vector_view column = gsl_matrix_column(A, j);
double d;
d = gsl_blas_dnrm2(&column.vector);
gsl_vector_set(As, j, d*d);
gsl_vector_set(invAs, j, 1./(d*d));
}
if (rank == 0) {
printf("=============================\n");
printf("|GRock start to solve Lasso!|\n");
printf("|---------------------------|\n");
printf("|lambda=%1.2f, m=%d, n=%d |\n", lambda, m, n*size);
if(mode==1) printf("| Mode: dynamic update P. |\n");
else printf("| Mode: fixed update P |\n");
printf("=============================\n");
printf("%3s %8s %8s %5s\n", "iter", "rel_err", "obj", "P");
startTime = MPI_Wtime();
sprintf(s, "results/test%d.m", size);
test = fopen(s, "w");
fprintf(test,"res = [ \n");
}
/* Main BCD loop */
total_start_time = MPI_Wtime();
int iter = 0;
while (iter < MAX_ITER) {
startTime = MPI_Wtime();
/*---------- restore the old x ------------*/
gsl_vector_memcpy(oldx, x);
/*------- calculate local_b = b - sum_{j \neq i} Aj*xj--------- */
gsl_blas_dgemv(CblasNoTrans, 1, A, x, 0, Ax); // Ax = A * x
MPI_Allreduce(Ax->data, z->data, m, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
gsl_vector_sub(z, b); // z = Ax - b
gsl_vector_memcpy(local_b, Ax);
gsl_vector_sub(local_b, z);
/* -------calculate beta ------------------*/
gsl_blas_dgemv(CblasTrans, -1, A, z, 0, beta); // beta = A'(b - Ax) + ||A.s||^2 * xs
gsl_vector_memcpy(tmp, As);
pointwise(tmp, x, n);
gsl_vector_add(beta, tmp);
shrink(beta, lambda);
// x = 1/|xs|^2 * shrink(beta, lambda)
gsl_vector_memcpy(x, beta);
pointwise(x, invAs, n);
/* ------calcuate proposed decrease -------- */
gsl_vector_memcpy(d,x);
gsl_vector_sub(d, oldx);
if(mode ==1){
gsl_vector_memcpy(absd, d);
abs_vector(absd, n);
// sort the local array d
gsl_vector_scale(absd, -1.0);
gsl_sort_vector_index(idx, absd);
// printf("|d(0)| = %lf, |d(1)| = %lf \n", gsl_vector_get(absd,0), gsl_vector_get(absd, 3));
// calculate current objective value;
local_old_obj = objective(oldx, lambda, z, size);
MPI_Allreduce(&local_old_obj, &global_old_obj, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
num_upd = fmin(num_upd+1, (int)(0.1*n));
gsl_vector_memcpy(tmpx, oldx);
int upd_idx;
double local_delta = 0, delta=0.0;
for(int i=0; i<num_upd; i++){
upd_idx = gsl_permutation_get(idx, i);
// printf("%d\n", upd_idx);
gsl_vector_set(tmpx, upd_idx, gsl_vector_get(x, upd_idx));
local_delta += gsl_vector_get(d, upd_idx) * gsl_vector_get(d, upd_idx);
}
MPI_Allreduce(&local_delta, &delta, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
gsl_blas_dgemv(CblasNoTrans, 1, A, tmpx, 0, Atmpx); // Ax = A * x
MPI_Allreduce(Atmpx->data, tmpz->data, m, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
gsl_vector_sub(tmpz, b); // z = Ax - b
local_new_obj = objective(tmpx, lambda, tmpz, size);
MPI_Allreduce(&local_new_obj, &global_new_obj, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
while(global_new_obj - global_old_obj> -sigma * delta){
num_upd = fmax(num_upd-1, 1);
for(int i=0; i<num_upd; i++){
upd_idx = gsl_permutation_get(idx, i);
gsl_vector_set(tmpx, upd_idx, gsl_vector_get(x, upd_idx));
local_delta += gsl_vector_get(d, upd_idx) * gsl_vector_get(d, upd_idx);
}
MPI_Allreduce(&delta, &local_delta, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
gsl_blas_dgemv(CblasNoTrans, 1, A, tmpx, 0, Atmpx); // Ax = A * x
MPI_Allreduce(Atmpx->data, tmpz->data, m, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
gsl_vector_sub(tmpz, b); // z = Ax - b
local_new_obj = objective(tmpx, lambda, tmpz, size);
MPI_Allreduce(&local_new_obj, &global_new_obj, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
if(num_upd==1)
break;
}
gsl_vector_memcpy(x, tmpx);
}
if(mode==0){
CBLAS_INDEX_t id = gsl_blas_idamax(d);
double *store = (double*)calloc(size, sizeof(double));
double foo[1];
foo[0] = gsl_vector_get(d,id);
MPI_Allgather(foo, 1, MPI_DOUBLE, store, 1, MPI_DOUBLE, MPI_COMM_WORLD);
for(int i=0;i<size;i++){
table[i].ID = i;
table[i].data = fabs(store[i]);
}
// quick sort to decide which block to update
qsort((void *) & table, size, sizeof(struct value), (compfn)compare );
gsl_vector_memcpy(x, oldx);
if(size>P){
for(int i=0;i<P;i++){
if(rank == table[i].ID)
gsl_vector_set(x, id, gsl_vector_get(oldx, id) + gsl_vector_get(d, id));
}
}else
gsl_vector_set(x, id, gsl_vector_get(oldx, id) + gsl_vector_get(d, id));
local_new_obj = objective(x, lambda, z, size);
MPI_Allreduce(&local_new_obj, &global_new_obj, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
}
/*------------------------------
calculate the relative error
------------------------------*/
gsl_vector_memcpy(xdiff,xs);
gsl_vector_sub(xdiff, x);
err = gsl_blas_dnrm2(xdiff);
send[0] = err*err;
MPI_Allreduce(send, recv, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
recv[0] = sqrt(recv[0])/xs_nrm[0];
endTime = MPI_Wtime();
if(mode==1) P = num_upd*size;
if (rank == 0) {
if(iter%5 == 0)
printf("%3d %10.2e %10.4f %3d\n", iter,
recv[0], global_new_obj, P);
fprintf(test, "%e \n",recv[0]);
}
/* termination check */
if(recv[0] < TOL){
break;
}
iter++;
}
total_end_time = MPI_Wtime();
/* Have the master write out the results to disk */
if (rank == 0) {
printf("=============================\n");
printf("| GRock solved Lasso! |\n");
printf("|---------------------------|\n");
printf("|Summary: |\n");
printf("| # of iteration: %d |\n", iter);
printf("| relative error: %4.2e|\n", recv[0]);
printf("| objective value: %4.2f |\n", global_new_obj);
printf("| time: %4.1es|\n", total_end_time - total_start_time);
printf("=============================\n");
fprintf(test,"] \n");
fprintf(test,"semilogy(1:length(res),res); \n");
fprintf(test,"xlabel('# of iteration'); ylabel('||x - xs||');\n");
fclose(test);
f = fopen("results/solution.dat", "w");
fprintf(f,"x = [ \n");
gsl_vector_fprintf(f, x, "%lf");
fprintf(f,"] \n");
fclose(f);
endTime = MPI_Wtime();
}
MPI_Finalize(); /* Shut down the MPI execution environment */
/* Clear memory */
gsl_matrix_free(A);
gsl_vector_free(b);
gsl_vector_free(x);
gsl_vector_free(z);
gsl_vector_free(xdiff);
gsl_vector_free(Ax);
gsl_vector_free(As);
gsl_vector_free(invAs);
gsl_vector_free(tmpx);
gsl_vector_free(oldx);
gsl_vector_free(local_b);
gsl_vector_free(beta);
gsl_vector_free(tmpz);
gsl_vector_free(absd);
gsl_vector_free(Atmpx);
gsl_permutation_free(idx);
return 0;
}
int main(int argc, char **argv) {
const int MAX_ITER = 50;
const double RELTOL = 1e-2;
const double ABSTOL = 1e-4;
/*
* Some bookkeeping variables for MPI. The 'rank' of a process is its numeric id
* in the process pool. For example, if we run a program via `mpirun -np 4 foo', then
* the process ranks are 0 through 3. Here, N and size are the total number of processes
* running (in this example, 4).
*/
int rank;
int size;
MPI_Init(&argc, &argv); // Initialize the MPI execution environment
MPI_Comm_rank(MPI_COMM_WORLD, &rank); // Determine current running process
MPI_Comm_size(MPI_COMM_WORLD, &size); // Total number of processes
double N = (double) size; // Number of subsystems/slaves for ADMM
/* Read in local data */
int skinny; // A flag indicating whether the matrix A is fat or skinny
FILE *f;
int m, n;
int row, col;
double entry;
/*
* Subsystem n will look for files called An.dat and bn.dat
* in the current directory; these are its local data and do not need to be
* visible to any other processes. Note that
* m and n here refer to the dimensions of the *local* coefficient matrix.
*/
/* Read A */
char s[20];
sprintf(s, "data/A%d.dat", rank + 1);
printf("[%d] reading %s\n", rank, s);
f = fopen(s, "r");
if (f == NULL) {
printf("[%d] ERROR: %s does not exist, exiting.\n", rank, s);
exit(EXIT_FAILURE);
}
mm_read_mtx_array_size(f, &m, &n);
gsl_matrix *A = gsl_matrix_calloc(m, n);
for (int i = 0; i < m*n; i++) {
row = i % m;
col = floor(i/m);
fscanf(f, "%lf", &entry);
gsl_matrix_set(A, row, col, entry);
}
fclose(f);
/* Read b */
sprintf(s, "data/b%d.dat", rank + 1);
printf("[%d] reading %s\n", rank, s);
f = fopen(s, "r");
if (f == NULL) {
printf("[%d] ERROR: %s does not exist, exiting.\n", rank, s);
exit(EXIT_FAILURE);
}
mm_read_mtx_array_size(f, &m, &n);
gsl_vector *b = gsl_vector_calloc(m);
for (int i = 0; i < m; i++) {
fscanf(f, "%lf", &entry);
gsl_vector_set(b, i, entry);
}
fclose(f);
m = A->size1;
n = A->size2;
skinny = (m >= n);
/*
* These are all variables related to ADMM itself. There are many
* more variables than in the Matlab implementation because we also
* require vectors and matrices to store various intermediate results.
* The naming scheme follows the Matlab version of this solver.
*/
double rho = 1.0;
gsl_vector *x = gsl_vector_calloc(n);
gsl_vector *u = gsl_vector_calloc(n);
gsl_vector *z = gsl_vector_calloc(n);
gsl_vector *y = gsl_vector_calloc(n);
gsl_vector *r = gsl_vector_calloc(n);
gsl_vector *zprev = gsl_vector_calloc(n);
gsl_vector *zdiff = gsl_vector_calloc(n);
gsl_vector *q = gsl_vector_calloc(n);
gsl_vector *w = gsl_vector_calloc(n);
gsl_vector *Aq = gsl_vector_calloc(m);
gsl_vector *p = gsl_vector_calloc(m);
gsl_vector *Atb = gsl_vector_calloc(n);
double send[3]; // an array used to aggregate 3 scalars at once
double recv[3]; // used to receive the results of these aggregations
double nxstack = 0;
double nystack = 0;
double prires = 0;
double dualres = 0;
double eps_pri = 0;
double eps_dual = 0;
/* Precompute and cache factorizations */
gsl_blas_dgemv(CblasTrans, 1, A, b, 0, Atb); // Atb = A^T b
/*
* The lasso regularization parameter here is just hardcoded
* to 0.5 for simplicity. Using the lambda_max heuristic would require
* network communication, since it requires looking at the *global* A^T b.
*/
double lambda = 0.5;
if (rank == 0) {
printf("using lambda: %.4f\n", lambda);
}
gsl_matrix *L;
/* Use the matrix inversion lemma for efficiency; see section 4.2 of the paper */
if (skinny) {
/* L = chol(AtA + rho*I) */
L = gsl_matrix_calloc(n,n);
gsl_matrix *AtA = gsl_matrix_calloc(n,n);
gsl_blas_dsyrk(CblasLower, CblasTrans, 1, A, 0, AtA);
gsl_matrix *rhoI = gsl_matrix_calloc(n,n);
gsl_matrix_set_identity(rhoI);
gsl_matrix_scale(rhoI, rho);
gsl_matrix_memcpy(L, AtA);
gsl_matrix_add(L, rhoI);
gsl_linalg_cholesky_decomp(L);
gsl_matrix_free(AtA);
gsl_matrix_free(rhoI);
} else {
/* L = chol(I + 1/rho*AAt) */
L = gsl_matrix_calloc(m,m);
gsl_matrix *AAt = gsl_matrix_calloc(m,m);
gsl_blas_dsyrk(CblasLower, CblasNoTrans, 1, A, 0, AAt);
gsl_matrix_scale(AAt, 1/rho);
gsl_matrix *eye = gsl_matrix_calloc(m,m);
gsl_matrix_set_identity(eye);
gsl_matrix_memcpy(L, AAt);
gsl_matrix_add(L, eye);
gsl_linalg_cholesky_decomp(L);
gsl_matrix_free(AAt);
gsl_matrix_free(eye);
}
/* Main ADMM solver loop */
int iter = 0;
if (rank == 0) {
printf("%3s %10s %10s %10s %10s %10s\n", "#", "r norm", "eps_pri", "s norm", "eps_dual", "objective");
}
double startAllTime, endAllTime;
startAllTime = MPI_Wtime();
while (iter < MAX_ITER) {
/* u-update: u = u + x - z */
gsl_vector_sub(x, z);
gsl_vector_add(u, x);
/* x-update: x = (A^T A + rho I) \ (A^T b + rho z - y) */
gsl_vector_memcpy(q, z);
gsl_vector_sub(q, u);
gsl_vector_scale(q, rho);
gsl_vector_add(q, Atb); // q = A^T b + rho*(z - u)
double tmp, tmpq;
gsl_blas_ddot(x, x, &tmp);
gsl_blas_ddot(q, q, &tmpq);
if (skinny) {
/* x = U \ (L \ q) */
gsl_linalg_cholesky_solve(L, q, x);
} else {
/* x = q/rho - 1/rho^2 * A^T * (U \ (L \ (A*q))) */
gsl_blas_dgemv(CblasNoTrans, 1, A, q, 0, Aq);
gsl_linalg_cholesky_solve(L, Aq, p);
gsl_blas_dgemv(CblasTrans, 1, A, p, 0, x); /* now x = A^T * (U \ (L \ (A*q)) */
gsl_vector_scale(x, -1/(rho*rho));
gsl_vector_scale(q, 1/rho);
gsl_vector_add(x, q);
}
/*
* Message-passing: compute the global sum over all processors of the
* contents of w and t. Also, update z.
*/
gsl_vector_memcpy(w, x);
gsl_vector_add(w, u); // w = x + u
gsl_blas_ddot(r, r, &send[0]);
gsl_blas_ddot(x, x, &send[1]);
gsl_blas_ddot(u, u, &send[2]);
send[2] /= pow(rho, 2);
gsl_vector_memcpy(zprev, z);
// could be reduced to a single Allreduce call by concatenating send to w
MPI_Allreduce(w->data, z->data, n, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
MPI_Allreduce(send, recv, 3, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
prires = sqrt(recv[0]); /* sqrt(sum ||r_i||_2^2) */
nxstack = sqrt(recv[1]); /* sqrt(sum ||x_i||_2^2) */
nystack = sqrt(recv[2]); /* sqrt(sum ||y_i||_2^2) */
gsl_vector_scale(z, 1/N);
soft_threshold(z, lambda/(N*rho));
/* Termination checks */
/* dual residual */
gsl_vector_memcpy(zdiff, z);
gsl_vector_sub(zdiff, zprev);
dualres = sqrt(N) * rho * gsl_blas_dnrm2(zdiff); /* ||s^k||_2^2 = N rho^2 ||z - zprev||_2^2 */
/* compute primal and dual feasibility tolerances */
eps_pri = sqrt(n*N)*ABSTOL + RELTOL * fmax(nxstack, sqrt(N)*gsl_blas_dnrm2(z));
eps_dual = sqrt(n*N)*ABSTOL + RELTOL * nystack;
if (rank == 0) {
printf("%3d %10.4f %10.4f %10.4f %10.4f %10.4f\n", iter,
prires, eps_pri, dualres, eps_dual, objective(A, b, lambda, z));
}
if (prires <= eps_pri && dualres <= eps_dual) {
break;
}
/* Compute residual: r = x - z */
gsl_vector_memcpy(r, x);
gsl_vector_sub(r, z);
iter++;
}
/* Have the master write out the results to disk */
if (rank == 0) {
endAllTime = MPI_Wtime();
printf("Elapsed time is: %lf \n", endAllTime - startAllTime);
f = fopen("data/solution.dat", "w");
gsl_vector_fprintf(f, z, "%lf");
fclose(f);
}
MPI_Finalize(); /* Shut down the MPI execution environment */
/* Clear memory */
gsl_matrix_free(A);
gsl_matrix_free(L);
gsl_vector_free(b);
gsl_vector_free(x);
gsl_vector_free(u);
gsl_vector_free(z);
gsl_vector_free(y);
gsl_vector_free(r);
gsl_vector_free(w);
gsl_vector_free(zprev);
gsl_vector_free(zdiff);
gsl_vector_free(q);
gsl_vector_free(Aq);
gsl_vector_free(Atb);
gsl_vector_free(p);
return EXIT_SUCCESS;
}
/** \brief Read a vector file.
*
* Reads a Matrix Market formatted vector from a file.
* Returns pointer to dense vector.
*
* @param file vector file name
* @param out_vec vector data
*
* @return result
*/
int read_mm_new_vector(const char *file, double **out_vec)
{
double *val;
int i, m, n, nz;
FILE *fp;
MM_typecode matcode;
if ((fp = fopen(file, "r")) == NULL)
{
fprintf(stderr,"ERROR: Could not open file: %s\n",file);
exit(1);
}
if (mm_read_banner(fp, &matcode) != 0)
{
fprintf(stderr,"ERROR: Could not process Matrix Market banner.\n");
exit(1);
}
if (!(mm_is_real(matcode) || mm_is_integer(matcode)))
{
fprintf(stderr,"ERROR: Market Market type: [%s] not supported\n",
mm_typecode_to_str(matcode));
exit(1);
}
if (mm_is_sparse(matcode))
{
if (mm_read_mtx_crd_size(fp, &m, &n, &nz) !=0)
{ /* find out size of sparse matrix */
exit(1);
}
if (n != 1)
{
fprintf(stderr,"ERROR: %s does not contain a column vector\n",file);
exit(1);
}
val = malloc(m*sizeof(double));
memset(val, 0, m*sizeof(double));
for (i = 0; i < nz; i++)
{
int cur_i, cur_j;
double cur_val;
fscanf(fp, "%d %d %lg\n", &cur_i, &cur_j, &cur_val);
val[cur_i-1] = cur_val;
}
}
else
{
if (mm_read_mtx_array_size(fp, &m, &n) !=0)
{ /* find out size of dense matrix */
exit(1);
}
if (n != 1)
{
fprintf(stderr,"ERROR: %s does not contain a column vector\n",file);
exit(1);
}
val = malloc(m*sizeof(double));
for (i = 0; i < m; i++)
{
double cur_val;
fscanf(fp, "%lg\n", &cur_val);
val[i] = cur_val;
}
}
*out_vec = val;
if (fp !=stdin) fclose(fp);
return 0;
}
int read_mm_new_matrix_transpose(const char *file, dmatrix **out_mat)
{
dmatrix *dmat;
int i, j, m, n, nz;
FILE *fp;
MM_typecode matcode;
if ((fp = fopen(file, "r")) == NULL)
{
fprintf(stderr,"ERROR: Could not open file: %s\n",file);
exit(1);
}
if (mm_read_banner(fp, &matcode) != 0)
{
fprintf(stderr,"ERROR: Could not process Matrix Market banner.\n");
exit(1);
}
if (!(mm_is_real(matcode) || mm_is_integer(matcode)))
{
fprintf(stderr,"ERROR: Market Market type: [%s] not supported\n",
mm_typecode_to_str(matcode));
exit(1);
}
if (mm_is_sparse(matcode))
{
int cur_i, cur_j;
double cur_val;
double *val, *vtmp;
int *idx, *jdx, *rdx;
int *itmp, *jtmp, *tmp;
if (mm_read_mtx_crd_size(fp, &n, &m, &nz) !=0)
{ /* find out size of sparse matrix */
exit(1);
}
dmat = malloc(sizeof(dmatrix));
itmp = malloc(nz*sizeof(int));
jtmp = malloc(nz*sizeof(int));
vtmp = malloc(nz*sizeof(double));
rdx = malloc((m+1)*sizeof(int));
idx = malloc(nz*sizeof(int));
jdx = malloc(nz*sizeof(int));
val = malloc(nz*sizeof(double));
tmp = malloc(m*sizeof(int));
memset(tmp, 0, sizeof(int)*m);
for (i = 0; i < nz; i++)
{
fscanf(fp, "%d %d %lg\n", &cur_j, &cur_i, &cur_val);
itmp[i] = --cur_i;
jtmp[i] = --cur_j;
vtmp[i] = cur_val;
tmp[ itmp[i] ]++ ;
}
rdx[0] = 0;
for (i = 0; i < m ; i++)
{
rdx[i+1] = rdx[i] + tmp[i];
tmp[i] = rdx[i];
}
for (i = 0; i < nz; i++)
{
int ii;
ii = tmp[ itmp[i] ]++;
idx[ii] = itmp[i];
jdx[ii] = jtmp[i];
val[ii] = vtmp[i];
}
dmat->n = n;
dmat->m = m;
dmat->nz = nz;
dmat->val = val;
dmat->idx = idx;
dmat->jdx = jdx;
dmat->rdx = rdx;
free(itmp);
free(jtmp);
free(vtmp);
free(tmp);
}
else
{
double *val;
if (mm_read_mtx_array_size(fp, &n, &m) !=0)
{ /* find out size of dense matrix */
exit(1);
}
dmat = malloc(sizeof(dmatrix));
val = malloc((m*n)*sizeof(double));
for (j = 0; j < n*m; j++)
{
fscanf(fp, "%lg\n", &val[j]);
}
dmat->n = n;
dmat->m = m;
dmat->nz = -1;
dmat->val = val;
dmat->idx = NULL;
dmat->jdx = NULL;
dmat->rdx = NULL;
}
*out_mat = dmat;
if (fp !=stdin) fclose(fp);
return 0;
}
