Ejemplo n.º 1
0
int main (int argc, char *argv[])
{

	int	taskid;              // a task identifier 
	int	numtasks;              // number of tasks in partition
	MPI_Comm comm;
	int m; // size of the matrix
	int	local_m;                  // rows of matrix A sent to each worker 
	double *A, *b,*exact_x, *x;
	double *temp_1, *temp_2;
	double *local_A,  *local_v,*local_u;
	double *local_M; // M is the preconditioner in this example, which is the diagonal element of A;
	int i,j,k;


	MPI_Init(&argc,&argv);
	comm=MPI_COMM_WORLD;
	MPI_Comm_rank(comm,&taskid);
	MPI_Comm_size(comm,&numtasks);
	if(taskid==MASTER){ // initilization: A and b
		/* start modification 1: read A and b from mtx files in node 0 */
		////m=64; // size of the matrix
		////A=(double *)malloc(sizeof(double)*(m*m));
		////// !!! A is in col-major
		////for(j=0;j<m;j++)
		////	for (i=0;i<m;i++){
		////		if(i==j)
		////			*(A+j*m+i)=m*100.0;
		////		else
		////			*(A+j*m+i)=i+1.0;
		////	}
		////exact_x=(double *)malloc(sizeof(double)*m);
		////for (i=0;i<m;i++)
		////	*(exact_x+i)=1.0;
		////b=(double *)malloc(sizeof(double)*m);
		////// b=A*ones(n,1)
		////cblas_dgemv(CblasColMajor, CblasNoTrans, m, m, 1.0, A, m, exact_x, 1, 0.0, b, 1); 	
		/////* end modification 1*/
	  mtxBLU mtxA(argv[1], NPROC);
    mtxBLU mtxb(argv[2]);
    m=mtxA.m;
    A = (double*)malloc(sizeof(double)*(m*m));
    b = (double*)malloc(sizeof(double)*m);
    for(i=0; i<m*m; i++)
      A[i]=mtxA.arr[i];
    for(i=0; i<m; i++)
      b[i]=mtxb.arr[i];
  }

	MPI_Bcast(&m,1,MPI_INT, MASTER, comm); // send m from node MASTER to all other nodes.
	local_m=m/numtasks;
	local_A=(double *)malloc(sizeof(double)*(local_m*m));
	local_u=(double *)malloc(sizeof(double)*(local_m));
	local_v=(double *)malloc(sizeof(double)*m);
	//	partition A and send A_i to local_A on  node i
	MPI_Scatter(A,local_m*m, MPI_DOUBLE, local_A, local_m*m, MPI_DOUBLE, MASTER, comm);

	if(taskid==MASTER){
		free(A);
		//free(exact_x);
		// do not free b, it wil be used  for GMRES
	}


	/* start modification 2: generate preconditioner M
	 * In this example, TA choose the diagonal elements of A as the preconditioner.
	 * In HW3 part b, you should generate L and U here.
	 */
	////local_M=(double *)malloc(sizeof(double)*local_m);
	////for(i=0;i<local_m;i++)
	////	*(local_M+i)=*(local_A+taskid*local_m+i*m+i);
  int mm=int( m/numtasks);
  double *U,*L,*UU,*LL;
  U = new double[numtasks*mm*mm]; //block col domain
  L = new double[numtasks*mm*mm]; //block col domain
  UU= new double[numtasks*mm*mm]; //full col domain
  LL= new double[numtasks*mm*mm]; //full col domain
  std::fill(L, L+numtasks*mm*mm, 0);
  std::fill(U, U+numtasks*mm*mm, 0);
  for(i=0; i<mm; i++)
    L[taskid*mm*mm+i+i*mm]=1;  //make Lii = eyes(mm)
  
  for(i=0; i<numtasks*mm*mm; i++) //backup A into U
    U[i]=local_A[i];

  blockLU(L, U, mm, taskid);
  for(k=0; k<numtasks; k++)
    for(j=0; j<mm; j++)
      for(i=0; i<mm; i++){
        UU[k*mm+i+j*mm*numtasks]= U[k*mm*mm+j*mm+i];
        LL[k*mm+i+j*mm*numtasks]= L[k*mm*mm+j*mm+i];
      }

   
  /* end  modification 2*/


	/*---------------------------------------------------------------------------
	 * GMRES: Allocate storage for the ?par parameters and the solution vectors
	 *---------------------------------------------------------------------------*/
	MKL_INT RCI_request;
	int RCI_flag;
	double dvar;
	int flag=0;

	MKL_INT ipar[128]; //specifies the integer set of data for the RCI FGMRES computations
	double dpar[128]; // specifies the double precision set of data
	double *tmp; //used to supply the double precision temporary space for theRCI FGMRES computations, specifically:
	double *computed_solution;
	double *residual;
	double *f;
	MKL_INT itercount, ierr=0;;
	MKL_INT  ivar;
	double b_2norm;
	char cvar='N';
	MKL_INT incx=1;
	if (taskid==MASTER){
		ipar[14]=RESTART; // restart iteration number
		int n_tmp = (2 * ipar[14] + 1) * m + ipar[14] * (ipar[14] + 9) / 2 + 1;
		tmp=(double *)malloc(sizeof(double)*n_tmp);
		computed_solution=(double *)malloc(sizeof(double)*m);
		residual=(double *)malloc(sizeof(double)*m);
		f=(double *)malloc(sizeof(double)*m);

		ivar=m;
		/*---------------------------------------------------------------------------
		 * Initialize the initial guess
		 *---------------------------------------------------------------------------*/
		for (i = 0; i < m; i++)
		{
			computed_solution[i] = 0.5;
		}

		b_2norm = cblas_dnrm2 (ivar, b, incx);
	//	printf("b_2norm=%f\n",b_2norm);
		/*---------------------------------------------------------------------------
		 * Initialize the solver
		 *---------------------------------------------------------------------------*/
		dfgmres_init (&ivar, computed_solution,b, &RCI_request, ipar, dpar, tmp);
		RCI_flag=RCI_request;
	}
	MPI_Bcast(&RCI_flag,1,MPI_INT, MASTER, comm);
	if (RCI_flag != 0)
		goto FAILED;

	if(taskid==MASTER){
		/*---------------------------------------------------------------------------
		 * GMRES: Set the desired parameters:
		 *---------------------------------------------------------------------------*/
		ipar[14] = RESTART; // restart iteration number
		ipar[7] = 1; //do the stopping test
		ipar[10] = 1; // use preconditioner
		dpar[0] = TOL;
		/*---------------------------------------------------------------------------
		 * Check the correctness and consistency of the newly set parameters
		 *---------------------------------------------------------------------------*/
		dfgmres_check (&ivar, computed_solution, b, &RCI_request, ipar, dpar,
				tmp);
		RCI_flag=RCI_request;
	}

	MPI_Bcast(&RCI_flag,1,MPI_INT, MASTER, comm);
	if (RCI_flag != 0)
		goto FAILED;

	if (taskid==MASTER){
		/*---------------------------------------------------------------------------
		 * Print the info about the RCI FGMRES method
		 *---------------------------------------------------------------------------*/
		printf ("Some info about the current run of RCI FGMRES method:\n\n");
		if (ipar[7])
		{
			printf ("As ipar[7]=%d, the automatic test for the maximal number of ", ipar[7]);
			printf ("iterations will be\nperformed\n");
		}
		else
		{
			printf ("As ipar[7]=%d, the automatic test for the maximal number of ", ipar[7]);
			printf ("iterations will be\nskipped\n");
		}
		printf ("+++\n");
		if (ipar[8])
		{
			printf ("As ipar[8]=%d, the automatic residual test will be performed\n", ipar[8]);
		}
		else
		{
			printf ("As ipar[8]=%d, the automatic residual test will be skipped\n", ipar[8]);
		}
		printf ("+++\n");
		if (ipar[9])
		{
			printf ("As ipar[9]=%d, the user-defined stopping test will be ", ipar[9]);
			printf ("requested via\nRCI_request=2\n");
		}
		else
		{
			printf ("As ipar[9]=%d, the user-defined stopping test will not be ", ipar[9]);
			printf ("requested, thus,\nRCI_request will not take the value 2\n");
		}
		printf ("+++\n");
		if (ipar[10])
		{
			printf ("As ipar[10]=%d, the Preconditioned FGMRES iterations will be ", ipar[10]);
			printf ("performed, thus,\nthe preconditioner action will be requested via ");
			printf ("RCI_request=3\n");
		}
		else
		{
			printf ("As ipar[10]=%d, the Preconditioned FGMRES iterations will not ", ipar[10]);
			printf ("be performed,\nthus, RCI_request will not take the value 3\n");
		}
		printf ("+++\n");
		if (ipar[11])
		{
			printf ("As ipar[11]=%d, the automatic test for the norm of the next ", ipar[11]);
			printf ("generated vector is\nnot equal to zero up to rounding and ");
			printf ("computational errors will be performed,\nthus, RCI_request will not ");
			printf ("take the value 4\n");
		}
		else
		{
			printf ("As ipar[11]=%d, the automatic test for the norm of the next ", ipar[11]);
			printf ("generated vector is\nnot equal to zero up to rounding and ");
			printf ("computational errors will be skipped,\nthus, the user-defined test ");
			printf ("will be requested via RCI_request=4\n");
		}
		printf ("+++\n\n");
	}
	/*---------------------------------------------------------------------------
	 * Compute the solution by RCI (P)FGMRES solver with preconditioning
	 * Reverse Communication starts here
	 *---------------------------------------------------------------------------*/
ONE:
	if(taskid==MASTER){
		dfgmres (&ivar, computed_solution,b, &RCI_request, ipar, dpar, tmp);
		RCI_flag=RCI_request;  
	}
	MPI_Bcast(&RCI_flag,1,MPI_INT, MASTER, comm); // send RCI_request from node MASTER to all other nodes.
	
	/*---------------------------------------------------------------------------
	 * If RCI_request=0, then the solution was found with the required precision
	 *---------------------------------------------------------------------------*/
	if (RCI_flag == 0)
		goto COMPLETE;
	/*---------------------------------------------------------------------------
	 * If RCI_request=1, then compute the vector A*tmp[ipar[21]-1]
	 * and put the result in vector tmp[ipar[22]-1]
	 *---------------------------------------------------------------------------
	 * NOTE that ipar[21] and ipar[22] contain FORTRAN style addresses,
	 * therefore, in C code it is required to subtract 1 from them to get C style
	 * addresses
	 *---------------------------------------------------------------------------*/
	if (RCI_flag == 1)
	{
		if (taskid==MASTER){
			temp_1=&tmp[ipar[21] - 1];
			temp_2=&tmp[ipar[22] - 1];
		}

		mpi_dgemv(m,local_m,local_A,temp_1, temp_2,local_u,local_v,taskid, comm);

		goto ONE;
	}
	/*---------------------------------------------------------------------------
	 * If RCI_request=2, then do the user-defined stopping test
	 * The residual stopping test for the computed solution is performed here
	 *---------------------------------------------------------------------------
	 */
	if (RCI_flag == 2)
	{
		/* Request to the dfgmres_get routine to put the solution into b[N] via ipar[12]
		   --------------------------------------------------------------------------------
WARNING: beware that the call to dfgmres_get routine with ipar[12]=0 at this
stage may destroy the convergence of the FGMRES method, therefore, only
advanced users should exploit this option with care */
		if (taskid==MASTER){
			ipar[12] = 1;
			/* Get the current FGMRES solution in the vector f */
			dfgmres_get (&ivar, computed_solution, f, &RCI_request, ipar, dpar, tmp, &itercount);
			temp_1=f;
			temp_2=residual;
		} 
		/* Compute the current true residual via mpi mat_vec multiplication */
		mpi_dgemv(m,local_m,local_A,temp_1,temp_2,local_u,local_v,taskid, comm);

		if(taskid==MASTER){
			dvar = -1.0E0;
			cblas_daxpy (ivar, dvar, b, incx, residual, incx);
			dvar = cblas_dnrm2 (ivar, residual, incx);
			printf("iteration %d, relative residual:%e\n",itercount, dvar);
		}

		MPI_Bcast(&dvar,1,MPI_DOUBLE, MASTER, comm); 
		if (dvar < TOL){
			goto COMPLETE;
		}
		else
			goto ONE;
	}
	/*---------------------------------------------------------------------------
	 * If RCI_request=3, then apply the preconditioner on the vector
	 * tmp[ipar[21]-1] and put the result in vector tmp[ipar[22]-1]
	 *---------------------------------------------------------------------------
	 * NOTE that ipar[21] and ipar[22] contain FORTRAN style addresses,
	 * therefore, in C code it is required to subtract 1 from them to get C style
	 * addresses
	 *---------------------------------------------------------------------------*/
	if (RCI_flag == 3)
	{
		if (taskid==MASTER){
			temp_1=&tmp[ipar[21] - 1];
			temp_2=&tmp[ipar[22] - 1];
		}
		/* start modification 3: solve L U temp_2 = temp_1   */
		////mpi_preconditioner_solver(m,local_m,local_M,temp_1, temp_2,local_u,taskid,comm);
	  //method1:
    //double *rhs = (double*) malloc( sizeof(double)*m);
    //for(i=0; i<m; i++)
    //  rhs[i]=temp_1[i];
    //method2:
    for(i=0; i<m; i++)
      temp_2[i]=temp_1[i];
    LUSolver(LL,UU, temp_2, m, mm, taskid, NPROC);	
    //free(rhs);
    
    /* end modification 3 */
		goto ONE;
	}
	/*---------------------------------------------------------------------------
	 * If RCI_request=4, then check if the norm of the next generated vector is
	 * not zero up to rounding and computational errors. The norm is contained
	 * in dpar[6] parameter
	 *---------------------------------------------------------------------------*/
	if (RCI_flag == 4)	
	{
		if(taskid==MASTER)
			dvar=dpar[6];
		MPI_Bcast(&dvar,1,MPI_DOUBLE, MASTER, comm); 
		if (dvar <1.0E-12 ){
			goto COMPLETE;
		}
		else
			goto ONE;
	}
	/*---------------------------------------------------------------------------
	 * If RCI_request=anything else, then dfgmres subroutine failed
	 * to compute the solution vector: computed_solution[N]
	 *---------------------------------------------------------------------------*/
	else
	{
		goto FAILED;
	}
	/*---------------------------------------------------------------------------
	 * Reverse Communication ends here
	 * Get the current iteration number and the FGMRES solution (DO NOT FORGET to
	 * call dfgmres_get routine as computed_solution is still containing
	 * the initial guess!). Request to dfgmres_get to put the solution
	 * into vector computed_solution[N] via ipar[12]
	 *---------------------------------------------------------------------------*/
COMPLETE:if(taskid==MASTER){
			 ipar[12] = 0;
			 dfgmres_get (&ivar, computed_solution,b, &RCI_request, ipar, dpar, tmp, &itercount);
			 /*---------------------------------------------------------------------------
			  * Print solution vector: computed_solution[N] and the number of iterations: itercount
			  *---------------------------------------------------------------------------*/
			 printf ("The system has been solved in %d iterations \n", itercount);
			 printf ("The following solution has been obtained (first 4 elements): \n");
			 for (i = 0; i < 4; i++)
			 {
				 printf ("computed_solution[%d]=", i);
				 printf ("%e\n", computed_solution[i]);
			 }

			 /*-------------------------------------------------------------------------*/
			 /* Release internal MKL memory that might be used for computations         */
			 /* NOTE: It is important to call the routine below to avoid memory leaks   */
			 /* unless you disable MKL Memory Manager                                   */
			 /*-------------------------------------------------------------------------*/
			 MKL_Free_Buffers ();
			 temp_1=computed_solution;
			 temp_2=residual;
		 }
		// compute the relative residual
		 mpi_dgemv(m,local_m,local_A,temp_1,temp_2,local_u,local_v,taskid, comm);
		 if(taskid==MASTER){
			 dvar = -1.0E0;
			 cblas_daxpy (ivar, dvar, b, incx, residual, incx);
			 dvar = cblas_dnrm2 (ivar, residual, incx);
			 printf("relative residual:%e\n",dvar/b_2norm);

			 if(itercount<MAXIT && dvar<TOL)
				 flag=0; //success
			 else 
				 flag=1; //fail

		 }

		 MPI_Bcast(&flag,1,MPI_INT, MASTER, comm); 
		
		free(local_A);
		free(local_M);
		free(local_u);
		free(local_v);
		if(taskid==MASTER){
			free(tmp);
			free(b);
			free(computed_solution);
			free(residual);
		}
		
		if(flag==0){
			 MPI_Finalize();
			 return 0;
		 }
		 else{
			 MPI_Finalize();
			 return 1;
		 }
		 /* Release internal MKL memory that might be used for computations         */
		 /* NOTE: It is important to call the routine below to avoid memory leaks   */
		 /* unless you disable MKL Memory Manager                                   */
		 /*-------------------------------------------------------------------------*/
FAILED:
		 if(taskid==MASTER){
			 printf ("\nThis example FAILED as the solver has returned the ERROR code %d", RCI_request);
			 MKL_Free_Buffers ();
		 }
		free(local_A);
		free(local_M);
		free(local_u);
		free(local_v);
		if(taskid==MASTER){
			free(tmp);
			free(b);
			free(computed_solution);
			free(residual);
		}

		 MPI_Finalize();
		 return 1;





}
Ejemplo n.º 2
0
void cMKLSolver::step(MatrixX1C &solvec, MatrixX1C &rhsvec) {
    // variables required by gmres
    MKL_INT RCI_request;
    MKL_INT itercount;
    MKL_INT ivar = size;
    MKL_INT ipar[128];
    double dpar[128];
    
    // pointers to vectors
    double *solution = solvec.data();
    double *rhs = rhsvec.data();
    
    // zero initial guess
    for (int i = 0; i < size; i++) {
        solution[i] = 0.0;
    }
    
    // initialise gmres
    dfgmres_init(&ivar, solution, rhs, &RCI_request, ipar, dpar, gmres_tmp);
    if (RCI_request != 0) {
        fatal_error("initialising gmres failed!");
    }
    
    // set desired parameters
    ipar[7] = 1;  // perform maximum iterations test
    ipar[8] = 1;  // perform residual stopping test
    ipar[9] = 0;  // do not perform the user defined stopping test
    ipar[10] = 1;  // run preconditioned gmres
    ipar[14] = gmres_restarts;  // how often to restart gmres
    dpar[0] = gmres_relative_tol;  // relative tolerance
    dpar[1] = gmres_absolute_tol;  // absolute tolerance
    
    // check correctness of parameters
    dfgmres_check(&ivar, solution, rhs, &RCI_request, ipar, dpar, gmres_tmp);
    if (RCI_request != 0) {
        fatal_error("param check failed!");
    }
    
    // start gmres reverse communication
    dfgmres(&ivar, solution, rhs, &RCI_request, ipar, dpar, gmres_tmp);
    bool complete = false;
    while (not complete) {
        // success
        if (RCI_request == 0) {
            complete = true;
        }
        // compute matrix vector multiplication
        else if (RCI_request == 1) {
            // compute gmres_tmp[ipar[22]-1] = A*gmres_tmp[ipar[21]-1]
            // note: ipar[21] and ipar[22] contain fortran style addresses so we must subtract 1
            char cvar = 'N';
            mkl_dcsrgemv(&cvar, &ivar, Acsr, Ai, Aj, &gmres_tmp[ipar[21] - 1], &gmres_tmp[ipar[22] - 1]);
        }
        // apply the preconditioner
        else if (RCI_request == 3) {
            char cvar = 'N';
            char cvar1 = 'L';
            char cvar2 = 'U';
            mkl_dcsrtrsv(&cvar1, &cvar, &cvar2, &ivar, bilut, ibilut, jbilut, &gmres_tmp[ipar[21] - 1], gmres_trvec);
            cvar1='U';
    		cvar='N';
    		cvar2='N';
    		mkl_dcsrtrsv(&cvar1, &cvar, &cvar2, &ivar, bilut, ibilut, jbilut, gmres_trvec, &gmres_tmp[ipar[22] - 1]);
        }
        // check norm of generated vector
        else if (RCI_request == 4) {
            if (dpar[6] < 1.e-12) {
                complete = true;
            }
        }
        // failed
        else {
            std::ostringstream msgstream;
            msgstream << "fgmres failed: RCI_request " << RCI_request << "!";
            fatal_error(msgstream.str());
        }
        
        // next gmres call
        if (not complete) {
            dfgmres(&ivar, solution, rhs, &RCI_request, ipar, dpar, gmres_tmp);
        }
    }
    
    // get the result and print iteration count
    ipar[12] = 0;
    dfgmres_get(&ivar, solution, rhs, &RCI_request, ipar, dpar, gmres_tmp, &itercount);
    std::cout << itercount << " " << dpar[4] << std::endl;
}
Ejemplo n.º 3
0
int main( int argc, char **argv ) 
{
    /* Matrix data. */
    double *aa;
    int N;
    int *ia, *ja;

    read_mtx_and_return_csr(argc, argv, &N, &N, &ia, &ja, &aa);
    printf("\nDone with reading mtx file.\n");
    printf("m = %d, n = %d, nz = %d\n\n", N, N, ia[N]);
	
        /*---------------------------------------------------------------------------
	/* Allocate storage for the ?par parameters and the solution/rhs vectors
	/*---------------------------------------------------------------------------*/

    MKL_INT ipar[size];
    double dpar[size]; 
    double *tmp = (double *)malloc((N * (2 * N + 1) + (N * (N + 9)) / 2 + 1) * sizeof(double));
    double rhs[N];
    double computed_solution[N];

        /*---------------------------------------------------------------------------
	/* Some additional variables to use with the RCI (P)FGMRES solver
	/*---------------------------------------------------------------------------*/
    
    MKL_INT itercount;
    MKL_INT RCI_request, i, ivar;
    double dvar;
    char cvar;

	/*---------------------------------------------------------------------------
	/* Initialize variables and the right hand side through matrix-vector product
	/*---------------------------------------------------------------------------*/
  ivar = N;
  cvar = 'N';
	/*---------------------------------------------------------------------------
	/* Initialize the initial guess
	/*---------------------------------------------------------------------------*/
  for (i = 0; i < N; i++)
    {
      computed_solution[i] = 1.0;
    }
	/*---------------------------------------------------------------------------
	/* Initialize the solver
	/*---------------------------------------------------------------------------*/
  dfgmres_init (&ivar, computed_solution, rhs, &RCI_request, ipar, dpar, tmp);
  if (RCI_request != 0) {
    printf("Going to FAILED\n");
    goto FAILED;
  }
	/*---------------------------------------------------------------------------
	/* Set the desired parameters:
	/* LOGICAL parameters:
	/* do residual stopping test
	/* do not request for the user defined stopping test
	/* do the check of the norm of the next generated vector automatically
	/* DOUBLE PRECISION parameters
	/* set the relative tolerance to 1.0D-3 instead of default value 1.0D-6
	/*---------------------------------------------------------------------------*/
  ipar[7] = 0;
  ipar[8] = 1;
  ipar[9] = 0;
  ipar[11] = 1;
  dpar[0] = 1.0E-3;
	/*---------------------------------------------------------------------------
	/* Check the correctness and consistency of the newly set parameters
	/*---------------------------------------------------------------------------*/
  dfgmres_check (&ivar, computed_solution, rhs, &RCI_request, ipar, dpar,
		 tmp);
  if (RCI_request != 0) {
    printf("Going to FAILED\n");
    goto FAILED;
  }
	/*---------------------------------------------------------------------------
	/* Print the info about the RCI FGMRES method
	/*---------------------------------------------------------------------------*/
  if (INFO == 1) {
  printf ("Some info about the current run of RCI FGMRES method:\n\n");
  if (ipar[7])
    {
      printf ("As ipar[7]=%d, the automatic test for the maximal number of ",
	      ipar[7]);
      printf ("iterations will be\nperformed\n");
    }
  else
    {
      printf ("As ipar[7]=%d, the automatic test for the maximal number of ",
	      ipar[7]);
      printf ("iterations will be\nskipped\n");
    }
  printf ("+++\n");
  if (ipar[8])
    {
      printf
	("As ipar[8]=%d, the automatic residual test will be performed\n",
	 ipar[8]);
    }
  else
    {
      printf ("As ipar[8]=%d, the automatic residual test will be skipped\n",
	      ipar[8]);
    }
  printf ("+++\n");
  if (ipar[9])
    {
      printf ("As ipar[9]=%d, the user-defined stopping test will be ",
	      ipar[9]);
      printf ("requested via\nRCI_request=2\n");
    }
  else
    {
      printf ("As ipar[9]=%d, the user-defined stopping test will not be ",
	      ipar[9]);
      printf ("requested, thus,\nRCI_request will not take the value 2\n");
    }
  printf ("+++\n");
  if (ipar[10])
    {
      printf ("As ipar[10]=%d, the Preconditioned FGMRES iterations will be ",
	      ipar[10]);
      printf
	("performed, thus,\nthe preconditioner action will be requested via");
      printf ("RCI_request=3\n");
    }
  else
    {
      printf
	("As ipar[10]=%d, the Preconditioned FGMRES iterations will not ",
	 ipar[10]);
      printf ("be performed,\nthus, RCI_request will not take the value 3\n");
    }
  printf ("+++\n");
  if (ipar[11])
    {
      printf ("As ipar[11]=%d, the automatic test for the norm of the next ",
	      ipar[11]);
      printf ("generated vector is\nnot equal to zero up to rounding and ");
      printf
	("computational errors will be performed,\nthus, RCI_request will not ");
      printf ("take the value 4\n");
    }
  else
    {
      printf ("As ipar[11]=%d, the automatic test for the norm of the next ",
	      ipar[11]);
      printf ("generated vector is\nnot equal to zero up to rounding and ");
      printf
	("computational errors will be skipped,\nthus, the user-defined test ");
      printf ("will be requested via RCI_request=4\n");
    }
  printf ("+++\n\n");
  }
	/*---------------------------------------------------------------------------
	/* Compute the solution by RCI (P)FGMRES solver without preconditioning
	/* Reverse Communication starts here
	/*---------------------------------------------------------------------------*/
ONE:dfgmres (&ivar, computed_solution, rhs, &RCI_request, ipar, dpar, tmp);
	/*---------------------------------------------------------------------------
	/* If RCI_request=0, then the solution was found with the required precision
	/*---------------------------------------------------------------------------*/
  if (RCI_request == 0) {
    printf("RCI_request = %d\n", RCI_request);
    printf("Going to COMPLETE\n");
    goto COMPLETE;
  }
	/*---------------------------------------------------------------------------
	/* If RCI_request=1, then compute the vector A*tmp[ipar[21]-1]
	/* and put the result in vector tmp[ipar[22]-1]
	/*---------------------------------------------------------------------------
	/* NOTE that ipar[21] and ipar[22] contain FORTRAN style addresses,
	/* therefore, in C code it is required to subtract 1 from them to get C style
	/* addresses
	/*---------------------------------------------------------------------------*/
  if (RCI_request == 1)
    {
      printf("RCI_request = %d\n", RCI_request);
      mkl_dcsrgemv (&cvar, &ivar, aa, ia, ja, &tmp[ipar[21] - 1],
		    &tmp[ipar[22] - 1]);
      printf("Going to ONE\n");
      goto ONE;
    }
	/*---------------------------------------------------------------------------
	/* If RCI_request=anything else, then dfgmres subroutine failed
	/* to compute the solution vector: computed_solution[N]
	/*---------------------------------------------------------------------------*/
  else
    {
    printf("Going to FAILED\n");
      goto FAILED;
    }
	/*---------------------------------------------------------------------------
	/* Reverse Communication ends here
	/* Get the current iteration number and the FGMRES solution (DO NOT FORGET to
	/* call dfgmres_get routine as computed_solution is still containing
	/* the initial guess!)
	/*---------------------------------------------------------------------------*/
COMPLETE:dfgmres_get (&ivar, computed_solution, rhs, &RCI_request, ipar, dpar, tmp,
	       &itercount);
  /*
     /*---------------------------------------------------------------------------
     /* Print solution vector: computed_solution[N] and the number of iterations:
     itercount
     /*--------------------------------------------------------------------------- */
  printf (" The system has been solved \n");
  printf ("\n The following solution has been obtained: \n");
  /*
  for (i = 0; i < N; i++)
    {
      printf ("computed_solution[%d]=", i);
      printf ("%e\n", computed_solution[i]);
    }
    */
  printf ("\n Number of iterations: %d\n", itercount);
  i = 1;

	/*-------------------------------------------------------------------------*/
  /* Release internal MKL memory that might be used for computations         */
  /* NOTE: It is important to call the routine below to avoid memory leaks   */
  /* unless you disable MKL Memory Manager                                   */
	/*-------------------------------------------------------------------------*/
  MKL_Free_Buffers ();
  return 0;

  /*if (itercount == expected_itercount && dvar < 1.0e-14)
    {
      printf ("\nThis example has successfully PASSED through all steps of ");
      printf ("computation!\n");
      return 0;
    }
  else
    {
      printf
	("\nThis example may have FAILED as either the number of iterations ");
      printf ("differs\nfrom the expected number of iterations %d, ",
	      expected_itercount);
      printf
	("or the computed solution\ndiffers much from the expected solution ");
      printf ("(Euclidean norm is %e), or both.\n", dvar);
      return 1;
    }*/
	/*-------------------------------------------------------------------------*/
  /* Release internal MKL memory that might be used for computations         */
  /* NOTE: It is important to call the routine below to avoid memory leaks   */
  /* unless you disable MKL Memory Manager                                   */
	/*-------------------------------------------------------------------------*/
FAILED:printf
    ("\nThis example FAILED as the solver has returned the ERROR ");
  printf ("code %d", RCI_request);
  MKL_Free_Buffers ();


    free (ia);
    free (ja);
    free (aa);
    free (tmp);

    printf("\nMemory deallocated...\n");

    return 0;
}