static void check_output(void)
{
/* compute C = C - AB */
CPU_GEMM("N", "N", ydim, xdim, zdim, (TYPE)-1.0f, A, ydim, B, zdim, (TYPE)1.0f, C, ydim);
/* make sure C = 0 */
TYPE err;
err = CPU_ASUM(xdim*ydim, C, 1);
if (err < xdim*ydim*0.001)
{
FPRINTF(stderr, "Results are OK\n");
}
else
{
int max;
max = CPU_IAMAX(xdim*ydim, C, 1);
FPRINTF(stderr, "There were errors ... err = %f\n", err);
FPRINTF(stderr, "Max error : %e\n", C[max]);
}
}
void STARPU_PLU(compute_lu_matrix)(unsigned size, unsigned nblocks, TYPE *Asaved)
{
TYPE *all_r = STARPU_PLU(reconstruct_matrix)(size, nblocks);
unsigned display = STARPU_PLU(display_flag)();
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (rank == 0)
{
TYPE *L = malloc((size_t)size*size*sizeof(TYPE));
TYPE *U = malloc((size_t)size*size*sizeof(TYPE));
memset(L, 0, size*size*sizeof(TYPE));
memset(U, 0, size*size*sizeof(TYPE));
/* only keep the lower part */
unsigned i, j;
for (j = 0; j < size; j++)
{
for (i = 0; i < j; i++)
{
L[j+i*size] = all_r[j+i*size];
}
/* diag i = j */
L[j+j*size] = all_r[j+j*size];
U[j+j*size] = 1.0;
for (i = j+1; i < size; i++)
{
U[j+i*size] = all_r[j+i*size];
}
}
STARPU_PLU(display_data_content)(L, size);
STARPU_PLU(display_data_content)(U, size);
/* now A_err = L, compute L*U */
CPU_TRMM("R", "U", "N", "U", size, size, 1.0f, U, size, L, size);
if (display)
fprintf(stderr, "\nLU\n");
STARPU_PLU(display_data_content)(L, size);
/* compute "LU - A" in L*/
CPU_AXPY(size*size, -1.0, Asaved, 1, L, 1);
TYPE err = CPU_ASUM(size*size, L, 1);
int max = CPU_IAMAX(size*size, L, 1);
if (display)
fprintf(stderr, "DISPLAY ERROR\n");
STARPU_PLU(display_data_content)(L, size);
fprintf(stderr, "(A - LU) Avg error : %e\n", err/(size*size));
fprintf(stderr, "(A - LU) Max error : %e\n", L[max]);
double residual = frobenius_norm(L, size);
double matnorm = frobenius_norm(Asaved, size);
fprintf(stderr, "||A-LU|| / (||A||*N) : %e\n", residual/(matnorm*size));
}
}
int main(int argc, char **argv)
{
int rank;
int world_size;
/*
* Initialization
*/
int thread_support;
if (MPI_Init_thread(&argc, &argv, MPI_THREAD_SERIALIZED, &thread_support) != MPI_SUCCESS) {
fprintf(stderr,"MPI_Init_thread failed\n");
exit(1);
}
if (thread_support == MPI_THREAD_FUNNELED)
fprintf(stderr,"Warning: MPI only has funneled thread support, not serialized, hoping this will work\n");
if (thread_support < MPI_THREAD_FUNNELED)
fprintf(stderr,"Warning: MPI does not have thread support!\n");
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &world_size);
starpu_srand48((long int)time(NULL));
parse_args(rank, argc, argv);
int ret = starpu_init(NULL);
STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
/* We disable sequential consistency in this example */
starpu_data_set_default_sequential_consistency_flag(0);
starpu_mpi_init(NULL, NULL, 0);
STARPU_ASSERT(p*q == world_size);
starpu_cublas_init();
int barrier_ret = MPI_Barrier(MPI_COMM_WORLD);
STARPU_ASSERT(barrier_ret == MPI_SUCCESS);
/*
* Problem Init
*/
init_matrix(rank);
fprintf(stderr, "Rank %d: allocated (%d + %d) MB = %d MB\n", rank,
(int)(allocated_memory/(1024*1024)),
(int)(allocated_memory_extra/(1024*1024)),
(int)((allocated_memory+allocated_memory_extra)/(1024*1024)));
display_grid(rank, nblocks);
TYPE *a_r = NULL;
// STARPU_PLU(display_data_content)(a_r, size);
TYPE *x, *y;
if (check)
{
x = calloc(size, sizeof(TYPE));
STARPU_ASSERT(x);
y = calloc(size, sizeof(TYPE));
STARPU_ASSERT(y);
if (rank == 0)
{
unsigned ind;
for (ind = 0; ind < size; ind++)
x[ind] = (TYPE)starpu_drand48();
}
a_r = STARPU_PLU(reconstruct_matrix)(size, nblocks);
if (rank == 0)
STARPU_PLU(display_data_content)(a_r, size);
// STARPU_PLU(compute_ax)(size, x, y, nblocks, rank);
}
barrier_ret = MPI_Barrier(MPI_COMM_WORLD);
STARPU_ASSERT(barrier_ret == MPI_SUCCESS);
double timing = STARPU_PLU(plu_main)(nblocks, rank, world_size);
/*
* Report performance
*/
int reduce_ret;
double min_timing = timing;
double max_timing = timing;
double sum_timing = timing;
reduce_ret = MPI_Reduce(&timing, &min_timing, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
STARPU_ASSERT(reduce_ret == MPI_SUCCESS);
reduce_ret = MPI_Reduce(&timing, &max_timing, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
STARPU_ASSERT(reduce_ret == MPI_SUCCESS);
reduce_ret = MPI_Reduce(&timing, &sum_timing, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
STARPU_ASSERT(reduce_ret == MPI_SUCCESS);
if (rank == 0)
{
fprintf(stderr, "Computation took: %f ms\n", max_timing/1000);
fprintf(stderr, "\tMIN : %f ms\n", min_timing/1000);
fprintf(stderr, "\tMAX : %f ms\n", max_timing/1000);
fprintf(stderr, "\tAVG : %f ms\n", sum_timing/(world_size*1000));
unsigned n = size;
double flop = (2.0f*n*n*n)/3.0f;
fprintf(stderr, "Synthetic GFlops : %2.2f\n", (flop/max_timing/1000.0f));
}
/*
* Test Result Correctness
*/
if (check)
{
/*
* Compute || A - LU ||
*/
STARPU_PLU(compute_lu_matrix)(size, nblocks, a_r);
#if 0
/*
* Compute || Ax - LUx ||
*/
unsigned ind;
y2 = calloc(size, sizeof(TYPE));
STARPU_ASSERT(y);
if (rank == 0)
{
for (ind = 0; ind < size; ind++)
{
y2[ind] = (TYPE)0.0;
}
}
STARPU_PLU(compute_lux)(size, x, y2, nblocks, rank);
/* Compute y2 = y2 - y */
CPU_AXPY(size, -1.0, y, 1, y2, 1);
TYPE err = CPU_ASUM(size, y2, 1);
int max = CPU_IAMAX(size, y2, 1);
fprintf(stderr, "(A - LU)X Avg error : %e\n", err/(size*size));
fprintf(stderr, "(A - LU)X Max error : %e\n", y2[max]);
#endif
}
/*
* Termination
*/
barrier_ret = MPI_Barrier(MPI_COMM_WORLD);
STARPU_ASSERT(barrier_ret == MPI_SUCCESS);
starpu_cublas_shutdown();
starpu_mpi_shutdown();
starpu_shutdown();
#if 0
MPI_Finalize();
#endif
return 0;
}
