irregular_array1(int rank)
{
int g_A, g_B;
int dims[DIM]={5,10}, dims2[DIM], ndim, type, value=5, block[DIM]={2,3}, map[5]={0,2,0,4,6}, val=7;
int n_block[DIM], block_dims[DIM], i;
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
g_B = NGA_Create_irreg(C_INT, DIM, dims, "array_B", block, map);
GA_Fill(g_A, &value);
GA_Print(g_A);
GA_Fill(g_B, &val);
GA_Print(g_B);
GA_Sync();
NGA_Inquire(g_A, &type, &ndim, dims2);
//printf(" %d -- %d,,\n", type, ndim);
/*
GA_Get_block_info(g_B, n_block, block_dims);
for(i=0; i<DIM; i++)
printf(" %d: %d ___ %d --- \n", rank, n_block[i], block_dims[i]);
*/
GA_Destroy(g_A);
GA_Destroy(g_B);
}
irregular_array2(int rank)
{
int g_A, g_B;
int dims[DIM]={GSIZE,GSIZE}, dims2[DIM], block[DIM]={3,2}, map[5]={0,2,6,0,4}, val_A=4, val_B=7;
int n_block[DIM], block_dims[DIM], i;
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
g_B = NGA_Create_irreg(C_INT, DIM, dims, "array_B", block, map);
GA_Fill(g_A, &val_A);
GA_Print(g_A);
GA_Fill(g_B, &val_B);
GA_Print(g_B);
GA_Sync();
/*
GA_Get_block_info(g_B, n_block, block_dims);
for(i=0; i<DIM; i++)
printf(" %d: %d ___ %d --- \n", rank, n_block[i], block_dims[i]);
*/
GA_Destroy(g_A);
GA_Destroy(g_B);
}
fillandscale(int rank, int nprocs)
{
int g_A, val1=5, val2=5, local_A[SIZE][SIZE], i, j;
int dims[DIM]={SIZE,SIZE}, alo[DIM]={1,1}, ahi[DIM]={2,2}, ld=5;
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
GA_Zero(g_A);
NGA_Fill_patch(g_A, alo, ahi, &val1);
GA_Print(g_A);
GA_Scale(g_A, &val2);
GA_Print(g_A);
NGA_Get(g_A, alo, ahi, local_A, &ld);
if(rank == 1)
{
for(i=0; i<DIM; i++)
{
for(j=0; j<DIM; j++) if(local_A[i][j]!=val1*val2)
printf(" GA ERROR: \n");
}
}
GA_Destroy(g_A);
}
main(int argc, char **argv)
{
int rank, nprocs, i, j;
int g_A, g_B, g_C, local_C[DIM][DIM], dims[DIM]={5,5};
int val_A=5, val_B=3, ld=DIM, max;
int lo[DIM]={2,2}, hi[DIM]={4,4}, blo[DIM]={0,0}, bhi[DIM]={2,2}, clo[DIM]={1,1}, chi[DIM]={3,3};
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
g_B = NGA_Create(C_INT, DIM, dims, "array_B", NULL);
g_C = NGA_Create(C_INT, DIM, dims, "array_C", NULL);
GA_Fill(g_A, &val_A);
GA_Fill(g_B, &val_B);
GA_Zero(g_C);
GA_Elem_maximum_patch(g_A, lo, hi, g_B, blo, bhi, g_C, clo, chi);
GA_Print(g_C);
GA_Sync();
NGA_Get(g_C, clo, chi, local_C, &ld);
if(rank==1)
{
for(i=0; i<DIM; i++)
{
for(j=0; j<DIM; j++)printf("%d ", local_C[i][j]);
printf("\n");
}
if(val_A>val_B) max=val_A;
else max=val_B;
for(i=0; i<DIM; i++)
{
for(j=0; j<DIM; j++)
if(local_C[i][j]!=max) printf("GA Error : \n");
}
}
GA_Sync();
if(rank == 0)
printf("Test Completed \n");
GA_Terminate();
MPI_Finalize();
}
main(int argc, char **argv)
{
int rank, nprocs;
int g_A, g_V, val1=5, val2=5, local_A[SIZE][SIZE], dims_V=SIZE, local_V[dims_V];
int dims[DIM]={SIZE,SIZE}, dims2[DIM], lo[DIM]={1,1}, hi[DIM]={2,2}, ld=5, i, j;
int loV=0, hiV=dims_V-1;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
g_V = NGA_Create(C_INT, 1, &dims_V, "array_A", NULL);
GA_Fill(g_A, &val1);
GA_Print(g_A);
printf("\n");
GA_Scale(g_A, &val2);
GA_Print(g_A);
GA_Get_diag(g_A, g_V);
GA_Print(g_V);
NGA_Get(g_A, lo, hi, local_A, &ld);
NGA_Get(g_V, &loV, &hiV, local_V, &ld);
if(rank==1)
{
for(i=0; i<dims_V; i++)
if(local_V[i]!=val1*val2) printf(" GA Error: \n");
}
if(rank == 0)
printf("Test Completed \n");
GA_Terminate();
MPI_Finalize();
}
main(int argc, char **argv)
{
int rank, nprocs;
int g_A, dims[D]={SIZE,SIZE}, *local_A=NULL, *local_G=NULL, **sub_array=NULL, **s_array=NULL;
int i, j, value=5;
MPI_Init(&argc, &argv);
GA_Initialize();
MA_init(C_INT, 1000, 1000);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
s_array=(int**)malloc(N*sizeof(int*));
for(i=0; i<N; i++)
{
s_array[i]=(int*)malloc(D*sizeof(int));
for(j=0; j<D; j++) s_array[i][j]=rand()%10;
}
sub_array=(int**)malloc(N*sizeof(int*));
for(i=0; i<N; i++)
{
sub_array[i]=(int*)malloc(D*sizeof(int));
for(j=0; j<D; j++) sub_array[i][j]=rand()%10;
}
for(i=0; i<N; i++) local_A=(int*)malloc(N*sizeof(int));
for(i=0; i<N; i++) local_G=(int*)malloc(N*sizeof(int));
g_A=NGA_Create(C_INT, D, dims, "array_A", NULL);
GA_Fill(g_A, &value);
GA_Sync();
NGA_Scatter(g_A, local_A, s_array, N);
NGA_Gather(g_A, local_G, s_array, N);
GA_Sync();
GA_Print(g_A);
if(rank==0)
{
for(i=0; i<N; i++)
if(local_G[i]!=local_A[i]) printf("GA Error: \n");
}
GA_Sync();
if(rank==0)
GA_PRINT_MSG();
GA_Terminate();
MPI_Finalize();
return 0;
}
auto_number2(int rank, int nprocs)
{
int g_A, g_B;
int dims[DIM]={GSIZE, GSIZE}, dims2[DIM], block[DIM], *map=NULL, val=7;
int n_block[DIM], block_dims[DIM], b_temp, i;
int b1, b2, inc=0;
do{
b2=DIM+inc;
b1=nprocs/b2;
inc++;
}while(nprocs/b2>=GSIZE);
block[0]=b1;
block[1]=b2;
map=(int*)malloc(nprocs*sizeof(int));
for(i=0; i<b1; i++)
map[i]=i;
for(i=b1; i<(b2+b1); i++)
map[i]=i-b1;
if(rank==0)
{
for(i=0; i<(b1+b2); i++)
printf("map[%d] - %d\n", i, map[i]);
for(i=0; i<DIM; i++)
printf("BLOCK[%d] - %d\n", i, block[i]);
}
g_B = NGA_Create_irreg(C_INT, DIM, dims, "array_B", block, map);
GA_Fill(g_B, &val);
GA_Print(g_B);
GA_Sync();
if(rank==1)
{
GA_Get_block_info(g_B, n_block, block_dims);
for(i=0; i<DIM; i++)
printf(" %d: %d --- %d ... %d\n", rank, n_block[i], block_dims[i], b_temp);
}
GA_Destroy(g_B);
}
int main(int argc, char **argv)
{
int size_dst = 15;
int g_a = 0;
int I_NEG_ONE = -1;
long L_NEG_ONE = -1;
long long LL_NEG_ONE = -1;
int FIVE = 5;
int TEN = 10;
int lo;
int hi;
int *ptr;
int i;
MP_INIT(argc,argv);
GA_INIT(argc,argv);
for (i=0; i<3; ++i) {
if (0 == i) {
g_a = NGA_Create(C_INT, 1, &size_dst, "dst", NULL);
GA_Fill(g_a, &I_NEG_ONE);
} else if (1 == i) {
g_a = NGA_Create(C_LONG, 1, &size_dst, "dst", NULL);
GA_Fill(g_a, &L_NEG_ONE);
} else if (2 == i) {
g_a = NGA_Create(C_LONGLONG, 1, &size_dst, "dst", NULL);
GA_Fill(g_a, &LL_NEG_ONE);
}
GA_Sync();
GA_Print(g_a);
NGA_Print_patch(g_a, &FIVE, &TEN, 0);
NGA_Print_patch(g_a, &FIVE, &TEN, 1);
NGA_Distribution(g_a, GA_Nodeid(), &lo, &hi);
NGA_Access(g_a, &lo, &hi, &ptr, NULL);
printf("[%d] (%d)=%d\n", GA_Nodeid(), lo, *ptr);
NGA_Release(g_a, &lo, &hi);
}
GA_Terminate();
MP_FINALIZE();
exit(EXIT_SUCCESS);
}
int main(int argc, char **argv)
{
int me;
int nproc;
int status;
int g_a;
int dims[NDIM];
int chunk[NDIM];
int pg_world;
size_t num = 10;
double *p1 = NULL;
double *p2 = NULL;
size_t i;
int num_mutex;
int lo[1];
int hi[1];
int ld[1]={1};
MPI_Comm comm;
MP_INIT(argc,argv);
GA_INIT(argc,argv);
me = GA_Nodeid();
nproc = GA_Nnodes();
comm = GA_MPI_Comm_pgroup_default();
printf("%d: Hello world!\n",me);
if (me==0) printf("%d: GA_Initialize\n",me);
/*if (me==0) printf("%d: ARMCI_Init\n",me);*/
/*ARMCI_Init();*/
/*if (me==0) printf("%d: MA_Init\n",me);*/
/*MA_init(MT_DBL, 8*1024*1024, 2*1024*1024);*/
if (me==0) printf("%d: GA_Create_handle\n",me);
g_a = GA_Create_handle();
if (me==0) printf("%d: GA_Set_array_name\n",me);
GA_Set_array_name(g_a,"test array A");
dims[0] = 30;
if (me==0) printf("%d: GA_Set_data\n",me);
GA_Set_data(g_a,NDIM,dims,MT_DBL);
chunk[0] = -1;
if (me==0) printf("%d: GA_Set_chunk\n",me);
GA_Set_chunk(g_a,chunk);
if (me==0) printf("%d: GA_Pgroup_get_world\n",me);
pg_world = GA_Pgroup_get_world();
if (me==0) printf("%d: GA_Set_pgroup\n",me);
GA_Set_pgroup(g_a,pg_world);
if (me==0) printf("%d: GA_Allocate\n",me);
status = GA_Allocate(g_a);
if(0 == status) MPI_Abort(comm,100);
if (me==0) printf("%d: GA_Zero\n",me);
GA_Zero(g_a);
if (me==0) printf("%d: GA_Sync\n",me);
GA_Sync();
num = 10;
p1 = malloc(num*sizeof(double));
/*double* p1 = ARMCI_Malloc_local(num*sizeof(double));*/
if (p1==NULL) MPI_Abort(comm,1000);
p2 = malloc(num*sizeof(double));
/*double* p2 = ARMCI_Malloc_local(num*sizeof(double));*/
if (p2==NULL) MPI_Abort(comm,2000);
for ( i=0 ; i<num ; i++ ) p1[i] = 7.0;
for ( i=0 ; i<num ; i++ ) p2[i] = 3.0;
num_mutex = 17;
status = GA_Create_mutexes(num_mutex);
if (me==0) printf("%d: GA_Create_mutexes = %d\n",me,status);
/***************************************************************/
if (me==0) {
printf("%d: before GA_Lock\n",me);
GA_Lock(0);
lo[0] = 0;
hi[0] = num-1;
GA_Init_fence();
NGA_Put(g_a,lo,hi,p1,ld);
GA_Fence();
GA_Unlock(0);
printf("%d: after GA_Unlock\n",me);
}
GA_Print(g_a);
if (me==1) {
printf("%d: before GA_Lock\n",me);
GA_Lock(0);
lo[0] = 0;
hi[0] = num-1;
GA_Init_fence();
NGA_Get(g_a,lo,hi,p2,ld);
GA_Fence();
GA_Unlock(0);
printf("%d: after GA_Unlock\n",me);
for ( i=0 ; i<num ; i++ ) printf("p2[%2lu] = %20.10f\n",
(long unsigned)i,p2[i]);
}
/***************************************************************/
status = GA_Destroy_mutexes();
if (me==0) printf("%d: GA_Destroy_mutexes = %d\n",me,status);
/*ARMCI_Free(p2);*/
/*ARMCI_Free(p1);*/
free(p2);
free(p1);
if (me==0) printf("%d: GA_Destroy\n",me);
GA_Destroy(g_a);
/*if (me==0) printf("%d: ARMCI_Finalize\n",me);*/
/*ARMCI_Finalize();*/
if (me==0) printf("%d: GA_Terminate\n",me);
GA_Terminate();
if (me==0) printf("%d: MPI_Finalize\n",me);
MPI_Finalize();
return(0);
}
main(int argc, char **argv)
{
int rank, nprocs, i, j;
int g_A, g_B, g_C, **local_C=NULL, dims[DIM]={SIZE,SIZE}, val1=5, val2=4, alpha=3, beta=2;
int clo[DIM]={SIZE-SIZE,SIZE-SIZE}, chi[DIM]={SIZE-1,SIZE-1}, ld=SIZE;
int **local_tm=NULL;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
local_C=(int**)malloc(SIZE*sizeof(int*));
for(i=0; i<SIZE; i++)
local_C[i]=(int*)malloc(SIZE*sizeof(int));
local_tm=(int**)malloc(SIZE*sizeof(int*));
for(i=0; i<SIZE; i++)
local_tm[i]=(int*)malloc(SIZE*sizeof(int));
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
g_B = GA_Duplicate(g_A, "array_B");
g_C = GA_Duplicate(g_A, "array_C");
GA_Fill(g_A, &val1);
GA_Fill(g_B, &val2);
GA_Add(&alpha, g_A, &beta, g_B, g_C);
GA_Sync();
GA_Print(g_A);
GA_Print(g_B);
GA_Print(g_C);
//printf("check 1\n");
NGA_Get(g_C, clo, chi, local_tm, &ld);
//printf("check 2\n");
// GA_Sync();
if(rank==0)
{
for(i=0; i<SIZE; i++)
{
for(j=0; j<SIZE; j++)printf("%d ", local_tm[i][j]);
printf("\n");
}
}
/*
if(rank==0)
{
NGA_Get(g_C, clo, chi, local_C, &ld);
printf("check 1 \n");
for(i=0; i<SIZE; i++)
{
for(j=0; j<SIZE; j++)printf("%d ", local_C[i][j]);
printf("\n");
}
printf("check 2\n");
for(i=0; i<SIZE; i++)
{
for(j=0; j<SIZE; j++)
if(local_C[i][j]!=(alpha*val1)+(beta*val2)) printf("GA Error : \n");
}
}
*/
//GA_Sync();
if(rank==0)
printf("Test Completed \n");
// GA_Sync();
/*
GA_Destroy(g_A);
GA_Destroy(g_B);
GA_Destroy(g_C);
*/
//*******************************************************************
/* what would be the possible reason for GA_destroy to get failed ..,
* solve this before consolidate the whole
*/
GA_Terminate();
MPI_Finalize();
}
static int test(int shape_idx, int type_idx, int dist_idx)
{
int type = TYPES[type_idx];
int *dims = SHAPES[shape_idx];
int ndim = SHAPES_NDIM[shape_idx];
mock_ga_t *mock_a, *result_a;
int g_a;
int buffer[100];
int lo[GA_MAX_DIM], hi[GA_MAX_DIM], ld[GA_MAX_DIM], shape[GA_MAX_DIM];
int result=0, error_index=-1, error_proc=-1;
mock_a = Mock_Create(type, ndim, dims, "mock", NULL);
result_a = Mock_Create(type, ndim, dims, "mock", NULL);
g_a = create_function[dist_idx](type, ndim, dims);
mock_data(mock_a, g_a);
mock_to_global(mock_a, g_a);
Mock_Zero(mock_a);
GA_Zero(g_a);
global_to_mock(g_a, result_a);
result = neq_mock(mock_a, result_a, &error_index);
if (0 != result) {
error_proc = GA_Nodeid();
}
GA_Igop(&result, 1, "+");
GA_Igop(&error_proc, 1, "max");
if (error_proc != GA_Nodeid()) {
error_index = 0;
}
GA_Igop(&error_index, 1, "+");
if (0 != result) {
if (error_proc == GA_Nodeid()) {
printf("ERROR: local result failed to compare to global result\n");
printf("\terror_proc=%d\n", error_proc);
printf("\terror_index=%d\n", error_index);
printf("***LOCAL RESULT***\n");
Mock_Print(mock_a);
printf("***GLOBAL RESULT***\n");
Mock_Print(result_a);
printf("\tprinting array distribution\n");
}
GA_Sync();
GA_Print(g_a);
GA_Print_distribution(g_a);
return 1;
}
Mock_Destroy(mock_a);
Mock_Destroy(result_a);
GA_Destroy(g_a);
return 0;
}
int main (int argc, char **argv)
{
double startTime;
int info; /* used to check for functions returning nonzeros */
GAVec ga_x; /* solution vector */
TAO_SOLVER tao; /* TAO_SOLVER solver context */
TAO_GA_APPLICATION taoapp; /* TAO application context */
TaoTerminateReason reason;
AppCtx user; /* user-defined application context */
/*initialize GA and MPI */
int heap = 400000, stack = 400000;
MPI_Init (&argc, &argv); /* initialize MPI */
GA_Initialize (); /* initialize GA */
user.me = GA_Nodeid ();
user.nproc = GA_Nnodes ();
startTime = MPI_Wtime();
if (user.me == 0) {
if (GA_Uses_fapi ())
GA_Error ("Program runs with C array API only", 0);
printf ("Using %ld processes\n", (long) user.nproc);
fflush (stdout);
}
heap /= user.nproc;
stack /= user.nproc;
if (!MA_init (MT_F_DBL, stack, heap))
GA_Error ("MA_init failed", stack + heap); /* initialize memory allocator */
/* Initialize TAO */
TaoInitialize (&argc, &argv, (char *) 0, help);
/* Initialize problem parameters */
user.ndim = NDIM;
user.natoms = NATOMS;
user.BlockSize = BLOCKSIZE;
/* Allocate vectors for the solution and gradient */
int dims[2];
dims[0] = user.ndim*user.natoms;
ga_x = NGA_Create (C_DBL, 1, dims, "GA_X", NULL);
if (!ga_x) GA_Error ("lennard-jones.main::NGA_Create ga_x", ga_x);
/* Set up structures for data distribution */
info = SetupBlocks(&user); CHKERRQ(info);
/* The TAO code begins here */
/* Create TAO solver with desired solution method */
info = TaoCreate (MPI_COMM_WORLD, "tao_lmvm", &tao); CHKERRQ(info);
info = TaoGAApplicationCreate (MPI_COMM_WORLD, &taoapp); CHKERRQ(info);
/* Set the initial solution */
info = InitializeVariables(ga_x, &user); CHKERRQ(info);
info = TaoGAAppSetInitialSolutionVec(taoapp, ga_x); CHKERRQ(info);
/* Set routines for function, gradient */
info = TaoGAAppSetObjectiveAndGradientRoutine (taoapp, FormFunctionGradient,
(void *) &user); CHKERRQ(info);
/* Check for TAO command line options */
info = TaoSetFromOptions (tao); CHKERRQ(info);
/* SOLVE THE APPLICATION */
info = TaoSolveGAApplication (taoapp, tao); CHKERRQ(info);
/* To View TAO solver information use */
info = TaoView(tao); CHKERRQ(info);
/* Get termination information */
info = TaoGetTerminationReason (tao, &reason);
if(info) GA_Error("lennard-jones.main.TaoGetTerminationReason",info);
if (user.me == 0) {
if (reason <= 0)
printf("Try a different TAO method, adjust some parameters, or check the function evaluation routines\n");
printf("WALL TIME TAKEN = %lf\n", MPI_Wtime()-startTime);
/*output the solutions */
printf ("The solution is :\n");
}
GA_Print (ga_x);
/* Free TAO data structures */
info = TaoDestroy (tao); CHKERRQ(info);
info = TaoGAAppDestroy (taoapp); CHKERRQ(info);
/* Free GA data structures */
GA_Destroy (ga_x);
if (!MA_pop_stack(user.memHandle))
ga_error("Main::MA_pop_stack for memHandle failed",0);
/* Finalize TAO, GA, and MPI */
TaoFinalize ();
GA_Terminate ();
MPI_Finalize ();
return 0;
}
static int test(int shape_idx, int type_idx, int dist_idx)
{
int type = TYPES[type_idx];
int *dims = SHAPES[shape_idx];
int ndim = SHAPES_NDIM[shape_idx];
mock_ga_t *mock_a, *result_a;
int g_a;
void *alpha = NULL;
int buffer[100];
int lo[GA_MAX_DIM], hi[GA_MAX_DIM], ld[GA_MAX_DIM], shape[GA_MAX_DIM];
int result=0, error_index=-1, error_proc=-1;
int ival = 6;
long lval = 7;
long long llval = 8;
float fval = 9;
double dval = 10;
SingleComplex cval = {11,12};
DoubleComplex zval = {13,14};
/* create the local array and result array */
mock_a = Mock_Create(type, ndim, dims, "mock", NULL);
result_a = Mock_Create(type, ndim, dims, "mock", NULL);
/* create the global array */
g_a = create_function[dist_idx](type, ndim, dims);
/* create meaningful data for local array */
mock_data(mock_a, g_a);
/* init global array with same data as local array */
mock_to_global(mock_a, g_a);
switch (type) {
case C_INT: alpha = &ival; break;
case C_LONG: alpha = &lval; break;
case C_LONGLONG: alpha = &llval; break;
case C_FLOAT: alpha = &fval; break;
case C_DBL: alpha = &dval; break;
case C_SCPL: alpha = &cval; break;
case C_DCPL: alpha = &zval; break;
}
/* call the local routine */
Mock_Add_constant(mock_a, alpha);
/* call the global routine */
GA_Add_constant(g_a, alpha);
/* get the results from the global array */
global_to_mock(g_a, result_a);
/* compare the results */
result = neq_mock(mock_a, result_a, &error_index);
if (0 != result) {
error_proc = GA_Nodeid();
}
/* make sure all procs get same result so they can die gracefully */
GA_Igop(&result, 1, "+");
/* if error occured, find the highest failing node ID */
GA_Igop(&error_proc, 1, "max");
/* clear the error index for all but the highest failing node ID */
if (error_proc != GA_Nodeid()) {
error_index = 0;
}
/* make sure all procs get the error index on the highest failing node ID */
GA_Igop(&error_index, 1, "+");
if (0 != result) {
if (error_proc == GA_Nodeid()) {
printf("ERROR: local result failed to compare to global result\n");
printf("\terror_proc=%d\n", error_proc);
printf("\terror_index=%d\n", error_index);
printf("***LOCAL RESULT***\n");
Mock_Print(mock_a);
printf("***GLOBAL RESULT***\n");
Mock_Print(result_a);
printf("\tprinting array distribution\n");
}
GA_Sync();
GA_Print(g_a);
GA_Print_distribution(g_a);
return 1;
}
/* clean up */
Mock_Destroy(mock_a);
Mock_Destroy(result_a);
GA_Destroy(g_a);
return 0;
}
main(int argc, char **argv)
{
int rank, nprocs, i, j;
int g_A, g_B, g_C, local_C[DIM][DIM], dims[DIM]={5,5}, val1=5, val2=4, alpha=3, beta=2, ld=5;
int alo[DIM]={2,2}, ahi[DIM]={3,3}, blo[DIM]={2,2}, bhi[DIM]={3,3}, clo[DIM]={1,1}, chi[DIM]={2,2};
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
g_B = GA_Duplicate(g_A, "array_B");
g_C = GA_Duplicate(g_A, "array_C");
GA_Fill(g_A, &val1);
GA_Fill(g_B, &val2);
GA_Zero(g_C);
NGA_Add_patch(&alpha, g_A, clo, chi, &beta, g_B, blo, bhi, g_C, clo, chi);
GA_Sync();
GA_Print(g_A);
GA_Print(g_B);
GA_Print(g_C);
NGA_Get(g_C, clo, chi, local_C, &ld);
//printf("check 1 \n");
for(i=0; i<DIM; i++)
{
for(j=0; j<DIM; j++)printf("%d ", local_C[i][j]);
printf("\n");
}
if(rank == 0)
{
printf("check 2\n");
for(i=0; i<DIM; i++)
{
for(j=0; j<DIM; j++)
if(local_C[i][j]!=(alpha*val1)+(beta*val2)) printf("GA Error : \n");
}
}
if(rank==0)
GA_PRINT_MSG();
GA_Sync();
/*
GA_Destroy(g_A);
GA_Destroy(g_B);
GA_Destroy(g_C);
*/
//*******************************************************************
/* what would be the possible reason for GA_destroy to get failed ..,
* solve this before consolidate the whole
*/
GA_Terminate();
MPI_Finalize();
}
int main (int argc, char **argv)
{
double startTime;
int info; /* used to check for functions returning nonzeros */
GAVec ga_x; /* solution vector */
TAO_SOLVER tao; /* TAO_SOLVER solver context */
TAO_GA_APPLICATION taoapp; /* TAO application context */
TaoTerminateReason reason;
AppCtx user; /* user-defined application context */
/*initialize GA and MPI */
int heap = 4000, stack = 4000;
MPI_Init (&argc, &argv); /* initialize MPI */
GA_Initialize (); /* initialize GA */
if (!MA_init(MT_F_DBL, stack, heap))
GA_Error((char*)"MA_init failed", stack+heap);
/* Initialize TAO */
TaoInitialize (&argc, &argv, (char *) 0, help);
startTime = MPI_Wtime();
/* Initialize problem parameters */
user.natoms = NATOMS;
user.ndim = NDIM;
user.n = user.natoms*user.ndim;
/* Create working space */
if (MA_push_stack(C_DBL, 2*user.n, "Vector buffers", &user.memHandle) == MA_FALSE)
GA_Error((char*)"MAIN::ma_alloc_get failed",2*user.n);
/* Allocate Global Array vector for the solution */
int dims[2];
dims[0] = user.n;
ga_x = NGA_Create (C_DBL, 1, dims, (char*)"GA_X", NULL);
if (!ga_x) GA_Error ((char*)"lennard-jones.main::NGA_Create ga_x", ga_x);
/* The TAO code begins here */
/* Create TAO solver with desired solution method */
info = TaoCreate (MPI_COMM_WORLD, "tao_cg", &tao); CHKERRQ(info);
info = TaoGAApplicationCreate (MPI_COMM_WORLD, &taoapp); CHKERRQ(info);
/* Set initial vector */
info = InitializeVariables(ga_x, &user); CHKERRQ(info);
info = TaoGAAppSetInitialSolutionVec(taoapp, ga_x); CHKERRQ(info);
/* Set routines for function, gradient */
info = TaoGAAppSetObjectiveAndGradientRoutine (taoapp, FormFunctionGradient, (void *) &user);
CHKERRQ(info);
/* Check for TAO command line options */
info = TaoSetFromOptions (tao); CHKERRQ(info);
/* SOLVE THE APPLICATION */
info = TaoSolveGAApplication (taoapp, tao); CHKERRQ(info);
/* To View TAO solver information use */
info = TaoView(tao); CHKERRQ(info);
/* Get termination information */
info = TaoGetTerminationReason (tao, &reason); CHKERRQ(info);
if (reason <= 0)
printf("Try a different TAO method, adjust some parameters, or check the function evaluation routines\n");
printf("TIME TAKEN = %lf\n", MPI_Wtime()-startTime);
/*output the solutions */
printf ("The solution is :\n");
GA_Print (ga_x);
/* Free TAO data structures */
info = TaoDestroy (tao); CHKERRQ(info);
info = TaoGAAppDestroy (taoapp); CHKERRQ(info);
/* Free GA data structures */
GA_Destroy (ga_x);
if (!MA_pop_stack(user.memHandle))
GA_Error((char*)"Main::MA_pop_stack failed",0);
/* Finalize TAO, GA, and MPI */
TaoFinalize ();
GA_Terminate ();
MPI_Finalize ();
return 0;
}
int main(int argc, char **argv)
{
int rank, nprocs;
int g_A;
int *local_A=NULL, *local_B=NULL, *output_A=NULL;
int dims[DIM]={SIZE,SIZE}, dims2[DIM], lo[DIM]={SIZE-SIZE,SIZE-SIZE}, hi[DIM]={SIZE-1,SIZE-1}, ld=SIZE;
int value=SIZE;
//int value=0;
#if defined(USE_ELEMENTAL)
// initialize Elemental (which will initialize MPI)
ElInitialize( &argc, &argv );
ElMPICommRank( MPI_COMM_WORLD, &rank );
ElMPICommSize( MPI_COMM_WORLD, &nprocs );
// instantiate el::global array
ElGlobalArraysConstruct_i( &eliga );
// initialize global arrays
ElGlobalArraysInitialize_i( eliga );
#else
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
#endif
local_A=(int*)malloc(SIZE*SIZE*sizeof(int));
output_A=(int*)malloc(SIZE*SIZE*sizeof(int));
memset (output_A, 0, SIZE*SIZE*sizeof(int));
for(int j=0; j<SIZE; j++)
for(int i=0; i<SIZE; i++) local_A[i+j*ld]=(i + j);
local_B=(int*)malloc(SIZE*SIZE*sizeof(int));
memset (local_B, 0, SIZE*SIZE*sizeof(int));
#if defined(USE_ELEMENTAL)
ElGlobalArraysCreate_i( eliga, DIM, dims, "array_A", NULL, NULL, &g_A );
ElGlobalArraysFill_i( eliga, g_A, &value );
ElGlobalArraysPrint_i( eliga, g_A );
// acc data
ElGlobalArraysPut_i( eliga, g_A, lo, hi, local_A, &ld );
ElGlobalArraysSync_i( eliga );
// get
ElGlobalArraysGet_i( eliga, g_A, lo, hi, local_B, &ld );
ElGlobalArraysSync_i( eliga );
ElGlobalArraysPrint_i( eliga, g_A );
#else
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
GA_Fill(g_A, &value);
GA_Print(g_A);
NGA_Put(g_A, lo, hi, local_A, &ld);
GA_Sync();
NGA_Get(g_A, lo, hi, local_B, &ld);
GA_Sync();
GA_Print(g_A);
#endif
// updated output
MPI_Reduce (local_A, output_A, SIZE*SIZE, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD);
if(rank==0)
{
printf(" Original local buffer to be accumulated: \n");
for(int i=0; i<SIZE; i++)
{
for(int j=0; j<SIZE; j++)
printf("%d ", local_A[i*ld+j]);
printf("\n");
}
printf("\n");
printf(" Get returns: \n");
for(int i=0; i<SIZE; i++)
{
for(int j=0; j<SIZE; j++)
printf("%d ", local_B[i*ld + j]);
printf("\n");
}
printf("\n");
for(int i=0; i<SIZE; i++)
{
for(int j=0; j<SIZE; j++)
{
if(local_B[i*ld+j]!=output_A[i*ld+j])
GA_Error("ERROR", -99);
}
}
}
#if defined(USE_ELEMENTAL)
ElGlobalArraysDestroy_i( eliga, g_A );
#else
GA_Destroy(g_A);
#endif
if(rank == 0)
printf ("OK. Test passed\n");
free (local_A);
free (local_B);
free (output_A);
#if defined(USE_ELEMENTAL)
ElGlobalArraysTerminate_i( eliga );
// call el::global arrays destructor
ElGlobalArraysDestruct_i( eliga );
ElFinalize();
#else
GA_Terminate();
MPI_Finalize();
#endif
}
int main(int argc, char **argv)
{
int rank, nprocs, i, j;
int g_A, g_B, dims[DIM]={SIZE,SIZE}, val1=5, val2=4;
int lo[DIM]={SIZE-SIZE,SIZE-SIZE}, hi[DIM]={SIZE-1,SIZE-1}, ld=SIZE;
#if defined(USE_ELEMENTAL)
// initialize Elemental (which will initialize MPI)
ElInitialize( &argc, &argv );
ElMPICommRank( MPI_COMM_WORLD, &rank );
ElMPICommSize( MPI_COMM_WORLD, &nprocs );
// instantiate el::global array
ElGlobalArraysConstruct_i( &eliga );
// initialize global arrays
ElGlobalArraysInitialize_i( eliga );
#else
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
#endif
// create global arrays
#if defined(USE_ELEMENTAL)
ElGlobalArraysCreate_i( eliga, DIM, dims, "array_A", NULL, &g_A );
#else
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
#endif
#if defined(USE_ELEMENTAL)
ElGlobalArraysDuplicate_i( eliga, g_A, "array_B", &g_B );
#else
g_B = GA_Duplicate(g_A, "array_B");
#endif
#if defined(USE_ELEMENTAL)
ElGlobalArraysFill_i( eliga, g_A, &val1 );
ElGlobalArraysFill_i( eliga, g_B, &val2 );
#else
GA_Fill(g_A, &val1);
GA_Fill(g_B, &val2);
#endif
int dot_AB = -99;
#if defined(USE_ELEMENTAL)
ElGlobalArraysDot_i( eliga, g_A, g_B, &dot_AB );
#else
dot_AB = GA_Idot(g_A, g_B);
#endif
#if defined(USE_ELEMENTAL)
ElGlobalArraysSync_i( eliga );
#else
GA_Sync();
#endif
#if defined(USE_ELEMENTAL)
ElGlobalArraysPrint_i( eliga, g_A );
ElGlobalArraysPrint_i( eliga, g_B );
#else
GA_Print(g_A);
GA_Print(g_B);
#endif
// Check
#if defined(USE_ELEMENTAL)
ElGlobalArraysSync_i( eliga );
#else
GA_Sync();
#endif
if(rank==0)
printf ("Integer dot product of g_A and g_B: %d\n", dot_AB);
#if defined(USE_ELEMENTAL)
ElGlobalArraysSync_i( eliga );
#else
GA_Sync();
#endif
if(rank==0)
printf("Test Completed \n");
#if defined(USE_ELEMENTAL)
ElGlobalArraysDestroy_i( eliga, g_A );
ElGlobalArraysDestroy_i( eliga, g_B );
#else
GA_Destroy(g_A);
GA_Destroy(g_B);
#endif
#if defined(USE_ELEMENTAL)
ElGlobalArraysTerminate_i( eliga );
// call el::global arrays destructor
ElGlobalArraysDestruct_i( eliga );
ElFinalize();
#else
GA_Terminate();
MPI_Finalize();
#endif
return 0;
}
main(int argc, char **argv)
{
int rank, nprocs;
int g_A, dims[D]={5,10}, local_A[N], local_G[N], **sub_array=NULL, **s_array=NULL;
int i, j, value=5;
MPI_Init(&argc, &argv);
GA_Initialize();
MA_init(C_INT, 1000, 1000);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
s_array=(int**)malloc(N*sizeof(int*));
for(i=0; i<N; i++)
{
s_array[i]=(int*)malloc(D*sizeof(int));
for(j=0; j<D; j++) s_array[i][j]=rand()%5;
}
sub_array=(int**)malloc(N*sizeof(int*));
for(i=0; i<N; i++)
{
sub_array[i]=(int*)malloc(D*sizeof(int));
for(j=0; j<D; j++) sub_array[i][j]=rand()%5;
}
for(i=0; i<N; i++)
//local_A=(int*)malloc(N*sizeof(int));
/*
* depends on the value of array ..we can generate the location values in randon
* we can also use the if-condition
*/
// PRINTing all the genrated array for reference
for(i=0; i<N; i++)
{
for(j=0; j<D; j++)printf("%d ",s_array[i][j]);
printf("\n");
}
printf("\n");
for(i=0; i<N; i++)
{
for(j=0; j<D; j++)printf("%d ",sub_array[i][j]);
printf("\n");
}
printf("\n");
for(i=0; i<N; i++)printf("%d \n",local_A[i]=rand()%5+1);
// PRINT done - now creating array
g_A=NGA_Create(C_INT, D, dims, "array_A", NULL);
GA_Fill(g_A, &value);
GA_Sync();
NGA_Scatter(g_A, local_A, s_array, N);
NGA_Gather(g_A, local_G, s_array, N);
GA_Sync();
GA_Print(g_A);
for(i=0; i<N; i++)printf("%d \n",local_G[i]);
printf("\n");
if(rank==0)
{
for(i=0; i<N; i++)
if(local_G[i]!=local_A[i]) printf("GA Error: \n");
}
GA_Sync();
if(rank==0)
GA_PRINT_MSG();
GA_Terminate();
MPI_Finalize();
return 0;
}
/* input is matrix size */
void ga_lu(double *A, int matrix_size)
{
int g_a, g_b, dims[2], type=C_DBL;
int lo[2], hi[2], ld;
int block_size[2], proc_grid[2];
double time, gflops;
/* create a 2-d GA (global matrix) */
dims[0] = matrix_size;
dims[1] = matrix_size;
block_size[0] = BLOCK_SIZE;
block_size[1] = BLOCK_SIZE;
#ifdef USE_SCALAPACK_DISTR
proc_grid[0] = 2;
proc_grid[1] = nprocs/2;
if(nprocs%2) GA_Error("For ScaLAPACK stle distribution, nprocs must be "
" divisible by 2", 0);
#endif
#ifndef BLOCK_CYCLIC
g_a = NGA_Create(type, 2, dims, "A", NULL);
g_b = GA_Duplicate(g_a, "transposed array B");
#else
g_a = GA_Create_handle();
GA_Set_data(g_a, 2, dims, type);
GA_Set_array_name(g_a,"A");
# ifdef USE_SCALAPACK_DISTR
GA_Set_block_cyclic_proc_grid(g_a, block_size, proc_grid);
# else
GA_Set_block_cyclic(g_a, block_size);
# endif
GA_Allocate(g_a);
g_b = GA_Create_handle();
GA_Set_data(g_b, 2, dims, type);
GA_Set_array_name(g_b,"B");
# ifdef USE_SCALAPACK_DISTR
GA_Set_block_cyclic_proc_grid(g_b, block_size, proc_grid);
# else
GA_Set_block_cyclic(g_b, block_size);
# endif
GA_Allocate(g_b);
#endif
/* copy the local matrix into GA */
if(me==0)
{
lo[0] = 0;
hi[0] = matrix_size - 1;
lo[1] = 0;
hi[1] = matrix_size - 1;
ld = matrix_size;
NGA_Put(g_a, lo, hi, A, &ld);
}
GA_Sync();
GA_Transpose(g_a, g_b);
time = CLOCK_();
GA_Lu('n', g_b);
time = CLOCK_() - time;
/* 2/3 N^3 - 1/2 N^2 flops for LU and 2*N^2 for solver */
gflops = ( (((double)matrix_size) * matrix_size)/(time*1.0e+9) *
(2.0/3.0 * (double)matrix_size - 0.5) );
if(me==0) printf("\nGA_Lu: N=%d flops=%2.5e Gflops, time=%2.5e secs\n\n",
matrix_size, gflops, time);
#if DEBUG
GA_Print(g_a);
GA_Print(g_b);
#endif
/* if(me==0) lu(A, matrix_size); */
GA_Destroy(g_a);
GA_Destroy(g_b);
}
int main(int argc, char **argv)
{
int rank, nprocs;
int g_A;
int *local_A=NULL, *local_B=NULL, *output_A=NULL;
int dims[DIM]={SIZE,SIZE}, dims2[DIM], lo[DIM]={SIZE-SIZE,SIZE-SIZE}, hi[DIM]={SIZE-1,SIZE-1}, ld=SIZE;
int value=SIZE;
#if defined(USE_ELEMENTAL)
// initialize Elemental (which will initialize MPI)
ElInitialize( &argc, &argv );
ElMPICommRank( MPI_COMM_WORLD, &rank );
ElMPICommSize( MPI_COMM_WORLD, &nprocs );
// instantiate el::global array
ElGlobalArraysConstruct_i( &eliga );
// initialize global arrays
ElGlobalArraysInitialize_i( eliga );
#else
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
#endif
local_A=(int*)malloc(SIZE*SIZE*sizeof(int));
output_A=(int*)malloc(SIZE*SIZE*sizeof(int));
memset (output_A, 0, SIZE*SIZE*sizeof(int));
for(int j=0; j<SIZE; j++)
for(int i=0; i<SIZE; i++) local_A[i+j*ld]=(i + j);
//for(int i=0; i<SIZE; i++) local_A[i+j*ld]=(rand()%10);
local_B=(int*)malloc(SIZE*SIZE*sizeof(int));
memset (local_B, 0, SIZE*SIZE*sizeof(int));
// nb handle
#if defined(USE_ELEMENTAL)
typedef ElInt ga_nbhdl_t;
#endif
ga_nbhdl_t nbnb;
#if defined(USE_ELEMENTAL)
ElGlobalArraysCreate_i( eliga, DIM, dims, "array_A", NULL, &g_A );
ElGlobalArraysFill_i( eliga, g_A, &value );
#else
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
GA_Fill(g_A, &value);
#endif
if (rank == 0) printf ("Initial global array:\n");
#if defined(USE_ELEMENTAL)
ElGlobalArraysPrint_i( eliga, g_A );
#else
GA_Print(g_A);
#endif
for (int i = 0; i < NITERS; i++)
{
// acc data
#if defined(USE_ELEMENTAL)
ElGlobalArraysNBAccumulate_i( eliga, g_A, lo, hi, local_A, &ld, &value, &nbnb );
#else
NGA_NbAcc(g_A, lo, hi, local_A, &ld, &value, &nbnb);
#endif
// updated output
MPI_Reduce (local_A, output_A, SIZE*SIZE, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
#if defined(USE_ELEMENTAL)
ElGlobalArraysNBWait_i( eliga, &nbnb );
#else
NGA_NbWait (&nbnb);
#endif
// get
if (rank == 0) printf ("Get in iter #%d\n", i);
#if defined(USE_ELEMENTAL)
ElGlobalArraysSync_i( eliga );
ElGlobalArraysGet_i( eliga, g_A, lo, hi, local_B, &ld );
ElGlobalArraysPrint_i( eliga, g_A );
#else
GA_Sync();
NGA_Get(g_A, lo, hi, local_B, &ld);
GA_Print(g_A);
#endif
} // end of iters
if(rank==0)
{
printf(" Alpha (multiplier): %d\n", value);
printf(" Original local buffer (before accumulation): \n");
for(int i=0; i<SIZE; i++)
{
for(int j=0; j<SIZE; j++)
printf("%d ", local_A[i*ld+j]);
printf("\n");
}
printf("\n");
printf(" Get returns: \n");
for(int i=0; i<SIZE; i++)
{
for(int j=0; j<SIZE; j++)
printf("%d ", local_B[i*ld + j]);
printf("\n");
}
printf("\n");
for(int i=0; i<SIZE; i++)
{
for(int j=0; j<SIZE; j++)
{
if(local_B[i*ld+j]!=(value + (NITERS * value * (output_A[i*ld+j]))))
GA_Error("ERROR", -99);
}
}
}
#if defined(USE_ELEMENTAL)
ElGlobalArraysDestroy_i( eliga, g_A );
#else
GA_Destroy(g_A);
#endif
if(rank == 0)
printf ("OK. Test passed\n");
free (local_A);
free (local_B);
free (output_A);
#if defined(USE_ELEMENTAL)
ElGlobalArraysTerminate_i( eliga );
// call el::global arrays destructor
ElGlobalArraysDestruct_i( eliga );
ElFinalize();
#else
GA_Terminate();
MPI_Finalize();
#endif
}
static int test(int shape_idx, int type_idx, int dist_idx)
{
int type = TYPES[type_idx];
int *dims = SHAPES[shape_idx];
int ndim = SHAPES_NDIM[shape_idx];
mock_ga_t *mock_a, *mock_b, *mock_c, *mock_r;
int g_a, g_b, g_c;
int buffer[100];
int lo[GA_MAX_DIM], hi[GA_MAX_DIM], ld[GA_MAX_DIM], shape[GA_MAX_DIM];
int result=0, error_index=-1, error_proc=-1;
mock_a = Mock_Create(type, ndim, dims, "mock", NULL);
mock_b = Mock_Create(type, ndim, dims, "mock", NULL);
mock_c = Mock_Create(type, ndim, dims, "mock", NULL);
mock_r = Mock_Create(type, ndim, dims, "mock", NULL);
g_a = create_function[dist_idx](type, ndim, dims);
g_b = create_function[dist_idx](type, ndim, dims);
g_c = create_function[dist_idx](type, ndim, dims);
/* create meaningful data for local array */
mock_data(mock_a, g_a);
mock_data(mock_b, g_b);
/* init global array with same data as local array */
mock_to_global(mock_a, g_a);
mock_to_global(mock_b, g_b);
/* call the local routine */
Mock_Elem_multiply(mock_a, mock_b, mock_c);
/* call the global routine */
GA_Elem_multiply(g_a, g_b, g_c);
/* get the results from the global array */
global_to_mock(g_c, mock_r);
/* compare the results */
result = neq_mock(mock_c, mock_r, &error_index);
if (0 != result) {
error_proc = GA_Nodeid();
}
/* make sure all procs get same result so they can die gracefully */
GA_Igop(&result, 1, "+");
/* if error occured, find the highest failing node ID */
GA_Igop(&error_proc, 1, "max");
/* clear the error index for all but the highest failing node ID */
if (error_proc != GA_Nodeid()) {
error_index = 0;
}
/* make sure all procs get the error index on the highest failing node ID */
GA_Igop(&error_index, 1, "+");
if (0 != result) {
if (error_proc == GA_Nodeid()) {
printf("ERROR: local result failed to compare to global result\n");
printf("\terror_proc=%d\n", error_proc);
printf("\terror_index=%d\n", error_index);
printf("***LOCAL RESULT***\n");
Mock_Print(mock_a);
printf("***GLOBAL RESULT***\n");
Mock_Print(mock_r);
printf("\tprinting array distribution\n");
}
GA_Sync();
GA_Print(g_a);
GA_Print_distribution(g_a);
return 1;
}
/* clean up */
Mock_Destroy(mock_a);
Mock_Destroy(mock_r);
GA_Destroy(g_a);
return 0;
}
