void verify_results(int op, int *elems) {
int i, j;
switch(op) {
case PUT:
if(!(me==0))
for(j=0; j<elems[1]; j++) {
if( ARMCI_ABS(ddst[me][j]-j*1.001) > 0.1) {
ARMCI_Error("put failed...Invalid Value Obtained..1", 0);
}
}
MP_BARRIER();
if(DEBUG) if(me==0) printf(" verifying put ..O.K.\n");
break;
case GET:
if(me==0) {
for(i=1; i<nproc; i++) {
for(j=0; j<elems[1]; j++) {
if( ARMCI_ABS(ddst[me][i*elems[1]+j]-j*1.001*(i+1)) > 0.1)
ARMCI_Error("get failed...Invalid Value Obtained..1", 0);
}
}
}
MP_BARRIER();
if(DEBUG) if(me==0) printf(" verifying get ..O.K.\n\n");
break;
case ACC:
if(me==0)
for(j=0; j<elems[1]; j++) {
/*printf("ddst[%d][%d] = %lf\n", me, j, ddst[me][j]);
fflush(stdout); */
if( ARMCI_ABS(ddst[me][j]-(double)nproc) > 0.1) {
ARMCI_Error("accumulate failed...Invalid Value Obtained..1", 0);
}
}
MP_BARRIER();
if(DEBUG)if(me==0) printf(" verifying accumulate ..O.K.\n");
break;
default:
ARMCI_Error("Invalid Operation", 0);
}
fflush(stdout);
}
void compare_patches(double eps, int ndim, double *patch1, int lo1[], int hi1[],
int dims1[],double *patch2, int lo2[], int hi2[],
int dims2[])
{
int i,j, elems=1;
int subscr1[MAXDIMS], subscr2[MAXDIMS];
double diff,max;
for(i=0;i<ndim;i++){
int diff = hi1[i]-lo1[i];
assert(diff == (hi2[i]-lo2[i]));
assert(diff < dims1[i]);
assert(diff < dims2[i]);
elems *= diff+1;
subscr1[i]= lo1[i];
subscr2[i]=lo2[i];
}
for(j=0; j< elems; j++){
int idx1=0, idx2=0, offset1=0, offset2=0;
idx1 = Index(ndim, subscr1, dims1);
idx2 = Index(ndim, subscr2, dims2);
if(j==0){
offset1 =idx1;
offset2 =idx2;
}
idx1 -= offset1;
idx2 -= offset2;
diff = patch1[idx1] - patch2[idx2];
max = ARMCI_MAX(ARMCI_ABS(patch1[idx1]),ARMCI_ABS(patch2[idx2]));
if(max == 0. || max <eps) max = 1.;
if(eps < ARMCI_ABS(diff)/max){
char msg[48];
sprintf(msg,"(proc=%d):%f",me,patch1[idx1]);
print_subscript("ERROR: a",ndim,subscr1,msg);
sprintf(msg,"%f\n",patch2[idx2]);
print_subscript(" b",ndim,subscr2,msg);
fflush(stdout);
sleep(1);
ARMCI_Error("Bailing out",0);
}
update_subscript(ndim, subscr1, lo1,hi1, dims1);
update_subscript(ndim, subscr2, lo2,hi2, dims2);
}
}
void test_one_group(ARMCI_Group *group, int *pid_list) {
int grp_me, grp_size;
int i,j,src_proc,dst_proc;
double *ddst_put[MAXPROC];
double dsrc[ELEMS];
int elems[2] = {MAXPROC,ELEMS};
int value = -1, bytes, world_me;
MP_MYID(&world_me);
ARMCI_Group_rank(group, &grp_me);
ARMCI_Group_size(group, &grp_size);
if(grp_me==0) printf("GROUP SIZE = %d\n", grp_size);
printf("%d:group rank = %d\n", me, grp_me);
src_proc = 0; dst_proc = grp_size-1;
bytes = ELEMS*sizeof(double);
ARMCI_Malloc_group((void **)ddst_put, bytes, group);
for(i=0; i<ELEMS; i++) dsrc[i]=i*1.001*(grp_me+1);
for(i=0; i<ELEMS; i++) ddst_put[grp_me][i]=-1.0;
armci_msg_group_barrier(group);
if(grp_me==src_proc) {
/* NOTE: make sure to specify absolute ids in ARMCI calls */
ARMCI_Put(dsrc, &ddst_put[dst_proc][0], bytes,
ARMCI_Absolute_id(group,dst_proc));
}
armci_msg_group_barrier(group);
/* NOTE: make sure to specify absolute ids in ARMCI calls */
ARMCI_Fence(ARMCI_Absolute_id(group,dst_proc));
sleep(1);
/* Verify*/
if(grp_me==dst_proc) {
for(j=0; j<ELEMS; j++) {
if(ARMCI_ABS(ddst_put[grp_me][j]-j*1.001*(src_proc+1)) > 0.1) {
printf("\t%d: ddst_put[%d][%d] = %lf and expected value is %lf\n",
me, grp_me, j, ddst_put[grp_me][j], j*1.001*(src_proc+1));
ARMCI_Error("groups: armci put failed...1", 0);
}
}
printf("\n%d(%d): Test O.K. Verified\n", dst_proc, world_me);
}
armci_msg_group_barrier(group);
ARMCI_Free_group(ddst_put[grp_me], group);
}
void check_result(double *src_buf, double *dst_buf, int *stride, int *count,
int stride_levels)
{
int i, j, size;
long idx;
int n1dim; /* number of 1 dim block */
int bvalue[ARMCI_MAX_STRIDE_LEVEL], bunit[ARMCI_MAX_STRIDE_LEVEL];
/* number of n-element of the first dimension */
n1dim = 1;
for(i=1; i<=stride_levels; i++)
n1dim *= count[i];
/* calculate the destination indices */
bvalue[0] = 0; bvalue[1] = 0; bunit[0] = 1; bunit[1] = 1;
for(i=2; i<=stride_levels; i++) {
bvalue[i] = 0;
bunit[i] = bunit[i-1] * count[i-1];
}
for(i=0; i<n1dim; i++) {
idx = 0;
for(j=1; j<=stride_levels; j++) {
idx += bvalue[j] * stride[j-1];
if((i+1) % bunit[j] == 0) bvalue[j]++;
if(bvalue[j] > (count[j]-1)) bvalue[j] = 0;
}
size = count[0] / sizeof(double);
for(j=0; j<size; j++)
if(ARMCI_ABS(((double *)((char *)src_buf+idx))[j] -
((double *)((char *)dst_buf+idx))[j]) > 0.000001 ){
fprintf(stdout,"Error:%s comparison failed: (%d) (%f :%f) %d\n",
check_type, j, ((double *)((char *)src_buf+idx))[j],
((double *)((char *)dst_buf+idx))[j], count[0]);
ARMCI_Error("failed",0);
}
}
}
void test_gop2_or_reduce(const int datatype, char * op, const int reduce_test)
{
void *a[6];
int len[6] = {1, 10, 100, 1000, 10000, 100000};
int len_length = 3;
int datatype_size = 0;
int i, j;
char * test_type;
int verbose = 0;
if(reduce_test == 0)
test_type = "gop2";
else
test_type = "reduce";
switch(datatype)
{
case ARMCI_INT:
datatype_size = sizeof(int);
for(i = 0; i < len_length; i++)
a[i] = malloc(len[i] * datatype_size);
for(i = 0; i < len_length; i++)
for(j = 0; j < len[i]; j++)
((int *) a[i])[j] = (int) (me + j) * (((me + j) % 2 == 0) ? 1 : -1);
for(i = 0; i < len_length; i++)
{
if(me == 0 && verbose != 0)
printf("testing %s %s message size = %d op = %s\n", test_type, "ARMCI_INT", len[i], op);
if(reduce_test == 0)
armci_msg_igop(a[i], len[i], op);
else
armci_msg_reduce(a[i], len[i], op, datatype);
}
if(me == 0 || reduce_test == 0)
for(i = 0; i < len_length; i++)
{
if(me == 0 && verbose != 0)
printf("checking %s %s message size = %d op = %s\n", test_type, "ARMCI_INT", len[i], op);
for(j = 0; j < len[i]; j++)
if(strncmp(op, "+", 1) == 0)
{
int compare = 0;
if(nproc % 2 == 0)
{
if(j % 2 == 0)
compare = -nproc / 2;
else
compare = nproc / 2;
}
else
{
if(j % 2 == 0)
compare = j + nproc / 2;
else
compare = -(j + nproc / 2);
}
if(((int *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %d != %d\n", test_type, "ARMCI_INT", op, i, j, ((int *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "*", 1) == 0)
{
int compare = 1;
int k = 0;
for(k = 0; k < nproc; k++)
compare *= (k + j) * (((k + j) % 2 == 0) ? 1 : -1);
if(((int *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %d != %d\n", test_type, "ARMCI_INT", op, i, j, ((int *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "min", 3) == 0)
{
int compare = -(j + nproc - 1);
if(compare % 2 == 0 && nproc > 1)
compare = -(j + nproc - 2);
if(((int *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %d != %d\n", test_type, "ARMCI_INT", op, i, j, ((int *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "max", 3) == 0)
{
int compare = j + nproc - 1;
if(compare % 2 != 0 && nproc > 1)
compare = j + nproc - 2;
if(((int *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %d != %d\n", test_type, "ARMCI_INT", op, i, j, ((int *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "absmax", 6) == 0)
{
int compare = j + nproc - 1;
if(((int *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %d != %d\n", test_type, "ARMCI_INT", op, i, j, ((int *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "absmin", 6) == 0)
{
int compare = j;
if(((int *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %d != %d\n", test_type, "ARMCI_INT", op, i, j, ((int *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "or", 2) == 0)
{
}
}
break;
case ARMCI_LONG:
datatype_size = sizeof(long);
for(i = 0; i < len_length; i++)
a[i] = malloc(len[i] * datatype_size);
for(i = 0; i < len_length; i++)
for(j = 0; j < len[i]; j++)
((long *) a[i])[j] = (long) (me + j) * (((me + j) % 2 == 0) ? 1 : -1);
for(i = 0; i < len_length; i++)
{
if(me == 0 && verbose != 0)
printf("testing %s %s message size = %d op = %s\n", test_type, "ARMCI_LONG", len[i], op);
if(reduce_test == 0)
armci_msg_lgop(a[i], len[i], op);
else
armci_msg_reduce(a[i], len[i], op, datatype);
}
if(me == 0 || reduce_test == 0)
for(i = 0; i < len_length; i++)
{
if(me == 0 && verbose != 0)
printf("checking %s %s message size = %d op = %s\n", test_type, "ARMCI_LONG", len[i], op);
for(j = 0; j < len[i]; j++)
if(strncmp(op, "+", 1) == 0)
{
int compare = 0;
if(nproc % 2 == 0)
{
if(j % 2 == 0)
compare = -nproc / 2;
else
compare = nproc / 2;
}
else
{
if(j % 2 == 0)
compare = j + nproc / 2;
else
compare = -(j + nproc / 2);
}
if(((long *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %ld != %d\n", test_type, "ARMCI_LONG", op, i, j, ((long *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "*", 1) == 0)
{
int compare = 1;
int k = 0;
for(k = 0; k < nproc; k++)
compare *= (k + j) * (((k + j) % 2 == 0) ? 1 : -1);
if(((long *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %ld != %d\n", test_type, "ARMCI_LONG", op, i, j, ((long *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "min", 3) == 0)
{
int compare = -(j + nproc - 1);
if(compare % 2 == 0 && nproc > 1)
compare = -(j + nproc - 2);
if(((long *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %ld != %d\n", test_type, "ARMCI_LONG", op, i, j, ((long *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "max", 3) == 0)
{
int compare = j + nproc - 1;
if(compare % 2 != 0 && nproc > 1)
compare = j + nproc - 2;
if(((long *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %ld != %d\n", test_type, "ARMCI_LONG", op, i, j, ((long *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "absmax", 6) == 0)
{
int compare = j + nproc - 1;
if(((long *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %ld != %d\n", test_type, "ARMCI_LONG", op, i, j, ((long *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "absmin", 6) == 0)
{
int compare = j;
if(((long *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %ld != %d\n", test_type, "ARMCI_LONG", op, i, j, ((long *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "or", 2) == 0)
{
}
}
break;
case ARMCI_FLOAT:
datatype_size = sizeof(float);
for(i = 0; i < len_length; i++)
a[i] = malloc(len[i] * datatype_size);
for(i = 0; i < len_length; i++)
for(j = 0; j < len[i]; j++)
((float *) a[i])[j] = (float) (me + j) * (((me + j) % 2 == 0) ? 1.0 / nproc : -1.0 / nproc);
for(i = 0; i < len_length; i++)
{
if(me == 0 && verbose != 0)
printf("testing %s ARMCI_FLOAT message size = %d op = %s\n", test_type, len[i], op);
if(reduce_test == 0)
armci_msg_fgop(a[i], len[i], op);
else
armci_msg_reduce(a[i], len[i], op, datatype);
}
if(me == 0 || reduce_test == 0)
for(i = 0; i < len_length; i++)
{
if(me == 0 && verbose != 0)
printf("checking %s ARMCI_FLOAT message size = %d op = %s\n", test_type, len[i], op);
for(j = 0; j < len[i]; j++)
if(strncmp(op, "+", 1) == 0)
{
float compare = 0.0;
if(nproc % 2 == 0)
{
if(j % 2 == 0)
compare = -(((int)nproc / 2) / (float) nproc);
else
compare = ((int)nproc / 2) / (float) nproc;
}
else
{
if(j % 2 == 0)
compare = ((int) j + nproc / 2) / (float) nproc;
else
compare = -(((int) j + nproc / 2) / (float) nproc);
}
if(ARMCI_ABS(((float *) a[i])[j] - compare) > ARMCI_ABS(compare) * FLOAT_EPS)
{
printf("ERROR %s %s %s a[%d][%d] = %f != %f\n", test_type, "ARMCI_FLOAT", op, i, j, ((float *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "*", 1) == 0)
{
float compare = 1.0;
int k = 0;
for(k = 0; k < nproc; k++)
compare *= ((float) k + j) / (float) nproc;
if((nproc / 2) % 2 != 0)
if(nproc % 2 != 0)
if(j % 2 == 0)
compare *= -1.0;
if(ARMCI_ABS(((float *) a[i])[j] - compare) > ARMCI_ABS(compare) * FLOAT_EPS)
{
printf("ERROR %s %s %s a[%d][%d] = %f != %f\n", test_type, "ARMCI_FLOAT", op, i, j, ((float *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "min", 3) == 0)
{
float compare = -((float) j + nproc - 1) / nproc;
if((j + nproc - 1)% 2 == 0 && nproc > 1)
compare = -((float) j + nproc - 2) / nproc;
if(((float *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %f != %f\n", test_type, "ARMCI_FLOAT", op, i, j, ((float *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "max", 3) == 0)
{
float compare = ((float) j + nproc - 1) / nproc;
if((j + nproc - 1) % 2 != 0 && nproc > 1)
compare = ((float) j + nproc - 2) / nproc;
if(((float *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %f != %f\n", test_type, "ARMCI_FLOAT", op, i, j, ((float *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "absmax", 6) == 0)
{
float compare = ((float) j + nproc - 1) / nproc;
if(((float *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %f != %f\n", test_type, "ARMCI_FLOAT", op, i, j, ((float *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "absmin", 6) == 0)
{
float compare = (float) j / nproc;
if(((float *) a[i])[j] != compare)
{
printf("ERROR %s %s %s a[%d][%d] = %f != %f\n", test_type, "ARMCI_FLOAT", op, i, j, ((float *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
}
break;
case ARMCI_DOUBLE:
datatype_size = sizeof(double);
for(i = 0; i < len_length; i++)
a[i] = malloc(len[i] * datatype_size);
for(i = 0; i < len_length; i++)
for(j = 0; j < len[i]; j++)
((double *) a[i])[j] = (double) (me + j) * (((me + j) % 2 == 0) ? 1.0 / nproc : -1.0 / nproc);
for(i = 0; i < len_length; i++)
{
if(me == 0 && verbose != 0)
printf("testing %s ARMCI_DOUBLE message size = %d op = %s\n", test_type, len[i], op);
if(reduce_test == 0)
armci_msg_dgop(a[i], len[i], op);
else
armci_msg_reduce(a[i], len[i], op, datatype);
}
if(me == 0 || reduce_test == 0)
for(i = 0; i < len_length; i++)
{
if(me == 0 && verbose != 0)
printf("checking %s ARMCI_DOUBLE message size = %d op = %s\n", test_type, len[i], op);
for(j = 0; j < len[i]; j++)
if(strncmp(op, "+", 1) == 0)
{
double compare = 0.0;
if(nproc % 2 == 0)
{
if(j % 2 == 0)
compare = -(((int)nproc / 2) / (double) nproc);
else
compare = ((int)nproc / 2) / (double) nproc;
}
else
{
if(j % 2 == 0)
compare = ((int) j + nproc / 2) / (double) nproc;
else
compare = -(((int) j + nproc / 2) / (double) nproc);
}
if(ARMCI_ABS(((double *) a[i])[j] - compare) > ARMCI_ABS(compare) * DOUBLE_EPS)
{
printf("ERROR %s %s %s a[%d][%d] = %f != %f\n", test_type, "ARMCI_DOUBLE", op, i, j, ((double *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "*", 1) == 0)
{
double compare = 1.0;
int k = 0;
for(k = 0; k < nproc; k++)
compare *= ((float) k + j) / (float) nproc;
if((nproc / 2) % 2 != 0)
if(nproc % 2 != 0)
if(j % 2 == 0)
compare *= -1.0;
if(ARMCI_ABS(((double *) a[i])[j] - compare) > ARMCI_ABS(compare) * DOUBLE_EPS)
{
printf("ERROR %s %s %s a[%d][%d] = %f != %f\n", test_type, "ARMCI_DOUBLE", op, i, j, ((double *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "min", 3) == 0)
{
double compare = -((double) j + nproc - 1) / nproc;
if((j + nproc - 1)% 2 == 0 && nproc > 1)
compare = -((double) j + nproc - 2) / nproc;
if(ARMCI_ABS(((double *) a[i])[j] - compare) > ARMCI_ABS(compare) * DOUBLE_EPS)
{
printf("ERROR %s %s %s a[%d][%d] = %f != %f\n", test_type, "ARMCI_DOUBLE", op, i, j, ((double *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "max", 3) == 0)
{
double compare = ((double) j + nproc - 1) / nproc;
if((j + nproc - 1) % 2 != 0 && nproc > 1)
compare = ((double) j + nproc - 2) / nproc;
if(ARMCI_ABS(((double *) a[i])[j] - compare) > ARMCI_ABS(compare) * DOUBLE_EPS)
{
printf("ERROR %s %s %s a[%d][%d] = %f != %f\n", test_type, "ARMCI_DOUBLE", op, i, j, ((double *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "absmax", 6) == 0)
{
double compare = ((double) j + nproc - 1) / nproc;
if(ARMCI_ABS(((double *) a[i])[j] - compare) > ARMCI_ABS(compare) * DOUBLE_EPS)
{
printf("ERROR %s %s %s a[%d][%d] = %f != %f\n", test_type, "ARMCI_DOUBLE", op, i, j, ((double *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
else if(strncmp(op, "absmin", 6) == 0)
{
double compare = (double) j / nproc;
if(ARMCI_ABS(((double *) a[i])[j] - compare) > ARMCI_ABS(compare) * DOUBLE_EPS)
{
printf("ERROR %s %s %s a[%d][%d] = %f != %f\n", test_type, "ARMCI_DOUBLE", op, i, j, ((double *) a[i])[j], compare);
ARMCI_Error("test_gop2_or_reduce failed\n",0);
}
}
}
break;
default:
break;
}
for(i = 0; i < len_length; i++)
free(a[i]);
}
void test_aggregate(int dryrun) {
int i, j, rc, bytes, elems[2] = {MAXPROC, MAXELEMS};
double *ddst_put[MAXPROC];
double *ddst_get[MAXPROC];
double *dsrc[MAXPROC];
armci_hdl_t aggr_hdl_put[MAXPROC];
armci_hdl_t aggr_hdl_get[MAXPROC];
armci_hdl_t hdl_put[MAXELEMS];
armci_hdl_t hdl_get[MAXELEMS];
armci_giov_t darr;
void *src_ptr[MAX_REQUESTS], *dst_ptr[MAX_REQUESTS];
int start = 0, end = 0;
double start_time;
create_array((void**)ddst_put, sizeof(double),2, elems);
create_array((void**)ddst_get, sizeof(double),2, elems);
create_array((void**)dsrc, sizeof(double),1, &elems[1]);
for(i=0; i<elems[1]; i++) dsrc[me][i]=i*1.001*(me+1);
for(i=0; i<elems[0]*elems[1]; i++) {
ddst_put[me][i]=0.0;
ddst_get[me][i]=0.0;
}
MP_BARRIER();
/* only proc 0 does the work */
if(me == 0) {
if(!dryrun)printf("Transferring %d doubles (Not an array of %d doubles)\n", MAXELEMS, MAXELEMS);
/* initializing non-blocking handles */
for(i=0; i<elems[1]; i++) ARMCI_INIT_HANDLE(&hdl_put[i]);
for(i=0; i<elems[1]; i++) ARMCI_INIT_HANDLE(&hdl_get[i]);
/* aggregate handles */
for(i=0; i<nproc; i++) ARMCI_INIT_HANDLE(&aggr_hdl_put[i]);
for(i=0; i<nproc; i++) ARMCI_INIT_HANDLE(&aggr_hdl_get[i]);
for(i=0; i<nproc; i++) ARMCI_SET_AGGREGATE_HANDLE(&aggr_hdl_put[i]);
for(i=0; i<nproc; i++) ARMCI_SET_AGGREGATE_HANDLE(&aggr_hdl_get[i]);
bytes = sizeof(double);
/* **************** PUT **************** */
/* register put */
start_time=MP_TIMER();
start = 0; end = elems[1];
for(i=1; i<nproc; i++) {
for(j=start; j<end; j++) {
ARMCI_NbPutValueDouble(dsrc[me][j], &ddst_put[i][me*elems[1]+j], i,
&hdl_put[j]);
}
for(j=start; j<end; j++) ARMCI_Wait(&hdl_put[j]);
}
if(!dryrun)printf("%d: Value Put time = %.2es\n", me, MP_TIMER()-start_time);
/* vector put */
start_time=MP_TIMER();
for(i=1; i<nproc; i++) {
for(j=start; j<end; j++) {
src_ptr[j] = (void *)&dsrc[me][j];
dst_ptr[j] = (void *)&ddst_put[i][me*elems[1]+j];
}
darr.src_ptr_array = src_ptr;
darr.dst_ptr_array = dst_ptr;
darr.bytes = sizeof(double);
darr.ptr_array_len = elems[1];
if((rc=ARMCI_NbPutV(&darr, 1, i, &hdl_put[i])))
ARMCI_Error("armci_nbputv failed\n",rc);
}
for(i=1; i<nproc; i++) ARMCI_Wait(&hdl_put[i]);
if(!dryrun)printf("%d: Vector Put time = %.2es\n", me, MP_TIMER()-start_time);
/* regular put */
start_time=MP_TIMER();
for(i=1; i<nproc; i++) {
for(j=start; j<end; j++) {
if((rc=ARMCI_NbPut(&dsrc[me][j], &ddst_put[i][me*elems[1]+j], bytes,
i, &hdl_put[j])))
ARMCI_Error("armci_nbput failed\n",rc);
}
for(j=start; j<end; j++) ARMCI_Wait(&hdl_put[j]);
}
if(!dryrun)printf("%d: Regular Put time = %.2es\n", me, MP_TIMER()-start_time);
/* aggregate put */
start_time=MP_TIMER();
for(i=1; i<nproc; i++) {
for(j=start; j<end; j++) {
if((rc=ARMCI_NbPut(&dsrc[me][j], &ddst_put[i][me*elems[1]+j], bytes,
i, &aggr_hdl_put[i])))
ARMCI_Error("armci_nbput failed\n",rc);
}
}
for(i=1; i<nproc; i++) ARMCI_Wait(&aggr_hdl_put[i]);
if(!dryrun)printf("%d: Aggregate Put time = %.2es\n\n", me, MP_TIMER()-start_time);
/* **************** GET **************** */
/* vector get */
start_time=MP_TIMER();
for(i=1; i<nproc; i++) {
for(j=start; j<end; j++) {
src_ptr[j] = (void *)&dsrc[i][j];
dst_ptr[j] = (void *)&ddst_get[me][i*elems[1]+j];
}
darr.src_ptr_array = src_ptr;
darr.dst_ptr_array = dst_ptr;
darr.bytes = sizeof(double);
darr.ptr_array_len = elems[1];
if((rc=ARMCI_NbGetV(&darr, 1, i, &hdl_get[i])))
ARMCI_Error("armci_nbgetv failed\n",rc);
ARMCI_Wait(&hdl_get[i]);
}
if(!dryrun)printf("%d: Vector Get time = %.2es\n", me, MP_TIMER()-start_time);
/* regular get */
start_time=MP_TIMER();
for(i=1; i<nproc; i++) {
for(j=start; j<end; j++) {
if((rc=ARMCI_NbGet(&dsrc[i][j], &ddst_get[me][i*elems[1]+j], bytes,
i, &hdl_get[j])))
ARMCI_Error("armci_nbget failed\n",rc);
}
for(j=start; j<end; j++) ARMCI_Wait(&hdl_get[j]);
}
if(!dryrun)printf("%d: Regular Get time = %.2es\n", me, MP_TIMER()-start_time);
/* aggregate get */
start_time=MP_TIMER();
for(i=1; i<nproc; i++) {
for(j=start; j<end; j++) {
ARMCI_NbGet(&dsrc[i][j], &ddst_get[me][i*elems[1]+j], bytes,
i, &aggr_hdl_get[i]);
}
}
for(i=1; i<nproc; i++) ARMCI_Wait(&aggr_hdl_get[i]);
if(!dryrun)printf("%d: Aggregate Get time = %.2es\n", me, MP_TIMER()-start_time);
}
MP_BARRIER();
ARMCI_AllFence();
MP_BARRIER();
/* Verify */
if(!(me==0))
for(j=0; j<elems[1]; j++) {
if( ARMCI_ABS(ddst_put[me][j]-j*1.001) > 0.1) {
ARMCI_Error("aggregate put failed...1", 0);
}
}
MP_BARRIER();
if(!dryrun)if(me==0) printf("\n aggregate put ..O.K.\n"); fflush(stdout);
if(me==0) {
for(i=1; i<nproc; i++) {
for(j=0; j<elems[1]; j++) {
if( ARMCI_ABS(ddst_get[me][i*elems[1]+j]-j*1.001*(i+1)) > 0.1) {
ARMCI_Error("aggregate get failed...1", 0);
}
}
}
}
MP_BARRIER();
if(!dryrun)if(me==0) printf(" aggregate get ..O.K.\n"); fflush(stdout);
ARMCI_AllFence();
MP_BARRIER();
if(!dryrun)if(me==0){printf("O.K.\n"); fflush(stdout);}
destroy_array((void **)ddst_put);
destroy_array((void **)ddst_get);
destroy_array((void **)dsrc);
}
void compare_patches(double eps, int ndim, double *patch1, int lo1[], int hi1[],
int dims1[], double *patch2, int lo2[], int hi2[],
int dims2[])
{
int i, j, elems = 1;
int subscr1[MAXDIMS], subscr2[MAXDIMS];
double diff, max;
int idx1, idx2, offset1, offset2;
for (i = 0; i < ndim; i++) { /* count # of elements & verify consistency of both patches */
int diff = hi1[i] - lo1[i];
assert(diff == (hi2[i] - lo2[i]));
assert(diff < dims1[i]);
assert(diff < dims2[i]);
elems *= diff + 1;
subscr1[i] = lo1[i];
subscr2[i] = lo2[i];
}
/* compare element values in both patches */
for (j = 0; j < elems; j++) {
idx1 = Index(ndim, subscr1, dims1); /* calculate element Index from a subscript */
idx2 = Index(ndim, subscr2, dims2);
if (j == 0) {
offset1 = idx1;
offset2 = idx2;
}
idx1 -= offset1;
idx2 -= offset2;
diff = patch1[idx1] - patch2[idx2];
max = ARMCI_MAX(ARMCI_ABS(patch1[idx1]), ARMCI_ABS(patch2[idx2]));
if (max == 0. || max < eps) {
max = 1.;
}
if (eps < ARMCI_ABS(diff) / max) {
char msg[48];
sprintf(msg, "(proc=%d):%f", me, patch1[idx1]);
print_subscript("ERROR: a", ndim, subscr1, msg);
sprintf(msg, "%f\n", patch2[idx2]);
print_subscript(" b", ndim, subscr2, msg);
fflush(stdout);
sleep(1);
ARMCI_Error("Bailing out", 0);
}
{ /* update subscript for the patches */
update_subscript(ndim, subscr1, lo1, hi1, dims1);
update_subscript(ndim, subscr2, lo2, hi2, dims2);
}
}
/* make sure we reached upper limit */
/*for(i=0;i<ndim;i++){
assert(subscr1[i]==hi1[i]);
assert(subscr2[i]==hi2[i]);
}*/
}
