static inline void swaptest(int me, int iterations, int T, int S, int P)
{
int i;
const int tswap = 5, sswap = 2;
target[T] = tswap;
source[S] = sswap;
shmem_barrier_all(); /* Ensure target/source initialization completed */
if (me == 0)
pre_op_check(__func__, source[S], iterations, 0);
if (me == 0) {
for (i = 0; i < iterations; i++)
source[S] = shmem_int_swap(&target[T], source[S], 1);
shmem_int_p(&sync_pes[P], i, 1);
if (debug)
printf("AFTER flag PE 0 value of source is %d"
" = 5?\n", source[S]);
if (((iterations % 2 == 1) && (source[S] != tswap)) ||
((iterations % 2 == 0) &&
(source[S] != sswap))) {
fprintf(stderr, "swap ERR: PE 0 source = %d\n",
source[S]);
shmem_global_exit(EXIT_FAILURE);
}
} else {
wait_until(&sync_pes[P], iterations, 1);
if (((iterations % 2 == 1) && (target[T] != sswap)) ||
((iterations % 2 == 0) &&
(target[T] != tswap))) {
fprintf(stderr, "swap ERR: PE 0 target = %d \n",
target[T]);
shmem_global_exit(EXIT_FAILURE);
}
}
if (verbose) {
if (me == 0)
printf("SHMEM %s finished\n", __func__);
}
}
double
benchmark_swap (struct pe_vars v, union data_types *buffer,
unsigned long iterations)
{
int64_t begin, end;
int i;
static double rate = 0, sum_rate = 0, lat = 0, sum_lat = 0;
/*
* Touch memory
*/
memset(buffer, CHAR_MAX * drand48(), sizeof(union data_types
[ITERATIONS]));
shmem_barrier_all();
if (v.me < v.pairs) {
int value = INT_MAX * drand48();
int old_value;
begin = TIME();
for (i = 0; i < iterations; i++) {
old_value = shmem_int_swap(&(buffer[i].int_type), value, v.nxtpe);
}
end = TIME();
rate = ((double)iterations * 1e6) / (end - begin);
lat = (end - begin) / (double)iterations;
}
shmem_double_sum_to_all(&sum_rate, &rate, 1, 0, 0, v.npes, pwrk1, psync1);
shmem_double_sum_to_all(&sum_lat, &lat, 1, 0, 0, v.npes, pwrk2, psync2);
print_operation_rate(v.me, "shmem_int_swap", sum_rate/1e6, sum_lat/v.pairs);
return 0;
}
int FORTRANIFY (shmem_int4_swap) (int *target, int *value, int *pe)
{
return shmem_int_swap (target, *value, *pe);
}
int main(int argc, char *argv[])
{
int size, rank, world_rank, my_group;
int num_lsms; // number of parallel LSMS instances
int size_lsms; // number of atoms in a lsms instance
int num_steps; // number of energy calculations
int initial_steps; // number of steps before sampling starts
int stepCount=0; // count the Monte Carlo steps executed
double max_time; // maximum walltime for this run in seconds
bool restrict_time = false; // was the maximum time specified?
bool restrict_steps = false; // or the max. numer of steps?
int align; // alignment of lsms_instances
double magnetization;
double energy_accumulator; // accumulates the enegy to calculate the mean
int energies_accumulated;
int new_peid,new_root;
static int op,flag;
double *evec,*r_values;
evec=(double *)shmalloc(sizeof(double)*3*size_lsms);
r_values=(double *)shmalloc(sizeof(double)*(R_VALUE_OFFSET+3*(size_lsms+1)));
energy_accumulator=0.0;
energies_accumulated=0;
double walltime_0,walltime;
double restartWriteFrequency=30.0*60.0;
double nextWriteTime=restartWriteFrequency;
MPI_Comm local_comm;
int *lsms_rank0;
MPI_Status status;
char prefix[40];
char i_lsms_name[64];
char gWL_in_name[64], gWL_out_name[64];
char mode_name[64];
char energy_calculation_name[64];
char stupid[37];
char step_out_name[64];
char wl_step_out_name[128];
char *wl_stepf=NULL;
bool step_out_flag=false;
std::ofstream step_out_file;
typedef enum {Constant, Random, WangLandau_1d, ExhaustiveIsing, WangLandau_2d} EvecGenerationMode;
typedef enum {MagneticMoment, MagneticMomentZ, MagneticMomentX, MagneticMomentY} SecondDimension;
EvecGenerationMode evec_generation_mode = Constant;
SecondDimension second_dimension = MagneticMoment;
double ev0[3];
bool return_moments_flag=true; // true-> return all magnetic moments from lsms run at each step.
bool generator_needs_moment=false;
typedef enum {OneStepEnergy, MultiStepEnergy, ScfEnergy} EnergyCalculationMode;
EnergyCalculationMode energyCalculationMode = OneStepEnergy;
int energyIndex=1; // index for the return value to use for the MC step (0: total energy, 1: band energy)
ev0[0]=ev0[1]=0.0; ev0[2]=1.0;
// size has to be align + size_lsms*num_lsms
align=1;
num_lsms=1;
size_lsms=-1;
my_group=-1;
num_steps=1;
initial_steps=0;
sprintf(i_lsms_name,"i_lsms");
gWL_in_name[0]=gWL_out_name[0]=0;
mode_name[0]=0;
energy_calculation_name[0]=0;
// check command line arguments
for(int i=0; i<argc; i++)
{
if(!strcmp("-num_lsms",argv[i])) num_lsms=atoi(argv[++i]);
if(!strcmp("-size_lsms",argv[i])) size_lsms=atoi(argv[++i]);
if(!strcmp("-align",argv[i])) align=atoi(argv[++i]);
if(!strcmp("-num_steps",argv[i])) {num_steps=atoi(argv[++i]); restrict_steps=true;}
if(!strcmp("-initial_steps",argv[i])) initial_steps=atoi(argv[++i]);
if(!strcmp("-walltime",argv[i])) {max_time=60.0*atof(argv[++i]); restrict_time=true;}
if(!strcmp("-i",argv[i])) strncpy(i_lsms_name,argv[++i],64);
if(!strcmp("-random_dir",argv[i])) {evec_generation_mode = Random;}
if(!strcmp("-step_out",argv[i]))
{strncpy(step_out_name,argv[++i],64); step_out_flag=true;
return_moments_flag=true;}
if(!strcmp("-wl_out", argv[i])) strncpy(gWL_out_name,argv[++i],64);
if(!strcmp("-wl_in", argv[i])) strncpy(gWL_in_name,argv[++i],64);
if(!strcmp("-mode", argv[i])) strncpy(mode_name,argv[++i],64);
if(!strcmp("-energy_calculation",argv[i])) strncpy(energy_calculation_name,argv[++i],64);
}
if(!(restrict_steps || restrict_time)) restrict_steps=true;
if(mode_name[0]!=0)
{
if(!strcmp("constant",mode_name)) evec_generation_mode = Constant;
if(!strcmp("random",mode_name)) evec_generation_mode = Random;
if(!strcmp("1d",mode_name)) evec_generation_mode = WangLandau_1d;
if(!strcmp("ising",mode_name)) evec_generation_mode = ExhaustiveIsing;
if(!strcmp("2d",mode_name)) evec_generation_mode = WangLandau_2d;
if(!strcmp("2d-m",mode_name)) {evec_generation_mode = WangLandau_2d; second_dimension=MagneticMoment;}
if(!strcmp("2d-x",mode_name)) {evec_generation_mode = WangLandau_2d; second_dimension=MagneticMomentX;}
if(!strcmp("2d-y",mode_name)) {evec_generation_mode = WangLandau_2d; second_dimension=MagneticMomentY;}
if(!strcmp("2d-z",mode_name)) {evec_generation_mode = WangLandau_2d; second_dimension=MagneticMomentZ;}
}
if(energy_calculation_name[0]!=0)
{
if(energy_calculation_name[0]=='o') { energyCalculationMode = OneStepEnergy; energyIndex=1; }
if(energy_calculation_name[0]=='m') { energyCalculationMode = MultiStepEnergy; energyIndex=1; }
if(energy_calculation_name[0]=='s') { energyCalculationMode = ScfEnergy; energyIndex=0; }
}
#ifdef USE_PAPI
#define NUM_PAPI_EVENTS 4
int hw_counters = PAPI_num_counters();
if(hw_counters>NUM_PAPI_EVENTS) hw_counters=NUM_PAPI_EVENTS;
int papi_events[NUM_PAPI_EVENTS]; // = {PAPI_TOT_INS,PAPI_TOT_CYC,PAPI_FP_OPS,PAPI_VEC_INS};
char *papi_event_name[] = {"PAPI_TOT_INS","PAPI_FP_OPS",
"RETIRED_SSE_OPERATIONS:DOUBLE_ADD_SUB_OPS:DOUBLE_MUL_OPS:DOUBLE_DIV_OPS:OP_TYPE",
"RETIRED_SSE_OPERATIONS:SINGLE_ADD_SUB_OPS:SINGLE_MUL_OPS:SINGLE_DIV_OPS:OP_TYPE"};
// "RETIRED_INSTRUCTIONS",
// "RETIRED_MMX_AND_FP_INSTRUCTIONS:PACKED_SSE_AND_SSE2",
// "RETIRED_SSE_OPERATIONS:DOUBLE_ADD_SUB_OPS:DOUBLE_MUL_OPS:DOUBLE_DIV_OPS:1",
// "RETIRED_SSE_OPERATIONS:SINGLE_ADD_SUB_OPS:SINGLE_MUL_OPS:SINGLE_DIV_OPS:1"
// get events from names:
for(int i=0; i<NUM_PAPI_EVENTS; i++)
{
if(PAPI_event_name_to_code(papi_event_name[i],&papi_events[i]) != PAPI_OK)
{
// printline("Error in obtaining PAPI event code for: "+ttos(papi_event_name[i]),
// std::cerr,parameters.myrankWorld);
// printline("Skipping all following events",
// std::cerr,parameters.myrankWorld);
if(hw_counters>i) hw_counters=i;
}
}
long long papi_values[NUM_PAPI_EVENTS+4];
// printline("PAPI: "+ttos(hw_counters)+" counters available",std::cout,parameters.myrankWorld);
if(hw_counters>NUM_PAPI_EVENTS) hw_counters=NUM_PAPI_EVENTS;
long long papi_real_cyc_0 = PAPI_get_real_cyc();
long long papi_real_usec_0 = PAPI_get_real_usec();
long long papi_virt_cyc_0 = PAPI_get_virt_cyc();
long long papi_virt_usec_0 = PAPI_get_virt_usec();
PAPI_start_counters(papi_events,hw_counters);
#endif
lsms_rank0=(int *)malloc(sizeof(int)*(num_lsms+1));
// initialize MPI:
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
world_rank=rank;
MPI_Comm_size(MPI_COMM_WORLD, &size);
walltime_0 = get_rtc();
#ifndef SVN_REV
#define SVN_REV "unknown"
#endif
// make sure 'return_moments_flag' is set correctly
switch(evec_generation_mode)
{
case Constant : break;
case Random : break;
case WangLandau_1d :
return_moments_flag = true;
generator_needs_moment = true;
break;
case ExhaustiveIsing : break;
case WangLandau_2d :
return_moments_flag = true;
generator_needs_moment = true;
break;
default: std::cout<<" ERROR: UNKNOWN EVEC GENERATION MODE\n"; exit(1);
}
if(rank==0)
{
std::cout<<"LSMS_3"<<std::endl;
std::cout<<" SVN revision "<<SVN_REV<<std::endl<<std::endl;
#ifdef USE_PAPI
std::cout<<" Using Papi counters"<<std::endl<<std::endl;
#endif
std::cout<<" Size of LSMS instances = "<<size_lsms<<" atoms\n";
std::cout<<" Number of LSMS instances = "<<num_lsms<<std::endl;
std::cout<<" LSMS Energy calculated using ";
switch(energyCalculationMode)
{
case OneStepEnergy: std::cout<<"oneStepEnergy [frozen potential band energy]"<<std::endl; break;
case MultiStepEnergy: std::cout<<"multiStepEnergy [frozen potential band energy with converged Fermi energy]"<<std::endl; break;
case ScfEnergy: std::cout<<"scfEnergy [self-consistent total energy]"<<std::endl; break;
default: std::cout<<"UNKNOWN ENERGY CALCULATION METHOD"<<std::endl; exit(1);
}
if(restrict_steps) std::cout<<" Number of gWL steps = "<<num_steps<<std::endl;
if(restrict_time) std::cout<<" Maximum walltime = "<<max_time<<"s\n";
std::cout<<" Processor alignment (process allocation quantization) = "<<align<<std::endl;
switch(evec_generation_mode)
{
case Constant : std::cout<<" Constant moments direction along "
<<ev0[0]<<" "<<ev0[1]<<" "<<ev0[2]<<std::endl;
break;
case Random : std::cout<<" Random distribution of moments (no Wang-Landau)"<<std::endl;
break;
case WangLandau_1d : std::cout<<" Wang-Landau for one continuous variable (energy)"<<std::endl;
// return_moments_flag = true;
// generator_needs_moment = true;
break;
case ExhaustiveIsing : std::cout<<" Exhaustive Ising sampling"<<std::endl; break;
case WangLandau_2d : std::cout<<" Wang-Landau for two continuous variable (energy, ";
switch(second_dimension)
{
case MagneticMoment : std::cout<<"magnitude of magnetization)"; break;
case MagneticMomentX : std::cout<<"x component of magnetization)"; break;
case MagneticMomentY : std::cout<<"y component of magnetization)"; break;
case MagneticMomentZ : std::cout<<"z component of magnetization)"; break;
}
std::cout<<std::endl;
// return_moments_flag = true;
// generator_needs_moment = true;
break;
default: std::cout<<" ERROR: UNKNOWN EVEC GENERATION MODE\n"; exit(1);
}
if(step_out_flag) std::cout<<" Step output written to: "<<step_out_name<<std::endl;
std::cout<<std::endl;
if(step_out_flag && (evec_generation_mode==WangLandau_1d))
{
// step_out_flag=false;
snprintf(wl_step_out_name,127,"wl1d_%s",step_out_name);
wl_stepf=wl_step_out_name;
}
if(step_out_flag)
{
step_out_file.open(step_out_name);
step_out_file<<"#";
for(int i=0; i<argc; i++) step_out_file<<" "<<argv[i];
step_out_file<<std::endl<<size_lsms<<std::endl;
}
}
if(generator_needs_moment) return_moments_flag=true;
if(num_lsms==1)
{
SHMEM_activeset local_comm;
local_comm.rank=shmem_my_pe();
local_comm.size=shmem_n_pes();
local_comm.start_pe=0;
local_comm.logPE_stride=0;
LSMS lsms_calc(local_comm,i_lsms_name,"1_");
if(rank==0)
{
std::cout<<"executing LSMS(C++) for "<<lsms_calc.numSpins()<<" atoms\n";
std::cout<<" LSMS version = "<<lsms_calc.version()<<std::endl;
}
if(energyCalculationMode==OneStepEnergy)
std::cout<<"one step Energy = "<<lsms_calc.oneStepEnergy()<<std::endl;
else if(energyCalculationMode==MultiStepEnergy)
std::cout<<"multi-step Energy = "<<lsms_calc.multiStepEnergy()<<std::endl;
else if(energyCalculationMode==ScfEnergy)
std::cout<<"self-consistent Energy = "<<lsms_calc.scfEnergy()<<std::endl;
else
{
printf("ERROR: Unknown energy calculation mode for lsms_calc in wl-lsms main!\n");
// MPI_Abort(MPI_COMM_WORLD,5);
exit(5);
}
}
else
{
// build the communicators
//int color=MPI_UNDEFINED;
//Assuming user passes a power of two while using "-align"
int s = align;
int comm_size=(size-align)/num_lsms;
int world_rank;
for(int i=0; i<num_lsms; i++)
{
if((world_rank>=s) && (world_rank<s+comm_size))
{
my_group=i;
//color=i;
new_peid=world_rank-s;
new_root=s;
}
lsms_rank0[i]=s;
s+=comm_size;
}
if(world_rank==0){
//color=num_lsms;
new_peid=0;
comm_size=1;
new_root=0;
}
//MPI_Comm_split(MPI_COMM_WORLD, color, 0, &local_comm);
SHMEM_activeset local_comm;
local_comm.rank=new_peid;
local_comm.size=comm_size;
local_comm.start_pe=new_root;
local_comm.logPE_stride=0;
std::cout<<"world_rank="<<world_rank<<" -> group="<<my_group<<std::endl;
snprintf(prefix,38,"Group %4d: ",my_group);
// now we get ready to do some calculations...
if(my_group>=0)
{
double energy;
double band_energy;
int static i_values[10];
double static r_values[10];
static int op;
//MPI_Comm_rank(local_comm, &rank);
rank = local_comm.rank;
snprintf(prefix,38,"%d_",my_group);
// to use the ramdisk on jaguarpf:
// snprintf(prefix,38,"/tmp/ompi/%d_",my_group);
LSMS lsms_calc(local_comm,i_lsms_name,prefix);
snprintf(prefix,38,"Group %4d: ",my_group);
if(rank==0 && my_group==0)
{
std::cout<<prefix<<"executing LSMS(C++) for "<<lsms_calc.numSpins()<<" atoms\n";
std::cout<<prefix<<" LSMS version = "<<lsms_calc.version()<<std::endl;
}
// wait for commands from master
bool finished=false;
while(!finished)
{
if(rank==0)
{
//MPI_Recv(evec,3*size_lsms,MPI_DOUBLE,0,MPI_ANY_TAG,MPI_COMM_WORLD,&status);
//op =status.MPI_TAG;
if (lsms_rank0[0]==world_rank)
shmem_barrier(0, lsms_rank0[0], 2, pSync1);
}
//MPI_Bcast(&op,1,MPI_INT,0,local_comm);
shmem_broadcast32(&op, &op, 1, local_comm.start_pe, local_comm.start_pe, local_comm.logPE_stride, local_comm.size, pSync2);
/* recognized opcodes:
5: calculate energy
recognized energy calculation modes:
OneStepEnergy : calclulate frozen potential band energy in one step (don't converge Ef)
use only if the Fermi energy will not change due to MC steps!
The only method available in LSMS_1.9
MultiStepEnergy : calculate frozen potential band energy after converging Fermi energy
This should be the new default method. If the Fermi energy doesn't change
multiStepEnergy only performs one step and should be equivalent to oneStepEnergy
The tolerance for Ef convergence can be set with LSMS::setEfTol(Real).
The default tolerance is set in the LSMS::LSMS constructor (currently 1.0e-6).
The maximum number of steps is read from the LSMS input file 'nscf' parameter.
ScfEnergy : this will calculate the selfconsistent total energy.
The maximum number of steps is read from the LSMS input file 'nscf' parameter.
NOT IMPLEMENTED YET!!!
10: get number of sites
*/
if(op==5)
{
lsms_calc.setEvec(evec);
if(energyCalculationMode==OneStepEnergy)
energy=lsms_calc.oneStepEnergy(&band_energy);
else if(energyCalculationMode==MultiStepEnergy)
band_energy=energy=lsms_calc.multiStepEnergy();
else if(energyCalculationMode==ScfEnergy)
energy=lsms_calc.scfEnergy(&band_energy);
else
{
printf("ERROR: Unknown energy calculation mode for lsms_calc in wl-lsms main!\n");
//MPI_Abort(MPI_COMM_WORLD,5);
exit(5);
}
r_values[0]=energy;
r_values[1]=band_energy;
if(return_moments_flag)
{
lsms_calc.getMag(&r_values[R_VALUE_OFFSET]);
}
if(rank==0)
{
if(return_moments_flag)
{
//MPI_Send(r_values,R_VALUE_OFFSET+3*size_lsms,MPI_DOUBLE,0,1005,MPI_COMM_WORLD);
shmem_double_put(r_values, r_values, R_VALUE_OFFSET+3*size_lsms, 0);
} else {
//MPI_Send(r_values,R_VALUE_OFFSET,MPI_DOUBLE,0,1005,MPI_COMM_WORLD);
shmem_double_put(r_values, r_values, R_VALUE_OFFSET, 0);
}
shmem_fence();
shmem_int_swap(&flag, world_rank, 0);
}
} else if(op==10) {
i_values[0]=lsms_calc.numSpins();
//MPI_Send(i_values,10,MPI_INT,0,1010,MPI_COMM_WORLD);
shmem_int_put(i_values, i_values, 10, 0);
} else {
// printf("world rank %d: recieved exit\n",world_rank);
finished=true;
}
}
shfree(evec);
//shfree(r_values);
}
else if(world_rank==0)
{
int running;
double **evecs;
//double *r_values;
//int i_values[10];
int *init_steps;
int total_init_steps;
bool accepted;
char *wl_inf=NULL;
char *wl_outf=NULL;
if(gWL_in_name) wl_inf=gWL_in_name;
if(gWL_out_name) wl_outf=gWL_out_name;
EvecGenerator *generator;
/*
// get number of spins from first LSMS instance
// temp r_values:
r_values=(double *)malloc(sizeof(double)*10);
MPI_Send(r_values,1,MPI_DOUBLE, lsms_rank0[0], 10, MPI_COMM_WORLD);
free(r_values);
MPI_Recv(i_values,10,MPI_INT,lsms_rank0[0],1010,MPI_COMM_WORLD,&status);
if(i_values[0]!=size_lsms)
{
printf("Size specified for Wang-Landau and in LSMS input file don't match!\n");
size_lsms=i_values[0];
}
*/
evecs=(double **)shmalloc(sizeof(double *)*num_lsms);
init_steps=(int *)shmalloc(sizeof(int)*num_lsms);
for(int i=0; i<num_lsms; i++)
{
evecs[i]=(double *)shmalloc(sizeof(double)*3*size_lsms);
init_steps[i]=initial_steps;
}
total_init_steps=num_lsms*initial_steps;
// Initialize the correct evec generator
switch(evec_generation_mode)
{
case Random : generator = new RandomEvecGenerator(size_lsms);
break;
case Constant: generator = new ConstantEvecGenerator(size_lsms, ev0, num_lsms);
break;
//case WangLandau_1d : generator = new WL1dEvecGenerator<std::mt19937>(size_lsms, num_lsms,
// evecs, wl_inf, wl_outf, wl_stepf);
case WangLandau_1d : generator = new WL1dEvecGenerator<boost::mt19937>(size_lsms, num_lsms,
evecs, wl_inf, wl_outf, wl_stepf);
break;
case ExhaustiveIsing : generator = new ExhaustiveIsing1dEvecGenerator(size_lsms, num_lsms,
evecs, wl_inf, wl_outf);
break;
//case WangLandau_2d : generator = new WL2dEvecGenerator<std::mt19937>(size_lsms, num_lsms,
// evecs, wl_inf, wl_outf, wl_stepf);
case WangLandau_2d : generator = new WL2dEvecGenerator<boost::mt19937>(size_lsms, num_lsms,
evecs, wl_inf, wl_outf, wl_stepf);
break;
default: std::cerr<<"The code should never arrive here: UNKNOWN EVEC GENERATION MODE\n";
exit(1);
}
for(int i=0; i<num_lsms; i++)
{
generator->initializeEvec(i,evecs[i]);
}
std::cout<<"This is the master node\n";
// issue initial commands to all LSMS instances
running=0;
bool more_work=true;
if(total_init_steps>0)
{
for(int i=0; i<num_lsms; i++)
{
std::cout<<"starting initial calculation in group "<<i<<std::endl;
//MPI_Send(evecs[i], 3*size_lsms, MPI_DOUBLE, lsms_rank0[i], 5, MPI_COMM_WORLD);
shmem_double_put(evec, evecs[i], 3*size_lsms, lsms_rank0[i]);
shmem_int_p(&op, 5, lsms_rank0[i]);
shmem_fence();
num_steps--; running++; stepCount++;
if(restrict_steps) std::cout<<" "<<num_steps<<" steps remaining\n";
}
shmem_barrier(0, lsms_rank0[0], 2, pSync1);
// first deal with the initial steps:
while(running>0)
{
//if(return_moments_flag)
// MPI_Recv(r_values,R_VALUE_OFFSET+3*size_lsms,MPI_DOUBLE,MPI_ANY_SOURCE,MPI_ANY_TAG,MPI_COMM_WORLD,&status);
//else
// MPI_Recv(r_values,R_VALUE_OFFSET,MPI_DOUBLE,MPI_ANY_SOURCE,MPI_ANY_TAG,MPI_COMM_WORLD,&status);
shmem_int_wait(&flag,-1);
running--;
// std::cout<<"received energy E_tot ="<<r_values[0]<<std::endl;
// std::cout<<" band energy E_band="<<r_values[1]<<std::endl;
if(total_init_steps>0)
{
//int r_group=(status.MPI_SOURCE-align)/comm_size;
int r_group=(flag-align)/comm_size;
std::cout<<"starting additional calculation in group "<<r_group<<std::endl;
if(init_steps[r_group]>0)
{
more_work = !(generator->generateUnsampledEvec(r_group,evecs[r_group],r_values[energyIndex]));
init_steps[r_group]--; total_init_steps--;
}
//MPI_Send(evecs[r_group], 3*size_lsms, MPI_DOUBLE, lsms_rank0[r_group], 5, MPI_COMM_WORLD);
shmem_double_put(r_values, evecs[r_group], 3*size_lsms, lsms_rank0[r_group]); //TODO check this
shmem_fence();
num_steps--; running++; stepCount++;
if(restrict_steps && num_steps<=0) more_work=false;
if(restrict_steps) std::cout<<" "<<num_steps<<" steps remaining\n";
walltime = get_rtc() - walltime_0;
if(restrict_time && walltime>=max_time) more_work=false;
if(restrict_time) std::cout<<" "<<max_time-walltime<<" seconds remaining\n";
}
}
}
more_work=true;
running=0;
for(int i=0; i<num_lsms; i++)
{
std::cout<<"starting main calculation in group "<<i<<std::endl;
//MPI_Send(evecs[i], 3*size_lsms, MPI_DOUBLE, lsms_rank0[i], 5, MPI_COMM_WORLD);
shmem_double_put(evec, evecs[i], 3*size_lsms, lsms_rank0[i]);
shmem_int_p(&op, 5, lsms_rank0[i]);
shmem_fence();
num_steps--; running++; stepCount++;
if(restrict_steps) std::cout<<" "<<num_steps<<" steps remaining\n";
}
shmem_barrier(0, lsms_rank0[0], 2, pSync1);
generator->startSampling();
// wait for results and issue new commands or wind down
while(running>0)
{
//MPI_Recv(r_values,R_VALUE_OFFSET+3*size_lsms,MPI_DOUBLE,MPI_ANY_SOURCE,MPI_ANY_TAG,MPI_COMM_WORLD,&status);
shmem_int_wait(&flag,-1);
running--;
std::cout<<"received energy E_tot ="<<r_values[0]<<std::endl;
std::cout<<" band energy E_band="<<r_values[1]<<std::endl;
// printf("from status.MPI_SOURCE=%d\n",status.MPI_SOURCE);
energy_accumulator+=r_values[0]; energies_accumulated++;
if(more_work)
{
int r_group=(status.MPI_SOURCE-align)/comm_size;
std::cout<<"starting additional calculation in group "<<r_group<<std::endl;
if(generator_needs_moment)
{
double m0,m1,m2;
m0=0.0; m1=0.0; m2=0.0;
for(int i=0; i<3*size_lsms; i+=3)
{
m0+=r_values[R_VALUE_OFFSET+i];
m1+=r_values[R_VALUE_OFFSET+i+1];
m2+=r_values[R_VALUE_OFFSET+i+2];
}
switch(second_dimension)
{
case MagneticMoment : magnetization=std::sqrt(m0*m0+m1*m1+m2*m2); break;
case MagneticMomentX : magnetization=m0; break;
case MagneticMomentY : magnetization=m1; break;
case MagneticMomentZ : magnetization=m2; break;
}
if(generator->generateEvec(r_group,evecs[r_group],r_values[energyIndex],magnetization, &accepted))
more_work=false;
} else {
if(generator->generateEvec(r_group,evecs[r_group],r_values[energyIndex], &accepted)) more_work=false;
}
//MPI_Send(evecs[r_group], 3*size_lsms, MPI_DOUBLE, lsms_rank0[r_group], 5, MPI_COMM_WORLD);
shmem_double_put(r_values, evecs[r_group], 3*size_lsms, lsms_rank0[r_group]); //TODO check this
shmem_fence();
num_steps--; running++; stepCount++;
if(restrict_steps && num_steps<=0) more_work=false;
if(restrict_steps) std::cout<<" "<<num_steps<<" steps remaining\n";
walltime = get_rtc() - walltime_0;
if(restrict_time && walltime>=max_time) more_work=false;
if(restrict_time) std::cout<<" "<<max_time-walltime<<" seconds remaining\n";
}
else
{
// send an exit message to this instance of LSMS
int r_group=(status.MPI_SOURCE-align)/comm_size;
MPI_Send(evecs[r_group], 3*size_lsms, MPI_DOUBLE, lsms_rank0[r_group], 2, MPI_COMM_WORLD);
}
if(step_out_flag && accepted)
{
step_out_file<<"# iteration "<<energies_accumulated<<std::endl;
step_out_file.precision(15);
step_out_file<<energies_accumulated<<std::endl;
step_out_file<<r_values[0]<<" "<<r_values[1]<<std::endl;
for(int j=0; j<3*size_lsms; j+=3)
{
step_out_file<<r_values[j+R_VALUE_OFFSET]<<" "<<r_values[j+R_VALUE_OFFSET+1]
<<" "<<r_values[j+R_VALUE_OFFSET+2]<<std::endl;
}
}
// write restart file every restartWriteFrequency seconds
if(walltime>nextWriteTime)
{
generator->writeState("WLrestart.jsn");
nextWriteTime+=restartWriteFrequency;
}
}
generator->writeState("WLrestart.jsn");
/*
if(evec_generation_mode==WangLandau_1d)
(static_cast<WL1dEvecGenerator<std::mt19937> *>(generator))->writeState("WLrestart.state");
if(evec_generation_mode==ExhaustiveIsing)
(static_cast<ExhaustiveIsing1dEvecGenerator *>(generator))->writeState("WLrestart.state");
*/
for(int i=0; i<num_lsms; i++) free(evecs[i]);
shfree(evecs);
//shfree(r_values);
}
}
if(world_rank==0)
{
if(step_out_flag)
{
step_out_file<<"# end\n-1\n"
<<energy_accumulator/double(energies_accumulated)<<std::endl;
step_out_file.close();
}
std::cout<<"Finished all scheduled calculations. Freeing resources.\n";
std::cout<<"Energy mean = "<<energy_accumulator/double(energies_accumulated)<<"Ry\n";
}
if(num_lsms>1)
{
// make sure averyone arrives here:
MPI_Bcast(stupid,37,MPI_CHAR,0,MPI_COMM_WORLD);
if(world_rank==0)
{
MPI_Comm_free(&local_comm);
}
else if(my_group>=0)
{
MPI_Comm_free(&local_comm);
}
}
if(world_rank==0)
{
double walltime = get_rtc() - walltime_0;
std::cout<<" WL-LSMS finished in "<<walltime<<" seconds.\n";
std::cout<<" Monte-Carlo steps / walltime = "
<<double(stepCount)/walltime<<"/sec\n";
}
#ifdef USE_PAPI
PAPI_stop_counters(papi_values,hw_counters);
papi_values[hw_counters ] = PAPI_get_real_cyc()-papi_real_cyc_0;
papi_values[hw_counters+1] = PAPI_get_real_usec()-papi_real_usec_0;
papi_values[hw_counters+2] = PAPI_get_virt_cyc()-papi_virt_cyc_0;
papi_values[hw_counters+3] = PAPI_get_virt_usec()-papi_virt_usec_0;
long long accumulated_counters[NUM_PAPI_EVENTS+4];
/*
for(int i=0; i<hw_counters; i++)
{
printline(ttos(papi_event_name[i])+" = "+ttos(papi_values[i]),
std::cout,parameters.myrankWorld);
}
printline("PAPI real cycles : "+ttos(papi_values[hw_counters]),
std::cout,parameters.myrankWorld);
printline("PAPI real usecs : "+ttos(papi_values[hw_counters+1]),
std::cout,parameters.myrankWorld);
printline("PAPI user cycles : "+ttos(papi_values[hw_counters+2]),
std::cout,parameters.myrankWorld);
printline("PAPI user usecs : "+ttos(papi_values[hw_counters+3]),
std::cout,parameters.myrankWorld);
*/
//MPI_Reduce(papi_values,accumulated_counters,hw_counters+4,
// MPI_LONG,MPI_SUM,0,MPI_COMM_WORLD);
shmem_long_sum_to_all(accumulated_counters, papi_values, hw_counters+4,
comm.pestart, comm.logPE_stride, comm.size, pWrk_i, pSync2);
if(world_rank==0)
{
for(int i=0; i<hw_counters; i++)
{
std::cout<<"Accumulated: "<<(papi_event_name[i])<<" = "<<(accumulated_counters[i])<<"\n";
}
std::cout<<"PAPI accumulated real cycles : "<<(accumulated_counters[hw_counters])<<"\n";
std::cout<<"PAPI accumulated user cycles : "<<(accumulated_counters[hw_counters+2])<<"\n";
double gflops_papi = ((double)accumulated_counters[1])/
(1000.0*(double)papi_values[hw_counters+1]);
double gflops_hw_double = ((double)accumulated_counters[2])/
(1000.0*(double)papi_values[hw_counters+1]);
double gflops_hw_single = ((double)accumulated_counters[3])/
(1000.0*(double)papi_values[hw_counters+1]);
double gips = ((double)accumulated_counters[0])/(1000.0*(double)papi_values[hw_counters+1]);
std::cout<<"PAPI_FP_OPS real GFLOP/s : "<<(gflops_papi)<<"\n";
std::cout<<"PAPI hw double real GFLOP/s : "<<(gflops_hw_double)<<"\n";
std::cout<<"PAPI hw single real GFLOP/s : "<<(gflops_hw_single)<<"\n";
std::cout<<"PAPI real GINST/s : "<<(gips)<<"\n";
}
#endif
//MPI_Finalize();
return 0;
}
int main(int argc, char **argv)
{
int i,j;
short oldjs, oldxs, my_pes;
int oldji, oldxi, my_pei;
long oldjl, oldxl, my_pel;
long long oldjll,oldxll,my_pell;
float oldjf, oldxf, my_pef;
double oldjd, oldxd, my_ped;
int my_pe,n_pes;
size_t max_elements,max_elements_bytes;
static short *xs;
static int *xi;
static long *xl;
static long long *xll;
static float *xf;
static double *xd;
shmem_init();
my_pe = shmem_my_pe();
n_pes = shmem_n_pes();
my_pes = (short) my_pe;
my_pei = (int) my_pe;
my_pel = (long) my_pe;
my_pell = (long long) my_pe;
my_pef = (float) my_pe;
my_ped = (double) my_pe;
#ifdef HAVE_SET_CACHE_INV
shmem_set_cache_inv();
#endif
/* fail if trying to use only one processor */
if ( n_pes <= 1 ){
fprintf(stderr, "FAIL - test requires at least two PEs\n");
exit(1);
}
if(my_pe == 0)
fprintf(stderr, "shmem_swap(%s) n_pes=%d\n", argv[0],n_pes);
/* test shmem_short_swap */
/* shmalloc xs on all pes (only check the ones on PE 0) */
max_elements_bytes = (size_t) (sizeof(short) * n_pes);
xs = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes; i++)
xs[i] = 0;
shmem_barrier_all();
oldjs = 0;
for(i=0; i<ITER; i++) {
if (my_pe != 0) {
my_pes = my_pes + (short) 1;
/* record PE value in xs[my_pe] -- save PE number */
oldxs = shmem_short_swap(&xs[my_pe], my_pes, 0);
/* printf("PE=%d,i=%d,my_pes=%d,oldxs=%d\n",my_pe,i,my_pes,oldxs); */
if (oldxs != oldjs)
fprintf(stderr, "FAIL PE %d of %d: i=%d, oldxs = %d expected = %d\n",
my_pe, n_pes, i, oldxs, oldjs);
oldjs = my_pes;
}
}
shmem_barrier_all();
if (my_pe == 0) { /* check xs[j] array vs PE# + ITER */
i = (int) ITER + 1;
for(j=1 ; j<n_pes; j++) {
/* printf("j=%d,xs[%d]=%d,i=%d\n",j,j,xs[j],i); */
if (xs[j] != (short) i)
fprintf(stderr, "FAIL PE %d of %d: xs[%d] = %d expected = %d\n",
my_pe, n_pes, j, xs[j],i);
i++;
}
}
shmem_free(xs);
/* test shmem_int_swap */
/* shmalloc xi on all pes (only check the ones on PE 0) */
max_elements_bytes = (size_t) (sizeof(int) * n_pes);
xi = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes; i++)
xi[i] = 0;
shmem_barrier_all();
oldji = 0;
for(i=0; i<ITER; i++) {
if (my_pe != 0) {
my_pei = my_pei + (int) 1;
/* record PE value in xi[my_pe] -- save PE number */
oldxi = shmem_int_swap(&xi[my_pe], my_pei, 0);
/* printf("PE=%d,i=%d,my_pei=%d,oldxi=%d\n",my_pe,i,my_pei,oldxi); */
if (oldxi != oldji)
fprintf(stderr, "FAIL PE %d of %d: i=%d, oldxi = %d expected = %d\n",
my_pe, n_pes, i, oldxi, oldji);
oldji = my_pei;
}
}
shmem_barrier_all();
if (my_pe == 0) { /* check xi[j] array vs PE# + ITER */
i = (int) ITER + 1;
for(j=1 ; j<n_pes; j++) {
/* printf("j=%d,xi[%d]=%d,i=%d\n",j,j,xi[j],i); */
if (xi[j] != i)
fprintf(stderr, "FAIL PE %d of %d: xi[%d] = %d expected = %d\n",
my_pe, n_pes, j, xi[j],i);
i++;
}
}
shmem_free(xi);
/* test shmem_long_swap */
/* shmalloc xl on all pes (only check the ones on PE 0) */
max_elements_bytes = (size_t) (sizeof(long) * n_pes);
xl = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes; i++)
xl[i] = 0;
shmem_barrier_all();
oldjl = 0;
for(i=0; i<ITER; i++) {
if (my_pe != 0) {
my_pel = my_pel + (long) 1;
/* record PE value in xl[my_pe] -- save PE number */
oldxl = shmem_long_swap(&xl[my_pe], my_pel, 0);
/* printf("PE=%d,i=%d,my_pel=%d,oldxl=%d\n",my_pe,i,my_pel,oldxl); */
if (oldxl != oldjl)
fprintf(stderr, "FAIL PE %d of %d: i=%d, oldxl = %d expected = %d\n",
my_pe, n_pes, i, oldxl, oldjl);
oldjl = my_pel;
}
}
shmem_barrier_all();
if (my_pe == 0) { /* check xl[j] array vs PE# + ITER */
i = (int) ITER + 1;
for(j=1 ; j<n_pes; j++) {
/* printf("j=%d,xl[%d]=%d,i=%d\n",j,j,xl[j],i); */
if (xl[j] != (long)i)
fprintf(stderr, "FAIL PE %d of %d: xl[%d] = %ld expected = %d\n",
my_pe, n_pes, j, xl[j],i);
i++;
}
}
shmem_free(xl);
/* test shmem_longlong_swap */
#ifdef HAVE_LONG_LONG
/* shmalloc xll on all pes (only check the ones on PE 0) */
max_elements_bytes = (size_t) (sizeof(long long) * n_pes);
xll = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes; i++)
xll[i] = 0;
shmem_barrier_all();
oldjll = 0;
for(i=0; i<ITER; i++) {
if (my_pe != 0) {
my_pell = my_pell + (long long) 1;
/* record PE value in xll[my_pe] -- save PE number */
oldxll = shmem_longlong_swap(&xll[my_pe], my_pell, 0);
/* printf("PE=%d,i=%d,my_pell=%ld,oldxll=%d\n",my_pe,i,my_pell,oldxll); */
if (oldxll != (long long) oldjll)
fprintf(stderr, "FAIL PE %d of %d: i=%d, oldxll = %ld expected = %ld\n",
my_pe, n_pes, i, oldxll, oldjll);
oldjll = my_pell;
}
}
shmem_barrier_all();
if (my_pe == 0) { /* check xll[j] array vs PE# + ITER */
i = (int) ITER + 1;
for(j=1 ; j<n_pes; j++) {
/* printf("j=%d,xll[%d]=%ld,i=%d\n",j,j,xll[j],i); */
if (xll[j] != (long long) i)
fprintf(stderr, "FAIL PE %d of %d: xll[%d] = %d expected = %d\n",
my_pe, n_pes, j, xll[j],i);
i++;
}
}
shmem_free(xll);
#endif
/* test shmem_float_swap */
/* shmalloc xf on all pes (only use the ones on PE 0) */
max_elements_bytes = (size_t) (sizeof(float) * n_pes);
xf = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes; i++)
xf[i] = (float) 0;
shmem_barrier_all();
oldjf = (float) 0;
for(i=0; i<ITER; i++) {
if (my_pe != 0) {
my_pef = my_pef + (float) 1;
/* record PE value in xf[my_pe] -- save PE number */
oldxf = shmem_float_swap(&xf[my_pe], my_pef, 0);
/* printf("PE=%d,i=%d,my_pef=%10.2f,oldxf=%10.2f\n",my_pe,i,my_pef,oldxf); */
if (oldxf != oldjf)
fprintf(stderr, "FAIL PE %d of %d: i=%d, oldxf = %10.2f expected = %10.2f\n",
my_pe, n_pes, i, oldxf, oldjf);
oldjf = my_pef;
}
}
shmem_barrier_all();
if (my_pe == 0) { /* check xs[j] array vs PE# + ITER */
i = (int) ITER + 1;
for(j=1 ; j<n_pes; j++) {
/* printf("j=%d,xf[%d]=%10.2f,i=%d\n",j,j,xf[j],i); */
if (xf[j] != (float) i)
fprintf(stderr, "FAIL PE %d of %d: xf[%d] = %10.2f expected = %10.2f\n",
my_pe, n_pes, j-1, xf[j], (float)i);
i++;
}
}
shmem_free(xf);
/* test shmem_double_swap */
/* shmalloc xd on all pes (only use the ones on PE 0) */
max_elements_bytes = (size_t) (sizeof(double) * n_pes);
xd = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes; i++)
xd[i] = (double) 0;
shmem_barrier_all();
oldjd = (double) 0;
for(i=0; i<ITER; i++) {
if (my_pe != 0) {
my_ped = my_ped + (double) 1;
/* record PE value in xd[my_pe] -- save PE number */
oldxd = shmem_double_swap(&xd[my_pe], my_ped, 0);
/* printf("PE=%d,i=%d,my_ped=%10.2f,oldxd=%10.2f\n",my_pe,i,my_ped,oldxd);
*/
if (oldxd != oldjd)
fprintf(stderr, "FAIL PE %d of %d: i=%d, oldxd = %10.2f expected = %10.2f\n",
my_pe, n_pes, i, oldxd, oldjd);
oldjd = my_ped;
}
}
shmem_barrier_all();
if (my_pe == 0) { /* check xd[j] array vs PE# + ITER */
i = (int) ITER + 1;
for(j=1 ; j<n_pes; j++) {
/* printf("j=%d,xd[%d]=%10.2f,i=%d\n",j,j,xd[j],i); */
if (xd[j] != (double) i)
fprintf(stderr, "FAIL PE %d of %d: xd[%d] = %10.2f expected = %10.2f\n",
my_pe, n_pes, j, xd[j], (double)i);
i++;
}
}
shmem_free(xd);
#ifdef SHMEM_C_GENERIC_32
/* test shmem_swap (GENERIC 32) */
my_pei = (int) my_pe;
/* shmalloc xi on all pes (only check the ones on PE 0) */
max_elements_bytes = (size_t) (sizeof(int) * n_pes);
xi = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes; i++)
xi[i] = 0;
shmem_barrier_all();
oldji = 0;
for(i=0; i<ITER; i++) {
if (my_pe != 0) {
my_pei = my_pei + (int) 1;
/* record PE value in xi[my_pe] -- save PE number */
oldxi = shmem_swap(&xi[my_pe], my_pei, 0);
/* printf("PE=%d,i=%d,my_pei=%d,oldxi=%d\n",my_pe,i,my_pei,oldxi); */
if (oldxi != oldji)
fprintf(stderr, "FAIL pe %d of %d: i=%d, oldxi = %d expected = %d\n",
my_pe, n_pes, i, oldxi, oldji);
oldji = my_pei;
}
}
shmem_barrier_all();
if (my_pe == 0) { /* check xi[j] array vs PE# + ITER */
i = (int) ITER + 1;
for(j=1 ; j<n_pes; j++) {
/* printf("j=%d,xi[%d]=%d,i=%d\n",j,j,xi[j],i); */
if (xi[j] != i)
fprintf(stderr, "FAIL pe %d of %d: xi[%d] = %d expected = %d\n",
my_pe, n_pes, j, xi[j],i);
i++;
}
}
shmem_free(xi);
#else
/* test shmem_swap (GENERIC 64) */
my_pel = (long) my_pe;
/* shmalloc xl on all pes (only check the ones on PE 0) */
max_elements_bytes = (size_t) (sizeof(long) * n_pes);
xl = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes; i++)
xl[i] = 0;
shmem_barrier_all();
oldjl = 0;
for(i=0; i<ITER; i++) {
if (my_pe != 0) {
my_pel = my_pel + (long) 1;
/* record PE value in xl[my_pe] -- save PE number */
oldxl = shmem_swap(&xl[my_pe], my_pel, 0);
/* printf("PE=%d,i=%d,my_pel=%d,oldxl=%d\n",my_pe,i,my_pel,oldxl); */
if (oldxl != oldjl)
fprintf(stderr, "FAIL pe %d of %d: i=%d, oldxl = %d expected = %d\n",
my_pe, n_pes, i, oldxl, oldjl);
oldjl = my_pel;
}
}
shmem_barrier_all();
if (my_pe == 0) { /* check xl[j] array vs PE# + ITER */
i = (int) ITER + 1;
for(j=1 ; j<n_pes; j++) {
/* printf("j=%d,xl[%d]=%d,i=%d\n",j,j,xl[j],i); */
if (xl[j] != (long)i)
fprintf(stderr, "FAIL pe %d of %d: xl[%d] = %ld expected = %d\n",
my_pe, n_pes, j, xl[j],i);
i++;
}
}
shmem_free(xl);
#endif
shmem_barrier_all();
#ifdef NEEDS_FINALIZE
shmem_finalize();
#endif
return 0;
}
int
main(int argc, char* argv[])
{
int me, num_procs, l, j;
int Verbose = 0;
start_pes(0);
me = _my_pe();
num_procs = _num_pes();
if ( num_procs < 2 ) {
if (me ==0)
printf("PE[0] requires 2 or more PEs?\n");
return 1;
}
for (l = 0 ; l < loops ; ++l) {
if ((src_int = shmalloc(2*num_procs*sizeof(int))) == NULL) {
printf("PE-%d int shmalloc() failed?\n", me);
exit(1);
}
dst_int = &src_int[num_procs];
for(j=0; j < num_procs; j++) {
src_int[j] = 4;
dst_int[j] = 0;
}
if ((src_float = shmalloc(2*num_procs*sizeof(float))) == NULL) {
printf("PE-%d float shmalloc() failed?\n", me);
exit(1);
}
dst_float = &src_float[num_procs];
for(j=0; j < num_procs; j++) {
src_float[j] = 4.0;
dst_float[j] = 0.0;
}
if ((src_double = shmalloc(2*num_procs*sizeof(double))) == NULL) {
printf("PE-%d double shmalloc() failed?\n", me);
exit(1);
}
dst_double = &src_double[num_procs];
for(j=0; j < num_procs; j++) {
src_double[j] = 8.0;
dst_double[j] = 0.0;
}
if ((src_long = shmalloc(2*num_procs*sizeof(long))) == NULL) {
printf("PE-%d long shmalloc() failed?\n", me);
exit(1);
}
dst_long = &src_long[num_procs];
for(j=0; j < num_procs; j++) {
src_long[j] = 8;
dst_long[j] = 0;
}
if ((src_llong = shmalloc(2*num_procs*sizeof(long long))) == NULL) {
printf("PE-%d long shmalloc() failed?\n", me);
exit(1);
}
dst_llong = &src_llong[num_procs];
for(j=0; j < num_procs; j++) {
src_llong[j] = 16;
dst_llong[j] = 0;
}
shmem_barrier_all();
if ( me != 0 ) {
/* is 'src_*' accessible from PE0? should be. */
if (!shmem_addr_accessible(src_int,0)) {
printf("PE-%d local src_int %p not accessible from PE-%d?\n",
me, (void*)src_int, 0);
exit(1);
}
if (!shmem_addr_accessible(src_float,0)) {
printf("PE-%d local src_float %p not accessible from PE-%d?\n",
me, (void*)src_float, 0);
exit(1);
}
if (!shmem_addr_accessible(src_double,0)) {
printf("PE-%d local src_double %p not accessible from PE-%d?\n",
me, (void*)src_double, 0);
exit(1);
}
if (!shmem_addr_accessible(src_long,0)) {
printf("PE-%d local src_long %p not accessible from PE-%d?\n",
me, (void*)src_long, 0);
exit(1);
}
if (!shmem_addr_accessible(src_llong,0)) {
printf("PE-%d local src_llong %p not accessible from PE-%d?\n",
me, (void*)src_llong, 0);
exit(1);
}
}
shmem_barrier_all();
if ( me == 0 ) {
shmem_quiet();
for(j=1; j < num_procs; j++) {
dst_int[j] = shmem_int_swap(src_int+j,0,j);
if (dst_int[j] != 4) {
printf("PE-%d dst_int[%d] %d != 4?\n",me,j,dst_int[j]);
exit(1);
}
}
shmem_barrier_all();
/* verify remote data */
for(j=1; j < num_procs; j++) {
itmp = shmem_int_g(src_int+j,j);
if (itmp != 0) {
printf("PE-0 int PE[%d] rem(%d) != 0?\n",j,itmp);
exit(1);
}
/* swap back */
dst_int[j] = shmem_int_swap(src_int+j,dst_int[j],j);
if (dst_int[j] != 0) {
printf("PE-0 dst_int[%d] %d != 0?\n",j,dst_int[j]);
exit(1);
}
itmp = shmem_int_g(src_int+j,j);
if (itmp != 4) {
printf("PE-0 PE[%d] rem %d != 4?\n",j,itmp);
exit(1);
}
}
for(j=1; j < num_procs; j++) {
dst_float[j] = shmem_float_swap(src_float+j,0.0,j);
if (dst_float[j] != 4.0) {
printf("PE-0 dst_float[%d] %f != 4.0?\n",j,dst_float[j]);
exit(1);
}
/* verify remote data */
ftmp = shmem_float_g(src_float+j,j);
if (ftmp != 0.0) {
printf("PE-0 float rem(%f) != 0.0?\n",ftmp);
exit(1);
}
/* swap back */
dst_float[j] = shmem_float_swap(src_float+j,dst_float[j],j);
if (dst_float[j] != 0.0) {
printf("PE-0 dst_float[%d] %f != 0.0?\n",j,dst_float[j]);
exit(1);
}
ftmp = shmem_float_g(src_float+j,j);
if (ftmp != 4.0) {
printf("PE-%d float rem(%f) != 4.0?\n",me,ftmp);
exit(1);
}
}
for(j=1; j < num_procs; j++) {
dst_double[j] = shmem_double_swap(src_double+j,0.0,j);
if (dst_double[j] != 8.0) {
printf("PE-0 dst_double[%d] %f != 8.0?\n",j,dst_double[j]);
exit(1);
}
/* verify remote data */
dtmp = shmem_double_g(src_double+j,j);
if (dtmp != 0.0) {
printf("PE-0 float rem(%f) != 0.0?\n",dtmp);
exit(1);
}
dst_double[j] = shmem_double_swap(src_double+j,dst_double[j],j);
if (dst_double[j] != 0.0) {
printf("PE-0 dst_double[%d] %f != 0.0?\n",j,dst_double[j]);
exit(1);
}
dtmp = shmem_double_g(src_double+j,j);
if (dtmp != 8.0) {
printf("PE-0 double rem(%f) != 8.0?\n",dtmp);
exit(1);
}
}
for(j=1; j < num_procs; j++) {
dst_long[j] = shmem_long_swap(src_long+j,0,j);
if (dst_long[j] != 8) {
printf("PE-0 dst_long[%d] %ld != 8?\n",j,dst_long[j]);
exit(1);
}
}
shmem_barrier_all();
/* verify remote data */
for(j=1; j < num_procs; j++) {
ltmp = shmem_long_g(src_long+j,j);
if (ltmp != 0) {
printf("PE-0 PE[%d]long rem(%ld) != 0?\n",j,ltmp);
exit(1);
}
/* swap back */
dst_long[j] = shmem_long_swap(src_long+j,dst_long[j],j);
if (dst_long[j] != 0) {
printf("PE-%d dst_long[%d] %ld != 0?\n",me,j,dst_long[j]);
exit(1);
}
ltmp = shmem_long_g(src_long+j,j);
if (ltmp != 8) {
printf("PE-%d long rem(%ld) != 8?\n",me,ltmp);
exit(1);
}
}
for(j=1; j < num_procs; j++) {
dst_llong[j] = shmem_longlong_swap(src_llong+j,0,j);
if (dst_llong[j] != 16) {
printf("PE-%d dst_llong[%d] %lld != 16?\n",me,j,dst_llong[j]);
exit(1);
}
}
shmem_barrier_all();
/* verify remote data */
for(j=1; j < num_procs; j++) {
lltmp = shmem_longlong_g(src_llong+j,j);
if (lltmp != 0) {
printf("PE-%d long long rem(%lld) != 0?\n",me,lltmp);
exit(1);
}
/* swap back */
dst_llong[j] = shmem_longlong_swap(src_llong+j,dst_llong[j],j);
if (dst_llong[j] != 0) {
printf("PE-%d dst_llong[%d] %lld != 0?\n", me,j,dst_llong[j]);
exit(1);
}
lltmp = shmem_longlong_g(src_llong+j,j);
if (lltmp != 16) {
printf("PE-%d longlong rem(%lld) != 16?\n",me,lltmp);
exit(1);
}
}
}
else {
shmem_int_wait_until(&src_int[me],SHMEM_CMP_EQ,0);
shmem_barrier_all();
shmem_long_wait_until(&src_long[me],SHMEM_CMP_EQ,0);
shmem_barrier_all();
shmem_longlong_wait_until(&src_llong[me],SHMEM_CMP_EQ,0);
shmem_barrier_all();
}
shmem_barrier_all();
shfree(src_int);
shfree(src_float);
shfree(src_double);
shfree(src_long);
shfree(src_llong);
}
if (Verbose)
fprintf(stderr,"[%d] exit\n",_my_pe());
return 0;
}
int main(int argc, char **argv)
{
int i,j;
short modjs, oldjs, oldxmodjs, oldxas, my_pes, vals;
int modji, oldji, oldxmodji, oldxai, my_pei, vali;
long modjl, oldjl, oldxmodjl, oldxal, my_pel, vall;
long long modjll,oldjll,oldxmodjll,oldxall,my_pell,valll;
int my_pe,n_pes;
size_t max_elements,max_elements_bytes;
static short *xs,*xas;
static int *xi,*xai;
static long *xl,*xal;
static long long *xll,*xall;
shmem_init();
my_pe = shmem_my_pe();
n_pes = shmem_n_pes();
my_pes = (short) my_pe;
my_pei = (int) my_pe;
my_pel = (long) my_pe;
my_pell = (long long) my_pe;
vals = 1; vali = 1; vall = 1; valll = 1;
#ifdef HAVE_SET_CACHE_INV
shmem_set_cache_inv();
#endif
/* fail if trying to use only one processor */
if ( n_pes <= 1 ){
fprintf(stderr, "FAIL - test requires at least two PEs\n");
exit(1);
}
if(my_pe == 0)
fprintf(stderr, "shmem_cswap(%s) n_pes=%d\n", argv[0],n_pes);
/* test shmem_short_finc & shmem_short_swap & shmem_short_cswap */
/* shmalloc xs & xas on all pes (only use the ones on PE 0) */
max_elements_bytes = (size_t) (sizeof(short) * n_pes);
xs = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes; i++)
xs[i] = 0;
max_elements_bytes = (size_t) (sizeof(short) * n_pes * ITER);
xas = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes*ITER; i++)
if (((i/(n_pes-1)) % 2) == 0) {
xas[i] = 1;
} else {
xas[i] = 0;
}
count_short = 0;
shmem_barrier_all();
for(i=0; i<ITER; i++) {
if (i == ITER-1) shmem_barrier_all(); /* all PEs participate last time */
if (my_pe != 0) {
oldjs = shmem_short_finc(&count_short, 0); /* get index oldjs from PE 0 */
modjs = (oldjs % (n_pes-1)); /* PE 0 is just the counter/checker */
/* conditionally record PE value in xas[oldjs] --
tells PE involved for each count */
oldxas = shmem_short_cswap(&xas[oldjs], vals, my_pes, 0);
/* printf("PE=%d,i=%d,oldjs=%d,oldxas=%d\n",my_pe,i,oldjs,oldxas); */
if (oldxas == 1) {
/* record PE value in xs[modjs] */
oldxmodjs = shmem_short_swap(&xs[modjs], my_pes, 0);
/* printf("PE=%d,oldjs=%d,modjs=%d,oldxmodjs=%d\n",
my_pe,oldjs,modjs,oldxmodjs); */
}
if (oldxas != 0 && oldxas != 1)
fprintf(stderr, "FAIL PE %d of %d: i=%d, oldxas = %d expected = 0\n",
my_pe, n_pes, i, oldxas);
}
}
shmem_barrier_all();
if (my_pe == 0) { /* check last xs[j] array PEs vs saved ans in xas[i] */
i = (ITER-2)*(n_pes-1);
for(j=1 ; j<n_pes; j++) {
/* printf("j=%d,xs[%d]=%d,xas[%d]=%d\n",j,j-1,xs[j-1],i,xas[i]); */
if (xs[j-1] != xas[i])
fprintf(stderr, "FAIL PE %d of %d: xs[%d] = %d expected = %d\n",
my_pe, n_pes, j-1, xs[j-1], xas[i]);
i++;
}
}
shmem_free(xs); shmem_free(xas);
/* test shmem_int_finc & shmem_int_swap & shmem_int_cswap */
/* shmalloc xi & xai on all pes (only use the ones on PE 0) */
max_elements_bytes = (size_t) (sizeof(int) * n_pes);
xi = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes; i++)
xi[i] = 0;
max_elements_bytes = (size_t) (sizeof(int) * n_pes * ITER);
xai = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes*ITER; i++)
if (((i/(n_pes-1)) % 2) == 0) {
xai[i] = 1;
} else {
xai[i] = 0;
}
count_int = 0;
shmem_barrier_all();
for(i=0; i<ITER; i++) {
if (i == ITER-1) shmem_barrier_all(); /* all PEs participate last time */
if (my_pe != 0) {
oldji = shmem_int_finc(&count_int, 0); /* get index oldji from PE 0 */
modji = (oldji % (n_pes-1)); /* PE 0 is just the counter/checker */
/* conditionally record PE value in xai[oldji] --
tells PE involved for each count */
oldxai = shmem_int_cswap(&xai[oldji], vali, my_pei, 0);
/* printf("PE=%d,i=%d,oldji=%d,oldxai=%d\n",my_pe,i,oldji,oldxai); */
if (oldxai == 1) {
/* record PE value in xi[modji] */
oldxmodji = shmem_int_swap(&xi[modji], my_pei, 0);
/* printf("PE=%d,oldji=%d,modji=%d,oldxmodji=%d\n",
my_pe,oldji,modji,oldxmodji); */
}
if (oldxai != 0 && oldxai != 1)
fprintf(stderr, "FAIL PE %d of %d: i=%d, oldxai = %d expected = 0\n",
my_pe, n_pes, i, oldxai);
}
}
shmem_barrier_all();
if (my_pe == 0) { /* check last xi[j] array PEs vs saved ans in xai[i] */
i = (ITER-2)*(n_pes-1);
for(j=1 ; j<n_pes; j++) {
/* printf("j=%d,xi[%d]=%d,xai[%d]=%d\n",j,j-1,xi[j-1],i,xai[i]); */
if (xi[j-1] != xai[i])
fprintf(stderr, "FAIL PE %d of %d: xi[%d] = %d expected = %d\n",
my_pe, n_pes, j-1, xi[j-1], xai[i]);
i++;
}
}
shmem_free(xi); shmem_free(xai);
/* test shmem_long_finc & shmem_long_swap & shmem_long_cswap */
/* shmalloc xl & xal on all pes (only use the ones on PE 0) */
max_elements_bytes = (size_t) (sizeof(long) * n_pes);
xl = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes; i++)
xl[i] = 0;
max_elements_bytes = (size_t) (sizeof(long) * n_pes * ITER);
xal = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes*ITER; i++)
if (((i/(n_pes-1)) % 2) == 0) {
xal[i] = 1;
} else {
xal[i] = 0;
}
count_long = 0;
shmem_barrier_all();
for(i=0; i<ITER; i++) {
if (i == ITER-1) shmem_barrier_all(); /* all PEs participate last time */
if (my_pe != 0) {
oldjl = shmem_long_finc(&count_long, 0); /* get index oldjl from PE 0 */
modjl = (oldjl % (n_pes-1)); /* PE 0 is just the counter/checker */
/* conditionally record PE value in xal[oldjl] --
tells PE involved for each count */
oldxal = shmem_long_cswap(&xal[oldjl], vall, my_pel, 0);
/* printf("PE=%d,i=%d,oldjl=%d,oldxal=%d\n",my_pe,i,oldjl,oldxal); */
if (oldxal == 1) {
/* record PE value in xl[modjl] */
oldxmodjl = shmem_long_swap(&xl[modjl], my_pel, 0);
/* printf("PE=%d,oldjl=%ld,modjl=%ld,oldxmodjl=%ld\n",
my_pe,oldjl,modjl,oldxmodjl); */
}
if (oldxal != 0 && oldxal != 1)
fprintf(stderr, "FAIL PE %d of %d: i=%d, oldxal = %ld expected = 0\n",
my_pe, n_pes, i, oldxal);
}
}
shmem_barrier_all();
if (my_pe == 0) { /* check last xl[j] array PEs vs saved ans in xal[i] */
i = (ITER-2)*(n_pes-1);
for(j=1 ; j<n_pes; j++) {
/* printf("j=%d,xl[%d]=%ld,xal[%d]=%ld\n",j,j-1,xl[j-1],i,xal[i]); */
if (xl[j-1] != xal[i])
fprintf(stderr, "FAIL PE %d of %d: xl[%d] = %ld expected = %ld\n",
my_pe, n_pes, j-1, xl[j-1], xal[i]);
i++;
}
}
shmem_free(xl); shmem_free(xal);
/* test shmem_longlong_finc & shmem_longlong_swap & shmem_longlong_cswap */
#ifdef HAVE_LONG_LONG
/* shmalloc xll & xall on all pes (only use the ones on PE 0) */
max_elements_bytes = (size_t) (sizeof(long long) * n_pes);
xll = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes; i++)
xll[i] = 0;
max_elements_bytes = (size_t) (sizeof(long long) * n_pes * ITER);
xall = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes*ITER; i++)
if (((i/(n_pes-1)) % 2) == 0) {
xall[i] = 1;
} else {
xall[i] = 0;
}
count_longlong = 0;
shmem_barrier_all();
for(i=0; i<ITER; i++) {
if (i == ITER-1) shmem_barrier_all(); /* all PEs participate last time */
if (my_pe != 0) {
oldjll = shmem_longlong_finc(&count_longlong, 0); /* get index oldjll from PE 0 */
modjll = (oldjll % (n_pes-1)); /* PE 0 is just the counter/checker */
/* conditionally record PE value in xall[oldjll] --
tells PE involved for each count */
oldxall = shmem_longlong_cswap(&xall[oldjll], valll, my_pell, 0);
/* printf("PE=%d,i=%d,oldjll=%d,oldxall=%d\n",my_pe,i,oldjll,oldxall); */
if (oldxall == 1) {
/* record PE value in xll[modjll] */
oldxmodjll = shmem_longlong_swap(&xll[modjll], my_pell, 0);
/* printf("PE=%d,oldjll=%ld,modjll=%ld,oldxmodjll=%ld\n",
my_pe,oldjll,modjll,oldxmodjll); */
}
if (oldxall != 0 && oldxall != 1)
fprintf(stderr, "FAIL PE %d of %d: i=%d, oldxall = %ld expected = 0\n",
my_pe, n_pes, i, oldxall);
}
}
shmem_barrier_all();
if (my_pe == 0) { /* check last xll[j] array PEs vs saved ans in xall[i] */
i = (ITER-2)*(n_pes-1);
for(j=1 ; j<n_pes; j++) {
/* printf("j=%d,xll[%d]=%ld,xall[%d]=%ld\n",j,j-1,xll[j-1],i,xall[i]); */
if (xll[j-1] != xall[i])
fprintf(stderr, "FAIL PE %d of %d: xll[%d] = %ld expected = %ld\n",
my_pe, n_pes, j-1, xll[j-1], xall[i]);
i++;
}
}
shmem_free(xll); shmem_free(xall);
#endif
#ifdef SHMEM_C_GENERIC_32
/* test shmem_finc & shmem_swap & shmem_cswap (GENERIC 32) */
/* shmalloc xi & xai on all pes (only use the ones on PE 0) */
max_elements_bytes = (size_t) (sizeof(int) * n_pes);
xi = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes; i++)
xi[i] = 0;
max_elements_bytes = (size_t) (sizeof(int) * n_pes * ITER);
xai = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes*ITER; i++)
if (((i/(n_pes-1)) % 2) == 0) {
xai[i] = 1;
} else {
xai[i] = 0;
}
count_int = 0;
shmem_barrier_all();
for(i=0; i<ITER; i++) {
if (i == ITER-1) shmem_barrier_all(); /* all PEs participate last time */
if (my_pe != 0) {
oldji = shmem_finc(&count_int, 0); /* get index oldji from PE 0 */
modji = (oldji % (n_pes-1)); /* PE 0 is just the counter/checker */
/* conditionally record PE value in xai[oldji] --
tells PE involved for each count */
oldxai = shmem_cswap(&xai[oldji], vali, my_pei, 0);
/* printf("PE=%d,i=%d,oldji=%d,oldxai=%d\n",my_pe,i,oldji,oldxai); */
if (oldxai == 1) {
/* record PE value in xi[modji] */
oldxmodji = shmem_swap(&xi[modji], my_pei, 0);
/* printf("PE=%d,oldji=%d,modji=%d,oldxmodji=%d\n",
my_pe,oldji,modji,oldxmodji); */
}
if (oldxai != 0 && oldxai != 1)
fprintf(stderr, "FAIL pe %d of %d: i=%d, oldxai = %d expected = 0\n",
my_pe, n_pes, i, oldxai);
}
}
shmem_barrier_all();
if (my_pe == 0) { /* check last xi[j] array PEs vs saved ans in xai[i] */
i = (ITER-2)*(n_pes-1);
for(j=1 ; j<n_pes; j++) {
/* printf("j=%d,xi[%d]=%d,xai[%d]=%d\n",j,j-1,xi[j-1],i,xai[i]); */
if (xi[j-1] != xai[i])
fprintf(stderr, "FAIL pe %d of %d: xi[%d] = %d expected = %d\n",
my_pe, n_pes, j-1, xi[j-1], xai[i]);
i++;
}
}
shmem_free(xi); shmem_free(xai);
#else
/* test shmem_finc & shmem_swap & shmem_cswap (GENERIC 64) */
/* shmalloc xl & xal on all pes (only use the ones on PE 0) */
max_elements_bytes = (size_t) (sizeof(long) * n_pes);
xl = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes; i++)
xl[i] = 0;
max_elements_bytes = (size_t) (sizeof(long) * n_pes * ITER);
xal = shmem_malloc( max_elements_bytes );
for(i=0; i<n_pes*ITER; i++)
if (((i/(n_pes-1)) % 2) == 0) {
xal[i] = 1;
} else {
xal[i] = 0;
}
count_long = 0;
shmem_barrier_all();
for(i=0; i<ITER; i++) {
if (i == ITER-1) shmem_barrier_all(); /* all PEs participate last time */
if (my_pe != 0) {
oldjl = shmem_finc(&count_long, 0); /* get index oldjl from PE 0 */
modjl = (oldjl % (n_pes-1)); /* PE 0 is just the counter/checker */
/* conditionally record PE value in xal[oldjl] --
tells PE involved for each count */
oldxal = shmem_cswap(&xal[oldjl], vall, my_pell, 0);
/* printf("PE=%d,i=%d,oldjl=%d,oldxal=%d\n",my_pe,i,oldjl,oldxal); */
if (oldxal == 1) {
/* record PE value in xl[modjl] */
oldxmodjl = shmem_swap(&xl[modjl], my_pell, 0);
/* printf("PE=%d,oldjl=%ld,modjl=%ld,oldxmodjl=%ld\n",
my_pe,oldjl,modjl,oldxmodjl); */
}
if (oldxal != 0 && oldxal != 1)
fprintf(stderr, "FAIL pe %d of %d: i=%d, oldxal = %ld expected = 0\n",
my_pe, n_pes, i, oldxal);
}
}
shmem_barrier_all();
if (my_pe == 0) { /* check last xl[j] array PEs vs saved ans in xal[i] */
i = (ITER-2)*(n_pes-1);
for(j=1 ; j<n_pes; j++) {
/* printf("j=%d,xl[%d]=%ld,xal[%d]=%ld\n",j,j-1,xl[j-1],i,xal[i]); */
if (xl[j-1] != xal[i])
fprintf(stderr, "FAIL pe %d of %d: xl[%d] = %ld expected = %10.2f\n",
my_pe, n_pes, j-1, xl[j-1], xal[i]);
i++;
}
}
shmem_free(xl); shmem_free(xal);
#endif
shmem_barrier_all();
#ifdef NEEDS_FINALIZE
shmem_finalize();
#endif
return 0;
}
void Worker ()
{
newshortestlen = INTMAX;
isdone = 0; isnewpath = 0;
Msg_t outbuf[NumCities];
shmem_barrier_all();
shmemx_am_request(MASTER_PE, hid_SUBSCRIBE, NULL, 0);
shmemx_am_quiet();
while (1) {
if (shmem_int_swap(&isnewpath,0,mype)) {
//printf("Worker %d just received new path\n",mype);
//fflush(stdout);
msg_in.visited++;
if (msg_in.visited==NumCities) {
//printf("Worker %d checking for short distance\n",mype);
//fflush(stdout);
int d1 = Dist[(msg_in.city[NumCities-2])*NumCities + msg_in.city[NumCities-1]];
int d2 = Dist[(msg_in.city[NumCities-1]) * NumCities ];
if (d1 * d2) {
// both edges exist
msg_in.length += d1 + d2;
// if path is good, send it to master
if (msg_in.length < newshortestlen)
shmemx_am_request(MASTER_PE, hid_BESTPATH, &msg_in, sizeof(Msg_t));
}
// not a valid path, ask for another partial path
}
else {
//printf("Worker %d evaluating path\n",mype);
//fflush(stdout);
// For each city not yet visited, extend the path:
// (use of symmetric buffer msg_in to compute every extended path)
int length = msg_in.length;
int pathcnt = 0;
for (int i=msg_in.visited-1; i<NumCities; i++) {
// swap city[i] and city[visted-1]
if (i > msg_in.visited-1) {
int tmp = msg_in.city[msg_in.visited-1];
msg_in.city[msg_in.visited-1] = msg_in.city[i];
msg_in.city[i] = tmp;
}
// visit city[visited-1]
if (int d = Dist[(msg_in.city[msg_in.visited-2])*NumCities +
msg_in.city[msg_in.visited-1] ]) {
msg_in.length = length + d;
if (msg_in.length < newshortestlen) {
memcpy(&outbuf[pathcnt],&msg_in,sizeof(Msg_t));
pathcnt++;
}
}
}
if(pathcnt)
shmemx_am_request(MASTER_PE, hid_PUTPATH, outbuf, pathcnt*sizeof(Msg_t));
}
shmemx_am_request(MASTER_PE, hid_SUBSCRIBE, NULL, 0);
shmemx_am_quiet();
} /* end of new path check */
if (shmem_int_swap(&isdone,0,mype)) {
printf("Worker %d received DONE_TAG ..\n", mype);
break;
}
} /* end of while(1) */
shmem_barrier_all();
}
int
main ()
{
int me, npes;
int *dest1;
float *dest2;
long *dest3;
double *dest4;
long long *dest5;
int swapped_val1, new_val1;
float swapped_val2, new_val2;
long swapped_val3, new_val3;
double swapped_val4, new_val4;
long long swapped_val5, new_val5;
int success = 1;
int success1_p1;
int success2_p1;
int success3_p1;
int success4_p1;
int success5_p1;
shmem_init ();
me = shmem_my_pe ();
npes = shmem_n_pes ();
shmem_barrier_all ();
/* Checks if there are atleast 2 executing PEs */
if (npes > 1) {
dest1 = (int *) shmem_malloc (sizeof (*dest1));
dest2 = (float *) shmem_malloc (sizeof (*dest2));
dest3 = (long *) shmem_malloc (sizeof (*dest3));
dest4 = (double *) shmem_malloc (sizeof (*dest4));
dest5 = (long long *) shmem_malloc (sizeof (*dest5));
*dest1 = *dest2 = *dest3 = *dest4 = *dest5 = me;
new_val1 = new_val2 = new_val3 = new_val4 = new_val5 = me;
success1_p1 = success1_p2 = success2_p1 = success2_p2 = success3_p1 =
success3_p2 = success4_p1 = success4_p2 = success5_p1 =
success5_p2 = -1;
shmem_barrier_all ();
swapped_val1 = shmem_int_swap (dest1, new_val1, (me + 1) % npes);
swapped_val2 = shmem_float_swap (dest2, new_val2, (me + 1) % npes);
swapped_val3 = shmem_long_swap (dest3, new_val3, (me + 1) % npes);
swapped_val4 = shmem_double_swap (dest4, new_val4, (me + 1) % npes);
swapped_val5 = shmem_longlong_swap (dest5, new_val5, (me + 1) % npes);
/* To validate the working of swap we need to check the value received
at the PE that initiated the swap as well as the dest PE */
if (me == 0) {
if (swapped_val1 == 1) {
success1_p1 = 1;
}
if (swapped_val2 == 1) {
success2_p1 = 1;
}
if (swapped_val3 == 1) {
success3_p1 = 1;
}
if (swapped_val4 == 1) {
success4_p1 = 1;
}
if (swapped_val5 == 1) {
success5_p1 = 1;
}
}
if (me == 1) {
if (*dest1 == 0) {
shmem_int_put (&success1_p2, &success, 1, 0);
}
if (*dest2 == 0) {
shmem_int_put (&success2_p2, &success, 1, 0);
}
if (*dest3 == 0) {
shmem_int_put (&success3_p2, &success, 1, 0);
}
if (*dest4 == 0) {
shmem_int_put (&success4_p2, &success, 1, 0);
}
if (*dest5 == 0) {
shmem_int_put (&success5_p2, &success, 1, 0);
}
}
shmem_barrier_all ();
if (me == 0) {
if (success1_p1 && success1_p2) {
printf ("Test shmem_int_swap: Passed\n");
}
else {
printf ("Test shmem_int_swap: Failed\n");
}
if (success2_p1 && success2_p2) {
printf ("Test shmem_float_swap: Passed\n");
}
else {
printf ("Test shmem_float_swap: Failed\n");
}
if (success3_p1 && success3_p2) {
printf ("Test shmem_long_swap: Passed\n");
}
else {
printf ("Test shmem_long_swap: Failed\n");
}
if (success4_p1 && success4_p2) {
printf ("Test shmem_double_swap: Passed\n");
}
else {
printf ("Test shmem_double_swap: Failed\n");
}
if (success5_p1 && success5_p2) {
printf ("Test shmem_longlong_swap: Passed\n");
}
else {
printf ("Test shmem_longlong_swap: Failed\n");
}
}
shmem_barrier_all ();
/* Test conditional swaps shmem_longlong_cswap, shmem_long_cswap,
shmem_int_cswap, */
*dest1 = *dest3 = *dest5 = me;
new_val1 = new_val3 = new_val5 = me;
success1_p1 = success1_p2 = success3_p1 = success3_p2 = success5_p1 =
success5_p2 = -1;
shmem_barrier_all ();
swapped_val1 = shmem_int_cswap (dest1, me + 1, (long) me, 1);
swapped_val3 = shmem_long_cswap (dest3, me + 1, (long) me, 1);
swapped_val5 = shmem_longlong_cswap (dest5, me + 1, (long) me, 1);
/* To validate the working of conditionalswap we need to check the
value received at the PE that initiated the conditional swap as
well as the dest PE */
if (me == 0) {
if (swapped_val1 == 1) {
success1_p1 = 1;
}
if (swapped_val3 == 1) {
success3_p1 = 1;
}
if (swapped_val5 == 1) {
success5_p1 = 1;
}
}
if (me == 1) {
if (*dest1 == 0) {
shmem_int_put (&success1_p2, &success, 1, 0);
}
if (*dest3 == 0) {
shmem_int_put (&success3_p2, &success, 1, 0);
}
if (*dest5 == 0) {
shmem_int_put (&success5_p2, &success, 1, 0);
}
}
shmem_barrier_all ();
if (me == 0) {
if (success1_p1 && success1_p2) {
printf ("Test shmem_int_cswap: Passed\n");
}
else {
printf ("Test shmem_int_cswap: Failed\n");
}
if (success3_p1 && success3_p2) {
printf ("Test shmem_long_cswap: Passed\n");
}
else {
printf ("Test shmem_long_cswap: Failed\n");
}
if (success5_p1 && success5_p2) {
printf ("Test shmem_longlong_cswap: Passed\n");
}
else {
printf ("Test shmem_longlong_cswap: Failed\n");
}
}
shmem_barrier_all ();
/* Test shmem_long_fadd, shmem_int_fadd, shmem_longlong_fadd */
*dest1 = *dest3 = *dest5 = me;
new_val1 = new_val3 = new_val5 = me;
success1_p1 = success1_p2 = success3_p1 = success3_p2 = success5_p1 =
success5_p2 = -1;
shmem_barrier_all ();
swapped_val1 = shmem_int_fadd (dest1, 1, 0);
swapped_val3 = shmem_long_fadd (dest3, 1, 0);
swapped_val5 = shmem_longlong_fadd (dest5, 1, 0);
/* To validate the working of fetch and add we need to check the old
value received at the PE that initiated the fetch and increment as
well as the new value on the dest PE */
if (me != 0) {
if (swapped_val1 == 0) {
success1_p1 = 1;
}
if (swapped_val3 == 0) {
success3_p1 = 1;
}
if (swapped_val5 == 0) {
success5_p1 = 1;
}
}
if (me == 0) {
if (*dest1 == npes - 1) {
shmem_int_put (&success1_p2, &success, 1, npes - 1);
}
if (*dest3 == npes - 1) {
shmem_int_put (&success3_p2, &success, 1, npes - 1);
}
if (*dest5 == npes - 1) {
shmem_int_put (&success5_p2, &success, 1, npes - 1);
}
}
shmem_barrier_all ();
if (me == npes - 1) {
if (success1_p1 && success1_p2) {
printf ("Test shmem_int_fadd: Passed\n");
}
else {
printf ("Test shmem_int_fadd: Failed\n");
}
if (success3_p1 && success3_p2) {
printf ("Test shmem_long_fadd: Passed\n");
}
else {
printf ("Test shmem_long_fadd: Failed\n");
}
if (success5_p1 && success5_p2) {
printf ("Test shmem_longlong_fadd: Passed\n");
}
else {
printf ("Test shmem_longlong_fadd: Failed\n");
}
}
shmem_barrier_all ();
/* Test shmem_long_finc, shmem_int_finc, shmem_longlong_finc */
*dest1 = *dest3 = *dest5 = me;
new_val1 = new_val3 = new_val5 = me;
success1_p1 = success1_p2 = success3_p1 = success3_p2 = success5_p1 =
success5_p2 = -1;
shmem_barrier_all ();
swapped_val1 = shmem_int_finc (dest1, 0);
swapped_val3 = shmem_long_finc (dest3, 0);
swapped_val5 = shmem_longlong_finc (dest5, 0);
/* To validate the working of fetch and increment we need to check the
old value received at the PE that initiated the fetch and increment
as well as the new value on the dest PE */
if (me != 0) {
if (swapped_val1 == 0) {
success1_p1 = 1;
}
if (swapped_val3 == 0) {
success3_p1 = 1;
}
if (swapped_val5 == 0) {
success5_p1 = 1;
}
}
if (me == 0) {
if (*dest1 == npes - 1) {
shmem_int_put (&success1_p2, &success, 1, npes - 1);
}
if (*dest3 == npes - 1) {
shmem_int_put (&success3_p2, &success, 1, npes - 1);
}
if (*dest5 == npes - 1) {
shmem_int_put (&success5_p2, &success, 1, npes - 1);
}
}
shmem_barrier_all ();
if (me == npes - 1) {
if (success1_p1 && success1_p2) {
printf ("Test shmem_int_finc: Passed\n");
}
else {
printf ("Test shmem_int_finc: Failed\n");
}
if (success3_p1 && success3_p2) {
printf ("Test shmem_long_finc: Passed\n");
}
else {
printf ("Test shmem_long_finc: Failed\n");
}
if (success5_p1 && success5_p2) {
printf ("Test shmem_longlong_finc: Passed\n");
}
else {
printf ("Test shmem_longlong_finc: Failed\n");
}
}
shmem_barrier_all ();
shmem_free (dest1);
shmem_free (dest2);
shmem_free (dest3);
shmem_free (dest4);
shmem_free (dest5);
}
else {
printf
("Number of PEs must be > 1 to test shmem atomics, test skipped\n");
}
shmem_finalize ();
return 0;
}
