Beispiel #1
0
int main(int argc, char **argv){

  int    *becell, *ecell,  *bound, *bedge, *edge, *cell;
  double  *x, *q, *qold, *adt, *res;

  int    niter;
  double  rms;

  // set constants and initialise flow field and residual
  printf("initialising flow field \n");

  gam = 1.4f;
  gm1 = gam - 1.0f;
  cfl = 0.9f;
  eps = 0.05f;

  double mach  = 0.4f;
  double alpha = 3.0f*atan(1.0f)/45.0f;  
  double p     = 1.0f;
  double r     = 1.0f;
  double u     = sqrt(gam*p/r)*mach;
  double e     = p/(r*gm1) + 0.5f*u*u;

  qinf[0] = r;
  qinf[1] = r*u;
  qinf[2] = 0.0f;
  qinf[3] = r*e;


  // OP initialisation

  op_init(argc,argv,2);

  char file[] = "new_grid.h5";//"new_grid-26mil.h5";
  
  // declare sets, pointers, datasets and global constants

  op_set nodes  = op_decl_set_hdf5(file, "nodes");
  op_set edges  = op_decl_set_hdf5(file,  "edges");
  op_set bedges = op_decl_set_hdf5(file, "bedges");
  op_set cells  = op_decl_set_hdf5(file,  "cells");

  op_map pedge   = op_decl_map_hdf5(edges, nodes, 2, file, "pedge");
  op_map pecell  = op_decl_map_hdf5(edges, cells,2, file, "pecell");
  op_map pbedge  = op_decl_map_hdf5(bedges,nodes,2, file, "pbedge");
  op_map pbecell = op_decl_map_hdf5(bedges,cells,1, file, "pbecell");
  op_map pcell   = op_decl_map_hdf5(cells, nodes,4, file, "pcell");

  op_dat p_bound = op_decl_dat_hdf5(bedges,1,"int"  ,file,"p_bound");
  op_dat p_x     = op_decl_dat_hdf5(nodes ,2,"double",file,"p_x");
  op_dat p_q     = op_decl_dat_hdf5(cells ,4,"double",file,"p_q");
  op_dat p_qold  = op_decl_dat_hdf5(cells ,4,"double",file,"p_qold");
  op_dat p_adt   = op_decl_dat_hdf5(cells ,1,"double",file,"p_adt");
  op_dat p_res   = op_decl_dat_hdf5(cells ,4,"double",file,"p_res");

  op_decl_const2("gam",1,"double",&gam  );
  op_decl_const2("gm1",1,"double",&gm1  );
  op_decl_const2("cfl",1,"double",&cfl  );
  op_decl_const2("eps",1,"double",&eps  );
  op_decl_const2("mach",1,"double",&mach );
  op_decl_const2("alpha",1,"double",&alpha);
  op_decl_const2("qinf",4,"double",qinf  );

  op_diagnostic_output();

  niter = 1000;

  for(int iter=1; iter<=niter; iter++) {

//  save old flow solution

    op_par_loop_save_soln("save_soln", cells,
                op_arg_dat(p_q,   -1,OP_ID, 4,"double",OP_READ ),
                op_arg_dat(p_qold,-1,OP_ID, 4,"double",OP_WRITE));

//  predictor/corrector update loop

    for(int k=0; k<2; k++) {

//    calculate area/timstep

      op_par_loop_adt_calc("adt_calc",cells,
                  op_arg_dat(p_x,   0,pcell, 2,"double",OP_READ ),
                  op_arg_dat(p_x,   1,pcell, 2,"double",OP_READ ),
                  op_arg_dat(p_x,   2,pcell, 2,"double",OP_READ ),
                  op_arg_dat(p_x,   3,pcell, 2,"double",OP_READ ),
                  op_arg_dat(p_q,  -1,OP_ID, 4,"double",OP_READ ),
                  op_arg_dat(p_adt,-1,OP_ID, 1,"double",OP_WRITE));

//    calculate flux residual

      op_par_loop_res_calc("res_calc",edges,
                  op_arg_dat(p_x,    0,pedge, 2,"double",OP_READ),
                  op_arg_dat(p_x,    1,pedge, 2,"double",OP_READ),
                  op_arg_dat(p_q,    0,pecell,4,"double",OP_READ),
                  op_arg_dat(p_q,    1,pecell,4,"double",OP_READ),
                  op_arg_dat(p_adt,  0,pecell,1,"double",OP_READ),
                  op_arg_dat(p_adt,  1,pecell,1,"double",OP_READ),
                  op_arg_dat(p_res,  0,pecell,4,"double",OP_INC ),
                  op_arg_dat(p_res,  1,pecell,4,"double",OP_INC ));

      op_par_loop_bres_calc("bres_calc",bedges,
                  op_arg_dat(p_x,     0,pbedge, 2,"double",OP_READ),
                  op_arg_dat(p_x,     1,pbedge, 2,"double",OP_READ),
                  op_arg_dat(p_q,     0,pbecell,4,"double",OP_READ),
                  op_arg_dat(p_adt,   0,pbecell,1,"double",OP_READ),
                  op_arg_dat(p_res,   0,pbecell,4,"double",OP_INC ),
                  op_arg_dat(p_bound,-1,OP_ID  ,1,"int",  OP_READ));

//    update flow field

      rms = 0.0;

      op_par_loop_update("update",cells,
                  op_arg_dat(p_qold,-1,OP_ID, 4,"double",OP_READ ),
                  op_arg_dat(p_q,   -1,OP_ID, 4,"double",OP_WRITE),
                  op_arg_dat(p_res, -1,OP_ID, 4,"double",OP_RW   ),
                  op_arg_dat(p_adt, -1,OP_ID, 1,"double",OP_READ ),
                  op_arg_gbl(&rms,1,"double",OP_INC));
    }

//  print iteration history

    rms = sqrt(rms/(double) cells->size);

    if (iter%100 == 0)
       printf(" %d  %10.5e \n",iter,rms);
  }

  op_timing_output();
  
}
int main(int argc, char **argv)
{
  // OP initialisation
  op_init(argc,argv,2);

  //MPI for user I/O
  int my_rank;
  int comm_size;
  MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
  MPI_Comm_size(MPI_COMM_WORLD, &comm_size);

  //timer
  double cpu_t1, cpu_t2, wall_t1, wall_t2;

  int    *becell, *ecell,  *bound, *bedge, *edge, *cell;
  double  *x, *q, *qold, *adt, *res;

  int    nnode,ncell,nedge,nbedge,niter;
  double  rms;

  /**------------------------BEGIN I/O and PARTITIONING -------------------**/

  op_timers(&cpu_t1, &wall_t1);

  /* read in grid from disk on root processor */
  FILE *fp;

  if ( (fp = fopen("new_grid.dat","r")) == NULL) {
    op_printf("can't open file new_grid.dat\n"); exit(-1);
  }

  int   g_nnode,g_ncell,g_nedge,g_nbedge;

  check_scan(fscanf(fp,"%d %d %d %d \n",&g_nnode, &g_ncell, &g_nedge, &g_nbedge), 4);

  int *g_becell = 0, *g_ecell = 0, *g_bound = 0, *g_bedge = 0, *g_edge = 0, *g_cell = 0;
  double *g_x = 0,*g_q = 0, *g_qold = 0, *g_adt = 0, *g_res = 0;

  // set constants

  op_printf("initialising flow field\n");
  gam = 1.4f;
  gm1 = gam - 1.0f;
  cfl = 0.9f;
  eps = 0.05f;

  double mach  = 0.4f;
  double alpha = 3.0f*atan(1.0f)/45.0f;
  double p     = 1.0f;
  double r     = 1.0f;
  double u     = sqrt(gam*p/r)*mach;
  double e     = p/(r*gm1) + 0.5f*u*u;

  qinf[0] = r;
  qinf[1] = r*u;
  qinf[2] = 0.0f;
  qinf[3] = r*e;

  op_printf("reading in grid \n");
  op_printf("Global number of nodes, cells, edges, bedges = %d, %d, %d, %d\n"
      ,g_nnode,g_ncell,g_nedge,g_nbedge);

  if(my_rank == MPI_ROOT) {
    g_cell   = (int *) malloc(4*g_ncell*sizeof(int));
    g_edge   = (int *) malloc(2*g_nedge*sizeof(int));
    g_ecell  = (int *) malloc(2*g_nedge*sizeof(int));
    g_bedge  = (int *) malloc(2*g_nbedge*sizeof(int));
    g_becell = (int *) malloc(  g_nbedge*sizeof(int));
    g_bound  = (int *) malloc(  g_nbedge*sizeof(int));

    g_x      = (double *) malloc(2*g_nnode*sizeof(double));
    g_q      = (double *) malloc(4*g_ncell*sizeof(double));
    g_qold   = (double *) malloc(4*g_ncell*sizeof(double));
    g_res    = (double *) malloc(4*g_ncell*sizeof(double));
    g_adt    = (double *) malloc(  g_ncell*sizeof(double));

    for (int n=0; n<g_nnode; n++){
      check_scan(fscanf(fp,"%lf %lf \n",&g_x[2*n], &g_x[2*n+1]), 2);
    }

    for (int n=0; n<g_ncell; n++) {
      check_scan(fscanf(fp,"%d %d %d %d \n",&g_cell[4*n  ], &g_cell[4*n+1],
            &g_cell[4*n+2], &g_cell[4*n+3]), 4);
    }

    for (int n=0; n<g_nedge; n++) {
      check_scan(fscanf(fp,"%d %d %d %d \n",&g_edge[2*n],&g_edge[2*n+1],
            &g_ecell[2*n],&g_ecell[2*n+1]), 4);
    }

    for (int n=0; n<g_nbedge; n++) {
      check_scan(fscanf(fp,"%d %d %d %d \n",&g_bedge[2*n],&g_bedge[2*n+1],
            &g_becell[n],&g_bound[n]), 4);
    }

    //initialise flow field and residual

    for (int n=0; n<g_ncell; n++) {
      for (int m=0; m<4; m++) {
        g_q[4*n+m] = qinf[m];
        g_res[4*n+m] = 0.0f;
      }
    }
  }

  fclose(fp);

  nnode = compute_local_size (g_nnode, comm_size, my_rank);
  ncell = compute_local_size (g_ncell, comm_size, my_rank);
  nedge = compute_local_size (g_nedge, comm_size, my_rank);
  nbedge = compute_local_size (g_nbedge, comm_size, my_rank);

  op_printf("Number of nodes, cells, edges, bedges on process %d = %d, %d, %d, %d\n"
      ,my_rank,nnode,ncell,nedge,nbedge);

  /*Allocate memory to hold local sets, mapping tables and data*/
  cell   = (int *) malloc(4*ncell*sizeof(int));
  edge   = (int *) malloc(2*nedge*sizeof(int));
  ecell  = (int *) malloc(2*nedge*sizeof(int));
  bedge  = (int *) malloc(2*nbedge*sizeof(int));
  becell = (int *) malloc(  nbedge*sizeof(int));
  bound  = (int *) malloc(  nbedge*sizeof(int));

  x      = (double *) malloc(2*nnode*sizeof(double));
  q      = (double *) malloc(4*ncell*sizeof(double));
  qold   = (double *) malloc(4*ncell*sizeof(double));
  res    = (double *) malloc(4*ncell*sizeof(double));
  adt    = (double *) malloc(  ncell*sizeof(double));

  /* scatter sets, mappings and data on sets*/
  scatter_int_array(g_cell, cell, comm_size, g_ncell,ncell, 4);
  scatter_int_array(g_edge, edge, comm_size, g_nedge,nedge, 2);
  scatter_int_array(g_ecell, ecell, comm_size, g_nedge,nedge, 2);
  scatter_int_array(g_bedge, bedge, comm_size, g_nbedge,nbedge, 2);
  scatter_int_array(g_becell, becell, comm_size, g_nbedge,nbedge, 1);
  scatter_int_array(g_bound, bound, comm_size, g_nbedge,nbedge, 1);

  scatter_double_array(g_x, x, comm_size, g_nnode,nnode, 2);
  scatter_double_array(g_q, q, comm_size, g_ncell,ncell, 4);
  scatter_double_array(g_qold, qold, comm_size, g_ncell,ncell, 4);
  scatter_double_array(g_res, res, comm_size, g_ncell,ncell, 4);
  scatter_double_array(g_adt, adt, comm_size, g_ncell,ncell, 1);

  /*Freeing memory allocated to gloabal arrays on rank 0
    after scattering to all processes*/
  if(my_rank == MPI_ROOT) {
    free(g_cell);
    free(g_edge);
    free(g_ecell);
    free(g_bedge);
    free(g_becell);
    free(g_bound);
    free(g_x );
    free(g_q);
    free(g_qold);
    free(g_adt);
    free(g_res);
  }

  op_timers(&cpu_t2, &wall_t2);
  op_printf("Max total file read time = %f\n", wall_t2-wall_t1);

  /**------------------------END I/O and PARTITIONING -----------------------**/

  // declare sets, pointers, datasets and global constants

  op_set nodes  = op_decl_set(nnode,  "nodes");
  op_set edges  = op_decl_set(nedge,  "edges");
  op_set bedges = op_decl_set(nbedge, "bedges");
  op_set cells  = op_decl_set(ncell,  "cells");

  op_map pedge   = op_decl_map(edges, nodes,2,edge,  "pedge");
  op_map pecell  = op_decl_map(edges, cells,2,ecell, "pecell");
  op_map pbedge  = op_decl_map(bedges,nodes,2,bedge, "pbedge");
  op_map pbecell = op_decl_map(bedges,cells,1,becell,"pbecell");
  op_map pcell   = op_decl_map(cells, nodes,4,cell,  "pcell");

  op_dat p_bound = op_decl_dat(bedges,1,"int"  ,bound,"p_bound");
  op_dat p_x     = op_decl_dat(nodes ,2,"double",x    ,"p_x");
  op_dat p_q     = op_decl_dat(cells ,4,"double",q    ,"p_q");
  //op_dat p_qold  = op_decl_dat(cells ,4,"double",qold ,"p_qold");
  //op_dat p_adt   = op_decl_dat(cells ,1,"double",adt  ,"p_adt");
  //op_dat p_res   = op_decl_dat(cells ,4,"double",res  ,"p_res");

  // p_res, p_adt and p_qold  now declared as a temp op_dats during
  // the execution of the time-marching loop

  op_decl_const2("gam",1,"double",&gam  );
  op_decl_const2("gm1",1,"double",&gm1  );
  op_decl_const2("cfl",1,"double",&cfl  );
  op_decl_const2("eps",1,"double",&eps  );
  op_decl_const2("mach",1,"double",&mach );
  op_decl_const2("alpha",1,"double",&alpha);
  op_decl_const2("qinf",4,"double",qinf  );

  op_diagnostic_output();

  //trigger partitioning and halo creation routines
  op_partition("PTSCOTCH", "KWAY", cells, pecell, p_x);

  //initialise timers for total execution wall time
  op_timers(&cpu_t1, &wall_t1);

  niter = 1000;
  for(int iter=1; iter<=niter; iter++) {

    double* tmp_elem = NULL;
    op_dat p_res   = op_decl_dat_temp(cells ,4,"double",tmp_elem,"p_res");
    op_dat p_adt   = op_decl_dat_temp(cells ,1,"double",tmp_elem,"p_adt");
    op_dat p_qold  = op_decl_dat_temp(cells ,4,"double",qold ,"p_qold");

    //save old flow solution
    op_par_loop_save_soln("save_soln",cells,
               op_arg_dat(p_q,-1,OP_ID,4,"double",OP_READ),
               op_arg_dat(p_qold,-1,OP_ID,4,"double",OP_WRITE));

    //  predictor/corrector update loop

    for(int k=0; k<2; k++) {

      //    calculate area/timstep
      op_par_loop_adt_calc("adt_calc",cells,
                 op_arg_dat(p_x,0,pcell,2,"double",OP_READ),
                 op_arg_dat(p_x,1,pcell,2,"double",OP_READ),
                 op_arg_dat(p_x,2,pcell,2,"double",OP_READ),
                 op_arg_dat(p_x,3,pcell,2,"double",OP_READ),
                 op_arg_dat(p_q,-1,OP_ID,4,"double",OP_READ),
                 op_arg_dat(p_adt,-1,OP_ID,1,"double",OP_WRITE));

      //    calculate flux residual
      op_par_loop_res_calc("res_calc",edges,
                 op_arg_dat(p_x,0,pedge,2,"double",OP_READ),
                 op_arg_dat(p_x,1,pedge,2,"double",OP_READ),
                 op_arg_dat(p_q,0,pecell,4,"double",OP_READ),
                 op_arg_dat(p_q,1,pecell,4,"double",OP_READ),
                 op_arg_dat(p_adt,0,pecell,1,"double",OP_READ),
                 op_arg_dat(p_adt,1,pecell,1,"double",OP_READ),
                 op_arg_dat(p_res,0,pecell,4,"double",OP_INC),
                 op_arg_dat(p_res,1,pecell,4,"double",OP_INC));

      op_par_loop_bres_calc("bres_calc",bedges,
                 op_arg_dat(p_x,0,pbedge,2,"double",OP_READ),
                 op_arg_dat(p_x,1,pbedge,2,"double",OP_READ),
                 op_arg_dat(p_q,0,pbecell,4,"double",OP_READ),
                 op_arg_dat(p_adt,0,pbecell,1,"double",OP_READ),
                 op_arg_dat(p_res,0,pbecell,4,"double",OP_INC),
                 op_arg_dat(p_bound,-1,OP_ID,1,"int",OP_READ));

      //    update flow field

      rms = 0.0;

      op_par_loop_update("update",cells,
                 op_arg_dat(p_qold,-1,OP_ID,4,"double",OP_READ),
                 op_arg_dat(p_q,-1,OP_ID,4,"double",OP_WRITE),
                 op_arg_dat(p_res,-1,OP_ID,4,"double",OP_RW),
                 op_arg_dat(p_adt,-1,OP_ID,1,"double",OP_READ),
                 op_arg_gbl(&rms,1,"double",OP_INC));

    }

    //print iteration history
    rms = sqrt(rms/(double) g_ncell);
    if (iter%100 == 0)
      op_printf("%d  %10.5e \n",iter,rms);

    if (op_free_dat_temp(p_res) < 0)
      op_printf("Error: temporary op_dat %s cannot be removed\n",p_res->name);
    if (op_free_dat_temp(p_adt) < 0)
      op_printf("Error: temporary op_dat %s cannot be removed\n",p_adt->name);
    if (op_free_dat_temp(p_qold) < 0)
      op_printf("Error: temporary op_dat %s cannot be removed\n",p_qold->name);
  }

  op_timers(&cpu_t2, &wall_t2);
  op_timing_output();

  //print total time for niter interations
  op_printf("Max total runtime = %f\n",wall_t2-wall_t1);
  op_exit();

  free(cell);
  free(edge);
  free(ecell);
  free(bedge);
  free(becell);
  free(bound);
  free(x);
  free(q);
  free(qold);
  free(res);
  free(adt);
}
Beispiel #3
0
int main(int argc, char **argv)
{
  // OP initialisation
  op_init(argc,argv,2);

  int    niter;
  double  rms;

  //timer
  double cpu_t1, cpu_t2, wall_t1, wall_t2;

  // set constants and initialise flow field and residual
  op_printf("initialising flow field \n");

  char file[] = "new_grid.h5";

  // declare sets, pointers, datasets and global constants

  op_set nodes  = op_decl_set_hdf5(file, "nodes");
  op_set edges  = op_decl_set_hdf5(file,  "edges");
  op_set bedges = op_decl_set_hdf5(file, "bedges");
  op_set cells  = op_decl_set_hdf5(file,  "cells");

  op_map pedge   = op_decl_map_hdf5(edges, nodes, 2, file, "pedge");
  op_map pecell  = op_decl_map_hdf5(edges, cells,2, file, "pecell");
  op_map pbedge  = op_decl_map_hdf5(bedges,nodes,2, file, "pbedge");
  op_map pbecell = op_decl_map_hdf5(bedges,cells,1, file, "pbecell");
  op_map pcell   = op_decl_map_hdf5(cells, nodes,4, file, "pcell");

  op_map m_test  = op_decl_map_hdf5(cells, nodes,4, file, "m_test");
  if (m_test == NULL) printf("m_test not found\n");

  op_dat p_bound = op_decl_dat_hdf5(bedges,1,"int"  ,file,"p_bound");
  op_dat p_x     = op_decl_dat_hdf5(nodes ,2,"double",file,"p_x");
  op_dat p_q     = op_decl_dat_hdf5(cells ,4,"double",file,"p_q");
  op_dat p_qold  = op_decl_dat_hdf5(cells ,4,"double",file,"p_qold");
  op_dat p_adt   = op_decl_dat_hdf5(cells ,1,"double",file,"p_adt");
  op_dat p_res   = op_decl_dat_hdf5(cells ,4,"double",file,"p_res");

  op_dat p_test  = op_decl_dat_hdf5(cells ,4,"double",file,"p_test");
  if (p_test == NULL) printf("p_test not found\n");

  op_get_const_hdf5("gam", 1, "double", (char *)&gam, "new_grid.h5");
  op_get_const_hdf5("gm1", 1, "double", (char *)&gm1, "new_grid.h5");
  op_get_const_hdf5("cfl", 1, "double", (char *)&cfl, "new_grid.h5");
  op_get_const_hdf5("eps", 1, "double", (char *)&eps, "new_grid.h5");
  op_get_const_hdf5("mach", 1, "double", (char *)&mach, "new_grid.h5");
  op_get_const_hdf5("alpha", 1, "double", (char *)&alpha, "new_grid.h5");
  op_get_const_hdf5("qinf", 4, "double", (char *)&qinf, "new_grid.h5");

  op_decl_const2("gam",1,"double",&gam);
  op_decl_const2("gm1",1,"double",&gm1);
  op_decl_const2("cfl",1,"double",&cfl);
  op_decl_const2("eps",1,"double",&eps);
  op_decl_const2("mach",1,"double",&mach);
  op_decl_const2("alpha",1,"double",&alpha);
  op_decl_const2("qinf",4,"double",qinf);

  op_diagnostic_output();

  //write back original data just to compare you read the file correctly
  //do an h5diff between new_grid_out.h5 and new_grid.h5 to
  //compare two hdf5 files
  op_dump_to_hdf5("new_grid_out.h5");

  op_write_const_hdf5("gam",1,"double",(char *)&gam,  "new_grid_out.h5");
  op_write_const_hdf5("gm1",1,"double",(char *)&gm1,  "new_grid_out.h5");
  op_write_const_hdf5("cfl",1,"double",(char *)&cfl,  "new_grid_out.h5");
  op_write_const_hdf5("eps",1,"double",(char *)&eps,  "new_grid_out.h5");
  op_write_const_hdf5("mach",1,"double",(char *)&mach,  "new_grid_out.h5");
  op_write_const_hdf5("alpha",1,"double",(char *)&alpha,  "new_grid_out.h5");
  op_write_const_hdf5("qinf",4,"double",(char *)qinf,  "new_grid_out.h5");

  //trigger partitioning and halo creation routines
  op_partition("PTSCOTCH", "KWAY", edges, pecell, p_x);
  //op_partition("PARMETIS", "KWAY", edges, pecell, p_x);

  int g_ncell = op_get_size(cells);


  //initialise timers for total execution wall time
  op_timers(&cpu_t1, &wall_t1);

  // main time-marching loop

  niter = 1000;

  for(int iter=1; iter<=niter; iter++) {

    //  save old flow solution

    op_par_loop_save_soln("save_soln",cells,
                op_arg_dat(p_q,-1,OP_ID,4,"double",OP_READ),
                op_arg_dat(p_qold,-1,OP_ID,4,"double",OP_WRITE));

    //  predictor/corrector update loop

    for(int k=0; k<2; k++) {

      //    calculate area/timstep

      op_par_loop_adt_calc("adt_calc",cells,
                  op_arg_dat(p_x,0,pcell,2,"double",OP_READ),
                  op_arg_dat(p_x,1,pcell,2,"double",OP_READ),
                  op_arg_dat(p_x,2,pcell,2,"double",OP_READ),
                  op_arg_dat(p_x,3,pcell,2,"double",OP_READ),
                  op_arg_dat(p_q,-1,OP_ID,4,"double",OP_READ),
                  op_arg_dat(p_adt,-1,OP_ID,1,"double",OP_WRITE));

      //    calculate flux residual

      op_par_loop_res_calc("res_calc",edges,
                  op_arg_dat(p_x,0,pedge,2,"double",OP_READ),
                  op_arg_dat(p_x,1,pedge,2,"double",OP_READ),
                  op_arg_dat(p_q,0,pecell,4,"double",OP_READ),
                  op_arg_dat(p_q,1,pecell,4,"double",OP_READ),
                  op_arg_dat(p_adt,0,pecell,1,"double",OP_READ),
                  op_arg_dat(p_adt,1,pecell,1,"double",OP_READ),
                  op_arg_dat(p_res,0,pecell,4,"double",OP_INC),
                  op_arg_dat(p_res,1,pecell,4,"double",OP_INC));

      op_par_loop_bres_calc("bres_calc",bedges,
                  op_arg_dat(p_x,0,pbedge,2,"double",OP_READ),
                  op_arg_dat(p_x,1,pbedge,2,"double",OP_READ),
                  op_arg_dat(p_q,0,pbecell,4,"double",OP_READ),
                  op_arg_dat(p_adt,0,pbecell,1,"double",OP_READ),
                  op_arg_dat(p_res,0,pbecell,4,"double",OP_INC),
                  op_arg_dat(p_bound,-1,OP_ID,1,"int",OP_READ));

      //    update flow field

      rms = 0.0;

      op_par_loop_update("update",cells,
                  op_arg_dat(p_qold,-1,OP_ID,4,"double",OP_READ),
                  op_arg_dat(p_q,-1,OP_ID,4,"double",OP_WRITE),
                  op_arg_dat(p_res,-1,OP_ID,4,"double",OP_RW),
                  op_arg_dat(p_adt,-1,OP_ID,1,"double",OP_READ),
                  op_arg_gbl(&rms,1,"double",OP_INC));
    }

    //  print iteration history

    rms = sqrt(rms/(double)g_ncell);

    if (iter%100 == 0)
      op_printf(" %d  %10.5e \n",iter,rms);
  }

  op_timers(&cpu_t2, &wall_t2);

  //write given op_dat's indicated segment of data to a memory block in the order it was originally
  //arranged (i.e. before partitioning and reordering)
  double* q = (double *)op_malloc(sizeof(double)*op_get_size(cells)*4);
  op_fetch_data_idx(p_q, q, 0, op_get_size(cells)-1);
  free(q);

  //write given op_dat's data to hdf5 file in the order it was originally arranged (i.e. before partitioning and reordering)
  op_fetch_data_hdf5_file(p_q, "file_name.h5");

  //printf("Root process = %d\n",op_is_root());

  //output the result dat array to files
  //op_dump_to_hdf5("new_grid_out.h5"); //writes data as it is held on each process (under MPI)

  //compress using
  // ~/hdf5/bin/h5repack -f GZIP=9 new_grid.h5 new_grid_pack.h5

  op_timing_output();
  op_printf("Max total runtime = %f\n",wall_t2-wall_t1);
  op_exit();
}
Beispiel #4
0
int main(int argc, char **argv)
{
  // OP initialisation
  op_init(argc,argv,2);

  int    *becell, *ecell,  *bound, *bedge, *edge, *cell;
  double  *x, *q, *qold, *adt, *res;

  int    nnode,ncell,nedge,nbedge,niter;
  double  rms;

  //timer
  double cpu_t1, cpu_t2, wall_t1, wall_t2;

  // read in grid

  op_printf("reading in grid \n");

  FILE *fp;
  if ( (fp = fopen("./new_grid.dat","r")) == NULL) {
    op_printf("can't open file new_grid.dat\n"); exit(-1);
  }

  if (fscanf(fp,"%d %d %d %d \n",&nnode, &ncell, &nedge, &nbedge) != 4) {
    op_printf("error reading from new_grid.dat\n"); exit(-1);
  }

  cell   = (int *) malloc(4*ncell*sizeof(int));
  edge   = (int *) malloc(2*nedge*sizeof(int));
  ecell  = (int *) malloc(2*nedge*sizeof(int));
  bedge  = (int *) malloc(2*nbedge*sizeof(int));
  becell = (int *) malloc(  nbedge*sizeof(int));
  bound  = (int *) malloc(  nbedge*sizeof(int));

  x      = (double *) malloc(2*nnode*sizeof(double));
  q      = (double *) malloc(4*ncell*sizeof(double));
  qold   = (double *) malloc(4*ncell*sizeof(double));
  res    = (double *) malloc(4*ncell*sizeof(double));
  adt    = (double *) malloc(  ncell*sizeof(double));

  for (int n=0; n<nnode; n++) {
    if (fscanf(fp,"%lf %lf \n",&x[2*n], &x[2*n+1]) != 2) {
      op_printf("error reading from new_grid.dat\n"); exit(-1);
    }
  }

  for (int n=0; n<ncell; n++) {
    if (fscanf(fp,"%d %d %d %d \n",&cell[4*n  ], &cell[4*n+1],
                                   &cell[4*n+2], &cell[4*n+3]) != 4) {
      op_printf("error reading from new_grid.dat\n"); exit(-1);
    }
  }

  for (int n=0; n<nedge; n++) {
    if (fscanf(fp,"%d %d %d %d \n",&edge[2*n], &edge[2*n+1],
                                   &ecell[2*n],&ecell[2*n+1]) != 4) {
      op_printf("error reading from new_grid.dat\n"); exit(-1);
    }
  }

  for (int n=0; n<nbedge; n++) {
    if (fscanf(fp,"%d %d %d %d \n",&bedge[2*n],&bedge[2*n+1],
                                   &becell[n], &bound[n]) != 4) {
      op_printf("error reading from new_grid.dat\n"); exit(-1);
    }
  }

  fclose(fp);

  // set constants and initialise flow field and residual

  op_printf("initialising flow field \n");

  gam = 1.4f;
  gm1 = gam - 1.0f;
  cfl = 0.9f;
  eps = 0.05f;

  double mach  = 0.4f;
  double alpha = 3.0f*atan(1.0f)/45.0f;
  double p     = 1.0f;
  double r     = 1.0f;
  double u     = sqrt(gam*p/r)*mach;
  double e     = p/(r*gm1) + 0.5f*u*u;

  qinf[0] = r;
  qinf[1] = r*u;
  qinf[2] = 0.0f;
  qinf[3] = r*e;

  for (int n=0; n<ncell; n++) {
    for (int m=0; m<4; m++) {
        q[4*n+m] = qinf[m];
      res[4*n+m] = 0.0f;
    }
  }

  // declare sets, pointers, datasets and global constants

  op_set nodes  = op_decl_set(nnode,  "nodes");
  op_set edges  = op_decl_set(nedge,  "edges");
  op_set bedges = op_decl_set(nbedge, "bedges");
  op_set cells  = op_decl_set(ncell,  "cells");

  op_map pedge   = op_decl_map(edges, nodes,2,edge,  "pedge");
  op_map pecell  = op_decl_map(edges, cells,2,ecell, "pecell");
  op_map pbedge  = op_decl_map(bedges,nodes,2,bedge, "pbedge");
  op_map pbecell = op_decl_map(bedges,cells,1,becell,"pbecell");
  op_map pcell   = op_decl_map(cells, nodes,4,cell,  "pcell");

  op_dat p_bound = op_decl_dat(bedges,1,"int"  ,bound,"p_bound");
  op_dat p_x     = op_decl_dat(nodes ,2,"double",x    ,"p_x");
  op_dat p_q     = op_decl_dat(cells ,4,"double",q    ,"p_q");
  //op_dat p_qold  = op_decl_dat(cells ,4,"double",qold ,"p_qold");
  //op_dat p_adt   = op_decl_dat(cells ,1,"double",adt  ,"p_adt");
  //op_dat p_res   = op_decl_dat(cells ,4,"double",res  ,"p_res");

  // p_res, p_adt and p_qold  now declared as a temp op_dats during
  // the execution of the time-marching loop

  op_decl_const2("gam",1,"double",&gam);
  op_decl_const2("gm1",1,"double",&gm1);
  op_decl_const2("cfl",1,"double",&cfl);
  op_decl_const2("eps",1,"double",&eps);
  op_decl_const2("mach",1,"double",&mach);
  op_decl_const2("alpha",1,"double",&alpha);
  op_decl_const2("qinf",4,"double",qinf);

  op_diagnostic_output();

  double g_ncell = op_get_size(cells);

  //initialise timers for total execution wall time
  op_timers(&cpu_t1, &wall_t1);

  // main time-marching loop

  niter = 1000;

  for(int iter=1; iter<=niter; iter++) {

    double* tmp_elem = NULL;
    op_dat p_res   = op_decl_dat_temp(cells ,4,"double",tmp_elem,"p_res");
    op_dat p_adt   = op_decl_dat_temp(cells ,1,"double",tmp_elem,"p_adt");
    op_dat p_qold  = op_decl_dat_temp(cells ,4,"double",qold ,"p_qold");

    // save old flow solution

    op_par_loop_save_soln("save_soln",cells,
                op_arg_dat(p_q,-1,OP_ID,4,"double",OP_READ),
                op_arg_dat(p_qold,-1,OP_ID,4,"double",OP_WRITE));

    // predictor/corrector update loop

    for(int k=0; k<2; k++) {

      // calculate area/timstep

      op_par_loop_adt_calc("adt_calc",cells,
                  op_arg_dat(p_x,0,pcell,2,"double",OP_READ),
                  op_arg_dat(p_x,1,pcell,2,"double",OP_READ),
                  op_arg_dat(p_x,2,pcell,2,"double",OP_READ),
                  op_arg_dat(p_x,3,pcell,2,"double",OP_READ),
                  op_arg_dat(p_q,-1,OP_ID,4,"double",OP_READ),
                  op_arg_dat(p_adt,-1,OP_ID,1,"double",OP_WRITE));

      // calculate flux residual

      op_par_loop_res_calc("res_calc",edges,
                  op_arg_dat(p_x,0,pedge,2,"double",OP_READ),
                  op_arg_dat(p_x,1,pedge,2,"double",OP_READ),
                  op_arg_dat(p_q,0,pecell,4,"double",OP_READ),
                  op_arg_dat(p_q,1,pecell,4,"double",OP_READ),
                  op_arg_dat(p_adt,0,pecell,1,"double",OP_READ),
                  op_arg_dat(p_adt,1,pecell,1,"double",OP_READ),
                  op_arg_dat(p_res,0,pecell,4,"double",OP_INC),
                  op_arg_dat(p_res,1,pecell,4,"double",OP_INC));

      op_par_loop_bres_calc("bres_calc",bedges,
                  op_arg_dat(p_x,0,pbedge,2,"double",OP_READ),
                  op_arg_dat(p_x,1,pbedge,2,"double",OP_READ),
                  op_arg_dat(p_q,0,pbecell,4,"double",OP_READ),
                  op_arg_dat(p_adt,0,pbecell,1,"double",OP_READ),
                  op_arg_dat(p_res,0,pbecell,4,"double",OP_INC),
                  op_arg_dat(p_bound,-1,OP_ID,1,"int",OP_READ));

      // update flow field

      rms = 0.0;

      op_par_loop_update("update",cells,
                  op_arg_dat(p_qold,-1,OP_ID,4,"double",OP_READ),
                  op_arg_dat(p_q,-1,OP_ID,4,"double",OP_WRITE),
                  op_arg_dat(p_res,-1,OP_ID,4,"double",OP_RW),
                  op_arg_dat(p_adt,-1,OP_ID,1,"double",OP_READ),
                  op_arg_gbl(&rms,1,"double",OP_INC));
    }

    // print iteration history
    rms = sqrt(rms/(double)g_ncell );
    if (iter%100 == 0)
      op_printf(" %d  %10.5e \n",iter,rms);

    if (iter%1000 == 0 && g_ncell == 720000){ //defailt mesh -- for validation testing
      //op_printf(" %d  %3.16f \n",iter,rms);
      double diff=fabs((100.0*(rms/0.0001060114637578))-100.0);
      op_printf("\n\nTest problem with %d cells is within %3.15E %% of the expected solution\n",720000, diff);
      if(diff < 0.00001) {
        op_printf("This test is considered PASSED\n");
      }
      else {
        op_printf("This test is considered FAILED\n");
      }
    }

    if (op_free_dat_temp(p_res) < 0)
      op_printf("Error: temporary op_dat %s cannot be removed\n",p_res->name);
    if (op_free_dat_temp(p_adt) < 0)
      op_printf("Error: temporary op_dat %s cannot be removed\n",p_adt->name);
    if (op_free_dat_temp(p_qold) < 0)
      op_printf("Error: temporary op_dat %s cannot be removed\n",p_qold->name);
  }

  op_timers(&cpu_t2, &wall_t2);
  op_timing_output();
  op_printf("Max total runtime = %f\n",wall_t2-wall_t1);

  op_exit();

  free(cell);
  free(edge);
  free(ecell);
  free(bedge);
  free(becell);
  free(bound);
  free(x);
  free(q);
  free(qold);
  free(res);
  free(adt);
}
Beispiel #5
0
int main(int argc, char **argv)
{
  // OP initialisation
  op_init(argc,argv,2);

  int    *becell, *ecell,  *bound, *bedge, *edge, *cell;
  float  *x, *q, *qold, *adt, *res;

  int    nnode,ncell,nedge,nbedge,niter;
  float  rms;

  //timer
  double cpu_t1, cpu_t2, wall_t1, wall_t2;

  // read in grid

  op_printf("reading in grid \n");

  FILE *fp;
  if ( (fp = fopen("./new_grid.dat","r")) == NULL) {
    op_printf("can't open file new_grid.dat\n"); exit(-1);
  }

  if (fscanf(fp,"%d %d %d %d \n",&nnode, &ncell, &nedge, &nbedge) != 4) {
    op_printf("error reading from new_grid.dat\n"); exit(-1);
  }

  cell   = (int *) malloc(4*ncell*sizeof(int));
  edge   = (int *) malloc(2*nedge*sizeof(int));
  ecell  = (int *) malloc(2*nedge*sizeof(int));
  bedge  = (int *) malloc(2*nbedge*sizeof(int));
  becell = (int *) malloc(  nbedge*sizeof(int));
  bound  = (int *) malloc(  nbedge*sizeof(int));

  x      = (float *) malloc(2*nnode*sizeof(float));
  q      = (float *) malloc(4*ncell*sizeof(float));
  qold   = (float *) malloc(4*ncell*sizeof(float));
  res    = (float *) malloc(4*ncell*sizeof(float));
  adt    = (float *) malloc(  ncell*sizeof(float));

  for (int n=0; n<nnode; n++) {
    if (fscanf(fp,"%f %f \n",&x[2*n], &x[2*n+1]) != 2) {
      op_printf("error reading from new_grid.dat\n"); exit(-1);
    }
  }

  for (int n=0; n<ncell; n++) {
    if (fscanf(fp,"%d %d %d %d \n",&cell[4*n  ], &cell[4*n+1],
                                   &cell[4*n+2], &cell[4*n+3]) != 4) {
      op_printf("error reading from new_grid.dat\n"); exit(-1);
    }
  }

  for (int n=0; n<nedge; n++) {
    if (fscanf(fp,"%d %d %d %d \n",&edge[2*n], &edge[2*n+1],
                                   &ecell[2*n],&ecell[2*n+1]) != 4) {
      op_printf("error reading from new_grid.dat\n"); exit(-1);
    }
  }

  for (int n=0; n<nbedge; n++) {
    if (fscanf(fp,"%d %d %d %d \n",&bedge[2*n],&bedge[2*n+1],
                                   &becell[n], &bound[n]) != 4) {
      op_printf("error reading from new_grid.dat\n"); exit(-1);
    }
  }

  fclose(fp);

  // set constants and initialise flow field and residual

  op_printf("initialising flow field \n");

  gam = 1.4f;
  gm1 = gam - 1.0f;
  cfl = 0.9f;
  eps = 0.05f;

  float mach  = 0.4f;
  float alpha = 3.0f*atan(1.0f)/45.0f;
  float p     = 1.0f;
  float r     = 1.0f;
  float u     = sqrt(gam*p/r)*mach;
  float e     = p/(r*gm1) + 0.5f*u*u;

  qinf[0] = r;
  qinf[1] = r*u;
  qinf[2] = 0.0f;
  qinf[3] = r*e;

  for (int n=0; n<ncell; n++) {
    for (int m=0; m<4; m++) {
        q[4*n+m] = qinf[m];
      res[4*n+m] = 0.0f;
    }
  }

  // declare sets, pointers, datasets and global constants

  op_set nodes  = op_decl_set(nnode,  "nodes");
  op_set edges  = op_decl_set(nedge,  "edges");
  op_set bedges = op_decl_set(nbedge, "bedges");
  op_set cells  = op_decl_set(ncell,  "cells");

  op_map pedge   = op_decl_map(edges, nodes,2,edge,  "pedge");
  op_map pecell  = op_decl_map(edges, cells,2,ecell, "pecell");
  op_map pbedge  = op_decl_map(bedges,nodes,2,bedge, "pbedge");
  op_map pbecell = op_decl_map(bedges,cells,1,becell,"pbecell");
  op_map pcell   = op_decl_map(cells, nodes,4,cell,  "pcell");

  op_dat p_bound = op_decl_dat(bedges,1,"int"  ,bound,"p_bound");
  op_dat p_x     = op_decl_dat(nodes ,2,"float",x    ,"p_x");
  op_dat p_q     = op_decl_dat(cells ,4,"float",q    ,"p_q");
  op_dat p_qold  = op_decl_dat(cells ,4,"float",qold ,"p_qold");
  op_dat p_adt   = op_decl_dat(cells ,1,"float",adt  ,"p_adt");
  op_dat p_res   = op_decl_dat(cells ,4,"float",res  ,"p_res");

  op_decl_const2("gam",1,"float",&gam);
  op_decl_const2("gm1",1,"float",&gm1);
  op_decl_const2("cfl",1,"float",&cfl);
  op_decl_const2("eps",1,"float",&eps);
  op_decl_const2("mach",1,"float",&mach);
  op_decl_const2("alpha",1,"float",&alpha);
  op_decl_const2("qinf",4,"float",qinf);

  op_diagnostic_output();

  //initialise timers for total execution wall time
  op_timers(&cpu_t1, &wall_t1);

  // main time-marching loop

  niter = 1000;

  for(int iter=1; iter<=niter; iter++) {

    // save old flow solution

    op_par_loop_save_soln("save_soln",cells,
                op_arg_dat(p_q,-1,OP_ID,4,"float",OP_READ),
                op_arg_dat(p_qold,-1,OP_ID,4,"float",OP_WRITE));

    // predictor/corrector update loop

    for(int k=0; k<2; k++) {

      // calculate area/timstep

      op_par_loop_adt_calc("adt_calc",cells,
                  op_arg_dat(p_x,0,pcell,2,"float",OP_READ),
                  op_arg_dat(p_x,1,pcell,2,"float",OP_READ),
                  op_arg_dat(p_x,2,pcell,2,"float",OP_READ),
                  op_arg_dat(p_x,3,pcell,2,"float",OP_READ),
                  op_arg_dat(p_q,-1,OP_ID,4,"float",OP_READ),
                  op_arg_dat(p_adt,-1,OP_ID,1,"float",OP_WRITE));

      // calculate flux residual

      op_par_loop_res_calc("res_calc",edges,
                  op_arg_dat(p_x,0,pedge,2,"float",OP_READ),
                  op_arg_dat(p_x,1,pedge,2,"float",OP_READ),
                  op_arg_dat(p_q,0,pecell,4,"float",OP_READ),
                  op_arg_dat(p_q,1,pecell,4,"float",OP_READ),
                  op_arg_dat(p_adt,0,pecell,1,"float",OP_READ),
                  op_arg_dat(p_adt,1,pecell,1,"float",OP_READ),
                  op_arg_dat(p_res,0,pecell,4,"float",OP_INC),
                  op_arg_dat(p_res,1,pecell,4,"float",OP_INC));

      op_par_loop_bres_calc("bres_calc",bedges,
                  op_arg_dat(p_x,0,pbedge,2,"float",OP_READ),
                  op_arg_dat(p_x,1,pbedge,2,"float",OP_READ),
                  op_arg_dat(p_q,0,pbecell,4,"float",OP_READ),
                  op_arg_dat(p_adt,0,pbecell,1,"float",OP_READ),
                  op_arg_dat(p_res,0,pbecell,4,"float",OP_INC),
                  op_arg_dat(p_bound,-1,OP_ID,1,"int",OP_READ));

      // update flow field

      rms = 0.0;

      op_par_loop_update("update",cells,
                  op_arg_dat(p_qold,-1,OP_ID,4,"float",OP_READ),
                  op_arg_dat(p_q,-1,OP_ID,4,"float",OP_WRITE),
                  op_arg_dat(p_res,-1,OP_ID,4,"float",OP_RW),
                  op_arg_dat(p_adt,-1,OP_ID,1,"float",OP_READ),
                  op_arg_gbl(&rms,1,"float",OP_INC));
    }

    // print iteration history
    rms = sqrt(rms/(float) op_get_size(cells));
    if (iter%100 == 0)
      op_printf(" %d  %10.5e \n",iter,rms);
  }

  op_timers(&cpu_t2, &wall_t2);
  op_timing_output();
  op_printf("Max total runtime = \n%f\n",wall_t2-wall_t1);

  op_exit();

  free(cell);
  free(edge);
  free(ecell);
  free(bedge);
  free(becell);
  free(bound);
  free(x);
  free(q);
  free(qold);
  free(res);
  free(adt);
}
Beispiel #6
0
int main(int argc, char **argv) {
  // OP initialisation
  op_init(argc, argv, 2);

  int niter;
  float rms;

  // timer
  double cpu_t1, cpu_t2, wall_t1, wall_t2;

  // set constants and initialise flow field and residual
  op_printf("initialising flow field \n");

  char file[] = "new_grid.h5";

  // declare sets, pointers, datasets and global constants

  op_set nodes = op_decl_set_hdf5(file, "nodes");
  op_set edges = op_decl_set_hdf5(file, "edges");
  op_set bedges = op_decl_set_hdf5(file, "bedges");
  op_set cells = op_decl_set_hdf5(file, "cells");

  op_map pedge = op_decl_map_hdf5(edges, nodes, 2, file, "pedge");
  op_map pecell = op_decl_map_hdf5(edges, cells, 2, file, "pecell");
  op_map pbedge = op_decl_map_hdf5(bedges, nodes, 2, file, "pbedge");
  op_map pbecell = op_decl_map_hdf5(bedges, cells, 1, file, "pbecell");
  op_map pcell = op_decl_map_hdf5(cells, nodes, 4, file, "pcell");

  op_dat p_bound = op_decl_dat_hdf5(bedges, 1, "int", file, "p_bound");
  op_dat p_x = op_decl_dat_hdf5(nodes, 2, "float", file, "p_x");
  op_dat p_q = op_decl_dat_hdf5(cells, 4, "float", file, "p_q");
  op_dat p_qold = op_decl_dat_hdf5(cells, 4, "float", file, "p_qold");
  op_dat p_adt = op_decl_dat_hdf5(cells, 1, "float", file, "p_adt");
  op_dat p_res = op_decl_dat_hdf5(cells, 4, "float", file, "p_res");

  op_get_const_hdf5("gam", 1, "float", (char *)&gam, "new_grid.h5");
  op_get_const_hdf5("gm1", 1, "float", (char *)&gm1, "new_grid.h5");
  op_get_const_hdf5("cfl", 1, "float", (char *)&cfl, "new_grid.h5");
  op_get_const_hdf5("eps", 1, "float", (char *)&eps, "new_grid.h5");
  op_get_const_hdf5("mach", 1, "float", (char *)&mach, "new_grid.h5");
  op_get_const_hdf5("alpha", 1, "float", (char *)&alpha, "new_grid.h5");
  op_get_const_hdf5("qinf", 4, "float", (char *)&qinf, "new_grid.h5");

  op_decl_const2("gam", 1, "float", &gam);
  op_decl_const2("gm1", 1, "float", &gm1);
  op_decl_const2("cfl", 1, "float", &cfl);
  op_decl_const2("eps", 1, "float", &eps);
  op_decl_const2("mach", 1, "float", &mach);
  op_decl_const2("alpha", 1, "float", &alpha);
  op_decl_const2("qinf", 4, "float", qinf);

  if (op_is_root())
    op_diagnostic_output();

  // trigger partitioning and halo creation routines
  op_partition("PTSCOTCH", "KWAY", edges, pecell, p_x);
  // op_partition("PARMETIS", "KWAY", edges, pecell, p_x);

  int g_ncell = op_get_size(cells);

  // initialise timers for total execution wall time
  op_timers(&cpu_t1, &wall_t1);

  // main time-marching loop

  niter = 1000;

  for (int iter = 1; iter <= niter; iter++) {

    //  save old flow solution

    op_par_loop_save_soln("save_soln", cells,
                          op_arg_dat(p_q, -1, OP_ID, 4, "float", OP_READ),
                          op_arg_dat(p_qold, -1, OP_ID, 4, "float", OP_WRITE));

    //  predictor/corrector update loop

    for (int k = 0; k < 2; k++) {

      //    calculate area/timstep

      op_par_loop_adt_calc("adt_calc", cells,
                           op_arg_dat(p_x, 0, pcell, 2, "float", OP_READ),
                           op_arg_dat(p_x, 1, pcell, 2, "float", OP_READ),
                           op_arg_dat(p_x, 2, pcell, 2, "float", OP_READ),
                           op_arg_dat(p_x, 3, pcell, 2, "float", OP_READ),
                           op_arg_dat(p_q, -1, OP_ID, 4, "float", OP_READ),
                           op_arg_dat(p_adt, -1, OP_ID, 1, "float", OP_WRITE));

      //    calculate flux residual

      op_par_loop_res_calc("res_calc", edges,
                           op_arg_dat(p_x, 0, pedge, 2, "float", OP_READ),
                           op_arg_dat(p_x, 1, pedge, 2, "float", OP_READ),
                           op_arg_dat(p_q, 0, pecell, 4, "float", OP_READ),
                           op_arg_dat(p_q, 1, pecell, 4, "float", OP_READ),
                           op_arg_dat(p_adt, 0, pecell, 1, "float", OP_READ),
                           op_arg_dat(p_adt, 1, pecell, 1, "float", OP_READ),
                           op_arg_dat(p_res, 0, pecell, 4, "float", OP_INC),
                           op_arg_dat(p_res, 1, pecell, 4, "float", OP_INC));

      op_par_loop_bres_calc("bres_calc", bedges,
                            op_arg_dat(p_x, 0, pbedge, 2, "float", OP_READ),
                            op_arg_dat(p_x, 1, pbedge, 2, "float", OP_READ),
                            op_arg_dat(p_q, 0, pbecell, 4, "float", OP_READ),
                            op_arg_dat(p_adt, 0, pbecell, 1, "float", OP_READ),
                            op_arg_dat(p_res, 0, pbecell, 4, "float", OP_INC),
                            op_arg_dat(p_bound, -1, OP_ID, 1, "int", OP_READ));

      //    update flow field

      rms = 0.0;

      op_par_loop_update("update", cells,
                         op_arg_dat(p_qold, -1, OP_ID, 4, "float", OP_READ),
                         op_arg_dat(p_q, -1, OP_ID, 4, "float", OP_WRITE),
                         op_arg_dat(p_res, -1, OP_ID, 4, "float", OP_RW),
                         op_arg_dat(p_adt, -1, OP_ID, 1, "float", OP_READ),
                         op_arg_gbl(&rms, 1, "float", OP_INC));
    }

    //  print iteration history

    rms = sqrtf(rms / (float)g_ncell);

    if (iter % 100 == 0)
      op_printf(" %d  %10.5e \n", iter, rms);
    if (iter % 1000 == 0 &&
        g_ncell == 720000) { // defailt mesh -- for validation testing
      op_printf(" %d  %3.16f \n", iter, rms);
      float diff = fabsf((100.0 * (rms / 0.000105987)) - 100.0);
      op_printf("\n\nTest problem with %d cells is within %3.15E %% of the "
                "expected solution\n",
                720000, diff);
      if (diff < 0.1) {
        op_printf("This test is considered PASSED\n");
      } else {
        op_printf("This test is considered FAILED\n");
      }
    }
  }

  op_timers(&cpu_t2, &wall_t2);

  op_timing_output();
  op_printf("Max total runtime = %f\n", wall_t2 - wall_t1);
  op_exit();
}
int main(int argc, char **argv){

  int    *becell, *ecell,  *bound, *bedge, *edge, *cell;
  float  *x, *q, *qold, *adt, *res;

  int    nnode,ncell,nedge,nbedge,niter;
  float  rms;

  // read in grid

  printf("reading in grid \n");

  FILE *fp;
  if ( (fp = fopen("/work/rr908/airfoil/new_grid.dat","r")) == NULL) {
    printf("can't open file new_grid.dat\n"); exit(-1);
  }

  if (fscanf(fp,"%d %d %d %d \n",&nnode, &ncell, &nedge, &nbedge) != 4) {
    printf("error reading from new_grid.dat\n"); exit(-1);
  }

  cell   = (int *) malloc(4*ncell*sizeof(int));
  edge   = (int *) malloc(2*nedge*sizeof(int));
  ecell  = (int *) malloc(2*nedge*sizeof(int));
  bedge  = (int *) malloc(2*nbedge*sizeof(int));
  becell = (int *) malloc(  nbedge*sizeof(int));
  bound  = (int *) malloc(  nbedge*sizeof(int));

  x      = (float *) malloc(2*nnode*sizeof(float));
  q      = (float *) malloc(4*ncell*sizeof(float));
  qold   = (float *) malloc(4*ncell*sizeof(float));
  res    = (float *) malloc(4*ncell*sizeof(float));
  adt    = (float *) malloc(  ncell*sizeof(float));

  for (int n=0; n<nnode; n++) {
    if (fscanf(fp,"%f %f \n",&x[2*n], &x[2*n+1]) != 2) {
      printf("error reading from new_grid.dat\n"); exit(-1);
    }
  }

  for (int n=0; n<ncell; n++) {
    if (fscanf(fp,"%d %d %d %d \n",&cell[4*n  ], &cell[4*n+1],
                                   &cell[4*n+2], &cell[4*n+3]) != 4) {
      printf("error reading from new_grid.dat\n"); exit(-1);
    }
  }

  for (int n=0; n<nedge; n++) {
    if (fscanf(fp,"%d %d %d %d \n",&edge[2*n], &edge[2*n+1],
                                   &ecell[2*n],&ecell[2*n+1]) != 4) {
      printf("error reading from new_grid.dat\n"); exit(-1);
    }
  }

  for (int n=0; n<nbedge; n++) {
    if (fscanf(fp,"%d %d %d %d \n",&bedge[2*n],&bedge[2*n+1],
                                   &becell[n], &bound[n]) != 4) {
      printf("error reading from new_grid.dat\n"); exit(-1);
    }
  }

  fclose(fp);

#ifdef DIAGNOSTIC
  print_array((float *) x, nnode, "initial_nodes");
  print_array((float *) cell, ncell, "initial_cells");
 
  FILE *flog;
  flog = fopen( "initial_cells_cellarray", "w" );
  for( int i=0; i< ncell; ++i ) {
    fprintf( flog, "%d %d %d %d\n", cell[4*i], cell[4*i+1], cell[4*i+2], cell[4*i+3] );
  }
  fclose( flog );


  print_array((float *) edge, nedge, "initial_edges");
  print_array((float *) ecell, nedge, "initiall_edges_for_cell");
  print_array((float *) bedge, nbedge, "initial_border_edges");
  print_array((float *) becell, nbedge, "initial_becell");
  print_array((float *) bound, nbedge, "initial bound");
#endif

  // set constants and initialise flow field and residual

  printf("initialising flow field \n");

  g_const.gam = 1.4f;
  g_const.gm1 = g_const.gam - 1.0f;
  g_const.cfl = 0.9f;
  g_const.eps = 0.05f;

  g_const.mach  = 0.4f;
  g_const.alpha = 3.0f*atan(1.0f)/45.0f;  
  float p     = 1.0f;
  float r     = 1.0f;
  float u     = sqrt(g_const.gam*p/r)*g_const.mach;
  float e     = p/(r*g_const.gm1) + 0.5f*u*u;

  g_const.qinf[0] = r;
  g_const.qinf[1] = r*u;
  g_const.qinf[2] = 0.0f;
  g_const.qinf[3] = r*e;

  for (int n=0; n<ncell; n++) {
    for (int m=0; m<4; m++) {
        q[4*n+m] = g_const.qinf[m];
      res[4*n+m] = 0.0f;
    }
  }




  // OP initialisation

  printf("OP initialisation\n");
  op_init(argc,argv,2);
  g_const_d = op_allocate_constant( &g_const, sizeof( struct global_constants ) );

  // declare sets, pointers, datasets and global constants

  op_set nodes  = op_decl_set(nnode,  "nodes");
  op_set edges  = op_decl_set(nedge,  "edges");
  op_set bedges = op_decl_set(nbedge, "bedges");
  op_set cells  = op_decl_set(ncell,  "cells");

  op_map pedge   = op_decl_map(edges, nodes,2,edge,  "pedge");
  op_map pecell  = op_decl_map(edges, cells,2,ecell, "pecell");
  op_map pbedge  = op_decl_map(bedges,nodes,2,bedge, "pbedge");
  op_map pbecell = op_decl_map(bedges,cells,1,becell,"pbecell");
  op_map pcell   = op_decl_map(cells, nodes,4,cell,  "pcell");

  op_dat p_bound = op_decl_dat(bedges,1,"int"  ,bound,"p_bound");
  op_dat p_x     = op_decl_dat(nodes ,2,"float",x    ,"p_x");
  op_dat p_q     = op_decl_dat(cells ,4,"float",q    ,"p_q");
  op_dat p_qold  = op_decl_dat(cells ,4,"float",qold ,"p_qold");
  op_dat p_adt   = op_decl_dat(cells ,1,"float",adt  ,"p_adt");
  op_dat p_res   = op_decl_dat(cells ,4,"float",res  ,"p_res");

  op_decl_const2("gam",1,"float",&g_const.gam  );
  op_decl_const2("gm1",1,"float",&g_const.gm1  );
  op_decl_const2("cfl",1,"float",&g_const.cfl  );
  op_decl_const2("eps",1,"float",&g_const.eps  );
  op_decl_const2("mach",1,"float",&g_const.mach );
  op_decl_const2("alpha",1,"float",&g_const.alpha);
  op_decl_const2("qinf",4,"float",g_const.qinf  );


  op_diagnostic_output();

#ifdef DIAGNOSTIC
  dump_array(p_bound, "initial_dat_p_bound");
  dump_array(p_x, "initial_dat_p_x");
  dump_array(p_q, "initiall_dat_p_q");
  dump_array(p_qold, "initial_dat_p_qold");
  dump_array(p_adt, "initial_dat_p_adt");
  dump_array(p_res, "initial_dat_res");
#endif

// main time-marching loop
  niter = 1000;
  for(int iter=1; iter<=niter; iter++) {

//  save old flow solution
 // dump_array(p_q, "p_q_iter_before");
 // dump_array(p_qold, "p_q_old_iter_before");

    op_par_loop_save_soln("save_soln", cells,
                op_arg_dat(p_q,   -1,OP_ID, 4,"float",OP_READ ),
                op_arg_dat(p_qold,-1,OP_ID, 4,"float",OP_WRITE));


//  dump_array(p_q, "p_q_iter_after");
//  dump_array(p_qold, "p_q_old_iter_after");

/*    if ( iter == 1 ) {
      dump_array( p_qold, "p_qold" );
    }
    */

#ifdef DIAGNOSTIC
    if (iter==1) {
      dump_array( p_qold, "p_qold" );
    }
#endif
    //dump_array( p_qold, "p_qold" );
    //op_fetch_data( p_qold );
    //print_array( ( float *) p_qold->data, 4*p_qold->set->size, "p_qold" );
//    print_array( p_q, "p_qold2" );
//    print_array( p_qold, "p_qold" );

    //assert( p_q->data[0] != 0.0f );

//  predictor/corrector update loop

  //  dump_array(p_adt, "p_adt_before");
    for(int k=0; k<2; k++) {
//    calculate area/timstep
      if(k == 0 && iter == 0) {
        printf("Dumping adt before adt_calc execution array");
         op_fetch_data( p_adt );
	 float* array = (float *) p_adt->data;
         long size = p_adt->set->size;
         for(long elem = 0; elem < size; ++elem) {
           printf("%lf",array[elem]);
         }
      }

      op_par_loop_adt_calc("adt_calc",cells,
                  op_arg_dat(p_x,   0,pcell, 2,"float",OP_READ ),
                  op_arg_dat(p_x,   1,pcell, 2,"float",OP_READ ),
                  op_arg_dat(p_x,   2,pcell, 2,"float",OP_READ ),
                  op_arg_dat(p_x,   3,pcell, 2,"float",OP_READ ),
                  op_arg_dat(p_q,  -1,OP_ID, 4,"float",OP_READ ),
                  op_arg_dat(p_adt,-1,OP_ID, 1,"float",OP_WRITE));
      
      if(k == 0 && iter == 0) {
        printf("Dumping adt after 1x adt_calc execution array");
         op_fetch_data( p_adt );
         float* array = (float *) p_adt->data;
         long size = p_adt->set->size;
         for(long elem = 0; elem < size; ++elem) {
           printf("%lf",array[elem]);
         }
      }


#ifdef DIAGNOSTIC
    if (iter==1 && k==0) {
      dump_array( p_adt, "p_adt0" );
    }
    if (iter==1 && k==1) {
      dump_array( p_adt, "p_adt1" );
    }
#endif
  //  dump_array(p_adt, "p_adt_after");
//    calculate flux residual

      op_par_loop_res_calc("res_calc",edges,
                  op_arg_dat(p_x,    0,pedge, 2,"float",OP_READ),
                  op_arg_dat(p_x,    1,pedge, 2,"float",OP_READ),
                  op_arg_dat(p_q,    0,pecell,4,"float",OP_READ),
                  op_arg_dat(p_q,    1,pecell,4,"float",OP_READ),
                  op_arg_dat(p_adt,  0,pecell,1,"float",OP_READ),
                  op_arg_dat(p_adt,  1,pecell,1,"float",OP_READ),
                  op_arg_dat(p_res,  0,pecell,4,"float",OP_INC ),
                  op_arg_dat(p_res,  1,pecell,4,"float",OP_INC ));

#ifdef DIAGNOSTIC
    if (iter==1 && k==0) {
      dump_array( p_res, "p_res0" );
    }
    if (iter==1 && k==1) {
      dump_array( p_res, "p_res1" );
    }
#endif

      op_par_loop_bres_calc("bres_calc",bedges,
                  op_arg_dat(p_x,     0,pbedge, 2,"float",OP_READ),
                  op_arg_dat(p_x,     1,pbedge, 2,"float",OP_READ),
                  op_arg_dat(p_q,     0,pbecell,4,"float",OP_READ),
                  op_arg_dat(p_adt,   0,pbecell,1,"float",OP_READ),
                  op_arg_dat(p_res,   0,pbecell,4,"float",OP_INC ),
                  op_arg_dat(p_bound,-1,OP_ID  ,1,"int",  OP_READ));

#ifdef DIAGNOSTIC
    if (iter==1 && k==0) {
      dump_array( p_res, "p_res_a0" );
    }
    if (iter==1 && k==0) {
      dump_array( p_res, "p_res_a1" );
    }
#endif
//    update flow field

      rms = 0.0;

      op_par_loop_update("update",cells,
                  op_arg_dat(p_qold,-1,OP_ID, 4,"float",OP_READ ),
                  op_arg_dat(p_q,   -1,OP_ID, 4,"float",OP_WRITE),
                  op_arg_dat(p_res, -1,OP_ID, 4,"float",OP_RW   ),
                  op_arg_dat(p_adt, -1,OP_ID, 1,"float",OP_READ ),
                  op_arg_gbl(&rms,1,"float",OP_INC));
    }

#ifdef DIAGNOSTIC
    if (iter==1) {
      dump_array( p_q, "p_q1" );
    }
#endif

//  print iteration history

    rms = sqrt(rms/(float) ncell);

    if (iter%100 == 0)
      printf(" %d  %10.5e \n",iter,rms);



  }

  op_timing_output();

#ifdef DIAGNOSTIC
  dump_array( p_q, "p_q" );
#endif



}