Example #1
0
int
main(int argc, char *argv[])
{
    /* Curvilinear mesh points stored x0,y0,z0,x1,y1,z1,...*/
    float pts[] = {0, 0.5, 0, 1, 0, 0, 2, 0, 0,
        3, 0.5, 0, 0, 1, 0, 1, 1, 0,
        2, 1, 0, 3, 1, 0, 0, 1.5, 0,
        1, 2, 0, 2, 2, 0, 3, 1.5, 0, 
        0, 0.5, 1, 1, 0, 1, 2, 0, 1,
        3, 0.5, 1, 0, 1, 1, 1, 1, 1,
        2, 1, 1, 3, 1, 1, 0, 1.5, 1,
        1, 2, 1, 2, 2, 1, 3, 1.5, 1
    };
    int dims[] = {NX, NY, NZ};
    /* Zonal and nodal variable data. */
    float zonal[NZ-1][NY-1][NX-1], nodal[NZ][NY][NX];
    /* Info about the variables to pass to visit_writer. */
    int nvars = 2;
    int vardims[] = {1, 1};
    int centering[] = {0, 1};
    const char *varnames[] = {"zonal", "nodal"};
    float *vars[] = {(float *)zonal, (float *)nodal};
    int i,j,k, index = 0;

    /* Create zonal variable */
    for(k = 0; k < NZ-1; ++k)
        for(j = 0; j < NY-1; ++j)
            for(i = 0; i < NX-1; ++i, ++index)
                zonal[k][j][i] = (float)index;

    /* Create nodal variable. */
    index = 0;
    for(k = 0; k < NZ; ++k)
        for(j = 0; j < NY; ++j)
            for(i = 0; i < NX; ++i, ++index)
                nodal[k][j][i] = index;

    /* Pass the data to visit_writer to write a binary VTK file. */
    write_curvilinear_mesh("vwcurv3d.vtk", 1, dims, pts, nvars,
        vardims, centering, varnames, vars);

    return 0;
}
Example #2
0
int main(int argc, char *argv[]) {
    int rank = 0;
#ifdef PAR
    int size;
    double start_time, end_time;
    /* record start time */
    start_time = MPI_Wtime();
    MPI_Init (&argc, &argv);      /* starts MPI */
    MPI_Comm_rank (MPI_COMM_WORLD, &rank);        /* get current process id */
    MPI_Comm_size (MPI_COMM_WORLD, &size);        /* get number of processes */
#endif
    struct rlimit rlp;
    float max_magnitude = 0;
    float max_u = 0;
    float max_v = 0;
    float max_w = 0;
    float min_u = 0;
    float min_v = 0;
    float min_w = 0;
    int NZ; //NZ is the total number of layers, read from the pickpoints file.
    int nx, ny, nz=0; //nz is the number of layers assigned to a particular processor.
    int lowerbound=0;
    char *root_folder = ".";
    char *output_folder = "../vtks/";
    if (argc == 2) {
        root_folder = argv[1];
    }

    //Open the pickpoints file
    int n_pickpoints, total_pickpoints;
    FILE *fp_pickpoints;
    fp_pickpoints=fopen( build_string(root_folder, "/pickpoints.dat"), "r");
    if( fp_pickpoints == NULL ) {
        printf("Cannot open pickpoints file\n") ;
        return(1) ;
    } else {
        printf("Opened pickpoints file\n");
    }
    char buff[200];
    //Skip the line with the extents of the domain as we are not interested.
    fgets( buff, sizeof buff, fp_pickpoints );
    //printf("%s", buff);
    fgets( buff, sizeof buff, fp_pickpoints );
    if (sscanf( buff, "%d %d %d", &nx, &ny, &NZ) != 3) {
        printf( "Couldn't read the number of pickpoints.\n"
                "The format expected for the pickpoints file is:\n"
                "min_x max_x min_y max_y min_z max_z\n"
                "n_pickpoints_x n_pickpoints_y n_pickpoints_z\n"
                "<list of pickpoints coordinates>\n"
                "<list of pickpoint file names>");
        return 1;
    }
    total_pickpoints=nx*ny*NZ;
    //create the output folder
    mkdir(build_string(root_folder,output_folder), S_IRWXU | S_IRWXG | S_IRWXO);

#ifdef PAR
    if(size > NZ) {
        if(rank == 0)
            printf("You have more processors than layers (nz = %i)!\nSince multiprocessor partitioning works on dividing layers amongs processors... well...\nI'm quitting!\nMake sure np < nz (np < %i, yes.. that's a lower than, not lower or equal than)\n", NZ, NZ);
        MPI_Finalize();
        return(1);
    }
    //VisIt won't interpolate between 2 time zones, therefoere we need to fill in the gap
    nz = NZ / size;
    lowerbound = nz * nx * ny * rank;
    if(rank == size - 1) {
        nz = NZ - (nz * (size - 1));
        n_pickpoints = nx * ny * nz;
    } else {
        nz = (NZ / size) + 1;
        n_pickpoints = nz * nx * ny;
    }
    printf("[%i] lowerbound: %i\n", rank, lowerbound);
#else
    nz = NZ;
    n_pickpoints = nz*nx*ny;
#endif
    printf("[%i] Will be reading %i pickpoints out of %i.\n", rank, n_pickpoints, total_pickpoints);
    float pts[n_pickpoints * 3];
    int i, counter;
    i = 0;
    counter = 0;
    printf("[%i] lowerbound: %i\n", rank, lowerbound);
    //while ( fgets( buff, sizeof buff, fp_pickpoints ) != NULL && counter < total_pickpoints) {
    for(counter = 0; counter < total_pickpoints && (fgets( buff, sizeof buff, fp_pickpoints ) != NULL); counter ++) {
//		if ( !sscanf( buff, "%f %f %f", &pts[i], &pts[i++], &pts[i++] ) == 3 ) {
        if(counter >= lowerbound && counter < lowerbound + n_pickpoints) {
            //printf("Through the if\n");
            if ( !sscanf( buff, "%f %f %f", &pts[i], &pts[i+1], &pts[i+2] ) == 3 ) {
                printf("Couldn't read the %d pickpoint. Aborting.\n", i);
#ifdef PAR
                MPI_Finalize();
#endif
                return 1;
            } else {
                //printf("[%i]\t%d\t%f\t%f\t%f\n", rank, i, pts[i], pts[i+1], pts[i+2]);
                i += 3;
            }
        }/* else {
			printf("Failed the if\n");
		}*/
    }
#ifdef PAR
    printf("[%i] read in %i coordinates (shoulde be n_points * 3)\n", rank, i);
#else
    printf("Read in %i coordinates (shoulde be n_points * 3)\n", i);
#endif

//Get the name of the files
    i=0;
    counter = 0;
    int len;
    //FILE *fp_inputs[n_pickpoints];
    char inputs_files[n_pickpoints+1][100];
    off_t inputs_offsets[n_pickpoints+1];
    printf("Getting pickpoint file name information.\n");

    //while ( fgets( buff, sizeof buff, fp_pickpoints ) != NULL && counter++ < total_pickpoints) {
    for(counter = 0; counter < total_pickpoints && (fgets(inputs_files[i], sizeof buff, fp_pickpoints ) != NULL); counter ++) {
        //printf("[%i] counter: %i lowerbound %i\n", rank, counter, lowerbound);
        if(counter >= lowerbound && counter < lowerbound + n_pickpoints) {
            printf("[%i] Got file name %s", rank, inputs_files[i]);
            //remove the \n at the end
            len = strlen(inputs_files[i]);
            if( inputs_files[i][len-1] == '\n' )
                inputs_files[i][len-1] = 0;
            i++;
        }
    }

    fclose (fp_pickpoints);
    /*
    fgets(buff,sizeof buff, fp_inputs[167]);
    printf("[%i] %s", rank, buff);
    fgets(buff,sizeof buff, fp_inputs[199]);
    printf("[%i] %s", rank, buff);
    MPI_Finalize();
    return(0);//*/
//Skip the information at the beginning
    printf("[%i]Skipping information at the beginning\n", rank);
    int j;
    FILE *fp_input;
    for(i=0; i < n_pickpoints; i++) {
        //printf("[%i] skipping for file %i - \n", rank, i);
        fp_input = fopen(build_string(root_folder,inputs_files[i]), "r");
        if (fp_input == NULL) {
            printf("[%i] File in position %d was null %s!", rank, i, inputs_files[i]);
#ifdef PAR
            MPI_Finalize();
#endif
            return(1);
        }
        for(j=0; j<5; j++) {
            fgets(buff,sizeof buff, fp_input);
            //printf("[%i] %i - %s\n", rank, i, buff);
        }
        inputs_offsets[i] = ftello(fp_input);
        //printf("Current location in the file: %i __ %i",  ftell(fp_inputs[i]), inputs_offsets[i]);
        fclose(fp_input);
    }
    printf("[%i] Completed skipping information at the beginning\n", rank);

    /*for(i=0; i < 10; i++) {
    	fgets(buff,sizeof buff, fp_inputs[i]);
    	printf("Before %s\n", buff );
    	fclose(fp_inputs[i]);
    	fp_inputs[i] = fopen(build_string(root_folder, inputs_files[i]), "r");
    	fseek(fp_inputs[i], inputs_offsets[i], SEEK_SET);
    	fgets(buff,sizeof buff, fp_inputs[i]);
    	printf("After %s\n\n", buff );
    }*/

    j=0;
    char outputFileName[100];
    float data[n_pickpoints][3], u[nz][ny][nx], v[nz][ny][nx], w[nz][ny][nx], uvw[nz][ny][nx][3], times[n_pickpoints];
    int dims[] = {nx, ny, nz};
    int nvars = 4, x=0, y=0, z=0;
    int vardims[] = {1, 1, 1, 3};
    int centering[] = {1, 1, 1, 1};
    const char *varnames[] = {"u", "v", "w", "uvw"};

    //float *vars[] = {(float *)pts, data, data2};
    float *vars[] = {(float *)u, (float *)v, (float *)w, (float *)uvw};

    printf("[%i] Starting processing\n", rank);
    j=0;
    int finished = 0;
    while (!finished) {
        x = 0;
        z = 0;
        y = 0;

        for (i=0; i<n_pickpoints; i++) {
            //printf("[%i] i = %i reading from file: \n", rank, i, inputs_files[i]);
            fp_input = fopen(build_string(root_folder,inputs_files[i]), "r");
            if (fp_input == NULL) {
                printf("[%i] Unable to open file %s\n", rank, inputs_files[i]);
            }
            fseek(fp_input, inputs_offsets[i], SEEK_SET);
            if (fgets(buff, sizeof buff, fp_input) == NULL) {
                finished = 1;
                printf("[%i] Finisde and exiting loop\n", i);
                fclose(fp_input);
                break;
            }
            //printf("[%i] Previous offset: %d\n",rank, inputs_offsets[i]);
            inputs_offsets[i] = ftello(fp_input);
            //printf("[%i] After offset: %d\n", i, inputs_offsets[i]);
            //printf("[%i] Read in %s\n", buff);
            if (!(sscanf(buff, "%f %f %f %f", &times[i], &data[i][0], &data[i][1], &data[i][2]) == 4)) {
                printf("Scanf for pickpoint %i didn't return 4.\nExiting.\n", i);
#ifdef PAR
                MPI_Finalize();
#endif
                return(1);
            }
            //printf("%f %f %f %f\n", &times[i], &data[i][0], &data[i][1], &data[i][2]);
            //printf("%d - %d - %d\n", x, y, z);
            u[z][y][x] = data[i][0];
            v[z][y][x] = data[i][1];
            w[z][y][x] = data[i][2];
            uvw[z][y][x][0] = data[i][0];
            uvw[z][y][x][1] = data[i][1];
            uvw[z][y][x][2] = data[i][2];

            if (u[z][y][x] > max_u) max_u = u[z][y][x];
            if (u[z][y][x] < min_u) min_u = u[z][y][x];
            if (v[z][y][x] > max_v) max_v = v[z][y][x];
            if (v[z][y][x] < min_v) min_v = v[z][y][x];
            if (w[z][y][x] > max_w) max_w = w[z][y][x];
            if (w[z][y][x] < min_w) min_w = w[z][y][x];

            if((++y % ny) == 0) {
                x++;
                y=0;
            }
            if (x == nx) {
                z++;
                x = 0;
            }
            fclose(fp_input);
        }
        //At this point I should have all the data for this time step.
        /* Pass the mesh and data to visit_writer. */
        //cd printf("I'm out of the loop!\n");
        if (!finished) {
            //printf("I'm in the if statement\n");
            sprintf(outputFileName, "%s/%s/proc-%03i.%08d.vtk", root_folder, output_folder, rank,j);
            //printf("Will be saving to %s\n", outputFileName );
            //printf("%i %i %i\n", nx, ny, nz);
            write_curvilinear_mesh(outputFileName, 0, dims, (float*)pts, nvars, vardims, centering, varnames, vars);
        }
        /*MPI_Barrier(MPI_COMM_WORLD);
        MPI_Finalize();
        return(0);*/
        j++;
    }

    /*for (i=0; i<n_pickpoints; i++){
    	fclose(fp_inputs[i]);
    }*/

    //printf("Maxi magnitude was %lf\n", max_magnitude);
    printf("Max || Min u: %lf || %lf \n", max_u, min_u);
    printf("Max || Min v: %lf || %lf \n", max_v, min_v);
    printf("Max || Min w: %lf || %lf \n", max_w, min_w);

#ifdef PAR
    //write out the .visit file. Make sure you have altered tha variable that was counting the timesteps (j in this case)...
    if (rank==0) {
        FILE *fp_visit_config;
        fp_visit_config = fopen(build_string(root_folder,build_string(output_folder, "config.visit")), "w")
                          //NBLOCKS tells visit that every block of 4 files is related in a single time step
                          fprintf(fp_visit_config, "!NBLOCKS %i\n", size);
        for( counter=0; counter < j; counter++ ) {
            for(i=0; i < size; i++) {
                fprintf(fp_visit_config, "proc-%03i.%08d.vtk\n", i, counter);
            }
        }
    }

    MPI_Barrier(MPI_COMM_WORLD);
    /* record end time */
    if (rank == 0) {
        end_time = MPI_Wtime();
        printf("time to compute = %g seconds\n", end_time - start_time);
    }
    MPI_Finalize();
#endif
    return(0);
}
Example #3
0
int main(int argc, char *argv[]) {
    //    float pts[NX*NY*NZ*3];
    FILE* file = fopen(argv[1], "r");
    //FILE* outFile = fopen(argv[3], "w");
    char str[FILE_SIZE];

    int dims[] = {NX, NY, NZ};
    /* Zonal and nodal variable data. */
    //    float zonal[NZ][NY][NX];
    //float nodal[NZ][NY][NX];
    /* Info about the variables to pass to visit_writer. */
    int nvars = 1;
    int vardims[] = {1};
    int centering[] = {1};
    int i,j,k;
    const char *varnames[] = {"zonal"};
    //float *vars[] = {(float *)zonal};

    float *pts = (float *) malloc(NX*NY*NZ*3*sizeof(float));

    float *zonal = (float *)malloc(NX*NY*NZ*sizeof(float));
    float *vars[] = {(float *)zonal};

    int x = 0;
    int y = 0;
    int z = 0;
    int index = 0;

    for (int i = 0; i < NZ; i++) {
        for (int j = 0; j < NY; j++) {
            for (int k = 0; k < NX; k++) {
                pts[index++] = k;
                pts[index++] = j;
                pts[index++] = i;
            }
        }
    }



    while(fgets(str, FILE_SIZE, file) != NULL) {
        char line[FILE_SIZE];
        strcpy(line, str);
        if ((index / 3) % 10000 == 0) {
            printf("Processed %d lines\n", index / 3);
        }
        if (strlen(line) >= 2) {
            int count = 0;
            for (char* token = strtok(line, "\t"); token != NULL; token = strtok(NULL, "\t")) {
                if (count < 3) {
                    double point_val = atoi(token);
                    if (count == 0) {
                        x = point_val;
                    } else if (count == 1) {
                        y = point_val;
                    } else {
                        z = point_val;
                    }
                    count++;
                } else {
                    double val = atof(token);
                    int pos = z * NY * NX + y * NX + x;
                    zonal[pos] = val;
                }
            }
            //fprintf(outFile, "%d\t%d\t%d\t%f\n", x, y, z, zonal[z][y][x]);
        }
    }
    int pos = 30 * NY * NX + 21 * NX + 50;
    printf("%f\n", zonal[pos]);


    /* Pass the data to visit_writer to write a binary VTK file. */
    write_curvilinear_mesh(argv[2], 1, dims, pts, nvars,vardims, centering, varnames, vars);
    free(zonal);
    free(pts);
}
Example #4
0
int main(int argc, char **argv){
  int i,j,k,n; 
  int nsteps=150;
  double dt =0.01;
  
  /* Computational (2D polar) grid coordinates */
  int nx1 = 400;
  int nx2 = 400;

  //number of ghost cells on both sides of each dimension
  //only need 1 for piecewise constant method
  //need 2 for piecewise linear reconstruction
  int num_ghost = 2;
  int nx1_r = nx1;
  int nx2_r = nx2;
  nx1 += 2*num_ghost; 
  nx2 += 2*num_ghost; 

  /*non-ghost indices */
  int is = num_ghost;
  int ie= is+nx1_r; 
  int js = num_ghost;
  int je = js+nx2_r; 

  //convention: these ranges refer to the non-ghost cells
  //however, the ghost cells have real coordinate interpretations
  //this means that we must be sure that the number of ghost cells makes sense with the range of coordinates
  //this is a big problem if the phi polar coordinate runs the whole range [0,2pi) for example

  //further, all mesh structures (zonal and nodal) will be nx1 x nx2, although we may not fill the ghost entries with anything meaningful
  //this is to standardize the loop indexing from is:ie

  /*another convention: when the phi coordinate is periodic, do we repeat the boundary mesh points? yes for now */

  double lx1 = 2.0;//these values are inclusive [x1_i, x1_f]
  double lx2 = M_PI;
  double x1_i = 0.5;
  double x2_i = 0.0;

  double dx2 = lx2/(nx2_r-1);   
  double x2_f = x2_i + lx2;

  if(X2_PERIODIC){
    dx2 = 2*M_PI/(nx2_r);   
    lx2 = dx2*(nx2_r-1); 
    x2_i = 0.0;
  }

  double dx1 = lx1/(nx1_r-1);   
  double x1_f = x1_i + lx1;

  printf("dx1=%lf dx2=%lf \n",dx1,dx2); 
  
  /*Cell centered (zonal) values of computational coordinate position */
  double *x1 = (double *) malloc(sizeof(double)*nx1); 
  double *x2 = (double *) malloc(sizeof(double)*nx2);
  x1[is] = x1_i;
  x2[js] = x2_i;
  for(i=is+1; i<ie; i++){ 
    x1[i] = x1[i-1] + dx1;
    //    printf("%lf\n",x1[i]);
  }
  for(i=js+1; i<je; i++){
    x2[i] = x2[i-1] + dx2;
    //printf("%lf\n",x2[i]);
  } 

  /*Mesh edge (nodal) values of computational coordinate position */
  double *x1_b = (double *) malloc(sizeof(double)*(nx1+1)); 
  double *x2_b = (double *) malloc(sizeof(double)*(nx2+1));
  x1_b[is] = x1_i - dx1/2;
  x2_b[js] = x2_i - dx2/2;
  for(i=is+1; i<=ie; i++){ 
    x1_b[i] = x1_b[i-1] + dx1;
    //    printf("%lf\n",x1_b[i]);
  }
  for(i=js+1; i<=je; i++){
    x2_b[i] = x2_b[i-1] + dx2;
    //printf("%lf\n",x2_b[i]);
  } 

  /*Cell centered (zonal) values of physical coordinate position */
  //These must be 2D arrays since coordinate transformation is not diagonal
  //indexed by x1 rows and x2 columns
  double **x = (double **) malloc(sizeof(double *)*nx1); 
  double **y = (double **) malloc(sizeof(double *)*nx1);
  double *dataX = (double *) malloc(sizeof(double)*nx1*nx2);
  double *dataY = (double *) malloc(sizeof(double)*nx1*nx2);
  for(i=0; i<nx1; i++){
    x[i] = &(dataX[nx2*i]);
    y[i] = &(dataY[nx2*i]);
  }

  /*precompute coordinate mappings */
  for(i=is; i<ie; i++){
    for(j=js; j<je; j++){
      x[i][j] = X_physical(x1[i],x2[j]); 
      y[i][j] = Y_physical(x1[i],x2[j]); 
    }
  }
  /*Mesh edge (nodal) values of physical coordinate position */
  double **x_b = (double **) malloc(sizeof(double *)*(nx1+1)); 
  double **y_b = (double **) malloc(sizeof(double *)*(nx1+1));
  double *dataXb = (double *) malloc(sizeof(double)*(nx1+1)*(nx2+1));
  double *dataYb = (double *) malloc(sizeof(double)*(nx1+1)*(nx2+1));
  for(i=0; i<=nx1; i++){
    x_b[i] = &(dataXb[(nx2+1)*i]);
    y_b[i] = &(dataYb[(nx2+1)*i]);
  }
  for(i=is; i<=ie; i++){
    for(j=js; j<=je; j++){
      x_b[i][j] = X_physical(x1_b[i],x2_b[j]); 
      y_b[i][j] = Y_physical(x1_b[i],x2_b[j]); 
    }
  }
  /*Edge normal vectors in Cartesian coordinates */
  // point radially outward and +\phi
  

  /*Coordinate cell capacity */
  double **kappa;
  double *datakappa;
  kappa = (double **) malloc(sizeof(double *)*nx1);
  datakappa = (double *) malloc(sizeof(double)*nx1*nx2);
  for(i=0; i<nx1; i++){
    kappa[i] = &(datakappa[nx2*i]);
  }
  
  for(i=is; i<ie; i++){
    for(j=js; j<je; j++){
      kappa[i][j] = x1[i]*dx1*dx2/(dx1*dx2); // C_ij/(dx1*dx2)
      //capacity in ghost cells
      kappa[ie][j] = (x1[ie-1]+dx1)*dx1*dx2/(dx1*dx2); 
    }
    kappa[i][je] = (x1[i]+dx1)*dx1*dx2/(dx1*dx2); 
  }
 
  /*Average normal edge velocities */
  //now we move from cell centered quantities to edge quantities 
  //the convention in this code is that index i refers to i-1/2 edge
  double **U, **V;
  double *dataU, *dataV; 
  U = (double **) malloc(sizeof(double *)*nx1);
  V = (double **) malloc(sizeof(double *)*nx1);
  dataU = (double *) malloc(sizeof(double)*nx1*nx2);
  dataV = (double *) malloc(sizeof(double)*nx1*nx2);
  for(i=0; i<nx1; i++){
    U[i] = &(dataU[nx2*i]);
    V[i] = &(dataV[nx2*i]);
  }
  
  /*Option #1 for computing edge velocities: path integral of Cartesian stream function */
  /* Stream function */ //probably dont need this array
  //this variable is cell centered stream 
  /* double **stream;
  double *datastream;
  stream = (double **) malloc(sizeof(double *)*nx1);
  datastream = (double *) malloc(sizeof(double)*nx1*nx2);
  for(i=0; i<nx1; i++){
    stream[i] = &(datastream[nx2*i]);
  }
  for(i=is; i<ie; i++){
    for(j=js; j<je; j++){
      stream[i][j] = stream_function(x[i][j],y[i][j]);
    }
  }
 
   for(i=is; i<ie; i++){
    for(j=js; j<je; j++){ //go an additional step to capture nx2_r+1/2
      //average the corner stream functions
      U[i][j]= (stream_function(X_physical(x1[i]-dx1/2,x2[j]+dx2/2),Y_physical(x1[i]-dx1/2,x2[j]+dx2/2)) - 
		stream_function(X_physical(x1[i]-dx1/2,x2[j]-dx2/2),Y_physical(x1[i]-dx1/2,x2[j]-dx2/2)))/(dx2);
      V[i][j]= -(stream_function(X_physical(x1[i]+dx1/2,x2[j]-dx2/2),Y_physical(x1[i]+dx1/2,x2[j]-dx2/2)) - 
      	 stream_function(X_physical(x1[i]-dx1/2,x2[j]-dx2/2),Y_physical(x1[i]-dx1/2,x2[j]-dx2/2)))/dx1;
      if(j==10)
	printf("i=%d, u,v=%lf,%lf\n",i,U[i][j],V[i][j]); 
    }
    j=je;
    U[i][j]= (stream_function(X_physical(x1[i]-dx1/2,x2[j-1]+3*dx2/2),Y_physical(x1[i]-dx1/2,x2[j-1]+3*dx2/2)) - 
	      stream_function(X_physical(x1[i]-dx1/2,x2[j-1]+dx2/2),Y_physical(x1[i]-dx1/2,x2[j-1]+dx2/2)))/(dx2);
    V[i][j]= -(stream_function(X_physical(x1[i]+dx1/2,x2[j-1]+dx2/2),Y_physical(x1[i]+dx1/2,x2[j-1]+dx2/2)) - 
	       stream_function(X_physical(x1[i]-dx1/2,x2[j-1]+dx2/2),Y_physical(x1[i]-dx1/2,x2[j-1]+dx2/2)))/dx1;
  }
  i=ie;
  for(j=js; j<je; j++){
    U[i][j]= (stream_function(X_physical(x1[i-1]+dx1/2,x2[j]+dx2/2),Y_physical(x1[i-1]+dx1/2,x2[j]+dx2/2)) - 
	      stream_function(X_physical(x1[i-1]+dx1/2,x2[j]-dx2/2),Y_physical(x1[i-1]+dx1/2,x2[j]-dx2/2)))/(dx2);
    V[i][j]= -(stream_function(X_physical(x1[i-1]+3*dx1/2,x2[j]-dx2/2),Y_physical(x1[i-1]+3*dx1/2,x2[j]-dx2/2)) - 
	      stream_function(X_physical(x1[i-1]+dx1/2,x2[j]-dx2/2),Y_physical(x1[i-1]+dx1/2,x2[j]-dx2/2)))/dx1;
  } 
  */
  double ux,vy,temp; 
  for(i=is; i<=ie; i++){
    for(j=js; j<=je; j++){ 
      /*Option #2: directly specify velocity in Cartesian coordinate basis as a function of cartesian position*/   
      
      //radial face i-1/2
      velocity_physical(X_physical(x1_b[i],x2_b[j]+dx2/2),Y_physical(x1_b[i],x2_b[j]+dx2/2),&ux,&vy);
      // Average normal edge velocity: just transform face center velocity to local orthonormal basis? 
      vector_physical_to_coordinate(ux,vy,x2_b[j]+dx2/2,&U[i][j],&temp); 
      
#ifdef HORIZONTAL
      //EXACT SOLUTION FOR EDGE VELOCITY FOR HORIZONTAL FLOW
      //      printf("U_before = %lf\n",U[i][j]);
      U[i][j] = (sin(x2_b[j]+dx2) - sin(x2_b[j]))/(dx2); 
      //      printf("U_after = %lf\n",U[i][j]);
#endif

      //phi face j-1/2
      velocity_physical(X_physical(x1_b[i]+dx1/2,x2_b[j]),Y_physical(x1_b[i]+dx1/2,x2_b[j]),&ux,&vy);
      vector_physical_to_coordinate(ux,vy,x2_b[j],&temp,&V[i][j]); 
#if defined(SEMI_CLOCK) || defined(GAUSS_CLOCK)
      velocity_coordinate(x1_b[i],x2_b[j],&U[i][j],&V[i][j]);
#endif
      //      printf("U,V = %lf,%lf\n",U[i][j],V[i][j]);
    }
  } 

  /* check normalization of velocities  */
  /* these are naturally not normalized because the cell has finite volume so the edge velocities arent the velocity of the same point */
  double norm; 
  double max_deviation =0.0; 
  for(i=is; i<=ie; i++){
    for(j=js; j<=je; j++){ 
      norm = sqrt(U[i][j]*U[i][j] + V[i][j]*V[i][j]); 
      if (fabs(norm-1.0) > max_deviation)
	max_deviation = fabs(norm-1.0); 
      //  printf("%0.12lf\n",norm);
    }
  }
  //  printf("maximum deviation from 1.0 = %0.12lf\n",max_deviation); 



  /*Option #3: specify velocity in polar coordinates */


  /*Check CFL condition, reset timestep */
  float **cfl_array;
  float *dataCFL;
  cfl_array = (float **) malloc(sizeof(float *)*nx1);
  dataCFL = (float *) malloc(sizeof(float)*nx1*nx2);
  for(i=0; i<nx1; i++){
    cfl_array[i] = &(dataCFL[nx2*i]);
  }
  for(i=1; i<nx1; i++){
    for(j=1; j<nx2; j++){ //based on edge velocities or cell centered u,v?
      if (i >=is && i< ie && j >=js && j <je)
	cfl_array[i][j] = fabs(U[i][j])*dt/dx1 + fabs(V[i][j])*dt/(x1_b[i]*dx2); //use boundary radius
      else
	cfl_array[i][j] =0.0; 
    }
  }
  //find maximum CFL value in domain
  float  max_cfl = find_max(dataCFL,nx1*nx2); 
  printf("Largest CFL number = %lf\n",max_cfl); 
  if (max_cfl > CFL || AUTO_TIMESTEP){//reset timestep if needed
    dt = CFL*dt/max_cfl; 
    for(i=1; i<nx1; i++){
      for(j=1; j<nx2; j++){ 
	if (i >=is && i< ie && j >=js && j <je)
	  cfl_array[i][j] = fabs(U[i][j])*dt/dx1 + fabs(V[i][j])*dt/(x1_b[i]*dx2);
	else
	  cfl_array[i][j] =0.0; 
      }
    } 
  }
  max_cfl = find_max(dataCFL,nx1*nx2); 
  printf("Largest CFL number = %lf\n",max_cfl); 
  
#ifdef SEMI_CLOCK
  nsteps = M_PI/dt;
  printf("nsteps = %d, dt = %lf, t_final = %lf\n",nsteps,dt,nsteps*dt);
#endif
  
#ifdef GAUSS_CLOCK
  nsteps = 2*M_PI/dt;
  //turn dt down until mod(2*MPI,nsteps) = 0
  double remainder = 2*M_PI -nsteps*dt;
  double extra = remainder/nsteps; 
  double dt_new = dt +extra; 
  printf("nsteps = %d, dt = %lf, dt_new =%lf, remainder = %lf, t_final = %lf t_final_new =%lf\n",nsteps,dt,dt_new,remainder,nsteps*dt,nsteps*dt_new);
  max_cfl *= dt_new/dt; 
  dt = dt_new; 
  printf("Largest CFL number = %lf\n",max_cfl); 
#endif

  /*Conserved variable on the computational coordinate mesh*/
  double **Q;
  double *dataQ;
  //make contiguous multiarray                                                                                                
  Q = (double **) malloc(sizeof(double *)*nx1);
  dataQ = (double *) malloc(sizeof(double)*nx1*nx2);
  for(i=0; i<nx1; i++){
    Q[i] = &(dataQ[nx2*i]);
  }
  /*Initial condition */
  //specified as a function of cartesian physical cooridnates, as is the stream function
  for(i=is; i<ie; i++){
    for(j=js; j<je; j++){
      Q[i][j] = initial_condition(x[i][j],y[i][j]); 
    }
  }

  //net fluctuations/fluxes
  double U_plus,U_minus,V_plus,V_minus;
  double **net_fluctuation;
  double *dataFlux;
  net_fluctuation = (double **) malloc(sizeof(double *)*nx1);
  dataFlux = (double *) malloc(sizeof(double)*nx1*nx2);
  for(i=0; i<nx1; i++){
    net_fluctuation[i] = &(dataFlux[nx2*i]);
  }

  /* Using Visit VTK writer */
  char filename[20];
  int dims[] = {nx1_r+1, nx2_r+1, 1}; //dont output ghost cells. //nodal variables have extra edge point
  int nvars = 2;
  int vardims[] = {1, 3}; //Q is a scalar, velocity is a 3-vector 
  int centering[] = {0, 1}; // Q is cell centered, velocity is defined at edges
  const char *varnames[] = {"Q", "edge_velocity"};
  /* Curvilinear mesh points stored x0,y0,z0,x1,y1,z1,...*/
  //An array of size nI*nJ*nK*3 . These points are nodal, not zonal... unfortunatley
  float *pts = (float *) malloc(sizeof(float)*(nx1_r+1)*(nx2_r+1)*3); 
  //The array should be layed out as (pt(i=0,j=0,k=0), pt(i=1,j=0,k=0), ...
  //pt(i=nI-1,j=0,k=0), pt(i=0,j=1,k=0), ...).
  int index=0; 
  for(k=0; k<1; k++){
    for(j=js; j<=je; j++){
      for(i=is; i<=ie; i++){
	pts[index] = x_b[i][j];
	pts[++index] = y_b[i][j];
	pts[++index] = 0.0;
	index++;
      }
    }
  }

  /* pack U and V into a vector */
  float *edge_vel = (float *) malloc(sizeof(float)*(nx1_r+1)*(nx2_r+1)*3); //An array of size nI*nJ*nK*3 
  index=0; 
  for(k=0; k<1; k++){
    for(j=js; j<=je; j++){
      for(i=is; i<=ie; i++){
	vector_coordinate_to_physical(U[i][j],V[i][j], x2[j], &ux,&vy);
	//edge_vel[index] = U[i][j];
	//edge_vel[++index] = V[i][j]; 
	edge_vel[index] = ux;
	edge_vel[++index] = vy;
	edge_vel[++index] = 0.0;
	index++;
      }
    }
  } 

  //  vars       An array of variables.  The size of vars should be nvars.
  //                 The size of vars[i] should be npts*vardim[i].
  float *realQ; 
  realQ =(float *) malloc(sizeof(double)*nx1_r*nx2_r);
  float *vars[] = {(float *) realQ, (float *)edge_vel};

  /*-----------------------*/
  /* Main timestepping loop */
  /*-----------------------*/
  for (n=0; n<nsteps; n++){
    /*Boundary conditions */
    //bcs are specified along a computational coord direction, but are a function of the physical coordinate of adjacent "real cells"
    for(k=0;k<num_ghost; k++){
      for (j=js; j<je; j++){
	Q[k][j] = bc_x1i(x[is][j],y[is][j]);
	Q[nx1-1-k][j] = bc_x1f(x[ie-1][j],y[ie-1][j],(n+1)*dt);
      }
      for (i=is; i<ie; i++){
	if(X2_PERIODIC){
	  Q[i][k] = Q[i][je-1-k];
	  Q[i][nx2-1-k] = Q[i][js+k];
	}
	else{
	Q[i][k] = bc_x2i(x[i][js],y[i][js]);
	Q[i][nx2-1-k] = bc_x2f(x[i][je-1],y[i][je-1]);
	}
      }
    }

    double flux_limiter =0.0; 
    double *qmu = (double *) malloc(sizeof(double)*3); //manually copy array for computing slope limiters
    /* Donor cell upwinding */
    for (i=is; i<ie; i++){
      for (j=js; j<je; j++){
	/* First coordinate */
	U_plus = fmax(U[i][j],0.0); // max{U_{i-1/2,j},0.0} LHS boundary
	U_minus = fmin(U[i+1][j],0.0); // min{U_{i+1/2,j},0.0} RHS boundary
	/*Fluctuations: A^+ \Delta Q_{i-1/2,j} + A^- \Delta Q_{i+1/2,j} */
	//	net_fluctuation[i][j] = dt/(kappa[i][j]*dx1)*(U_plus*(Q[i][j] - Q[i-1][j]) + U_minus*(Q[i+1][j] - Q[i][j]));
	/* First order fluxes: F_i+1/2 - F_i-1/2 */
	net_fluctuation[i][j] = dt/(kappa[i][j]*dx1)*(x1_b[i+1]*(fmax(U[i+1][j],0.0)*Q[i][j] + U_minus*Q[i+1][j])-x1_b[i]*(U_plus*Q[i-1][j] + fmin(U[i][j],0.0)*Q[i][j]));
#ifdef SECOND_ORDER
	/* Second order fluxes */
	if (U[i+1][j] > 0.0){ //middle element is always the upwind element
	  qmu[0] = Q[i-1][j];
	  qmu[1] = Q[i][j];  
	  qmu[2] = Q[i+1][j];  
	  flux_limiter= flux_PLM(dx1,qmu);
	}
	else{
	  qmu[0] = Q[i+2][j];
	  qmu[1] = Q[i+1][j];  
	  qmu[2] = Q[i][j];  
	  flux_limiter= flux_PLM(dx1,qmu);
	  /*	  if (flux_limiter != 0.0){
	    printf("i,j: %d,%d F_{i+1/2} flux limiter: %lf\n",i,j,flux_limiter);
	    printf("Q0 = %lf Q1= %lf Q2 = %lf\n",qmu[0],qmu[1],qmu[2]); } */
	}
	//F^H_{i+1/2,j}
	net_fluctuation[i][j] -= dt/(kappa[i][j]*dx1)*(x1_b[i+1]*(1-dt*fabs(U[i+1][j])/(dx1))*fabs(U[i+1][j])*flux_limiter/2);
	//	net_fluctuation[i][j] -= dt/(kappa[i][j]*dx1)*((kappa[i][j]/x1_b[i+1]-dt*fabs(U[i+1][j])/(x1_b[i+1]*dx1))*fabs(U[i+1][j])*flux_limiter/2);
	//	net_fluctuation[i][j] += dt/(kappa[i][j]*dx1)*(x1_b[i+1]*(kappa[i][j]/x1_b[i+1]-dt*fabs(U[i+1][j])/(x1_b[i+1]*dx1))*fabs(U[i+1][j])*flux_limiter/2);
	if (U[i][j] > 0.0){
	  qmu[0] = Q[i-2][j];  //points to the two preceeding bins; 
	  qmu[1] = Q[i-1][j];  
	  qmu[2] = Q[i][j];  
	  flux_limiter= flux_PLM(dx1,qmu);
	}
	else{
	  qmu[0] = Q[i+1][j]; //centered around current bin
	  qmu[1] = Q[i][j];  
	  qmu[2] = Q[i-1][j];  
	  flux_limiter= flux_PLM(dx1,qmu);
	}
	//F^H_{i-1/2,j}
	net_fluctuation[i][j] += dt/(kappa[i][j]*dx1)*(x1_b[i]*(1-dt*fabs(U[i][j])/(dx1))*fabs(U[i][j])*flux_limiter/2);
	//	net_fluctuation[i][j] += dt/(kappa[i][j]*dx1)*((kappa[i-1][j]/x1_b[i]-dt*fabs(U[i][j])/(x1_b[i]*dx1))*fabs(U[i][j])*flux_limiter/2);
	//net_fluctuation[i][j] -= dt/(kappa[i][j]*dx1)*(x1_b[i]*(kappa[i-1][j]/x1_b[i]-dt*fabs(U[i][j])/(x1_b[i]*dx1))*fabs(U[i][j])*flux_limiter/2);
#endif
	/* Second coordinate */
	V_plus = fmax(V[i][j],0.0); // max{V_{i,j-1/2},0.0} LHS boundary
	V_minus = fmin(V[i][j+1],0.0); // min{V_{i,j+1/2},0.0} RHS boundary
	/*Fluctuations: B^+ \Delta Q_{i,j-1/2} + B^- \Delta Q_{i,j+1/2} */
	//net_fluctuation[i][j] += dt/(kappa[i][j]*dx2)*(V_plus*(Q[i][j] - Q[i][j-1]) + V_minus*(Q[i][j+1] - Q[i][j]));	
	/* Fluxes: G_i,j+1/2 - G_i,j-1/2 */
	net_fluctuation[i][j] += dt/(kappa[i][j]*dx2)*((fmax(V[i][j+1],0.0)*Q[i][j] + V_minus*Q[i][j+1])-(V_plus*Q[i][j-1] + fmin(V[i][j],0.0)*Q[i][j]));
#ifdef SECOND_ORDER
	/* Second order fluxes */
	if (V[i][j+1] > 0.0){
	  qmu[0] = Q[i][j-1];  //points to the two preceeding bins; 
	  qmu[1] = Q[i][j];  
	  qmu[2] = Q[i][j+1];  
	  flux_limiter= flux_PLM(dx2,qmu);
	}
	else{
	  qmu[0] = Q[i][j+2]; //centered around current bin
	  qmu[1] = Q[i][j+1];  
	  qmu[2] = Q[i][j];  
	  flux_limiter= flux_PLM(dx2,qmu);
	  /*	  if (flux_limiter != 0.0){
	    printf("i,j: %d,%d F_{i+1/2} flux limiter: %lf\n",i,j,flux_limiter);
	    printf("Q0 = %lf Q1= %lf Q2 = %lf\n",qmu[0],qmu[1],qmu[2]); } */
	}
	//G^H_{i,j+1/2}
	net_fluctuation[i][j] -= dt/(kappa[i][j]*dx2)*((1-dt*fabs(V[i][j+1])/(kappa[i][j]*dx2))*fabs(V[i][j+1])*flux_limiter/2);
	//net_fluctuation[i][j] -= dt/(kappa[i][j]*dx2)*((1-dt*fabs(V[i][j+1])/(dx2))*fabs(V[i][j+1])*flux_limiter/2);
	if (V[i][j] > 0.0){
	  qmu[0] = Q[i][j-2];  //points to the two preceeding bins; 
	  qmu[1] = Q[i][j-1];  
	  qmu[2] = Q[i][j];  
	  flux_limiter = flux_PLM(dx2,qmu);
	}
	else{
	  qmu[0] = Q[i][j+1]; //centered around current bin
	  qmu[1] = Q[i][j];  
	  qmu[2] = Q[i][j-1];  
	  flux_limiter= flux_PLM(dx2,qmu);
	}
	//G^H_{i,j-1/2}
	net_fluctuation[i][j] += dt/(kappa[i][j]*dx2)*((1-dt*fabs(V[i][j])/(kappa[i][j]*dx2))*fabs(V[i][j])*flux_limiter/2);
	//net_fluctuation[i][j] += dt/(kappa[i][j]*dx2)*((1-dt*fabs(V[i][j])/(dx2))*fabs(V[i][j])*flux_limiter/2);
#endif
      }
    }

    /*Apply fluctuations */
    for (i=is; i<ie; i++)
      for (j=js; j<je; j++){
	Q[i][j] -= net_fluctuation[i][j];
      }
    
    /*Source terms */
    for (i=is; i<ie; i++)
      for (j=js; j<je; j++){
      }

    /*Output */
    //for now, explicitly copy subarray corresponding to real zonal info:
    index=0; 
    for (k=0; k<1; k++){
      for (j=js; j<je; j++){
	for (i=is; i<ie; i++){
	  //index =(j-num_ghost)*nx2_r + (i-num_ghost); 
	  realQ[index] = (float) Q[i][j];//*kappa[i][j]; //\bar{q}=qk density in computational space
	  index++;
	}
      }
    }
    //debug only horizontal flow
    /*    if (find_max(realQ,nx1_r*nx2_r) > 1.1){
      printf("Q greater than 1.0!\n"); 
      for (i=1;i<nx1; i++){
	for (j=1; j<nx2; j++){
	  if (Q[i][j] > 1.0){
	    printf("i=%d j=%d Q[i][j] = %0.10lf\n",i,j,Q[i][j]); 
	    return(0); 
	  }
	}
      }
      }*/

    sprintf(filename,"advect-%.3d.vtk",n); 
    if(!OUTPUT_INTERVAL){
      if (n==nsteps-1) //for only the final result
	write_curvilinear_mesh(filename,3,dims, pts, nvars,vardims, centering, varnames, vars);}
    else{
      if (!(n%OUTPUT_INTERVAL)) //HAVENT CHECKED THIS
	write_curvilinear_mesh(filename,3,dims, pts, nvars,vardims, centering, varnames, vars);}
      
      printf("step: %d time: %lf max{Q} = %0.7lf min{Q} = %0.7lf sum{Q} = %0.7lf \n",
	     n+1,(n+1)*dt,find_max(realQ,nx1_r*nx2_r),find_min(realQ,nx1_r*nx2_r),sum(realQ,nx1_r*nx2_r));
  }
  return(0); 
}