void Cio_data::write(vector<Cparticle> &ps, int timestep) {
H5PartFile *writer = H5PartOpenFile(filename.c_str(),H5PART_WRITE);
H5PartSetStep(writer,timestep);
int n = ps.size();
H5PartSetNumParticles(writer,n);
double r[NDIM][n],v[NDIM][n],mass[n],u[n];
long long tag[n];
double zero[n];
for (int i=0; i<n; i++) {
zero[i] = 0.0;
for (int j=0; j<NDIM; j++) {
r[j][i] = ps[i].r[j];
v[j][i] = ps[i].v[j];
}
mass[i] = ps[i].mass;
u[i] = ps[i].u;
tag[i] = ps[i].tag;
}
for (int i=0; i<NDIM; i++) {
H5PartWriteDataFloat64(writer,dimNames[i],r[i]);
H5PartWriteDataFloat64(writer,vdimNames[i],v[i]);
}
for (int i=NDIM; i<3; i++) {
H5PartWriteDataFloat64(writer,dimNames[i],zero);
H5PartWriteDataFloat64(writer,vdimNames[i],zero);
}
H5PartWriteDataFloat64(writer,"mass",mass);
H5PartWriteDataFloat64(writer,"u",u);
H5PartWriteDataInt64(writer,"id",tag);
}
static void
test_read_data64(H5PartFile *file, int nparticles, int step)
{
int i,t;
int rank, nprocs;
h5part_int64_t status, val, start, end, type, size;
char name1[4];
char name2[8];
h5part_int64_t indices[8];
double *x,*y,*z;
double *px,*py,*pz;
h5part_int64_t *id;
x=(double*)malloc(nparticles*sizeof(double));
y=(double*)malloc(nparticles*sizeof(double));
z=(double*)malloc(nparticles*sizeof(double));
px=(double*)malloc(nparticles*sizeof(double));
py=(double*)malloc(nparticles*sizeof(double));
pz=(double*)malloc(nparticles*sizeof(double));
id=(h5part_int64_t*)malloc(nparticles*sizeof(h5part_int64_t));
TEST("Verifying dataset info");
#if PARALLEL_IO
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
#else
nprocs = 1;
rank = 2;
#endif
val = H5PartGetNumParticles(file);
IVALUE(val, nprocs*nparticles, "particle count");
val = H5PartGetNumDatasets(file);
IVALUE(val, 7, "dataset count");
for (i=0; i<7; i++) {
status = H5PartGetDatasetName(file, i, name1, 2);
RETURN(status, H5PART_SUCCESS, "H5PartGetDatasetName");
status = H5PartGetDatasetInfo(
file, i, name2, 4, &type, &size);
RETURN(status, H5PART_SUCCESS, "H5PartGetDatasetInfo");
CVALUE(name1[0], name2[0], "dataset name");
status = H5PartGetDatasetName(file, i, name1, 4);
RETURN(status, H5PART_SUCCESS, "H5PartGetDatasetName");
CVALUE(name1[1], name2[1], "dataset name");
IVALUE(size, nprocs*nparticles, "dataset size");
if (name1[0] == 'i') IVALUE(type, H5PART_INT64, "dataset type");
else IVALUE(type, H5PART_FLOAT64, "dataset type");
}
#if PARALLEL_IO
TEST("Setting throttle");
status = H5PartSetThrottle(file, 3);
RETURN(status, H5PART_SUCCESS, "H5PartSetThrottle");
#endif
TEST("Reading 64-bit data");
for (t=step; t<step+NTIMESTEPS; t++)
{
val = H5PartHasStep(file, t);
IVALUE(val, 1, "has step");
status = H5PartSetStep(file, t);
RETURN(status, H5PART_SUCCESS, "H5PartSetStep");
test_read_step_attribs(file, t);
status = H5PartSetNumParticles(file, nparticles);
RETURN(status, H5PART_SUCCESS, "H5PartSetNumParticles");
status = H5PartResetView(file);
RETURN(status, H5PART_SUCCESS, "H5PartResetView");
start = rank;
end = -1;
status = H5PartSetView(file, start, end);
RETURN(status, H5PART_SUCCESS, "H5PartSetView");
status = H5PartReadDataFloat64(file, "x", x);
RETURN(status, H5PART_SUCCESS, "H5PartReadDataFloat64");
FVALUE(x[rank], (double)(start+rank+nparticles*t), "x data");
val = H5PartGetView(file, &start, &end);
IVALUE(val, nprocs*nparticles-start, "particle count");
IVALUE(start, rank, "view start");
IVALUE(end, nprocs*nparticles-1, "view end");
status = H5PartSetView(file, -1, -1);
RETURN(status, H5PART_SUCCESS, "H5PartSetView");
status = H5PartReadDataFloat64(file, "x", x);
RETURN(status, H5PART_SUCCESS, "H5PartReadDataFloat64");
IVALUE(x[rank], (double)(rank+nparticles*t), "x data");
indices[0] = rank*2 + 0;
indices[1] = rank*2 + 3;
indices[2] = rank*2 + 9;
indices[3] = rank*2 + 7;
status = H5PartSetViewIndices(file, indices, -1);
RETURN(status, H5PART_SUCCESS, "H5PartSetViewIndices");
status = H5PartReadDataFloat64(file, "x", x);
RETURN(status, H5PART_SUCCESS, "H5PartReadDataFloat64");
FVALUE(x[2*rank], (double)(2*rank+nparticles*t), "x data");
status = H5PartResetView(file);
RETURN(status, H5PART_SUCCESS, "H5PartResetView");
status = H5PartSetViewIndices(file, indices, 4);
RETURN(status, H5PART_SUCCESS, "H5PartSetViewIndices");
val = H5PartGetNumParticles(file);
IVALUE(val, 4, "particle count");
status = H5PartReadDataFloat64(file, "x", x);
RETURN(status, H5PART_SUCCESS, "H5PartReadDataFloat64");
FVALUE(x[2], (double)(rank*2+9+nparticles*t), "x data");
val = H5PartGetNumParticles(file);
IVALUE(val, 4, "particle count");
status = H5PartSetViewIndices(file, NULL, 4);
RETURN(status, H5PART_SUCCESS, "H5PartSetViewIndices");
status = H5PartReadDataFloat64(file, "x", x);
RETURN(status, H5PART_SUCCESS, "H5PartReadDataFloat64");
status = H5PartSetCanonicalView(file);
RETURN(status, H5PART_SUCCESS, "H5PartSetCanonicalView");
status = H5PartReadParticleStep (
file, t, x, y, z, px, py, pz, id);
RETURN(status, H5PART_SUCCESS, "H5PartReadParticleStep");
for (i=0; i<nparticles; i++)
{
FVALUE(x[i] , 0.0 + (double)(i+nparticles*t), " x data");
FVALUE(y[i] , 0.1 + (double)(i+nparticles*t), " y data");
FVALUE(z[i] , 0.2 + (double)(i+nparticles*t), " z data");
FVALUE(px[i], 0.3 + (double)(i+nparticles*t), " px data");
FVALUE(py[i], 0.4 + (double)(i+nparticles*t), " py data");
FVALUE(pz[i], 0.5 + (double)(i+nparticles*t), " pz data");
IVALUE(id[i], (i+nparticles*t), " id data");
}
}
}
static void
test_write_data64(H5PartFile *file, int nparticles, int step)
{
int i,t;
h5part_int64_t status, val;
double *x,*y,*z;
double *px,*py,*pz;
h5part_int64_t *id;
x=(double*)malloc(nparticles*sizeof(double));
y=(double*)malloc(nparticles*sizeof(double));
z=(double*)malloc(nparticles*sizeof(double));
px=(double*)malloc(nparticles*sizeof(double));
py=(double*)malloc(nparticles*sizeof(double));
pz=(double*)malloc(nparticles*sizeof(double));
id=(h5part_int64_t*)malloc(nparticles*sizeof(h5part_int64_t));
/* invalid stride will produce a warning */
status = H5PartSetNumParticlesStrided(file, nparticles, -1);
RETURN(status, H5PART_SUCCESS, "H5PartSetNumParticlesStrided");
/* invalid nparticles will produce an error */
status = H5PartSetNumParticlesStrided(file, -1, 2);
RETURN(status, H5PART_ERR_INVAL, "H5PartSetNumParticlesStrided");
#if PARALLEL_IO
TEST("Setting throttle");
status = H5PartSetThrottle(file, 2);
RETURN(status, H5PART_SUCCESS, "H5PartSetThrottle");
#endif
TEST("Writing 64-bit data");
for (t=step; t<step+NTIMESTEPS; t++)
{
for (i=0; i<nparticles; i++)
{
x[i] = 0.0 + (double)(i+nparticles*t);
y[i] = 0.1 + (double)(i+nparticles*t);
z[i] = 0.2 + (double)(i+nparticles*t);
px[i] = 0.3 + (double)(i+nparticles*t);
py[i] = 0.4 + (double)(i+nparticles*t);
pz[i] = 0.5 + (double)(i+nparticles*t);
id[i] = i + nparticles*t;
}
val = H5PartHasStep(file, t);
status = H5PartSetStep(file, t);
RETURN(status, H5PART_SUCCESS, "H5PartSetStep");
if (val == 0) test_write_step_attribs(file, t);
status = H5PartSetNumParticles(file, nparticles);
RETURN(status, H5PART_SUCCESS, "H5PartSetNumParticles");
status = H5PartWriteDataFloat64(file, "x", x);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat64");
status = H5PartWriteDataFloat64(file, "y", y);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat64");
status = H5PartWriteDataFloat64(file, "z", z);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat64");
status = H5PartWriteDataFloat64(file, "px", px);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat64");
status = H5PartWriteDataFloat64(file, "py", py);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat64");
status = H5PartWriteDataFloat64(file, "pz", pz);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat64");
status = H5PartWriteDataInt64(file, "id", id);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataInt64");
}
}
static void
test_write_data32(H5PartFile *file, int nparticles, int step)
{
int i,t;
h5part_int32_t status, val;
int rank, nprocs;
float *x,*y,*z;
float *px,*py,*pz;
int *id;
x=(float*)malloc(nparticles*sizeof(float));
y=(float*)malloc(nparticles*sizeof(float));
z=(float*)malloc(nparticles*sizeof(float));
px=(float*)malloc(nparticles*sizeof(float));
py=(float*)malloc(nparticles*sizeof(float));
pz=(float*)malloc(nparticles*sizeof(float));
id=(int*)malloc(nparticles*sizeof(int));
status = H5PartSetNumParticles(file, nparticles);
RETURN(status, H5PART_SUCCESS, "H5PartSetNumParticles");
#if PARALLEL_IO
/* will generate a warning since we are in MPI-IO Collective mode */
TEST("Setting throttle");
status = H5PartSetThrottle(file, 2);
RETURN(status, H5PART_SUCCESS, "H5PartSetThrottle");
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
#else
rank = 0;
nprocs = 1;
#endif
TEST("Writing 32-bit data");
for (t=step; t<step+NTIMESTEPS; t++)
{
for (i=0; i<nparticles; i++)
{
x[i] = 0.0F + (float)(i+nparticles*t);
y[i] = 0.1F + (float)(i+nparticles*t);
z[i] = 0.2F + (float)(i+nparticles*t);
px[i] = 0.3F + (float)(i+nparticles*t);
py[i] = 0.4F + (float)(i+nparticles*t);
pz[i] = 0.5F + (float)(i+nparticles*t);
id[i] = i + nparticles*t;
}
val = H5PartHasStep(file, t);
if (val == 0) {
status = H5PartSetStep(file, t);
RETURN(status, H5PART_SUCCESS, "H5PartSetStep");
}
/* test a two-part write using views */
status = H5PartSetView(file,
rank*nparticles,
rank*nparticles + 31);
RETURN(status, H5PART_SUCCESS, "H5PartSetView");
status = H5PartWriteDataFloat32(file, "x", x);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat32");
test_write_step_attribs(file, t);
status = H5PartWriteDataFloat32(file, "y", y);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat32");
status = H5PartWriteDataFloat32(file, "z", z);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat32");
status = H5PartWriteDataFloat32(file, "px", px);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat32");
status = H5PartWriteDataFloat32(file, "py", py);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat32");
status = H5PartWriteDataFloat32(file, "pz", pz);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat32");
status = H5PartWriteDataInt32(file, LONGNAME, id);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataInt32");
/* the second write phase... */
status = H5PartSetView(file,
rank*nparticles + 32,
rank*nparticles + nparticles - 1);
RETURN(status, H5PART_SUCCESS, "H5PartSetView");
/* offset the input arrays */
status = H5PartWriteDataFloat32(file, "x", x+32);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat32");
status = H5PartWriteDataFloat32(file, "y", y+32);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat32");
status = H5PartWriteDataFloat32(file, "z", z+32);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat32");
status = H5PartWriteDataFloat32(file, "px", px+32);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat32");
status = H5PartWriteDataFloat32(file, "py", py+32);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat32");
status = H5PartWriteDataFloat32(file, "pz", pz+32);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataFloat32");
status = H5PartWriteDataInt32(file, LONGNAME, id+32);
RETURN(status, H5PART_SUCCESS, "H5PartWriteDataInt32");
}
}
double write_data (float* data, int iter, Params* p)
{
int i,j;
double start_time;
double open_time;
double write_time;
double close_time;
double total_time;
double sum_time;
double open_mean;
double write_mean;
double close_mean;
double bandwidth;
float* segment;
char var_name[64];
char* filename;
H5PartFile* file;
h5part_int64_t status;
if (p->verbosity >= VERBOSITY_HIGH) {
printf ("rank %d: writing data\n", p->rank);
}
start_time = MPI_Wtime();
filename = (char*) malloc (strlen (p->filename) + 64);
sprintf (filename, "%s/%d.h5", p->filename, iter);
file = H5PartOpenFileParallelAlign (filename,
p->flags, MPI_COMM_WORLD, p->alignment);
if (!file) {
fprintf (stderr,
"rank %d: could not open H5Part file!\n", p->rank);
MPI_Barrier (MPI_COMM_WORLD);
exit (EXIT_FAILURE);
}
H5PartSetNumParticles (file, p->particles);
if (p->verbosity >= VERBOSITY_HIGH) {
printf ("rank %d: %ld particles\n", p->rank, p->particles);
}
open_time = MPI_Wtime() - start_time;
segment = data;
for (i=1; i<=p->segments; i++) {
status = H5PartSetStep (file, i);
if (status != H5PART_SUCCESS) {
fprintf (stderr, "rank %d: H5PartSetStep error!", p->rank);
}
for (j=0; j<p->blocks; j++) {
sprintf (var_name, "test%d", j);
status = H5PartWriteDataFloat32 (file, var_name, segment);
if (status != H5PART_SUCCESS) {
fprintf (stderr,
"rank %d: H5PartWriteDataFloat32 error!", p->rank);
}
segment += p->particles;
}
}
write_time = (MPI_Wtime() - start_time) - open_time;
H5PartCloseFile (file);
close_time = (MPI_Wtime() - start_time) - write_time - open_time;
total_time = open_time + write_time + close_time;
if (p->verbosity >= VERBOSITY_HIGH) {
printf ("rank %d: write\t%.3f\t%.3f\t%.3f\t%.3f\n", p->rank,
open_time, write_time, close_time, total_time);
}
MPI_Barrier (MPI_COMM_WORLD);
MPI_Reduce (&open_time, &sum_time, 1, MPI_DOUBLE,
MPI_SUM, 0, MPI_COMM_WORLD);
open_mean = sum_time / p->procs;
MPI_Reduce (&write_time, &sum_time, 1, MPI_DOUBLE,
MPI_SUM, 0, MPI_COMM_WORLD);
write_mean = sum_time / p->procs;
MPI_Reduce (&close_time, &sum_time, 1, MPI_DOUBLE,
MPI_SUM, 0, MPI_COMM_WORLD);
close_mean = sum_time / p->procs;
bandwidth = p->aggregate_size / total_time;
if (p->verbosity >= VERBOSITY_MEDIUM && p->rank == 0) {
printf ("write\t%.1f\t%.3f\t%.3f\t%.3f\t%.3f\n", bandwidth,
open_mean, write_mean, close_mean, total_time);
}
free (filename);
return bandwidth;
}
void H5output::WriteParticles(int ispec, long long np, const double *q, const double *x, const double *y, const double *z, const double *u, const double *v, const double *w, MPI_Comm CART_COMM){
/* --------------------------------------------------------------------- */
/* Find out the total number of particles of species i in all the domain */
/* --------------------------------------------------------------------- */
long long ntpart;
MPI_Allreduce(&np, &ntpart, 1, MPI_LONG_LONG, MPI_SUM, CART_COMM);
const h5_int64_t ntp = ntpart;
/* --------------------------------------------- */
/* Write the number of particles as an attribute */
/* --------------------------------------------- */
std::stringstream sstm;
sstm << "npart_" << ispec;
std::string nparti = sstm.str();
H5WriteStepAttribInt64(partfile, nparti.c_str(), &ntp, 1);
sstm.str("");
H5PartSetNumParticles(partfile,np);
/* ------------------ */
/* Write the datasets */
/* ------------------ */
sstm << "q_" << ispec;
std::string dtset = sstm.str();
H5PartWriteDataFloat64(partfile,dtset.c_str(),q);
sstm.str("");
sstm << "x_" << ispec;
dtset = sstm.str();
H5PartWriteDataFloat64(partfile,dtset.c_str(),x);
sstm.str("");
sstm << "y_" << ispec;
dtset = sstm.str();
H5PartWriteDataFloat64(partfile,dtset.c_str(),y);
sstm.str("");
sstm << "z_" << ispec;
dtset = sstm.str();
H5PartWriteDataFloat64(partfile,dtset.c_str(),z);
sstm.str("");
sstm << "u_" << ispec;
dtset = sstm.str();
H5PartWriteDataFloat64(partfile,dtset.c_str(),u);
sstm.str("");
sstm << "v_" << ispec;
dtset = sstm.str();
H5PartWriteDataFloat64(partfile,dtset.c_str(),v);
sstm.str("");
sstm << "w_" << ispec;
dtset = sstm.str();
H5PartWriteDataFloat64(partfile,dtset.c_str(),w);
sstm.str("");
}