int main(int argc, char **argv) {
MPI_Offset i, j, k;
int status;
int ncid;
int dimid1, dimid2, dimid3, udimid;
int square_dim[2], cube_dim[3], xytime_dim[3], time_dim[1];
MPI_Offset square_start[2], cube_start[3] = {0, 0, 0};
MPI_Offset square_count[2] = {50, 50}, cube_count[3] = {100, 50, 50};
MPI_Offset xytime_start[3] = {0, 0, 0};
MPI_Offset xytime_count[3] = {100, 50, 50};
MPI_Offset time_start[1], time_count[1] = {25};
int square_id, cube_id, xytime_id, time_id;
static char title[] = "example netCDF dataset";
static char description[] = "2-D integer array";
int data[100][50][50], buffer[100];
int rank;
int nprocs;
MPI_Comm comm = MPI_COMM_WORLD;
double TotalWriteTime;
params opts;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (rank == 0)
fprintf(stderr, "Testing write ... \n");
parse_write_args(argc, argv, rank, &opts);
MPI_Barrier(MPI_COMM_WORLD);
TotalWriteTime = MPI_Wtime();
/********** START OF NETCDF ACCESS **************/
/**
* Create the dataset
* File name: "testwrite.nc"
* Dataset API: Collective
*/
status = ncmpi_create(comm, opts.outfname, NC_CLOBBER|NC_64BIT_OFFSET, MPI_INFO_NULL, &ncid);
if (status != NC_NOERR) handle_error(status);
/**
* Create a global attribute:
* :title = "example netCDF dataset";
*/
sprintf(title, "%s:%d of %d", title, rank, nprocs);
printf("title:%s\n", title);
status = ncmpi_put_att_text (ncid, NC_GLOBAL, "title",
strlen(title), title);
if (status != NC_NOERR) handle_error(status);
/**
* Add 4 pre-defined dimensions:
* x = 100, y = 100, z = 100, time = NC_UNLIMITED
*/
status = ncmpi_def_dim(ncid, "x", 100L, &dimid1);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_dim(ncid, "y", 100L, &dimid2);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_dim(ncid, "z", 100L, &dimid3);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &udimid);
if (status != NC_NOERR) handle_error(status);
/**
* Define the dimensionality and then add 4 variables:
* square(x, y), cube(x,y,z), time(time), xytime(time, x, y)
*/
square_dim[0] = cube_dim[0] = xytime_dim[1] = dimid1;
square_dim[1] = cube_dim[1] = xytime_dim[2] = dimid2;
cube_dim[2] = dimid3;
xytime_dim[0] = udimid;
time_dim[0] = udimid;
status = ncmpi_def_var (ncid, "square", NC_INT, 2, square_dim, &square_id);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_var (ncid, "cube", NC_INT, 3, cube_dim, &cube_id);
if (status != NC_NOERR) handle_error(status);
// status = ncmpi_def_var (ncid, "time", NC_INT, 1, time_dim, &time_id);
status = ncmpi_def_var (ncid, "time", NC_INT, 1, time_dim, &time_id);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_var (ncid, "xytime", NC_INT, 3, xytime_dim, &xytime_id);
if (status != NC_NOERR) handle_error(status);
/**
* Add an attribute for variable:
* square: decsription = "2-D integer array"
*/
status = ncmpi_put_att_text (ncid, square_id, "description",
strlen(description), description);
if (status != NC_NOERR) handle_error(status);
/**
* End Define Mode (switch to data mode)
* Dataset API: Collective
*/
status = ncmpi_enddef(ncid);
if (status != NC_NOERR){
handle_error(status);
status = ncmpi_close(ncid);
if (status != NC_NOERR) handle_error(status);
if (rank == 0) {
fprintf(stderr, "Fatal Error: file header is inconsistent!\n");
}
}
/**
* Data Partition (Assume 4 processors):
* square: 2-D, (Block, Block), 50*50 from 100*100
* cube: 3-D, (*, Block, Block), 100*50*50 from 100*100*100
* xytime: 3-D, (*, Block, Block), 100*50*50 from 100*100*100
* time: 1-D, Block-wise, 25 from 100
*/
else {
square_start[0] = cube_start[1] = xytime_start[1] = (rank/2) * 50;
square_start[1] = cube_start[2] = xytime_start[2] = (rank%2) * 50;
time_start[0] = (rank%4) * 25;
/**
* Packing data in the buffer
*/
/* Data for variable: time */
for ( i = time_start[0]; i < time_start[0] + time_count[0]; i++ )
buffer[i - time_start[0]] = i;
/* Data for variable: square, cube and xytime */
for ( i = 0; i < 100; i++ )
for ( j = square_start[0]; j < square_start[0]+square_count[0]; j++ )
for ( k = square_start[1]; k < square_start[1]+square_count[1]; k++ )
data[i][j-square_start[0]][k-square_start[1]] = i*100*100 + j*100 + k;
/**
* Write data into variables: square, cube, time and xytime
* Access Method: subarray
* Data Mode API: collective
*/
status = ncmpi_put_vara_int_all(ncid, square_id,
square_start, square_count,
&data[0][0][0]);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_int_all(ncid, cube_id,
cube_start, cube_count,
&data[0][0][0]);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_int_all(ncid, time_id,
time_start, time_count,
(void *)buffer);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_int_all(ncid, xytime_id,
xytime_start, xytime_count,
&data[0][0][0]);
if (status != NC_NOERR) handle_error(status);
/*
status = ncmpi_sync(ncid);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_redef(ncid);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_del_att(ncid, square_id, "description");
if (status != NC_NOERR) handle_error(status);
status = ncmpi_enddef(ncid);
if (status != NC_NOERR) handle_error(status);
*/
/**
* Close the dataset
* Dataset API: collective
*/
status = ncmpi_close(ncid);
if (status != NC_NOERR) handle_error(status);
/******************* END OF NETCDF ACCESS ****************/
MPI_Barrier(MPI_COMM_WORLD);
TotalWriteTime = MPI_Wtime() - TotalWriteTime;
if (rank == 0) {
fprintf(stderr, "OK\nFile written to: %s!\n", opts.outfname);
fprintf(stderr, "Total Write Time = %10.8f\n", TotalWriteTime);
}
}
MPI_Finalize();
return 0;
}
int main(int argc, char **argv) {
int i, j, k;
int status;
int ncid;
int dimid1, dimid2, dimid3, udimid;
int square_dim[2], cube_dim[3], xytime_dim[3], time_dim[1];
MPI_Offset square_start[2], cube_start[3] = {0, 0, 0};
MPI_Offset square_count[2] = {50, 50}, cube_count[3] = {100, 50, 50};
MPI_Offset square_stride[2] = {2, 2};
MPI_Offset xytime_start[3] = {0, 0, 0};
MPI_Offset xytime_count[3] = {100, 50, 50};
MPI_Offset time_start[1], time_count[1] = {25};
int square_id, cube_id, xytime_id, time_id;
static char title[] = "example netCDF dataset";
static char description[] = "2-D integer array";
double data[100][50][50], buffer[100];
double stride_2d_data[50][50];
int rank;
int nprocs;
MPI_Comm comm = MPI_COMM_WORLD;
params opts;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (rank == 0)
fprintf(stderr, "Testing write ... ");
parse_write_args(argc, argv, rank, &opts);
/********** START OF NETCDF ACCESS **************/
/**
* Create the dataset
* File name: "testwrite.nc"
* Dataset API: Collective
*/
status = ncmpi_create(comm, opts.outfname, NC_CLOBBER, MPI_INFO_NULL, &ncid);
if (status != NC_NOERR) handle_error(status);
/**
* Create a global attribute:
* :title = "example netCDF dataset";
*/
status = ncmpi_put_att_text (ncid, NC_GLOBAL, "title",
strlen(title), title);
if (status != NC_NOERR) handle_error(status);
/**
* Add 4 pre-defined dimensions:
* x = 100, y = 100, z = 100, time = NC_UNLIMITED
*/
status = ncmpi_def_dim(ncid, "x", 100L, &dimid1);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_dim(ncid, "y", 100L, &dimid2);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_dim(ncid, "z", 100L, &dimid3);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &udimid);
if (status != NC_NOERR) handle_error(status);
/**
* Define the dimensionality and then add 4 variables:
* square(x, y), cube(x,y,z), time(time), xytime(time, x, y)
*/
square_dim[0] = cube_dim[0] = xytime_dim[1] = dimid1;
square_dim[1] = cube_dim[1] = xytime_dim[2] = dimid2;
cube_dim[2] = dimid3;
xytime_dim[0] = udimid;
time_dim[0] = udimid;
status = ncmpi_def_var (ncid, "square", NC_DOUBLE, 2, square_dim, &square_id);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_var (ncid, "cube", NC_DOUBLE, 3, cube_dim, &cube_id);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_var (ncid, "time", NC_DOUBLE, 1, time_dim, &time_id);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_var (ncid, "xytime", NC_DOUBLE, 3, xytime_dim, &xytime_id);
if (status != NC_NOERR) handle_error(status);
/**
* Add an attribute for variable:
* square: decsription = "2-D integer array"
*/
status = ncmpi_put_att_text (ncid, square_id, "description",
strlen(description), description);
if (status != NC_NOERR) handle_error(status);
/**
* End Define Mode (switch to data mode)
* Dataset API: Collective
*/
status = ncmpi_enddef(ncid);
if (status != NC_NOERR) handle_error(status);
/**
* Data Partition (Assume 4 processors):
* square: 2-D, (Cyclic, Cyclic), 50*50 from 100*100, strided access
* cube: 3-D, (*, Block, Block), 100*50*50 from 100*100*100
* xytime: 3-D, (*, Block, Block), 100*50*50 from 100*100*100
* time: 1-D, Block-wise, 25 from 100
*/
/* square_start[0] = */
cube_start[1] = xytime_start[1] = (rank/2) * 50;
/* square_start[1] = */
cube_start[2] = xytime_start[2] = (rank%2) * 50;
time_start[0] = (rank%4) * 25;
square_start[0] = rank/2;
square_start[1] = rank%2;
/**
* Packing data in the buffer
*/
/* Data for variable: time */
for ( i = time_start[0]; i < time_start[0] + time_count[0]; i++ )
buffer[i - time_start[0]] = i;
/* Data for variable: cube and xytime */
for ( i = 0; i < 100; i++ )
for ( j = cube_start[1]; j < cube_start[1]+cube_count[1]; j++ )
for ( k = cube_start[2]; k < cube_start[2]+cube_count[2]; k++ )
data[i][j-cube_start[1]][k-cube_start[2]] = i*100*100 + j*100 + k;
/* Data for variable: square */
for ( i = 0; i < 50; i ++ )
for ( j = 0; j < 50; j++ )
stride_2d_data[i][j] = (2*i + rank/2)*100 + (2*j + rank%2);
/**
* Write data into variables: square, cube, time and xytime
* Access Method: subarray
* Data Mode API: collective
*/
status = ncmpi_put_vars_double_all(ncid, square_id,
square_start, square_count, square_stride,
&stride_2d_data[0][0]);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_double_all(ncid, cube_id,
cube_start, cube_count,
&data[0][0][0]);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_double_all(ncid, time_id,
time_start, time_count,
(void *)buffer);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_double_all(ncid, xytime_id,
xytime_start, xytime_count,
&data[0][0][0]);
if (status != NC_NOERR) handle_error(status);
/**
* Close the dataset
* Dataset API: collective
*/
status = ncmpi_close(ncid);
if (status != NC_NOERR) handle_error(status);
/******************* END OF NETCDF ACCESS ****************/
if (rank == 0)
fprintf(stderr, "OK\nFile written to: %s!\n", opts.outfname);
MPI_Finalize();
return 0;
}
static int writehtk_l (lua_State *L){
// luaL_openlibs(L);
// Parse the arguments and write them out
if (!parse_write_args(L)){
return 0;
}
std::ofstream outputfile;
outputfile.open(luaL_checkstring(L,1),std::ios::binary);
short feat_dim = luaL_checknumber(L,4);
// 4 bytes for the feature dimensions
short featurelength = feat_dim*4;
if(outputfile.is_open()){
// HTK uses big endian, c++ uses little endian, so we need
// to convert the big to the little endian using the buildin
// gcc function bswap
int n_samples=luaL_checknumber(L,2);
n_samples = __builtin_bswap32(n_samples);
int sampleperiod=luaL_checknumber(L,3);
sampleperiod=__builtin_bswap32(sampleperiod);
featurelength = _swap16b(featurelength);
std::string featuretypestring=lua_tostring(L,5);
std::vector<std::string> featuresplit= split(featuretypestring,'_');
std::map<std::string,int>::iterator it;
// Check if given feature type exists in HTK
it = HTKFEATURETYPE.find(featuresplit[0]);
if (it == HTKFEATURETYPE.end()){
std::string err = "Feature type unknown! Possible types are:\n";
for (std::map<std::string,int>::iterator i = HTKFEATURETYPE.begin(); i != HTKFEATURETYPE.end(); ++i)
{
err += i->first + "\n";
}
error(L,err);
return 0;
}
// Init the featuretype without any classifiers
short featuretype = it->second;
std::map<std::string,int>::iterator classit;
for(auto i = 1u; i < featuresplit.size();i++){
classit = HTKCLASSIFIER.find(featuresplit[i]);
if (classit == HTKCLASSIFIER.end()){
std::string classifier = "Classifier ";
std::string unrecog = " unrecognized";
std::string err = classifier + featuresplit[i] + unrecog;
error(L,err);
return 0;
}
featuretype += classit->second;
}
featuretype = _swap16b(featuretype);
// get size of the data
auto size_n = luaL_getn(L,6);
// Write out the HTK header, which is two 4 byte ints and two 2 byte shorts
// in big endian format
outputfile.write(reinterpret_cast<char*>(&n_samples),sizeof(int));
outputfile.write(reinterpret_cast<char*>(&sampleperiod),sizeof(int));
outputfile.write(reinterpret_cast<char*>(&featurelength),sizeof(short));
outputfile.write(reinterpret_cast<char*>(&featuretype),sizeof(short));
// Write out the data ( we use as default floats here )
// Lua indices start at 1
for (auto i = 1; i <= size_n; ++i)
{
// Get the current index i of the array on the stack
lua_rawgeti(L,6, i);
// Convert the current value into a float
float val = lua_tonumber(L,-1);
// Pop the value from the stack
lua_pop(L,1);
val = chtk::swapfloatendian(val);
outputfile.write(reinterpret_cast<char*>(&val),sizeof(float));
}
outputfile.close();
}else{
std::cerr<< " File is already open or cant be opened !" <<std::endl;
return 0;
}
return 0;
}
int main(int argc, char **argv) {
int i, j, k;
int status;
int ncid;
int dimid1, dimid2, dimid3, udimid;
int square_dim[2], cube_dim[3], xytime_dim[3], time_dim[1];
MPI_Offset square_start[2], cube_start[3] = {0, 0, 0};
MPI_Offset square_count[2] = {50, 50}, cube_count[3] = {100, 50, 50};
MPI_Offset xytime_start[3] = {0, 0, 0};
MPI_Offset xytime_count[3] = {100, 50, 50};
MPI_Offset time_start[1], time_count[1] = {25};
int square_id, cube_id, xytime_id, time_id;
static char title[] = "example netCDF dataset";
static char description[] = "2-D integer array";
double data[100][50][50], buffer[100];
int rank;
int nprocs;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info;
params opts;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (rank == 0)
fprintf(stderr, "Testing independent write ... ");
parse_write_args(argc, argv, rank, &opts);
/********** START OF NETCDF ACCESS **************/
MPI_Info_create(&info);
MPI_Info_set(info, "striping_factor", "4");
MPI_Info_set(info, "striping_unit", "20000");
MPI_Info_set(info, "start_iodevice", "0");
/**
* Create the dataset
* File name: "testwrite.nc"
* Dataset API: Collective
*/
status = ncmpi_create(comm, opts.outfname, NC_CLOBBER, info, &ncid);
if (status != NC_NOERR) handle_error(status);
/**
* Create a global attribute:
* :title = "example netCDF dataset";
*/
status = ncmpi_put_att_text (ncid, NC_GLOBAL, "title",
strlen(title), title);
if (status != NC_NOERR) handle_error(status);
/**
* Add 4 pre-defined dimensions:
* x = 100, y = 100, z = 100, time = NC_UNLIMITED
*/
status = ncmpi_def_dim(ncid, "x", 100L, &dimid1);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_dim(ncid, "y", 100L, &dimid2);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_dim(ncid, "z", 100L, &dimid3);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &udimid);
if (status != NC_NOERR) handle_error(status);
/**
* Define the dimensionality and then add 4 variables:
* square(x, y), cube(x,y,z), time(time), xytime(time, x, y)
*/
square_dim[0] = cube_dim[0] = xytime_dim[1] = dimid1;
square_dim[1] = cube_dim[1] = xytime_dim[2] = dimid2;
cube_dim[2] = dimid3;
xytime_dim[0] = udimid;
time_dim[0] = udimid;
status = ncmpi_def_var (ncid, "square", NC_DOUBLE, 2, square_dim, &square_id);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_var (ncid, "cube", NC_DOUBLE, 3, cube_dim, &cube_id);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_var (ncid, "time", NC_DOUBLE, 1, time_dim, &time_id);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_def_var (ncid, "xytime", NC_DOUBLE, 3, xytime_dim, &xytime_id);
if (status != NC_NOERR) handle_error(status);
/**
* Add an attribute for variable:
* square: decsription = "2-D integer array"
*/
status = ncmpi_put_att_text (ncid, square_id, "description",
strlen(description), description);
if (status != NC_NOERR) handle_error(status);
/**
* End Define Mode (switch to data mode)
* Dataset API: Collective
*/
status = ncmpi_enddef(ncid);
if (status != NC_NOERR) handle_error(status);
/**
* Data Partition (Assume 4 processors):
* square: 2-D, (Block, Block), 50*50 from 100*100
* cube: 3-D, (*, Block, Block), 100*50*50 from 100*100*100
* xytime: 3-D, (*, Block, Block), 100*50*50 from 100*100*100
* time: 1-D, Block-wise, 25 from 100
*/
square_start[0] = cube_start[1] = xytime_start[1] = (rank/2) * 50;
square_start[1] = cube_start[2] = xytime_start[2] = (rank%2) * 50;
time_start[0] = (rank%4) * 25;
/**
* Packing data in the buffer
*/
/* Data for variable: time */
for ( i = time_start[0]; i < time_start[0] + time_count[0]; i++ )
buffer[i - time_start[0]] = i;
/* Data for variable: square, cube and xytime */
for ( i = 0; i < 100; i++ )
for ( j = square_start[0]; j < square_start[0]+square_count[0]; j++ )
for ( k = square_start[1]; k < square_start[1]+square_count[1]; k++ )
data[i][j-square_start[0]][k-square_start[1]] = i*100*100 + j*100 + k;
/**
* Write data into variables: square, cube, time and xytime
* Access Method: subarray
* Data Mode API: non-collective
*/
status = ncmpi_begin_indep_data(ncid);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_double(ncid, square_id,
square_start, square_count,
&data[0][0][0]);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_double(ncid, cube_id,
cube_start, cube_count,
&data[0][0][0]);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_double(ncid, time_id,
time_start, time_count,
(double *)buffer);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_vara_double(ncid, xytime_id,
xytime_start, xytime_count,
&data[0][0][0]);
if (status != NC_NOERR) handle_error(status);
{
/**
* Change a single element and then change it back
* Access Method: single value
* Data Mode API: non-collective
*/
double singlevalue = 0;
ncmpi_sync(ncid);
status = ncmpi_put_var1_double(ncid, square_id,
square_start, &singlevalue);
if (status != NC_NOERR) handle_error(status);
status = ncmpi_put_var1_double(ncid, time_id,
time_start, &singlevalue);
if (status != NC_NOERR) handle_error(status);
ncmpi_sync(ncid);
singlevalue = square_start[0]*100 + square_start[1];
status = ncmpi_put_var1_double(ncid, square_id,
square_start, &singlevalue);
if (status != NC_NOERR) handle_error(status);
singlevalue = time_start[0];
status = ncmpi_put_var1_double(ncid, time_id,
time_start, &singlevalue);
if (status != NC_NOERR) handle_error(status);
}
{
/**
* Change the whole array for time[] and then change it back
* Access Method: whole array
* Data Mode API: non-collective
*/
ncmpi_sync(ncid);
for (i = 0; i < 100; i++ )
buffer[i] = 0;
if (rank == 0) {
status = ncmpi_put_var_double(ncid, time_id, buffer);
if (status != NC_NOERR) handle_error(status);
}
ncmpi_sync(ncid);
for (i=0; i<100; i++)
buffer[i] = i;
if (rank == 1) {
status = ncmpi_put_var_double(ncid, time_id, buffer);
if (status != NC_NOERR) handle_error(status);
}
}
status = ncmpi_end_indep_data(ncid);
if (status != NC_NOERR) handle_error(status);
/**
* Close the dataset
* Dataset API: collective
*/
status = ncmpi_close(ncid);
if (status != NC_NOERR) handle_error(status);
MPI_Info_free(&info);
/******************* END OF NETCDF ACCESS ****************/
if (rank == 0)
fprintf(stderr, "OK\nFile written to: %s!\n", opts.outfname);
MPI_Finalize();
return 0;
}
