/*
* mpi_io_shared
*
* creates a single-shared-file
* writes with independent-io
* reads with independent-io
* writes with collective-io
* reads with collective-io
*/
int mpi_io_shared (char *path, int size, int rank)
{
MPI_File fh;
char filepath[512];
MPI_Offset offset;
MPI_Status status;
void *buf;
int bufcount = BYTES_PER_RANK;
int rc;
buf = malloc(bufcount);
if (!buf) { return 0; }
memset(buf, 0xa, bufcount);
sprintf(filepath, "%s/%s", path, "cp-bench-mpio-shared");
rc = MPI_File_open(MPI_COMM_WORLD,
filepath,
(MPI_MODE_CREATE|MPI_MODE_RDWR|MPI_MODE_DELETE_ON_CLOSE),
MPI_INFO_NULL,
&fh);
MPI_CHECK(rc,"MPI_File_open");
/* Indep Write */
offset = rank * bufcount;
rc = MPI_File_write_at(fh,offset,buf,bufcount,MPI_BYTE,&status);
MPI_CHECK(rc,"MPI_File_write_at");
MPI_Barrier(MPI_COMM_WORLD);
/* Indep Read */
offset = ((rank+1)%size) * bufcount;
rc = MPI_File_read_at(fh,offset,buf,bufcount,MPI_BYTE,&status);
MPI_CHECK(rc,"MPI_File_read_at");
/* Collective Write */
offset = rank * bufcount;
rc = MPI_File_write_at_all(fh, offset, buf, bufcount, MPI_BYTE, &status);
MPI_CHECK(rc,"MPI_File_write_at_all");
/* Collective Read */
offset = ((rank+1)%size) * bufcount;
rc = MPI_File_read_at_all(fh, offset, buf, bufcount, MPI_BYTE, &status);
MPI_CHECK(rc,"MPI_File_read_at_all");
rc = MPI_File_close(&fh);
MPI_CHECK(rc,"MPI_File_close");
free(buf);
return 1;
}
/* This test checks if datareps given are little- or big-endian */
int main( int argc, char* argv[] ) {
int sample_i = 123456789, i, j;
char sample_i_le[4] = {0x15,0xcd,0x5b,0x07}, c[4];
const char* datarep[3] = { "native", "external32", "internal" };
MPI_File fileh;
int rank;
FILE* fileh_std;
if( sizeof(int) != 4 ) { printf( "non-supported sizeof(int)=%ld\n", sizeof(int) ); return (-1); }
MPI_Init( &argc, &argv );
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
/* For each datarep */
for( i = 0; i < 3; i++ ) {
/* Open file */
CHECK(MPI_File_open( MPI_COMM_WORLD, TEST_FILENAME,
MPI_MODE_RDWR | MPI_MODE_CREATE, MPI_INFO_NULL, &fileh ) );
/* Set view */
CHECK(MPI_File_set_view( fileh, 0, MPI_INT, MPI_INT, datarep[i], MPI_INFO_NULL ));
/* Write into file */
CHECK(MPI_File_write_at( fileh, (MPI_Offset)rank, (void*)&sample_i, 1,
MPI_INT, MPI_STATUS_IGNORE ));
/* Close file */
CHECK(MPI_File_close( &fileh ));
/* Check if your datarep is little or big endian */
MPI_Barrier( MPI_COMM_WORLD );
if( rank == 0 ) {
fileh_std = fopen( TEST_FILENAME, "r" );
for( j = 0; j < 4; j++ ) {
if( feof( fileh_std ) ) { printf( "unexpected eof, aborted\n" ); return (-1); }
fscanf( fileh_std, "%c", &c[j] );
}
is_little_or_big_endian( datarep[i], c, sample_i_le, 4 );
fclose( fileh_std );
}
/* Delete file */
if( rank == 0 ) {
CHECK(MPI_File_delete( TEST_FILENAME, MPI_INFO_NULL ));
}
}
MPI_Finalize();
return 0;
}
FORT_DLL_SPEC void FORT_CALL mpi_file_write_at_ ( MPI_Fint *v1, MPI_Offset *v2, void*v3, MPI_Fint *v4, MPI_Fint *v5, MPI_Fint *v6, MPI_Fint *ierr ){
#ifdef MPI_MODE_RDONLY
#ifndef HAVE_MPI_F_INIT_WORKS_WITH_C
if (MPIR_F_NeedInit){ mpirinitf_(); MPIR_F_NeedInit = 0; }
#endif
if (v6 == MPI_F_STATUS_IGNORE) { v6 = (MPI_Fint*)MPI_STATUS_IGNORE; }
*ierr = MPI_File_write_at( MPI_File_f2c(*v1), (MPI_Offset)*v2, v3, (int)*v4, (MPI_Datatype)(*v5), (MPI_Status *)v6 );
#else
*ierr = MPI_ERR_INTERN;
#endif
}
void dump_time_field(char* file_prefix, grid_parms grid, double field)
{
MPI_Status status;
MPI_Offset displacement = 0;
MPI_File fw;
char* buffer = (char*)checked_malloc(NUM_DBL_TO_CHAR_BYTES * sizeof(char), SRC_LOC);
char* write_buffer;
int root = 0;
char filename[50];
int length = sprintf(buffer, "%2.12lf\n", field);
int *recv_count = (int*) checked_malloc(grid.num_ranks_branch * sizeof(int), SRC_LOC);
int *disp = (int*) checked_malloc(grid.num_ranks_branch * sizeof(int), SRC_LOC);
/// Gathering the lengths of buffer from each MPI process.
CHECK_MPI_ERROR(MPI_Gather(&length, 1, MPI_INT, recv_count, 1, MPI_INT, root, grid.cart_comm));
int total_buffer_length = 0;
for (int i = 0; i < grid.num_ranks_branch; i++)
{
disp[i] = total_buffer_length;
total_buffer_length += recv_count[i];
}
if (grid.rank_branch == 0)
{
write_buffer = (char*) checked_malloc(total_buffer_length * sizeof(char), SRC_LOC);
}
// Gathering the buffers from all MPI processes.
CHECK_MPI_ERROR(MPI_Gatherv(buffer, length, MPI_CHAR, write_buffer, recv_count, disp, MPI_CHAR, root, grid.cart_comm));
if (grid.rank_branch == 0)
{
sprintf(filename, "%s/%s_%d_%d.txt", grid.time_profiling_dir, file_prefix, grid.domain_index, grid.branch_tag);
CHECK_MPI_ERROR(MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fw));
CHECK_MPI_ERROR(MPI_File_write_at(fw, 0, write_buffer, total_buffer_length, MPI_CHAR, &status));
MPI_File_close(&fw);
}
if (grid.rank_branch == 0)
{
free(write_buffer);
}
free(recv_count);
free(buffer);
free(disp);
}
void Mpi2Evolution::OpenFiles(int is_restart){
World* pW= World::instance();
if (pW->saveEvolStepSize == 0) return;
#ifndef HAVE_MPI_THREADS
if (pW->m_myRank == 0)
cout << "\n!!! Warning: !!!\nSaving spin evolution in parallel JEMRIS mode requires MPI2.0 (parallel file I/O). no evol files will be written.\nUse MPI2.0 or sequential jemris.\n" << endl;
#else
long M = pW->TotalADCNumber / pW->saveEvolStepSize ;
string fname;
int SpinNo = pW->TotalSpinNumber;
MPI_Bcast(&is_restart,1,MPI_INT,0,MPI_COMM_WORLD);
MPI_Bcast(&SpinNo,1,MPI_INT,0,MPI_COMM_WORLD);
MPI_Offset filesize;
filesize = (SpinNo * 7 +2)* sizeof(double);
MPI_Status status;
for (int i=0; i<M; i++) {
stringstream sF;
sF << pW->saveEvolFileName << "_" << setw(3) << setfill('0') << i+1 << ".bin";
if (is_restart != 1){
// delete existing old file; (c) by Rolf Rabenseifer...
/* MPI::File fh=MPI::File::Open(MPI::COMM_WORLD,(sF.str()).c_str(),MPI::MODE_DELETE_ON_CLOSE | MPI::MODE_CREATE | MPI::MODE_WRONLY, MPI::INFO_NULL );
fh.Close();*/
// above lines lead to trouble on our cluster. try different approach:
if (pW->m_myRank == 0) {
ofstream myfile;
myfile.open ((sF.str()).c_str(),ios::out | ios::binary| ios::trunc);
myfile.close();
}
MPI_Barrier(MPI_COMM_WORLD);
}
MPI_File mpifh;
MPI_File_open (MPI_COMM_WORLD, charstar(sF.str()),MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &mpifh);
m_files.push_back(mpifh);
MPI_File_preallocate(m_files[i], filesize);
if (pW->m_myRank == 1) {
double dummy = (double) SpinNo;
MPI_File_write_at(m_files[i], 0, &dummy, 1, MPI_DOUBLE,&status);
m_first_write.push_back(true);
}
}
#endif
}
void dump_coords(grid_parms grid, celltype2** ec, checkpoint_handle* check, const char* message){
MPI_Status status;
MPI_Offset disp;
int write_element_count = grid.num_ec_axially;
double buffer[write_element_count];
int k = 0;
int i =1;
for (int j = 1; j <= grid.num_ec_axially; j++) {
buffer[k] = ec[i][j].z_coord;
k++;
}
int offset = grid.rank/grid.n;
disp = (offset * write_element_count * sizeof(double));
CHECK(MPI_File_write_at(check->coords, disp, &buffer, write_element_count, MPI_DOUBLE, &status));
}
void dump_ec(grid_parms grid, celltype2 **ec, checkpoint_handle *check, int write_count){
MPI_Status status[8];
MPI_Offset disp;
int write_element_count, time_offset_in_file;
write_element_count = grid.num_ec_axially * grid.num_ec_circumferentially;
time_offset_in_file = write_count * write_element_count * grid.tasks
* sizeof(double);
double b1[grid.num_ec_circumferentially * grid.num_ec_axially],
b2[grid.num_ec_circumferentially * grid.num_ec_axially],
b3[grid.num_ec_circumferentially * grid.num_ec_axially],
b4[grid.num_ec_circumferentially * grid.num_ec_axially],
b5[grid.num_ec_circumferentially * grid.num_ec_axially],
b6[grid.num_ec_circumferentially * grid.num_ec_axially],
b7[grid.num_ec_circumferentially * grid.num_ec_axially];
int k;
k = 0;
for (int i = 1; i <= grid.num_ec_circumferentially; i++) {
for (int j = 1; j <= grid.num_ec_axially; j++) {
b1[k] = ec[i][j].q[ec_Ca];
b2[k] = ec[i][j].q[ec_SR];
b3[k] = ec[i][j].q[ec_Vm];
b4[k] = ec[i][j].q[ec_IP3];
b5[k] = ec[i][j].B[cpl_Ca];
b6[k] = ec[i][j].B[cpl_Vm];
b7[k] = ec[i][j].B[cpl_IP3];
k++;
}
}
disp = time_offset_in_file
+ (grid.rank * write_element_count * sizeof(double));
CHECK(
MPI_File_write_at(check->cj, disp, &b1, write_element_count, MPI_DOUBLE, &status[0]));
CHECK(
MPI_File_write_at(check->sj, disp, &b2, write_element_count, MPI_DOUBLE, &status[1]));
CHECK(
MPI_File_write_at(check->vj, disp, &b3, write_element_count, MPI_DOUBLE, &status[2]));
CHECK(
MPI_File_write_at(check->Ij, disp, &b4, write_element_count, MPI_DOUBLE, &status[3]));
CHECK(
MPI_File_write_at(check->cpCj, disp, &b5, write_element_count, MPI_DOUBLE, &status[4]));
CHECK(
MPI_File_write_at(check->cpVj, disp, &b6, write_element_count, MPI_DOUBLE, &status[5]));
CHECK(
MPI_File_write_at(check->cpIj, disp, &b7, write_element_count, MPI_DOUBLE, &status[6]));
}
int savematrix_cross_rows(MPI_File *fh, float *data, int numrows, int rank, int numtasks, int m, int n)
{
MPI_Offset offset = 0;
int i = 0, j = 0;
MPI_Status status;
MPI_Datatype rowtype;
MPI_Datatype filetype;
int result = 0;
MPI_Type_contiguous(n, MPI_FLOAT, &rowtype);
MPI_Type_commit(&rowtype);
MPI_Type_vector(numrows, 1, numtasks, rowtype, &filetype);
MPI_Type_commit(&filetype);
offset = rank;
MPI_File_set_view(*fh, offset*n*sizeof(float), rowtype, filetype, "native", MPI_INFO_NULL);
result = MPI_File_write_at(*fh, 0, data, numrows, rowtype, &status);
if(result != MPI_SUCCESS)
{
printf("Proc %d write at %d error!\n", rank, offset);
}
#if 1
if(rank == 0)
{
for(i = 0; i < numrows; i++)
{
printf("Proc %d write row %d: ", rank, i);
for(j = 0; j < n; j++)
{
printf("%f, ", data[i*n+j]);
}
printf("\n");
}
}
#endif
MPI_Type_free(&rowtype);
MPI_Type_free(&filetype);
return n*numrows;
}
JNIEXPORT void JNICALL Java_mpi_File_writeAt(
JNIEnv *env, jobject jthis, jlong fh, jlong fileOffset,
jobject buf, jboolean db, jint off, jint count,
jlong jType, jint bType, jlongArray stat)
{
MPI_Datatype type = (MPI_Datatype)jType;
void *ptr;
ompi_java_buffer_t *item;
ompi_java_getReadPtr(&ptr, &item, env, buf, db, off, count, type, bType);
MPI_Status status;
int rc = MPI_File_write_at((MPI_File)fh, (MPI_Offset)fileOffset,
ptr, count, type, &status);
ompi_java_exceptionCheck(env, rc);
ompi_java_releaseReadPtr(ptr, item, buf, db);
ompi_java_status_set(env, stat, &status);
}
void dump_JPLC(grid_parms grid, celltype2 **ec, checkpoint_handle *check, const char *message){
/* MPI_Status status;
MPI_Offset disp;
int write_element_count = grid.num_ec_circumferentially * grid.num_ec_axially;
double buffer[write_element_count];
int k = 0;
for (int i = 1; i <= grid.num_ec_circumferentially; i++) {
for (int j = 1; j <= grid.num_ec_axially; j++) {
buffer[k] = ec[i][j].JPLC;
k++;
}
}
int k = 0;
int i=1;
for (int j = 1; j <= grid.num_ec_axially; j++) {
buffer[k] = ec[i][j].JPLC;
k++;
}
int offset = grid.rank/grid.n;
//disp = (grid.rank * write_element_count * sizeof(double));
disp = (offset * write_element_count * sizeof(double));
CHECK(MPI_File_write_at(check->jplc, disp, &buffer, write_element_count, MPI_DOUBLE, &status));*/
MPI_Status status;
MPI_Offset disp;
int write_element_count = grid.num_ec_axially;
double buffer[write_element_count];
int k = 0;
int i=1;
for (int j = 1; j <= grid.num_ec_axially; j++) {
buffer[k] = ec[i][j].JPLC;
k++;
}
int offset = grid.rank/grid.n;
disp = (offset * write_element_count * sizeof(double));
CHECK(MPI_File_write_at(check->jplc, disp, &buffer, write_element_count, MPI_DOUBLE, &status));
}
/*!
Writes the geometry into given open file starting at given offset.
Returns true on success, false otherwise.
The number of bytes written by this function can be obtained
from geometry_data_size().
*/
bool write(MPI_File file, MPI_Offset offset) const
{
const int
temp_id = No_Geometry::geometry_id,
ret_val = MPI_File_write_at(
file,
offset,
(void*) &temp_id,
1,
MPI_INT,
MPI_STATUS_IGNORE
);
if (ret_val != MPI_SUCCESS) {
std::cerr << __FILE__ << ":" << __LINE__
<< " Couldn't write geometry data to given file: " << Error_String()(ret_val)
<< std::endl;
return false;
}
return true;
}
int main(int argc, char *argv[])
{
int i, myrank, buf[BUFSIZE];
int count, nprocs;
char *filename;
MPI_File thefile;
MPI_Offset offset;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (argc < 3) {
fprintf(stderr, "Usage: %s <count> <output file>\n", argv[0]);
exit(1);
}
count = atoi(argv[1]);
filename = argv[2];
for (i = 0; i < BUFSIZE; i++)
buf[i] = myrank * BUFSIZE + i;
MPI_File_open(MPI_COMM_WORLD, filename, (MPI_MODE_WRONLY | MPI_MODE_CREATE),
MPI_INFO_NULL, &thefile);
MPI_File_set_view(thefile, 0, MPI_INT, MPI_INT, "native", MPI_INFO_NULL);
offset = myrank * BUFSIZE;
for (i=0; i<count; i++) {
#ifdef DEBUG
printf("Wrote round %d\n", i);
#endif
MPI_File_write_at(thefile, offset, buf, BUFSIZE, MPI_INT, MPI_STATUS_IGNORE);
offset += nprocs*BUFSIZE;
}
MPI_File_close(&thefile);
MPI_Finalize();
return 0;
}
int main(int argc, char* argv[])
{
if (argc != 3) {
fprintf(stderr, "%s\n", usage);
return 0;
}
/* Initialisation */
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
MPI_Comm_size(MPI_COMM_WORLD, &nb_proc);
MPI_Status status;
int nb_elem = atoi(argv[1]); // nombre d'éléments total
int sort_type = atoi(argv[2]); // type de l'algorithme de tri
k = nb_elem / nb_proc;
int *tab_sort = (int *)malloc(k*sizeof(int));
int *tab_tmp = (int *)malloc(k*sizeof(int));
if ((tab_tmp == NULL) || (tab_sort == NULL)) {
fprintf(stderr, "Erreur allocation mémoire du tableau \n");
MPI_Finalize();
exit(1);
}
int left = my_rank-1;
int right = my_rank+1;
// initialise le tableau local
P0 printf("Initialisation du tableau...");
switch (sort_type) {
case 1:
case 2: init_rand(tab_tmp, k); break;
case 3:
case 4:
case 5: init_rand(tab_sort, k); break;
}
P0 printf(" OK!\n");
P0 printf("Calcul...\n");
// début du chronométrage
double start, end;
start = MPI_Wtime();
// choix de l'algorithme de tri
switch (sort_type) {
case 1: PRAM(tab_tmp, tab_sort); break;
case 2: PRAM_omp(tab_tmp, tab_sort); break;
case 3: quick_sort(tab_sort, k); break;
case 4: quick_sort_omp(tab_sort, k); break;
case 5: qsort(tab_sort, k, sizeof(int), compare); break;
}
// tri pair-impair
int step;
for (step = 0; step < nb_proc; step++) {
if ((my_rank%2) - (step%2) == 0) {
if (my_rank != nb_proc-1) {
MPI_Recv(tab_tmp, k, MPI_INT, right, TAG_TAB, MPI_COMM_WORLD, &status);
merge_sort(tab_sort, tab_tmp);
MPI_Send(tab_tmp, k, MPI_INT, right, TAG_TAB, MPI_COMM_WORLD);
}
}
else {
if (my_rank != 0) {
MPI_Send(tab_sort, k, MPI_INT, left, TAG_TAB, MPI_COMM_WORLD);
MPI_Recv(tab_sort, k, MPI_INT, left, TAG_TAB, MPI_COMM_WORLD, &status);
}
}
}
// fin du chronométrage
end = MPI_Wtime();
print_results(end - start);
// écriture dans le fichier
if (nb_elem <= NMAX) {
P0 printf("Ecriture du fichier...");
MPI_File file;
MPI_Offset my_offset;
char filename[strlen("/Vrac/ppar_cassat_ducamain_sort")+1];
strcpy(filename, "/Vrac/ppar_cassat_ducamain_sort");
my_offset = my_rank * sizeof(int) * k;
MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &file);
MPI_File_write_at(file, my_offset, tab_sort, k, MPI_INT, &status);
// printf("(%d) a écrit: ", my_rank);
// print_tab(tab_sort, k);
// attends que tous aient écrit
MPI_Barrier(MPI_COMM_WORLD);
MPI_File_read_ordered(file, tab_sort, k, MPI_INT, &status);
MPI_File_close(&file);
// printf("(%d) a lu: ", my_rank);
// print_tab(tab_sort, k);
P0 printf(" OK!\n");
}
// Vérification du tri
P0 printf("Vérification du tri...\n");
if (!check_sort(tab_sort, k) && (my_rank == 0))
printf("\tTri correct!\n");
#ifdef BENCH
P0 fprintf(stderr, "\t%d\t%d\t%d", sort_type, nb_elem, nb_proc);
#ifdef _OPENMP
#pragma omp parallel
{
if ((my_rank == 0) && (omp_get_thread_num() == 0))
fprintf(stderr, "\t%d", omp_get_num_threads());
}
#endif
P0 fprintf(stderr, "\n");
#endif
free(tab_sort);
free(tab_tmp);
/* Desactivation */
MPI_Finalize();
return 0;
}
int main(int argc, char *argv[])
{
MPI_File fh;
char emsg[MPI_MAX_ERROR_STRING];
int emsglen, err, ec, errs = 0;
int amode, rank;
char *name = 0;
MPI_Status st;
int outbuf[BUFLEN], inbuf[BUFLEN];
MTest_Init(&argc, &argv);
name = "not-a-file-to-use";
/* Try to open a file that does/should not exist */
/* Note that no error message should be printed by MPI_File_open,
* even when there is an error */
err = MPI_File_open(MPI_COMM_WORLD, name, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
if (err == MPI_SUCCESS) {
errs++;
printf("Did not return error when opening a file that does not exist\n");
MPI_File_close(&fh);
MPI_File_delete(name, MPI_INFO_NULL);
} else {
MPI_Error_class(err, &ec);
MPI_Error_string(err, emsg, &emsglen);
MTestPrintfMsg(2, "Error msg from open: %s\n", emsg);
if (ec != MPI_ERR_NO_SUCH_FILE && ec != MPI_ERR_IO) {
errs++;
printf("Did not return class ERR_NO_SUCH_FILE or ERR_IO\n");
printf("Returned class %d, message %s\n", ec, emsg);
}
}
/* Now, create a file, write data into it; close, then reopen as
* read only and try to write to it */
amode = MPI_MODE_CREATE | MPI_MODE_WRONLY;
name = "mpio-test-openerrs";
err = MPI_File_open(MPI_COMM_WORLD, name, amode, MPI_INFO_NULL, &fh);
if (err) {
errs++;
MTestPrintErrorMsg("Unable to open file for writing", err);
} else {
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
memset(outbuf, 'A' + rank, BUFLEN);
err = MPI_File_write_at(fh, rank * BUFLEN, outbuf, BUFLEN, MPI_BYTE, &st);
if (err) {
errs++;
MTestPrintErrorMsg("Unable to write file", err);
}
MPI_File_close(&fh);
}
/* Now, open for read only, and delete on close */
amode = MPI_MODE_RDONLY | MPI_MODE_DELETE_ON_CLOSE;
err = MPI_File_open(MPI_COMM_WORLD, name, amode, MPI_INFO_NULL, &fh);
if (err) {
errs++;
MTestPrintErrorMsg("Unable to reopen file for reading", err);
} else {
/* Try to read it */
/* Clear buffer before reading into it */
memset(inbuf, 0, BUFLEN);
err = MPI_File_read_at(fh, rank * BUFLEN, inbuf, BUFLEN, MPI_BYTE, &st);
if (err) {
errs++;
MTestPrintErrorMsg("Unable to read file", err);
}
/* Try to write it (should fail) */
err = MPI_File_write_at(fh, rank * BUFLEN, outbuf, BUFLEN, MPI_BYTE, &st);
if (err == MPI_SUCCESS) {
errs++;
printf("Write operation succeeded to read-only file\n");
} else {
/* Look at error class */
MPI_Error_class(err, &ec);
if (ec != MPI_ERR_READ_ONLY && ec != MPI_ERR_ACCESS) {
errs++;
printf("Unexpected error class %d when writing to read-only file\n", ec);
MTestPrintErrorMsg("Error msg is: ", err);
}
}
err = MPI_File_close(&fh);
if (err) {
errs++;
MTestPrintErrorMsg("Failed to close", err);
}
/* No MPI_Barrier is required here */
/*
* Test open without CREATE to see if DELETE_ON_CLOSE worked.
* This should fail if file was deleted correctly.
*/
amode = MPI_MODE_RDONLY;
err = MPI_File_open(MPI_COMM_WORLD, name, amode, MPI_INFO_NULL, &fh);
if (err == MPI_SUCCESS) {
errs++;
printf("File was not deleted!\n");
MPI_File_close(&fh);
} else {
MPI_Error_class(err, &ec);
if (ec != MPI_ERR_NO_SUCH_FILE && ec != MPI_ERR_IO) {
errs++;
printf("Did not return class ERR_NO_SUCH_FILE or ERR_IO\n");
printf("Returned class %d, message %s\n", ec, emsg);
}
}
}
/* */
/* Find out how many errors we saw */
MTest_Finalize(errs);
return MTestReturnValue(errs);
}
void dump_rank_info(checkpoint_handle *check, conductance cpl_cef,
grid_parms grid) {
MPI_Status status;
MPI_Offset disp;
int bytes = 2 * 1024; // 2kB space
char* buffer;
buffer = (char*) checked_malloc(bytes,
"allocation for logfile segment space\n");
sprintf(buffer,
"BRANCH_TAG = %d\n[Universal_Rank, Cart_Rank= (%d,%d)] \tcoords= %d,%d\t nbrs: local (u,d,l,r)=(%d %d %d %d) \t "
"remote: (up1,up2,down1,down2)=(%d %d %d %d)\n\n flip_array: (%d,%d,%d,%d)\n\n"
"Boundary_tag = %c\n(T = Top\t B= Bottom\t I=Interior edges of the bifurcation segmensts, parent or children\t N=Interior of the subdomain)\n"
"COUPLING COEFFICIENTS\n"
"Vm_hm_smc=%2.5lf\nVm_hm_ec=%2.5lf\nCa_hm_smc=%2.5lf\nCa_hm_ec=%2.5lf\nIP3_hm_smc=%2.5lf\n"
"IP3_hm_ec=%2.5lf\nVm_ht_smc=%2.5lf\nVm_ht_ec=%2.5lf\nCa_ht_smc=%2.5lf\nCa_ht_ec=%2.5lf\n"
"IP3_ht_smc=%2.5lf\nIP3_ht_ec=%2.5lf\n\n"
"Spatial Gradient info:\nUniform JPLC\t=%2.5lf\nMinimum JPLC\t=%2.5lf\nMaximum JPLC\t=%2.5lf\nGradient\t=%2.5lf\n"
"Total Tasks=%d\n"
"Number of grid points in axial direction =%d\n"
"Number of grid points in circumferential direction =%d\n"
"Number of ECs per node (axially) =%d\n"
"Number of SMCs per node (circumferentially) =%d\n"
"Total ECs on this node =%d\n"
"Total SMCs on this node =%d\n"
"Total number of cells on this node =%d\n"
"\nTotal ECs in the full computational domain =%d\n"
"Total SMCs in the full computational domain =%d\n"
"Total number of cells in the full computational domain =%d\n"
"Total number of equations in the full computational domain =%d\n "
"z_coordinates: start = %lf end = %lf\n local_z_start = %lf local_z_end = %lf\n"
"------------------------------------------------------------------",
grid.branch_tag, grid.universal_rank, grid.rank, grid.coords[0],
grid.coords[1], grid.nbrs[local][UP], grid.nbrs[local][DOWN],
grid.nbrs[local][LEFT], grid.nbrs[local][RIGHT],
grid.nbrs[remote][UP1], grid.nbrs[remote][UP2],
grid.nbrs[remote][DOWN1], grid.nbrs[remote][DOWN2],
grid.flip_array[0], grid.flip_array[1], grid.flip_array[2],
grid.flip_array[3], grid.my_domain.internal_info.boundary_tag,
cpl_cef.Vm_hm_smc, cpl_cef.Vm_hm_ec, cpl_cef.Ca_hm_smc,
cpl_cef.Ca_hm_ec, cpl_cef.IP3_hm_smc, cpl_cef.IP3_hm_ec,
cpl_cef.Vm_ht_smc, cpl_cef.Vm_ht_ec, cpl_cef.Ca_ht_smc,
cpl_cef.Ca_ht_ec, cpl_cef.IP3_ht_smc, cpl_cef.IP3_ht_ec,
grid.uniform_jplc, grid.min_jplc, grid.max_jplc, grid.gradient,
grid.numtasks,grid.m,grid.n,
grid.num_ec_axially, grid.num_smc_circumferentially,
grid.num_ec_axially * grid.num_ec_circumferentially,
grid.num_smc_axially * grid.num_smc_circumferentially,
(grid.num_ec_axially * grid.num_ec_circumferentially)
+ (grid.num_smc_axially * grid.num_smc_circumferentially),
(grid.num_ec_circumferentially * grid.num_ec_axially * grid.numtasks),
(grid.num_smc_circumferentially * grid.num_smc_axially
* grid.numtasks),
((grid.num_ec_axially * grid.num_ec_circumferentially)
+ (grid.num_smc_axially * grid.num_smc_circumferentially))
* grid.numtasks, grid.NEQ * grid.numtasks,
grid.my_domain.z_offset_start,grid.my_domain.z_offset_end,
grid.my_domain.local_z_start, grid.my_domain.local_z_end);
disp = grid.rank * bytes;
CHECK(MPI_File_write_at(check->logptr, disp, buffer, bytes, MPI_CHAR, &status));
}
int main (int argc, char *argv[]) {
int rank, size;
MPI_File fh_in, fh_out;
MPI_Offset offset;
MPI_Status status;
MPI_Group origin_group, new_group;
MPI_Comm custom_world = MPI_COMM_WORLD;
MPI_Init(&argc, &argv);
MPI_Comm_size(custom_world, &size);
MPI_Comm_rank(custom_world, &rank);
// read command
if (argc < 4) {
if (rank == MASTER_RANK) {
fprintf(stderr, "Insufficient args\n");
fprintf(stderr, "Usage: %s N input_file output_file", argv[0]);
}
return 0;
}
const int N = atoi(argv[1]);
const char *INPUT_NAME = argv[2];
const char *OUTPUT_NAME = argv[3];
// Deal with the case where (N < size)
if (N < size) {
// obtain the group of proc. in the world communicator
MPI_Comm_group(custom_world, &origin_group);
// remove unwanted ranks
int ranges[][3] = {{N, size-1, 1}};
MPI_Group_range_excl(origin_group, 1, ranges, &new_group);
// create a new communicator
MPI_Comm_create(custom_world, new_group, &custom_world);
if (custom_world == MPI_COMM_NULL) {
// terminate those unwanted processes
MPI_Finalize();
exit(0);
}
size = N;
}
// Read file using MPI-IO
int *local_buf;
int num_per_node = N / size;
offset = rank * num_per_node * sizeof(int);
if (rank == (size - 1)) {
num_per_node += N % size;
}
local_buf = malloc(num_per_node * sizeof(int));
MPI_File_open(custom_world, INPUT_NAME, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh_in);
MPI_File_read_at(fh_in, offset, local_buf, num_per_node, MPI_INT, &status);
MPI_File_close(&fh_in);
// Odd-even sort
int sorted = false, all_sorted = false;
int recv;
while (!sorted || !all_sorted) {
sorted = true;
// local sorting
int i;
// odd-phase
for (i = 1; i < num_per_node; i += 2) {
if (local_buf[i] < local_buf[i-1]) {
swap(&local_buf[i], &local_buf[i-1]);
sorted = false;
}
}
// even-phase
for (i = 0; i < num_per_node; i += 2) {
if(i == 0) { continue; }
if (local_buf[i] < local_buf[i-1]) {
swap(&local_buf[i], &local_buf[i-1]);
sorted = false;
}
}
// transportation
// odd phase
if (rank % 2) {
MPI_Send(&local_buf[0], 1, MPI_INT, rank - 1, MSG_RECV, custom_world);
MPI_Recv(&recv, 1, MPI_INT, rank - 1, MSG_RECV, custom_world, &status);
if (recv > local_buf[0]) {
local_buf[0] = recv;
sorted = false;
}
} else if (rank != (size - 1)) {
MPI_Recv(&recv, 1, MPI_INT, rank + 1, MSG_RECV, custom_world, &status);
if(recv < local_buf[num_per_node - 1]) {
swap(&recv, &local_buf[num_per_node - 1]);
sorted = false;
}
MPI_Send(&recv, 1, MPI_INT, rank + 1, MSG_RECV, custom_world);
}
// even phase
if ((rank % 2) == 0 && rank != MASTER_RANK) {
MPI_Send(&local_buf[0], 1, MPI_INT, rank - 1, MSG_RECV, custom_world);
MPI_Recv(&recv, 1, MPI_INT, rank - 1, MSG_RECV, custom_world, &status);
if (recv > local_buf[0]) {
local_buf[0] = recv;
sorted = false;
}
} else if(rank > MASTER_RANK && rank != (size - 1)) {
MPI_Recv(&recv, 1, MPI_INT, rank + 1, MSG_RECV, custom_world, &status);
if(recv < local_buf[num_per_node - 1]) {
swap(&recv, &local_buf[num_per_node - 1]);
sorted = false;
}
MPI_Send(&recv, 1, MPI_INT, rank + 1, MSG_RECV, custom_world);
}
MPI_Allreduce(&sorted, &all_sorted, 1, MPI_INT, MPI_LAND, custom_world);
}
// Write file using MPI-IO
MPI_File_open(custom_world, OUTPUT_NAME, MPI_MODE_CREATE|MPI_MODE_WRONLY, MPI_INFO_NULL, &fh_out);
MPI_File_write_at(fh_out, offset, local_buf, num_per_node, MPI_INT, &status);
MPI_File_close(&fh_out);
free(local_buf);
MPI_Barrier(custom_world);
MPI_Finalize();
return 0;
}
int main( int argc, char *argv[] )
{
unsigned int itr;
int operacao;
int verbose;
int juntar;
char * chave_file;
char * entrada_file;
char * saida_file;
octeto Nb,Nk,Nr;
octeto bloco[4*8];
octeto chave[4*8*15];
int worldsize, rank;
MPI_Status status;
MPI_File chave_handle;
MPI_File entrada_handle;
MPI_File saida_handle;
MPI_Offset entrada_bytes;
unsigned int numero_blocos;
unsigned int blocos_processo;
MPI_Offset bloco_byte_inicio;
MPI_Offset bloco_byte_fim;
MPI_Offset iterador;
Tabela * tabela;
octeto * tabelaEmpacotada;
unsigned int proc;
unsigned int tamanho_tabela;
Tabela * tabela2;
unsigned int no_proc;
unsigned int no_resto;
unsigned int i;
BTreeNode * node;
Indice * indice;
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD,&worldsize);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
operacao = INDEFINIDA;
verbose = 0;
juntar = 0;
chave_file = NULL;
entrada_file = NULL;
saida_file = NULL;
for(itr = 1;itr < (unsigned int)argc;itr++)
{
/* Instrucoes de uso */
if( strcmp(argv[itr],"-a") == 0 || strcmp(argv[itr],"--ajuda") == 0 ||
strcmp(argv[itr],"-h") == 0 || strcmp(argv[itr],"--help") == 0 )
{
if(rank == 0)
{
printf(" Uso: mpiexec -n [PROCESSOS] ./sm-rijndael [ARGUMENTO VALOR].\n");
printf(" Encripta/Decripta um arquivo usando o algoritmo Rijndael(AES) extendido,\n");
printf(" realizando um pre-processamento de blocos repetidos.\n");
printf(" Argumentos opcionais:\n");
printf(" -v,--verbose: Exibe mensagens de conclusao da operacao.\n");
printf(" -j,--juntar: Concatena as tabelas de cada processo em um mestre.\n");
printf(" Argumentos obrigatorios:\n");
printf(" -op,--operacao: Informa se o objetivo da execucao eh encriptar ou decriptar.\n");
printf(" * Os valores possiveis sao: \'encriptar\' e \'decriptar\'.\n");
printf(" -e,-i,--entrada,--input: Caminho e nome do arquivo a ser criptografado.\n");
printf(" -s,-o,--saida,--output: Caminho e nome do arquivo resultante do processo de criptografia da entrada.\n");
printf(" -c,-k,--chave,--key: Caminho e nome do arquivo contendo a chave.\n");
printf(" O arquivo contendo a chave eh em formato binario de acordo com a seguinte especificacao:\n");
printf(" - O primeiro byte deve conter o tamanho do bloco (em palavras de 4 bytes).\n");
printf(" * O bloco pode possuir tamanho: 4, 5, 6, 7 ou 8.\n");
printf(" - O segundo byte deve conter o tamanho da chave (em palavras de 4 bytes).\n");
printf(" * Esta aplicacao aceita chaves com tamanho: 4, 5, 6, 7 ou 8.\n");
printf(" - Os proximos 4*[tamanho da chave] bytes do arquivo sao os bytes componentes da chave, que\n");
printf(" devem estar (obrigatoriamente) escritos no formato hexadecimal da linguagem C (0xff).\n");
printf(" * Eh recomendavel o uso de um editor hexadecimal na construcao do arquivo chave.\n");
}
goto finalizando;
}
/* Juntar: Concatena as tabelas de cada processo em um mestre */
else
if( strcmp(argv[itr],"-j") == 0 || strcmp(argv[itr],"--juntar") == 0)
{
juntar = 1;
}
/* Verbose: exibir mensagens de finalizacao */
else
if( strcmp(argv[itr],"-v") == 0 || strcmp(argv[itr],"--verbose") == 0)
{
verbose = 1;
}
/* Operacao a ser realizada */
else
if( strcmp(argv[itr],"-op") == 0 || strcmp(argv[itr],"--operacao") == 0 )
{
if( itr+1 < argc )
{
if( strcmp(argv[itr+1],"encriptar") == 0 )
{
operacao = ENCRIPTAR;
}
else
if( strcmp(argv[itr+1],"decriptar") == 0 )
{
operacao = DECRIPTAR;
}
itr++;
}
else
{
goto sempar;
}
}
/* Arquivo com a chave */
else
if( strcmp(argv[itr],"-c") == 0 || strcmp(argv[itr],"--chave") == 0 ||
strcmp(argv[itr],"-k") == 0 || strcmp(argv[itr],"--key") == 0 )
{
if(itr+1 < argc)
{
chave_file = argv[itr+1];
itr++;
}
else
{
goto sempar;
}
}
/* Arquivo de entrada */
else
if( strcmp(argv[itr],"-e") == 0 || strcmp(argv[itr],"--entrada") == 0 ||
strcmp(argv[itr],"-i") == 0 || strcmp(argv[itr],"--input") == 0 )
{
if(itr+1 < argc)
{
entrada_file = argv[itr+1];
itr++;
}
else
{
goto sempar;
}
}
/* Arquivo de saida */
else
if( strcmp(argv[itr],"-s") == 0 || strcmp(argv[itr],"--saida") == 0 ||
strcmp(argv[itr],"-o") == 0 || strcmp(argv[itr],"--output") == 0 )
{
if(itr+1 < argc)
{
saida_file = argv[itr+1];
itr++;
}
else
{
goto sempar;
}
}
/* Erro desconhecido */
else
{
if(rank == 0)
{
printf("Erro nos argumentos passados.\n");
}
goto help;
}
}
/* Fim da leitura dos argumentos */
if( operacao == INDEFINIDA || chave_file == NULL || entrada_file == NULL || saida_file == NULL )
{
if(rank == 0)
{
if( operacao == INDEFINIDA )
printf("A operacao a ser realizada eh invalida ou nao foi especificada.\n");
if( chave_file == NULL )
printf("Esta faltando especificar o arquivo com a chave.\n");
if( entrada_file == NULL )
printf("Esta faltando especificar o arquivo de entrada.\n");
if( saida_file == NULL )
printf("Esta faltando especificar o arquivo de saida.\n");
}
goto help;
}
/* Fim do tratamento dos argumentos */
if( MPI_File_open( MPI_COMM_WORLD, chave_file, MPI_MODE_RDONLY, MPI_INFO_NULL, &chave_handle ) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro na abertura do arquivo com a chave (%s).\n",chave_file);
}
goto help;
}
if( MPI_File_read(chave_handle,&Nb,1, MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro na leitura do tamanho de um bloco no arquivo com a chave (%s).\n",chave_file);
}
goto help;
}
if( Nb< 4 || Nb > 8 )
{
if( rank == 0 )
{
printf("Tamanho de bloco invalido no arquivo com a chave (%s).\n",chave_file);
}
goto help;
}
if( MPI_File_read(chave_handle,&Nk,1, MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro na leitura do tamanho da chave no arquivo com a chave (%s).\n",chave_file);
}
goto help;
}
if( Nk< 4 || Nk > 8 )
{
if( rank == 0 )
{
printf("Tamanho de chave invalido no arquivo com a chave (%s).\n",chave_file);
}
goto help;
}
if( MPI_File_read(chave_handle,chave,4*Nk,MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro na leitura da chave no arquivo com a chave (%s).\n",chave_file);
}
goto help;
}
MPI_File_close( &chave_handle );
Nr = numero_rodadas(Nb,Nk);
KeyExpansion(chave,Nb,Nk);
if( MPI_File_open( MPI_COMM_WORLD, entrada_file,
MPI_MODE_RDONLY,
MPI_INFO_NULL, &entrada_handle ) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro na abertura do arquivo de entrada (%s).\n",entrada_file);
}
goto help;
}
MPI_File_get_size(entrada_handle,&entrada_bytes);
if( MPI_File_open( MPI_COMM_WORLD, saida_file,
MPI_MODE_RDWR | MPI_MODE_CREATE | MPI_MODE_EXCL,
MPI_INFO_NULL, &saida_handle ) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro na criacao do arquivo de saida (%s).\n",saida_file);
printf("Uma possivel causa eh que o arquivo ja exista.\n");
}
goto help;
}
numero_blocos = ( entrada_bytes / (Nb*4) );
blocos_processo = numero_blocos / worldsize;
if( operacao == ENCRIPTAR || operacao == DECRIPTAR )
{
bloco_byte_inicio = 4*Nb*blocos_processo*rank;
bloco_byte_fim = 4*Nb*blocos_processo*(rank+1);
tabela = novaTabela(Nb*4);
for( iterador = bloco_byte_inicio ; iterador < bloco_byte_fim ; iterador += (4*Nb) )
{
if( MPI_File_read_at(entrada_handle,iterador,bloco,(4*Nb),MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro ao ler do arquivo de entrada (%s).\n",entrada_file);
}
goto help;
}
novaOcorrenciaTabela(tabela,bloco,iterador);
}
iterador = 4*Nb*blocos_processo*worldsize + 4*Nb*rank;
if( iterador < numero_blocos*4*Nb )
{
if( MPI_File_read_at(entrada_handle,iterador,bloco,(4*Nb),MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro ao ler do arquivo de entrada (%s).\n",entrada_file);
}
goto help;
}
novaOcorrenciaTabela(tabela,bloco,iterador);
}
else if( operacao == ENCRIPTAR && iterador == numero_blocos*4*Nb )
{
if( MPI_File_read_at(entrada_handle,iterador,bloco,(4*Nb),MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro ao ler do arquivo de entrada (%s).\n",entrada_file);
}
goto help;
}
bloco[ 4*Nb - 1 ] = (octeto)(entrada_bytes - numero_blocos*4*Nb);
novaOcorrenciaTabela(tabela,bloco,iterador);
}
if( juntar == 1 )
{
tabelaEmpacotada = (octeto*)malloc( entrada_bytes );
if( rank == 0 ) /* Mestre que vai concatenar todas as arvores*/
{
for(proc=1;proc<worldsize;proc++)
{
MPI_Recv( tabelaEmpacotada, entrada_bytes, MPI_BYTE, MPI_ANY_SOURCE, TAG_TABELA_EMPACOTADA, MPI_COMM_WORLD, &status );
desempacotarInserindo(tabelaEmpacotada,tabela);
}
tamanho_tabela = numeroBlocosTabela(tabela);
no_proc = (tamanho_tabela / worldsize);
no_resto = (tamanho_tabela % worldsize);
tabela2 = novaTabela(Nb*4);
for(proc=1;proc<worldsize;proc++)
{
for(i=0;i<no_proc;i++)
{
soInsiraTabela(tabela2, popLastTabelaNode(tabela) );
}
if( no_resto > 1 )
{
soInsiraTabela(tabela2, popLastTabelaNode(tabela) );
no_resto--;
}
empacotarTabela(tabela2,tabelaEmpacotada);
MPI_Send(tabelaEmpacotada,numeroBytesTabela(tabela2), MPI_BYTE, proc, TAG_TABELA_EMPACOTADA_2, MPI_COMM_WORLD );
destruirArvore(tabela2->root);
tabela2->root = NULL;
}
destruirTabela(tabela2);
}
else
{
empacotarTabela(tabela,tabelaEmpacotada);
MPI_Send(tabelaEmpacotada,numeroBytesTabela(tabela), MPI_BYTE, 0, TAG_TABELA_EMPACOTADA, MPI_COMM_WORLD );
destruirArvore(tabela->root);
tabela->root = NULL;
MPI_Recv( tabelaEmpacotada, entrada_bytes, MPI_BYTE, 0, TAG_TABELA_EMPACOTADA_2, MPI_COMM_WORLD, &status );
desempacotarInserindo(tabelaEmpacotada,tabela);
}
free(tabelaEmpacotada);
}
if( operacao == ENCRIPTAR )
MPI_File_set_size(saida_handle,(MPI_Offset)( (numero_blocos+1)*(Nb*4) ) );
else if( operacao == DECRIPTAR )
MPI_File_set_size(saida_handle,entrada_bytes);
tamanho_tabela = numeroBlocosTabela(tabela);
for( i=0 ; i<tamanho_tabela ; i++ )
{
node = popLastTabelaNode(tabela);
// memcpy (bloco,node->bloco,4*Nb);
if( operacao == ENCRIPTAR )
AES_encriptar_bloco(node->bloco,Nb,chave,Nr);
else if( operacao == DECRIPTAR )
AES_decriptar_bloco(node->bloco,Nb,chave,Nr);
indice = node->ocorrencias;
while( indice != NULL )
{
if( MPI_File_write_at(saida_handle,indice->indice,node->bloco,(4*Nb),MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro ao escrever no arquivo de saida (%s).\n",saida_file);
}
goto help;
}
indice = indice->next;
}
destruirArvore(node);
}
destruirTabela(tabela);
if( operacao == DECRIPTAR )
{
MPI_Barrier( MPI_COMM_WORLD ); /*Barreira q impede q alguem leia antes do valor decriptografado ser escrito */
if( MPI_File_read_at(saida_handle,entrada_bytes-1,bloco,1,MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro ao realizar leitura no arquivo de saida (%s).\n",saida_file);
}
goto help;
}
MPI_Barrier( MPI_COMM_WORLD ); /* Barreira q impede q alqum processo trunque o arquivo antes de outro processo ler*/
MPI_File_set_size(saida_handle,entrada_bytes - 4*Nb + bloco[0]);
}
if( rank == 0 && verbose==1)
{
if( operacao == ENCRIPTAR )
printf("A encriptacao do arquivo foi realizada com sucesso.\n");
else if( operacao == DECRIPTAR )
printf("A decriptacao do arquivo foi realizada com sucesso.\n");
}
}
goto finalizando;
sempar:
if( rank == 0 )
{
printf("Sem par correspondente para a opcao %s.\n",argv[itr]);
}
help:
if( rank == 0 )
{
printf("Use a opcao --help para melhor entendimento do uso da aplicacao.\n");
}
finalizando:
MPI_Finalize( );
return 0;
}
void do_ffApply(SEXP ans,
double *data,
SEXP margin,
SEXP function,
int my_start,
int my_end,
int nrows,
int ncols,
int worldRank,
char *out_filename) {
MPI_Status stat;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_File fh;
SEXP data_chunk, R_fcall, parsedCmd = R_NilValue;
double *rchunk;
int i,k, offset=0, count=0;
/* Open the file handler */
MPI_File_open(comm, out_filename, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
/* The MPI_FILE_SET_VIEW routine changes the process's view of the data in the file */
MPI_File_set_view(fh, 0, MPI_DOUBLE, MPI_DOUBLE, "native", MPI_INFO_NULL);
/* Parse the command, returns a function object */
PROTECT(parsedCmd = parseExpression(function));
/* Create R LANGSXP Vector, R function holder
length of the vector is 1 + number of arguments */
PROTECT(R_fcall = lang2(VECTOR_ELT(parsedCmd, 0), R_NilValue));
if (INTEGER(margin)[0] == 1) {
PROTECT(data_chunk = allocVector(REALSXP, ncols));
rchunk = REAL(data_chunk);
for(i=my_start, k=0; i<my_end; i++, k++) {
for(int j=0; j<ncols; j++) {
rchunk[j] = data[j*nrows+i];
}
SETCADR(R_fcall, data_chunk);
SET_VECTOR_ELT(ans, 0, eval(R_fcall, R_GlobalEnv));
count = length(VECTOR_ELT(ans, 0));
offset = i*count;
MPI_File_write_at(fh, offset, REAL(VECTOR_ELT(ans, 0)), count, MPI_DOUBLE, &stat);
}
}
if (INTEGER(margin)[0] == 2) {
PROTECT(data_chunk = allocVector(REALSXP, nrows));
rchunk = REAL(data_chunk);
for(i=my_start, k=0; i<my_end; i++, k++) {
for(int j=0; j<nrows; j++) {
rchunk[j] = data[j+nrows*i];
}
SETCADR(R_fcall, data_chunk);
SET_VECTOR_ELT(ans, 0, eval(R_fcall, R_GlobalEnv));
count = length(VECTOR_ELT(ans, 0));
offset = i*count;
MPI_File_write_at(fh, offset, REAL(VECTOR_ELT(ans, 0)), count, MPI_DOUBLE, &stat);
}
}
/* Close file handler */
MPI_File_close(&fh);
return;
}
int main(int argc, char *argv[])
{
int iarrayOfSizes[2], iarrayOfSubsizes[2], iarrayOfStarts[2], ilocal_size;
int nproc[2], periods[2], icoord[2];
int m, n, i, j, wsize, wrank, crank, ndims, lrows, lcols, grow, gcol, err;
MPI_Datatype filetype;
MPI_File fh;
MPI_Comm cartcomm;
MPI_Info info0, info3;
double t, topen, twrite, tclose, wrate;
double *local_array;
char nstripesStr[12], stripeUnitStr[12];
int nstripes = -1;
int stripeUnit = -1;
MPI_Offset headerSize = 0;
MPI_Init(0,0);
MPI_Comm_rank(MPI_COMM_WORLD, &wrank);
/* Get global array size */
m = n = 128; /* Set default size */
/* ioda [ n ] [ m ] [ nstripes ] [ stripeunit ] [ headersize ] */
if (argc > 0) {
if (argc > 1) m = atoi(argv[1]);
if (argc > 2) n = atoi(argv[2]);
if (argc > 3) nstripes = atoi(argv[3]);
if (argc > 4) stripeUnit = atoi(argv[4]);
if (argc > 5) headerSize = atoi(argv[5]);
if (argc > 6) {
if (wrank == 0)
fprintf(stderr,"Unrecognized argument %s\n", argv[6]);
MPI_Abort(MPI_COMM_WORLD,1);
}
}
if (wrank == 0) printf("Matrix is [%d,%d]; file dir = %s\n", m, n, MYSCRATCHDIR );
/* The default number of stripes = totalsize/1M */
if (nstripes < 0) {
nstripes = n * m * sizeof(double) / (1024*1024);
if (nstripes < 1) nstripes = 1;
}
if (wrank == 0) printf("nstripes = %d, stripeUnit = %d, header size = %d\n",
nstripes, stripeUnit, (int)headerSize);
/* Use topology routines to get decomposition and coordinates */
MPI_Comm_size(MPI_COMM_WORLD, &wsize);
nproc[0] = 0; nproc[1] = 0;
ndims = 2;
MPI_Dims_create(wsize, ndims, nproc);
periods[0] = 0; periods[1] = 0;
MPI_Cart_create(MPI_COMM_WORLD, ndims, nproc, periods, 1, &cartcomm);
MPI_Comm_rank(cartcomm, &crank);
MPI_Cart_coords(cartcomm, crank, ndims, icoord);
iarrayOfSizes[0] = m;
iarrayOfSizes[1] = n;
iarrayOfSubsizes[0] = m/nproc[0];
iarrayOfSubsizes[1] = n/nproc[1];
iarrayOfStarts[0] = icoord[0] * iarrayOfSubsizes[0];
iarrayOfStarts[1] = icoord[1] * iarrayOfSubsizes[1];
/* Initialize my block of the data */
ilocal_size = iarrayOfSubsizes[0] * iarrayOfSubsizes[1];
lrows = iarrayOfSubsizes[0];
lcols = iarrayOfSubsizes[1];
local_array = (double *)malloc(lrows*lcols*sizeof(double));
gcol = iarrayOfStarts[1];
grow = iarrayOfStarts[0];
for (i=0; i<lrows; i++) {
for (j=0; j<lcols; j++) {
local_array[j*lrows+i] = (grow+i) + (gcol+j)*m;
}
}
/* Fortran order simply means the data is stored by columns */
MPI_Type_create_subarray(ndims, iarrayOfSizes, iarrayOfSubsizes,
iarrayOfStarts, MPI_ORDER_FORTRAN, MPI_DOUBLE,
&filetype);
MPI_Type_commit(&filetype);
info0 = MPI_INFO_NULL;
info3 = MPI_INFO_NULL;
if (nstripes > 0 || stripeUnit > 0) {
MPI_Info_create(&info0);
if (nstripes > 0) {
snprintf(nstripesStr, sizeof(nstripesStr), "%d", nstripes);
MPI_Info_set(info0, "striping_factor", nstripesStr);
MPI_Info_set(info0, "cb_nodes", nstripesStr);
}
if (stripeUnit > 0) {
snprintf(stripeUnitStr, sizeof(stripeUnitStr), "%d", stripeUnit);
MPI_Info_set(info0, "striping_unit", stripeUnitStr);
}
MPI_Info_dup(info0, &info3);
MPI_Info_set(info3, "romio_no_indep_rw", "true");
/* Other hints to consider:
direct_io=true
The default cb_buffer_size is 16777216 , but is overridden by the
striping unit, which is smaller by default.
*/
}
/* level - 3 */
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
err = MPI_File_open(cartcomm, MYSCRATCHDIR "testfile-3.out",
MPI_MODE_CREATE | MPI_MODE_RDWR, info3, &fh);
topen = MPI_Wtime() - t;
if (err != MPI_SUCCESS) myAbort(err, "open testfile-3.out");
if (headerSize > 0) {
/* Simulate writing a header */
if (wrank == 0) {
char *header;
header = (char *)calloc(1,(size_t)headerSize);
MPI_File_write(fh, header, headerSize, MPI_BYTE, MPI_STATUS_IGNORE);
free(header);
}
MPI_Barrier(cartcomm);
}
MPI_File_set_view(fh, headerSize, MPI_DOUBLE, filetype, "native", MPI_INFO_NULL);
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
err = MPI_File_write_all(fh, local_array, ilocal_size, MPI_DOUBLE,
MPI_STATUS_IGNORE);
twrite = MPI_Wtime() - t;
if (err != MPI_SUCCESS) myAbort(err, "collective write");
err = MPI_File_close(&fh);
tclose = MPI_Wtime() - t;
/* tclose is the time for the write(s) + the close, in case the
implementation delays (some of) the writes until the close */
if (err != MPI_SUCCESS) myAbort(err, "close testfile-3.out");
MPI_Allreduce(MPI_IN_PLACE, &topen, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &twrite, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &tclose, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
if (twrite > 0)
wrate = (double)m * (double)n * sizeof(double)/twrite;
if (wrank == 0)
printf("%d\t[%d,%d]\t%d\t%.2e\t%.2e\t%.2e\t%.2e\n", wsize, m, n, nstripes, topen,
twrite, tclose, wrate);
/* level - 0 */
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
err = MPI_File_open(cartcomm, MYSCRATCHDIR "testfile-0.out",
MPI_MODE_CREATE | MPI_MODE_RDWR, info0, &fh);
topen = MPI_Wtime() - t;
if (err != MPI_SUCCESS) myAbort(err, "open testfile-0.out");
if (headerSize > 0) {
/* Simulate writing a header */
if (wrank == 0) {
char *header;
header = (char *)calloc(1,(size_t)headerSize);
MPI_File_write(fh, header, headerSize, MPI_BYTE, MPI_STATUS_IGNORE);
free(header);
}
MPI_Barrier(cartcomm);
}
MPI_Barrier(MPI_COMM_WORLD);
t = MPI_Wtime();
gcol = iarrayOfStarts[1];
grow = iarrayOfStarts[0];
for (j=0; j<lcols; j++) {
MPI_Offset offset = headerSize +
((MPI_Offset)(grow) + (MPI_Offset)(gcol+j)*m) * sizeof(double);
err = MPI_File_write_at(fh, offset, local_array+j*lrows, lrows, MPI_DOUBLE,
MPI_STATUS_IGNORE);
if (err != MPI_SUCCESS) myAbort(err, "write at");
}
twrite = MPI_Wtime() - t;
err = MPI_File_close(&fh);
tclose = MPI_Wtime() - t;
/* tclose is the time for the write(s) + the close, in case the
implementation delays (some of) the writes until the close */
if (err != MPI_SUCCESS) myAbort(err, "close testfile-0");
MPI_Allreduce(MPI_IN_PLACE, &topen, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &twrite, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &tclose, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
if (twrite > 0)
wrate = (double)m * (double)n * sizeof(double)/twrite;
if (wrank == 0)
printf("%d\t[%d,%d]\t%d\t%.2e\t%.2e\t%.2e\t%.2e\n", wsize, m, n, nstripes, topen,
twrite, tclose, wrate);
if (info0 != MPI_INFO_NULL) {
MPI_Info_free(&info0);
MPI_Info_free(&info3);
}
free(local_array);
MPI_Finalize();
return 0;
}
int main(int argc, char **argv)
{
char *buf, *tmp, *buf2, *tmp2, *check;
int i, j, mynod=0, nprocs=1, err, my_correct = 1, correct, myerrno;
double stim, etim;
double write_tim = 0;
double read_tim = 0;
double read_bw, write_bw;
double max_read_tim, max_write_tim;
double min_read_tim, min_write_tim;
double ave_read_tim, ave_write_tim;
int64_t iter_jump = 0;
int64_t seek_position = 0;
MPI_File fh;
MPI_Status status;
int nchars;
/* startup MPI and determine the rank of this process */
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
/* parse the command line arguments */
parse_args(argc, argv);
if (mynod == 0) printf("# Using mpi-io calls.\n");
/* kindof a weird hack- if the location of the pvfstab file was
* specified on the command line, then spit out this location into
* the appropriate environment variable: */
#if H5_HAVE_SETENV
/* no setenv or unsetenv */
if (opt_pvfstab_set) {
if((setenv("PVFSTAB_FILE", opt_pvfstab, 1)) < 0){
perror("setenv");
goto die_jar_jar_die;
}
}
#endif
/* this is how much of the file data is covered on each iteration of
* the test. used to help determine the seek offset on each
* iteration */
iter_jump = nprocs * opt_block;
/* setup a buffer of data to write */
if (!(tmp = (char *) malloc(opt_block + 256))) {
perror("malloc");
goto die_jar_jar_die;
}
buf = tmp + 128 - (((long)tmp) % 128); /* align buffer */
if (opt_correct) {
/* do the same buffer setup for verifiable data */
if (!(tmp2 = (char *) malloc(opt_block + 256))) {
perror("malloc2");
goto die_jar_jar_die;
}
buf2 = tmp + 128 - (((long)tmp) % 128);
}
/* open the file for writing */
err = MPI_File_open(MPI_COMM_WORLD, opt_file,
MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
if (err < 0) {
fprintf(stderr, "node %d, open error: %s\n", mynod, strerror(errno));
goto die_jar_jar_die;
}
/* now repeat the write operations the number of times
* specified on the command line */
for (j=0; j < opt_iter; j++) {
/* calculate the appropriate position depending on the iteration
* and rank of the current process */
seek_position = (j*iter_jump)+(mynod*opt_block);
if (opt_correct) /* fill in buffer for iteration */ {
for (i=mynod+j, check=buf; i<opt_block; i++,check++) *check=(char)i;
}
/* discover the starting time of the operation */
MPI_Barrier(MPI_COMM_WORLD);
stim = MPI_Wtime();
/* write out the data */
nchars = opt_block/sizeof(char);
err = MPI_File_write_at(fh, seek_position, buf, nchars, MPI_CHAR, &status);
if(err){
fprintf(stderr, "node %d, write error: %s\n", mynod,
strerror(errno));
}
/* discover the ending time of the operation */
etim = MPI_Wtime();
write_tim += (etim - stim);
/* we are done with this "write" iteration */
}
err = MPI_File_close(&fh);
if(err){
fprintf(stderr, "node %d, close error after write\n", mynod);
}
/* wait for everyone to synchronize at this point */
MPI_Barrier(MPI_COMM_WORLD);
/* reopen the file to read the data back out */
err = MPI_File_open(MPI_COMM_WORLD, opt_file,
MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
if (err < 0) {
fprintf(stderr, "node %d, open error: %s\n", mynod, strerror(errno));
goto die_jar_jar_die;
}
/* we are going to repeat the read operation the number of iterations
* specified */
for (j=0; j < opt_iter; j++) {
/* calculate the appropriate spot give the current iteration and
* rank within the MPI processes */
seek_position = (j*iter_jump)+(mynod*opt_block);
/* discover the start time */
MPI_Barrier(MPI_COMM_WORLD);
stim = MPI_Wtime();
/* read in the file data */
if (!opt_correct){
err = MPI_File_read_at(fh, seek_position, buf, nchars, MPI_CHAR, &status);
}
else{
err = MPI_File_read_at(fh, seek_position, buf2, nchars, MPI_CHAR, &status);
}
myerrno = errno;
/* discover the end time */
etim = MPI_Wtime();
read_tim += (etim - stim);
if (err < 0) fprintf(stderr, "node %d, read error, loc = %Ld: %s\n",
mynod, mynod*opt_block, strerror(myerrno));
/* if the user wanted to check correctness, compare the write
* buffer to the read buffer */
if (opt_correct && memcmp(buf, buf2, opt_block)) {
fprintf(stderr, "node %d, correctness test failed\n", mynod);
my_correct = 0;
MPI_Allreduce(&my_correct, &correct, 1, MPI_INT, MPI_MIN,
MPI_COMM_WORLD);
}
/* we are done with this read iteration */
}
/* close the file */
err = MPI_File_close(&fh);
if(err){
fprintf(stderr, "node %d, close error after write\n", mynod);
}
/* compute the read and write times */
MPI_Allreduce(&read_tim, &max_read_tim, 1, MPI_DOUBLE, MPI_MAX,
MPI_COMM_WORLD);
MPI_Allreduce(&read_tim, &min_read_tim, 1, MPI_DOUBLE, MPI_MIN,
MPI_COMM_WORLD);
MPI_Allreduce(&read_tim, &ave_read_tim, 1, MPI_DOUBLE, MPI_SUM,
MPI_COMM_WORLD);
/* calculate the average from the sum */
ave_read_tim = ave_read_tim / nprocs;
MPI_Allreduce(&write_tim, &max_write_tim, 1, MPI_DOUBLE, MPI_MAX,
MPI_COMM_WORLD);
MPI_Allreduce(&write_tim, &min_write_tim, 1, MPI_DOUBLE, MPI_MIN,
MPI_COMM_WORLD);
MPI_Allreduce(&write_tim, &ave_write_tim, 1, MPI_DOUBLE, MPI_SUM,
MPI_COMM_WORLD);
/* calculate the average from the sum */
ave_write_tim = ave_write_tim / nprocs;
/* print out the results on one node */
if (mynod == 0) {
read_bw = ((int64_t)(opt_block*nprocs*opt_iter))/(max_read_tim*1000000.0);
write_bw = ((int64_t)(opt_block*nprocs*opt_iter))/(max_write_tim*1000000.0);
printf("nr_procs = %d, nr_iter = %d, blk_sz = %ld\n", nprocs,
opt_iter, (long)opt_block);
printf("# total_size = %ld\n", (long)(opt_block*nprocs*opt_iter));
printf("# Write: min_time = %f, max_time = %f, mean_time = %f\n",
min_write_tim, max_write_tim, ave_write_tim);
printf("# Read: min_time = %f, max_time = %f, mean_time = %f\n",
min_read_tim, max_read_tim, ave_read_tim);
printf("Write bandwidth = %f Mbytes/sec\n", write_bw);
printf("Read bandwidth = %f Mbytes/sec\n", read_bw);
if (opt_correct) {
printf("Correctness test %s.\n", correct ? "passed" : "failed");
}
}
die_jar_jar_die:
#if H5_HAVE_SETENV
/* no setenv or unsetenv */
/* clear the environment variable if it was set earlier */
if (opt_pvfstab_set){
unsetenv("PVFSTAB_FILE");
}
#endif
free(tmp);
if (opt_correct) free(tmp2);
MPI_Finalize();
return(0);
}
static int write_read_samples(PIDX_file_io_id io_id, int variable_index, uint64_t hz_start_index, uint64_t hz_count, unsigned char* hz_buffer, int64_t buffer_offset, PIDX_block_layout layout, int MODE)
{
int samples_per_file, block_number, file_index, file_count, ret = 0, block_negative_offset = 0, file_number;
int bytes_per_sample, bytes_per_datatype;
int i = 0;
char file_name[PATH_MAX];
off_t data_offset = 0;
samples_per_file = io_id->idx_d->samples_per_block * io_id->idx->blocks_per_file;
bytes_per_datatype = (io_id->idx->variable[variable_index]->bits_per_value / 8) * (io_id->idx->chunk_size[0] * io_id->idx->chunk_size[1] * io_id->idx->chunk_size[2] * io_id->idx->chunk_size[3] * io_id->idx->chunk_size[4]) / (io_id->idx->compression_factor);
#if !SIMULATE_IO
hz_buffer = hz_buffer + buffer_offset * bytes_per_datatype * io_id->idx->variable[variable_index]->values_per_sample;
#endif
while (hz_count)
{
block_number = hz_start_index / io_id->idx_d->samples_per_block;
file_number = hz_start_index / samples_per_file;
file_index = hz_start_index % samples_per_file;
file_count = samples_per_file - file_index;
if ((int64_t)file_count > hz_count)
file_count = hz_count;
// build file name
int adjusted_file_index = 0;
int l = pow(2, ((int)log2((unsigned int) file_number * io_id->idx->blocks_per_file)));
adjusted_file_index = (l * (io_id->idx_d->idx_count[0] * io_id->idx_d->idx_count[1] * io_id->idx_d->idx_count[2]) + (((unsigned int) file_number * io_id->idx->blocks_per_file) - l) + (io_id->idx_d->color * l)) / io_id->idx->blocks_per_file;
ret = generate_file_name(io_id->idx->blocks_per_file, io_id->idx->filename_template, /*file_number*/adjusted_file_index, file_name, PATH_MAX);
if (ret == 1)
{
fprintf(stderr, "[%s] [%d] generate_file_name() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
data_offset = 0;
bytes_per_sample = io_id->idx->variable[variable_index]->bits_per_value / 8;
data_offset = file_index * bytes_per_sample * io_id->idx->variable[variable_index]->values_per_sample;
data_offset += io_id->idx_d->start_fs_block * io_id->idx_d->fs_block_size;
block_negative_offset = PIDX_blocks_find_negative_offset(io_id->idx->blocks_per_file, block_number, layout);
data_offset -= block_negative_offset * io_id->idx_d->samples_per_block * bytes_per_sample * io_id->idx->variable[variable_index]->values_per_sample;
for (l = 0; l < variable_index; l++)
{
bytes_per_sample = io_id->idx->variable[l]->bits_per_value / 8;
for (i = 0; i < io_id->idx->blocks_per_file; i++)
if (PIDX_blocks_is_block_present((i + (io_id->idx->blocks_per_file * file_number)), layout))
data_offset = data_offset + (io_id->idx->variable[l]->values_per_sample * bytes_per_sample * io_id->idx_d->samples_per_block);
}
if(MODE == PIDX_WRITE)
{
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
#ifdef PIDX_DUMP_IO
if (io_id->idx_d->dump_io_info == 1 && io_id->idx->current_time_step == 0)
{
fprintf(io_dump_fp, "[A] Count %lld Target Disp %d (%d %d)\n", (long long)file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), (file_index * bytes_per_sample * io_id->idx->variable[variable_index]->values_per_sample - block_negative_offset * io_id->idx_d->samples_per_block * bytes_per_sample * io_id->idx->variable[variable_index]->values_per_sample)/8, (int)io_id->idx_d->start_fs_block, (int)io_id->idx_d->fs_block_size);
fflush(io_dump_fp);
}
#endif
if (io_id->idx_d->parallel_mode == 1)
{
int rank = 0;
MPI_Comm_rank(io_id->comm, &rank);
MPI_File fh;
MPI_Status status;
int mpi_ret;
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed. (%s) [%d]\n", __FILE__, __LINE__, file_name, file_number);
return PIDX_err_io;
}
/*
printf("[%d] Data Offset %d Count %d\n", rank, data_offset, (file_count));
int x = 0;
for (x = 0; x < file_count; x++)
{
double x1;
memcpy(&x1, hz_buffer + x * sizeof(double), sizeof(double));
printf("Values %d %f\n", x, x1);
}
*/
mpi_ret = MPI_File_write_at(fh, data_offset, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
int write_count;
MPI_Get_count(&status, MPI_BYTE, &write_count);
if (write_count != file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8))
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
MPI_File_close(&fh);
}
else
{
int fh;
fh = open(file_name, O_WRONLY);
ssize_t write_count = pwrite(fh, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), data_offset);
if (write_count != file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8))
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
close(fh);
}
#else
int fh;
fh = open(file_name, O_WRONLY);
/*
double x1, x2, x3, x4;
memcpy(&x1, hz_buffer, sizeof(double));
memcpy(&x2, hz_buffer + sizeof(double), sizeof(double));
memcpy(&x3, hz_buffer + 2*sizeof(double), sizeof(double));
memcpy(&x4, hz_buffer + 3*sizeof(double), sizeof(double));
printf("[%d] [%d %d] Values %f %f %f %f\n", variable_index, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), data_offset, x1, x2, x3, x4);
*/
ssize_t write_count = pwrite(fh, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), data_offset);
if (write_count != file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8))
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
close(fh);
#endif
#endif
}
if(MODE == PIDX_READ)
{
#if PIDX_HAVE_MPI
if (io_id->idx_d->parallel_mode == 1)
{
MPI_File fh;
MPI_Status status;
int mpi_ret;
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
mpi_ret = MPI_File_read_at(fh, data_offset, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
MPI_File_close(&fh);
}
else
{
int fh;
fh = open(file_name, O_RDONLY);
ssize_t read_count = pread(fh, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), data_offset);
if (read_count != file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8))
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
close(fh);
}
#else
int fh;
fh = open(file_name, O_RDONLY);
ssize_t read_count = pread(fh, hz_buffer, file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8), data_offset);
if (read_count != file_count * io_id->idx->variable[variable_index]->values_per_sample * (io_id->idx->variable[variable_index]->bits_per_value/8))
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
close(fh);
#endif
}
hz_count -= file_count;
hz_start_index += file_count;
hz_buffer += file_count * io_id->idx->variable[variable_index]->values_per_sample * bytes_per_datatype;
#if PIDX_HAVE_MPI
#else
#endif
}
return PIDX_success;
}
/*
* sweep_access_size
*
* read and write from file using various access sizes that
* fall into the 10 buckets.
*
* mpi-io
* mpi contiguous data type
* shared file
*/
int sweep_access_size(char *path, int size, int rank)
{
MPI_File fh;
MPI_Offset offset;
MPI_Datatype dt;
MPI_Status status;
int rc;
int blksize;
int pblksize;
int count;
void* buf;
char filepath[512];
sprintf(filepath, "%s/%s", path, "access_sweep");
rc = MPI_File_open(MPI_COMM_WORLD, filepath,
(MPI_MODE_CREATE|MPI_MODE_RDWR|MPI_MODE_DELETE_ON_CLOSE),
MPI_INFO_NULL,
&fh);
MPI_CHECK(rc,"MPI_File_open");
for (pblksize=0,count=6,blksize=(((int)pow(2,count))+1);
count <= 30;
count++,pblksize=blksize,blksize=(((int)pow(2,count))+1))
{
int test;
rc = MPI_Type_contiguous(blksize, MPI_BYTE, &dt);
MPI_CHECK(rc,"MPI_Type_contiguous");
rc = MPI_Type_commit(&dt);
MPI_CHECK(rc,"MPI_Type_commit");
buf = malloc(blksize);
if (!buf) { return 0; }
offset = ((MPI_Offset)pblksize*(MPI_Offset)size)+
((MPI_Offset)rank*(MPI_Offset)blksize);
rc = MPI_File_write_at(fh, offset, buf, 1, dt, &status);
MPI_CHECK(rc,"MPI_File_write_at");
MPI_Barrier(MPI_COMM_WORLD);
offset = ((MPI_Offset)pblksize*(MPI_Offset)size)+
((MPI_Offset)((rank+1)%size)*(MPI_Offset)blksize);
rc = MPI_File_read_at(fh, offset, buf, 1, dt, &status);
MPI_CHECK(rc,"MPI_File_read_at");
rc = MPI_Type_free(&dt);
MPI_CHECK(rc,"MPI_Type_free");
free(buf);
}
rc = MPI_File_close(&fh);
MPI_CHECK(rc,"MPI_File_close");
return 1;
}
int main(int argc, char** argv) {
MPI_Init(&argc, &argv);
setup_globals();
/* Parse arguments. */
int SCALE = 16;
int edgefactor = 16; /* nedges / nvertices, i.e., 2*avg. degree */
// if (argc >= 2) SCALE = atoi(argv[1]);
// if (argc >= 3) edgefactor = atoi(argv[2]);
char* name = argv[1];
if (argc >= 3) SCALE = atoi(argv[2]);
if (argc >= 4) edgefactor = atoi(argv[3]);
// if (argc <= 1 || argc >= 4 || SCALE == 0 || edgefactor == 0) {
// if (rank == 0) {
// fprintf(stderr, "Usage: %s SCALE edgefactor\n SCALE = log_2(# vertices) [integer, required]\n edgefactor = (# edges) / (# vertices) = .5 * (average vertex degree) [integer, defaults to 16]\n(Random number seed and Kronecker initiator are in main.c)\n", argv[0]);
// }
if (argc <= 2 || argc >= 5 || SCALE == 0 || edgefactor == 0) {
if (rank == 0) {
fprintf(stderr, "Usage: %s filename SCALE edgefactor\n SCALE = log_2(# vertices) [integer, required]\n edgefactor = (# edges) / (# vertices) = .5 * (average vertex degree) [integer, defaults to 16]\n(Random number seed and Kronecker initiator are in main.c)\n", argv[0]);
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
uint64_t seed1 = 2, seed2 = 3;
// const char* filename = getenv("TMPFILE");
const char* filename = name;
/* If filename is NULL, store data in memory */
tuple_graph tg;
tg.nglobaledges = (int64_t)(edgefactor) << SCALE;
int64_t nglobalverts = (int64_t)(1) << SCALE;
tg.data_in_file = (filename != NULL);
if (tg.data_in_file) {
printf("data in file \n");
MPI_File_set_errhandler(MPI_FILE_NULL, MPI_ERRORS_ARE_FATAL);
// MPI_File_open(MPI_COMM_WORLD, (char*)filename, MPI_MODE_RDWR | MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_DELETE_ON_CLOSE | MPI_MODE_UNIQUE_OPEN, MPI_INFO_NULL, &tg.edgefile);
MPI_File_open(MPI_COMM_WORLD, (char*)filename, MPI_MODE_RDWR | MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_UNIQUE_OPEN, MPI_INFO_NULL, &tg.edgefile);
MPI_File_set_size(tg.edgefile, tg.nglobaledges * sizeof(packed_edge));
MPI_File_set_view(tg.edgefile, 0, packed_edge_mpi_type, packed_edge_mpi_type, "native", MPI_INFO_NULL);
MPI_File_set_atomicity(tg.edgefile, 0);
}
/* Make the raw graph edges. */
/* Get roots for BFS runs, plus maximum vertex with non-zero degree (used by
* validator). */
int num_bfs_roots = 64;
int64_t* bfs_roots = (int64_t*)xmalloc(num_bfs_roots * sizeof(int64_t));
int64_t max_used_vertex = 0;
double make_graph_start = MPI_Wtime();
{
/* Spread the two 64-bit numbers into five nonzero values in the correct
* range. */
uint_fast32_t seed[5];
make_mrg_seed(seed1, seed2, seed);
/* As the graph is being generated, also keep a bitmap of vertices with
* incident edges. We keep a grid of processes, each row of which has a
* separate copy of the bitmap (distributed among the processes in the
* row), and then do an allreduce at the end. This scheme is used to avoid
* non-local communication and reading the file separately just to find BFS
* roots. */
MPI_Offset nchunks_in_file = (tg.nglobaledges + FILE_CHUNKSIZE - 1) / FILE_CHUNKSIZE;
int64_t bitmap_size_in_bytes = int64_min(BITMAPSIZE, (nglobalverts + CHAR_BIT - 1) / CHAR_BIT);
if (bitmap_size_in_bytes * size * CHAR_BIT < nglobalverts) {
bitmap_size_in_bytes = (nglobalverts + size * CHAR_BIT - 1) / (size * CHAR_BIT);
}
int ranks_per_row = ((nglobalverts + CHAR_BIT - 1) / CHAR_BIT + bitmap_size_in_bytes - 1) / bitmap_size_in_bytes;
int nrows = size / ranks_per_row;
int my_row = -1, my_col = -1;
unsigned char* restrict has_edge = NULL;
MPI_Comm cart_comm;
{
int dims[2] = {size / ranks_per_row, ranks_per_row};
int periods[2] = {0, 0};
MPI_Cart_create(MPI_COMM_WORLD, 2, dims, periods, 1, &cart_comm);
}
int in_generating_rectangle = 0;
if (cart_comm != MPI_COMM_NULL) {
in_generating_rectangle = 1;
{
int dims[2], periods[2], coords[2];
MPI_Cart_get(cart_comm, 2, dims, periods, coords);
my_row = coords[0];
my_col = coords[1];
}
MPI_Comm this_col;
MPI_Comm_split(cart_comm, my_col, my_row, &this_col);
MPI_Comm_free(&cart_comm);
has_edge = (unsigned char*)xMPI_Alloc_mem(bitmap_size_in_bytes);
memset(has_edge, 0, bitmap_size_in_bytes);
/* Every rank in a given row creates the same vertices (for updating the
* bitmap); only one writes them to the file (or final memory buffer). */
packed_edge* buf = (packed_edge*)xmalloc(FILE_CHUNKSIZE * sizeof(packed_edge));
MPI_Offset block_limit = (nchunks_in_file + nrows - 1) / nrows;
// fprintf(stderr, "%d: nchunks_in_file = %" PRId64 ", block_limit = %" PRId64 " in grid of %d rows, %d cols\n", rank, (int64_t)nchunks_in_file, (int64_t)block_limit, nrows, ranks_per_row);
if (tg.data_in_file) {
tg.edgememory_size = 0;
tg.edgememory = NULL;
} else {
int my_pos = my_row + my_col * nrows;
int last_pos = (tg.nglobaledges % ((int64_t)FILE_CHUNKSIZE * nrows * ranks_per_row) != 0) ?
(tg.nglobaledges / FILE_CHUNKSIZE) % (nrows * ranks_per_row) :
-1;
int64_t edges_left = tg.nglobaledges % FILE_CHUNKSIZE;
int64_t nedges = FILE_CHUNKSIZE * (tg.nglobaledges / ((int64_t)FILE_CHUNKSIZE * nrows * ranks_per_row)) +
FILE_CHUNKSIZE * (my_pos < (tg.nglobaledges / FILE_CHUNKSIZE) % (nrows * ranks_per_row)) +
(my_pos == last_pos ? edges_left : 0);
/* fprintf(stderr, "%d: nedges = %" PRId64 " of %" PRId64 "\n", rank, (int64_t)nedges, (int64_t)tg.nglobaledges); */
tg.edgememory_size = nedges;
tg.edgememory = (packed_edge*)xmalloc(nedges * sizeof(packed_edge));
}
MPI_Offset block_idx;
for (block_idx = 0; block_idx < block_limit; ++block_idx) {
/* fprintf(stderr, "%d: On block %d of %d\n", rank, (int)block_idx, (int)block_limit); */
MPI_Offset start_edge_index = int64_min(FILE_CHUNKSIZE * (block_idx * nrows + my_row), tg.nglobaledges);
MPI_Offset edge_count = int64_min(tg.nglobaledges - start_edge_index, FILE_CHUNKSIZE);
packed_edge* actual_buf = (!tg.data_in_file && block_idx % ranks_per_row == my_col) ?
tg.edgememory + FILE_CHUNKSIZE * (block_idx / ranks_per_row) :
buf;
/* fprintf(stderr, "%d: My range is [%" PRId64 ", %" PRId64 ") %swriting into index %" PRId64 "\n", rank, (int64_t)start_edge_index, (int64_t)(start_edge_index + edge_count), (my_col == (block_idx % ranks_per_row)) ? "" : "not ", (int64_t)(FILE_CHUNKSIZE * (block_idx / ranks_per_row))); */
if (!tg.data_in_file && block_idx % ranks_per_row == my_col) {
assert (FILE_CHUNKSIZE * (block_idx / ranks_per_row) + edge_count <= tg.edgememory_size);
}
// debug
char* wtxbuf = (char*)xmalloc(FILE_CHUNKSIZE * sizeof(packed_edge));
// generate_kronecker_range(seed, SCALE, start_edge_index, start_edge_index + edge_count, actual_buf);
generate_kronecker_range(seed, SCALE, start_edge_index, start_edge_index + edge_count, actual_buf);
if (tg.data_in_file && my_col == (block_idx % ranks_per_row)) { /* Try to spread writes among ranks */
// MPI_File_write_at(tg.edgefile, start_edge_index, actual_buf, edge_count, packed_edge_mpi_type, MPI_STATUS_IGNORE);
// debug
printf("%d: %d, %d\n", rank, start_edge_index, edge_count);
int i;
// for (i = start_edge_index; i < start_edge_index + 3; i++) {
// if(block_idx == 0) {
// for (i = 0; i < 3; i++) {
// if (edge_count > 3)
// printf("%d: %d\t%d\n", rank, actual_buf[i].v0, actual_buf[i].v1);
// }
// }
MPI_File_write_at(tg.edgefile, start_edge_index, actual_buf, edge_count, packed_edge_mpi_type, MPI_STATUS_IGNORE);
}
ptrdiff_t i;
#ifdef _OPENMP
#pragma omp parallel for
#endif
for (i = 0; i < edge_count; ++i) {
int64_t src = get_v0_from_edge(&actual_buf[i]);
int64_t tgt = get_v1_from_edge(&actual_buf[i]);
if (src == tgt) continue;
if (src / bitmap_size_in_bytes / CHAR_BIT == my_col) {
#ifdef _OPENMP
#pragma omp atomic
#endif
has_edge[(src / CHAR_BIT) % bitmap_size_in_bytes] |= (1 << (src % CHAR_BIT));
}
if (tgt / bitmap_size_in_bytes / CHAR_BIT == my_col) {
#ifdef _OPENMP
#pragma omp atomic
#endif
has_edge[(tgt / CHAR_BIT) % bitmap_size_in_bytes] |= (1 << (tgt % CHAR_BIT));
}
}
}
free(buf);
#if 0
/* The allreduce for each root acts like we did this: */
MPI_Allreduce(MPI_IN_PLACE, has_edge, bitmap_size_in_bytes, MPI_UNSIGNED_CHAR, MPI_BOR, this_col);
#endif
MPI_Comm_free(&this_col);
} else {
tg.edgememory = NULL;
tg.edgememory_size = 0;
}
MPI_Allreduce(&tg.edgememory_size, &tg.max_edgememory_size, 1, MPI_INT64_T, MPI_MAX, MPI_COMM_WORLD);
#ifndef GEN_ONLY
/* Find roots and max used vertex */
{
uint64_t counter = 0;
int bfs_root_idx;
for (bfs_root_idx = 0; bfs_root_idx < num_bfs_roots; ++bfs_root_idx) {
int64_t root;
while (1) {
double d[2];
make_random_numbers(2, seed1, seed2, counter, d);
root = (int64_t)((d[0] + d[1]) * nglobalverts) % nglobalverts;
counter += 2;
if (counter > 2 * nglobalverts) break;
int is_duplicate = 0;
int i;
for (i = 0; i < bfs_root_idx; ++i) {
if (root == bfs_roots[i]) {
is_duplicate = 1;
break;
}
}
if (is_duplicate) continue; /* Everyone takes the same path here */
int root_ok = 0;
if (in_generating_rectangle && (root / CHAR_BIT / bitmap_size_in_bytes) == my_col) {
root_ok = (has_edge[(root / CHAR_BIT) % bitmap_size_in_bytes] & (1 << (root % CHAR_BIT))) != 0;
}
MPI_Allreduce(MPI_IN_PLACE, &root_ok, 1, MPI_INT, MPI_LOR, MPI_COMM_WORLD);
if (root_ok) break;
}
bfs_roots[bfs_root_idx] = root;
}
num_bfs_roots = bfs_root_idx;
/* Find maximum non-zero-degree vertex. */
{
int64_t i;
max_used_vertex = 0;
if (in_generating_rectangle) {
for (i = bitmap_size_in_bytes * CHAR_BIT; i > 0; --i) {
if (i > nglobalverts) continue;
if (has_edge[(i - 1) / CHAR_BIT] & (1 << ((i - 1) % CHAR_BIT))) {
max_used_vertex = (i - 1) + my_col * CHAR_BIT * bitmap_size_in_bytes;
break;
}
}
}
MPI_Allreduce(MPI_IN_PLACE, &max_used_vertex, 1, MPI_INT64_T, MPI_MAX, MPI_COMM_WORLD);
}
}
#endif
if (in_generating_rectangle) {
MPI_Free_mem(has_edge);
}
if (tg.data_in_file) {
MPI_File_sync(tg.edgefile);
}
}
double make_graph_stop = MPI_Wtime();
double make_graph_time = make_graph_stop - make_graph_start;
if (rank == 0) { /* Not an official part of the results */
fprintf(stderr, "graph_generation: %f s\n", make_graph_time);
}
//debug
#ifndef GEN_ONLY //!GEN_ONLY
/* Make user's graph data structure. */
double data_struct_start = MPI_Wtime();
make_graph_data_structure(&tg);
double data_struct_stop = MPI_Wtime();
double data_struct_time = data_struct_stop - data_struct_start;
if (rank == 0) { /* Not an official part of the results */
fprintf(stderr, "construction_time: %f s\n", data_struct_time);
}
/* Number of edges visited in each BFS; a double so get_statistics can be
* used directly. */
double* edge_counts = (double*)xmalloc(num_bfs_roots * sizeof(double));
/* Run BFS. */
int validation_passed = 1;
double* bfs_times = (double*)xmalloc(num_bfs_roots * sizeof(double));
double* validate_times = (double*)xmalloc(num_bfs_roots * sizeof(double));
uint64_t nlocalverts = get_nlocalverts_for_pred();
int64_t* pred = (int64_t*)xMPI_Alloc_mem(nlocalverts * sizeof(int64_t));
int bfs_root_idx;
for (bfs_root_idx = 0; bfs_root_idx < num_bfs_roots; ++bfs_root_idx) {
int64_t root = bfs_roots[bfs_root_idx];
if (rank == 0) fprintf(stderr, "Running BFS %d\n", bfs_root_idx);
/* Clear the pred array. */
memset(pred, 0, nlocalverts * sizeof(int64_t));
/* Do the actual BFS. */
double bfs_start = MPI_Wtime();
run_bfs(root, &pred[0]);
double bfs_stop = MPI_Wtime();
bfs_times[bfs_root_idx] = bfs_stop - bfs_start;
if (rank == 0) fprintf(stderr, "Time for BFS %d is %f\n", bfs_root_idx, bfs_times[bfs_root_idx]);
/* Validate result. */
if (rank == 0) fprintf(stderr, "Validating BFS %d\n", bfs_root_idx);
double validate_start = MPI_Wtime();
int64_t edge_visit_count;
int validation_passed_one = validate_bfs_result(&tg, max_used_vertex + 1, nlocalverts, root, pred, &edge_visit_count);
double validate_stop = MPI_Wtime();
validate_times[bfs_root_idx] = validate_stop - validate_start;
if (rank == 0) fprintf(stderr, "Validate time for BFS %d is %f\n", bfs_root_idx, validate_times[bfs_root_idx]);
edge_counts[bfs_root_idx] = (double)edge_visit_count;
if (rank == 0) fprintf(stderr, "TEPS for BFS %d is %g\n", bfs_root_idx, edge_visit_count / bfs_times[bfs_root_idx]);
if (!validation_passed_one) {
validation_passed = 0;
if (rank == 0) fprintf(stderr, "Validation failed for this BFS root; skipping rest.\n");
break;
}
}
MPI_Free_mem(pred);
free(bfs_roots);
free_graph_data_structure();
#endif //!GEN_ONLY
if (tg.data_in_file) {
MPI_File_close(&tg.edgefile);
} else {
free(tg.edgememory); tg.edgememory = NULL;
}
#ifndef GEN_ONLY
/* Print results. */
if (rank == 0) {
if (!validation_passed) {
fprintf(stdout, "No results printed for invalid run.\n");
} else {
int i;
fprintf(stdout, "SCALE: %d\n", SCALE);
fprintf(stdout, "edgefactor: %d\n", edgefactor);
fprintf(stdout, "NBFS: %d\n", num_bfs_roots);
fprintf(stdout, "graph_generation: %g\n", make_graph_time);
fprintf(stdout, "num_mpi_processes: %d\n", size);
fprintf(stdout, "construction_time: %g\n", data_struct_time);
double stats[s_LAST];
get_statistics(bfs_times, num_bfs_roots, stats);
fprintf(stdout, "min_time: %g\n", stats[s_minimum]);
fprintf(stdout, "firstquartile_time: %g\n", stats[s_firstquartile]);
fprintf(stdout, "median_time: %g\n", stats[s_median]);
fprintf(stdout, "thirdquartile_time: %g\n", stats[s_thirdquartile]);
fprintf(stdout, "max_time: %g\n", stats[s_maximum]);
fprintf(stdout, "mean_time: %g\n", stats[s_mean]);
fprintf(stdout, "stddev_time: %g\n", stats[s_std]);
get_statistics(edge_counts, num_bfs_roots, stats);
fprintf(stdout, "min_nedge: %.11g\n", stats[s_minimum]);
fprintf(stdout, "firstquartile_nedge: %.11g\n", stats[s_firstquartile]);
fprintf(stdout, "median_nedge: %.11g\n", stats[s_median]);
fprintf(stdout, "thirdquartile_nedge: %.11g\n", stats[s_thirdquartile]);
fprintf(stdout, "max_nedge: %.11g\n", stats[s_maximum]);
fprintf(stdout, "mean_nedge: %.11g\n", stats[s_mean]);
fprintf(stdout, "stddev_nedge: %.11g\n", stats[s_std]);
double* secs_per_edge = (double*)xmalloc(num_bfs_roots * sizeof(double));
for (i = 0; i < num_bfs_roots; ++i) secs_per_edge[i] = bfs_times[i] / edge_counts[i];
get_statistics(secs_per_edge, num_bfs_roots, stats);
fprintf(stdout, "min_TEPS: %g\n", 1. / stats[s_maximum]);
fprintf(stdout, "firstquartile_TEPS: %g\n", 1. / stats[s_thirdquartile]);
fprintf(stdout, "median_TEPS: %g\n", 1. / stats[s_median]);
fprintf(stdout, "thirdquartile_TEPS: %g\n", 1. / stats[s_firstquartile]);
fprintf(stdout, "max_TEPS: %g\n", 1. / stats[s_minimum]);
fprintf(stdout, "harmonic_mean_TEPS: %g\n", 1. / stats[s_mean]);
/* Formula from:
* Title: The Standard Errors of the Geometric and Harmonic Means and
* Their Application to Index Numbers
* Author(s): Nilan Norris
* Source: The Annals of Mathematical Statistics, Vol. 11, No. 4 (Dec., 1940), pp. 445-448
* Publisher(s): Institute of Mathematical Statistics
* Stable URL: http://www.jstor.org/stable/2235723
* (same source as in specification). */
fprintf(stdout, "harmonic_stddev_TEPS: %g\n", stats[s_std] / (stats[s_mean] * stats[s_mean] * sqrt(num_bfs_roots - 1)));
free(secs_per_edge); secs_per_edge = NULL;
free(edge_counts); edge_counts = NULL;
get_statistics(validate_times, num_bfs_roots, stats);
fprintf(stdout, "min_validate: %g\n", stats[s_minimum]);
fprintf(stdout, "firstquartile_validate: %g\n", stats[s_firstquartile]);
fprintf(stdout, "median_validate: %g\n", stats[s_median]);
fprintf(stdout, "thirdquartile_validate: %g\n", stats[s_thirdquartile]);
fprintf(stdout, "max_validate: %g\n", stats[s_maximum]);
fprintf(stdout, "mean_validate: %g\n", stats[s_mean]);
fprintf(stdout, "stddev_validate: %g\n", stats[s_std]);
#if 0
for (i = 0; i < num_bfs_roots; ++i) {
fprintf(stdout, "Run %3d: %g s, validation %g s\n", i + 1, bfs_times[i], validate_times[i]);
}
#endif
}
}
free(bfs_times);
free(validate_times);
#endif
cleanup_globals();
MPI_Finalize();
return 0;
}
int PIDX_aggregated_io(PIDX_file_io_id io_id, Agg_buffer agg_buf, PIDX_block_layout block_layout, int MODE)
{
int64_t data_offset = 0;
char file_name[PATH_MAX];
int i = 0, k = 0;
uint32_t *headers;
int total_header_size = 0;
#ifdef PIDX_RECORD_TIME
double t1, t2, t3, t4, t5;
#endif
#if PIDX_HAVE_MPI
int mpi_ret;
MPI_File fh;
MPI_Status status;
#else
int fh;
#endif
int total_chunk_size = (io_id->idx->chunk_size[0] * io_id->idx->chunk_size[1] * io_id->idx->chunk_size[2] * io_id->idx->chunk_size[3] * io_id->idx->chunk_size[4]);
if (enable_caching == 1 && agg_buf->var_number == io_id->init_index && agg_buf->sample_number == 0)
{
#ifdef PIDX_RECORD_TIME
t1 = PIDX_get_time();
#endif
int adjusted_file_index = 0;
int l = pow(2, ((int)log2((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file)));
adjusted_file_index = (l * (io_id->idx_d->idx_count[0] * io_id->idx_d->idx_count[1] * io_id->idx_d->idx_count[2]) + (((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file) - l) + (io_id->idx_d->color * l)) / io_id->idx->blocks_per_file;
generate_file_name(io_id->idx->blocks_per_file, io_id->idx->filename_template, (unsigned int) /*agg_buf->file_number*/adjusted_file_index, file_name, PATH_MAX);
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed filename %s.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
#else
fh = open(file_name, O_WRONLY);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t2 = PIDX_get_time();
#endif
data_offset = 0;
total_header_size = (10 + (10 * io_id->idx->blocks_per_file)) * sizeof (uint32_t) * io_id->idx->variable_count;
headers = (uint32_t*)malloc(total_header_size);
memset(headers, 0, total_header_size);
#if !SIMULATE_IO
if (enable_caching == 1)
memcpy (headers, cached_header_copy, total_header_size);
else
{
//TODO
}
#endif
#ifdef PIDX_RECORD_TIME
t3 = PIDX_get_time();
#endif
uint64_t header_size = (io_id->idx_d->start_fs_block * io_id->idx_d->fs_block_size);
#if !SIMULATE_IO
unsigned char* temp_buffer = (unsigned char*)realloc(agg_buf->buffer, agg_buf->buffer_size + header_size);
if (temp_buffer == NULL)
{
fprintf(stderr, "[%s] [%d] realloc() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
else
{
agg_buf->buffer = temp_buffer;
memmove(agg_buf->buffer + header_size, agg_buf->buffer, agg_buf->buffer_size);
memcpy(agg_buf->buffer, headers, total_header_size);
memset(agg_buf->buffer + total_header_size, 0, (header_size - total_header_size));
}
#endif
free(headers);
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_write_at(fh, 0, agg_buf->buffer, agg_buf->buffer_size + header_size, MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed for filename %s.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
int write_count;
MPI_Get_count(&status, MPI_BYTE, &write_count);
if (write_count != agg_buf->buffer_size + header_size)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#else
ssize_t write_count = pwrite(fh, agg_buf->buffer, agg_buf->buffer_size + header_size, 0);
if (write_count != agg_buf->buffer_size + header_size)
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#endif
#endif
#ifdef PIDX_RECORD_TIME
t4 = PIDX_get_time();
#endif
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_close(&fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#else
close(fh);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t5 = PIDX_get_time();
#endif
#ifdef PIDX_RECORD_TIME
printf("V0. [R %d] [O 0 C %lld] [FVS %d %d %d] Time: O %f H %f W %f C %f\n", rank, (long long)agg_buf->buffer_size + header_size, agg_buf->file_number, agg_buf->var_number, agg_buf->sample_number, (t2-t1), (t3-t2), (t4-t3), (t5-t4));
#else
#endif
}
else if (agg_buf->var_number != -1 && agg_buf->sample_number != -1 && agg_buf->file_number != -1)
{
#ifdef PIDX_RECORD_TIME
t1 = PIDX_get_time();
#endif
int adjusted_file_index = 0;
int l = pow(2, ((int)log2((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file)));
adjusted_file_index = (l * (io_id->idx_d->idx_count[0] * io_id->idx_d->idx_count[1] * io_id->idx_d->idx_count[2]) + (((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file) - l) + (io_id->idx_d->color * l)) / io_id->idx->blocks_per_file;
generate_file_name(io_id->idx->blocks_per_file, io_id->idx->filename_template, (unsigned int) adjusted_file_index/*agg_buf->file_number*/, file_name, PATH_MAX);
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
if (MODE == PIDX_WRITE)
{
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() filename %s failed.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
}
else
{
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() filename %s failed.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
}
#else
if (MODE == PIDX_WRITE)
fh = open(file_name, O_WRONLY);
else
fh = open(file_name, O_RDONLY);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t2 = PIDX_get_time();
#endif
data_offset = 0;
data_offset += io_id->idx_d->start_fs_block * io_id->idx_d->fs_block_size;
if (MODE == PIDX_WRITE)
{
for (k = 0; k < agg_buf->var_number; k++)
{
PIDX_variable vark = io_id->idx->variable[k];
int bytes_per_datatype = ((vark->bits_per_value/8) * total_chunk_size) / (io_id->idx->compression_factor);
int64_t prev_var_sample = (int64_t) block_layout->block_count_per_file[agg_buf->file_number] * io_id->idx_d->samples_per_block * bytes_per_datatype * io_id->idx->variable[k]->values_per_sample;
data_offset = (int64_t) data_offset + prev_var_sample;
}
for (i = 0; i < agg_buf->sample_number; i++)
data_offset = (int64_t) data_offset + agg_buf->buffer_size;
}
else
{
int total_header_size = (10 + (10 * io_id->idx->blocks_per_file)) * sizeof (uint32_t) * io_id->idx->variable_count;
headers = malloc(total_header_size);
memset(headers, 0, total_header_size);
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_read_at(fh, 0, headers, total_header_size , MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "Data offset = %lld [%s] [%d] MPI_File_write_at() failed for filename %s.\n", (long long) data_offset, __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
#endif
}
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
if (MODE == PIDX_WRITE)
{
//int rank;
//MPI_Comm_rank(io_id->comm, &rank);
//printf("W [%d] [%d %d %d] size = %d and offset = %d\n", rank, agg_buf->file_number, agg_buf->var_number, agg_buf->sample_number, agg_buf->buffer_size, data_offset);
mpi_ret = MPI_File_write_at(fh, data_offset, agg_buf->buffer, agg_buf->buffer_size , MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "Data offset = %lld [%s] [%d] MPI_File_write_at() failed for filename %s.\n", (long long) data_offset, __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
int write_count = 0;
MPI_Get_count(&status, MPI_BYTE, &write_count);
if (write_count != agg_buf->buffer_size)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
}
else
{
int data_size = 0;
int block_count = 0;
for (i = 0; i < io_id->idx->blocks_per_file; i++)
{
if (PIDX_blocks_is_block_present(agg_buf->file_number * io_id->idx->blocks_per_file + i, block_layout))
{
data_offset = htonl(headers[12 + ((i + (io_id->idx->blocks_per_file * agg_buf->var_number))*10 )]);
data_size = htonl(headers[14 + ((i + (io_id->idx->blocks_per_file * agg_buf->var_number))*10 )]);
mpi_ret = MPI_File_read_at(fh, data_offset, agg_buf->buffer + (block_count * io_id->idx_d->samples_per_block * (io_id->idx->variable[agg_buf->var_number]->bits_per_value/8) * io_id->idx->variable[agg_buf->var_number]->values_per_sample * io_id->idx->chunk_size[0] * io_id->idx->chunk_size[1] * io_id->idx->chunk_size[2]) / io_id->idx->compression_factor, /*agg_buf->buffer_size*/data_size , MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "Data offset = %lld [%s] [%d] MPI_File_write_at() failed for filename %s.\n", (long long) data_offset, __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
int read_count = 0;
MPI_Get_count(&status, MPI_BYTE, &read_count);
if (read_count != /*agg_buf->buffer_size*/data_size)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed. %d != %lldd\n", __FILE__, __LINE__, read_count, (long long)agg_buf->buffer_size);
return PIDX_err_io;
}
block_count++;
}
}
free(headers);
}
#else
if (MODE == PIDX_WRITE)
{
ssize_t write_count = pwrite(fh, agg_buf->buffer, agg_buf->buffer_size, data_offset);
if (write_count != agg_buf->buffer_size)
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
}
else
{
ssize_t read_count = pread(fh, agg_buf->buffer, agg_buf->buffer_size, data_offset);
if (read_count != agg_buf->buffer_size)
{
fprintf(stderr, "[%s] [%d] pread() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
}
#endif
#endif
#ifdef PIDX_RECORD_TIME
t3 = PIDX_get_time();
#endif
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_close(&fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#else
close(fh);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t4 = PIDX_get_time();
#endif
#ifdef PIDX_RECORD_TIME
printf("V. [R %d] [O %lld C %lld] [FVS %d %d %d] Time: O %f H %f W %f C %f\n", rank, (long long) data_offset, (long long)agg_buf->buffer_size, agg_buf->file_number, agg_buf->var_number, agg_buf->sample_number, (t2-t1), (t2-t2), (t3-t2), (t4-t3));
#endif
}
return PIDX_success;
}
static void writePLY(
MPI_Comm comm, std::string fname,
int nvertices, int nverticesPerObject,
int ntriangles, int ntrianglesPerObject,
int nObjects,
const std::vector<int3>& mesh,
const std::vector<float3>& vertices)
{
int rank;
MPI_Check( MPI_Comm_rank(comm, &rank) );
int totalVerts = 0;
MPI_Check( MPI_Reduce(&nvertices, &totalVerts, 1, MPI_INT, MPI_SUM, 0, comm) );
int totalTriangles = 0;
MPI_Check( MPI_Reduce(&ntriangles, &totalTriangles, 1, MPI_INT, MPI_SUM, 0, comm) );
MPI_File f;
MPI_Check( MPI_File_open(comm, fname.c_str(), MPI_MODE_CREATE|MPI_MODE_DELETE_ON_CLOSE|MPI_MODE_WRONLY, MPI_INFO_NULL, &f) );
MPI_Check( MPI_File_close(&f) );
MPI_Check( MPI_File_open(comm, fname.c_str(), MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &f) );
int headerSize = 0;
MPI_Offset fileOffset = 0;
if (rank == 0)
{
std::stringstream ss;
ss << "ply\n";
ss << "format binary_little_endian 1.0\n";
ss << "element vertex " << totalVerts << "\n";
ss << "property float x\nproperty float y\nproperty float z\n";
//ss << "property float xnormal\nproperty float ynormal\nproperty float znormal\n";
ss << "element face " << totalTriangles << "\n";
ss << "property list int int vertex_index\n";
ss << "end_header\n";
std::string content = ss.str();
headerSize = content.length();
MPI_Check( MPI_File_write_at(f, fileOffset, content.c_str(), headerSize, MPI_CHAR, MPI_STATUS_IGNORE) );
}
MPI_Check( MPI_Bcast(&headerSize, 1, MPI_INT, 0, comm) );
fileOffset += headerSize;
fileOffset += writeToMPI(vertices, f, fileOffset, comm);
int verticesOffset = 0;
MPI_Check( MPI_Exscan(&nvertices, &verticesOffset, 1, MPI_INT, MPI_SUM, comm));
std::vector<int4> connectivity;
for(int j = 0; j < nObjects; ++j)
for(int i = 0; i < ntrianglesPerObject; ++i)
{
int3 vertIds = mesh[i] + nverticesPerObject * j + verticesOffset;
connectivity.push_back({3, vertIds.x, vertIds.y, vertIds.z});
}
fileOffset += writeToMPI(connectivity, f, fileOffset, comm);
MPI_Check( MPI_File_close(&f));
}
int main(int argc, char * argv[])
{
int i;
int ret;
MPI_File file;
MPI_Status status;
char * filename = "tokufs:/hellobasicmpiio";
MPI_Init(&argc, &argv);
ret = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_CREATE | MPI_MODE_RDWR,
MPI_INFO_NULL, &file);
printf("open ret = %d\n", ret);
if (ret != MPI_SUCCESS) {
char * estr = malloc(1000);
int l;
MPI_Error_string(ret, estr, &l);
printf("%s\n", estr);
free(estr);
}
assert(ret == MPI_SUCCESS);
/* Generate the usual 0-99 repeating buffer */
char write_buf[BUF_SIZE];
for (i = 0; i < BUF_SIZE; i++) {
write_buf[i] = i % 100;
}
/* Write to the file as 10 seperate chunks with explicit offsets. */
int num_chunks = 10;
assert(BUF_SIZE % num_chunks == 0);
int write_size = BUF_SIZE / num_chunks;
for (i = 0; i < 10; i++) {
ret = MPI_File_write_at(file, i * write_size,
write_buf + i * write_size,
write_size, MPI_CHAR, &status);
assert(ret == MPI_SUCCESS);
}
/* Close the file and open it again */
ret = MPI_File_close(&file);
assert(ret == MPI_SUCCESS);
ret = MPI_File_open(MPI_COMM_WORLD, filename,
MPI_MODE_EXCL | MPI_MODE_RDWR,
MPI_INFO_NULL, &file);
assert(ret == MPI_SUCCESS);
/* Read back the file in a similar fashion, into a new buffer. */
char read_buf[BUF_SIZE];
int read_size = write_size;
for (i = 0; i < 10; i++) {
ret = MPI_File_read_at(file, i * read_size,
read_buf + i * read_size,
read_size, MPI_CHAR, &status);
assert(ret == MPI_SUCCESS);
}
/* Verify the read buf is the same as the write buf */
for (i = 0; i < BUF_SIZE; i++) {
if (read_buf[i] != write_buf[i]) {
printf("%s:%s: buf[%d]: expected %d, got %d\n",
__FILE__, __FUNCTION__,
i, write_buf[i], read_buf[i]);
}
assert(read_buf[i] == write_buf[i]);
}
ret = MPI_File_close(&file);
assert(ret == 0);
ret = MPI_Finalize();
assert(ret == 0);
return 0;
}
int main(int argc, char *argv[]) {
if(argc != 5) {
printf("ERROR: Not enough arguments.\n");
return EXIT_FAILURE;
}
// Example MPI startup and using CLCG4 RNG
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &mpi_commsize);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_myrank);
// assign cli args to globals
g_matrix_size = atoi(argv[1]);
g_threads = atoi(argv[2]);
g_file_ranks = atoi(argv[3]);
g_out = argv[4];
g_row_size = g_matrix_size / mpi_commsize;
// Init 16,384 RNG streams - each rank has an independent stream
InitDefault();
MPI_Barrier(MPI_COMM_WORLD);
// my_matrix stores a local slice of the 128gb matrix
// my_transpose stores the transpose of that slice
my_matrix = calloc(g_row_size * g_matrix_size, sizeof(unsigned int));
my_transpose = calloc(g_row_size * g_matrix_size, sizeof(unsigned int));
// initialize and randomize matrix thru mpi ranks
// each rank holds some number of rows
// held in a 1d array to make mpi sending easier
unsigned int i, j, k, l;
for(i = 0; i < g_row_size; ++i) {
for(j = 0; j < g_matrix_size; ++j) {
my_matrix[i * g_matrix_size + j] = (unsigned int)(GenVal(mpi_myrank) * 100.0) + 1;
}
}
// populate transpose with own values
unsigned int start_idx = mpi_myrank * g_row_size;
for(i = 0; i < g_row_size; ++i) {
for(j = start_idx; j < g_row_size * g_matrix_size; j = j + g_matrix_size) {
// calculation for the matrix
k = (j - start_idx) / g_matrix_size + start_idx;
my_transpose[i + j] = my_matrix[i * g_matrix_size + k];
}
}
// initialize and allocate buffers
unsigned int bufsize = g_row_size * g_row_size;
unsigned int *sendbuf = calloc(bufsize, sizeof(unsigned int *));
unsigned int **recvbuf = calloc(mpi_commsize-1, sizeof(unsigned int *));
for(i = 0; i < mpi_commsize-1; ++i) {
recvbuf[i] = calloc(bufsize, sizeof(unsigned int));
}
// mpi stuff
unsigned int num_reqs = 2 * (mpi_commsize - 1);
unsigned int cur_req = 0;
MPI_Request *requests = (MPI_Request *)malloc(num_reqs * sizeof(MPI_Request));
MPI_Status *statuses = (MPI_Status *)malloc(num_reqs * sizeof(MPI_Status));
// send to all other ranks
for(i = 0; i < mpi_commsize; ++i) {
if(i != mpi_myrank) {
// store relevant data for the receiving rank
int cnt = 0;
int tx = 0;
start_idx = i * g_row_size;
for(j = 0; j < g_row_size; ++j) {
for(k = start_idx; k < g_row_size * g_matrix_size; k = k + g_matrix_size) {
// calculation for the matrix (a little messy, could be optimized)
l = (k - start_idx) / g_matrix_size + start_idx;
if(cnt >= bufsize) {
// handles the overflow, after which we offset it (new column)
cnt = ++tx;
}
sendbuf[cnt] = my_matrix[j * g_matrix_size + l];
cnt += g_row_size;
}
}
MPI_Isend(sendbuf, bufsize, MPI_UNSIGNED, i, 0, MPI_COMM_WORLD, &requests[cur_req++]);
}
}
// recv from all other rows
// handling a little messy since irecv is nonblocking
int cnt = 0;
for(i = 0; i < mpi_commsize; ++i) {
if(i != mpi_myrank) {
MPI_Irecv(recvbuf[cnt++], bufsize, MPI_UNSIGNED, i, 0, MPI_COMM_WORLD, &requests[cur_req++]);
}
}
// wait on MPI messages
MPI_Waitall(num_reqs, requests, statuses);
// store relevant values
k = 0;
l = 0;
for(i = 0; i < g_row_size; ++i) {
for(j = 0; j < g_matrix_size; ++j) {
if(my_transpose[i * g_matrix_size + j] == 0) {
my_transpose[i * g_matrix_size + j] = recvbuf[k][l++];
if(l >= bufsize) {
k++;
l = 0;
}
}
}
}
MPI_Barrier(MPI_COMM_WORLD);
unsigned long long t1 = 0, t2 = 0;
if(mpi_myrank == 0) {
t1 = GetTimeBase();
}
// split into pthreads
pthread_t *call_thd;
call_thd = (pthread_t *)malloc(g_threads * sizeof(pthread_t));
void *status;
pthread_attr_t attr;
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
pthread_mutex_init(&mutexsum, NULL);
long x;
for(x = 0; x < g_threads; ++x) {
pthread_create(&call_thd[i], &attr, mtx_sum, (void *)x);
}
// wait on threads
for(x = 0; x < g_threads; ++x) {
pthread_join(call_thd[i], &status);
}
MPI_Barrier(MPI_COMM_WORLD);
if(mpi_myrank == 0) {
t2 = GetTimeBase();
float tmp = (t2-t1) / 1600000;
printf("Elapsed compute time: %f\n", tmp);
}
// I/O
if(mpi_myrank == 0) {
t1 = GetTimeBase();
}
MPI_Offset offset = (mpi_myrank % g_file_ranks) * g_row_size * g_matrix_size * sizeof(unsigned int);
MPI_File file;
MPI_Status iostatus;
MPI_Datatype localarray;
/* create a type describing our piece of the array */
int globalsizes[2] = {g_matrix_size, g_matrix_size};
int localsizes [2] = {g_row_size, g_matrix_size};
int starts[2] = {mpi_myrank * g_row_size, 0};
int order = MPI_ORDER_C;
MPI_Type_create_subarray(2, globalsizes, localsizes, starts, order, MPI_UNSIGNED, &localarray);
MPI_Type_commit(&localarray);
// open the file, and set the view
MPI_File_open(MPI_COMM_WORLD, g_out,
MPI_MODE_CREATE|MPI_MODE_WRONLY,
MPI_INFO_NULL, &file);
MPI_File_set_view(file, 0, MPI_UNSIGNED, localarray, "native", MPI_INFO_NULL);
// write to file at specified offset
MPI_File_write_at(file, offset * mpi_myrank, my_matrix, g_row_size * g_matrix_size, MPI_UNSIGNED, &iostatus);
MPI_File_close(&file);
MPI_Barrier(MPI_COMM_WORLD);
if(mpi_myrank == 0) {
t2 = GetTimeBase();
float tmp = (t2-t1) / 1600000;
printf("Elapsed IO time: %f\n", tmp);
}
// cleanup routine
MPI_Type_free(&localarray);
pthread_attr_destroy(&attr);
pthread_mutex_destroy(&mutexsum);
free(call_thd);
free(my_matrix);
free(my_transpose);
for(i = 0; i < mpi_commsize-1; ++i) free(recvbuf[i]);
free(recvbuf);
free(sendbuf);
free(requests);
free(statuses);
// END -Perform a barrier and then leave MPI
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
return EXIT_SUCCESS;
}
// this is currently slower than it could be because we write in chunks (data.size()..) with barriers in between - instead we could take care of correct format of matrix on the read-in side and write everything at once, or assume same format of data and only do one barrier
void Storage::SaveDataFloatMPIBin(char* filename, vector<vector<float> > data, int mpiRank, int mpiSize, MPI_Comm comm)
{
MPI_File fh;
// assumes specific data distribution for processes
MPI_File_delete(filename,MPI_INFO_NULL);
MPI_File_open(comm,filename,MPI_MODE_RDWR|MPI_MODE_CREATE,MPI_INFO_NULL,&fh);
int globalOffset = 0;
// (!) Currently assumes same size of all items in data
int size = 0;
for(int i=0;i<data.size();i++) // assumes same nr items each process
{
size+= data[i].size();
}
vector<float> tempData(size);
int index=0;
for(int i=0;i<data.size();i++) // assumes same nr items each process
{
for(int j=0;j<data[i].size();j++)
{
tempData[index] = data[i][j];
index++;
}
}
vector<int> allSizes(mpiSize);
MPI_Allgather(&size,1,MPI_INT,&allSizes[0],1,MPI_INT,comm);
int startPos = 0;
for(int j=0;j<mpiRank;j++)
startPos+=allSizes[j];
//for(int i=0;i<data.size();i++) // assumes same nr items each process
//{
// /*int size = data[i].size(); // each item can be of different size
// vector<int> allSizes(mpiSize);
// MPI_Allgather(&size,1,MPI_INT,&allSizes[0],1,MPI_INT,comm);
// int startPos = 0;
// for(int j=0;j<mpiRank;j++)
// startPos+=allSizes[j];*/
// MPI_Status status;
// MPI_File_write_at(fh,(startPos+globalOffset)*sizeof(MPI_FLOAT),&data[i][0],data[i].size(),MPI_FLOAT,&status);
// //if(i==0)
// //MPI_File_write_at(fh,(startPos+globalStartPos),&data[i][0],data[i].size(),MPI_FLOAT,&status);
// for(int j=0;j<mpiSize;j++)
// globalOffset+=allSizes[j];
// //globalOffset = 0;
// //for(int j=mpiRank;j<mpiSize;j++)
// // globalOffset+=allSizes[j];
//}
MPI_Status status;
if(size>0)
MPI_File_write_at(fh,(startPos+globalOffset)*sizeof(MPI_FLOAT),&tempData[0],size,MPI_FLOAT,&status);//&data[0][0],size,MPI_FLOAT,&status);
//MPI_File_write_at(fh,(startPos+globalOffset)*sizeof(MPI_FLOAT),&data[0][0],size,MPI_FLOAT,&status);
MPI_File_close(&fh);
}
static int populate_meta_data(PIDX_header_io_id header_io_id, PIDX_block_layout block_layout, int file_number, char* bin_file, int mode)
{
int block_negative_offset = 0;
int i = 0, j = 0, k = 0, all_scalars = 0;
off_t data_offset = 0, base_offset = 0;
//int total_file_size = 0;
int64_t total_chunk_size = (header_io_id->idx->chunk_size[0] * header_io_id->idx->chunk_size[1] * header_io_id->idx->chunk_size[2] * header_io_id->idx->chunk_size[3] * header_io_id->idx->chunk_size[4]);// / (64 / header_io_id->idx->compression_bit_rate);
/*
for (i = 0; i < header_io_id->idx->variable_count; i++)
{
if (header_io_id->idx->variable[i]->values_per_sample != 1)
{
all_scalars = 0;
break;
}
}
*/
int total_header_size = (10 + (10 * header_io_id->idx->blocks_per_file)) * sizeof (uint32_t) * header_io_id->idx->variable_count;
//memset(headers, 0, total_header_size);
//int nprocs;
//MPI_Comm_size(header_io_id->comm, &nprocs);
for (i = 0; i < header_io_id->idx->blocks_per_file; i++)
{
//if (nprocs == 1)
// printf("[B] [%d %d] : %d\n", header_io_id->idx->variable[header_io_id->first_index]->global_block_layout->resolution_from, header_io_id->idx->variable[header_io_id->first_index]->global_block_layout->resolution_to, i);
if (PIDX_blocks_is_block_present((i + (header_io_id->idx->blocks_per_file * file_number)), header_io_id->idx->variable[header_io_id->first_index]->global_block_layout))
{
//if (nprocs == 1)
// printf("[A] %d\n", i);
block_negative_offset = PIDX_blocks_find_negative_offset(header_io_id->idx->blocks_per_file, (i + (header_io_id->idx->blocks_per_file * file_number)), header_io_id->idx->variable[header_io_id->first_index]->global_block_layout);
for (j = header_io_id->first_index; j < header_io_id->last_index; j++)
//for (j = 0; j < header_io_id->idx->variable_count; j++)
{
base_offset = 0;
if (all_scalars == 0)
{
for (k = 0; k < j; k++)
base_offset = base_offset + ((block_layout->block_count_per_file[file_number]) * (header_io_id->idx->variable[k]->bits_per_value / 8) * total_chunk_size * header_io_id->idx_d->samples_per_block * header_io_id->idx->variable[k]->values_per_sample) / (header_io_id->idx->compression_factor);
//base_offset = base_offset + ((header_io_id->idx->variable[header_io_id->first_index]->block_count_per_file[file_number]) * (header_io_id->idx->variable[k]->bits_per_value / 8) * total_chunk_size * header_io_id->idx_d->samples_per_block * header_io_id->idx->variable[k]->values_per_sample) / (header_io_id->idx->variable[k]->bits_per_value / header_io_id->idx->compression_bit_rate);
}
else
base_offset = j * (block_layout->block_count_per_file[file_number]) * (header_io_id->idx->variable[header_io_id->first_index]->bits_per_value / 8) * total_chunk_size * header_io_id->idx_d->samples_per_block * header_io_id->idx->variable[header_io_id->first_index]->values_per_sample / (header_io_id->idx->compression_factor);
//base_offset = j * (header_io_id->idx->variable[header_io_id->first_index]->block_count_per_file[file_number]) * (header_io_id->idx->variable[header_io_id->first_index]->bits_per_value / 8) * total_chunk_size * header_io_id->idx_d->samples_per_block * header_io_id->idx->variable[header_io_id->first_index]->values_per_sample / (header_io_id->idx->variable[j]->bits_per_value / header_io_id->idx->compression_bit_rate);
data_offset = (((i) - block_negative_offset) * header_io_id->idx_d->samples_per_block) * (header_io_id->idx->variable[j]->bits_per_value / 8) * total_chunk_size * header_io_id->idx->variable[j]->values_per_sample / (header_io_id->idx->compression_factor);
//printf("BLOCK %d = %d + %d + %d (%d %d)\n", i, (int)base_offset, (int)data_offset, (int)(header_io_id->idx_d->start_fs_block * header_io_id->idx_d->fs_block_size), header_io_id->idx_d->start_fs_block, header_io_id->idx_d->fs_block_size);
data_offset = base_offset + data_offset + header_io_id->idx_d->start_fs_block * header_io_id->idx_d->fs_block_size;
//TODO
//printf("%d %d: %d (%d * %d * %d * %d)\n", i, j, header_io_id->idx_d->samples_per_block * (header_io_id->idx->variable[j]->bits_per_value / 8) * total_chunk_size * header_io_id->idx->variable[j]->values_per_sample, header_io_id->idx_d->samples_per_block, (header_io_id->idx->variable[j]->bits_per_value / 8), total_chunk_size, header_io_id->idx->variable[j]->values_per_sample);
//if (file_number == 2)
//if (nprocs == 1)
//printf("[%d] offset : count = %lld %lld\n", i, (unsigned long long)data_offset, (unsigned long long)(header_io_id->idx_d->samples_per_block * (header_io_id->idx->variable[j]->bits_per_value / 8) * total_chunk_size * header_io_id->idx->variable[j]->values_per_sample));
headers[12 + ((i + (header_io_id->idx->blocks_per_file * j))*10 )] = htonl(data_offset);
headers[14 + ((i + (header_io_id->idx->blocks_per_file * j))*10)] = htonl(header_io_id->idx_d->samples_per_block * (header_io_id->idx->variable[j]->bits_per_value / 8) * total_chunk_size * header_io_id->idx->variable[j]->values_per_sample / (header_io_id->idx->compression_factor));
//total_file_size = data_offset + header_io_id->idx_d->samples_per_block * (header_io_id->idx->variable[j]->bits_per_value / 8) * total_chunk_size * header_io_id->idx->variable[j]->values_per_sample / (header_io_id->idx->compression_factor);
}
}
}
#if 1
if (mode == 1)
{
#if PIDX_HAVE_MPI
if (header_io_id->idx_d->parallel_mode == 1)
{
MPI_File fh;
MPI_Status status;
int ret = 0;
ret = MPI_File_open(MPI_COMM_SELF, bin_file, MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
if (ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed on %s\n", __FILE__, __LINE__, bin_file);
return PIDX_err_io;
}
/*
if(total_file_size != 0)
{
ret = MPI_File_set_size(fh, total_file_size);
if(ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
}
*/
ret = MPI_File_write_at(fh, 0, headers, total_header_size, MPI_BYTE, &status);
if (ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
ret = MPI_File_close(&fh);
if (ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed on %s\n", __FILE__, __LINE__, bin_file);
return PIDX_err_io;
}
}
else
{
int fh = 0;
fh = open(bin_file, O_WRONLY, 0664);
//ftruncate(fh, total_file_size);
ssize_t ret_size = pwrite(fh, headers, total_header_size, 0);
if (ret_size != total_header_size)
{
fprintf(stderr, "[%s] [%d] pwrite() failed on %s\n", __FILE__, __LINE__, bin_file);
return PIDX_err_io;
}
close(fh);
}
#else
int fh = 0;
fh = open(bin_file, O_WRONLY, 0664);
//ftruncate(fh, total_file_size);
ssize_t ret_size = pwrite(fh, headers, total_header_size, 0);
if (ret_size != total_header_size)
{
fprintf(stderr, "[%s] [%d] pwrite() failed on %s\n", __FILE__, __LINE__, bin_file);
return PIDX_err_io;
}
close(fh);
#endif
//int total_header_size = (10 + (10 * header_io_id->idx->blocks_per_file)) * sizeof (uint32_t) * header_io_id->idx->variable_count;
}
#endif
return PIDX_success;
}
int main( int argc, char *argv[] )
{
int itr;
int operacao;
char * chave_file;
char * entrada_file;
char * saida_file;
octeto Nb,Nk,Nr;
octeto bloco[4*8];
octeto chave[4*8*15];
int worldsize, rank;
MPI_Status status;
MPI_File chave_handle;
MPI_File entrada_handle;
MPI_File saida_handle;
MPI_Offset entrada_bytes;
unsigned int numero_blocos;
unsigned int blocos_processo;
MPI_Offset bloco_byte_inicio;
MPI_Offset bloco_byte_fim;
MPI_Offset iterador;
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD,&worldsize);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
operacao = INDEFINIDA;
chave_file = NULL;
entrada_file = NULL;
saida_file = NULL;
for(itr = 1;itr < argc;itr++)
{
/* Instrucoes de uso */
if( strcmp(argv[itr],"-a") == 0 || strcmp(argv[itr],"--ajuda") == 0 ||
strcmp(argv[itr],"-h") == 0 || strcmp(argv[itr],"--help") == 0 )
{
if(rank == 0)
{
printf(" Uso: mpiexec -n [PROCESSOS] ./rijndael [ARGUMENTO VALOR].\n");
printf(" Encripta/Decripta um arquivo usando o algoritmo Rijndael(AES) extendido.\n");
printf(" Argumentos obrigatorios:\n");
printf(" -op,--operacao: Informa se o objetivo da execucao eh encriptar ou decriptar.\n");
printf(" * Os valores possiveis sao: \'encriptar\' e \'decriptar\'.\n");
printf(" -e,-i,--entrada,--input: Caminho e nome do arquivo a ser criptografado.\n");
printf(" -s,-o,--saida,--output: Caminho e nome do arquivo resultante do processo de criptografia da entrada.\n");
printf(" -c,-k,--chave,--key: Caminho e nome do arquivo contendo a chave.\n");
printf(" O arquivo contendo a chave eh em formato binario de acordo com a seguinte especificacao:\n");
printf(" - O primeiro byte deve conter o tamanho do bloco (em palavras de 4 bytes).\n");
printf(" * O bloco pode possuir tamanho: 4, 5, 6, 7 ou 8.\n");
printf(" - O segundo byte deve conter o tamanho da chave (em palavras de 4 bytes).\n");
printf(" * Esta aplicacao aceita chaves com tamanho: 4, 5, 6, 7 ou 8.\n");
printf(" - Os proximos 4*[tamanho da chave] bytes do arquivo sao os bytes componentes da chave, que\n");
printf(" devem estar (obrigatoriamente) escritos no formato hexadecimal da linguagem C (0xff).\n");
printf(" * Eh recomendavel o uso de um editor hexadecimal na construcao do arquivo chave.\n");
}
goto finalizando;
}
/* Operacao a ser realizada */
else
if( strcmp(argv[itr],"-op") == 0 || strcmp(argv[itr],"--operacao") == 0 )
{
if( itr+1 < argc )
{
if( strcmp(argv[itr+1],"encriptar") == 0 )
{
operacao = ENCRIPTAR;
}
else
if( strcmp(argv[itr+1],"decriptar") == 0 )
{
operacao = DECRIPTAR;
}
itr++;
}
else
{
goto sempar;
}
}
/* Arquivo com a chave */
else
if( strcmp(argv[itr],"-c") == 0 || strcmp(argv[itr],"--chave") == 0 ||
strcmp(argv[itr],"-k") == 0 || strcmp(argv[itr],"--key") == 0 )
{
if(itr+1 < argc)
{
chave_file = argv[itr+1];
itr++;
}
else
{
goto sempar;
}
}
/* Arquivo de entrada */
else
if( strcmp(argv[itr],"-e") == 0 || strcmp(argv[itr],"--entrada") == 0 ||
strcmp(argv[itr],"-i") == 0 || strcmp(argv[itr],"--input") == 0 )
{
if(itr+1 < argc)
{
entrada_file = argv[itr+1];
itr++;
}
else
{
goto sempar;
}
}
/* Arquivo de saida */
else
if( strcmp(argv[itr],"-s") == 0 || strcmp(argv[itr],"--saida") == 0 ||
strcmp(argv[itr],"-o") == 0 || strcmp(argv[itr],"--output") == 0 )
{
if(itr+1 < argc)
{
saida_file = argv[itr+1];
itr++;
}
else
{
goto sempar;
}
}
/* Erro desconhecido */
else
{
if(rank == 0)
{
printf("Erro nos argumentos passados.\n");
}
goto help;
}
}
/* Fim da leitura dos argumentos */
if( operacao == INDEFINIDA || chave_file == NULL || entrada_file == NULL || saida_file == NULL )
{
if(rank == 0)
{
if( operacao == INDEFINIDA )
printf("A operacao a ser realizada eh invalida ou nao foi especificada.\n");
if( chave_file == NULL )
printf("Esta faltando especificar o arquivo com a chave.\n");
if( entrada_file == NULL )
printf("Esta faltando especificar o arquivo de entrada.\n");
if( saida_file == NULL )
printf("Esta faltando especificar o arquivo de saida.\n");
}
goto help;
}
/* Fim do tratamento dos argumentos */
if( MPI_File_open( MPI_COMM_WORLD, chave_file, MPI_MODE_RDONLY, MPI_INFO_NULL, &chave_handle ) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro na abertura do arquivo com a chave (%s).\n",chave_file);
}
goto help;
}
if( MPI_File_read(chave_handle,&Nb,1, MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro na leitura do tamanho de um bloco no arquivo com a chave (%s).\n",chave_file);
}
goto help;
}
if( Nb< 4 || Nb > 8 )
{
if( rank == 0 )
{
printf("Tamanho de bloco invalido no arquivo com a chave (%s).\n",chave_file);
}
goto help;
}
if( MPI_File_read(chave_handle,&Nk,1, MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro na leitura do tamanho da chave no arquivo com a chave (%s).\n",chave_file);
}
goto help;
}
if( Nk< 4 || Nk > 8 )
{
if( rank == 0 )
{
printf("Tamanho de chave invalido no arquivo com a chave (%s).\n",chave_file);
}
goto help;
}
if( MPI_File_read(chave_handle,chave,4*Nk,MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro na leitura da chave no arquivo com a chave (%s).\n",chave_file);
}
goto help;
}
MPI_File_close( &chave_handle );
Nr = numero_rodadas(Nb,Nk);
KeyExpansion(chave,Nb,Nk);
if( MPI_File_open( MPI_COMM_WORLD, entrada_file,
MPI_MODE_RDONLY,
MPI_INFO_NULL, &entrada_handle ) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro na abertura do arquivo de entrada (%s).\n",entrada_file);
}
goto help;
}
MPI_File_get_size(entrada_handle,&entrada_bytes);
if( MPI_File_open( MPI_COMM_WORLD, saida_file,
MPI_MODE_RDWR | MPI_MODE_CREATE | MPI_MODE_EXCL,
MPI_INFO_NULL, &saida_handle ) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro na criacao do arquivo de saida (%s).\n",saida_file);
printf("Uma possivel causa eh que o arquivo ja exista.\n");
}
goto help;
}
numero_blocos = ( entrada_bytes / (Nb*4) );
blocos_processo = numero_blocos / worldsize;
if( operacao == ENCRIPTAR )
{
MPI_File_set_size(saida_handle,(MPI_Offset)( (numero_blocos+1)*(Nb*4) ) );
bloco_byte_inicio = 4*Nb*blocos_processo*rank;
bloco_byte_fim = 4*Nb*blocos_processo*(rank+1);
for( iterador = bloco_byte_inicio ; iterador < bloco_byte_fim ; iterador += (4*Nb) )
{
if( MPI_File_read_at(entrada_handle,iterador,bloco,(4*Nb),MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro ao ler do arquivo de entrada (%s).\n",entrada_file);
}
goto help;
}
AES_encriptar_bloco(bloco,Nb,chave,Nr);
if( MPI_File_write_at(saida_handle,iterador,bloco,(4*Nb),MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro ao escrever no arquivo de saida (%s).\n",saida_file);
}
goto help;
}
}
iterador = 4*Nb*blocos_processo*worldsize + 4*Nb*rank;
if( iterador <= numero_blocos*4*Nb )
{
if( MPI_File_read_at(entrada_handle,iterador,bloco,(4*Nb),MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro ao ler do arquivo de entrada (%s).\n",entrada_file);
}
goto help;
}
if( iterador == numero_blocos*4*Nb )
bloco[ 4*Nb - 1 ] = (octeto)(entrada_bytes - numero_blocos*4*Nb);
AES_encriptar_bloco(bloco,Nb,chave,Nr);
if( MPI_File_write_at(saida_handle,iterador,bloco,(4*Nb),MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro ao escrever no arquivo de saida (%s).\n",saida_file);
}
goto help;
}
}
if( rank == 0 )
{
// printf("A encriptacao do arquivo foi realizada com sucesso.\n");
}
}
else
if( operacao == DECRIPTAR )
{
MPI_File_set_size(saida_handle,entrada_bytes);
bloco_byte_inicio = 4*Nb*blocos_processo*rank;
bloco_byte_fim = 4*Nb*blocos_processo*(rank+1);
for( iterador = bloco_byte_inicio ; iterador < bloco_byte_fim ; iterador += (4*Nb) )
{
if( MPI_File_read_at(entrada_handle,iterador,bloco,(4*Nb),MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro ao ler do arquivo de entrada (%s).\n",entrada_file);
}
goto help;
}
AES_decriptar_bloco(bloco,Nb,chave,Nr);
if( MPI_File_write_at(saida_handle,iterador,bloco,(4*Nb),MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro ao escrever no arquivo de saida (%s).\n",saida_file);
}
goto help;
}
}
iterador = 4*Nb*blocos_processo*worldsize + 4*Nb*rank;
if( iterador < numero_blocos*4*Nb )
{
if( MPI_File_read_at(entrada_handle,iterador,bloco,(4*Nb),MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro ao ler do arquivo de entrada (%s).\n",entrada_file);
}
goto help;
}
AES_decriptar_bloco(bloco,Nb,chave,Nr);
if( MPI_File_write_at(saida_handle,iterador,bloco,(4*Nb),MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro ao escrever no arquivo de saida (%s).\n",saida_file);
}
goto help;
}
}
MPI_Barrier( MPI_COMM_WORLD ); /*Barreira q impede q alguem leia antes do valor decriptografado ser escrito */
if( MPI_File_read_at(saida_handle,entrada_bytes-1,bloco,1,MPI_BYTE,&status) != MPI_SUCCESS )
{
if( rank == 0 )
{
printf("Erro ao realizar leitura no arquivo de saida (%s).\n",saida_file);
}
goto help;
}
MPI_Barrier( MPI_COMM_WORLD ); /* Barreira q impede q alqum processo trunque o arquivo antes de outro processo ler*/
MPI_File_set_size(saida_handle,entrada_bytes - 4*Nb + bloco[0]);
if( rank == 0 )
{
// printf("A decriptacao do arquivo foi realizada com sucesso.\n");
}
}
goto finalizando;
sempar:
if( rank == 0 )
{
printf("Sem par correspondente para a opcao %s.\n",argv[itr]);
}
help:
if( rank == 0 )
{
printf("Use a opcao --help para melhor entendimento do uso da aplicacao.\n");
}
finalizando:
MPI_Finalize( );
return 0;
}