// write global checksum, and runinfo in new file
void write_1st_msg(const char* filename, GlobalDat& dat,
size_t chksum){
// open file
FILE* fp;
fp = fopen(filename, "w");
LimeWriter* w;
w = limeCreateWriter(fp);
// first record is global checksum
int MB_flag = 1;
int ME_flag = 0;
LimeRecordHeader* chk;
n_uint64_t chk_bytes = sizeof(chksum);
chk = limeCreateHeader(MB_flag, ME_flag, "Global Checksum", chk_bytes);
limeWriteRecordHeader(chk, w);
limeDestroyHeader(chk);
limeWriteRecordData(&chksum, &chk_bytes, w);
// second record is data on calculation
MB_flag = 0; ME_flag = 1;
LimeRecordHeader* run_id;
n_uint64_t run_id_bytes = sizeof(dat);
run_id = limeCreateHeader(MB_flag, ME_flag, "Runinfo", run_id_bytes);
limeWriteRecordHeader(run_id,w);
limeDestroyHeader(run_id);
limeWriteRecordData(&dat, &run_id_bytes, w);
limeDestroyWriter(w);
fclose(fp);
}
/****************************************************
* write_binary_contraction_data
****************************************************/
int write_binary_contraction_data_3d(double * const s, LimeWriter * limewriter, const int prec, const int N, DML_Checksum * ans) {
#ifdef MPI
fprintf(stderr, "[write_binary_contraction_data_3d] No mpi version.\n");
return(1);
#else
int x, y, z, i=0, mu, status=0;
double *tmp;
float *tmp2;
int proc_coords[4], tloc,xloc,yloc,zloc, proc_id;
n_uint64_t bytes;
DML_SiteRank rank;
int words_bigendian = big_endian();
unsigned int VOL3 = LX*LY*LZ;
DML_checksum_init(ans);
tmp = (double*)malloc(2*N*sizeof(double));
tmp2 = (float*)malloc(2*N*sizeof(float));
if(prec == 32) bytes = (n_uint64_t)2*N*sizeof(float);
else bytes = (n_uint64_t)2*N*sizeof(double);
if(g_cart_id==0) {
for(x = 0; x < LX; x++) {
for(y = 0; y < LY; y++) {
for(z = 0; z < LZ; z++) {
/* Rank should be computed by proc 0 only */
rank = (DML_SiteRank) (( x * LY + y)*LZ + z);
for(mu=0; mu<N; mu++) {
i = _GWI(mu, g_ipt[0][x][y][z], VOL3);
if(!words_bigendian) {
if(prec == 32) {
byte_swap_assign_double2single( (tmp2+2*mu), (s + i), 2);
} else {
byte_swap_assign( (tmp+2*mu), (s + i), 2);
}
} else {
if(prec == 32) {
double2single((float*)(tmp2+2*mu), (s + i), 2);
} else {
tmp[2*mu ] = s[i ];
tmp[2*mu+1] = s[i+1];
}
}
}
if(prec == 32) {
DML_checksum_accum(ans,rank,(char *) tmp2,2*N*sizeof(float));
status = limeWriteRecordData((void*)tmp2, &bytes, limewriter);
}
else {
status = limeWriteRecordData((void*)tmp, &bytes, limewriter);
DML_checksum_accum(ans,rank,(char *) tmp, 2*N*sizeof(double));
}
}}}
}
free(tmp2);
free(tmp);
return(0);
#endif
}
void lime_write_info(FILE **fout, char* binary_data, n_uint64_t data_size_per_site, n_uint64_t *offset) {
LimeWriter *limewriter;
LimeRecordHeader *limeheader = NULL;
n_uint64_t message_length;
char tmp_string[STRINGLENGTH];
ASSERT((limewriter = limeCreateWriter(*fout))!=(LimeWriter*)NULL);
sprintf(tmp_string, "Vector_by_DDalphaAMG");
message_length=(n_uint64_t) strlen(tmp_string);
limeheader = limeCreateHeader(1, 1, "vector-type", message_length);
ASSERT(limeheader != (LimeRecordHeader*)NULL);
ASSERT(limeWriteRecordHeader(limeheader, limewriter)>=0);
ASSERT(limeWriteRecordData(tmp_string, &message_length, limewriter)>=0);
limeDestroyHeader(limeheader);
sprintf(tmp_string, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<etmcFormat>\n\t<field>diracFermion</field>\n\t<precision>64</precision>\n\t<flavours>1</flavours>\n\t<lx>%d</lx>\n\t<ly>%d</ly>\n\t<lz>%d</lz>\n\t<lt>%d</lt>\n\t<spin>4</spin>\n\t<colour>3</colour>\n</etmcFormat>", g.global_lattice[0][X], g.global_lattice[0][Y], g.global_lattice[0][Z], g.global_lattice[0][T]);
message_length=(n_uint64_t) strlen(tmp_string);
limeheader = limeCreateHeader(1, 1, "etmc-propagator-format", message_length);
ASSERT(limeheader != (LimeRecordHeader*)NULL);
ASSERT(limeWriteRecordHeader(limeheader, limewriter)>=0);
ASSERT(limeWriteRecordData(tmp_string, &message_length, limewriter)>=0);
limeDestroyHeader(limeheader);
message_length=(n_uint64_t) strlen(tmp_string);
limeheader = limeCreateHeader(1, 1, "DDalphaAMG-header", message_length);
ASSERT(limeheader != (LimeRecordHeader*)NULL);
ASSERT(limeWriteRecordHeader(limeheader, limewriter)>=0);
ASSERT(limeWriteRecordData(tmp_string, &message_length, limewriter)>=0);
limeDestroyHeader(limeheader);
int vol=1, mu;
for ( mu=0; mu<4; mu++ ){
vol*=g.global_lattice[0][mu];
}
message_length=data_size_per_site*vol;
limeheader = limeCreateHeader(1, 1, "scidac-binary-data", message_length);
ASSERT(limeWriteRecordHeader(limeheader, limewriter)>=0);
limeDestroyHeader(limeheader);
message_length=1;
//make one fake record-write to set offset
ASSERT(limeWriteRecordData(tmp_string, &message_length, limewriter)>=0);
*offset = (n_uint64_t) ftell(*fout)-1;
limeDestroyWriter(limewriter);
}
//append correlation functions to file
void append_msgs(const char* filename, std::vector<vec>& corr, std::vector<Tag>& tags,
LimeWriter* w, bool be){
// Each message contains three records:
// 1st: Checksum for the Correlator
// 2nd: Tag for the Correlator
// 3rd: Correlationfunction itself
LimeRecordHeader* corr_chk;
LimeRecordHeader* id;
LimeRecordHeader* corr_hd;
boost::uint64_t corr_chksum;
n_uint64_t tag_bytes = sizeof(tags[0]);
n_uint64_t data_bytes = (corr[0]).size()*2*sizeof(double);
char headername[100];
// Access flags for record and message: MB (Message Begin): 1 if true, ME
// (Message End): 1 if true
int MB_flag, ME_flag;
for(size_t el = 0; el < corr.size(); ++el){
// 1st record for checksum
corr_chksum = checksum <vec> (corr[el], corr[el].size());
if(be) corr_chksum = swap_endian<boost::uint64_t>(corr_chksum);
n_uint64_t chk_bytes = sizeof(corr_chksum);
ME_flag = 0; MB_flag = 1;
corr_chk = limeCreateHeader(MB_flag, ME_flag,"Correlator checksum", chk_bytes);
limeWriteRecordHeader( corr_chk, w );
limeDestroyHeader( corr_chk );
limeWriteRecordData( &corr_chksum, &chk_bytes, w );
// 2nd record for Tag as struct of 3 2dim integer-arrays
ME_flag = 0; MB_flag = 0;
sprintf(headername,"Tag of Correlator with p^2 = %zd", el);
id = limeCreateHeader( MB_flag, ME_flag, headername , tag_bytes );
limeWriteRecordHeader( id, w );
limeDestroyHeader( id );
limeWriteRecordData( &tags[el], &tag_bytes, w );
// 3rd record for correlator belonging to tag
ME_flag = 1; MB_flag = 0;
sprintf(headername,"Correlator with p^2 = %zd", el);
corr_hd = limeCreateHeader( MB_flag, ME_flag, headername, data_bytes );
limeWriteRecordHeader( corr_hd, w );
limeDestroyHeader( corr_hd );
limeWriteRecordData( corr[el].data(), &data_bytes, w );
}
}
/***********************************************************************
* Copyright (C) 2002,2003,2004,2005,2006,2007,2008 Carsten Urbach
*
* This file is part of tmLQCD.
*
* tmLQCD is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* tmLQCD is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with tmLQCD. If not, see <http://www.gnu.org/licenses/>.
***********************************************************************/
#include "ranlux.ih"
#ifdef HAVE_LIBLEMON
// this cannot work like this...
int write_binary_ranlux_data(int * const s,
LemonWriter * lemonwriter, DML_Checksum *checksum) {
// int x, y, z, t, i = 0, xG, yG, zG, tG, status = 0;
// int latticeSize[] = {1, g_nproc_x*LX, g_nproc_y*LY, g_nproc_z*LZ};
// int scidacMapping[] = {0, 3, 2, 1};
// unsigned long bufoffset = 0;
// char *filebuffer = NULL;
// uint64_t bytes;
// DML_SiteRank rank;
// double tick = 0, tock = 0;
// char measure[64];
//
// DML_checksum_init(checksum);
// bytes = (uint64_t)sizeof(su3_vector);
// if (prec == 32) {
// bytes /= 2;
// }
// if((void*)(filebuffer = malloc(VOLUME * bytes)) == NULL) {
// fprintf (stderr, "malloc errno in write_binary_su3_vector_data_parallel: %d\n", errno);
// fflush(stderr);
// errno = 0;
// /* do we need to abort here? */
// return 1;
// }
//
// tG = g_proc_coords[0]*T;
// zG = g_proc_coords[3]*LZ;
// yG = g_proc_coords[2]*LY;
// xG = g_proc_coords[1]*LX;
// for(z = 0; z < LZ; z++) {
// for(y = 0; y < LY; y++) {
// for(x = 0; x < LX; x++) {
// rank = (DML_SiteRank) (((zG + z)*L + yG + y)*L + xG + x);
// i = g_ipt[t][x][y][z];
//
// if (prec == 32)
// be_to_cpu_assign_double2single((float*)(filebuffer + bufoffset), (double*)(s + i), sizeof(su3_vector) / 8);
// else
// be_to_cpu_assign((double*)(filebuffer + bufoffset), (double*)(s + i), sizeof(su3_vector) / 8);
// DML_checksum_accum(checksum, rank, (char*) filebuffer + bufoffset, bytes);
// bufoffset += bytes;
// }
// }
// }
//}
//
//if (g_debug_level > 0) {
// MPI_Barrier(g_cart_grid);
// tick = MPI_Wtime();
//}
//
//status = lemonWriteLatticeParallelMapped(lemonwriter, filebuffer, bytes, latticeSize, scidacMapping);
//
//if (status != LEMON_SUCCESS)
//{
// free(filebuffer);
// fprintf(stderr, "LEMON write error occurred with status = %d, while in write_binary_su3_vector_data_l (su3_vector_write_binary.c)!\n", status);
// return(-2);
//}
//
//if (g_debug_level > 0) {
// MPI_Barrier(g_cart_grid);
// tock = MPI_Wtime();
//
// if (g_cart_id == 0) {
// engineering(measure, latticeSize[0] * latticeSize[1] * latticeSize[2] * latticeSize[3] * bytes, "b");
// fprintf(stdout, "# Time spent writing %s ", measure);
// engineering(measure, tock - tick, "s");
// fprintf(stdout, "was %s.\n", measure);
// engineering(measure, latticeSize[0] * latticeSize[1] * latticeSize[2] * latticeSize[3] * bytes / (tock - tick), "b/s");
// fprintf(stdout, "# Writing speed: %s", measure);
// engineering(measure, latticeSize[0] * latticeSize[1] * latticeSize[2] * latticeSize[3] * bytes / (g_nproc * (tock - tick)), "b/s");
// fprintf(stdout, " (%s per MPI process).\n", measure);
// fflush(stdout);
// }
//}
//
//lemonWriterCloseRecord(lemonwriter);
//
//DML_global_xor(&checksum->suma);
//DML_global_xor(&checksum->sumb);
//
//free(filebuffer);
return 0;
}
#else /* HAVE_LIBLEMON */
int write_binary_ranlux_data(int * const state, LimeWriter * limewriter,
DML_Checksum * checksum, int const length_total) {
int i = 0, tmp, status = 0;
n_uint64_t bytes = sizeof(int);
if (g_cart_id == 0) {
for (i = 0; i < length_total; i++) {
tmp = state[i];
DML_checksum_accum(checksum, (uint32_t) i, (char*) &tmp,
sizeof(int));
status = limeWriteRecordData((int *) &tmp, &bytes, limewriter);
if (status < 0) {
fprintf(stderr,
"LIME write error occurred with status = %d, while in write_binary_ranlux_data (ranlux_write_binary.c)!\n",
status);
#ifdef MPI
MPI_Abort(MPI_COMM_WORLD, 1);
MPI_Finalize();
#endif
exit(500);
}
}
}
return (0);
}
int write_binary_gauge_data(LimeWriter * limewriter, const int prec, DML_Checksum * checksum)
{
int x, X, y, Y, z, Z, tt, t0, tag=0, id=0, status=0;
int latticeSize[] = {T_global, g_nproc_x*LX, g_nproc_y*LY, g_nproc_z*LZ};
su3 tmp[4];
su3 tmp3[4];
float tmp2[72];
int coords[4];
n_uint64_t bytes;
DML_SiteRank rank;
double tick = 0, tock = 0;
char measure[64];
#ifdef TM_USE_MPI
MPI_Status mpi_status;
#endif
DML_checksum_init(checksum);
if (g_debug_level > 0) {
#ifdef TM_USE_MPI
MPI_Barrier(g_cart_grid);
#endif
tick = gettime();
}
if(prec == 32) bytes = (n_uint64_t)2*sizeof(su3);
else bytes = (n_uint64_t)4*sizeof(su3);
for(t0 = 0; t0 < T*g_nproc_t; t0++) {
tt = t0 - g_proc_coords[0]*T;
coords[0] = t0 / T;
for(z = 0; z < LZ*g_nproc_z; z++) {
Z = z - g_proc_coords[3]*LZ;
coords[3] = z / LZ;
for(y = 0; y < LY*g_nproc_y; y++) {
tag = 0;
Y = y - g_proc_coords[2]*LY;
coords[2] = y / LY;
for(x = 0; x < LX*g_nproc_x; x++) {
X = x - g_proc_coords[1]*LX;
coords[1] = x / LX;
#ifdef TM_USE_MPI
MPI_Cart_rank(g_cart_grid, coords, &id);
#endif
if(g_cart_id == 0) {
/* Rank should be computed by proc 0 only */
rank = (DML_SiteRank) (((t0*LZ*g_nproc_z + z)*LY*g_nproc_y + y)*LX*g_nproc_x + x);
if(g_cart_id == id) {
memcpy(&tmp3[0], &g_gauge_field[ g_ipt[tt][X][Y][Z] ][1], sizeof(su3));
memcpy(&tmp3[1], &g_gauge_field[ g_ipt[tt][X][Y][Z] ][2], sizeof(su3));
memcpy(&tmp3[2], &g_gauge_field[ g_ipt[tt][X][Y][Z] ][3], sizeof(su3));
memcpy(&tmp3[3], &g_gauge_field[ g_ipt[tt][X][Y][Z] ][0], sizeof(su3));
if(prec == 32) {
be_to_cpu_assign_double2single(tmp2, tmp3, 4*sizeof(su3)/8);
DML_checksum_accum(checksum, rank, (char*) tmp2, 4*sizeof(su3)/2);
status = limeWriteRecordData((void*)&tmp2, &bytes, limewriter);
}
else {
be_to_cpu_assign(tmp, tmp3, 4*sizeof(su3)/8);
DML_checksum_accum(checksum, rank, (char*) tmp, 4*sizeof(su3));
status = limeWriteRecordData((void*)&tmp, &bytes, limewriter);
}
}
#ifdef TM_USE_MPI
else {
if(prec == 32) {
MPI_Recv(tmp2, 4*sizeof(su3)/8, MPI_FLOAT, id, tag, g_cart_grid, &mpi_status);
DML_checksum_accum(checksum, rank, (char*) tmp2, 4*sizeof(su3)/2);
status = limeWriteRecordData((void*)&tmp2, &bytes, limewriter);
}
else {
MPI_Recv(tmp, 4*sizeof(su3)/8, MPI_DOUBLE, id, tag, g_cart_grid, &mpi_status);
DML_checksum_accum(checksum, rank, (char*) tmp, 4*sizeof(su3));
status = limeWriteRecordData((void*)&tmp, &bytes, limewriter);
}
}
#endif
if(status < 0 ) {
fprintf(stderr, "LIME write error occurred with status = %d, while writing in gauge_write_binary.c!\n", status);
fprintf(stderr, "x %d, y %d, z %d, t %d (%d,%d,%d,%d)\n",x,y,z,tt,X,Y,Z,tt);
fprintf(stderr, "id = %d, bytes = %lu, size = %d\n", g_cart_id, bytes, (int)(4*sizeof(su3)/8));
#ifdef TM_USE_MPI
MPI_Abort(MPI_COMM_WORLD, 1);
MPI_Finalize();
#endif
exit(500);
}
}
#ifdef TM_USE_MPI
else {
if(g_cart_id == id){
memcpy(&tmp3[0], &g_gauge_field[ g_ipt[tt][X][Y][Z] ][1], sizeof(su3));
memcpy(&tmp3[1], &g_gauge_field[ g_ipt[tt][X][Y][Z] ][2], sizeof(su3));
memcpy(&tmp3[2], &g_gauge_field[ g_ipt[tt][X][Y][Z] ][3], sizeof(su3));
memcpy(&tmp3[3], &g_gauge_field[ g_ipt[tt][X][Y][Z] ][0], sizeof(su3));
if(prec == 32) {
be_to_cpu_assign_double2single(tmp2, tmp3, 4*sizeof(su3)/8);
MPI_Send((void*) tmp2, 4*sizeof(su3)/8, MPI_FLOAT, 0, tag, g_cart_grid);
}
else {
be_to_cpu_assign(tmp, tmp3, 4*sizeof(su3)/8);
MPI_Send((void*) tmp, 4*sizeof(su3)/8, MPI_DOUBLE, 0, tag, g_cart_grid);
}
}
}
#endif
tag++;
}
#ifdef TM_USE_MPI
MPI_Barrier(g_cart_grid);
#endif
}
}
}
if (g_debug_level > 0) {
#ifdef TM_USE_MPI
MPI_Barrier(g_cart_grid);
#endif
tock = gettime();
if (g_cart_id == 0) {
engineering(measure, latticeSize[0] * latticeSize[1] * latticeSize[2] * latticeSize[3] * bytes, "b");
fprintf(stdout, "# Time spent writing %s ", measure);
engineering(measure, tock-tick, "s");
fprintf(stdout, "was %s.\n", measure);
engineering(measure, latticeSize[0] * latticeSize[1] * latticeSize[2] * latticeSize[3] * bytes / (tock-tick), "b/s");
fprintf(stdout, "# Writing speed: %s", measure);
engineering(measure, latticeSize[0] * latticeSize[1] * latticeSize[2] * latticeSize[3] * bytes / (g_nproc * (tock-tick)), "b/s");
fprintf(stdout, " (%s per MPI process).\n", measure);
}
}
return(0);
}
