/*----------------------------------------------------------------------*/
void w_serial_finish_lattice(w_serial_site_writer *state)
{
/* gf = file descriptor as opened by w_serial_i */
gauge_file *gf = state->gf;
fsu3_matrix *lbuf = state->lbuf;
FILE *fp = gf->fp;
gauge_header *gh = gf->header;
off_t checksum_offset = state->checksum_offset; /* Location of checksum */
free(lbuf);
printf("Saved gauge configuration serially to binary file %s\n",
gf->filename);
printf("Time stamp %s\n",gh->time_stamp);
/* Write checksum */
/* Position file pointer */
if( g_seek(fp,checksum_offset,SEEK_SET) < 0 )
{
printf("w_serial: Node %d g_seek %lld failed error %d file %s\n",
this_node,(long long)checksum_offset,errno,gf->filename);
fflush(stdout);terminate(1);
}
write_checksum(SERIAL,gf);
} /* w_serial_finish_lattice */
/*----------------------------------------------------------------------*/
void w_serial_start_lattice(gauge_file *gf, w_serial_site_writer *state,
int input_prec)
{
/* gf = file descriptor as opened by w_serial_i */
/* state of the writer for a single site */
FILE *fp;
gauge_header *gh;
off_t offset; /* File stream pointer */
off_t coord_list_size; /* Size of coordinate list in bytes */
off_t head_size; /* Size of header plus coordinate list */
off_t checksum_offset; /* Location of checksum */
off_t gauge_check_size; /* Size of checksum record */
if(gf->parallel)
printf("w_serial_start_lattice: Attempting serial write to parallel file \n");
outputbuf = (fsu3_matrix *)malloc(MAX_BUF_LENGTH*4*sizeof(fsu3_matrix));
if(outputbuf == NULL)
{
printf("w_serial: Node 0 can't malloc outputbuf\n");
fflush(stdout);terminate(1);
}
fp = gf->fp;
gh = gf->header;
/* No coordinate list was written because fields are to be written
in standard coordinate list order */
coord_list_size = 0;
head_size = gh->header_bytes + coord_list_size;
checksum_offset = head_size;
gauge_check_size = sizeof(gf->check.sum29) + sizeof(gf->check.sum31);
offset = head_size + gauge_check_size;
if( g_seek(fp,offset,SEEK_SET) < 0 )
{
printf("w_serial: Node %d g_seek %lld failed error %d file %s\n",
this_node,(long long)offset,errno,gf->filename);
fflush(stdout);terminate(1);
}
/* initialize checksums */
gf->check.sum31 = 0;
gf->check.sum29 = 0;
state->gf = gf;
state->buf_length = 0;
state->siterank = 0;
state->rank29 = 0;
state->rank31 = 0;
state->lbuf = outputbuf;
state->checksum_offset = checksum_offset;
state->prec = input_prec;
} /* w_serial_start_lattice */
// -----------------------------------------------------------------
// Only node 0 reads the gauge configuration gf from a binary file
static void r_serial(gauge_file *gf) {
FILE *fp = gf->fp;
gauge_header *gh = gf->header;
char *filename = gf->filename;
int byterevflag = gf->byterevflag;
off_t offset = 0; // File stream pointer
off_t gauge_check_size; // Size of gauge configuration checksum record
off_t coord_list_size; // Size of coordinate list in bytes
off_t head_size; // Size of header plus coordinate list
off_t checksum_offset = 0; // Where we put the checksum
int rcv_rank, rcv_coords, destnode, stat, idest = 0;
int k, x, y, z, t;
int buf_length = 0, where_in_buf = 0;
gauge_check test_gc;
u_int32type *val;
int rank29, rank31;
fmatrix *lbuf = NULL; // Only allocate on node0
fmatrix tmat[4];
if (this_node == 0) {
// Compute offset for reading gauge configuration
if (gh->magic_number == GAUGE_VERSION_NUMBER)
gauge_check_size = sizeof(gf->check.sum29) + sizeof(gf->check.sum31);
else
gauge_check_size = 0;
if (gf->header->order == NATURAL_ORDER)
coord_list_size = 0;
else
coord_list_size = sizeof(int32type) * volume;
checksum_offset = gf->header->header_bytes + coord_list_size;
head_size = checksum_offset + gauge_check_size;
// Allocate single-precision read buffer
if (gf->parallel)
printf("r_serial: Attempting serial read from parallel file\n");
lbuf = malloc(MAX_BUF_LENGTH * 4 * sizeof(*lbuf));
if (lbuf == NULL) {
printf("r_serial: node%d can't malloc lbuf\n", this_node);
fflush(stdout);
terminate(1);
}
/* Position file for reading gauge configuration */
offset = head_size;
if (g_seek(fp, offset, SEEK_SET) < 0) {
printf("r_serial: node0 g_seek %lld failed error %d file %s\n",
(long long)offset, errno, filename);
fflush(stdout);
terminate(1);
}
buf_length = 0;
where_in_buf = 0;
}
// All nodes initialize checksums
test_gc.sum29 = 0;
test_gc.sum31 = 0;
// Count 32-bit words mod 29 and mod 31 in order of appearance on file
// Here all nodes see the same sequence because we read serially
rank29 = 0;
rank31 = 0;
g_sync();
// node0 reads and deals out the values
for (rcv_rank = 0; rcv_rank < volume; rcv_rank++) {
/* If file is in coordinate natural order, receiving coordinate
is given by rank. Otherwise, it is found in the table */
if (gf->header->order == NATURAL_ORDER)
rcv_coords = rcv_rank;
else
rcv_coords = gf->rank2rcv[rcv_rank];
x = rcv_coords % nx;
rcv_coords /= nx;
y = rcv_coords % ny;
rcv_coords /= ny;
z = rcv_coords % nz;
rcv_coords /= nz;
t = rcv_coords % nt;
// The node that gets the next set of gauge links
destnode = node_number(x, y, z, t);
// node0 fills its buffer, if necessary
if (this_node == 0) {
if (where_in_buf == buf_length) { /* get new buffer */
/* new buffer length = remaining sites, but never bigger
than MAX_BUF_LENGTH */
buf_length = volume - rcv_rank;
if (buf_length > MAX_BUF_LENGTH)
buf_length = MAX_BUF_LENGTH;
// Now do read
stat = (int)g_read(lbuf, 4 * sizeof(fmatrix), buf_length, fp);
if (stat != buf_length) {
printf("r_serial: node%d gauge configuration read error %d file %s\n",
this_node, errno, filename);
fflush(stdout);
terminate(1);
}
where_in_buf = 0; // Reset counter
} // End of the buffer read
if (destnode == 0) { // Just copy links
idest = node_index(x, y, z, t);
// Save 4 matrices in tmat for further processing
memcpy(tmat, &lbuf[4 * where_in_buf], 4 * sizeof(fmatrix));
}
else { // Send to correct node
send_field((char *)&lbuf[4 * where_in_buf],
4 * sizeof(fmatrix), destnode);
}
where_in_buf++;
}
// The node that contains this site reads the message
else { // All nodes other than node 0
if (this_node == destnode) {
idest = node_index(x, y, z, t);
// Receive 4 matrices in temporary space for further processing
get_field((char *)tmat, 4 * sizeof(fmatrix), 0);
}
}
/* The receiving node does the byte reversal and then checksum,
if needed. At this point tmat contains the input matrices
and idest points to the destination site structure. */
if (this_node == destnode) {
if (byterevflag == 1)
byterevn((int32type *)tmat,
4 * sizeof(fmatrix) / sizeof(int32type));
// Accumulate checksums
for (k = 0, val = (u_int32type *)tmat;
k < 4*(int)sizeof(fmatrix) / (int)sizeof(int32type);
k++, val++) {
test_gc.sum29 ^= (*val)<<rank29 | (*val)>>(32 - rank29);
test_gc.sum31 ^= (*val)<<rank31 | (*val)>>(32 - rank31);
rank29++;
if (rank29 >= 29)
rank29 = 0;
rank31++;
if (rank31 >= 31)
rank31 = 0;
}
// Copy 4 matrices to generic-precision lattice[idest]
f2d_4mat(tmat, &lattice[idest].link[0]);
}
else {
rank29 += 4 * sizeof(fmatrix) / sizeof(int32type);
rank31 += 4 * sizeof(fmatrix) / sizeof(int32type);
rank29 %= 29;
rank31 %= 31;
}
}
void r_check(gauge_file *gf, float *max_deviation)
{
/* gf = gauge configuration file structure */
FILE *fp;
gauge_header *gh;
char *filename;
int byterevflag;
off_t offset ; /* File stream pointer */
off_t gauge_check_size; /* Size of gauge configuration checksum record */
off_t coord_list_size; /* Size of coordinate list in bytes */
off_t head_size; /* Size of header plus coordinate list */
off_t checksum_offset; /* Where we put the checksum */
int rcv_rank, rcv_coords;
int destnode;
int i,k;
int x,y,z,t;
int buf_length,where_in_buf;
gauge_check test_gc;
u_int32type *val;
int rank29,rank31;
su3_matrix *lbuf;
su3_matrix work[4];
float deviation;
char myname[] = "r_check";
fp = gf->fp;
gh = gf->header;
filename = gf->filename;
byterevflag = gf->byterevflag;
if(this_node == 0)
{
/* Compute offset for reading gauge configuration */
/* (1996 gauge configuration files had a 32-bit unused checksum
record before the gauge link data) */
if(gh->magic_number == GAUGE_VERSION_NUMBER)
gauge_check_size = sizeof(gf->check.sum29) +
sizeof(gf->check.sum31);
else if(gh->magic_number == GAUGE_VERSION_NUMBER_1996)
gauge_check_size = 4;
else
gauge_check_size = 0;
if(gf->header->order == NATURAL_ORDER)coord_list_size = 0;
else coord_list_size = sizeof(int32type)*volume;
checksum_offset = gf->header->header_bytes + coord_list_size;
head_size = checksum_offset + gauge_check_size;
/* Allocate space for read buffer */
if(gf->parallel)
printf("%s: Attempting serial read from parallel file \n",myname);
lbuf = (su3_matrix *)malloc(MAX_BUF_LENGTH*4*sizeof(su3_matrix));
if(lbuf == NULL)
{
printf("%s: Node %d can't malloc lbuf\n",myname,this_node);
fflush(stdout);
terminate(1);
}
/* Position file for reading gauge configuration */
offset = head_size;
if( g_seek(fp,offset,SEEK_SET) < 0 )
{
printf("%s: Node 0 g_seek %ld failed error %d file %s\n",
myname,(long)offset,errno,filename);
fflush(stdout);terminate(1);
}
buf_length = 0;
where_in_buf = 0;
}
/* all nodes initialize checksums */
test_gc.sum29 = 0;
test_gc.sum31 = 0;
/* counts 32-bit words mod 29 and mod 31 in order of appearance
on file */
/* Here all nodes see the same sequence because we read serially */
rank29 = 0;
rank31 = 0;
*max_deviation = 0;
g_sync();
/* Node 0 reads and deals out the values */
for(rcv_rank=0; rcv_rank<volume; rcv_rank++)
{
/* If file is in coordinate natural order, receiving coordinate
is given by rank. Otherwise, it is found in the table */
if(gf->header->order == NATURAL_ORDER)
rcv_coords = rcv_rank;
else
rcv_coords = gf->rank2rcv[rcv_rank];
x = rcv_coords % nx; rcv_coords /= nx;
y = rcv_coords % ny; rcv_coords /= ny;
z = rcv_coords % nz; rcv_coords /= nz;
t = rcv_coords % nt;
/* The node that gets the next set of gauge links */
destnode=node_number(x,y,z,t);
if(this_node==0){
/* Node 0 fills its buffer, if necessary */
if(where_in_buf == buf_length)
{ /* get new buffer */
/* new buffer length = remaining sites, but never bigger
than MAX_BUF_LENGTH */
buf_length = volume - rcv_rank;
if(buf_length > MAX_BUF_LENGTH)buf_length = MAX_BUF_LENGTH;
/* then do read */
if( (int)g_read(lbuf,4*sizeof(su3_matrix),buf_length,fp) != buf_length)
{
printf("%s: node %d gauge configuration read error %d file %s\n",
myname,this_node,errno,filename);
fflush(stdout); terminate(1);
}
where_in_buf = 0; /* reset counter */
} /*** end of the buffer read ****/
if(destnode==0){ /* just copy links */
i = node_index(x,y,z,t);
memcpy((void *)&work[0],
(void *)&lbuf[4*where_in_buf], 4*sizeof(su3_matrix));
}
else { /* send to correct node */
send_field((char *)&lbuf[4*where_in_buf],
4*sizeof(su3_matrix),destnode);
}
where_in_buf++;
}
/* The node which contains this site reads message */
else { /* for all nodes other than node 0 */
if(this_node==destnode){
i = node_index(x,y,z,t);
get_field((char *)&work[0],4*sizeof(su3_matrix),0);
}
}
/* The receiving node does the byte reversal and then checksum,
if needed */
if(this_node==destnode)
{
if(byterevflag==1)
byterevn((int32type *)&work[0],
4*sizeof(su3_matrix)/sizeof(int32type));
/* Accumulate checksums */
for(k = 0, val = (u_int32type *)&work[0];
k < 4*(int)sizeof(su3_matrix)/(int)sizeof(int32type); k++, val++)
{
test_gc.sum29 ^= (*val)<<rank29 | (*val)>>(32-rank29);
test_gc.sum31 ^= (*val)<<rank31 | (*val)>>(32-rank31);
rank29++; if(rank29 >= 29)rank29 = 0;
rank31++; if(rank31 >= 31)rank31 = 0;
}
deviation = ck_unitarity(work,x,y,z,t);
if(deviation > *max_deviation)*max_deviation = deviation;
}
else
{
rank29 += 4*sizeof(su3_matrix)/sizeof(int32type);
rank31 += 4*sizeof(su3_matrix)/sizeof(int32type);
rank29 %= 29;
rank31 %= 31;
}
}
// -----------------------------------------------------------------
// Only node0 writes the gauge configuration to a binary file gf
static void w_serial(gauge_file *gf) {
register int i, j;
int rank29, rank31, buf_length, tbuf_length;
int x, y, z, t, currentnode, newnode;
FILE *fp = NULL;
gauge_header *gh = NULL;
fmatrix *lbuf = NULL;
fmatrix *tbuf = malloc(nx * 4 * sizeof(*tbuf));
off_t offset; // File stream pointer
off_t coord_list_size; // Size of coordinate list in bytes
off_t head_size; // Size of header plus coordinate list
off_t checksum_offset = 0; // Location of checksum
off_t gauge_check_size; // Size of checksum record
// tbuf holds message buffer space for the x dimension
// of the local hypercube (needs at most 4nx matrices)
if (tbuf == NULL) {
printf("w_serial: node%d can't malloc tbuf\n", this_node);
terminate(1);
}
// Only allocate lbuf on node0
if (this_node == 0) {
if (gf->parallel)
printf("w_serial: Attempting serial write to parallel file\n");
lbuf = malloc(MAX_BUF_LENGTH * 4 * sizeof(*lbuf));
if (lbuf == NULL) {
printf("w_serial: Node 0 can't malloc lbuf\n");
fflush(stdout);
terminate(1);
}
fp = gf->fp;
gh = gf->header;
// No coordinate list written
// Fields to be written in standard coordinate list order
coord_list_size = 0;
head_size = gh->header_bytes + coord_list_size;
checksum_offset = head_size;
gauge_check_size = sizeof(gf->check.sum29) + sizeof(gf->check.sum31);
offset = head_size + gauge_check_size;
if (fseeko(fp, offset, SEEK_SET) < 0) {
printf("w_serial: node%d g_seek %lld failed error %d file %s\n",
this_node, (long long)offset, errno, gf->filename);
fflush(stdout);
terminate(1);
}
}
// Buffered algorithm for writing fields in serial (lexicographic) order
// Initialize checksums
gf->check.sum31 = 0;
gf->check.sum29 = 0;
// Count 32-bit words mod 29 and mod 31 in order of appearance on file
// Here only node 0 uses these values -- both start at 0
i = 4 * sizeof(fmatrix) / sizeof(int32type) * sites_on_node * this_node;
rank29 = i % 29;
rank31 = i % 31;
g_sync();
currentnode = 0; // The node delivering data
buf_length = 0;
tbuf_length = 0;
for (j = 0, t = 0; t < nt; t++) {
for (z = 0; z < nz; z++) {
for (y = 0; y < ny; y++) {
for (x = 0; x < nx; x++, j++) {
// The node providing the next site
newnode = node_number(x, y, z, t);
if (newnode != currentnode || x == 0) {
// We are switching to a new node or have exhausted a line of nx
// Sweep any data in the retiring node's tbuf to the node0 lbuf
if (tbuf_length > 0) {
if (currentnode != 0)
send_buf_to_node0(tbuf, tbuf_length, currentnode);
// node0 flushes tbuf, accumulates checksum
// and writes lbuf if it is full
if (this_node == 0) {
flush_tbuf_to_lbuf(gf, &rank29, &rank31, lbuf, &buf_length,
tbuf, tbuf_length);
if (buf_length > MAX_BUF_LENGTH - nx)
flush_lbuf_to_file(gf, lbuf, &buf_length);
}
tbuf_length = 0;
}
// node0 sends a few bytes to newnode as a clear to send signal
if (newnode != currentnode) {
if (this_node == 0 && newnode != 0)
send_field((char *)tbuf, 32, newnode);
if (this_node == newnode && newnode != 0)
get_field((char *)tbuf, 32, 0);
currentnode = newnode;
}
}
// The node with the data just appends to its tbuf
if (this_node == currentnode) {
i = node_index(x, y, z, t);
d2f_4mat(&lattice[i].link[0], &tbuf[4 * tbuf_length]);
}
if (this_node == currentnode || this_node == 0)
tbuf_length++;
}
}
}
}
// Purge any remaining data
if (tbuf_length > 0) {
if (currentnode != 0)
send_buf_to_node0(tbuf, tbuf_length, currentnode);
}
if (this_node == 0) {
flush_tbuf_to_lbuf(gf, &rank29, &rank31, lbuf, &buf_length,
tbuf, tbuf_length);
flush_lbuf_to_file(gf, lbuf, &buf_length);
}
g_sync();
free(tbuf);
if (this_node == 0) {
free(lbuf);
printf("Saved gauge configuration serially to binary file %s\n",
gf->filename);
printf("Time stamp %s\n", gh->time_stamp);
// Write checksum
// Position file pointer
if (g_seek(fp, checksum_offset, SEEK_SET) < 0) {
printf("w_serial: node%d g_seek %lld failed error %d file %s\n",
this_node, (long long)checksum_offset, errno, gf->filename);
fflush(stdout);
terminate(1);
}
write_checksum(SERIAL, gf);
}
}
