Ejemplo n.º 1
0
/* main test functions */
int create(void)
{
   char buf[4096];
   char *s;
   int i;
   int ret = 1;
   char *spec;
   if ( ret && createDir(testDir) )
      ret = 0;

   if ( totalSize < maxSize )
   {
       maxSize = totalSize;
   }
   int nb = maxSize;
   spec = specToName(0, nb, csv);

   STARTTIME();
   for(i=0;i < nb &&createFileBefore;i++)
   {
      snprintf(buf,sizeof(buf), "%s/f_%d",testDir,i);
      int fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC, 0666);
      if ( fd > 0 )
      {
         close(fd);
      }
      else
      {
         ret = 0;
         break;
      }
   }
   ENDTIME();
   SETDT();
   double createPerSecond = nb*1000000.0/dt;

   fprintf(stdout,"Create %-18s time %8.3f sec %8.0f     F/s\n",
           spec,
           (double)dt/1000000,
           createPerSecond
           );
   STARTTIME();
   for(i=0;i < nb; i++)
   {
      snprintf(buf,sizeof(buf),"%s/f_%d",testDir,i);
      deleteFile(buf);
   }
   ENDTIME();
   SETDT();
   createPerSecond = nb*1000000.0/dt;
   fprintf(stdout,"Delete %-18s time %8.3f sec %8.0f     F/s\n",
           spec,
           (double)dt/1000000,
           createPerSecond
           );
   
   deleteTestDir(1);
   return ret;
}
Ejemplo n.º 2
0
int
pktchan_to_tracebuf( PktChannel *pktchan,
		     TracePacket *tp,
		     double starttime, 
		     int *nbytes )
{
	char	*datap;
	int	dsize_bytes;
	STACHAN *stachan;

	strcpy( tp->trh.datatype, UNSTUFFPKT_DATATYPE );

	if( ( stachan = lookup_stachan( pktchan->sta, pktchan->chan ) ) == NULL )
	{
		logit( "et", "Couldn't find %s %s in station info\n",
			pktchan->sta, pktchan->chan );
		return -1;
	}
	else
	{
		tp->trh.pinno = stachan->pinno;
	}

	strcpy( tp->trh.sta, pktchan->sta );
	strcpy( tp->trh.chan, pktchan->chan );
	strcpy( tp->trh.net, pktchan->net );
	tp->trh.samprate = pktchan->samprate;
	tp->trh.nsamp = pktchan->nsamp;
	tp->trh.starttime = starttime;
	tp->trh.endtime = ENDTIME( starttime,
				   pktchan->samprate,
				   pktchan->nsamp );
	strcpy( tp->trh.quality, "" );
	strcpy( tp->trh.pad, "" );

	datap = &tp->msg[0] + sizeof( TRACE_HEADER );
	dsize_bytes = datasize( tp->trh.datatype ) * tp->trh.nsamp;

	*nbytes = dsize_bytes + sizeof( TRACE_HEADER );

	memcpy( datap, pktchan->data, dsize_bytes );

	return 0;
}
Ejemplo n.º 3
0
int list()
{
   int i = 0;
   int nb = 0;
   int max = 0;
   char buf[4096];

   createDir(testDir);
   if ( stepSize > 0 )
      nb = minSize;

   if ( csv )
      fprintf(stdout,"\"files\",\"first call\",\"second call\",\n");

   while (nb <= maxSize)
   {
      for (; i < nb;i++)
      {
         for(i=0;i < nb;i++)
         {
            snprintf(buf,sizeof(buf), "%s/f_%d",testDir,i);
            createFile(buf);
            /*truncate(buf, i);*/
         }
      }

      /* remount */
      if ( remount )
      {
         system(remount);
      }

      /* get attributes for all files */
      STARTTIME();
      DIR *dir = opendir(testDir);
      if ( dir != NULL )
      {
         struct dirent *ent;
         while(ent=readdir(dir))
         {
            struct stat st;
            snprintf(buf, sizeof(buf),"%s/%s",
                     testDir,ent->d_name);
            stat(buf,&st);
         }
         closedir(dir);
      }
      ENDTIME();
      SETDT();
      if (csv)
         fprintf(stdout,"\"%d,%.6f\"",
                 nb,
                 (double)dt/1000000
                 );
      else
         fprintf(stdout,"List %18d files time %8.3f sec first call\n",
                 nb,
                 (double)dt/1000000
                 );

      /* get attributes for all files */
      STARTTIME();

      dir = opendir(testDir);
      if ( dir != NULL )
      {
         struct dirent *ent;
         while(ent=readdir(dir))
         {
            struct stat st;
            snprintf(buf, sizeof(buf),"%s/%s",
                     testDir,ent->d_name);
            stat(buf,&st);
         }
         closedir(dir);
      }
      ENDTIME();
      SETDT();
      if (csv)
         fprintf(stdout,"\"%.6f\"\n",
                 (double)dt/1000000
                 );
      else
         fprintf(stdout,"List %18d files time %8.3f sec second call\n",
                 nb,
                 (double)dt/1000000
                 );
       nb += stepSize;
   }
   /* cleanup */
   deleteTestDir(1);
   return 0;
}
Ejemplo n.º 4
0
int copy(void)
{
   int   i;
   int   nb;
   char buf[4096];
   char *s;
   char *spec;
   int first = 1;
   off_t actSize = 0;
   int  ret = 1;
   int result;

   if ( totalSize < maxSize )
   {
       maxSize = totalSize;
   }
   
   /* generate source file on client and target dir on server */
   if (numberOp > 0 )
   {
      for (i=0; i < numberOp;i++)
      {
         snprintf(buf,sizeof(buf),"sourceFile_%d",i);
         if ( createFile(buf) == 0)
            ret = 0;
      }
   }
   else
   {
      fprintf(stderr,"Wrong argument!\n");
      return 0;
   }

   if ( ret && createDir(testDir) )
      ret = 0;

   if ( ret && csv )
   {
      printf("\"size\",\"write\",\"read\"\n");
   }

   while (actSize <= maxSize && ret )
   {
      if ( stepType == 'x' )
      {
         if ( first )
         {
            actSize = minSize;
            first = 0;
         }
         else
         {
            actSize *= stepSize;
         }
      }
      else
      {
         if ( first )
         {
            actSize = minSize;
            first = 0;
         }
         else
         {
            actSize += stepSize;
         }
      }

      if ( actSize > maxSize )
         break;

      if ( actSize > 0 )
         nb = totalSize / actSize;
      if ( nb > maxFile )
         nb = maxFile;
      if ( numberOp > 1 )
      {
         int r = nb % numberOp;
         nb -= r;
         if ( nb == 0 )
         {
            break;
         }
      }

      /* set size of source file(s) */
      for (i=0; i < numberOp;i++)
      {
         snprintf(buf,sizeof(buf),"sourceFile_%d",i);
         setFileSize(buf,actSize);
      }


      s = sizeToName(actSize);
      spec = specToName(actSize, nb, csv);

      /* create all target file */
      STARTTIME();
      for(i=0;i < nb &&createFileBefore;i++)
      {
         snprintf(buf,sizeof(buf), "%s/%s_%d",testDir,s,i);
         if ( createFile(buf) == 0 )
         {
            ret = 0;
            break;
         }
      }

      if ( printCreateTime && createFileBefore)
      {
         ENDTIME();
         SETDT();
         int createPerSecond = nb*1000000/dt;
         fprintf(stdout,"Make  %-18s time %8.3f sec %8d     F/s\n",
                 spec,
                 (double)dt/1000000,
                 createPerSecond
                 );
      }
      if ( doSleep && createFileBefore)
      {
         sleep(doSleep);
      }
      
      /*************************/
      /* copy source to target */
      /*************************/
      STARTTIME();
      for ( i=0;i < nb ; i += numberOp )
      {
         if ( numberOp == 1 )
         {
            snprintf(buf,sizeof(buf), "%s/%s_%d",testDir,s,i);
            result = copyFile("sourceFile_0", buf);
            
         }
         else
         {
            snprintf(buf,sizeof(buf), "%s/%s_",testDir,s);
            result = copyNFile("sourceFile_", 0, buf, i, numberOp<nb?numberOp:nb);
         }
         if ( result == 0 )
         {
            ret = 0;
            break;
         }
      }
      ENDTIME();
      SETDT();

      /* end of copy print out results */
      if ( ! csv )
         fprintf(stdout,"Write %-18s time %8.3f sec %12.3f MBps\n",
                 spec,
                 (double)dt/1000000.0,
                 (double)(nb*actSize)/((double)dt/1000000.0)/(1024.0*1024.0)
                 );
      else
          fprintf(stdout,"\"%s\",\"%.3f\",",
                 spec,
                 (double)(nb*actSize)/((double)dt/1000000.0)/(1024.0*1024.0)
                 );

      /*******************/
      /* check file size */
      /*******************/
      for( i=0;i < nb; i++)
      {
         struct stat st;
         snprintf(buf,sizeof(buf), "%s/%s_%d", testDir,s, i);
         if (stat(buf,&st) == -1)
         {
            fprintf(stderr,"File %s not found\n",buf);
            ret = 0;
         }
         else
         {
            if ( st.st_size != actSize )
            {
               fprintf(stderr,"Wrong size (%llu) for file %s\n",st.st_size, buf);
               ret = 0;
            }
         }
      }
      
      if ( remount && ret )
      {
         system(remount);
      }

      /*************************/
      /* copy target to source */
      /*************************/
      STARTTIME();
      for(i=0;i < nb;i += numberOp)
      {
         if ( numberOp == 1 )
         {
            snprintf(buf,sizeof(buf), "%s/%s_%d",testDir,s,i);
            if ( toNull == 0 )
               result = copyFile(buf, "sourceFile_0");
            else
               result = readFile(buf, "sourceFile_0");
         }
         else
         {
            snprintf(buf,sizeof(buf), "%s/%s_",testDir,s);
            result = copyNFile(buf, i, "sourceFile_", 0, numberOp<nb?numberOp:nb);
         }
         if ( result == 0 )
         {
            ret = 0;
            break;
         }
      }
      ENDTIME();

      /*******************/
      /* check file size */
      /*******************/
      for( i=0;i < numberOp && toNull == 0; i++)
      {
         struct stat st;
         snprintf(buf,sizeof(buf), "sourceFile_%d", i);
         if (stat(buf,&st) == -1)
         {
            fprintf(stderr,"File %s not found\n",buf);
            ret = 0;
         }
         else
         {
            if ( st.st_size != actSize )
            {
               fprintf(stderr,"Wrong size (%llu) for file %s\n",st.st_size, buf);
               ret = 0;
            }
         }
      }

      /* print out results */
      SETDT();
      if ( ! csv )
         fprintf(stdout,"Read  %-18s time %8.3f sec %12.3f MBps\n",
                 spec,
                 (double)dt/1000000,
                 (double)(nb*actSize)/((double)dt/1000000)/(1024.0*1024.0)
                 );
      else
          fprintf(stdout,"\"%.3f\"\n",
                 (double)(nb*actSize)/((double)dt/1000000)/(1024.0*1024.0)
                 );
      /* remove files from test directory */
      deleteTestDir(0);
   }

   /* cleanup */
   for (i=0; i < numberOp;i++)
   {
      snprintf(buf,sizeof(buf),"sourceFile_%d",i);
      deleteFile(buf);
   }
   deleteTestDir(1);

   return ret;
}
Ejemplo n.º 5
0
int main(int argc, char *argv[])
{
  int prompt;
  int i, j, iq0, iq1;
#ifdef CLOV_LEAN
  int oldiq0, oldiq1, oldip0;
#endif
  double starttime, endtime;
#ifdef PRTIME
  double dtime;
#endif
  wilson_prop_field *prop[MAX_PROP];
  wilson_prop_field *quark[MAX_QK];
  
  initialize_machine(&argc,&argv);

  /* Remap standard I/O */
  if(remap_stdio_from_args(argc, argv) == 1)terminate(1);
  
  g_sync();
  
  starttime=dclock();
    
  /* set up */
  STARTTIME;
  prompt = setup();
  ENDTIME("setup");

  /* loop over input sets */

  while( readin(prompt) == 0){

    if(prompt == 2)continue;
    
    total_iters=0;
    
#ifdef HISQ_SVD_COUNTER
    hisq_svd_counter = 0;
#endif
    
    /**************************************************************/
    /* Set up gauge field */
    
    if( param.fixflag == COULOMB_GAUGE_FIX)
      {
	if(this_node == 0) 
	  printf("Fixing to Coulomb gauge\n");

	STARTTIME;
	gaugefix(TUP,(Real)1.5,500,GAUGE_FIX_TOL);
	ENDTIME("gauge fix");

	/* (Re)construct APE smeared links after gauge fixing.  
	   No KS phases here! */
	destroy_ape_links_3D(ape_links);
	ape_links = ape_smear_3D( param.staple_weight, param.ape_iter );

	invalidate_this_clov(gen_clov);
      }
    else
      if(this_node == 0)printf("COULOMB GAUGE FIXING SKIPPED.\n");
    
    /* save lattice if requested */
    if( param.saveflag != FORGET ){
      savelat_p = save_lattice( param.saveflag, param.savefile, 
				param.stringLFN );
    } else {
      savelat_p = NULL;
    }

    if(this_node==0)printf("END OF HEADER\n");
    
    /**************************************************************/


    /* Loop over the propagators */

    STARTTIME;
    for(i=0; i<param.num_prop; i++){
      node0_printf("******* Creating propagator %d ********\n",i);fflush(stdout);
      
      /**************************************************************/
      /* Read and/or generate quark propagator */

      if(param.prop_type[i] == CLOVER_TYPE)
	{
	  int ncolor = convert_ksource_to_color(param.src_qs[i].nsource);
	  
	  prop[i] = create_wp_field(ncolor);
      
	  node0_printf("Generate Dirac propagator\n");
	  node0_printf("Kappa= %g source %s residue= %g rel= %g\n",
		       (double)param.dcp[i].Kappa,
		       param.src_qs[i].descrp,
		       (double)param.qic[i].resid,
		       (double)param.qic[i].relresid);
	  
	  /* For clover_info */
	  wqstmp = param.src_qs[i];
	  dcptmp = param.dcp[i];

	  total_iters += get_wprop_to_wp_field(param.prop_type[i],
					       param.startflag_w[i], 
					       param.startfile_w[i], 
					       param.saveflag_w[i], 
					       param.savefile_w[i],
					       prop[i], 
					       &param.src_qs[i],
					       &param.qic[i], 
					       (void *)&param.dcp[i],
					       param.bdry_phase[i],
					       param.coord_origin,
					       param.check[i]);
#ifdef CLOV_LEAN
	  /* Free clover prop memory if we have saved the prop to disk */
	  if(param.saveflag_w[i] != FORGET){
	    free_wp_field(prop[i]);
	    clear_qs(&param.src_qs[i]);
	    node0_printf("destroy prop[%d]\n",i);
	  }
#endif
	}
	
      /* ------------------------------------------- */
      else if(param.prop_type[i] == IFLA_TYPE)
	{
	  int ncolor = convert_ksource_to_color(param.src_qs[i].nsource);
	  

	  prop[i] = create_wp_field(ncolor);
      
	  node0_printf("Generate Dirac IFLA propagator\n");
	  if(this_node==0)printf("Kappa= %g source %s residue= %g rel= %g\n",
				 (double)param.nap[i].kapifla,
				 param.src_qs[i].descrp,
				 (double)param.qic[i].resid,
				 (double)param.qic[i].relresid);
	 
	  /* For clover_info */
	  wqstmp = param.src_qs[i];
	  naptmp = param.nap[i];
	  
	  total_iters += get_wprop_to_wp_field(param.prop_type[i],
					       param.startflag_w[i], 
					       param.startfile_w[i], 
					       param.saveflag_w[i], 
					       param.savefile_w[i],
					       prop[i], 
					       &param.src_qs[i], 
					       &param.qic[i], 
					       (void *)&param.nap[i],
					       param.bdry_phase[i],
					       param.coord_origin,
					       param.check[i]);
#ifdef CLOV_LEAN
	  /* Free clover prop memory if we have saved the prop to disk */
	  if(param.saveflag_w[i] != FORGET){
	    free_wp_field(prop[i]);
	    clear_qs(&param.src_qs[i]);
	    node0_printf("destroy prop[%d]\n",i);
	  }
#endif
	}
      else if(param.prop_type[i] == KS_TYPE || param.prop_type[i] == KS0_TYPE ) /* KS_TYPE */
	{
	  prop[i] = create_wp_field(param.src_qs[i].ncolor);

	  if(this_node==0)printf("Mass= %g source %s residue= %g rel= %g\n",
				 (double)param.ksp[i].mass,
				 param.src_qs[i].descrp,
				 (double)param.qic[i].resid,
				 (double)param.qic[i].relresid);
	  
	  total_iters += get_ksprop_to_wp_field(param.startflag_ks[i], 
						param.startfile_ks[i], 
						param.saveflag_ks[i], 
						param.savefile_ks[i],
						prop[i], 
						&param.src_qs[i],
						&param.qic[i], 
						&param.ksp[i],
						param.bdry_phase[i],
						param.coord_origin,
						param.check[i],
						param.prop_type[i] == KS0_TYPE);
#ifdef CLOV_LEAN
	  /* (We don't free naive prop memory, since we don't save it
	     as a clover prop in get_ksprop_to_wp_field) */
#endif
	}
      else /* KS4_TYPE */
	{
	  prop[i] = create_wp_field(param.src_qs[i].ncolor);

	  if(this_node==0)printf("Mass= %g source %s residue= %g rel= %g\n",
				 (double)param.ksp[i].mass,
				 param.src_qs[i].descrp,
				 (double)param.qic[i].resid,
				 (double)param.qic[i].relresid);
	  
	  total_iters += get_ksprop4_to_wp_field(param.startflag_w[i], 
						 param.startfile_w[i], 
						 param.saveflag_w[i], 
						 param.savefile_w[i],
						 prop[i], 
						 &param.src_qs[i],
						 &param.qic[i], 
						 &param.ksp[i],
						 param.bdry_phase[i],
						 param.coord_origin,
						 param.check[i]);
	}
      
    } /* propagators */
    ENDTIME("compute propagators");

    /*****************************************************************/
    /* Complete the quark propagators by applying the sink operators
       to either the raw propagator or by building on an existing quark
       propagator */
    
    STARTTIME;
#ifdef CLOV_LEAN
    oldip0 = -1;
    oldiq0 = -1;
    oldiq1 = -1;
#endif
    for(j=0; j<param.num_qk; j++){
      node0_printf("******* Creating quark %d ********\n",j); fflush(stdout);
      i = param.prop_for_qk[j];

      if(param.parent_type[j] == PROP_TYPE){
#ifdef CLOV_LEAN
	/* Restore clover prop[i] from file. */
	/* But first destroy the old one, unless we still need it */
	if(oldip0 >= 0 && oldip0 != i)
	  if(param.prop_type[oldip0] == CLOVER_TYPE &&
	     param.saveflag_w[oldip0] != FORGET){
	    free_wp_field(prop[oldip0]);
	    node0_printf("destroy prop[%d]\n",oldip0);
	  }
	
	/* In this case we won't need any old quarks */
	if(oldiq0 >= 0)
	  if(param.saveflag_q[oldiq0] != FORGET){
	    free_wp_field(quark[oldiq0]);
	    node0_printf("destroy quark[%d]\n",oldiq0);
	  }

	if(oldiq1 >= 0)
	  if(param.saveflag_q[oldiq1] != FORGET){
	    free_wp_field(quark[oldiq1]);
	    node0_printf("destroy quark[%d]\n",oldiq1);
	  }

	if(prop[i]->swv[0] == NULL)
	  reread_wprop_to_wp_field(param.saveflag_w[i], param.savefile_w[i], prop[i]);
#endif
	/* Before applying operator, apply momentum twist to
	   ape_links.  Use the momentum twist of the parent
	   propagator */
	momentum_twist_ape_links(i, +1);
	/* Apply sink operator quark[j] <- Op[j] prop[i] */
	quark[j] = create_wp_field_copy(prop[i]);
	wp_sink_op(&param.snk_qs_op[j], quark[j]);
	/* Remove twist */
	momentum_twist_ape_links(i, -1);
#ifdef CLOV_LEAN
	oldip0 = i;
	oldiq0 = -1;
#endif
      }
      else if(param.parent_type[j] == QUARK_TYPE) { /* QUARK_TYPE */
#ifdef CLOV_LEAN
	/* Restore quark[i] from file */
	/* But first destroy the old ones, unless we still need one of them */
	
	/* In this case we won't need the old prop */
	if(oldip0 >= 0)
	   if(param.prop_type[oldip0] == CLOVER_TYPE &&
	      param.saveflag_w[oldip0] != FORGET){
	     free_wp_field(prop[oldip0]);
	     node0_printf("destroy prop[%d]\n",oldip0);
	   }

	if(oldiq0 >= 0 && oldiq0 != i)
	  if(param.saveflag_q[oldiq0] != FORGET){
	    free_wp_field(quark[oldiq0]);
	    node0_printf("destroy quark[%d]\n",oldiq0);
	  }

	if(oldiq1 >= 0 && oldiq1 != i)
	  if(param.saveflag_q[oldiq1] != FORGET){
	    free_wp_field(quark[oldiq1]);
	    node0_printf("destroy quark[%d]\n",oldiq1);
	  }

	if(quark[i]->swv[0] == NULL)
	  reread_wprop_to_wp_field(param.saveflag_q[i], param.savefile_q[i], quark[i]);
	
#endif
	/* Apply sink operator quark[j] <- Op[j] quark[i] */
	momentum_twist_ape_links(i, +1);
	quark[j] = create_wp_field_copy(quark[i]);
	wp_sink_op(&param.snk_qs_op[j], quark[j]);
	momentum_twist_ape_links(i, -1);
#ifdef CLOV_LEAN
	oldip0 = -1;
	oldiq0 = i;
#endif
      } else { /* COMBO_TYPE */
	int k;
	int nc = quark[param.combo_qk_index[j][0]]->nc;
	/* Create a zero field */
	quark[j] = create_wp_field(nc);
	/* Compute the requested linear combination */
	for(k = 0; k < param.num_combo[j]; k++){
	  wilson_prop_field *q = quark[param.combo_qk_index[j][k]];
	  if(nc != q->nc){
	    printf("Error: Attempting to combine an inconsistent number of colors: %d != %d\n",nc, q->nc);
	    terminate(1);
	  }
	  scalar_mult_add_wprop_field(quark[j], q, param.combo_coeff[j][k], quark[j]);
	}
      }
	
      /* Save the resulting quark[j] if requested */
      dump_wprop_from_wp_field( param.saveflag_q[j], param.savetype_q[j],
				param.savefile_q[j], quark[j]);

      /* Can we delete any props and quarks now? */
      /* If nothing later depends on a prop or quark, free it up. */
      for(i = 0; i < param.num_prop; i++)
	if( prop[i]->swv[0] != NULL  &&  param.prop_dep_qkno[i] < j ){
	  free_wp_field(prop[i]);
	  node0_printf("free prop[%d]\n",i);
	}
      
      for(i = 0; i < j; i++)
	if( quark[i]->swv[0] != NULL  &&  param.quark_dep_qkno[i] < j ){
	  free_wp_field(quark[i]);
	  node0_printf("free quark[%d]\n",i);
	}
#ifdef CLOV_LEAN
      oldiq1 = j;
#endif
    }

#ifdef CLOV_LEAN
    /* Free remaining memory */
    if(oldip0 >= 0)
       if(param.prop_type[oldip0] == CLOVER_TYPE &&
	  param.saveflag_w[oldip0] != FORGET){
	 free_wp_field(prop[oldip0]);
	 node0_printf("destroy prop[%d]\n",oldip0);
       }
    
    if(oldiq0 >= 0)
      if(param.saveflag_q[oldiq0] != FORGET){
	free_wp_field(quark[oldiq0]);
	node0_printf("destroy quark[%d]\n",oldiq0);
      }
    
    if(oldiq1 >= 0)
      if(param.saveflag_q[oldiq1] != FORGET){
	free_wp_field(quark[oldiq1]);
	node0_printf("destroy quark[%d]\n",oldiq1);
      }
#endif

    /* Now destroy all remaining propagator fields */

    for(i = 0; i < param.num_prop; i++){
      if(prop[i] != NULL)node0_printf("destroy prop[%d]\n",i);
      destroy_wp_field(prop[i]);
      prop[i] = NULL;
    }
    ENDTIME("generate quarks");
    
    /****************************************************************/
    /* Compute the meson propagators */

    STARTTIME;
    for(i = 0; i < param.num_pair; i++){

      /* Index for the quarks making up this meson */
      iq0 = param.qkpair[i][0];
      iq1 = param.qkpair[i][1];

      node0_printf("Mesons for quarks %d and %d\n",iq0,iq1);

#ifdef CLOV_LEAN
      /* Restore quarks from file and free old memory */
      /* We try to reuse props that are already in memory, so we don't
         destroy them immediately, but wait to see if we need
         them again for the next pair. */
      if(i > 0 && oldiq0 != iq0 && oldiq0 != iq1)
	if(param.saveflag_q[oldiq0] != FORGET){
	  free_wp_field(quark[oldiq0]);
	  node0_printf("destroy quark[%d]\n",oldiq0);
	}
      
      if(i > 0 && oldiq1 != iq0 && oldiq1 != iq1)
	if(param.saveflag_q[oldiq1] != FORGET){
	  free_wp_field(quark[oldiq1]);
	  node0_printf("destroy quark[%d]\n",oldiq1);
	}

      if(quark[iq0]->swv[0] == NULL)
	reread_wprop_to_wp_field(param.saveflag_q[iq0], param.savefile_q[iq0], quark[iq0]);

      if(quark[iq1]->swv[0] == NULL){
	reread_wprop_to_wp_field(param.saveflag_q[iq1], param.savefile_q[iq1], quark[iq1]);
      }
#endif

      /* Tie together to generate hadron spectrum */
      spectrum_cl(quark[iq0], quark[iq1], i);

      /* Remember, in case we need to free memory */
#ifdef CLOV_LEAN
      oldiq0 = iq0;
      oldiq1 = iq1;
#endif
    }
#ifdef CLOV_LEAN
    /* Free any remaining quark prop memory */
    if(quark[oldiq0]->swv[0] != NULL)
      if(param.saveflag_q[oldiq0] != FORGET){
	free_wp_field(quark[oldiq0]);
	node0_printf("destroy quark[%d]\n",oldiq0);
      }
    if(quark[oldiq1]->swv[0] != NULL)
      if(param.saveflag_q[oldiq1] != FORGET){
	free_wp_field(quark[oldiq1]);
	node0_printf("destroy quark[%d]\n",oldiq1);
      }
#endif
    ENDTIME("tie hadron correlators");

    node0_printf("RUNNING COMPLETED\n");
    endtime=dclock();

    node0_printf("Time = %e seconds\n",(double)(endtime-starttime));
    node0_printf("total_iters = %d\n",total_iters);
#ifdef HISQ_SVD_COUNTER
    printf("hisq_svd_counter = %d\n",hisq_svd_counter);
#endif
    fflush(stdout);

    for(i = 0; i < param.num_qk; i++){
      if(quark[i] != NULL)node0_printf("destroy quark[%d]\n",i);
      destroy_wp_field(quark[i]);
      quark[i] = NULL;
    }

    destroy_ape_links_3D(ape_links);

    /* Destroy fermion links (possibly created in make_prop()) */

#if FERM_ACTION == HISQ
    destroy_fermion_links_hisq(fn_links);
#else
    destroy_fermion_links(fn_links);
#endif
    fn_links = NULL;

  } /* readin(prompt) */

#ifdef HAVE_QUDA
  qudaFinalize();
#endif
  normal_exit(0);

  return 0;
}