Exemple #1
0
//  BPW -- this really doesn't have anything to do with "edgemate" but
//  it's commonly called with check_symmetry_of_the_edge_mates(), and
//  they're both disabled.
//
void count_fragment_and_edge_labels(Tfragment frags[],
                                    Tedge     edges[],
                                    char      comment[]) {
  FILE *fout = stderr;

  fprintf(stderr, "count_fragment_and_edge_labels()--  Disabled.\n");
  return;

  const int nsample=500;
  const int nbucket=500;

  IntFragment_ID nfrag = GetNumFragments(frags);
  IntFragment_ID vid;
  Histogram_t *frag_lab_histogram = create_histogram(nsample,nbucket,TRUE,FALSE);

  fprintf(fout,"*** Histogram Fragment Labels <%s> ***\n",comment);

  for(vid=0; vid<nfrag; vid++) {
    const Tlab ilab = get_lab_fragment(frags,vid);
    add_to_histogram(frag_lab_histogram, (int)ilab, NULL);
  }

  fprintf(fout,"Histogram of the fragment label \n");
  print_histogram(fout,frag_lab_histogram, 0, 1);
  free_histogram(frag_lab_histogram);

  IntEdge_ID ie;
  IntEdge_ID nedge = GetNumEdges(edges);
  Histogram_t *inter_chunk_edge_nes_histogram = create_histogram(nsample,nbucket,TRUE,FALSE);
  Histogram_t *intra_chunk_edge_nes_histogram = create_histogram(nsample,nbucket,TRUE,FALSE);

  fprintf(fout,
          "*** Histogram Edge Labels (2 edges/overlap) <%s> ***\n",
          comment);

  for(ie=0; ie<nedge; ie++) {
    const Tnes nes = get_nes_edge(edges,ie);
    const IntFragment_ID avx = get_avx_edge(edges,ie);
    const IntFragment_ID bvx = get_bvx_edge(edges,ie);
    const IntChunk_ID a_cid = get_cid_fragment(frags,avx);
    const IntChunk_ID b_cid = get_cid_fragment(frags,bvx);
    if( a_cid == b_cid ) {
      add_to_histogram(intra_chunk_edge_nes_histogram, (int)nes, NULL);
    } else {
      add_to_histogram(inter_chunk_edge_nes_histogram, (int)nes, NULL);
    }
  }

  fprintf(fout,"Histogram of the inter-chunk overlap labels \n");
  print_histogram(fout,inter_chunk_edge_nes_histogram, 0, 1);
  free_histogram(inter_chunk_edge_nes_histogram);

  fprintf(fout,"Histogram of the intra-chunk overlap labels \n");
  print_histogram(fout,intra_chunk_edge_nes_histogram, 0, 1);
  free_histogram(intra_chunk_edge_nes_histogram);
}
Exemple #2
0
int read_histograms_x (HISTOGRAM **phisto, int nhisto, const long *xcld_ids, int nxcld, IO_BUFFER *iobuf)
{
   int mhisto, ihisto, rc, ncounts;
   char title[256];
   char type, cdummy;
   long ident;
   double rlower[2] = {0., 0.}, rupper[2] = {0., 0.}, 
          rsum[2] = {0., 0.}, rtsum[2] = {0., 0.};
   long ilower[2] = {0,0}, iupper[2] = {0,0}, isum[2] = {0,0}, itsum[2] = {0,0};
   long entries=0, tentries=0, underflow[2] = {0,0}, overflow[2] = {0,0};
   int nbins=0, nbins_2d=0, ibin, mbins[2] = {0,0};
   HISTOGRAM *thisto=NULL, *ohisto=NULL;
   IO_ITEM_HEADER item_header;
   int adding = 0;
   
   if  ( nhisto < 0 )
   {
      adding = 1;
      nhisto = -nhisto;
   }

   if ( iobuf == (IO_BUFFER *) NULL )
      return -1;

   item_header.type = 100;
   if ( (rc = get_item_begin(iobuf,&item_header)) != 0 )
      return rc;

   if ( item_header.version < 1 || item_header.version > 2 )
   {
      Warning("Wrong version no. of histogram data to be read");
      return -1;
   }
// fprintf(stderr,"Read histograms called, with %d histograms excluded\n",nxcld);
   mhisto = get_short(iobuf);

   for (ihisto=0; ihisto<mhisto; ihisto++)
   {
      int add_this = 0, exclude_this = 0;
      type = (char) get_byte(iobuf);
      if ( get_string(title,sizeof(title)-1,iobuf) % 2 == 0 )
         cdummy = get_byte(iobuf); // Compiler may warn about it but this is OK.
      ident = get_long(iobuf);
      nbins = (int) get_short(iobuf);
      nbins_2d = (int) get_short(iobuf);
      entries = (uint32_t) get_long(iobuf);
      tentries = (uint32_t) get_long(iobuf);
      underflow[0] = (uint32_t) get_long(iobuf);
      overflow[0] = (uint32_t) get_long(iobuf);
      if ( type == 'R' || type == 'r' || type == 'F' || type == 'D' )
      {
         rlower[0] = get_real(iobuf);
         rupper[0] = get_real(iobuf);
         rsum[0] = get_real(iobuf);
         rtsum[0] = get_real(iobuf);
      }
      else
      {
         ilower[0] = get_long(iobuf);
         iupper[0] = get_long(iobuf);
         isum[0] = get_long(iobuf);
         itsum[0] = get_long(iobuf);
      }
      if ( nbins_2d > 0 )
      {
         underflow[1] = (uint32_t) get_long(iobuf);
         overflow[1] = (uint32_t) get_long(iobuf);
         if ( type == 'R' || type == 'r' || type == 'F' || type == 'D' )
         {
            rlower[1] = get_real(iobuf);
            rupper[1] = get_real(iobuf);
            rsum[1] = get_real(iobuf);
            rtsum[1] = get_real(iobuf);
         }
         else
         {
            ilower[1] = get_long(iobuf);
            iupper[1] = get_long(iobuf);
            isum[1] = get_long(iobuf);
            itsum[1] = get_long(iobuf);
         }
         ncounts = nbins * nbins_2d;
      }
      else
         ncounts = nbins;

      /* Don't attempt to allocate histograms without data. */
      if ( ncounts <= 0 )
         continue;

      if ( xcld_ids != (const long *) NULL && nxcld > 0 )
      {
         int ixcld;
         for ( ixcld=0; ixcld<nxcld && xcld_ids[ixcld] > 0; ixcld++ )
         {
            if ( xcld_ids[ixcld] == ident )
            {
               exclude_this = 1;
               break;
            }
         }
      }

      /* If the histogram has a numerical identifier delete a */
      /* previously existing histogram with the same identifier. */
      ohisto = NULL;
      if ( ident != 0 )
      {
         if ( (ohisto=get_histogram_by_ident(ident)) !=
               (HISTOGRAM *)NULL )
         {
            if ( adding && ! exclude_this )
               add_this = 1;
            else
               free_histogram(ohisto);
         }
      }

      /* (Re-) Allocate the new histogram according to its type. */
      thisto = NULL;
      // if ( ! exclude_this ) /* Would really exclude all histograms of this ID but we just don't want to add it up */
      {
         if ( nbins_2d > 0 )
         {
            if ( type == 'R' || type == 'r' )
               thisto = alloc_2d_real_histogram(rlower[0],rupper[0],nbins,
                   rlower[1],rupper[1],nbins_2d);
            else if ( type == 'F' || type == 'D' )
            {
               mbins[0] = nbins;
               mbins[1] = nbins_2d;
               thisto = allocate_histogram(&type,2,rlower,rupper,mbins);
            }
            else
               thisto = alloc_2d_int_histogram(ilower[0],iupper[0],nbins,
                   ilower[1],iupper[1],nbins_2d);
         }
         else
         {
            if ( type == 'R' || type == 'r' )
               thisto = alloc_real_histogram(rlower[0],rupper[0],nbins);
            else if ( type == 'F' || type == 'D' )
               thisto = allocate_histogram(&type,1,rlower,rupper,&nbins);
            else
               thisto = alloc_int_histogram(ilower[0],iupper[0],nbins);
         }
      }
      
      /* If the allocation failed or the histogram should be excluded, skip the histogram contents. */
      /* This should guarantee that reading the input doesn't get */
      /* confused when there is not enough memory available to allocate */
      /* a histogram. The drawback is that, so far, there is no failure */
      /* indicator for the caller. */
      if ( thisto == (HISTOGRAM *) NULL )
      {
         if ( type == 'F' || type == 'D' )
            for (ibin=0; ibin<10; ibin++ )
               (void) get_real(iobuf); /* contents... in histogram extension */
         if ( tentries > 0 )
            for (ibin=0; ibin<ncounts; ibin++)
               (void) get_long(iobuf); /* long and real is the same length */
         continue;
      }
      else
         thisto->type = type;

      /* Give the histogram its title and identifier. */
      if ( *title )
         describe_histogram(thisto,title,add_this?0:ident);

#ifdef _REENTRANT
      histogram_lock(thisto);
#endif

      /* Set the values for histogram statistics. */
      thisto->entries = entries;
      thisto->tentries = tentries;
      thisto->underflow = underflow[0];
      thisto->overflow = overflow[0];
      if ( type == 'R' || type == 'r' || type == 'F' || type == 'D' )
      {
         thisto->specific.real.sum = rsum[0];
         thisto->specific.real.tsum = rtsum[0];
      }
      else
      {
         thisto->specific.integer.sum = isum[0];
         thisto->specific.integer.tsum = itsum[0];
      }
      if ( nbins_2d > 0 )
      {
         thisto->underflow_2d = underflow[1];
         thisto->overflow_2d = overflow[1];
         if ( type == 'R' || type == 'r' || type == 'F' || type == 'D' )
         {
            thisto->specific_2d.real.sum = rsum[1];
            thisto->specific_2d.real.tsum = rtsum[1];
         }
         else
         {
            thisto->specific_2d.integer.sum = isum[1];
            thisto->specific_2d.integer.tsum = itsum[1];
         }
      }

      /* If wanted and possible, return the pointer to caller. */
      if ( phisto != (HISTOGRAM **) NULL && ihisto < nhisto )
         phisto[ihisto] = (add_this ? ohisto : thisto);

      /* Finally, read the histogram contents. */
      if ( type == 'F' || type == 'D' )
      {
         struct Histogram_Extension *he = thisto->extension;
         he->content_all = get_real(iobuf);
         he->content_inside = get_real(iobuf);
         get_vector_of_real(he->content_outside,8,iobuf);
         if ( type == 'F' )
         {
            if ( thisto->tentries > 0 )
               for ( ibin=0; ibin<ncounts; ibin++ )
                  he->fdata[ibin] = (float) get_real(iobuf);
            else
               for ( ibin=0; ibin<ncounts; ibin++ )
                  he->fdata[ibin] = (float) 0.;
         }
         else
         {
            if ( thisto->tentries > 0 )
               get_vector_of_real(he->ddata,ncounts,iobuf);
            else
               for ( ibin=0; ibin<ncounts; ibin++ )
                  he->ddata[ibin] = 0.;
         }
      }
      else
      {
         if ( thisto->tentries > 0 )
            get_vector_of_long((long *)thisto->counts,ncounts,iobuf);
         else
            for ( ibin=0; ibin<nbins; ibin++ )
               thisto->counts[ibin] = 0;
      }
#ifdef _REENTRANT
      histogram_unlock(thisto);
#endif

      if ( add_this )
      {
// fprintf(stderr,"Adding histogram ID %ld\n", ident);
         add_histogram(ohisto,thisto);
         free_histogram(thisto);
      }

   }

   if ( (rc = get_item_end(iobuf,&item_header)) != 0 )
      return rc;

   return(mhisto);
}
Exemple #3
0
int main( int argc , char *argv[] )
{
   THD_3dim_dataset *xset , *cset, *mset=NULL ;
   int nopt=1 , method=PEARSON , do_autoclip=0 ;
   int nvox , nvals , ii, jj, kout, kin, polort=1 ;
   int ix1,jy1,kz1, ix2, jy2, kz2 ;
   char *prefix = "degree_centrality" ;
   byte *mask=NULL;
   int   nmask , abuc=1 ;
   int   all_source=0;        /* output all source voxels  25 Jun 2010 [rickr] */
   char str[32] , *cpt ;
   int *imap = NULL ; MRI_vectim *xvectim ;
   float (*corfun)(int,float *,float*) = NULL ;
   /* djc - add 1d file output for similarity matrix */
   FILE *fout1D=NULL;

   /* CC - we will have two subbricks: binary and weighted centrality */
   int nsubbriks = 2;
   int subbrik = 0;
   float * bodset;
   float * wodset;

   int nb_ctr = 0;

   /* CC - added flags for thresholding correlations */
   double thresh = 0.0;
   double othresh = 0.0;
   int dothresh = 0;
   double sparsity = 0.0;
   int dosparsity = 0;
  
   /* variables for calculating degree centrality */
   long * binaryDC = NULL;
   double * weightedDC = NULL;

   /* variables for histogram */
   hist_node_head* histogram=NULL;
   hist_node* hptr=NULL;
   hist_node* pptr=NULL;
   int bottom_node_idx = 0;
   int totNumCor = 0;
   long totPosCor = 0;
   int ngoal = 0;
   int nretain = 0;
   float binwidth = 0.0;
   int nhistnodes = 50;

   /*----*/

   AFNI_SETUP_OMP(0) ;  /* 24 Jun 2013 */

   if( argc < 2 || strcmp(argv[1],"-help") == 0 ){
      printf(
"Usage: 3dDegreeCentrality [options] dset\n"
"  Computes voxelwise weighted and binary degree centrality and\n"
"  stores the result in a new 3D bucket dataset as floats to\n"
"  preserve their values. Degree centrality reflects the strength and\n"
"  extent of the correlation of a voxel with every other voxel in\n"
"  the brain.\n\n"
"  Conceptually the process involves: \n"
"      1. Calculating the correlation between voxel time series for\n"
"         every pair of voxels in the brain (as determined by masking)\n"
"      2. Applying a threshold to the resulting correlations to exclude\n"
"         those that might have arisen by chance, or to sparsify the\n"
"         connectivity graph.\n"
"      3. At each voxel, summarizing its correlation with other voxels\n"
"         in the brain, by either counting the number of voxels correlated\n"
"         with the seed voxel (binary) or by summing the correlation \n"
"         coefficients (weighted).\n"
"   Practically the algorithm is ordered differently to optimize for\n"
"   computational time and memory usage.\n\n"
"   The threshold can be supplied as a correlation coefficient, \n"
"   or a sparsity threshold. The sparsity threshold reflects the fraction\n"
"   of connections that should be retained after the threshold has been\n"
"   applied. To minimize resource consumption, using a sparsity threshold\n"
"   involves a two-step procedure. In the first step, a correlation\n"
"   coefficient threshold is applied to substantially reduce the number\n"
"   of correlations. Next, the remaining correlations are sorted and a\n"
"   threshold is calculated so that only the specified fraction of \n"
"   possible correlations are above threshold. Due to ties between\n"
"   correlations, the fraction of correlations that pass the sparsity\n"
"   threshold might be slightly more than the number specified.\n\n"
"   Regardless of the thresholding procedure employed, negative \n"
"   correlations are excluded from the calculations.\n" 
"\n"
"Options:\n"
"  -pearson  = Correlation is the normal Pearson (product moment)\n"
"               correlation coefficient [default].\n"
   #if 0
"  -spearman = Correlation is the Spearman (rank) correlation\n"
"               coefficient.\n"
"  -quadrant = Correlation is the quadrant correlation coefficient.\n"
   #else
"  -spearman AND -quadrant are disabled at this time :-(\n"
   #endif
"\n"
"  -thresh r = exclude correlations <= r from calculations\n"
"  -sparsity s = only use top s percent of correlations in calculations\n"
"                s should be an integer between 0 and 100. Uses an\n"
"                an adaptive thresholding procedure to reduce memory.\n"
"                The speed of determining the adaptive threshold can\n"
"                be improved by specifying an initial threshold with\n"
"                the -thresh flag.\n"
"\n"
"  -polort m = Remove polynomical trend of order 'm', for m=-1..3.\n"
"               [default is m=1; removal is by least squares].\n"
"               Using m=-1 means no detrending; this is only useful\n"
"               for data/information that has been pre-processed.\n"
"\n"
"  -autoclip = Clip off low-intensity regions in the dataset,\n"
"  -automask =  so that the correlation is only computed between\n"
"               high-intensity (presumably brain) voxels.  The\n"
"               mask is determined the same way that 3dAutomask works.\n"
"\n"
"  -mask mmm = Mask to define 'in-brain' voxels. Reducing the number\n"
"               the number of voxels included in the calculation will\n"
"               significantly speedup the calculation. Consider using\n"
"               a mask to constrain the calculations to the grey matter\n"
"               rather than the whole brain. This is also preferrable\n"
"               to using -autoclip or -automask.\n"
"\n"
"  -prefix p = Save output into dataset with prefix 'p', this file will\n"
"               contain bricks for both 'weighted' or 'degree' centrality\n"
"               [default prefix is 'deg_centrality'].\n"
"\n"
"  -out1D f = Save information about the above threshold correlations to\n"
"              1D file 'f'. Each row of this file will contain:\n"
"               Voxel1 Voxel2 i1 j1 k1 i2 j2 k2 Corr\n"
"              Where voxel1 and voxel2 are the 1D indices of the pair of\n"
"              voxels, i j k correspond to their 3D coordinates, and Corr\n"
"              is the value of the correlation between the voxel time courses.\n" 
"\n"
"Notes:\n"
" * The output dataset is a bucket type of floats.\n"
" * The program prints out an estimate of its memory used\n"
"    when it ends.  It also prints out a progress 'meter'\n"
"    to keep you pacified.\n"
"\n"
"-- RWCox - 31 Jan 2002 and 16 Jul 2010\n"
"-- Cameron Craddock - 26 Sept 2015 \n"
            ) ;
      PRINT_AFNI_OMP_USAGE("3dDegreeCentrality",NULL) ;
      PRINT_COMPILE_DATE ; exit(0) ;
   }

   mainENTRY("3dDegreeCentrality main"); machdep(); PRINT_VERSION("3dDegreeCentrality");
   AFNI_logger("3dDegreeCentrality",argc,argv);

   /*-- option processing --*/

   while( nopt < argc && argv[nopt][0] == '-' ){

      if( strcmp(argv[nopt],"-time") == 0 ){
         abuc = 0 ; nopt++ ; continue ;
      }

      if( strcmp(argv[nopt],"-autoclip") == 0 ||
          strcmp(argv[nopt],"-automask") == 0   ){

         do_autoclip = 1 ; nopt++ ; continue ;
      }

      if( strcmp(argv[nopt],"-mask") == 0 ){
         mset = THD_open_dataset(argv[++nopt]);
         CHECK_OPEN_ERROR(mset,argv[nopt]);
         nopt++ ; continue ;
      }

      if( strcmp(argv[nopt],"-pearson") == 0 ){
         method = PEARSON ; nopt++ ; continue ;
      }

#if 0
      if( strcmp(argv[nopt],"-spearman") == 0 ){
         method = SPEARMAN ; nopt++ ; continue ;
      }

      if( strcmp(argv[nopt],"-quadrant") == 0 ){
         method = QUADRANT ; nopt++ ; continue ;
      }
#endif

      if( strcmp(argv[nopt],"-eta2") == 0 ){
         method = ETA2 ; nopt++ ; continue ;
      }

      if( strcmp(argv[nopt],"-prefix") == 0 ){
         prefix = strdup(argv[++nopt]) ;
         if( !THD_filename_ok(prefix) ){
            ERROR_exit("Illegal value after -prefix!") ;
         }
         nopt++ ; continue ;
      }

      if( strcmp(argv[nopt],"-thresh") == 0 ){
         double val = (double)strtod(argv[++nopt],&cpt) ;
         if( *cpt != '\0' || val >= 1.0 || val < 0.0 ){
            ERROR_exit("Illegal value (%f) after -thresh!", val) ;
         }
         dothresh = 1;
         thresh = val ; othresh = val ; nopt++ ; continue ;
      }
      if( strcmp(argv[nopt],"-sparsity") == 0 ){
         double val = (double)strtod(argv[++nopt],&cpt) ;
         if( *cpt != '\0' || val > 100 || val <= 0 ){
            ERROR_exit("Illegal value (%f) after -sparsity!", val) ;
         }
         if( val > 5.0 )
         {
             WARNING_message("Sparsity %3.2f%% is large and will require alot of memory and time, consider using a smaller value. ", val);
         }
         dosparsity = 1 ;
         sparsity = val ; nopt++ ; continue ;
      }
      if( strcmp(argv[nopt],"-polort") == 0 ){
         int val = (int)strtod(argv[++nopt],&cpt) ;
         if( *cpt != '\0' || val < -1 || val > 3 ){
            ERROR_exit("Illegal value after -polort!") ;
         }
         polort = val ; nopt++ ; continue ;
      }
      if( strcmp(argv[nopt],"-mem_stat") == 0 ){
         MEM_STAT = 1 ; nopt++ ; continue ;
      }
      if( strncmp(argv[nopt],"-mem_profile",8) == 0 ){
         MEM_PROF = 1 ; nopt++ ; continue ;
      }
      /* check for 1d argument */
      if ( strcmp(argv[nopt],"-out1D") == 0 ){
          if (!(fout1D = fopen(argv[++nopt], "w"))) {
             ERROR_message("Failed to open %s for writing", argv[nopt]);
             exit(1);
          }
          nopt++ ; continue ;
      }

      ERROR_exit("Illegal option: %s",argv[nopt]) ;
   }

   /*-- open dataset, check for legality --*/

   if( nopt >= argc ) ERROR_exit("Need a dataset on command line!?") ;

   xset = THD_open_dataset(argv[nopt]); CHECK_OPEN_ERROR(xset,argv[nopt]);


   if( DSET_NVALS(xset) < 3 )
     ERROR_exit("Input dataset %s does not have 3 or more sub-bricks!",argv[nopt]) ;
   DSET_load(xset) ; CHECK_LOAD_ERROR(xset) ;

   /*-- compute mask array, if desired --*/
   nvox = DSET_NVOX(xset) ; nvals = DSET_NVALS(xset) ;
   INC_MEM_STATS((nvox * nvals * sizeof(double)), "input dset");
   PRINT_MEM_STATS("inset");

   /* if a mask was specified make sure it is appropriate */
   if( mset ){

      if( DSET_NVOX(mset) != nvox )
         ERROR_exit("Input and mask dataset differ in number of voxels!") ;
      mask  = THD_makemask(mset, 0, 1.0, 0.0) ;

      /* update running memory statistics to reflect loading the image */
      INC_MEM_STATS( mset->dblk->total_bytes, "mask dset" );
      PRINT_MEM_STATS( "mset load" );

      nmask = THD_countmask( nvox , mask ) ;
      INC_MEM_STATS( nmask * sizeof(byte), "mask array" );
      PRINT_MEM_STATS( "mask" );

      INFO_message("%d voxels in -mask dataset",nmask) ;
      if( nmask < 2 ) ERROR_exit("Only %d voxels in -mask, exiting...",nmask);

      /* update running memory statistics to reflect loading the image */
      DEC_MEM_STATS( mset->dblk->total_bytes, "mask dset" );
      DSET_unload(mset) ;
      PRINT_MEM_STATS( "mset unload" );
   } 
   /* if automasking is requested, handle that now */
   else if( do_autoclip ){
      mask  = THD_automask( xset ) ;
      nmask = THD_countmask( nvox , mask ) ;
      INFO_message("%d voxels survive -autoclip",nmask) ;
      if( nmask < 2 ) ERROR_exit("Only %d voxels in -automask!",nmask);
   }
   /* otherwise we use all of the voxels in the image */
   else {
      nmask = nvox ;
      INFO_message("computing for all %d voxels",nmask) ;
   }
   
   if( method == ETA2 && polort >= 0 )
      WARNING_message("Polort for -eta2 should probably be -1...");

    /* djc - 1d file out init */
    if (fout1D != NULL) {
        /* define affine matrix */
        mat44 affine_mat = xset->daxes->ijk_to_dicom;

        /* print command line statement */
        fprintf(fout1D,"#Similarity matrix from command:\n#");
        for(ii=0; ii<argc; ++ii) fprintf(fout1D,"%s ", argv[ii]);

        /* Print affine matrix */
        fprintf(fout1D,"\n");
        fprintf(fout1D,"#[ ");
        int mi, mj;
        for(mi = 0; mi < 4; mi++) {
            for(mj = 0; mj < 4; mj++) {
                fprintf(fout1D, "%.6f ", affine_mat.m[mi][mj]);
            }
        }
        fprintf(fout1D, "]\n");

        /* Print image extents*/
        THD_dataxes *xset_daxes = xset->daxes;
        fprintf(fout1D, "#Image dimensions:\n");
        fprintf(fout1D, "#[%d, %d, %d]\n",
                xset_daxes->nxx, xset_daxes->nyy, xset_daxes->nzz);

        /* Similarity matrix headers */
        fprintf(fout1D,"#Voxel1 Voxel2 i1 j1 k1 i2 j2 k2 Corr\n");
    }


   /* CC calculate the total number of possible correlations, will be 
       usefule down the road */
   totPosCor = (.5*((float)nmask))*((float)(nmask-1));

   /**  For the case of Pearson correlation, we make sure the  **/
   /**  data time series have their mean removed (polort >= 0) **/
   /**  and are normalized, so that correlation = dot product, **/
   /**  and we can use function zm_THD_pearson_corr for speed. **/

   switch( method ){
     default:
     case PEARSON: corfun = zm_THD_pearson_corr ; break ;
     case ETA2:    corfun = my_THD_eta_squared  ; break ;
   }

   /*-- create vectim from input dataset --*/
   INFO_message("vectim-izing input dataset") ;

   /*-- CC added in mask to reduce the size of xvectim -- */
   xvectim = THD_dset_to_vectim( xset , mask , 0 ) ;
   if( xvectim == NULL ) ERROR_exit("Can't create vectim?!") ;

   /*-- CC update our memory stats to reflect vectim -- */
   INC_MEM_STATS((xvectim->nvec*sizeof(int)) +
                       ((xvectim->nvec)*(xvectim->nvals))*sizeof(float) +
                       sizeof(MRI_vectim), "vectim");
   PRINT_MEM_STATS( "vectim" );

   /*--- CC the vectim contains a mapping between voxel index and mask index, 
         tap into that here to avoid duplicating memory usage ---*/

   if( mask != NULL )
   {
       imap = xvectim->ivec;

       /* --- CC free the mask */
       DEC_MEM_STATS( nmask*sizeof(byte), "mask array" );
       free(mask); mask=NULL;
       PRINT_MEM_STATS( "mask unload" );
   }

   /* -- CC unloading the dataset to reduce memory usage ?? -- */
   DEC_MEM_STATS((DSET_NVOX(xset) * DSET_NVALS(xset) * sizeof(double)), "input dset");
   DSET_unload(xset) ;
   PRINT_MEM_STATS("inset unload");

   /* -- CC configure detrending --*/
   if( polort < 0 && method == PEARSON ){
     polort = 0; WARNING_message("Pearson correlation always uses polort >= 0");
   }
   if( polort >= 0 ){
     for( ii=0 ; ii < xvectim->nvec ; ii++ ){  /* remove polynomial trend */
       DETREND_polort(polort,nvals,VECTIM_PTR(xvectim,ii)) ;
     }
   }


   /* -- this procedure does not change time series that have zero variance -- */
   if( method == PEARSON ) THD_vectim_normalize(xvectim) ;  /* L2 norm = 1 */

    /* -- CC create arrays to hold degree and weighted centrality while
          they are being calculated -- */
    if( dosparsity == 0 )
    {
        if( ( binaryDC = (long*)calloc( nmask, sizeof(long) )) == NULL )
        {
            ERROR_message( "Could not allocate %d byte array for binary DC calculation\n",
                nmask*sizeof(long)); 
        }

        /* -- update running memory estimate to reflect memory allocation */ 
        INC_MEM_STATS( nmask*sizeof(long), "binary DC array" );
        PRINT_MEM_STATS( "binaryDC" );

        if( ( weightedDC = (double*)calloc( nmask, sizeof(double) )) == NULL )
        {
            if (binaryDC){ free(binaryDC); binaryDC = NULL; }
            ERROR_message( "Could not allocate %d byte array for weighted DC calculation\n",
                nmask*sizeof(double)); 
        }
        /* -- update running memory estimate to reflect memory allocation */ 
        INC_MEM_STATS( nmask*sizeof(double), "weighted DC array" );
        PRINT_MEM_STATS( "weightedDC" );
    }


    /* -- CC if we are using a sparsity threshold, build a histogram to calculate the 
         threshold */
    if (dosparsity == 1)
    {
        /* make sure that there is a bin for correlation values that == 1.0 */
        binwidth = (1.005-thresh)/nhistnodes;

        /* calculate the number of correlations we wish to retain */
        ngoal = nretain = (int)(((double)totPosCor)*((double)sparsity) / 100.0);

        /* allocate memory for the histogram bins */
        if(( histogram = (hist_node_head*)malloc(nhistnodes*sizeof(hist_node_head))) == NULL )
        {
            /* if the allocation fails, free all memory and exit */
            if (binaryDC){ free(binaryDC); binaryDC = NULL; }
            if (weightedDC){ free(weightedDC); weightedDC = NULL; }
            ERROR_message( "Could not allocate %d byte array for histogram\n",
                nhistnodes*sizeof(hist_node_head)); 
        }
        else {
            /* -- update running memory estimate to reflect memory allocation */ 
            INC_MEM_STATS( nhistnodes*sizeof(hist_node_head), "hist bins" );
            PRINT_MEM_STATS( "hist1" );
        }

        /* initialize history bins */
        for( kout = 0; kout < nhistnodes; kout++ )
        {
            histogram[ kout ].bin_low = thresh+kout*binwidth;
            histogram[ kout ].bin_high = histogram[ kout ].bin_low+binwidth;
            histogram[ kout ].nbin = 0;
            histogram[ kout ].nodes = NULL; 
            /*INFO_message("Hist bin %d [%3.3f, %3.3f) [%d, %p]\n",
                kout, histogram[ kout ].bin_low, histogram[ kout ].bin_high,
                histogram[ kout ].nbin, histogram[ kout ].nodes );*/
        }
    }

    /*-- tell the user what we are about to do --*/
    if (dosparsity == 0 )
    {
        INFO_message( "Calculating degree centrality with threshold = %f.\n", thresh);
    }
    else
    {
        INFO_message( "Calculating degree centrality with threshold = %f and sparsity = %3.2f%% (%d)\n",
            thresh, sparsity, nretain);
    }

    /*---------- loop over mask voxels, correlate ----------*/
    AFNI_OMP_START ;
#pragma omp parallel if( nmask > 999 )
    {
       int lii,ljj,lin,lout,ithr,nthr,vstep,vii ;
       float *xsar , *ysar ;
       hist_node* new_node = NULL ;
       hist_node* tptr = NULL ;
       hist_node* rptr = NULL ;
       int new_node_idx = 0;
       double car = 0.0 ; 

       /*-- get information about who we are --*/
#ifdef USE_OMP
       ithr = omp_get_thread_num() ;
       nthr = omp_get_num_threads() ;
       if( ithr == 0 ) INFO_message("%d OpenMP threads started",nthr) ;
#else
       ithr = 0 ; nthr = 1 ;
#endif

       /*-- For the progress tracker, we want to print out 50 numbers,
            figure out a number of loop iterations that will make this easy */
       vstep = (int)( nmask / (nthr*50.0f) + 0.901f ) ; vii = 0 ;
       if((MEM_STAT==0) && (ithr == 0 )) fprintf(stderr,"Looping:") ;

#pragma omp for schedule(static, 1)
       for( lout=0 ; lout < xvectim->nvec ; lout++ ){  /*----- outer voxel loop -----*/

          if( ithr == 0 && vstep > 2 ) /* allow small dsets 16 Jun 2011 [rickr] */
          { vii++ ; if( vii%vstep == vstep/2 && MEM_STAT == 0 ) vstep_print(); }

          /* get ref time series from this voxel */
          xsar = VECTIM_PTR(xvectim,lout) ;

          /* try to make calculation more efficient by only calculating the unique 
             correlations */
          for( lin=(lout+1) ; lin < xvectim->nvec ; lin++ ){  /*----- inner loop over voxels -----*/

             /* extract the voxel time series */
             ysar = VECTIM_PTR(xvectim,lin) ;

             /* now correlate the time series */
             car = (double)(corfun(nvals,xsar,ysar)) ;

             if ( car <= thresh )
             {
                 continue ;
             }

/* update degree centrality values, hopefully the pragma
   will handle mutual exclusion */
#pragma omp critical(dataupdate)
             {
                 /* if the correlation is less than threshold, ignore it */
                 if ( car > thresh )
                 {
                     totNumCor += 1;
               
                     if ( dosparsity == 0 )
                     { 
                         binaryDC[lout] += 1; binaryDC[lin] += 1;
                         weightedDC[lout] += car; weightedDC[lin] += car;

                         /* print correlation out to the 1D file */
                         if ( fout1D != NULL )
                         {
                             /* determine the i,j,k coords */
                             ix1 = DSET_index_to_ix(xset,lii) ;
                             jy1 = DSET_index_to_jy(xset,lii) ;
                             kz1 = DSET_index_to_kz(xset,lii) ;
                             ix2 = DSET_index_to_ix(xset,ljj) ;
                             jy2 = DSET_index_to_jy(xset,ljj) ;
                             kz2 = DSET_index_to_kz(xset,ljj) ;
                             /* add source, dest, correlation to 1D file */
                             fprintf(fout1D, "%d %d %d %d %d %d %d %d %.6f\n",
                                lii, ljj, ix1, jy1, kz1, ix2, jy2, kz2, car);
                        }
                    }
                    else
                    {
                        /* determine the index in the histogram to add the node */
                        new_node_idx = (int)floor((double)(car-othresh)/(double)binwidth);
                        if ((new_node_idx > nhistnodes) || (new_node_idx < bottom_node_idx))
                        {
                            /* this error should indicate a programming error and should not happen */
                            WARNING_message("Node index %d is out of range [%d,%d)!",new_node_idx,
                            bottom_node_idx, nhistnodes);
                        }
                        else
                        {
                            /* create a node to add to the histogram */
                            new_node = (hist_node*)calloc(1,sizeof(hist_node));
                            if( new_node == NULL )
                            {
                                /* allocate memory for this node, rather than fiddling with 
                                   error handling here, lets just move on */
                                WARNING_message("Could not allocate a new node!");
                            }
                            else
                            {
                 
                                /* populate histogram node */
                                new_node->i = lout; 
                                new_node->j = lin;
                                new_node->corr = car;
                                new_node->next = NULL;

                                /* -- update running memory estimate to reflect memory allocation */ 
                                INC_MEM_STATS( sizeof(hist_node), "hist nodes" );
                                if ((totNumCor % (1024*1024)) == 0) PRINT_MEM_STATS( "hist nodes" );

                                /* populate histogram */
                                new_node->next = histogram[new_node_idx].nodes;
                                histogram[new_node_idx].nodes = new_node;
                                histogram[new_node_idx].nbin++; 

                                /* see if there are enough correlations in the histogram
                                   for the sparsity */
                                if ((totNumCor - histogram[bottom_node_idx].nbin) > nretain)
                                { 
                                    /* delete the list of nodes */
                                    rptr = histogram[bottom_node_idx].nodes;
                                    while(rptr != NULL)
                                    {
                                        tptr = rptr;
                                        rptr = rptr->next;
                                        /* check that the ptr is not null before freeing it*/
                                        if(tptr!= NULL)
                                        {
                                            DEC_MEM_STATS( sizeof(hist_node), "hist nodes" );
                                            free(tptr);
                                        }
                                    }
                                    PRINT_MEM_STATS( "unloaded hist nodes - thresh increase" );

                                    histogram[bottom_node_idx].nodes = NULL;
                                    totNumCor -= histogram[bottom_node_idx].nbin;
                                    histogram[bottom_node_idx].nbin=0;
 
                                    /* get the new threshold */
                                    thresh = (double)histogram[++bottom_node_idx].bin_low;
                                    if(MEM_STAT == 1) INFO_message("Increasing threshold to %3.2f (%d)\n",
                                        thresh,bottom_node_idx); 
                                }

                            } /* else, newptr != NULL */
                        } /* else, new_node_idx in range */
                    } /* else, do_sparsity == 1 */
                 } /* car > thresh */
             } /* this is the end of the critical section */
          } /* end of inner loop over voxels */
       } /* end of outer loop over ref voxels */

       if( ithr == 0 ) fprintf(stderr,".\n") ;

    } /* end OpenMP */
    AFNI_OMP_END ;

    /* update the user so that they know what we are up to */
    INFO_message ("AFNI_OMP finished\n");
    INFO_message ("Found %d (%3.2f%%) correlations above threshold (%f)\n",
       totNumCor, 100.0*((float)totNumCor)/((float)totPosCor), thresh);

   /*----------  Finish up ---------*/

   /*if( dosparsity == 1 )
   {
       for( kout = 0; kout < nhistnodes; kout++ )
       {
           INFO_message("Hist bin %d [%3.3f, %3.3f) [%d, %p]\n",
                kout, histogram[ kout ].bin_low, histogram[ kout ].bin_high,
                histogram[ kout ].nbin, histogram[ kout ].nodes );
       }
   }*/

   /*-- create output dataset --*/
   cset = EDIT_empty_copy( xset ) ;

   /*-- configure the output dataset */
   if( abuc ){
     EDIT_dset_items( cset ,
                        ADN_prefix    , prefix         ,
                        ADN_nvals     , nsubbriks      , /* 2 subbricks, degree and weighted centrality */
                        ADN_ntt       , 0              , /* no time axis */
                        ADN_type      , HEAD_ANAT_TYPE ,
                        ADN_func_type , ANAT_BUCK_TYPE ,
                        ADN_datum_all , MRI_float      ,
                      ADN_none ) ;
   } else {
     EDIT_dset_items( cset ,
                        ADN_prefix    , prefix         ,
                        ADN_nvals     , nsubbriks      , /* 2 subbricks, degree and weighted centrality */
                        ADN_ntt       , nsubbriks      ,  /* num times */
                        ADN_ttdel     , 1.0            ,  /* fake TR */
                        ADN_nsl       , 0              ,  /* no slice offsets */
                        ADN_type      , HEAD_ANAT_TYPE ,
                        ADN_func_type , ANAT_EPI_TYPE  ,
                        ADN_datum_all , MRI_float      ,
                      ADN_none ) ;
   }

   /* add history information to the hearder */
   tross_Make_History( "3dDegreeCentrality" , argc,argv , cset ) ;

   ININFO_message("creating output dataset in memory") ;

   /* -- Configure the subbriks: Binary Degree Centrality */
   subbrik = 0;
   EDIT_BRICK_TO_NOSTAT(cset,subbrik) ;                     /* stat params  */
   /* CC this sets the subbrik scaling factor, which we will probably want
      to do again after we calculate the voxel values */
   EDIT_BRICK_FACTOR(cset,subbrik,1.0) ;                 /* scale factor */

   sprintf(str,"Binary Degree Centrality") ;

   EDIT_BRICK_LABEL(cset,subbrik,str) ;
   EDIT_substitute_brick(cset,subbrik,MRI_float,NULL) ;   /* make array   */


   /* copy measure data into the subbrik */
   bodset = DSET_ARRAY(cset,subbrik);
 
   /* -- Configure the subbriks: Weighted Degree Centrality */
   subbrik = 1;
   EDIT_BRICK_TO_NOSTAT(cset,subbrik) ;                     /* stat params  */
   /* CC this sets the subbrik scaling factor, which we will probably want
      to do again after we calculate the voxel values */
   EDIT_BRICK_FACTOR(cset,subbrik,1.0) ;                 /* scale factor */

   sprintf(str,"Weighted Degree Centrality") ;

   EDIT_BRICK_LABEL(cset,subbrik,str) ;
   EDIT_substitute_brick(cset,subbrik,MRI_float,NULL) ;   /* make array   */

   /* copy measure data into the subbrik */
   wodset = DSET_ARRAY(cset,subbrik);

   /* increment memory stats */
   INC_MEM_STATS( (DSET_NVOX(cset)*DSET_NVALS(cset)*sizeof(float)), "output dset");
   PRINT_MEM_STATS( "outset" );

   /* pull the values out of the histogram */
   if( dosparsity == 0 )
   {
       for( kout = 0; kout < nmask; kout++ )
       {
          if ( imap != NULL )
          {
              ii = imap[kout] ;  /* ii= source voxel (we know that ii is in the mask) */
          }
          else
          {
              ii = kout ;
          }
   
          if( ii >= DSET_NVOX(cset) )
          {
              WARNING_message("Avoiding bodset, wodset overflow %d > %d (%s,%d)\n",
                  ii,DSET_NVOX(cset),__FILE__,__LINE__ );
          }
          else
          {
              bodset[ ii ] = (float)(binaryDC[kout]);
              wodset[ ii ] = (float)(weightedDC[kout]);
          }
       }

       /* we are done with this memory, and can kill it now*/
       if(binaryDC)
       {
           free(binaryDC);
           binaryDC=NULL;
           /* -- update running memory estimate to reflect memory allocation */ 
           DEC_MEM_STATS( nmask*sizeof(long), "binary DC array" );
           PRINT_MEM_STATS( "binaryDC" );
       }
       if(weightedDC)
       {
           free(weightedDC);
           weightedDC=NULL;
           /* -- update running memory estimate to reflect memory allocation */ 
           DEC_MEM_STATS( nmask*sizeof(double), "weighted DC array" );
           PRINT_MEM_STATS( "weightedDC" );
       }
   }
   else
   {

       /* add in the values from the histogram, this is a two stage procedure:
             at first we add in values a whole bin at the time until we get to a point
             where we need to add in a partial bin, then we create a new histogram
             to sort the values in the bin and then add those bins at a time */
       kout = nhistnodes - 1;
       while (( histogram[kout].nbin < nretain ) && ( kout >= 0 ))
       {
           hptr = pptr = histogram[kout].nodes;
           while( hptr != NULL )
           {

               /* determine the indices corresponding to this node */
               if ( imap != NULL )
               {
                   ii = imap[hptr->i] ;  /* ii= source voxel (we know that ii is in the mask) */
               }
               else 
               {
                   ii = hptr->i ;
               }
               if ( imap != NULL )
               {
                   jj = imap[hptr->j] ;  /* ii= source voxel (we know that ii is in the mask) */
               }
               else
               {
                   jj = hptr->j ;
               }

               /* add in the values */
               if(( ii >= DSET_NVOX(cset) ) || ( jj >= DSET_NVOX(cset)))
               {
                   if( ii >= DSET_NVOX(cset))
                   {
                       WARNING_message("Avoiding bodset, wodset overflow (ii) %d > %d\n (%s,%d)\n",
                           ii,DSET_NVOX(cset),__FILE__,__LINE__ );
                   }
                   if( jj >= DSET_NVOX(cset))
                   {
                       WARNING_message("Avoiding bodset, wodset overflow (jj) %d > %d\n (%s,%d)\n",
                           jj,DSET_NVOX(cset),__FILE__,__LINE__ );
                   }
               }
               else
               {
                   bodset[ ii ] += 1.0 ;
                   wodset[ ii ] += (float)(hptr->corr);
                   bodset[ jj ] += 1.0 ;
                   wodset[ jj ] += (float)(hptr->corr);
               }

               if( fout1D != NULL )
               {
                   /* add source, dest, correlation to 1D file */
                   ix1 = DSET_index_to_ix(cset,ii) ;
                   jy1 = DSET_index_to_jy(cset,ii) ;
                   kz1 = DSET_index_to_kz(cset,ii) ;
                   ix2 = DSET_index_to_ix(cset,jj) ;
                   jy2 = DSET_index_to_jy(cset,jj) ;
                   kz2 = DSET_index_to_kz(cset,jj) ;
                   fprintf(fout1D, "%d %d %d %d %d %d %d %d %.6f\n",
                           ii, jj, ix1, jy1, kz1, ix2, jy2, kz2, (float)(hptr->corr));
               }

               /* increment node pointers */
               pptr = hptr;
               hptr = hptr->next;

               /* delete the node */
               if(pptr)
               {
                   /* -- update running memory estimate to reflect memory allocation */ 
                   DEC_MEM_STATS(sizeof( hist_node ), "hist nodes" );
                   /* free the mem */
                   free(pptr);
                   pptr=NULL;
               }
           } 
           /* decrement the number of correlations we wish to retain */
           nretain -= histogram[kout].nbin;
           histogram[kout].nodes = NULL;

           /* go on to the next bin */
           kout--;
       }
       PRINT_MEM_STATS( "hist1 bins free - inc into output" );

        /* if we haven't used all of the correlations that are available, go through and 
           add a subset of the voxels from the remaining bin */
        if(( nretain > 0 ) && (kout >= 0))
        {

            hist_node_head* histogram2 = NULL; 
            hist_node_head* histogram2_save = NULL; 
            int h2nbins = 100;
            float h2binwidth = 0.0;
            int h2ndx=0;

            h2binwidth = (((1.0+binwidth/((float)h2nbins))*histogram[kout].bin_high) - histogram[kout].bin_low) /
               ((float)h2nbins);

            /* allocate the bins */
            if(( histogram2 = (hist_node_head*)malloc(h2nbins*sizeof(hist_node_head))) == NULL )
            {
                if (binaryDC){ free(binaryDC); binaryDC = NULL; }
                if (weightedDC){ free(weightedDC); weightedDC = NULL; }
                if (histogram){ histogram = free_histogram(histogram, nhistnodes); }
                ERROR_message( "Could not allocate %d byte array for histogram2\n",
                    h2nbins*sizeof(hist_node_head)); 
            }
            else {
                /* -- update running memory estimate to reflect memory allocation */ 
                histogram2_save = histogram2;
                INC_MEM_STATS(( h2nbins*sizeof(hist_node_head )), "hist bins");
                PRINT_MEM_STATS( "hist2" );
            }
   
            /* initiatize the bins */ 
            for( kin = 0; kin < h2nbins; kin++ )
            {
                histogram2[ kin ].bin_low = histogram[kout].bin_low + kin*h2binwidth;
                histogram2[ kin ].bin_high = histogram2[ kin ].bin_low + h2binwidth;
                histogram2[ kin ].nbin = 0;
                histogram2[ kin ].nodes = NULL; 
                /*INFO_message("Hist2 bin %d [%3.3f, %3.3f) [%d, %p]\n",
                    kin, histogram2[ kin ].bin_low, histogram2[ kin ].bin_high,
                    histogram2[ kin ].nbin, histogram2[ kin ].nodes );*/
            }

            /* move correlations from histogram to histgram2 */
            INFO_message ("Adding %d nodes from histogram to histogram2",histogram[kout].nbin);
            while ( histogram[kout].nodes != NULL )
            {
                hptr = histogram[kout].nodes;
                h2ndx = (int)floor((double)(hptr->corr - histogram[kout].bin_low)/(double)h2binwidth);
                if(( h2ndx < h2nbins ) && ( h2ndx >= 0 ))
                {
                    histogram[kout].nodes = hptr->next;
                    hptr->next = histogram2[h2ndx].nodes;
                    histogram2[h2ndx].nodes = hptr; 
                    histogram2[h2ndx].nbin++;
                    histogram[kout].nbin--;
                }
                else
                {
                    WARNING_message("h2ndx %d is not in range [0,%d) :: %.10f,%.10f\n",h2ndx,h2nbins,hptr->corr, histogram[kout].bin_low);
                }
               
            }

            /* free the remainder of histogram */
            {
                int nbins_rem = 0;
                for(ii = 0; ii < nhistnodes; ii++) nbins_rem+=histogram[ii].nbin;
                histogram = free_histogram(histogram, nhistnodes);
                PRINT_MEM_STATS( "free remainder of histogram1" );
            }

            kin = h2nbins - 1;
            while (( nretain > 0 ) && ( kin >= 0 ))
            {
                hptr = pptr = histogram2[kin].nodes;
                while( hptr != NULL )
                {
     
                    /* determine the indices corresponding to this node */
                    if ( imap != NULL )
                    {
                        ii = imap[hptr->i] ;  
                    }
                    else
                    {
                        ii = hptr->i ;
                    }
                    if ( imap != NULL )
                    {
                        jj = imap[hptr->j] ; 
                    }
                    else
                    {
                        jj = hptr->j ;
                    }

                    /* add in the values */
                    if(( ii >= DSET_NVOX(cset) ) || ( jj >= DSET_NVOX(cset)))
                    {
                        if( ii >= DSET_NVOX(cset))
                        {
                            WARNING_message("Avoiding bodset, wodset overflow (ii) %d > %d\n (%s,%d)\n",
                                ii,DSET_NVOX(cset),__FILE__,__LINE__ );
                        }
                        if( jj >= DSET_NVOX(cset))
                        {
                            WARNING_message("Avoiding bodset, wodset overflow (jj) %d > %d\n (%s,%d)\n",
                                jj,DSET_NVOX(cset),__FILE__,__LINE__ );
                        }
                    }
                    else
                    {
                        bodset[ ii ] += 1.0 ;
                        wodset[ ii ] += (float)(hptr->corr);
                        bodset[ jj ] += 1.0 ;
                        wodset[ jj ] += (float)(hptr->corr);
                    }
                    if( fout1D != NULL )
                    {
                        /* add source, dest, correlation to 1D file */
                        ix1 = DSET_index_to_ix(cset,ii) ;
                        jy1 = DSET_index_to_jy(cset,ii) ;
                        kz1 = DSET_index_to_kz(cset,ii) ;
                        ix2 = DSET_index_to_ix(cset,jj) ;
                        jy2 = DSET_index_to_jy(cset,jj) ;
                        kz2 = DSET_index_to_kz(cset,jj) ;
                        fprintf(fout1D, "%d %d %d %d %d %d %d %d %.6f\n",
                            ii, jj, ix1, jy1, kz1, ix2, jy2, kz2, (float)(hptr->corr));
                    }

                    /* increment node pointers */
                    pptr = hptr;
                    hptr = hptr->next;

                    /* delete the node */
                    if(pptr)
                    {
                        free(pptr);
                        DEC_MEM_STATS(( sizeof(hist_node) ), "hist nodes");
                        pptr=NULL;
                    }
                }
 
                /* decrement the number of correlations we wish to retain */
                nretain -= histogram2[kin].nbin;
                histogram2[kin].nodes = NULL;

                /* go on to the next bin */
                kin--;
            }
            PRINT_MEM_STATS("hist2 nodes free - incorporated into output");

            /* we are finished with histogram2 */
            {
                histogram2 = free_histogram(histogram2, h2nbins);
                /* -- update running memory estimate to reflect memory allocation */ 
                PRINT_MEM_STATS( "free hist2" );
            }

            if (nretain < 0 )
            {
                WARNING_message( "Went over sparsity goal %d by %d, with a resolution of %f",
                      ngoal, -1*nretain, h2binwidth);
            }
        }
        if (nretain > 0 )
        {
            WARNING_message( "Was not able to meet goal of %d (%3.2f%%) correlations, %d (%3.2f%%) correlations passed the threshold of %3.2f, maybe you need to change the threshold or the desired sparsity?",
                  ngoal, 100.0*((float)ngoal)/((float)totPosCor), totNumCor, 100.0*((float)totNumCor)/((float)totPosCor),  thresh);
        }
   }

   INFO_message("Done..\n") ;

   /* update running memory statistics to reflect freeing the vectim */
   DEC_MEM_STATS(((xvectim->nvec*sizeof(int)) +
                       ((xvectim->nvec)*(xvectim->nvals))*sizeof(float) +
                       sizeof(MRI_vectim)), "vectim");

   /* toss some trash */
   VECTIM_destroy(xvectim) ;
   DSET_delete(xset) ;
   if(fout1D!=NULL)fclose(fout1D);

   PRINT_MEM_STATS( "vectim unload" );

   if (weightedDC) free(weightedDC) ; weightedDC = NULL;
   if (binaryDC) free(binaryDC) ; binaryDC = NULL;
   
   /* finito */
   INFO_message("Writing output dataset to disk [%s bytes]",
                commaized_integer_string(cset->dblk->total_bytes)) ;

   /* write the dataset */
   DSET_write(cset) ;
   WROTE_DSET(cset) ;

   /* increment our memory stats, since we are relying on the header for this
      information, we update the stats before actually freeing the memory */
   DEC_MEM_STATS( (DSET_NVOX(cset)*DSET_NVALS(cset)*sizeof(float)), "output dset");

   /* free up the output dataset memory */
   DSET_unload(cset) ;
   DSET_delete(cset) ;

   /* force a print */
   MEM_STAT = 1;
   PRINT_MEM_STATS( "Fin" );

   exit(0) ;
}
Exemple #4
0
/* function for creating a sparse array from the thresholded
   correlation matrix of a vectim.

   This approach uses a histogram approach to implement a sparsity threshold if
   it is so desired.

   inputs:
       xvectim: the input time courses that will be correlated
       sparsity: the percentage of the top correlations that should be retained
       threshold: a threshold that should be applied to determine if a correlation 
           should be retained. For sparsity thresholding this value will be used
           as an initial guess to speed calculation and a higher threshold may
           ultimately be calculated through the adaptive process.

    output:
        sparse_array_node: the list of remaining correlation values, or NULL if there
             was an error

    note:

        this function can use a _lot_ of memory if you the sparsity is too high, we tell
        the user how much memory we anticipate using, but this doesn't work for threshold only!*/
sparse_array_head_node* create_sparse_corr_array( MRI_vectim* xvectim, double sparsity, double thresh,
     double (*corfun)(long,float*,float*), long mem_allowance )
{

    /* random counters etc... */
    long kout = 0;

    /* variables for histogram */
    hist_node_head* histogram=NULL;
    sparse_array_head_node* sparse_array=NULL;
    sparse_array_node* recycled_nodes=NULL;
    long bottom_node_idx = 0;
    long totNumCor = 0;
    long totPosCor = 0;
    long ngoal = 0;
    long nretain = 0;
    float binwidth = 0.0;
    long nhistbins = 10000;
    long mem_budget = 0;

    /* retain the original threshold*/
    double othresh = thresh;

    /* set the memory budget from the allowance */
    mem_budget = mem_allowance;

    INFO_message( "Starting create_sparse_corr_array with a memory allowance of %ld", mem_budget);
 
    /* calculate the total number of possible correlations */
    totPosCor = .5 * ( xvectim->nvec -1 ) * ( xvectim->nvec );

    /* create a head node for the sparse array */
    sparse_array = (sparse_array_head_node*)calloc(1,sizeof(sparse_array_head_node));

    if( sparse_array == NULL )
    {
        ERROR_message( "Could not allocate header for sparse array\n" );
        return(NULL);
    }

    /* decrement the memory budget to account for the sparse array header */
    mem_budget = mem_budget - sizeof(sparse_array_head_node);

    /* check if we can do what is asked of us with the budget provided */
    if( sparsity < 100.0 )
    {
        /* figure the cost of the histogram into the memory budget */
        mem_budget = mem_budget - nhistbins*sizeof(hist_node_head);

        /* and the number of desired correlations */
        ngoal = nretain = (long)ceil(((double)totPosCor)*((double)sparsity) / 100.0);

        /* check to see if we want to use more memory than would be used by the 
           full correlation matrix, if so, the we should probably just use full
           correlation - or min memory func */
        if((ngoal * sizeof( sparse_array_node )) > mem_budget)
        {
           WARNING_message( "The sparse array with %3.2lf%% of the %ld total"
                            " would exceed the memory budget (%3.2lf MB) refusing to proceed\n",
                            sparsity, totPosCor,((double)mem_budget)/(1024.0*1024.0));
           return( NULL );
        }
        else
        {
           INFO_message( "The sparse array with %ld values will take %3.2lf"
                         " MB of memory (budget = %3.2lf MB)\n", ngoal,
                         (double)(ngoal * sizeof(sparse_array_node))
                         /(1024.0*1024.0), ((double)mem_budget)/(1024.0*1024.0));
        }
    }
    else
    {
        WARNING_message( "Cannot pre-calculate the memory required for a sparse"
                         " matrix when only a correlation threshold is used. "
                         "Instead the mem is tracked and if we exceed what "
                         "would be used by the non-sparse array, the operation"
                         " will be aborted.");
    }

    INFO_message( "Extracting sparse correlation array with threshold = %f and"
                  " sparsity = %3.2f%% (%d)\n", thresh, sparsity, nretain);

    /* if we are using a sparsity threshold, setup the histogram to sort the values */
    if ( sparsity < 100.0 )
    {
        /* make sure that there is a bin for correlation values that == 1.0 */
        binwidth = (1.005-thresh)/nhistbins;

        /* allocate memory for the histogram bins */
        if(( histogram = (hist_node_head*)malloc(nhistbins*sizeof(hist_node_head))) == NULL )
        {
            /* if the allocation fails, free all memory and exit */
            ERROR_message( "Could not allocate %d byte array for histogram\n",
                nhistbins*sizeof(hist_node_head)); 
            return( NULL );
        }

        /* initialize history bins */
        for( kout = 0; kout < nhistbins; kout++ )
        {
            histogram[ kout ].bin_low = thresh+kout*binwidth;
            histogram[ kout ].bin_high = histogram[ kout ].bin_low+binwidth;
            histogram[ kout ].nbin = 0;
            histogram[ kout ].nodes = NULL; 
            histogram[ kout ].tail = NULL; 
            /*
                INFO_message("Hist bin %d [%3.3f, %3.3f) [%d, %p]\n",
                    kout, histogram[ kout ].bin_low, histogram[ kout ].bin_high,
                    histogram[ kout ].nbin, histogram[ kout ].nodes );
            */
        }
    }

    /*---------- loop over mask voxels, correlate ----------*/
    AFNI_OMP_START ;
#pragma omp parallel if( xvectim->nvec > 999 )
    {
        int lii,ljj,lin,lout,ithr,nthr,vstep,vii ;
        float *xsar , *ysar ;
        sparse_array_node* new_node = NULL ;
        int new_node_idx = 0;
        double car = 0.0 ; 

        /*-- get information about who we are --*/
#ifdef USE_OMP
        ithr = omp_get_thread_num() ;
        nthr = omp_get_num_threads() ;
        if( ithr == 0 ) INFO_message("%d OpenMP threads started",nthr) ;
#else
        ithr = 0 ; nthr = 1 ;
#endif

        /*-- For the progress tracker, we want to print out 50 numbers,
             figure out a number of loop iterations that will make this easy */
        vstep = (int)( xvectim->nvec / (nthr*50.0f) + 0.901f ) ; vii = 0 ;
        if(ithr == 0 ) fprintf(stderr,"Looping:") ;

#pragma omp for schedule(static, 1)
        for( lout=0 ; lout < xvectim->nvec ; lout++ )
        {  /*----- outer voxel loop -----*/

            if( ithr == 0 && vstep > 2 ) /* allow small dsets 16 Jun 2011 [rickr] */
            {
                vii++;
                if( vii%vstep == vstep/2 )
                {
                    vstep_print();
                }
            }

            /* if the amount of memory exceeds budget, dont do anything more */
            if ( mem_budget >= 0 )
            {
                /* get ref time series from this voxel */
                xsar = VECTIM_PTR(xvectim,lout);

                /* try to make calculation more efficient by only calculating the unique 
                   correlations */
                for( lin=(lout+1) ; lin < xvectim->nvec ; lin++ )
                {  /*----- inner loop over voxels -----*/

                    if ( mem_budget >= 0 )
                    {
                        /* extract the voxel time series */
                        ysar = VECTIM_PTR(xvectim,lin);

                        /* now correlate the time series */
                        car = (double)(corfun(xvectim->nvals,xsar,ysar));

                        if ( car < thresh )
                        {
                            continue;
                        }

#pragma omp critical(dataupdate)
                        {
                            /* the threshold might have changed while we were waiting,
                               so check it again */
                            if (car >= thresh )
                            {
                                /* create a node to add to the histogram, try to use a 
                                   recycled node to save time and memory */
                                if ( recycled_nodes == NULL )
                                {
                                    mem_budget = mem_budget - sizeof(sparse_array_node);
                                    if( mem_budget >= 0 )
                                    {
                                        new_node = (sparse_array_node*)calloc(1,sizeof(sparse_array_node));
                                    }
                                    else
                                    {
                                        new_node = NULL; 
                                    }
                                }
                                else
                                {
                                    new_node = recycled_nodes;
                                    recycled_nodes = recycled_nodes->next;
                                    new_node->next = NULL;
                                }
                                if( new_node == NULL )
                                {
                                    /* allocate memory for this node, rather than fiddling with 
                                       error handling here, lets just move on */
                                    WARNING_message("Could not allocate a new node!");
                                }
                                else
                                {
                                    new_node->weight = car;
                                    new_node->row = lout;
                                    new_node->column = lin;

                                    totNumCor += 1;

                                    /* if keeping all connections, just add to linked list */
                                    if ( sparsity >= 100.0 )
                                    {
                                        new_node->next = sparse_array->nodes;
                                        sparse_array->nodes = new_node;
                                        sparse_array->num_nodes = sparse_array->num_nodes + 1;
                                        new_node = NULL; 
                                    }
                                    /* otherwise, populate to proper bin of histogram */
                                    else
                                    {
                                        /* determine the index in the histogram to add the node */
                                        new_node_idx = (int)floor((double)(car-othresh)/(double)binwidth);
                                        if ((new_node_idx > nhistbins) || (new_node_idx < bottom_node_idx))
                                        {
                                            /* this error should indicate a programming error and should not happen */
                                            /*WARNING_message("Node index %d (%3.4lf >= %3.4lf) is out of range [%d,%d)"
                                                " {[%3.4lf, %3.4lf)}!",new_node_idx, car, thresh, bottom_node_idx,
                                                nhistbins, histogram[bottom_node_idx].bin_low,
                                                histogram[bottom_node_idx].bin_high ); */
                                        }
                                        else
                                        {
                                            /* populate histogram node */
                                            new_node->row = lout; 
                                            new_node->column = lin;
                                            new_node->weight = car;
                                            new_node->next = NULL;

                                            /* update histogram bin linked-list */
                                            new_node->next = histogram[new_node_idx].nodes;
                                            histogram[new_node_idx].nodes = new_node;
                                            /* if first node in bin, point tail to node */
                                            if (histogram[new_node_idx].tail == NULL)
                                            {
                                                histogram[new_node_idx].tail = new_node;
                                            }
                                            /* increment bin count */
                                            histogram[new_node_idx].nbin++; 

                                            /* see if there are enough correlations in the histogram
                                               for the sparsity - prune un-needed hist bins*/
                                            while ((totNumCor - histogram[bottom_node_idx].nbin) > nretain)
                                            { 
                                                /* push the histogram nodes onto the list of recycled nodes, it could be
                                                   that this hist bin is empty, in which case we have nothing to add */
                                                if( histogram[bottom_node_idx].tail != NULL )
                                                {
                                                    histogram[bottom_node_idx].tail->next = recycled_nodes;
                                                    recycled_nodes = histogram[bottom_node_idx].nodes;
                                                }
                                                else
                                                {
                                                    if( histogram[bottom_node_idx].nbin != 0 )
                                                    {
                                                         WARNING_message("Trying to remove histogram bin that contains"
                                                                         " %d values, but whose tail pointer is NULL\n",
                                                                         histogram[bottom_node_idx].nbin);
                                                    }
                                                }
        
                                                /* bookkeeping */ 
                                                histogram[bottom_node_idx].nodes = NULL;
                                                histogram[bottom_node_idx].tail = NULL;
                                                totNumCor -= histogram[bottom_node_idx].nbin;
                                                histogram[bottom_node_idx].nbin = 0;
             
                                                /* get the new threshold */
                                                thresh = (double)histogram[++bottom_node_idx].bin_low;
                                                /*INFO_message("Increasing threshold to %3.2f (%d)\n",
                                                    thresh,bottom_node_idx); */
                                            } /* while */
                                        } /* else, new_node_idx in range */
                                    } /* else, sparsity >= 100.0 */
                                } /* else, new_node != NULL */
                            } /* if (car >= thresh ) */
                        } /* this is the end of the critical section */
                    } /* if ( mem_budget >= 0 ) */
                } /* end of inner loop over voxels */
            } /* if ( mem_budget >= 0 ) */
        } /* end of outer loop over ref voxels */

        if( ithr == 0 ) fprintf(stderr,".\n") ;

    } /* end OpenMP */
    AFNI_OMP_END ;


    /* check to see if we exceeded memory or didn't get any
       correlations > threshold */
    if (( mem_budget < 0 ) || ( totNumCor == 0 ))
    {
        if ( mem_budget < 0 )
        {
            ERROR_message( "Memory budget (%lf MB) exceeded, consider using a"
                "higher correlation or lower sparsity threshold",
                ((double)mem_allowance/(1024.0*1024.0)));
        }
        else
        {
            ERROR_message( "No correlations exceeded threshold, consider using"
                           " a lower correlation threshold");
        }
        sparse_array = free_sparse_array( sparse_array );
    }
    else
    {
        /* if using sparsity threshold, construct sparse array from
           the histogram */
        if ( sparsity < 100.0 )
        {

            /* pull the requested number of nodes off of the histogram */
            for ( kout = (nhistbins-1); kout >= bottom_node_idx; kout-- )
            { 
                if((histogram[ kout ].nodes != NULL ) &&
                   (histogram[ kout ].nbin > 0))
                {
                    if( histogram[ kout ].tail == NULL )
                    {
                        ERROR_message("Head is not null, but tail is?? (%ld)\n",
                            kout);
                    }
                    /* push the list onto sparse array */
                    histogram[ kout ].tail->next = sparse_array->nodes;
                    sparse_array->nodes = histogram[ kout ].nodes;

                    /* increment the number of nodes */
                    sparse_array->num_nodes = sparse_array->num_nodes +
                        histogram[ kout ].nbin;
      
                    /* remove the references from the histogram,
                       this is super important considering we don't want
                       to accidently free any nodes that are on the sparse_array
                       when we free the histogram later */
                    histogram[ kout ].nodes = NULL;
                    histogram[ kout ].tail = NULL;
                    histogram[ kout ].nbin = 0;
                } 
                /* dont take more than we want */
                if ( sparse_array->num_nodes > nretain ) break;
            }

            INFO_message( "Sparsity requested %ld and received %ld correlations"
                          " (%3.2lf%% sparsity) final threshold = %3.4lf.\n",
                nretain, sparse_array->num_nodes,
                100.0*((double)sparse_array->num_nodes)/((double)totPosCor),
                thresh);

            if( sparse_array->num_nodes < nretain )
            {
                INFO_message( "Consider lowering the initial correlation"
                    "threshold (%3.2lf) to retain more correlations.\n",
                    othresh);
            }
        }
        else
        {
            INFO_message( "Correlation threshold (%3.2lf) resulted in %ld"
                " correlations (%3.2lf%% sparsity).\n", thresh, 
                sparse_array->num_nodes,
                100.0*((double)sparse_array->num_nodes)/((double)totPosCor));
        }
    }

    /* free residual mem */
    histogram = free_histogram( histogram, nhistbins );
    recycled_nodes = free_sparse_list( recycled_nodes );

    return( sparse_array );
}