int main( int argc , char *argv[] ) { THD_3dim_dataset *xset , *cset, *mset=NULL ; int nopt=1 , method=PEARSON , do_autoclip=0 ; int nvox , nvals , ii, jj, kout, kin, polort=1 ; int ix1,jy1,kz1, ix2, jy2, kz2 ; char *prefix = "degree_centrality" ; byte *mask=NULL; int nmask , abuc=1 ; int all_source=0; /* output all source voxels 25 Jun 2010 [rickr] */ char str[32] , *cpt ; int *imap = NULL ; MRI_vectim *xvectim ; float (*corfun)(int,float *,float*) = NULL ; /* djc - add 1d file output for similarity matrix */ FILE *fout1D=NULL; /* CC - we will have two subbricks: binary and weighted centrality */ int nsubbriks = 2; int subbrik = 0; float * bodset; float * wodset; int nb_ctr = 0; /* CC - added flags for thresholding correlations */ double thresh = 0.0; double othresh = 0.0; int dothresh = 0; double sparsity = 0.0; int dosparsity = 0; /* variables for calculating degree centrality */ long * binaryDC = NULL; double * weightedDC = NULL; /* variables for histogram */ hist_node_head* histogram=NULL; hist_node* hptr=NULL; hist_node* pptr=NULL; int bottom_node_idx = 0; int totNumCor = 0; long totPosCor = 0; int ngoal = 0; int nretain = 0; float binwidth = 0.0; int nhistnodes = 50; /*----*/ AFNI_SETUP_OMP(0) ; /* 24 Jun 2013 */ if( argc < 2 || strcmp(argv[1],"-help") == 0 ){ printf( "Usage: 3dDegreeCentrality [options] dset\n" " Computes voxelwise weighted and binary degree centrality and\n" " stores the result in a new 3D bucket dataset as floats to\n" " preserve their values. Degree centrality reflects the strength and\n" " extent of the correlation of a voxel with every other voxel in\n" " the brain.\n\n" " Conceptually the process involves: \n" " 1. Calculating the correlation between voxel time series for\n" " every pair of voxels in the brain (as determined by masking)\n" " 2. Applying a threshold to the resulting correlations to exclude\n" " those that might have arisen by chance, or to sparsify the\n" " connectivity graph.\n" " 3. At each voxel, summarizing its correlation with other voxels\n" " in the brain, by either counting the number of voxels correlated\n" " with the seed voxel (binary) or by summing the correlation \n" " coefficients (weighted).\n" " Practically the algorithm is ordered differently to optimize for\n" " computational time and memory usage.\n\n" " The threshold can be supplied as a correlation coefficient, \n" " or a sparsity threshold. The sparsity threshold reflects the fraction\n" " of connections that should be retained after the threshold has been\n" " applied. To minimize resource consumption, using a sparsity threshold\n" " involves a two-step procedure. In the first step, a correlation\n" " coefficient threshold is applied to substantially reduce the number\n" " of correlations. Next, the remaining correlations are sorted and a\n" " threshold is calculated so that only the specified fraction of \n" " possible correlations are above threshold. Due to ties between\n" " correlations, the fraction of correlations that pass the sparsity\n" " threshold might be slightly more than the number specified.\n\n" " Regardless of the thresholding procedure employed, negative \n" " correlations are excluded from the calculations.\n" "\n" "Options:\n" " -pearson = Correlation is the normal Pearson (product moment)\n" " correlation coefficient [default].\n" #if 0 " -spearman = Correlation is the Spearman (rank) correlation\n" " coefficient.\n" " -quadrant = Correlation is the quadrant correlation coefficient.\n" #else " -spearman AND -quadrant are disabled at this time :-(\n" #endif "\n" " -thresh r = exclude correlations <= r from calculations\n" " -sparsity s = only use top s percent of correlations in calculations\n" " s should be an integer between 0 and 100. Uses an\n" " an adaptive thresholding procedure to reduce memory.\n" " The speed of determining the adaptive threshold can\n" " be improved by specifying an initial threshold with\n" " the -thresh flag.\n" "\n" " -polort m = Remove polynomical trend of order 'm', for m=-1..3.\n" " [default is m=1; removal is by least squares].\n" " Using m=-1 means no detrending; this is only useful\n" " for data/information that has been pre-processed.\n" "\n" " -autoclip = Clip off low-intensity regions in the dataset,\n" " -automask = so that the correlation is only computed between\n" " high-intensity (presumably brain) voxels. The\n" " mask is determined the same way that 3dAutomask works.\n" "\n" " -mask mmm = Mask to define 'in-brain' voxels. Reducing the number\n" " the number of voxels included in the calculation will\n" " significantly speedup the calculation. Consider using\n" " a mask to constrain the calculations to the grey matter\n" " rather than the whole brain. This is also preferrable\n" " to using -autoclip or -automask.\n" "\n" " -prefix p = Save output into dataset with prefix 'p', this file will\n" " contain bricks for both 'weighted' or 'degree' centrality\n" " [default prefix is 'deg_centrality'].\n" "\n" " -out1D f = Save information about the above threshold correlations to\n" " 1D file 'f'. Each row of this file will contain:\n" " Voxel1 Voxel2 i1 j1 k1 i2 j2 k2 Corr\n" " Where voxel1 and voxel2 are the 1D indices of the pair of\n" " voxels, i j k correspond to their 3D coordinates, and Corr\n" " is the value of the correlation between the voxel time courses.\n" "\n" "Notes:\n" " * The output dataset is a bucket type of floats.\n" " * The program prints out an estimate of its memory used\n" " when it ends. It also prints out a progress 'meter'\n" " to keep you pacified.\n" "\n" "-- RWCox - 31 Jan 2002 and 16 Jul 2010\n" "-- Cameron Craddock - 26 Sept 2015 \n" ) ; PRINT_AFNI_OMP_USAGE("3dDegreeCentrality",NULL) ; PRINT_COMPILE_DATE ; exit(0) ; } mainENTRY("3dDegreeCentrality main"); machdep(); PRINT_VERSION("3dDegreeCentrality"); AFNI_logger("3dDegreeCentrality",argc,argv); /*-- option processing --*/ while( nopt < argc && argv[nopt][0] == '-' ){ if( strcmp(argv[nopt],"-time") == 0 ){ abuc = 0 ; nopt++ ; continue ; } if( strcmp(argv[nopt],"-autoclip") == 0 || strcmp(argv[nopt],"-automask") == 0 ){ do_autoclip = 1 ; nopt++ ; continue ; } if( strcmp(argv[nopt],"-mask") == 0 ){ mset = THD_open_dataset(argv[++nopt]); CHECK_OPEN_ERROR(mset,argv[nopt]); nopt++ ; continue ; } if( strcmp(argv[nopt],"-pearson") == 0 ){ method = PEARSON ; nopt++ ; continue ; } #if 0 if( strcmp(argv[nopt],"-spearman") == 0 ){ method = SPEARMAN ; nopt++ ; continue ; } if( strcmp(argv[nopt],"-quadrant") == 0 ){ method = QUADRANT ; nopt++ ; continue ; } #endif if( strcmp(argv[nopt],"-eta2") == 0 ){ method = ETA2 ; nopt++ ; continue ; } if( strcmp(argv[nopt],"-prefix") == 0 ){ prefix = strdup(argv[++nopt]) ; if( !THD_filename_ok(prefix) ){ ERROR_exit("Illegal value after -prefix!") ; } nopt++ ; continue ; } if( strcmp(argv[nopt],"-thresh") == 0 ){ double val = (double)strtod(argv[++nopt],&cpt) ; if( *cpt != '\0' || val >= 1.0 || val < 0.0 ){ ERROR_exit("Illegal value (%f) after -thresh!", val) ; } dothresh = 1; thresh = val ; othresh = val ; nopt++ ; continue ; } if( strcmp(argv[nopt],"-sparsity") == 0 ){ double val = (double)strtod(argv[++nopt],&cpt) ; if( *cpt != '\0' || val > 100 || val <= 0 ){ ERROR_exit("Illegal value (%f) after -sparsity!", val) ; } if( val > 5.0 ) { WARNING_message("Sparsity %3.2f%% is large and will require alot of memory and time, consider using a smaller value. ", val); } dosparsity = 1 ; sparsity = val ; nopt++ ; continue ; } if( strcmp(argv[nopt],"-polort") == 0 ){ int val = (int)strtod(argv[++nopt],&cpt) ; if( *cpt != '\0' || val < -1 || val > 3 ){ ERROR_exit("Illegal value after -polort!") ; } polort = val ; nopt++ ; continue ; } if( strcmp(argv[nopt],"-mem_stat") == 0 ){ MEM_STAT = 1 ; nopt++ ; continue ; } if( strncmp(argv[nopt],"-mem_profile",8) == 0 ){ MEM_PROF = 1 ; nopt++ ; continue ; } /* check for 1d argument */ if ( strcmp(argv[nopt],"-out1D") == 0 ){ if (!(fout1D = fopen(argv[++nopt], "w"))) { ERROR_message("Failed to open %s for writing", argv[nopt]); exit(1); } nopt++ ; continue ; } ERROR_exit("Illegal option: %s",argv[nopt]) ; } /*-- open dataset, check for legality --*/ if( nopt >= argc ) ERROR_exit("Need a dataset on command line!?") ; xset = THD_open_dataset(argv[nopt]); CHECK_OPEN_ERROR(xset,argv[nopt]); if( DSET_NVALS(xset) < 3 ) ERROR_exit("Input dataset %s does not have 3 or more sub-bricks!",argv[nopt]) ; DSET_load(xset) ; CHECK_LOAD_ERROR(xset) ; /*-- compute mask array, if desired --*/ nvox = DSET_NVOX(xset) ; nvals = DSET_NVALS(xset) ; INC_MEM_STATS((nvox * nvals * sizeof(double)), "input dset"); PRINT_MEM_STATS("inset"); /* if a mask was specified make sure it is appropriate */ if( mset ){ if( DSET_NVOX(mset) != nvox ) ERROR_exit("Input and mask dataset differ in number of voxels!") ; mask = THD_makemask(mset, 0, 1.0, 0.0) ; /* update running memory statistics to reflect loading the image */ INC_MEM_STATS( mset->dblk->total_bytes, "mask dset" ); PRINT_MEM_STATS( "mset load" ); nmask = THD_countmask( nvox , mask ) ; INC_MEM_STATS( nmask * sizeof(byte), "mask array" ); PRINT_MEM_STATS( "mask" ); INFO_message("%d voxels in -mask dataset",nmask) ; if( nmask < 2 ) ERROR_exit("Only %d voxels in -mask, exiting...",nmask); /* update running memory statistics to reflect loading the image */ DEC_MEM_STATS( mset->dblk->total_bytes, "mask dset" ); DSET_unload(mset) ; PRINT_MEM_STATS( "mset unload" ); } /* if automasking is requested, handle that now */ else if( do_autoclip ){ mask = THD_automask( xset ) ; nmask = THD_countmask( nvox , mask ) ; INFO_message("%d voxels survive -autoclip",nmask) ; if( nmask < 2 ) ERROR_exit("Only %d voxels in -automask!",nmask); } /* otherwise we use all of the voxels in the image */ else { nmask = nvox ; INFO_message("computing for all %d voxels",nmask) ; } if( method == ETA2 && polort >= 0 ) WARNING_message("Polort for -eta2 should probably be -1..."); /* djc - 1d file out init */ if (fout1D != NULL) { /* define affine matrix */ mat44 affine_mat = xset->daxes->ijk_to_dicom; /* print command line statement */ fprintf(fout1D,"#Similarity matrix from command:\n#"); for(ii=0; ii<argc; ++ii) fprintf(fout1D,"%s ", argv[ii]); /* Print affine matrix */ fprintf(fout1D,"\n"); fprintf(fout1D,"#[ "); int mi, mj; for(mi = 0; mi < 4; mi++) { for(mj = 0; mj < 4; mj++) { fprintf(fout1D, "%.6f ", affine_mat.m[mi][mj]); } } fprintf(fout1D, "]\n"); /* Print image extents*/ THD_dataxes *xset_daxes = xset->daxes; fprintf(fout1D, "#Image dimensions:\n"); fprintf(fout1D, "#[%d, %d, %d]\n", xset_daxes->nxx, xset_daxes->nyy, xset_daxes->nzz); /* Similarity matrix headers */ fprintf(fout1D,"#Voxel1 Voxel2 i1 j1 k1 i2 j2 k2 Corr\n"); } /* CC calculate the total number of possible correlations, will be usefule down the road */ totPosCor = (.5*((float)nmask))*((float)(nmask-1)); /** For the case of Pearson correlation, we make sure the **/ /** data time series have their mean removed (polort >= 0) **/ /** and are normalized, so that correlation = dot product, **/ /** and we can use function zm_THD_pearson_corr for speed. **/ switch( method ){ default: case PEARSON: corfun = zm_THD_pearson_corr ; break ; case ETA2: corfun = my_THD_eta_squared ; break ; } /*-- create vectim from input dataset --*/ INFO_message("vectim-izing input dataset") ; /*-- CC added in mask to reduce the size of xvectim -- */ xvectim = THD_dset_to_vectim( xset , mask , 0 ) ; if( xvectim == NULL ) ERROR_exit("Can't create vectim?!") ; /*-- CC update our memory stats to reflect vectim -- */ INC_MEM_STATS((xvectim->nvec*sizeof(int)) + ((xvectim->nvec)*(xvectim->nvals))*sizeof(float) + sizeof(MRI_vectim), "vectim"); PRINT_MEM_STATS( "vectim" ); /*--- CC the vectim contains a mapping between voxel index and mask index, tap into that here to avoid duplicating memory usage ---*/ if( mask != NULL ) { imap = xvectim->ivec; /* --- CC free the mask */ DEC_MEM_STATS( nmask*sizeof(byte), "mask array" ); free(mask); mask=NULL; PRINT_MEM_STATS( "mask unload" ); } /* -- CC unloading the dataset to reduce memory usage ?? -- */ DEC_MEM_STATS((DSET_NVOX(xset) * DSET_NVALS(xset) * sizeof(double)), "input dset"); DSET_unload(xset) ; PRINT_MEM_STATS("inset unload"); /* -- CC configure detrending --*/ if( polort < 0 && method == PEARSON ){ polort = 0; WARNING_message("Pearson correlation always uses polort >= 0"); } if( polort >= 0 ){ for( ii=0 ; ii < xvectim->nvec ; ii++ ){ /* remove polynomial trend */ DETREND_polort(polort,nvals,VECTIM_PTR(xvectim,ii)) ; } } /* -- this procedure does not change time series that have zero variance -- */ if( method == PEARSON ) THD_vectim_normalize(xvectim) ; /* L2 norm = 1 */ /* -- CC create arrays to hold degree and weighted centrality while they are being calculated -- */ if( dosparsity == 0 ) { if( ( binaryDC = (long*)calloc( nmask, sizeof(long) )) == NULL ) { ERROR_message( "Could not allocate %d byte array for binary DC calculation\n", nmask*sizeof(long)); } /* -- update running memory estimate to reflect memory allocation */ INC_MEM_STATS( nmask*sizeof(long), "binary DC array" ); PRINT_MEM_STATS( "binaryDC" ); if( ( weightedDC = (double*)calloc( nmask, sizeof(double) )) == NULL ) { if (binaryDC){ free(binaryDC); binaryDC = NULL; } ERROR_message( "Could not allocate %d byte array for weighted DC calculation\n", nmask*sizeof(double)); } /* -- update running memory estimate to reflect memory allocation */ INC_MEM_STATS( nmask*sizeof(double), "weighted DC array" ); PRINT_MEM_STATS( "weightedDC" ); } /* -- CC if we are using a sparsity threshold, build a histogram to calculate the threshold */ if (dosparsity == 1) { /* make sure that there is a bin for correlation values that == 1.0 */ binwidth = (1.005-thresh)/nhistnodes; /* calculate the number of correlations we wish to retain */ ngoal = nretain = (int)(((double)totPosCor)*((double)sparsity) / 100.0); /* allocate memory for the histogram bins */ if(( histogram = (hist_node_head*)malloc(nhistnodes*sizeof(hist_node_head))) == NULL ) { /* if the allocation fails, free all memory and exit */ if (binaryDC){ free(binaryDC); binaryDC = NULL; } if (weightedDC){ free(weightedDC); weightedDC = NULL; } ERROR_message( "Could not allocate %d byte array for histogram\n", nhistnodes*sizeof(hist_node_head)); } else { /* -- update running memory estimate to reflect memory allocation */ INC_MEM_STATS( nhistnodes*sizeof(hist_node_head), "hist bins" ); PRINT_MEM_STATS( "hist1" ); } /* initialize history bins */ for( kout = 0; kout < nhistnodes; kout++ ) { histogram[ kout ].bin_low = thresh+kout*binwidth; histogram[ kout ].bin_high = histogram[ kout ].bin_low+binwidth; histogram[ kout ].nbin = 0; histogram[ kout ].nodes = NULL; /*INFO_message("Hist bin %d [%3.3f, %3.3f) [%d, %p]\n", kout, histogram[ kout ].bin_low, histogram[ kout ].bin_high, histogram[ kout ].nbin, histogram[ kout ].nodes );*/ } } /*-- tell the user what we are about to do --*/ if (dosparsity == 0 ) { INFO_message( "Calculating degree centrality with threshold = %f.\n", thresh); } else { INFO_message( "Calculating degree centrality with threshold = %f and sparsity = %3.2f%% (%d)\n", thresh, sparsity, nretain); } /*---------- loop over mask voxels, correlate ----------*/ AFNI_OMP_START ; #pragma omp parallel if( nmask > 999 ) { int lii,ljj,lin,lout,ithr,nthr,vstep,vii ; float *xsar , *ysar ; hist_node* new_node = NULL ; hist_node* tptr = NULL ; hist_node* rptr = NULL ; int new_node_idx = 0; double car = 0.0 ; /*-- get information about who we are --*/ #ifdef USE_OMP ithr = omp_get_thread_num() ; nthr = omp_get_num_threads() ; if( ithr == 0 ) INFO_message("%d OpenMP threads started",nthr) ; #else ithr = 0 ; nthr = 1 ; #endif /*-- For the progress tracker, we want to print out 50 numbers, figure out a number of loop iterations that will make this easy */ vstep = (int)( nmask / (nthr*50.0f) + 0.901f ) ; vii = 0 ; if((MEM_STAT==0) && (ithr == 0 )) fprintf(stderr,"Looping:") ; #pragma omp for schedule(static, 1) for( lout=0 ; lout < xvectim->nvec ; lout++ ){ /*----- outer voxel loop -----*/ if( ithr == 0 && vstep > 2 ) /* allow small dsets 16 Jun 2011 [rickr] */ { vii++ ; if( vii%vstep == vstep/2 && MEM_STAT == 0 ) vstep_print(); } /* get ref time series from this voxel */ xsar = VECTIM_PTR(xvectim,lout) ; /* try to make calculation more efficient by only calculating the unique correlations */ for( lin=(lout+1) ; lin < xvectim->nvec ; lin++ ){ /*----- inner loop over voxels -----*/ /* extract the voxel time series */ ysar = VECTIM_PTR(xvectim,lin) ; /* now correlate the time series */ car = (double)(corfun(nvals,xsar,ysar)) ; if ( car <= thresh ) { continue ; } /* update degree centrality values, hopefully the pragma will handle mutual exclusion */ #pragma omp critical(dataupdate) { /* if the correlation is less than threshold, ignore it */ if ( car > thresh ) { totNumCor += 1; if ( dosparsity == 0 ) { binaryDC[lout] += 1; binaryDC[lin] += 1; weightedDC[lout] += car; weightedDC[lin] += car; /* print correlation out to the 1D file */ if ( fout1D != NULL ) { /* determine the i,j,k coords */ ix1 = DSET_index_to_ix(xset,lii) ; jy1 = DSET_index_to_jy(xset,lii) ; kz1 = DSET_index_to_kz(xset,lii) ; ix2 = DSET_index_to_ix(xset,ljj) ; jy2 = DSET_index_to_jy(xset,ljj) ; kz2 = DSET_index_to_kz(xset,ljj) ; /* add source, dest, correlation to 1D file */ fprintf(fout1D, "%d %d %d %d %d %d %d %d %.6f\n", lii, ljj, ix1, jy1, kz1, ix2, jy2, kz2, car); } } else { /* determine the index in the histogram to add the node */ new_node_idx = (int)floor((double)(car-othresh)/(double)binwidth); if ((new_node_idx > nhistnodes) || (new_node_idx < bottom_node_idx)) { /* this error should indicate a programming error and should not happen */ WARNING_message("Node index %d is out of range [%d,%d)!",new_node_idx, bottom_node_idx, nhistnodes); } else { /* create a node to add to the histogram */ new_node = (hist_node*)calloc(1,sizeof(hist_node)); if( new_node == NULL ) { /* allocate memory for this node, rather than fiddling with error handling here, lets just move on */ WARNING_message("Could not allocate a new node!"); } else { /* populate histogram node */ new_node->i = lout; new_node->j = lin; new_node->corr = car; new_node->next = NULL; /* -- update running memory estimate to reflect memory allocation */ INC_MEM_STATS( sizeof(hist_node), "hist nodes" ); if ((totNumCor % (1024*1024)) == 0) PRINT_MEM_STATS( "hist nodes" ); /* populate histogram */ new_node->next = histogram[new_node_idx].nodes; histogram[new_node_idx].nodes = new_node; histogram[new_node_idx].nbin++; /* see if there are enough correlations in the histogram for the sparsity */ if ((totNumCor - histogram[bottom_node_idx].nbin) > nretain) { /* delete the list of nodes */ rptr = histogram[bottom_node_idx].nodes; while(rptr != NULL) { tptr = rptr; rptr = rptr->next; /* check that the ptr is not null before freeing it*/ if(tptr!= NULL) { DEC_MEM_STATS( sizeof(hist_node), "hist nodes" ); free(tptr); } } PRINT_MEM_STATS( "unloaded hist nodes - thresh increase" ); histogram[bottom_node_idx].nodes = NULL; totNumCor -= histogram[bottom_node_idx].nbin; histogram[bottom_node_idx].nbin=0; /* get the new threshold */ thresh = (double)histogram[++bottom_node_idx].bin_low; if(MEM_STAT == 1) INFO_message("Increasing threshold to %3.2f (%d)\n", thresh,bottom_node_idx); } } /* else, newptr != NULL */ } /* else, new_node_idx in range */ } /* else, do_sparsity == 1 */ } /* car > thresh */ } /* this is the end of the critical section */ } /* end of inner loop over voxels */ } /* end of outer loop over ref voxels */ if( ithr == 0 ) fprintf(stderr,".\n") ; } /* end OpenMP */ AFNI_OMP_END ; /* update the user so that they know what we are up to */ INFO_message ("AFNI_OMP finished\n"); INFO_message ("Found %d (%3.2f%%) correlations above threshold (%f)\n", totNumCor, 100.0*((float)totNumCor)/((float)totPosCor), thresh); /*---------- Finish up ---------*/ /*if( dosparsity == 1 ) { for( kout = 0; kout < nhistnodes; kout++ ) { INFO_message("Hist bin %d [%3.3f, %3.3f) [%d, %p]\n", kout, histogram[ kout ].bin_low, histogram[ kout ].bin_high, histogram[ kout ].nbin, histogram[ kout ].nodes ); } }*/ /*-- create output dataset --*/ cset = EDIT_empty_copy( xset ) ; /*-- configure the output dataset */ if( abuc ){ EDIT_dset_items( cset , ADN_prefix , prefix , ADN_nvals , nsubbriks , /* 2 subbricks, degree and weighted centrality */ ADN_ntt , 0 , /* no time axis */ ADN_type , HEAD_ANAT_TYPE , ADN_func_type , ANAT_BUCK_TYPE , ADN_datum_all , MRI_float , ADN_none ) ; } else { EDIT_dset_items( cset , ADN_prefix , prefix , ADN_nvals , nsubbriks , /* 2 subbricks, degree and weighted centrality */ ADN_ntt , nsubbriks , /* num times */ ADN_ttdel , 1.0 , /* fake TR */ ADN_nsl , 0 , /* no slice offsets */ ADN_type , HEAD_ANAT_TYPE , ADN_func_type , ANAT_EPI_TYPE , ADN_datum_all , MRI_float , ADN_none ) ; } /* add history information to the hearder */ tross_Make_History( "3dDegreeCentrality" , argc,argv , cset ) ; ININFO_message("creating output dataset in memory") ; /* -- Configure the subbriks: Binary Degree Centrality */ subbrik = 0; EDIT_BRICK_TO_NOSTAT(cset,subbrik) ; /* stat params */ /* CC this sets the subbrik scaling factor, which we will probably want to do again after we calculate the voxel values */ EDIT_BRICK_FACTOR(cset,subbrik,1.0) ; /* scale factor */ sprintf(str,"Binary Degree Centrality") ; EDIT_BRICK_LABEL(cset,subbrik,str) ; EDIT_substitute_brick(cset,subbrik,MRI_float,NULL) ; /* make array */ /* copy measure data into the subbrik */ bodset = DSET_ARRAY(cset,subbrik); /* -- Configure the subbriks: Weighted Degree Centrality */ subbrik = 1; EDIT_BRICK_TO_NOSTAT(cset,subbrik) ; /* stat params */ /* CC this sets the subbrik scaling factor, which we will probably want to do again after we calculate the voxel values */ EDIT_BRICK_FACTOR(cset,subbrik,1.0) ; /* scale factor */ sprintf(str,"Weighted Degree Centrality") ; EDIT_BRICK_LABEL(cset,subbrik,str) ; EDIT_substitute_brick(cset,subbrik,MRI_float,NULL) ; /* make array */ /* copy measure data into the subbrik */ wodset = DSET_ARRAY(cset,subbrik); /* increment memory stats */ INC_MEM_STATS( (DSET_NVOX(cset)*DSET_NVALS(cset)*sizeof(float)), "output dset"); PRINT_MEM_STATS( "outset" ); /* pull the values out of the histogram */ if( dosparsity == 0 ) { for( kout = 0; kout < nmask; kout++ ) { if ( imap != NULL ) { ii = imap[kout] ; /* ii= source voxel (we know that ii is in the mask) */ } else { ii = kout ; } if( ii >= DSET_NVOX(cset) ) { WARNING_message("Avoiding bodset, wodset overflow %d > %d (%s,%d)\n", ii,DSET_NVOX(cset),__FILE__,__LINE__ ); } else { bodset[ ii ] = (float)(binaryDC[kout]); wodset[ ii ] = (float)(weightedDC[kout]); } } /* we are done with this memory, and can kill it now*/ if(binaryDC) { free(binaryDC); binaryDC=NULL; /* -- update running memory estimate to reflect memory allocation */ DEC_MEM_STATS( nmask*sizeof(long), "binary DC array" ); PRINT_MEM_STATS( "binaryDC" ); } if(weightedDC) { free(weightedDC); weightedDC=NULL; /* -- update running memory estimate to reflect memory allocation */ DEC_MEM_STATS( nmask*sizeof(double), "weighted DC array" ); PRINT_MEM_STATS( "weightedDC" ); } } else { /* add in the values from the histogram, this is a two stage procedure: at first we add in values a whole bin at the time until we get to a point where we need to add in a partial bin, then we create a new histogram to sort the values in the bin and then add those bins at a time */ kout = nhistnodes - 1; while (( histogram[kout].nbin < nretain ) && ( kout >= 0 )) { hptr = pptr = histogram[kout].nodes; while( hptr != NULL ) { /* determine the indices corresponding to this node */ if ( imap != NULL ) { ii = imap[hptr->i] ; /* ii= source voxel (we know that ii is in the mask) */ } else { ii = hptr->i ; } if ( imap != NULL ) { jj = imap[hptr->j] ; /* ii= source voxel (we know that ii is in the mask) */ } else { jj = hptr->j ; } /* add in the values */ if(( ii >= DSET_NVOX(cset) ) || ( jj >= DSET_NVOX(cset))) { if( ii >= DSET_NVOX(cset)) { WARNING_message("Avoiding bodset, wodset overflow (ii) %d > %d\n (%s,%d)\n", ii,DSET_NVOX(cset),__FILE__,__LINE__ ); } if( jj >= DSET_NVOX(cset)) { WARNING_message("Avoiding bodset, wodset overflow (jj) %d > %d\n (%s,%d)\n", jj,DSET_NVOX(cset),__FILE__,__LINE__ ); } } else { bodset[ ii ] += 1.0 ; wodset[ ii ] += (float)(hptr->corr); bodset[ jj ] += 1.0 ; wodset[ jj ] += (float)(hptr->corr); } if( fout1D != NULL ) { /* add source, dest, correlation to 1D file */ ix1 = DSET_index_to_ix(cset,ii) ; jy1 = DSET_index_to_jy(cset,ii) ; kz1 = DSET_index_to_kz(cset,ii) ; ix2 = DSET_index_to_ix(cset,jj) ; jy2 = DSET_index_to_jy(cset,jj) ; kz2 = DSET_index_to_kz(cset,jj) ; fprintf(fout1D, "%d %d %d %d %d %d %d %d %.6f\n", ii, jj, ix1, jy1, kz1, ix2, jy2, kz2, (float)(hptr->corr)); } /* increment node pointers */ pptr = hptr; hptr = hptr->next; /* delete the node */ if(pptr) { /* -- update running memory estimate to reflect memory allocation */ DEC_MEM_STATS(sizeof( hist_node ), "hist nodes" ); /* free the mem */ free(pptr); pptr=NULL; } } /* decrement the number of correlations we wish to retain */ nretain -= histogram[kout].nbin; histogram[kout].nodes = NULL; /* go on to the next bin */ kout--; } PRINT_MEM_STATS( "hist1 bins free - inc into output" ); /* if we haven't used all of the correlations that are available, go through and add a subset of the voxels from the remaining bin */ if(( nretain > 0 ) && (kout >= 0)) { hist_node_head* histogram2 = NULL; hist_node_head* histogram2_save = NULL; int h2nbins = 100; float h2binwidth = 0.0; int h2ndx=0; h2binwidth = (((1.0+binwidth/((float)h2nbins))*histogram[kout].bin_high) - histogram[kout].bin_low) / ((float)h2nbins); /* allocate the bins */ if(( histogram2 = (hist_node_head*)malloc(h2nbins*sizeof(hist_node_head))) == NULL ) { if (binaryDC){ free(binaryDC); binaryDC = NULL; } if (weightedDC){ free(weightedDC); weightedDC = NULL; } if (histogram){ histogram = free_histogram(histogram, nhistnodes); } ERROR_message( "Could not allocate %d byte array for histogram2\n", h2nbins*sizeof(hist_node_head)); } else { /* -- update running memory estimate to reflect memory allocation */ histogram2_save = histogram2; INC_MEM_STATS(( h2nbins*sizeof(hist_node_head )), "hist bins"); PRINT_MEM_STATS( "hist2" ); } /* initiatize the bins */ for( kin = 0; kin < h2nbins; kin++ ) { histogram2[ kin ].bin_low = histogram[kout].bin_low + kin*h2binwidth; histogram2[ kin ].bin_high = histogram2[ kin ].bin_low + h2binwidth; histogram2[ kin ].nbin = 0; histogram2[ kin ].nodes = NULL; /*INFO_message("Hist2 bin %d [%3.3f, %3.3f) [%d, %p]\n", kin, histogram2[ kin ].bin_low, histogram2[ kin ].bin_high, histogram2[ kin ].nbin, histogram2[ kin ].nodes );*/ } /* move correlations from histogram to histgram2 */ INFO_message ("Adding %d nodes from histogram to histogram2",histogram[kout].nbin); while ( histogram[kout].nodes != NULL ) { hptr = histogram[kout].nodes; h2ndx = (int)floor((double)(hptr->corr - histogram[kout].bin_low)/(double)h2binwidth); if(( h2ndx < h2nbins ) && ( h2ndx >= 0 )) { histogram[kout].nodes = hptr->next; hptr->next = histogram2[h2ndx].nodes; histogram2[h2ndx].nodes = hptr; histogram2[h2ndx].nbin++; histogram[kout].nbin--; } else { WARNING_message("h2ndx %d is not in range [0,%d) :: %.10f,%.10f\n",h2ndx,h2nbins,hptr->corr, histogram[kout].bin_low); } } /* free the remainder of histogram */ { int nbins_rem = 0; for(ii = 0; ii < nhistnodes; ii++) nbins_rem+=histogram[ii].nbin; histogram = free_histogram(histogram, nhistnodes); PRINT_MEM_STATS( "free remainder of histogram1" ); } kin = h2nbins - 1; while (( nretain > 0 ) && ( kin >= 0 )) { hptr = pptr = histogram2[kin].nodes; while( hptr != NULL ) { /* determine the indices corresponding to this node */ if ( imap != NULL ) { ii = imap[hptr->i] ; } else { ii = hptr->i ; } if ( imap != NULL ) { jj = imap[hptr->j] ; } else { jj = hptr->j ; } /* add in the values */ if(( ii >= DSET_NVOX(cset) ) || ( jj >= DSET_NVOX(cset))) { if( ii >= DSET_NVOX(cset)) { WARNING_message("Avoiding bodset, wodset overflow (ii) %d > %d\n (%s,%d)\n", ii,DSET_NVOX(cset),__FILE__,__LINE__ ); } if( jj >= DSET_NVOX(cset)) { WARNING_message("Avoiding bodset, wodset overflow (jj) %d > %d\n (%s,%d)\n", jj,DSET_NVOX(cset),__FILE__,__LINE__ ); } } else { bodset[ ii ] += 1.0 ; wodset[ ii ] += (float)(hptr->corr); bodset[ jj ] += 1.0 ; wodset[ jj ] += (float)(hptr->corr); } if( fout1D != NULL ) { /* add source, dest, correlation to 1D file */ ix1 = DSET_index_to_ix(cset,ii) ; jy1 = DSET_index_to_jy(cset,ii) ; kz1 = DSET_index_to_kz(cset,ii) ; ix2 = DSET_index_to_ix(cset,jj) ; jy2 = DSET_index_to_jy(cset,jj) ; kz2 = DSET_index_to_kz(cset,jj) ; fprintf(fout1D, "%d %d %d %d %d %d %d %d %.6f\n", ii, jj, ix1, jy1, kz1, ix2, jy2, kz2, (float)(hptr->corr)); } /* increment node pointers */ pptr = hptr; hptr = hptr->next; /* delete the node */ if(pptr) { free(pptr); DEC_MEM_STATS(( sizeof(hist_node) ), "hist nodes"); pptr=NULL; } } /* decrement the number of correlations we wish to retain */ nretain -= histogram2[kin].nbin; histogram2[kin].nodes = NULL; /* go on to the next bin */ kin--; } PRINT_MEM_STATS("hist2 nodes free - incorporated into output"); /* we are finished with histogram2 */ { histogram2 = free_histogram(histogram2, h2nbins); /* -- update running memory estimate to reflect memory allocation */ PRINT_MEM_STATS( "free hist2" ); } if (nretain < 0 ) { WARNING_message( "Went over sparsity goal %d by %d, with a resolution of %f", ngoal, -1*nretain, h2binwidth); } } if (nretain > 0 ) { WARNING_message( "Was not able to meet goal of %d (%3.2f%%) correlations, %d (%3.2f%%) correlations passed the threshold of %3.2f, maybe you need to change the threshold or the desired sparsity?", ngoal, 100.0*((float)ngoal)/((float)totPosCor), totNumCor, 100.0*((float)totNumCor)/((float)totPosCor), thresh); } } INFO_message("Done..\n") ; /* update running memory statistics to reflect freeing the vectim */ DEC_MEM_STATS(((xvectim->nvec*sizeof(int)) + ((xvectim->nvec)*(xvectim->nvals))*sizeof(float) + sizeof(MRI_vectim)), "vectim"); /* toss some trash */ VECTIM_destroy(xvectim) ; DSET_delete(xset) ; if(fout1D!=NULL)fclose(fout1D); PRINT_MEM_STATS( "vectim unload" ); if (weightedDC) free(weightedDC) ; weightedDC = NULL; if (binaryDC) free(binaryDC) ; binaryDC = NULL; /* finito */ INFO_message("Writing output dataset to disk [%s bytes]", commaized_integer_string(cset->dblk->total_bytes)) ; /* write the dataset */ DSET_write(cset) ; WROTE_DSET(cset) ; /* increment our memory stats, since we are relying on the header for this information, we update the stats before actually freeing the memory */ DEC_MEM_STATS( (DSET_NVOX(cset)*DSET_NVALS(cset)*sizeof(float)), "output dset"); /* free up the output dataset memory */ DSET_unload(cset) ; DSET_delete(cset) ; /* force a print */ MEM_STAT = 1; PRINT_MEM_STATS( "Fin" ); exit(0) ; }
THD_3dim_dataset * THD_localhistog( int nsar , THD_3dim_dataset **insar , int numval , int *rlist , MCW_cluster *nbhd , int do_prob , int verb ) { THD_3dim_dataset *outset=NULL , *inset ; int nvox=DSET_NVOX(insar[0]) ; int ids, iv, bb, nnpt=nbhd->num_pt ; MRI_IMAGE *bbim ; int btyp ; float **outar , **listar ; ENTRY("THD_localhistog") ; /*---- create output dataset ----*/ outset = EDIT_empty_copy(insar[0]) ; EDIT_dset_items( outset , ADN_nvals , numval , ADN_datum_all , MRI_float , ADN_nsl , 0 , ADN_brick_fac , NULL , ADN_none ) ; outar = (float **)malloc(sizeof(float *)*numval) ; for( bb=0 ; bb < numval ; bb++ ){ EDIT_substitute_brick( outset , bb , MRI_float , NULL ) ; outar[bb] = DSET_BRICK_ARRAY(outset,bb) ; } /*---- make mapping between values and arrays to get those values ----*/ listar = (float **)malloc(sizeof(float *)*TWO16) ; for( bb=0 ; bb < TWO16 ; bb++ ) listar[bb] = outar[0] ; for( bb=1 ; bb < numval ; bb++ ){ listar[ rlist[bb] + TWO15 ] = outar[bb] ; } /*----------- loop over datasets, add in counts for all voxels -----------*/ for( ids=0 ; ids < nsar ; ids++ ){ /* dataset loop */ inset = insar[ids] ; DSET_load(inset) ; for( iv=0 ; iv < DSET_NVALS(inset) ; iv++ ){ /* sub-brick loop */ if( verb ) fprintf(stderr,".") ; bbim = DSET_BRICK(inset,iv) ; btyp = bbim->kind ; if( nnpt == 1 ){ /* only 1 voxel in nbhd */ int qq,ii,jj,kk,ib,nb ; switch( bbim->kind ){ case MRI_short:{ short *sar = MRI_SHORT_PTR(bbim) ; for( qq=0 ; qq < nvox ; qq++ ) listar[sar[qq]+TWO15][qq]++ ; } break ; case MRI_byte:{ byte *bar = MRI_BYTE_PTR(bbim) ; for( qq=0 ; qq < nvox ; qq++ ) listar[bar[qq]+TWO15][qq]++ ; } break ; case MRI_float:{ float *far = MRI_FLOAT_PTR(bbim) ; short ss ; for( qq=0 ; qq < nvox ; qq++ ){ ss = SHORTIZE(far[qq]); listar[ss+TWO15][qq]++; } } break ; } } else { /* multiple voxels in nbhd */ AFNI_OMP_START ; #pragma omp parallel { int qq,ii,jj,kk,ib,nb ; void *nar ; short *sar,ss ; byte *bar ; float *far ; nar = malloc(sizeof(float)*nnpt) ; sar = (short *)nar ; bar = (byte *)nar ; far = (float *)nar ; #pragma omp for for( qq=0 ; qq < nvox ; qq++ ){ /* qq=voxel index */ ii = DSET_index_to_ix(inset,qq) ; jj = DSET_index_to_jy(inset,qq) ; kk = DSET_index_to_kz(inset,qq) ; nb = mri_get_nbhd_array( bbim , NULL , ii,jj,kk , nbhd , nar ) ; if( nb == 0 ) continue ; switch( btyp ){ case MRI_short: for( ib=0 ; ib < nb ; ib++ ) listar[sar[ib]+TWO15][qq]++ ; break ; case MRI_byte: for( ib=0 ; ib < nb ; ib++ ) listar[bar[ib]+TWO15][qq]++ ; break ; case MRI_float: for( ib=0 ; ib < nb ; ib++ ){ ss = SHORTIZE(far[ib]); listar[ss+TWO15][qq]++; } break ; } } /* end of voxel loop */ free(nar) ; } /* end of OpenMP */ AFNI_OMP_END ; } } /* end of sub-brick loop */ DSET_unload(inset) ; } /* end of dataset loop */ if( verb ) fprintf(stderr,"\n") ; free(listar) ; /*---- post-process output ---*/ if( do_prob ){ byte **bbar ; int pp ; if( verb ) INFO_message("Conversion to probabilities") ; AFNI_OMP_START ; #pragma omp parallel { int qq , ib ; float pfac , val ; byte **bbar ; #pragma omp for for( qq=0 ; qq < nvox ; qq++ ){ pfac = 0.0001f ; for( ib=0 ; ib < numval ; ib++ ) pfac += outar[ib][qq] ; pfac = 250.0f / pfac ; for( ib=0 ; ib < numval ; ib++ ){ val = outar[ib][qq]*pfac ; outar[ib][qq] = BYTEIZE(val) ; } } } /* end OpenMP */ AFNI_OMP_END ; bbar = (byte **)malloc(sizeof(byte *)*numval) ; for( bb=0 ; bb < numval ; bb++ ){ bbar[bb] = (byte *)malloc(sizeof(byte)*nvox) ; for( pp=0 ; pp < nvox ; pp++ ) bbar[bb][pp] = (byte)outar[bb][pp] ; EDIT_substitute_brick(outset,bb,MRI_byte,bbar[bb]) ; EDIT_BRICK_FACTOR(outset,bb,0.004f) ; } free(bbar) ; } /* end of do_prob */ free(outar) ; RETURN(outset) ; }
int main( int argc , char *argv[] ) { THD_3dim_dataset *dset , *oset=NULL , *tset=NULL ; int nvals , iv , nxyz , ii,jj,kk , iarg , kz,kzold ; float cut1=2.5,cut2=4.0 , sq2p,sfac , fq ; MRI_IMAGE *flim ; char *prefix="despike" , *tprefix=NULL ; int corder=-1 , nref , ignore=0 , polort=2 , nuse , nomask=0 ; int nspike, nbig, nproc ; float **ref ; float c21,ic21 , pspike,pbig ; short *sar , *qar ; byte *tar , *mask=NULL ; float *zar , *yar ; int datum ; int localedit=0 ; /* 04 Apr 2007 */ int verb=1 ; int do_NEW = 0 ; /* 29 Nov 2013 */ MRI_IMAGE *NEW_psinv=NULL ; int dilate = 4 ; /* 04 Dec 2013 */ int ctim = 0 ; /*----- Read command line -----*/ AFNI_SETUP_OMP(0) ; /* 24 Jun 2013 */ if( argc < 2 || strcmp(argv[1],"-help") == 0 ){ printf("Usage: 3dDespike [options] dataset\n" "Removes 'spikes' from the 3D+time input dataset and writes\n" "a new dataset with the spike values replaced by something\n" "more pleasing to the eye.\n" "\n" "Method:\n" " * L1 fit a smooth-ish curve to each voxel time series\n" " [see -corder option for description of the curve]\n" " [see -NEW option for a different & faster fitting method]\n" " * Compute the MAD of the difference between the curve and\n" " the data time series (the residuals).\n" " * Estimate the standard deviation 'sigma' of the residuals\n" " as sqrt(PI/2)*MAD.\n" " * For each voxel value, define s = (value-curve)/sigma.\n" " * Values with s > c1 are replaced with a value that yields\n" " a modified s' = c1+(c2-c1)*tanh((s-c1)/(c2-c1)).\n" " * c1 is the threshold value of s for a 'spike' [default c1=2.5].\n" " * c2 is the upper range of the allowed deviation from the curve:\n" " s=[c1..infinity) is mapped to s'=[c1..c2) [default c2=4].\n" "\n" "Options:\n" " -ignore I = Ignore the first I points in the time series:\n" " these values will just be copied to the\n" " output dataset [default I=0].\n" " -corder L = Set the curve fit order to L:\n" " the curve that is fit to voxel data v(t) is\n" "\n" " k=L [ (2*PI*k*t) (2*PI*k*t) ]\n" " f(t) = a+b*t+c*t*t + SUM [ d * sin(--------) + e * cos(--------) ]\n" " k=1 [ k ( T ) k ( T ) ]\n" "\n" " where T = duration of time series;\n" " the a,b,c,d,e parameters are chosen to minimize\n" " the sum over t of |v(t)-f(t)| (L1 regression);\n" " this type of fitting is is insensitive to large\n" " spikes in the data. The default value of L is\n" " NT/30, where NT = number of time points.\n" "\n" " -cut c1 c2 = Alter default values for the spike cut values\n" " [default c1=2.5, c2=4.0].\n" " -prefix pp = Save de-spiked dataset with prefix 'pp'\n" " [default pp='despike']\n" " -ssave ttt = Save 'spikiness' measure s for each voxel into a\n" " 3D+time dataset with prefix 'ttt' [default=no save]\n" " -nomask = Process all voxels\n" " [default=use a mask of high-intensity voxels, ]\n" " [as created via '3dAutomask -dilate 4 dataset'].\n" " -dilate nd = Dilate 'nd' times (as in 3dAutomask). The default\n" " value of 'nd' is 4.\n" " -q[uiet] = Don't print '++' informational messages.\n" "\n" " -localedit = Change the editing process to the following:\n" " If a voxel |s| value is >= c2, then replace\n" " the voxel value with the average of the two\n" " nearest non-spike (|s| < c2) values; the first\n" " one previous and the first one after.\n" " Note that the c1 cut value is not used here.\n" "\n" " -NEW = Use the 'new' method for computing the fit, which\n" " should be faster than the L1 method for long time\n" " series (200+ time points); however, the results\n" " are similar but NOT identical. [29 Nov 2013]\n" " * You can also make the program use the 'new'\n" " method by setting the environment variable\n" " AFNI_3dDespike_NEW\n" " to the value YES; as in\n" " setenv AFNI_3dDespike_NEW YES (csh)\n" " export AFNI_3dDespike_NEW=YES (bash)\n" " * If this variable is set to YES, you can turn off\n" " the '-NEW' processing by using the '-OLD' option.\n" " -->>* For time series more than 500 points long, the\n" " '-OLD' algorithm is tremendously slow. You should\n" " use the '-NEW' algorith in such cases.\n" " ** At some indeterminate point in the future, the '-NEW'\n" " method will become the default!\n" " -->>* As of 29 Sep 2016, '-NEW' is the default if there\n" " is more than 500 points in the time series dataset.\n" "\n" " -NEW25 = A slightly more aggressive despiking approach than\n" " the '-NEW' method.\n" "\n" "Caveats:\n" "* Despiking may interfere with image registration, since head\n" " movement may produce 'spikes' at the edge of the brain, and\n" " this information would be used in the registration process.\n" " This possibility has not been explored or calibrated.\n" "* [LATER] Actually, it seems like the registration problem\n" " does NOT happen, and in fact, despiking seems to help!\n" "* Check your data visually before and after despiking and\n" " registration!\n" " [Hint: open 2 AFNI controllers, and turn Time Lock on.]\n" ) ; PRINT_AFNI_OMP_USAGE("3dDespike",NULL) ; PRINT_COMPILE_DATE ; exit(0) ; } /** AFNI package setup and logging **/ mainENTRY("3dDespike main"); machdep(); AFNI_logger("3dDespike",argc,argv); PRINT_VERSION("3dDespike") ; AUTHOR("RW Cox") ; /** parse options **/ if( AFNI_yesenv("AFNI_3dDespike_NEW") ) do_NEW = 1 ; /* 29 Nov 2013 */ iarg = 1 ; while( iarg < argc && argv[iarg][0] == '-' ){ if( strncmp(argv[iarg],"-q",2) == 0 ){ /* 04 Apr 2007 */ verb = 0 ; iarg++ ; continue ; } if( strncmp(argv[iarg],"-v",2) == 0 ){ verb++ ; iarg++ ; continue ; } if( strcmp(argv[iarg],"-NEW") == 0 ){ /* 29 Nov 2013 */ do_NEW = 1 ; iarg++ ; continue ; } if( strcmp(argv[iarg],"-NEW25") == 0 ){ /* 29 Sep 2016 */ do_NEW = 1 ; use_des25 = 1 ; cut1 = 2.5f ; cut2 = 3.2f ; iarg++ ; continue ; } if( strcmp(argv[iarg],"-OLD") == 0 ){ do_NEW = 0 ; iarg++ ; continue ; } /** -localedit **/ if( strcmp(argv[iarg],"-localedit") == 0 ){ /* 04 Apr 2007 */ localedit = 1 ; iarg++ ; continue ; } /** don't use masking **/ if( strcmp(argv[iarg],"-nomask") == 0 ){ nomask = 1 ; iarg++ ; continue ; } /** dilation count [04 Dec 2013] **/ if( strcmp(argv[iarg],"-dilate") == 0 ){ dilate = (int)strtod(argv[++iarg],NULL) ; if( dilate <= 0 ) dilate = 1 ; else if( dilate > 99 ) dilate = 99 ; iarg++ ; continue ; } /** output dataset prefix **/ if( strcmp(argv[iarg],"-prefix") == 0 ){ prefix = argv[++iarg] ; if( !THD_filename_ok(prefix) ) ERROR_exit("-prefix is not good"); iarg++ ; continue ; } /** ratio dataset prefix **/ if( strcmp(argv[iarg],"-ssave") == 0 ){ tprefix = argv[++iarg] ; if( !THD_filename_ok(tprefix) ) ERROR_exit("-ssave prefix is not good"); iarg++ ; continue ; } /** trigonometric polynomial order **/ if( strcmp(argv[iarg],"-corder") == 0 ){ corder = strtol( argv[++iarg] , NULL , 10 ) ; if( corder < 0 ) ERROR_exit("Illegal value of -corder"); iarg++ ; continue ; } /** how much to ignore at start **/ if( strcmp(argv[iarg],"-ignore") == 0 ){ ignore = strtol( argv[++iarg] , NULL , 10 ) ; if( ignore < 0 ) ERROR_exit("Illegal value of -ignore"); iarg++ ; continue ; } /** thresholds for s ratio **/ if( strcmp(argv[iarg],"-cut") == 0 ){ cut1 = strtod( argv[++iarg] , NULL ) ; cut2 = strtod( argv[++iarg] , NULL ) ; if( cut1 < 1.0 || cut2 < cut1+0.5 ) ERROR_exit("Illegal values after -cut"); iarg++ ; continue ; } ERROR_exit("Unknown option: %s",argv[iarg]) ; } c21 = cut2-cut1 ; ic21 = 1.0/c21 ; /*----- read input dataset -----*/ if( iarg >= argc ) ERROR_exit("No input dataset!!??"); dset = THD_open_dataset( argv[iarg] ) ; CHECK_OPEN_ERROR(dset,argv[iarg]) ; datum = DSET_BRICK_TYPE(dset,0) ; if( (datum != MRI_short && datum != MRI_float) || !DSET_datum_constant(dset) ) ERROR_exit("Can't process non-short, non-float dataset!") ; if( verb ) INFO_message("Input data type = %s\n",MRI_TYPE_name[datum]) ; nvals = DSET_NUM_TIMES(dset) ; nuse = nvals - ignore ; if( nuse < 15 ) ERROR_exit("Can't use dataset with < 15 time points per voxel!") ; if( nuse > 500 && !do_NEW ){ INFO_message("Switching to '-NEW' method since number of time points = %d > 500",nuse) ; do_NEW = 1 ; } if( use_des25 && nuse < 99 ) use_des25 = 0 ; if( verb ) INFO_message("ignoring first %d time points, using last %d",ignore,nuse); if( corder > 0 && 4*corder+2 > nuse ){ ERROR_exit("-corder %d is too big for NT=%d",corder,nvals) ; } else if( corder < 0 ){ corder = rint(nuse/30.0) ; if( corder > 50 && !do_NEW ) corder = 50 ; if( verb ) INFO_message("using %d time points => -corder %d",nuse,corder) ; } else { if( verb ) INFO_message("-corder %d set from command line",corder) ; } nxyz = DSET_NVOX(dset) ; if( verb ) INFO_message("Loading dataset %s",argv[iarg]) ; DSET_load(dset) ; CHECK_LOAD_ERROR(dset) ; /*-- create automask --*/ if( !nomask ){ mask = THD_automask( dset ) ; if( verb ){ ii = THD_countmask( DSET_NVOX(dset) , mask ) ; INFO_message("%d voxels in the automask [out of %d in dataset]",ii,DSET_NVOX(dset)) ; } for( ii=0 ; ii < dilate ; ii++ ) THD_mask_dilate( DSET_NX(dset), DSET_NY(dset), DSET_NZ(dset), mask, 3 ) ; if( verb ){ ii = THD_countmask( DSET_NVOX(dset) , mask ) ; INFO_message("%d voxels in the dilated automask [out of %d in dataset]",ii,DSET_NVOX(dset)) ; } } else { if( verb ) INFO_message("processing all %d voxels in dataset",DSET_NVOX(dset)) ; } /*-- create empty despiked dataset --*/ oset = EDIT_empty_copy( dset ) ; EDIT_dset_items( oset , ADN_prefix , prefix , ADN_brick_fac , NULL , ADN_datum_all , datum , ADN_none ) ; if( THD_deathcon() && THD_is_file(DSET_HEADNAME(oset)) ) ERROR_exit("output dataset already exists: %s",DSET_HEADNAME(oset)); tross_Copy_History( oset , dset ) ; tross_Make_History( "3dDespike" , argc , argv , oset ) ; /* create bricks (will be filled with zeros) */ for( iv=0 ; iv < nvals ; iv++ ) EDIT_substitute_brick( oset , iv , datum , NULL ) ; /* copy the ignored bricks */ switch( datum ){ case MRI_short: for( iv=0 ; iv < ignore ; iv++ ){ sar = DSET_ARRAY(oset,iv) ; qar = DSET_ARRAY(dset,iv) ; memcpy( sar , qar , DSET_BRICK_BYTES(dset,iv) ) ; DSET_unload_one(dset,iv) ; } break ; case MRI_float: for( iv=0 ; iv < ignore ; iv++ ){ zar = DSET_ARRAY(oset,iv) ; yar = DSET_ARRAY(dset,iv) ; memcpy( zar , yar , DSET_BRICK_BYTES(dset,iv) ) ; DSET_unload_one(dset,iv) ; } break ; } /*-- setup to save a threshold statistic dataset, if desired --*/ if( tprefix != NULL ){ float *fac ; tset = EDIT_empty_copy( dset ) ; fac = (float *) malloc( sizeof(float) * nvals ) ; for( ii=0 ; ii < nvals ; ii++ ) fac[ii] = TFAC ; EDIT_dset_items( tset , ADN_prefix , tprefix , ADN_brick_fac , fac , ADN_datum_all , MRI_byte , ADN_func_type , FUNC_FIM_TYPE , ADN_none ) ; free(fac) ; tross_Copy_History( tset , dset ) ; tross_Make_History( "3dDespike" , argc , argv , tset ) ; #if 0 if( THD_is_file(DSET_HEADNAME(tset)) ) ERROR_exit("-ssave dataset already exists"); #endif tross_Copy_History( tset , dset ) ; tross_Make_History( "3dDespike" , argc , argv , tset ) ; for( iv=0 ; iv < nvals ; iv++ ) EDIT_substitute_brick( tset , iv , MRI_byte , NULL ) ; } /*-- setup to find spikes --*/ sq2p = sqrt(0.5*PI) ; sfac = sq2p / 1.4826f ; /* make ref functions */ nref = 2*corder+3 ; ref = (float **) malloc( sizeof(float *) * nref ) ; for( jj=0 ; jj < nref ; jj++ ) ref[jj] = (float *) malloc( sizeof(float) * nuse ) ; /* r(t) = 1 */ for( iv=0 ; iv < nuse ; iv++ ) ref[0][iv] = 1.0 ; jj = 1 ; /* r(t) = t - tmid */ { float tm = 0.5 * (nuse-1.0) ; float fac = 2.0 / nuse ; for( iv=0 ; iv < nuse ; iv++ ) ref[1][iv] = (iv-tm)*fac ; jj = 2 ; /* r(t) = (t-tmid)**jj */ for( ; jj <= polort ; jj++ ) for( iv=0 ; iv < nuse ; iv++ ) ref[jj][iv] = pow( (iv-tm)*fac , (double)jj ) ; } for( kk=1 ; kk <= corder ; kk++ ){ fq = (2.0*PI*kk)/nuse ; /* r(t) = sin(2*PI*k*t/N) */ for( iv=0 ; iv < nuse ; iv++ ) ref[jj][iv] = sin(fq*iv) ; jj++ ; /* r(t) = cos(2*PI*k*t/N) */ for( iv=0 ; iv < nuse ; iv++ ) ref[jj][iv] = cos(fq*iv) ; jj++ ; } /****** setup for the NEW solution method [29 Nov 2013] ******/ if( do_NEW ){ NEW_psinv = DES_get_psinv(nuse,nref,ref) ; INFO_message("Procesing time series with NEW model fit algorithm") ; } else { INFO_message("Procesing time series with OLD model fit algorithm") ; } /*--- loop over voxels and do work ---*/ #define Laplace_t2p(val) ( 1.0 - nifti_stat2cdf( (val), 15, 0.0, 1.4427 , 0.0 ) ) if( verb ){ if( !localedit ){ INFO_message("smash edit thresholds: %.1f .. %.1f MADs",cut1*sq2p,cut2*sq2p) ; ININFO_message(" [ %.3f%% .. %.3f%% of normal distribution]", 200.0*qg(cut1*sfac) , 200.0*qg(cut2*sfac) ) ; ININFO_message(" [ %.3f%% .. %.3f%% of Laplace distribution]" , 100.0*Laplace_t2p(cut1) , 100.0*Laplace_t2p(cut2) ) ; } else { INFO_message("local edit threshold: %.1f MADS",cut2*sq2p) ; ININFO_message(" [ %.3f%% of normal distribution]", 200.0*qg(cut2*sfac) ) ; ININFO_message(" [ %.3f%% of Laplace distribution]", 100.0*Laplace_t2p(cut1) ) ; } INFO_message("%d slices to process",DSET_NZ(dset)) ; } kzold = -1 ; nspike = 0 ; nbig = 0 ; nproc = 0 ; ctim = NI_clock_time() ; AFNI_OMP_START ; #pragma omp parallel if( nxyz > 6666 ) { int ii , iv , iu , id , jj ; float *far , *dar , *var , *fitar , *ssp , *fit , *zar ; short *sar , *qar ; byte *tar ; float fsig , fq , cls , snew , val ; float *NEW_wks=NULL ; #pragma omp critical (DESPIKE_malloc) { far = (float *) malloc( sizeof(float) * nvals ) ; dar = (float *) malloc( sizeof(float) * nvals ) ; var = (float *) malloc( sizeof(float) * nvals ) ; fitar = (float *) malloc( sizeof(float) * nvals ) ; ssp = (float *) malloc( sizeof(float) * nvals ) ; fit = (float *) malloc( sizeof(float) * nref ) ; if( do_NEW ) NEW_wks = (float *)malloc(sizeof(float)*DES_workspace_size(nuse,nref)) ; } #ifdef USE_OMP INFO_message("start OpenMP thread #%d",omp_get_thread_num()) ; #endif #pragma omp for for( ii=0 ; ii < nxyz ; ii++ ){ /* ii = voxel index */ if( mask != NULL && mask[ii] == 0 ) continue ; /* skip this voxel */ #ifndef USE_OMP kz = DSET_index_to_kz(dset,ii) ; /* starting a new slice */ if( kz != kzold ){ if( verb ){ fprintf(stderr, "++ start slice %2d",kz ) ; if( nproc > 0 ){ pspike = (100.0*nspike)/nproc ; pbig = (100.0*nbig )/nproc ; fprintf(stderr, "; so far %d data points, %d edits [%.3f%%], %d big edits [%.3f%%]", nproc,nspike,pspike,nbig,pbig ) ; } fprintf(stderr,"\n") ; } kzold = kz ; } #else if( verb && ii % 2345 == 1234 ) fprintf(stderr,".") ; #endif /*** extract ii-th time series into far[] ***/ switch( datum ){ case MRI_short: for( iv=0 ; iv < nuse ; iv++ ){ qar = DSET_ARRAY(dset,iv+ignore) ; /* skip ignored data */ far[iv] = (float)qar[ii] ; } break ; case MRI_float: for( iv=0 ; iv < nuse ; iv++ ){ zar = DSET_ARRAY(dset,iv+ignore) ; far[iv] = zar[ii] ; } break ; } AAmemcpy(dar,far,sizeof(float)*nuse) ; /* copy time series into dar[] */ /*** solve for L1 fit ***/ if( do_NEW ) cls = DES_solve( NEW_psinv , far , fit , NEW_wks ) ; /* 29 Nov 2013 */ else cls = cl1_solve( nuse , nref , far , ref , fit,0 ) ; /* the slow part */ if( cls < 0.0f ){ /* fit failed! */ #if 0 fprintf(stderr,"curve fit fails at voxel %d %d %d\n", DSET_index_to_ix(dset,ii) , DSET_index_to_jy(dset,ii) , DSET_index_to_kz(dset,ii) ) ; #endif continue ; /* skip this voxel */ } for( iv=0 ; iv < nuse ; iv++ ){ /* detrend */ val = fit[0] + fit[1]*ref[1][iv] /* quadratic part of curve fit */ + fit[2]*ref[2][iv] ; for( jj=3 ; jj < nref ; jj++ ) /* rest of curve fit */ val += fit[jj] * ref[jj][iv] ; fitar[iv] = val ; /* save curve fit value */ var[iv] = dar[iv]-val ; /* remove fitted value = resid */ far[iv] = fabsf(var[iv]) ; /* abs value of resid */ } /*** compute estimate standard deviation of detrended data ***/ fsig = sq2p * qmed_float(nuse,far) ; /* also mangles far array */ /*** process time series for spikes, editing data in dar[] ***/ if( fsig > 0.0f ){ /* data wasn't fit perfectly */ /* find spikiness for each point in time */ fq = 1.0f / fsig ; for( iv=0 ; iv < nuse ; iv++ ){ ssp[iv] = fq * var[iv] ; /* spikiness s = how many sigma out */ } /* save spikiness in -ssave datset */ if( tset != NULL ){ for( iv=0 ; iv < nuse ; iv++ ){ tar = DSET_ARRAY(tset,iv+ignore) ; snew = ITFAC*fabsf(ssp[iv]) ; /* scale for byte storage */ tar[ii] = BYTEIZE(snew) ; /* cf. mrilib.h */ } } /* process values of |s| > cut1, editing dar[] */ for( iv=0 ; iv < nuse ; iv++ ){ /* loop over time points */ if( !localedit ){ /** classic 'smash' edit **/ if( ssp[iv] > cut1 ){ snew = cut1 + c21*mytanh((ssp[iv]-cut1)*ic21) ; /* edit s down */ dar[iv] = fitar[iv] + snew*fsig ; #pragma omp critical (DESPIKE_counter) { nspike++ ; if( ssp[iv] > cut2 ) nbig++ ; } } else if( ssp[iv] < -cut1 ){ snew = -cut1 + c21*mytanh((ssp[iv]+cut1)*ic21) ; /* edit s up */ dar[iv] = fitar[iv] + snew*fsig ; #pragma omp critical (DESPIKE_counter) { nspike++ ; if( ssp[iv] < -cut2 ) nbig++ ; } } } else { /** local edit: 04 Apr 2007 **/ if( ssp[iv] >= cut2 || ssp[iv] <= -cut2 ){ for( iu=iv+1 ; iu < nuse ; iu++ ) /* find non-spike above */ if( ssp[iu] < cut2 && ssp[iu] > -cut2 ) break ; for( id=iv-1 ; id >= 0 ; id-- ) /* find non-spike below */ if( ssp[id] < cut2 && ssp[id] > -cut2 ) break ; switch( (id>=0) + 2*(iu<nuse) ){ /* compute replacement val */ case 3: val = 0.5*(dar[iu]+dar[id]); break; /* iu and id OK */ case 2: val = dar[iu] ; break; /* only iu OK */ case 1: val = dar[id] ; break; /* only id OK */ default: val = fitar[iv] ; break; /* shouldn't be */ } dar[iv] = val ; #pragma omp critical (DESPIKE_counter) { nspike++ ; nbig++ ; } } } } /* end of loop over time points */ #pragma omp atomic nproc += nuse ; /* number data points processed */ } /* end of processing time series when fsig is positive */ /* put dar[] time series (possibly edited above) into output bricks */ switch( datum ){ case MRI_short: for( iv=0 ; iv < nuse ; iv++ ){ sar = DSET_ARRAY(oset,iv+ignore) ; /* output brick */ sar[ii] = (short)dar[iv] ; /* original or mutated data */ } break ; case MRI_float: for( iv=0 ; iv < nuse ; iv++ ){ zar = DSET_ARRAY(oset,iv+ignore) ; /* output brick */ zar[ii] = dar[iv] ; /* original or mutated data */ } break ; } } /* end of loop over voxels #ii */ #pragma omp critical (DESPIKE_malloc) { free(fit); free(ssp); free(fitar); free(var); free(dar); free(far); if( do_NEW ) free(NEW_wks) ; } } /* end OpenMP */ AFNI_OMP_END ; #ifdef USE_OMP if( verb ) fprintf(stderr,"\n") ; #endif ctim = NI_clock_time() - ctim ; INFO_message( "Elapsed despike time = %s" , nice_time_string(ctim) ) ; if( ctim > 345678 && !do_NEW ) ININFO_message("That was SLOW -- try the '-NEW' option for a speedup") ; #ifdef USE_OMP if( verb ) fprintf(stderr,"\n") ; #endif /*--- finish up ---*/ if( do_NEW ) mri_free(NEW_psinv) ; DSET_delete(dset) ; /* delete input dataset */ if( verb ){ if( nproc > 0 ){ pspike = (100.0*nspike)/nproc ; pbig = (100.0*nbig )/nproc ; INFO_message("FINAL: %d data points, %d edits [%.3f%%], %d big edits [%.3f%%]", nproc,nspike,pspike,nbig,pbig ) ; } else { INFO_message("FINAL: no good voxels found to process!!??") ; } } /* write results */ DSET_write(oset) ; if( verb ) WROTE_DSET(oset) ; DSET_delete(oset) ; if( tset != NULL ){ DSET_write(tset) ; if( verb ) WROTE_DSET(tset) ; DSET_delete(tset) ; } exit( THD_get_write_error_count() ) ; }