real show_moment(Moment *m, int mom)
{
if (m->mom < 0) error("Cannot show moment for mom=%d",m->mom);
if (mom >= 0 && mom <= m->mom)
return m->sum[mom];
else if (mom == -1)
return mean_moment(m);
else if (mom == -2)
return sigma_moment(m);
else if (mom == -3)
return skewness_moment(m);
else if (mom == -4)
return kurtosis_moment(m);
else
warning("show_moment: out-of-range moment %d %d",mom);
return 0.0;
}
nemo_main()
{
int i, j, k;
real x, xmin, xmax, mean, sigma, skew, kurt, median, bad, w, *data;
real sum, sov;
Moment m;
bool Qmin, Qmax, Qbad, Qw, Qmedian, Qmmcount = getbparam("mmcount");
real nu, nppb = getdparam("nppb");
int npar = getiparam("npar");
int ngood = 0;
int min_count, max_count;
instr = stropen (getparam("in"), "r");
read_image (instr,&iptr);
strclose(instr);
if (hasvalue("win")) {
instr = stropen (getparam("win"), "r");
read_image (instr,&wptr);
strclose(instr);
if (Nx(iptr) != Nx(wptr)) error("X sizes of in/win don't match");
if (Ny(iptr) != Ny(wptr)) error("X sizes of in/win don't match");
if (Nz(iptr) != Nz(wptr)) error("X sizes of in/win don't match");
Qw = TRUE;
} else
Qw = FALSE;
nx = Nx(iptr);
ny = Ny(iptr);
nz = Nz(iptr);
Qmin = hasvalue("min");
if (Qmin) xmin = getdparam("min");
Qmax = hasvalue("max");
if (Qmax) xmax = getdparam("max");
Qbad = hasvalue("bad");
if (Qbad) bad = getdparam("bad");
Qmedian = getbparam("median");
if (Qmedian)
data = (real *) allocate(nx*ny*nz*sizeof(real));
sov = Dx(iptr)*Dy(iptr)*Dz(iptr); /* voxel volume; TODO: should we do 2D vs. 3D ? */
ini_moment(&m,4,0);
for (i=0; i<nx; i++) {
for (j=0; j<ny; j++) {
for (k=0; k<nz; k++) {
x = CubeValue(iptr,i,j,k);
if (Qmin && x<xmin) continue;
if (Qmax && x>xmax) continue;
if (Qbad && x==bad) continue;
w = Qw ? CubeValue(wptr,i,j,k) : 1.0;
accum_moment(&m,x,w);
if (Qmedian) data[ngood++] = x;
}
}
}
if (npar > 0) {
nu = n_moment(&m)/nppb - npar;
if (nu < 1) error("%g: No degrees of freedom",nu);
printf("chi2= %g\n", show_moment(&m,2)/nu/nppb);
printf("df= %g\n", nu);
} else {
nsize = nx * ny * nz;
mean = mean_moment(&m);
sigma = sigma_moment(&m);
skew = skewness_moment(&m);
kurt = kurtosis_moment(&m);
sum = show_moment(&m,1);
printf ("Min=%f Max=%f\n",min_moment(&m), max_moment(&m));
printf ("Number of points : %d\n",n_moment(&m));
printf ("Mean and dispersion : %f %f\n",mean,sigma);
printf ("Skewness and kurtosis : %f %f\n",skew,kurt);
printf ("Sum and Sum*Dx*Dy*Dz : %f %f\n",sum, sum*sov);
if (Qmedian)
printf ("Median : %f\n",get_median(ngood,data));
if (Qmmcount) {
min_count = max_count = 0;
xmin = min_moment(&m);
xmax = max_moment(&m);
for (i=0; i<nx; i++) {
for (j=0; j<ny; j++) {
for (k=0; k<nz; k++) {
x = CubeValue(iptr,i,j,k);
if (x==xmin) min_count++;
if (x==xmax) max_count++;
}
}
} /* i */
printf("Min_Max_count : %d %d\n",min_count,max_count);
}
printf ("%d/%d out-of-range points discarded\n",nsize-n_moment(&m), nsize);
}
}
real h3_moment(Moment *m)
{
return skewness_moment(m) / (4*sqrt(3.0));
}
nemo_main()
{
int i, j, k, ki;
real x, y, z, xmin, xmax, mean, sigma, skew, kurt, bad, w, *data;
real dmin, dmax;
real sum, sov, q1, q2, q3;
Moment m;
bool Qmin, Qmax, Qbad, Qw, Qmedian, Qrobust, Qtorben, Qmmcount = getbparam("mmcount");
bool Qx, Qy, Qz, Qone, Qall, Qign = getbparam("ignore");
bool Qhalf = getbparam("half");
bool Qmaxpos = getbparam("maxpos");
real nu, nppb = getdparam("nppb");
int npar = getiparam("npar");
int ngood;
int ndat = 0;
int nplanes;
int min_count, max_count;
int maxmom = getiparam("maxmom");
int maxpos[2];
char slabel[32];
instr = stropen (getparam("in"), "r");
read_image (instr,&iptr);
strclose(instr);
nx = Nx(iptr);
ny = Ny(iptr);
nz = Nz(iptr);
dprintf(1,"# data order debug: %f %f\n",Frame(iptr)[0], Frame(iptr)[1]);
if (hasvalue("tab")) tabstr = stropen(getparam("tab"),"w");
planes = (int *) allocate((nz+1)*sizeof(int));
nplanes = nemoinpi(getparam("planes"),planes,nz+1);
Qall = (planes[0]==-1);
if (planes[0]==0) {
Qone = FALSE;
nplanes = nz;
for (k=0; k<nz; k++)
planes[k] = k;
} else if (!Qall) {
Qone = (!Qall && nplanes==1);
for (k=0; k<nplanes; k++) {
if (planes[k]<1 || planes[k]>nz) error("%d is an illegal plane [1..%d]",planes[k],nz);
planes[k]--;
}
}
if (hasvalue("win")) {
instr = stropen (getparam("win"), "r");
read_image (instr,&wptr);
strclose(instr);
if (Nx(iptr) != Nx(wptr)) error("X sizes of in=/win= don't match");
if (Ny(iptr) != Ny(wptr)) error("Y sizes of in=/win= don't match");
if (Nz(iptr) != Nz(wptr)) error("Z sizes of in=/win= don't match");
Qw = TRUE;
} else
Qw = FALSE;
Qmin = hasvalue("min");
if (Qmin) xmin = getdparam("min");
Qmax = hasvalue("max");
if (Qmax) xmax = getdparam("max");
Qbad = hasvalue("bad");
if (Qbad) bad = getdparam("bad");
Qmedian = getbparam("median");
Qrobust = getbparam("robust");
Qtorben = getbparam("torben");
if (Qtorben) Qmedian = TRUE;
if (Qmedian || Qrobust || Qtorben) {
ndat = nx*ny*nz;
data = (real *) allocate(ndat*sizeof(real));
}
sov = 1.0; /* volume of a pixel/voxel */
Qx = Qign && Nx(iptr)==1;
Qy = Qign && Ny(iptr)==1;
Qz = Qign && Nz(iptr)==1;
sov *= Qx ? 1.0 : Dx(iptr);
sov *= Qy ? 1.0 : Dy(iptr);
sov *= Qz ? 1.0 : Dz(iptr);
strcpy(slabel,"*");
if (!Qx) strcat(slabel,"Dx*");
if (!Qy) strcat(slabel,"Dy*");
if (!Qz) strcat(slabel,"Dz*");
if (maxmom < 0) {
warning("No work done, maxmom<0");
stop(0);
}
if (Qall) { /* treat cube as one data block */
ini_moment(&m,maxmom,ndat);
ngood = 0;
for (k=0; k<nz; k++) {
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
x = CubeValue(iptr,i,j,k);
if (Qhalf && x>=0.0) continue;
if (Qmin && x<xmin) continue;
if (Qmax && x>xmax) continue;
if (Qbad && x==bad) continue;
w = Qw ? CubeValue(wptr,i,j,k) : 1.0;
accum_moment(&m,x,w);
if (Qhalf && x<0) accum_moment(&m,-x,w);
if (Qmedian) data[ngood++] = x;
if (tabstr) fprintf(tabstr,"%g\n",x);
}
}
}
if (npar > 0) {
nu = n_moment(&m)/nppb - npar;
if (nu < 1) error("%g: No degrees of freedom",nu);
printf("chi2= %g\n", show_moment(&m,2)/nu/nppb);
printf("df= %g\n", nu);
} else {
nsize = nx * ny * nz;
sum = mean = sigma = skew = kurt = 0;
if (maxmom > 0) {
mean = mean_moment(&m);
sum = show_moment(&m,1);
}
if (maxmom > 1)
sigma = sigma_moment(&m);
if (maxmom > 2)
skew = skewness_moment(&m);
if (maxmom > 3)
kurt = kurtosis_moment(&m);
printf ("Number of points : %d\n",n_moment(&m));
printf ("Min and Max : %f %f\n",min_moment(&m), max_moment(&m));
printf ("Mean and dispersion : %f %f\n",mean,sigma);
printf ("Skewness and kurtosis : %f %f\n",skew,kurt);
printf ("Sum and Sum*%s : %f %f\n",slabel, sum, sum*sov);
if (Qmedian) {
if (Qtorben) {
printf ("Median Torben : %f (%d)\n",median_torben(ngood,data,min_moment(&m),max_moment(&m)),ngood);
} else {
printf ("Median : %f\n",get_median(ngood,data));
q2 = median(ngood,data);
q1 = median_q1(ngood,data);
q3 = median_q3(ngood,data);
printf ("Q1,Q2,Q3 : %f %f %f\n",q1,q2,q3);
}
#if 1
if (ndat>0)
printf ("MedianL : %f\n",median_moment(&m));
#endif
}
if (Qrobust) {
compute_robust_moment(&m);
printf ("Mean Robust : %f\n",mean_robust_moment(&m));
printf ("Sigma Robust : %f\n",sigma_robust_moment(&m));
printf ("Median Robust : %f\n",median_robust_moment(&m));
}
if (Qmmcount) {
min_count = max_count = 0;
xmin = min_moment(&m);
xmax = max_moment(&m);
for (i=0; i<nx; i++) {
for (j=0; j<ny; j++) {
for (k=0; k<nz; k++) {
x = CubeValue(iptr,i,j,k);
if (x==xmin) min_count++;
if (x==xmax) max_count++;
}
}
} /* i */
printf("Min_Max_count : %d %d\n",min_count,max_count);
}
printf ("%d/%d out-of-range points discarded\n",nsize-n_moment(&m), nsize);
}
} else { /* treat each plane seperately */
/* tabular output, one line per (selected) plane */
printf("# iz z min max N mean sigma skew kurt sum sumsov ");
if (Qmedian) printf(" [med med]");
if (Qrobust) printf(" robust[N mean sig med]");
if (Qmaxpos) printf(" maxposx maxposy");
printf("\n");
ini_moment(&m,maxmom,ndat);
for (ki=0; ki<nplanes; ki++) {
reset_moment(&m);
k = planes[ki];
z = Zmin(iptr) + k*Dz(iptr);
ngood = 0;
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
x = CubeValue(iptr,i,j,k);
if (Qmin && x<xmin) continue;
if (Qmax && x>xmax) continue;
if (Qbad && x==bad) continue;
if (Qmaxpos) {
if (i==0 && j==0) { dmax = x; maxpos[0] = 0; maxpos[1] = 0;}
else if (x>dmax) { dmax = x; maxpos[0] = i; maxpos[1] = j;}
}
w = Qw ? CubeValue(wptr,i,j,k) : 1.0;
accum_moment(&m,x,w);
if (Qmedian) data[ngood++] = x;
}
}
nsize = nx * ny * nz;
sum = mean = sigma = skew = kurt = 0;
if (maxmom > 0) {
mean = mean_moment(&m);
sum = show_moment(&m,1);
}
if (maxmom > 1)
sigma = sigma_moment(&m);
if (maxmom > 2)
skew = skewness_moment(&m);
if (maxmom > 3)
kurt = kurtosis_moment(&m);
if (n_moment(&m) == 0) {
printf("# %d no data\n",k+1);
continue;
}
printf("%d %f %f %f %d %f %f %f %f %f %f",
k+1, z, min_moment(&m), max_moment(&m), n_moment(&m),
mean,sigma,skew,kurt,sum,sum*sov);
if (Qmedian) {
printf (" %f",get_median(ngood,data));
if (ndat>0) printf (" %f",median_moment(&m));
}
if (Qrobust) {
compute_robust_moment(&m);
printf (" %d %f %f %f",n_robust_moment(&m), mean_robust_moment(&m),
sigma_robust_moment(&m), median_robust_moment(&m));
}
if (Qmaxpos) {
printf(" %d %d",maxpos[0],maxpos[1]);
}
#if 0
if (Qmmcount) {
min_count = max_count = 0;
xmin = min_moment(&m);
xmax = max_moment(&m);
for (i=0; i<nx; i++) {
for (j=0; j<ny; j++) {
x = CubeValue(iptr,i,j,k);
if (x==xmin) min_count++;
if (x==xmax) max_count++;
}
} /* i,j */
printf(" %d %d",min_count,max_count);
}
printf ("%d/%d out-of-range points discarded\n",nsize-n_moment(&m), nsize);
#endif
printf("\n");
} /* ki */
}
}
local void histogram(void)
{
int i,j,k, l, kmin, kmax, lcount = 0;
real count[MAXHIST], under, over;
real xdat,ydat,xplt,yplt,dx,r,sum,sigma2, q, qmax;
real mean, sigma, mad, skew, kurt, h3, h4, lmin, lmax, median;
real rmean, rsigma, rrange[2];
Moment m;
dprintf (0,"read %d values\n",npt);
dprintf (0,"min and max value in column(s) %s: %g %g\n",getparam("xcol"),xmin,xmax);
if (!Qauto) {
xmin = xrange[0];
xmax = xrange[1];
dprintf (0,"min and max value reset to : %g %g\n",xmin,xmax);
lmin = xmax;
lmax = xmin;
for (i=0; i<npt; i++) {
if (x[i]>xmin && x[i]<=xmax) {
lmin = MIN(lmin, x[i]);
lmax = MAX(lmax, x[i]);
}
}
dprintf (0,"min and max value in range : %g %g\n",lmin,lmax);
}
for (k=0; k<nsteps; k++)
count[k] = 0; /* init histogram */
under = over = 0;
ini_moment(&m, 4, Qrobust||Qmad ? npt : 0);
for (i=0; i<npt; i++) {
if (Qbin) {
k=ring_index(nsteps,bins,x[i]);
} else {
if (xmax != xmin)
k = (int) floor((x[i]-xmin)/(xmax-xmin)*nsteps);
else
k = 0;
dprintf(2,"%d k=%d %g\n",i,k,x[i]);
}
if (k==nsteps && x[i]==xmax) k--; /* include upper edge */
if (k<0) { under++; continue; }
if (k>=nsteps) { over++; continue; }
count[k] = count[k] + 1;
dprintf (4,"%d : %f %d\n",i,x[i],k);
accum_moment(&m,x[i],1.0);
}
if (under > 0) error("bug: under = %d",under);
if (over > 0) error("bug: over = %d",over);
under = Nunder;
over = Nover;
mean = mean_moment(&m);
sigma = sigma_moment(&m);
skew = skewness_moment(&m);
kurt = kurtosis_moment(&m);
h3 = h3_moment(&m);
h4 = h4_moment(&m);
if (Qmad) mad = mad_moment(&m);
if (nsigma > 0) { /* remove outliers iteratively, starting from the largest */
iq = (int *) allocate(npt*sizeof(int));
for (i=0; i<npt; i++) {
#if 1
iq[i] = x[i] < xmin || x[i] > xmax;
#else
iq[i] = 0;
#endif
}
lcount = 0;
do { /* loop to remove outliers one by one */
qmax = -1.0;
for (i=0, l=-1; i<npt; i++) { /* find largest deviation from current mean */
if (iq[i]) continue; /* but skip previously flagged points */
q = (x[i]-mean)/sigma;
q = ABS(q);
if (q > qmax) {
qmax = q;
l = i;
}
}
if (qmax > nsigma) {
lcount++;
iq[l] = 1;
decr_moment(&m,x[l],1.0);
mean = mean_moment(&m);
sigma = sigma_moment(&m);
skew = skewness_moment(&m);
kurt = kurtosis_moment(&m);
h3 = h3_moment(&m);
h4 = h4_moment(&m);
if (Qmad) mad = mad_moment(&m);
dprintf(1,"%d/%d: removing point %d, m/s=%g %g qmax=%g\n",
lcount,npt,l,mean,sigma,qmax);
if (sigma <= 0) {
/* RELATED TO presetting MINMAX */
warning("BUG");
accum_moment(&m,x[l],1.0);
mean = mean_moment(&m);
sigma = sigma_moment(&m);
skew = skewness_moment(&m);
kurt = kurtosis_moment(&m);
h3 = h3_moment(&m);
h4 = h4_moment(&m);
dprintf(1,"%d/%d: LAST removing point %d, m/s=%g %g qmax=%g\n",
lcount,npt,l,mean,sigma,qmax);
break;
}
} else
dprintf(1,"%d/%d: keeping point %d, m/s=%g %g qmax=%g\n",
lcount,npt,l,mean,sigma,qmax);
/* if (lcount > npt/2) break; */
} while (qmax > nsigma);
dprintf(0,"Removed %d/%d points for nsigma=%g\n",lcount,npt,nsigma);
/* @algorithm left shift array values from mask array */
/* now shift all points into the array, decreasing npt */
/* otherwise the median is not correctly computed */
for (i=0, k=0; i<npt; i++) {
dprintf(1,"iq->%d\n",iq[i]);
if (iq[i]) k++;
if (k==0) continue; /* ?? */
if (i-k < 0) continue;
dprintf(1,"SHIFT: %d <= %d\n",i-k,i);
x[i-k] = x[i];
}
npt -= lcount; /* correct for outliers */
free(iq);
} /* nsigma > 0 */
if (npt != n_moment(&m))
error("Counting error, probably in removing outliers...");
dprintf (0,"Number of points : %d\n",npt);
if (npt>1)
dprintf (0,"Mean and dispersion : %g %g %g\n",mean,sigma,sigma/sqrt(npt-1.0));
else
dprintf (0,"Mean and dispersion : %g %g 0.0\n",mean,sigma);
if (Qmad) dprintf (0,"MAD : %g\n",mad);
dprintf (0,"Skewness and kurtosis: %g %g\n",skew,kurt);
dprintf (0,"h3 and h4 : %g %g\n", h3, h4);
if (Qmedian) {
if (npt % 2)
median = x[(npt-1)/2];
else
median = 0.5 * (x[npt/2] + x[npt/2-1]);
dprintf (0,"Median : %g\n",median);
} else if (Qtorben) {
median = median_torben(npt,x,xmin,xmax);
dprintf (0,"Median_torben : %g\n",median);
}
dprintf (0,"Sum : %g\n",show_moment(&m,1));
if (Qrobust) {
compute_robust_moment(&m);
rmean = mean_robust_moment(&m);
rsigma = sigma_robust_moment(&m);
robust_range(&m, rrange);
dprintf (0,"Robust N : %d\n",n_robust_moment(&m));
dprintf (0,"Robust Mean Disp : %g %g\n",rmean,rsigma);
dprintf (0,"Robust Range : %g %g\n",rrange[0],rrange[1]);
if (outstr) {
for (i=0; i<npt; i++) {
if (x[i]<rrange[0] || x[i]>rrange[1]) continue;
fprintf(outstr,"%g %d\n",x[i],i+1);
}
}
}
if (lcount > 0) {
warning("Recompute histogram because of outlier removals");
/* recompute histogram if we've lost some outliers */
for (k=0; k<nsteps; k++)
count[k] = 0; /* init histogram */
under = over = 0;
for (i=0; i<npt; i++) {
if (xmax != xmin)
k = (int) floor((x[i]-xmin)/(xmax-xmin)*nsteps);
else
k = 0;
if (k==nsteps && x[i]==xmax) k--; /* include upper edge */
if (k<0) { under++; continue; }
if (k>=nsteps) { over++; continue; }
count[k] = count[k] + 1;
dprintf (4,"%d : %f %d\n",i,x[i],k);
}
if (under > 0 || over > 0) error("under=%d over=%d in recomputed histo",under,over);
}
dprintf (3,"Histogram values : \n");
dx=(xmax-xmin)/nsteps;
kmax=0;
sum=0.0;
for (k=0; k<nsteps; k++) {
sum = sum + dx*count[k];
if (ylog) {
if (count[k]>0.0)
count[k] = log10(count[k]);
else
count[k] = -1.0;
}
if (count[k]>kmax)
kmax=count[k];
dprintf (3,"%f ",count[k]);
if (Qcumul) {
if (k==0)
count[k] += under;
else
count[k] += count[k-1];
}
}
dprintf (3,"\n");
sigma2 = 2.0 * sigma * sigma; /* gaussian */
sum /= sigma * sqrt(2*PI); /* scaling factor for equal area gauss */
if (ylog && over>0) over = log10(over);
if (ylog && under>0) under = log10(under);
kmax *= 1.1; /* add 10% */
if (Qcumul) kmax = npt;
if (maxcount>0) /* force scaling by user ? */
kmax=maxcount;
if (Qtab) {
maxcount = 0;
for (k=0; k<nsteps; k++)
maxcount = MAX(maxcount,count[k]);
if (maxcount>0)
r = 29.0/maxcount;
else
r = 1.0;
printf(" Bin Value Number\n");
printf(" Underflow %d\n",Nunder);
for (k=0; k<nsteps; k++) {
j = (int) (r*count[k]) + 1;
if (ylog) printf("%3d %13.6g %13.6g ",
k+1, xmin+(k+0.5)*dx, count[k]);
else printf("%3d %13.6g %8d ",
k+1, xmin+(k+0.5)*dx, (int)count[k]);
while (j-- > 0) printf("*");
printf("\n");
}
printf(" Overflow %d\n",Nover);
stop(0);
}
#ifdef YAPP
/* PLOTTING */
plinit("***",0.0,20.0,0.0,20.0);
xplot[0] = xmin;
xplot[1] = xmax;
yplot[0] = 0.0;
yplot[1] = (real) kmax;
xaxis (2.0, 2.0, 16.0, xplot, -7, xtrans, xlab);
xaxis (2.0, 18.0,16.0, xplot, -7, xtrans, NULL);
yaxis (2.0, 2.0, 16.0, yplot, -7, ytrans, ylab);
yaxis (18.0, 2.0, 16.0, yplot, -7, ytrans, NULL);
pljust(-1); /* set to left just */
pltext(input,2.0,18.2,0.32,0.0); /* filename */
pljust(1);
pltext(headline,18.0,18.2,0.24,0.0); /* headline */
pljust(-1); /* return to left just */
xdat=xmin;
dx=(xmax-xmin)/nsteps;
plmove(xtrans(xmin),ytrans(0.0));
for (k=0; k<nsteps; k++) { /* nsteps= */
xplt = xtrans(xdat);
yplt = ytrans((real)count[k]);
plline (xplt,yplt);
xdat += dx;
xplt = xtrans(xdat);
plline (xplt,yplt);
}
plline(xplt,ytrans(0.0));
for (i=0; i<nxcoord; i++) {
plmove(xtrans(xcoord[i]),ytrans(yplot[0]));
plline(xtrans(xcoord[i]),ytrans(yplot[1]));
}
if (Qgauss) { /* plot model and residuals */
if (ylog)
plmove(xtrans(xmin),ytrans(-1.0));
else
plmove(xtrans(xmin),ytrans(0.0));
for (k=0; k<100; k++) {
xdat = xmin + (k+0.5)*(xmax-xmin)/100.0;
ydat = sum * exp( -sqr(xdat-mean)/sigma2);
if (ylog) ydat = log10(ydat);
plline(xtrans(xdat), ytrans(ydat));
}
}
if (Qresid) {
plltype(0,2); /* dotted residuals */
xdat = xmin+0.5*dx;
dprintf(1,"# residuals from gauss\n");
for (k=0; k<nsteps; k++, xdat +=dx) {
ydat = sum * exp( -sqr(xdat-mean)/sigma2);
dprintf(1,"%g %g %g\n",xdat,count[k],ydat);
if (ylog) ydat = log10(ydat);
ydat = count[k] - ydat;
if (k==0)
plmove(xtrans(xdat),ytrans(ydat));
else
plline(xtrans(xdat),ytrans(ydat));
}
plltype(0,1); /* back to normal line type */
}
plstop();
#endif
}
