static void
Relative(struct dataset *ds, struct dataset *rs, int confidx)
{
double spool, s, d, e, t;
int i;
i = ds->n + rs->n - 2;
if (i > NSTUDENT)
t = student[0][confidx];
else
t = student[i][confidx];
spool = (ds->n - 1) * Var(ds) + (rs->n - 1) * Var(rs);
spool /= ds->n + rs->n - 2;
spool = sqrt(spool);
s = spool * sqrt(1.0 / ds->n + 1.0 / rs->n);
d = Avg(ds) - Avg(rs);
e = t * s;
if (fabs(d) > e) {
printf("Difference at %.1f%% confidence\n", studentpct[confidx]);
printf(" %g +/- %g\n", d, e);
printf(" %g%% +/- %g%%\n", d * 100 / Avg(rs), e * 100 / Avg(rs));
printf(" (Student's t, pooled s = %g)\n", spool);
} else {
printf("No difference proven at %.1f%% confidence\n",
studentpct[confidx]);
}
}
static void
DimPlot(struct dataset *ds)
{
AdjPlot(Min(ds));
AdjPlot(Max(ds));
AdjPlot(Avg(ds) - Stddev(ds));
AdjPlot(Avg(ds) + Stddev(ds));
}
bool NeighborClustCount(Neighbor** vNeighbors,vector<int>& vClustIDs,vector<ClusterInfo>& vPrct,int iNNToFind,int iClusts)
{
int iSz = vClustIDs.size();
if(vPrct.size()!=iClusts+1)
return false;
int i;
vector<int> vCounts(iClusts+1);
for(i=1;i<=iClusts;i++)
vCounts[i]=count(vClustIDs.begin(),vClustIDs.end(),i);
vector< vector<prob_t> > vprct(iClusts+1);
for(i=1;i<=iClusts;i++)
vprct[i]=vector<prob_t>(vCounts[i],0.0);
vector<int> vIndex(iClusts+1,0);
for(i=0;i<iSz;i++)
{
int j, iCount = 0, iClust = vClustIDs[i];
if(!iClust)continue;//skip background vectors
int idx = vIndex[iClust]++;
for(j=0;j<iNNToFind;j++)
{
if(vClustIDs[vNeighbors[i][j].m_id]==iClust)
vprct[iClust][idx]+=1.0f;
}
vprct[iClust][idx] /= iNNToFind;
}
for(i=1;i<=iClusts;i++)
vPrct[i].m_fPrctKNNInClust=Avg(vprct[i]);
return true;
}
static void
Vitals(struct dataset *ds, int flag)
{
printf("%c %3d %13.8g %13.8g %13.8g %13.8g %13.8g", symbol[flag],
ds->n, Min(ds), Max(ds), Median(ds), Avg(ds), Stddev(ds));
printf("\n");
}
double DataSource::Variance(Getdatafun getdata, double avg) {
if (IsNull(avg))
avg = Avg(getdata);
double ret = 0;
for (int64 i = 0; i < GetCount(); ++i) {
double val = Membercall(getdata)(i) - avg;
ret += val*val;
}
return ret/(GetCount() - 1);
}
void CMaxMin::Dump (FILE *f)
{
int i;
fprintf (f, "N=%d Avg=%d\n", n, Avg());
fprintf (f, "min: ");
for (i=0; i<n_min; i++)
fprintf (f,"%d ", min[i]);
fprintf (f, "\nmax: ");
for (i=0; i<n_max; i++)
fprintf (f, "%d ", max[i]);
fprintf (f, "\n");
}
int main (void)
{
struct MyTime first_time = { 10, 501};
struct MyTime second_time = { 2, 500};
struct MyTime min_time = { 99999, 999999};
struct MyTime max_time = { 0, 0};
struct MyTime diff_time = { 0, 0};
print_time ("First:", first_time);
print_time ("Second:", second_time);
print_time ("Min:", Min(first_time,second_time));
print_time ("Max:", Max(first_time,second_time));
print_time ("Diff:", Diff(second_time, first_time));
getCurrentMyTime(& first_time);
sleep (10);
getCurrentMyTime(& second_time);
print_time ("Average:", Avg(Diff(second_time,first_time), 10));
diff_time.SEC = first_time.SEC-second_time.SEC;
diff_time.FRAC_SEC = first_time.FRAC_SEC-second_time.FRAC_SEC;
/* If the substraction results in negative answer */
if(diff_time.FRAC_SEC < 0)
{
diff_time.FRAC_SEC = SCALE + diff_time.FRAC_SEC;
if(diff_time.SEC > 0 )
diff_time.SEC = diff_time.SEC-1;
else
diff_time.FRAC_SEC*=-1;
}
if(diff_time.FRAC_SEC < min_time.FRAC_SEC)
min_time.FRAC_SEC = diff_time.FRAC_SEC;
if(diff_time.FRAC_SEC > max_time.FRAC_SEC)
max_time.FRAC_SEC = diff_time.FRAC_SEC;
if(diff_time.SEC < min_time.SEC)
min_time.SEC = diff_time.SEC;
if(diff_time.SEC > max_time.SEC)
max_time.SEC = diff_time.SEC;
print_time("Old Min:", min_time);
print_time("Old Max:", max_time);
return 0;
}
/** Calculate Pearson product-moment correlation between DataSets.
* \D1 DataSet to caclculate correlation for.
* \D2 DataSet to caclulate correlation to.
* \return Pearson product-moment correlation coefficient.
*/
double DS_Math::CorrCoeff( DataSet& D1, DataSet& D2 ) {
// Check if D1 and D2 are valid types
if ( !GoodCalcType(D1) ) return 0;
if ( !GoodCalcType(D2) ) return 0;
// Check that D1 and D2 have same # data points.
int Nelements = D1.Size();
if (Nelements != D2.Size()) {
mprinterr("Error: Corr: # elements in dataset %s (%i) not equal to\n",
D1.Legend().c_str(), Nelements);
mprinterr("Error: # elements in dataset %s (%i)\n",
D2.Legend().c_str(), D2.Size());
return 0;
}
// Calculate averages
double avg1 = Avg(D1);
double avg2 = Avg(D2);
// Calculate average deviations.
double sumdiff1_2 = 0.0;
double sumdiff2_2 = 0.0;
double corr_coeff = 0.0;
//mprinterr("DATASETS %s and %s\n", c_str(), D2.c_str());
for (int i = 0; i < Nelements; i++) {
double diff1 = D1.Dval(i) - avg1;
double diff2 = D2.Dval(i) - avg2;
sumdiff1_2 += (diff1 * diff1);
sumdiff2_2 += (diff2 * diff2);
corr_coeff += (diff1 * diff2);
}
if (sumdiff1_2 == 0.0 || sumdiff2_2 == 0.0) {
mprintf("Warning: Corr: %s to %s, Normalization is 0\n",
D1.Legend().c_str(), D2.Legend().c_str());
return 0;
}
// Correlation coefficient
corr_coeff /= ( sqrt( sumdiff1_2 ) * sqrt( sumdiff2_2 ) );
//mprintf(" CORRELATION COEFFICIENT %6s to %6s IS %10.4f\n",
// D1_->c_str(), D2_->c_str(), corr_coeff );
return corr_coeff;
}
double DataSource::Variance(Getdatafun getdata, double avg) {
if (IsNull(avg))
avg = Avg(getdata);
if (IsNull(avg))
return Null;
double ret = 0;
int count = 0;
for (int64 i = 0; i < GetCount(); ++i) {
double d = Membercall(getdata)(i);
if (!IsNull(d)) {
d -= avg;
ret += d*d;
count++;
}
}
if (count <= 0)
return Null;
return ret/(count - 1);
}
void stdMatSDEV( CvMat* src, CvMat* dst )
{
int nrows = src->rows;
int ncols = src->cols;
if ( nrows != dst->rows || ncols != dst->cols )
throw std::string("stdMatSDEV - src and dst matrices are not same size");
for ( int r = 0; r < src->rows; r++ )
{
double avg = Avg(src->data.fl + (r*ncols), ncols);
double sdev = sDev(src->data.fl + (r*ncols), ncols, avg);
for ( int c = 0; c < ncols; c++ )
{
int index = r*ncols+c;
dst->data.fl[index] = ( (src->data.fl[index] - avg) / sdev );
}
}
}
int32_t ComputeFunc(MEM_POOL_PTR mem_pool, enum function_type agr_type,
struct low_data_struct *cur_value, struct low_data_struct* new_value) {
int32_t result_code = MILE_RETURN_SUCCESS;
switch (agr_type) {
case FUNC_MIN:
result_code = Min(mem_pool, cur_value, new_value);
break;
case FUNC_MAX:
result_code = Max(mem_pool, cur_value, new_value);
break;
case FUNC_SUM:
result_code = Sum(mem_pool, cur_value, new_value);
break;
case FUNC_COUNT:
result_code = Count(mem_pool, cur_value, new_value);
break;
case FUNC_AVG:
result_code = Avg(mem_pool, cur_value, new_value);
break;
case FUNC_VAR:
result_code = Var(mem_pool, cur_value, new_value);
break;
case FUNC_STD:
result_code = Var(mem_pool, cur_value, new_value);
break;
case FUNC_SQUARE_SUM:
result_code = SquareSum(mem_pool, cur_value, new_value);
break;
case FUNC_COUNT_DISTINCT:
result_code = CountDistinct(mem_pool, cur_value, new_value);
break;
default:
return ERROR_UNSUPPORTED_AGRFUNC_TYPE;
}
return result_code;
}
static void
PlotSet(struct dataset *ds, int val)
{
struct plot *pl;
int i, j, m, x;
unsigned n;
int bar;
pl = &plot;
if (pl->span == 0)
return;
if (pl->separate_bars)
bar = val-1;
else
bar = 0;
if (pl->bar == NULL) {
pl->bar = malloc(sizeof(char *) * pl->num_datasets);
memset(pl->bar, 0, sizeof(char*) * pl->num_datasets);
}
if (pl->bar[bar] == NULL) {
pl->bar[bar] = malloc(pl->width);
memset(pl->bar[bar], 0, pl->width);
}
m = 1;
i = -1;
j = 0;
for (n = 0; n < ds->n; n++) {
x = (ds->points[n] - pl->x0) / pl->dx;
if (x == i) {
j++;
if (j > m)
m = j;
} else {
j = 1;
i = x;
}
}
m += 1;
if (m > pl->height) {
pl->data = realloc(pl->data, pl->width * m);
memset(pl->data + pl->height * pl->width, 0,
(m - pl->height) * pl->width);
}
pl->height = m;
i = -1;
for (n = 0; n < ds->n; n++) {
x = (ds->points[n] - pl->x0) / pl->dx;
if (x == i) {
j++;
} else {
j = 1;
i = x;
}
pl->data[j * pl->width + x] |= val;
}
if (!isnan(Stddev(ds))) {
x = ((Avg(ds) - Stddev(ds)) - pl->x0) / pl->dx;
m = ((Avg(ds) + Stddev(ds)) - pl->x0) / pl->dx;
pl->bar[bar][m] = '|';
pl->bar[bar][x] = '|';
for (i = x + 1; i < m; i++)
if (pl->bar[bar][i] == 0)
pl->bar[bar][i] = '_';
}
x = (Median(ds) - pl->x0) / pl->dx;
pl->bar[bar][x] = 'M';
x = (Avg(ds) - pl->x0) / pl->dx;
pl->bar[bar][x] = 'A';
}
double DS_Math::Avg(DataSet& ds) {
return Avg(ds, 0);
}
/** Calculate time correlation between two DataSets.
* \D1 DataSet to calculate correlation for.
* \D2 DataSet to calculate correlation to.
* \Ct DataSet to store time correlation fn, must be DOUBLE.
* \lagmaxIn Max lag to calculate corr. -1 means use size of dataset.
* \calccovar If true calculate covariance (devation from avg).
* \return 0 on success, 1 on error.
*/
int DS_Math::CrossCorr( DataSet& D1, DataSet& D2, DataSet& Ct, int lagmaxIn,
bool calccovar, bool usefft )
{
int lagmax;
double ct;
// Check if D1 and D2 are valid types
if ( !GoodCalcType(D1) ) return 1;
if ( !GoodCalcType(D2) ) return 1;
// Check that D1 and D2 have same # data points.
int Nelements = D1.Size();
if (Nelements != D2.Size()) {
mprinterr("Error: CrossCorr: # elements in dataset %s (%i) not equal to\n",
D1.Legend().c_str(), Nelements);
mprinterr("Error: # elements in dataset %s (%i)\n",
D2.Legend().c_str(), D2.Size());
return 1;
}
if (Nelements < 2) {
mprinterr("Error: CrossCorr: # elements is less than 2 (%i)\n", Nelements);
return 1;
}
// Check return dataset type
if ( Ct.Type() != DataSet::DOUBLE ) {
mprinterr("Internal Error: CrossCorr: Ct must be of type DataSet::DOUBLE.\n");
return 1;
}
// Check if lagmaxIn makes sense. Set default lag to be Nelements
// if not specified.
if (lagmaxIn == -1)
lagmax = Nelements;
else if (lagmaxIn > Nelements) {
mprintf("Warning: CrossCorr [%s][%s]: max lag (%i) > Nelements (%i), setting to Nelements.\n",
D1.Legend().c_str(), D2.Legend().c_str(), lagmaxIn, Nelements);
lagmax = Nelements;
} else
lagmax = lagmaxIn;
// If calculating covariance calculate averages
double avg1 = 0;
double avg2 = 0;
if ( calccovar ) {
avg1 = Avg(D1);
avg2 = Avg(D2);
}
// Calculate correlation
double norm = 1.0;
if ( usefft ) {
// Calc using FFT
CorrF_FFT pubfft1(Nelements);
ComplexArray data1 = pubfft1.Array();
data1.PadWithZero(Nelements);
for (int i = 0; i < Nelements; ++i)
data1[i*2] = D1.Dval(i) - avg1;
if (&D2 == &D1)
pubfft1.AutoCorr(data1);
else {
// Populate second dataset if different
ComplexArray data2 = pubfft1.Array();
data2.PadWithZero(Nelements);
for (int i = 0; i < Nelements; ++i)
data2[i*2] = D2.Dval(i) - avg2;
pubfft1.CrossCorr(data1, data2);
}
// Put real components of data1 in output DataSet
norm = 1.0 / fabs( data1[0] );
for (int i = 0; i < lagmax; ++i) {
ct = data1[i*2] * norm;
Ct.Add(i, &ct);
}
} else {
// Direct calc
for (int lag = 0; lag < lagmax; ++lag) {
ct = 0;
int jmax = Nelements - lag;
for (int j = 0; j < jmax; ++j)
ct += ((D1.Dval(j) - avg1) * (D2.Dval(j+lag) - avg2));
if (lag == 0) {
if (ct != 0)
norm = fabs( ct );
}
ct /= norm;
Ct.Add(lag, &ct);
}
}
return 0;
}
void CrossNLMFilter::Apply(
float sigmaR,
const vector<TwoDArray<float> > &mseArray,
const vector<TwoDArray<float> > &priArray,
const TwoDArray<Color> &rImg,
const TwoDArray<Feature> &featureImg,
const TwoDArray<Feature> &featureVarImg,
vector<TwoDArray<float> > &outMSE,
vector<TwoDArray<float> > &outPri) const {
float mseScaleR = -0.5f/(sigmaR*sigmaR);
#pragma omp parallel for num_threads(PbrtOptions.nCores) schedule(static)
for(int taskId = 0; taskId < nTasks; taskId++) {
int txs, txe, tys, tye;
ComputeSubWindow(taskId, nTasks, width, height,
&txs, &txe, &tys, &tye);
for(int y = tys; y < tye; y++)
for(int x = txs; x < txe; x++) {
vector<float> mseSum(mseArray.size(), 0.f);
vector<float> priSum(priArray.size(), 0.f);
vector<float> wSum(mseArray.size(), 0.f);
Feature feature = featureImg(x, y);
Feature featureVar = featureVarImg(x, y);
// Filter using pixels within search range
for(int dy = -searchRadius; dy <= searchRadius; dy++)
for(int dx = -searchRadius; dx <= searchRadius; dx++) {
int xx = x + dx;
int yy = y + dy;
if(xx < 0 || yy < 0 || xx >= width || yy >= height)
continue;
Color diffSqSum(0.f, 0.f, 0.f);
// Calculate block distance
for(int by = -patchRadius; by <= patchRadius; by++)
for(int bx = -patchRadius; bx <= patchRadius; bx++) {
int xbx = x + bx;
int yby = y + by;
int xxbx = xx + bx;
int yyby = yy + by;
if( xbx < 0 || xbx >= width ||
yby < 0 || yby >= height ||
xxbx < 0 || xxbx >= width ||
yyby < 0 || yyby >= height)
continue;
Color diff = rImg(xbx, yby) - rImg(xxbx, yyby);
diffSqSum += (diff*diff);
}
diffSqSum *= invPatchSize;
float dist = Avg(diffSqSum);
Feature fDiff = feature - featureImg(xx, yy);
Feature fVarSum = featureVar + featureVarImg(xx, yy);
Feature fDist = (fDiff*fDiff)/fVarSum.Max(c_VarMax);
float weight = fmath::exp(dist*mseScaleR +
Sum(fDist*scaleF));
for(size_t i = 0; i < mseArray.size(); i++) {
mseSum[i] += weight * mseArray[i](xx, yy);
priSum[i] += weight * priArray[i](xx, yy);
wSum[i] += weight;
}
}
for(size_t i = 0; i < mseArray.size(); i++) {
outMSE[i](x, y) = mseSum[i] / wSum[i];
outPri[i](x, y) = priSum[i] / wSum[i];
}
}
}
}
void CrossNLMFilter::Apply(const TwoDArray<Color> &img,
const TwoDArray<Feature> &featureImg,
const TwoDArray<Feature> &featureVarImg,
const TwoDArray<Color> &rImg,
const TwoDArray<Color> &varImg,
vector<TwoDArray<Color> > &fltArray,
vector<TwoDArray<float> > &mseArray,
vector<TwoDArray<float> > &priArray) const {
#pragma omp parallel for num_threads(PbrtOptions.nCores) schedule(static)
for(int taskId = 0; taskId < nTasks; taskId++) {
int txs, txe, tys, tye;
ComputeSubWindow(taskId, nTasks, width, height,
&txs, &txe, &tys, &tye);
for(int y = tys; y < tye; y++)
for(int x = txs; x < txe; x++) {
vector<Color> sum(scaleR.size(), Color(0.f));
vector<Color> rSum(scaleR.size(), Color(0.f));
vector<Color> rSumSq(scaleR.size(), Color(0.f));
vector<float> wSum(scaleR.size(), 0.f);
vector<vector<float> > wArray(scaleR.size());
for(size_t p = 0; p < wArray.size(); p++) {
wArray[p].resize(patchWidth*patchWidth);
}
Feature feature = featureImg(x, y);
Feature featureVar = featureVarImg(x, y);
// Filter using pixels within search range
for(int dy = -searchRadius; dy <= searchRadius; dy++)
for(int dx = -searchRadius; dx <= searchRadius; dx++) {
int xx = x + dx;
int yy = y + dy;
if(xx < 0 || yy < 0 || xx >= width || yy >= height)
continue;
Color diffSqSum(0.f, 0.f, 0.f);
// Calculate block distance
for(int by = -patchRadius; by <= patchRadius; by++)
for(int bx = -patchRadius; bx <= patchRadius; bx++) {
int xbx = x + bx;
int yby = y + by;
int xxbx = xx + bx;
int yyby = yy + by;
if( xbx < 0 || xbx >= width ||
yby < 0 || yby >= height ||
xxbx < 0 || xxbx >= width ||
yyby < 0 || yyby >= height)
continue;
Color diff = rImg(xbx, yby) - rImg(xxbx, yyby);
diffSqSum += (diff*diff);
}
diffSqSum *= invPatchSize;
float dist = Avg(diffSqSum);
Feature fDiff = feature - featureImg(xx, yy);
Feature fVarSum = featureVar + featureVarImg(xx, yy);
Feature fDist = (fDiff*fDiff)/fVarSum.Max(c_VarMax);
// For each parameter, calculate information necessary for
// filtering and SURE estimation
for(size_t p = 0; p < scaleR.size(); p++) {
if(scaleR[p] == 0.f) {
continue;
}
float weight = fmath::exp(dist*scaleR[p] +
Sum(fDist*scaleF));
sum[p] += weight * img(xx, yy);
rSum[p] += weight * rImg(xx, yy);
rSumSq[p] += weight * rImg(xx, yy) * rImg(xx, yy);
wSum[p] += weight;
if(dy >= -patchRadius && dy <= patchRadius &&
dx >= -patchRadius && dx <= patchRadius) {
wArray[p][(dy+patchRadius)*patchWidth+(dx+patchRadius)] =
weight;
}
}
}
for(size_t p = 0; p < scaleR.size(); p++) {
if(scaleR[p] == 0.f) {
fltArray[p](x, y) = img(x, y);
mseArray[p](x, y) = Avg(2.f*varImg(x, y));
continue;
}
float invWSum = 1.f/wSum[p];
Color xl = sum[p] * invWSum;
Color rxl = rSum[p] * invWSum;
Color rxlSq = rSumSq[p] * invWSum;
Color ryl = rImg(x, y);
Color dxdy = (-2.f*scaleR[p])*(rxlSq - rxl*rxl)*invPatchSize + Color(invWSum);
Color tmp;
for(int by = -patchRadius; by <= patchRadius; by++)
for(int bx = -patchRadius; bx <= patchRadius; bx++) {
int xbpx = x+bx;
int ybpy = y+by;
int xbmx = x-bx;
int ybmy = y-by;
if( xbpx < 0 || xbpx >= width ||
ybpy < 0 || ybpy >= height ||
xbmx < 0 || xbmx >= width ||
ybmy < 0 || ybmy >= height)
continue;
Color rylpb = rImg(xbpx, ybpy);
Color rylmb = rImg(xbmx, ybmy);
float w = wArray[p][(-by+patchRadius)*patchWidth+(-bx+patchRadius)];
tmp += w*(ryl - rylpb)*(rxl - rylmb);
}
tmp *= (-2.f*scaleR[p])*invPatchSize*invWSum;
dxdy += tmp;
Color mse = (rxl-ryl)*(rxl-ryl) + 2.f*varImg(x, y)*dxdy - varImg(x, y);
Color pri = (mse + varImg(x, y));
fltArray[p](x, y) = xl;
mseArray[p](x, y) = Avg(mse);
priArray[p](x, y) = Avg(pri) / (xl.Y()*xl.Y() + 1e-2f);
}
}
}
}