void ribi::SimpleLinearRegression::Test() noexcept
{
{
static bool is_tested { false };
if (is_tested) return;
is_tested = true;
}
const TestTimer test_timer(__func__,__FILE__,1.0);
{
const std::vector<double> v { 75.0, 83.0, 96.0, 100.0, 121.0, 125.0 };
const double variance { CalculateVariance(v) };
const double expected { 332.666667 };
assert(std::abs(variance - expected) < 0.0001);
}
{
const std::vector<double> v { 0.23, 0.37, 0.45, 0.49, 0.56, 0.63, 0.63, 0.70, 0.72, 0.82 };
const double variance { CalculateVariance(v) };
const double expected { 0.02846 };
assert(std::abs(variance - expected) < 0.0001);
}
for (int i=1; i!=5; ++i) //Human-based counting, following the Ansombe's Quartet indices
{
const std::vector<double> xs { GetAnscombesQuartetX(i) };
const std::vector<double> ys { GetAnscombesQuartetY(i) };
const std::pair<double,double> p { CalculateBestFit(xs,ys) };
const double mean_x { CalculateMean(xs) };
const double mean_y { CalculateMean(ys) };
const double slope { p.first };
const double intercept { p.second };
//const double variance_x = CalculateVariance(xs);
//const double variance_y = CalculateVariance(ys);
const double expected_mean_x { 9.0 };
const double expected_mean_y { 7.5 }; //to 2 decimal places
const double expected_slope { 0.500 }; //to 3 decimal places
const double expected_intercept { 3.00 }; //to 2 decimal places
const double e { 0.01 };
assert(std::abs(expected_mean_x - mean_x) < e);
assert(std::abs(expected_mean_y - mean_y) < e);
//const double expected_variance_x = 11.0;
//const double expected_variance_y = 4.125; //4.122 or 4.127 (to 3 decimal places)
//const double expected_correlation = 0.816; //to 3 decimal places)
//assert(std::abs(expected_variance_x - variance_x) < e);
//assert(std::abs(expected_variance_y - variance_y) < e);
assert(std::abs(expected_slope - slope) < e);
assert(std::abs(expected_intercept - intercept) < e);
}
}
tResult TemporalImage::ProcessImage()
{
if (bufferSize == 0)
{
outputImage = workingImage;
RETURN_NOERROR;
}
buffer[bufferIndex] = workingImage.clone();
bufferIndex = (bufferIndex + 1) % bufferSize;
switch (method)
{
case (METHOD_AND):
outputImage = CalculateAnd();
break;
case (METHOD_OR):
outputImage = CalculateOr();
break;
case (METHOD_MEAN):
outputImage = CalculateMean();
break;
default:
logger.Log(cString::Format("Invalid method %d", method).GetPtr(), false);
outputImage = workingImage;
break;
}
RETURN_NOERROR;
}
double ribi::SimpleLinearRegression::CalculateCovariance(
const std::vector<double>& xs,
const std::vector<double>& ys)
{
assert(xs.size() == ys.size());
const std::size_t sz { xs.size() };
std::vector<double> ps(sz);
for (std::size_t i=0; i!=sz; ++i)
{
ps[i] = xs[i] * ys[i];
}
const double mean_x { CalculateMean(xs) };
const double mean_y { CalculateMean(ys) };
const double mean_p { CalculateMean(ps) };
return mean_p - (mean_x * mean_y);
}
double CCalculate::CalculateVariance(unsigned char *bitmap, int *labelmap, int width, int height, int maxarea, int labelIndex)
{
m_Variance = 0.0;
double m_Mean = CalculateMean(bitmap, labelmap, width, height, maxarea, labelIndex);
for(int i = 0; i < width*height; i++)
{
if(labelmap[i] == labelIndex)
m_Variance += (double)sqrt((bitmap[i]-m_Mean)*(bitmap[i]-m_Mean)) / maxarea;
}
return m_Variance;
}
double CalculateSampleVariane()
{
mean = CalculateMean();
double temp = 0;
for(int i = 0; i < max; i++)
{
temp += (value[i] - mean) * (value[i] - mean) ;
}
return temp / (max - 1);
}
double ribi::SimpleLinearRegression::CalculateVariance(
const std::vector<double>& v)
{
assert(!v.empty());
const double mean { CalculateMean(v) };
const double mean_of_squares {
std::accumulate(v.begin(),v.end(),0.0,
[](const double init, const double x)
{
return init + (x * x);
}
) / static_cast<double>(v.size())
};
return mean_of_squares - (mean * mean);
}
/*
See "change detection" wikis and http://people.irisa.fr/Michele.Basseville/
The letter-detection needs to be done online, but offline may be doable as well.
Implements an event oriented state machine in which it is easier to exit states than enter them.
This helps detect the confounded edge between neighbor-key transitions, a difficult case.
The only event parameter is stDev; other attributes tend to lead stDev, so a better trigger function
could probably be devised.
*/
void SingularityBuilder::Process3(vector<Point>& inData, vector<PointMu>& outData)
{
bool trig = false;
//char c;
vector<PointMu> midData;
int i, sampleRate, segmentWidth, trigger, eventStart, eventEnd, stDevTrigger;
double dDist, dist, dydx, lastDist, coVar, stDev, nKeyHits, avgDist, avgTheta, lastTheta, dTheta;
char currentAlpha = '!', prevAlpha = '!';
//segment width is the sample radius for assessing a trigger
sampleRate = 1; //sample every two ticks. Thus, there will be n/2 analyses
segmentWidth = 3; //every k ticks, grab next k point for analysis
lastTheta = dTheta = avgTheta = 0.0;
int triggerThreshold = 4;
int highFocus = 45;
double stDev_HardTrigger = 225;
double stDev_SoftTrigger = 14;
cout << "processing " << inData.size() << " data points in sb.process(), dxThreshold=" << dxThreshold << endl;
//TODO: sleep if no data
trigger = 0;
for(i = 0; i < (inData.size() - segmentWidth - 1); i += sampleRate){
//ignore points outside of the active region, including the <start/stop> region
if(InBounds(inData[i]) && InBounds(inData[i+segmentWidth-1])){
//measures absolute change in distance, a dumb attribute
//lastDist = dist;
//dist = layoutManager->DoubleDistance(inData[i],inData[i+segmentWidth-1]);
//dDist = dist - lastDist;
//dydx = layoutManager->DyDx(inData[i],inData[i+segmentWidth-1]); //slope
//dydx = layoutManager->AvgDyDx(inData,i,segmentWidth);
//avgDist = layoutManager->AvgDistance(inData,i,segmentWidth);
//measures clustering of a group of points, but only the points themselves
//coVar = layoutManager->CoVariance(inData,i,segmentWidth);
stDev = layoutManager->CoStdDeviation(inData,i,segmentWidth); //(sigmaX*sigmaY)^2
//lastTheta = avgTheta;
//avgTheta = layoutManager->AvgTheta(inData,i,segmentWidth); //get the avg direction for a sequence of points
//dTheta = (avgTheta - lastTheta) / 2.0;
//printf("stDev,avgDist,dDist,dydx,avgTheta,dTheta: %9.3f %9.3f %9.3f %9.3f %9.3f %9.3f\n",stDev,avgDist,dDist,dydx,avgTheta,dTheta);
currentAlpha = layoutManager->FindNearestKey(inData[i]);
if(stDev < stDev_HardTrigger){
printf("curAlpha, prevAlpha, pt-stDev: %c %c %9.3f %d %d pretrig\n",currentAlpha,prevAlpha,stDev,inData[i].X,inData[i].Y);
}
else{
printf("curAlpha, prevAlpha, pt-stDev: %c %c %9.3f %d %d\n",currentAlpha,prevAlpha,stDev,inData[i].X,inData[i].Y);
}
/*
Implements an event oriented state machine in which it is easier to exit states than enter them.
This helps detect the confounded edge between neighbor-key transitions, a difficult case.
*/
if(stDev < stDev_HardTrigger){
//gives the trigger faster progress when confidence is higher; there ought to be a more continuous way to do this
if(stDev < highFocus){
trigger += 2;
}
else{
trigger++;
}
//trigger and collect event
if(trigger > triggerThreshold){
cout << "trig" << endl;
//this optimistically assumes current position has (intentional) focus on some key
prevAlpha = currentAlpha = layoutManager->FindNearestKey(inData[i]);
eventStart = i;
i++;
stDev = layoutManager->CoStdDeviation(inData,i,segmentWidth);
//hold state, unless there is a stDev change and an alpha change. Only if both alpha changes and stDev throws, will we exit.
// A new event will immediately be thrown to catch the neighbor key event.
while(i < inData.size() && (stDev < stDev_SoftTrigger || prevAlpha == currentAlpha) && (stDev < stDev_HardTrigger)){
//while(i < inData.size() && (stDev < stDev_SoftTrigger && prevAlpha == currentAlpha) && (stDev < stDev_HardTrigger)){
//while(i < inData.size() && (stDev < stDev_SoftTrigger || prevAlpha == currentAlpha) && (stDev < stDev_HardTrigger)){
stDev = layoutManager->CoStdDeviation(inData,i,segmentWidth);
currentAlpha = layoutManager->FindNearestKey(inData[i]);
printf("curAlpha, prevAlpha, pt-stDev: %c %c %9.3f trig\n",currentAlpha,prevAlpha,stDev);
i++;
}
//exit state either by hard/fast exit, or soft-exit to an adjacent key
eventEnd = i;
i--;
//capture event data and push it, then continue monitoring for new ones. Event is pushed only if unique from previous one..
PointMu outPoint;
outPoint.ticks = eventEnd - eventStart + trigger; //ticks can be used as a confidence measure of the event
CalculateMean(eventStart,eventEnd,inData,outPoint);
outPoint.alpha = layoutManager->FindNearestKey(outPoint.pt);
cout << "hit mean for " << outPoint.alpha << " ticks: " << outPoint.ticks << endl;
trigger = 0;
//NOTE A new cluster is appended only if it is a unique letter; this prevents repeated chars.
if(midData.empty()){ //this is just an exception check, so we don't deref a -1 index in the next if-stmt, when the vec is empty
midData.push_back(outPoint);
}
//verify incoming alpha cluster is unique from previous alpha; accumulate ticks if not
else if(outPoint.alpha == midData[midData.size()-1].alpha){
midData[midData.size()-1].ticks += outPoint.ticks;
}
else{
midData.push_back(outPoint);
}
}
}
else{
trigger = 0;
}
}
}
cout << "clusters before merging..." << endl;
PrintOutData(midData);
MergeClusters(midData,outData);
//dbg
//PrintOutData(outData);
}
/*
More advanced event detection, using any attributes that are meaningful (dTheta, coVar(X,Y), etc.).
-dTheta(eventBegin,eventEnd)
-coVar(X,Y) (for some last k-inputs: this is like assessing when a set of point converges to a likely mean
-nKeyHits: num discrete key hits in a single key region; this may be no different than coVar()
-dist(pt[i],pt[i+]) Distance between points some k-ticks apart
This starts with analysis, to figure out what parameters look meaningful, by the data
*/
void SingularityBuilder::Process2(vector<Point>& inData, vector<PointMu>& outData)
{
bool trig = false;
//char c;
vector<PointMu> midData;
int i, sampleRate, segmentWidth, trigger, eventStart, eventEnd, stDevTrigger;
double dx, dist, lastDist, coVar, stDev, nKeyHits, avgDist, avgTheta, lastTheta, dTheta;
//segment width is the sample radius for assessing a trigger
sampleRate = 1; //sample every two ticks. Thus, there will be n/2 analyses
segmentWidth = 3; //every two ticks, grab next four point for analysis
lastTheta = dTheta = avgTheta = 0.0;
double stDevThreshold = 30;
cout << "processing " << inData.size() << " data points in sb.process(), dxThreshold=" << dxThreshold << endl;
//TODO: sleep if no data
trigger = 0;
for(i = 0; i < (inData.size() - segmentWidth - 1); i += sampleRate){
//ignore points outside of the active region, including the <start/stop> region
if(InBounds(inData[i]) && InBounds(inData[i+segmentWidth-1])){
//measures absolute change in distance, a dumb attribute
lastDist = dist;
dist = layoutManager->DoubleDistance(inData[i],inData[i+segmentWidth-1]);
dx = dist - lastDist;
avgDist = layoutManager->AvgDistance(inData,i,segmentWidth);
//measures clustering of a group of points, but only the points themselves
//coVar = layoutManager->CoVariance(inData,i,segmentWidth);
stDev = layoutManager->CoStdDeviation(inData,i,segmentWidth); //(sigmaX*sigmaY)^2
//only update theta when on the move (bounds should be the same as event capture)
if(stDev > 100){ //stop capturing a little ahead of event capture. capture only at med/hi velocity
lastTheta = avgTheta;
avgTheta = layoutManager->AvgTheta(inData,i,segmentWidth); //get the avg direction for a sequence of points
dTheta = (avgTheta - lastTheta) / 2.0;
}
else{
//avgTheta = 0.0;
dTheta = 0.0;
}
//dTheta = layoutManager->CosineSimilarity(inData,i,segmentWidth); // record change in angle or two sequences of points
//correlates points with nearest key. Something like this is important to map kays to points, instead of points to keys... get creative.
//cout << "(x,y) (" << inData[i].X << "," << inData[i].Y << ")" << endl;
//cout << "stDev, dx, 1/dx, coVar, 1/coVar: " << (int)stDev << " " << (int)dx << " " << (dx == 0 ? 0 : (1/dx)) << " " << (int)coVar << " " << (coVar == 0 ? 0 : (1/coVar)) << endl;
//cout << "pt. stDev, avgDist, dx, avgTheta: " << (int)stDev << " " << avgDist << " " << (int)dx << " " << avgTheta << endl;
printf("pt. stDev, avgDist, dist, dx, avgTheta, dTheta: %9.3f %9.3f %9.3f %9.3f %9.3f %9.3f\n",stDev,avgDist,dist,dx,avgTheta,dTheta);
//cout << "stDev: " << (int)stDev << endl;
if(stDev < stDevThreshold){
stDevTrigger++;
if(stDevTrigger > 2){
eventStart = i;
//event triggered, so gather event data
i++;
//stDev = layoutManager->CoStdDeviation(inData,i,segmentWidth);
while(i < inData.size() && stDev < stDevThreshold){
stDev = layoutManager->CoStdDeviation(inData,i,segmentWidth);
printf("pt. stDev, avgDist, dist, dx, avgTheta, dTheta: %9.3f\n",stDev);
i++;
}
eventEnd = i;
i--;
PointMu outPoint;
outPoint.ticks = eventEnd - eventStart;
CalculateMean(eventStart,eventEnd,inData,outPoint);
outPoint.alpha = layoutManager->FindNearestKey(outPoint.pt);
cout << "hit mean for " << outPoint.alpha << endl;
//NOTE A new cluster is appended only if it is a unique letter; this prevents repeated chars.
if(midData.empty()){ //this is just an exception check, so we don't deref a -1 index in the next if-stmt, when the vec is empty
midData.push_back(outPoint);
}
//verify incoming alpha cluster is unique from previous alpha
else if(outPoint.alpha != midData[midData.size()-1].alpha){
midData.push_back(outPoint);
}
stDevTrigger = 0;
}
}
/*
if(!trig && dx < dxThreshold){
cout << "Trigger!" << endl;
trig = true;
}
else{
trig = false;
}
*/
/*
//debug output, to view how data vals change
cout << "dx: " << dx;
if(dx > 0){
cout << " 1/dx: " << (1.0f / (float)dx);
}
cout << endl;
*/
/*
//TODO: advancing the index i below is done without InBounds() checks
if(dx < dxThreshold){ //determine velocity: distance of points three ticks apart
trigger++; //receive n-triggers before fully triggering, to buffer noise; like using a timer, but event-based
if(trigger >= triggerThreshold){ //trigger event and start collecting event data
//capture the event
eventStart = i;
while(i < (inData.size() - 4) && dx < innerDxThreshold){ //event state. remain in this state until dX (inter-reading) exceeds some threshold
dx = layoutManager->IntDistance(inData[i],inData[i+3]);
i++;
}
eventEnd = i; // exited Event state, so store right bound of the event cluster
//TODO: below is a small optimization to skip some data points following an event, eg, perhaps by dx difference of 3 data points.
//i += 2;
//get the mean point w/in the event cluster and store it
PointMu outPoint;
outPoint.ticks = eventEnd - eventStart;
CalculateMean(eventStart,eventEnd,inData,outPoint);
cout << "hit mean" << endl;
outPoint.alpha = layoutManager->FindNearestKey(outPoint.pt);
//NOTE A new cluster is appended only if it is a unique letter; this prevents repeated chars.
if(midData.empty()){ //this is just an exception check, so we don't deref a -1 index in the next if-stmt, when the vec is empty
midData.push_back(outPoint);
}
//verify incoming alpha cluster is unique from previous alpha
else if(outPoint.alpha != midData[midData.size()-1].alpha){
midData.push_back(outPoint);
}
//reset trigger for next event detection
trigger = 0;
} //exit the event-capture state, and continue streaming (reading inData and detecting the next event)
}
*/
}
}
cout << "clusters before merging..." << endl;
PrintOutData(midData);
MergeClusters(midData,outData);
//dbg
//PrintOutData(outData);
}
/*
It will take a lot of test data to figure out what event parameters are even useful, such that we can
optimize the precision of this function and build a really good state machine for the event triggers.
For instance, for some test input path zigzag (as a list of xy coordinates in some file), leverage everything:
build another file of theta, dtheta, velocity, angular acceleration, etc, per that path. Analyze the xy coordinates
for every possible attribute you might think of, and print these to some other file. Then analyze which of those
attributes is merely noise, and which corresponds to data that might be useful to event detection.
You might think of every point in time (each mouse reading, each tick) as a vector, and the current system
state as a set of k-vectors. Each vector contains a bunch of attributes. A cluster is identified when the
system (the matrix) suddenly changes in some meaningful way. The matrix analogy is intentional, since it comes from
control theory, so existing solution undoubtedly exist for this simple problem of geometric key-point detection.
And use python for these experiments, to avoid the labor overhead of c++.
Otherwise the highlevel behavior/state machine of this function is simple:
-detect event
-capture event
-process event (clustering)
-pass event (key-cluster)
-repeat
TODO: recode this using int32, not short. Cast to short where needed.
*/
void SingularityBuilder::Process(vector<Point>& inData, vector<PointMu>& outData)
{
//char c;
vector<PointMu> midData;
int i, trigger, dx, eventStart, eventEnd;
cout << "processing " << inData.size() << " data points in sb.process()" << endl;
//TODO: sleep if no data
trigger = 0;
for(i = 0; i < inData.size() - 4; i++){
//ignore points outside of the active region, including the <start/stop> region
if(InBounds(inData[i]) && InBounds(inData[i+3])){
dx = layoutManager->IntDistance(inData[i],inData[i+3]);
/*
//debug output, to view how data vals change
cout << "dx: " << dx;
if(dx > 0){
cout << " 1/dx: " << (1.0f / (float)dx);
}
cout << endl;
*/
//TODO: advancing the index i below is done without InBounds() checks
if(dx < dxThreshold){ //determine velocity: distance of points three ticks apart
trigger++; //receive n-triggers before fully triggering, to buffer noise; like using a timer, but event-based
if(trigger >= triggerThreshold){ //trigger event and start collecting event data
//capture the event
eventStart = i;
while(i < (inData.size() - 4) && dx < innerDxThreshold){ //event state. remain in this state until dX (inter-reading) exceeds some threshold
dx = layoutManager->IntDistance(inData[i],inData[i+3]);
i++;
}
eventEnd = i; // exited Event state, so store right bound of the event cluster
//TODO: below is a small optimization to skip some data points following an event, eg, perhaps by dx difference of 3 data points.
//i += 2;
//get the mean point w/in the event cluster and store it
PointMu outPoint;
//outPoint.ticks = eventEnd - eventStart;
CalculateMean(eventStart,eventEnd,inData,outPoint);
cout << "hit mean" << endl;
outPoint.alpha = layoutManager->FindNearestKey(outPoint.pt);
//NOTE A new cluster is appended only if it is a unique letter; this prevents repeated chars.
if(midData.empty()){ //this is just an exception check, so we don't deref a -1 index in the next if-stmt, when the vec is empty
midData.push_back(outPoint);
}
//verify incoming alpha cluster is unique from previous alpha
else if(outPoint.alpha != midData[midData.size()-1].alpha){
midData.push_back(outPoint);
}
//reset trigger for next event detection
trigger = 0;
} //exit the event-capture state, and continue streaming (reading inData and detecting the next event)
}
}
}
cout << "clusters before merging..." << endl;
PrintOutData(midData);
MergeClusters(midData,outData);
//dbg
//PrintOutData(outData);
}
