static int
outlierFilterShutdown(OutlierFilter *filter) {
if(filter->rawStats != NULL ) {
freeDoubleMovingStdDev(&filter->rawStats);
}
if(filter->filteredStats != NULL ) {
freeDoubleMovingMean(&filter->filteredStats);
}
resetDoublePermanentMean(&filter->outlierStats);
resetDoublePermanentMean(&filter->acceptedStats);
return 1;
}
static int
outlierFilterReset(OutlierFilter *filter) {
resetDoubleMovingStdDev(filter->rawStats);
resetDoubleMovingMean(filter->filteredStats);
filter->lastOutlier = FALSE;
filter->threshold = filter->config.threshold;
resetDoublePermanentMean(&filter->outlierStats);
resetDoublePermanentMean(&filter->acceptedStats);
filter->delay = 0;
filter->totalDelay = 0;
filter->delayCredit = filter->config.delayCredit;
filter->blocking = FALSE;
return 1;
}
void
resetDoublePermanentStdDev(DoublePermanentStdDev* container)
{
if(container == NULL)
return;
resetDoublePermanentMean(&container->meanContainer);
container->squareSum = 0.0;
container->stdDev = 0.0;
}
static int
outlierFilterInit(OutlierFilter *filter, OutlierFilterConfig *config, const char* id) {
filter->config = *config;
if (config->enabled) {
filter->rawStats = createDoubleMovingStdDev(config->capacity);
strncpy(filter->id, id, OUTLIERFILTER_MAX_DESC);
strncpy(filter->rawStats->identifier, id, 10);
filter->filteredStats = createDoubleMovingMean(config->capacity);
filter->threshold = config->threshold;
} else {
filter->rawStats = NULL;
filter->filteredStats = NULL;
}
resetDoublePermanentMean(&filter->outlierStats);
resetDoublePermanentMean(&filter->acceptedStats);
filter->delayCredit = filter->config.delayCredit;
return 1;
}
/* 2 x fairy dust, 3 x unicorn droppings, 1 x magic beanstalk juice. blend, spray on the affected area twice per day */
static Boolean
outlierFilterFilter(OutlierFilter *filter, double sample)
{
/* true = accepted - this is to tell the user if we advised to throw away the sample */
Boolean ret = TRUE;
/* step change: outlier mean - accepted mean from last sampling period */
double step = 0.0;
if(!filter->config.enabled) {
filter->output = sample;
return TRUE;
}
step = fabs(filter->outlierStats.mean - filter->acceptedStats.bufferedMean);
if(filter->config.autoTune) {
filter->autoTuneSamples++;
}
/* no outlier first - more convenient this way */
if(!isDoublePeircesOutlier(filter->rawStats, sample, filter->threshold) && (filter->delay == 0)) {
filter->lastOutlier = FALSE;
filter->output = sample;
/* filter is about to accept after a blocking period */
if(filter->consecutiveOutliers) {
DBG_LOCAL_ID(filter,"consecutive: %d, mean: %.09fm accepted bmean: %.09f\n", filter->consecutiveOutliers,
filter->outlierStats.mean,filter->acceptedStats.bufferedMean);
/* we are about to open up but the offset has risen above step level, we will block again, but not forever */
if(filter->config.stepDelay &&
(fabs(filter->acceptedStats.bufferedMean) < ((filter->config.stepThreshold + 0.0) / 1E9)) &&
(step > ((filter->config.stepLevel + 0.0) / 1E9))) {
/* if we're to enter blocking, we need 2 * consecutiveOutliers credit */
/* if we're already blocking, we just need enough credit */
/* if we're already blocking, make sure we block no more than maxDelay */
if((filter->blocking && ((filter->config.maxDelay > filter->totalDelay) && (filter->delayCredit >= filter->consecutiveOutliers))) ||
(!filter->blocking && (filter->delayCredit >= filter->consecutiveOutliers * 2 ))) {
if(!filter->blocking) {
INFO_LOCAL_ID(filter,"%.03f us step detected, filter will now block\n", step * 1E6);
}
DBG_LOCAL_ID(filter,"step: %.09f, credit left %d, requesting %d\n",step,
filter->delayCredit, filter->consecutiveOutliers);
filter->delay = filter->consecutiveOutliers;
filter->totalDelay += filter->consecutiveOutliers;
filter->delayCredit -= filter->consecutiveOutliers;
filter->blocking = TRUE;
resetDoublePermanentMean(&filter->outlierStats);
filter->lastOutlier = TRUE;
DBG_LOCAL_ID(filter,"maxdelay: %d, totaldelay: %d\n",filter->config.maxDelay, filter->totalDelay);
return FALSE;
/* much love for the ultra magnetic, cause everybody knows you never got enough credit */
/* we either ran out of credit while blocking, or we did not have enough to start with */
} else {
if(filter->blocking) {
INFO_LOCAL_ID(filter,"blocking time exhausted, filter will stop blocking\n");
} else {
INFO_LOCAL_ID(filter,"%.03f us step detected but filter cannot block\n", step * 1E6);
}
DBG_LOCAL_ID(filter,"credit out (has %d, needed %d)\n", filter->delayCredit, filter->consecutiveOutliers);
}
/* NO STEP */
} else {
if (filter->blocking) {
INFO_LOCAL_ID(filter,"step event over, filter will stop blocking\n");
}
filter->blocking = FALSE;
}
if(filter->totalDelay != 0) {
DBG_LOCAL_ID(filter,"Total waited %d\n", filter->totalDelay);
filter->totalDelay = 0;
}
}
filter->consecutiveOutliers = 0;
resetDoublePermanentMean(&filter->outlierStats);
feedDoublePermanentMean(&filter->acceptedStats, sample);
/* it's an outlier, Sir! */
} else {
filter->lastOutlier = TRUE;
feedDoublePermanentMean(&filter->outlierStats, sample);
if(filter->delay) {
DBG_LOCAL_ID(filter,"delay left: %d\n", filter->delay);
filter->delay--;
return FALSE;
}
filter->autoTuneOutliers++;
filter->consecutiveOutliers++;
if(filter->config.discard) {
ret = FALSE;
} else {
filter->output = filter->filteredStats->mean;
}
DBG_LOCAL_ID(filter,"Outlier: %.09f\n", sample);
/* Allow [weight] * [deviation from mean] to influence std dev in the next outlier checks */
sample = filter->rawStats->meanContainer->mean + filter->config.weight * ( sample - filter->rawStats->meanContainer->mean);
}
/* keep stats containers updated */
feedDoubleMovingStdDev(filter->rawStats, sample);
feedDoubleMovingMean(filter->filteredStats, filter->output);
/* re-tune filter twice per window */
if( (filter->rawStats->meanContainer->counter % ( filter->rawStats->meanContainer->capacity / 2)) == 0) {
outlierFilterTune(filter);
}
/* replenish filter credit once per window */
if( filter->config.stepDelay && ((filter->rawStats->meanContainer->counter % filter->rawStats->meanContainer->capacity) == 0)) {
filter->delayCredit += filter->config.creditIncrement;
if(filter->delayCredit >= filter->config.delayCredit) {
filter->delayCredit = filter->config.delayCredit;
}
DBG_LOCAL_ID(filter,"credit added, now %d\n", filter->delayCredit);
}
return ret;
}
