static int outlierFilterShutdown(OutlierFilter *filter) { if(filter->rawStats != NULL ) { freeDoubleMovingStdDev(&filter->rawStats); } if(filter->filteredStats != NULL ) { freeDoubleMovingMean(&filter->filteredStats); } resetDoublePermanentMean(&filter->outlierStats); resetDoublePermanentMean(&filter->acceptedStats); return 1; }
static int outlierFilterReset(OutlierFilter *filter) { resetDoubleMovingStdDev(filter->rawStats); resetDoubleMovingMean(filter->filteredStats); filter->lastOutlier = FALSE; filter->threshold = filter->config.threshold; resetDoublePermanentMean(&filter->outlierStats); resetDoublePermanentMean(&filter->acceptedStats); filter->delay = 0; filter->totalDelay = 0; filter->delayCredit = filter->config.delayCredit; filter->blocking = FALSE; return 1; }
void resetDoublePermanentStdDev(DoublePermanentStdDev* container) { if(container == NULL) return; resetDoublePermanentMean(&container->meanContainer); container->squareSum = 0.0; container->stdDev = 0.0; }
static int outlierFilterInit(OutlierFilter *filter, OutlierFilterConfig *config, const char* id) { filter->config = *config; if (config->enabled) { filter->rawStats = createDoubleMovingStdDev(config->capacity); strncpy(filter->id, id, OUTLIERFILTER_MAX_DESC); strncpy(filter->rawStats->identifier, id, 10); filter->filteredStats = createDoubleMovingMean(config->capacity); filter->threshold = config->threshold; } else { filter->rawStats = NULL; filter->filteredStats = NULL; } resetDoublePermanentMean(&filter->outlierStats); resetDoublePermanentMean(&filter->acceptedStats); filter->delayCredit = filter->config.delayCredit; return 1; }
/* 2 x fairy dust, 3 x unicorn droppings, 1 x magic beanstalk juice. blend, spray on the affected area twice per day */ static Boolean outlierFilterFilter(OutlierFilter *filter, double sample) { /* true = accepted - this is to tell the user if we advised to throw away the sample */ Boolean ret = TRUE; /* step change: outlier mean - accepted mean from last sampling period */ double step = 0.0; if(!filter->config.enabled) { filter->output = sample; return TRUE; } step = fabs(filter->outlierStats.mean - filter->acceptedStats.bufferedMean); if(filter->config.autoTune) { filter->autoTuneSamples++; } /* no outlier first - more convenient this way */ if(!isDoublePeircesOutlier(filter->rawStats, sample, filter->threshold) && (filter->delay == 0)) { filter->lastOutlier = FALSE; filter->output = sample; /* filter is about to accept after a blocking period */ if(filter->consecutiveOutliers) { DBG_LOCAL_ID(filter,"consecutive: %d, mean: %.09fm accepted bmean: %.09f\n", filter->consecutiveOutliers, filter->outlierStats.mean,filter->acceptedStats.bufferedMean); /* we are about to open up but the offset has risen above step level, we will block again, but not forever */ if(filter->config.stepDelay && (fabs(filter->acceptedStats.bufferedMean) < ((filter->config.stepThreshold + 0.0) / 1E9)) && (step > ((filter->config.stepLevel + 0.0) / 1E9))) { /* if we're to enter blocking, we need 2 * consecutiveOutliers credit */ /* if we're already blocking, we just need enough credit */ /* if we're already blocking, make sure we block no more than maxDelay */ if((filter->blocking && ((filter->config.maxDelay > filter->totalDelay) && (filter->delayCredit >= filter->consecutiveOutliers))) || (!filter->blocking && (filter->delayCredit >= filter->consecutiveOutliers * 2 ))) { if(!filter->blocking) { INFO_LOCAL_ID(filter,"%.03f us step detected, filter will now block\n", step * 1E6); } DBG_LOCAL_ID(filter,"step: %.09f, credit left %d, requesting %d\n",step, filter->delayCredit, filter->consecutiveOutliers); filter->delay = filter->consecutiveOutliers; filter->totalDelay += filter->consecutiveOutliers; filter->delayCredit -= filter->consecutiveOutliers; filter->blocking = TRUE; resetDoublePermanentMean(&filter->outlierStats); filter->lastOutlier = TRUE; DBG_LOCAL_ID(filter,"maxdelay: %d, totaldelay: %d\n",filter->config.maxDelay, filter->totalDelay); return FALSE; /* much love for the ultra magnetic, cause everybody knows you never got enough credit */ /* we either ran out of credit while blocking, or we did not have enough to start with */ } else { if(filter->blocking) { INFO_LOCAL_ID(filter,"blocking time exhausted, filter will stop blocking\n"); } else { INFO_LOCAL_ID(filter,"%.03f us step detected but filter cannot block\n", step * 1E6); } DBG_LOCAL_ID(filter,"credit out (has %d, needed %d)\n", filter->delayCredit, filter->consecutiveOutliers); } /* NO STEP */ } else { if (filter->blocking) { INFO_LOCAL_ID(filter,"step event over, filter will stop blocking\n"); } filter->blocking = FALSE; } if(filter->totalDelay != 0) { DBG_LOCAL_ID(filter,"Total waited %d\n", filter->totalDelay); filter->totalDelay = 0; } } filter->consecutiveOutliers = 0; resetDoublePermanentMean(&filter->outlierStats); feedDoublePermanentMean(&filter->acceptedStats, sample); /* it's an outlier, Sir! */ } else { filter->lastOutlier = TRUE; feedDoublePermanentMean(&filter->outlierStats, sample); if(filter->delay) { DBG_LOCAL_ID(filter,"delay left: %d\n", filter->delay); filter->delay--; return FALSE; } filter->autoTuneOutliers++; filter->consecutiveOutliers++; if(filter->config.discard) { ret = FALSE; } else { filter->output = filter->filteredStats->mean; } DBG_LOCAL_ID(filter,"Outlier: %.09f\n", sample); /* Allow [weight] * [deviation from mean] to influence std dev in the next outlier checks */ sample = filter->rawStats->meanContainer->mean + filter->config.weight * ( sample - filter->rawStats->meanContainer->mean); } /* keep stats containers updated */ feedDoubleMovingStdDev(filter->rawStats, sample); feedDoubleMovingMean(filter->filteredStats, filter->output); /* re-tune filter twice per window */ if( (filter->rawStats->meanContainer->counter % ( filter->rawStats->meanContainer->capacity / 2)) == 0) { outlierFilterTune(filter); } /* replenish filter credit once per window */ if( filter->config.stepDelay && ((filter->rawStats->meanContainer->counter % filter->rawStats->meanContainer->capacity) == 0)) { filter->delayCredit += filter->config.creditIncrement; if(filter->delayCredit >= filter->config.delayCredit) { filter->delayCredit = filter->config.delayCredit; } DBG_LOCAL_ID(filter,"credit added, now %d\n", filter->delayCredit); } return ret; }