void VideoWriter_ufmf::addFrame(StampedImage stampedImg)
{
bool skipFrame = false;
bool haveNewMedianImage = false;
currentImage_ = stampedImg;
// On first call - setup output file, background modeling, start compressors, ...
if (isFirst_)
{
checkImageFormat(stampedImg);
// Set output file and write header
setupOutputFile(stampedImg);
writeHeader();
// Set initial bg median image - just use current image.
bgMedianImage_ = stampedImg.image;
bgMembershipImage_.create(stampedImg.image.rows, stampedImg.image.cols,CV_8UC1);
cv::add(bgMedianImage_, backgroundThreshold_, bgUpperBoundImage_);
cv::subtract(bgMedianImage_, backgroundThreshold_, bgLowerBoundImage_);
// Start background model and frame compressors
startBackgroundModeling();
startCompressors();
writeKeyFrame();
isFirst_ = false;
}
// Process frames or skip
if (frameCount_%frameSkip_==0)
{
// Add frame to background image queue if it is empty
bgImageQueuePtr_ -> acquireLock();
if (bgImageQueuePtr_ -> empty())
{
bgImageQueuePtr_ -> push(stampedImg);
bgImageQueuePtr_ -> signalNotEmpty();
}
bgImageQueuePtr_ -> releaseLock();
// Get median image if available
medianMatQueuePtr_ -> acquireLock();
if (!(medianMatQueuePtr_ -> empty()))
{
bgMedianImage_ = medianMatQueuePtr_ -> front();
medianMatQueuePtr_ -> pop();
haveNewMedianImage = true;
}
medianMatQueuePtr_ -> releaseLock();
// When new median image available re-calculate thresholds
if (haveNewMedianImage)
{
cv::add(bgMedianImage_, backgroundThreshold_, bgUpperBoundImage_);
cv::subtract(bgMedianImage_, backgroundThreshold_, bgLowerBoundImage_);
bgUpdateCount_++;
bgModelTimeStamp_ = currentImage_.timeStamp;
bgModelFrameCount_ = currentImage_.frameCount;
writeKeyFrame();
}
// Create compressed frame and set its data using the current frame
CompressedFrame_ufmf compressedFrame(boxLength_);
compressedFrame.dilateEnabled(dilateState_);
compressedFrame.setDilateWindowSize(dilateWindowSize_);
if (!(framesWaitQueuePtr_ -> empty()))
{
// Take pre-allocated compressed frame if available
compressedFrame = framesWaitQueuePtr_ -> front();
framesWaitQueuePtr_ -> pop();
}
compressedFrame.setData(
currentImage_,
bgLowerBoundImage_,
bgUpperBoundImage_,
bgUpdateCount_
);
framesToDoQueuePtr_ -> acquireLock();
unsigned int framesToDoQueueSize = framesToDoQueuePtr_ -> size();
if (framesToDoQueueSize < FRAMES_TODO_MAX_QUEUE_SIZE)
{
// Insert new (uncalculated) compressed frame into "to do" queue.
framesToDoQueuePtr_ -> push(compressedFrame);
framesToDoQueuePtr_ -> wakeOne();
}
else
{
skipFrame = true;
}
framesToDoQueuePtr_ -> releaseLock();
if (skipFrame)
{
// Queue is full - skip frame
framesSkippedIndexListPtr_ -> acquireLock();
framesSkippedIndexListPtr_ -> push_back(stampedImg.frameCount);
framesSkippedIndexListPtr_ -> releaseLock();
skipReported_ = true;
}
} // if (frameCount_%frameSkip_==0)
// Remove frames from compressed frames "finished" set and write to disk
//framesFinishedSetSize = clearFinishedFrames();
clearFinishedFrames();
frameCount_++;
// Cull framesWaitQueue if it starts to grow too large
unsigned int framesWaitQueueSize = framesWaitQueuePtr_ -> size();
if ( framesWaitQueueSize > FRAMES_WAIT_MAX_QUEUE_SIZE )
{
unsigned int numToCull = framesWaitQueueSize - FRAMES_WAIT_MAX_QUEUE_SIZE/2;
for (unsigned int i=0; i<numToCull; i++)
{
framesWaitQueuePtr_ -> pop();
}
}
// Report skipped frame
if ((skipFrame) && (!skipReported_))
{
std::cout << "warning: logging overflow - skipped frame -" << std::endl;
unsigned int errorId = ERROR_FRAMES_TODO_MAX_QUEUE_SIZE;
QString errorMsg("logger framesToDoQueue has exceeded the maximum allowed size");
emit imageLoggingError(errorId, errorMsg);
skipReported_ = true;
}
}
int main(int argc, char **argv)
{
double tolerance, result;
long nEval;
int32_t nEvalMax = 5000, nPassMax = 25;
double a[4], da[4];
double alo[4], ahi[4];
long n_dimen = 4, zeroes;
SDDS_DATASET InputTable, OutputTable;
SCANNED_ARG *s_arg;
long i_arg, i, clue, fullOutput;
char *input, *output, *xName, *yName;
int32_t xIndex, yIndex, fitIndex, residualIndex;
long retval;
double *fitData, *residualData, rmsResidual;
unsigned long guessFlags, dummyFlags, pipeFlags;
double constantGuess, factorGuess, freqGuess, phaseGuess;
double firstZero, lastZero;
unsigned long simplexFlags = 0;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&s_arg, argc, argv);
if (argc<2 || argc>(2+N_OPTIONS))
bomb(NULL, USAGE);
input = output = NULL;
tolerance = 1e-6;
verbosity = fullOutput = 0;
clue = -1;
xName = yName = NULL;
guessFlags = 0;
pipeFlags = 0;
for (i_arg=1; i_arg<argc; i_arg++) {
if (s_arg[i_arg].arg_type==OPTION) {
switch (match_string(s_arg[i_arg].list[0], option, N_OPTIONS, 0)) {
case SET_TOLERANCE:
if (s_arg[i_arg].n_items!=2 || sscanf(s_arg[i_arg].list[1], "%lf", &tolerance)!=1)
SDDS_Bomb("incorrect -tolerance syntax");
break;
case SET_VERBOSITY:
if (s_arg[i_arg].n_items!=2 || sscanf(s_arg[i_arg].list[1], "%ld", &verbosity)!=1)
SDDS_Bomb("incorrect -verbosity syntax");
break;
case SET_GUESS:
if (s_arg[i_arg].n_items<2)
SDDS_Bomb("incorrect -guess syntax");
s_arg[i_arg].n_items -= 1;
if (!scanItemList(&guessFlags, s_arg[i_arg].list+1, &s_arg[i_arg].n_items, 0,
"constant", SDDS_DOUBLE, &constantGuess, 1, GUESS_CONSTANT_GIVEN,
"factor", SDDS_DOUBLE, &factorGuess, 1, GUESS_FACTOR_GIVEN,
"frequency", SDDS_DOUBLE, &freqGuess, 1, GUESS_FREQ_GIVEN,
"phase", SDDS_DOUBLE, &phaseGuess, 1, GUESS_PHASE_GIVEN,
NULL))
SDDS_Bomb("invalid -guess syntax");
break;
case SET_COLUMNS:
if (s_arg[i_arg].n_items!=3)
SDDS_Bomb("invalid -columns syntax");
xName = s_arg[i_arg].list[1];
yName = s_arg[i_arg].list[2];
break;
case SET_FULLOUTPUT:
fullOutput = 1;
break;
case SET_LIMITS:
if (s_arg[i_arg].n_items<2)
SDDS_Bomb("incorrect -limits syntax");
s_arg[i_arg].n_items -= 1;
if (!scanItemList(&dummyFlags, s_arg[i_arg].list+1, &s_arg[i_arg].n_items, 0,
"evaluations", SDDS_LONG, &nEvalMax, 1, 0,
"passes", SDDS_LONG, &nPassMax, 1, 0,
NULL) ||
nEvalMax<=0 || nPassMax<=0)
SDDS_Bomb("invalid -limits syntax");
break;
case SET_PIPE:
if (!processPipeOption(s_arg[i_arg].list+1, s_arg[i_arg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
default:
fprintf(stderr, "error: unknown/ambiguous option: %s\n", s_arg[i_arg].list[0]);
exit(1);
break;
}
}
else {
if (input==NULL)
input = s_arg[i_arg].list[0];
else if (output==NULL)
output = s_arg[i_arg].list[0];
else
SDDS_Bomb("too many filenames");
}
}
processFilenames("sddssinefit", &input, &output, pipeFlags, 0, NULL);
if (!xName || !yName)
SDDS_Bomb("-columns option must be given");
if (!SDDS_InitializeInput(&InputTable, input) ||
SDDS_GetColumnIndex(&InputTable, xName)<0 || SDDS_GetColumnIndex(&InputTable, yName)<0)
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
setupOutputFile(&OutputTable, &xIndex, &yIndex, &fitIndex, &residualIndex, output, fullOutput,
&InputTable, xName, yName);
fitData = residualData = NULL;
alo[0] = - (ahi[0] = DBL_MAX);
alo[1] = alo[2] = 0;
ahi[1] = ahi[2] = DBL_MAX;
alo[3] = - (ahi[3] = PIx2);
firstZero = lastZero = 0;
while ((retval=SDDS_ReadPage(&InputTable))>0) {
if (!(xData = SDDS_GetColumnInDoubles(&InputTable, xName)) ||
!(yData = SDDS_GetColumnInDoubles(&InputTable, yName)))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if ((nData = SDDS_CountRowsOfInterest(&InputTable))<4)
continue;
find_min_max(&yMin, &yMax, yData, nData);
find_min_max(&xMin, &xMax, xData, nData);
zeroes = 0;
for (i=1; i<nData; i++)
if (yData[i]*yData[i-1]<=0) {
i++;
if (!zeroes)
firstZero = (xData[i]+xData[i-1])/2;
else
lastZero = (xData[i]+xData[i-1])/2;
zeroes ++;
}
a[0] = (yMin+yMax)/2;
a[1] = (yMax-yMin)/2;
if (!zeroes)
a[2] = 2/fabs(xMax-xMin);
else
a[2] = zeroes/(2*fabs(lastZero-firstZero));
a[3] = 0;
if (guessFlags&GUESS_CONSTANT_GIVEN)
a[0] = constantGuess;
if (guessFlags&GUESS_FACTOR_GIVEN)
a[1] = factorGuess;
if (guessFlags&GUESS_FREQ_GIVEN)
a[2] = freqGuess;
if (guessFlags&GUESS_PHASE_GIVEN)
a[3] = phaseGuess;
alo[1] = a[1]/2;
if (!(da[0] = a[0]*0.1))
da[0] = 0.01;
if (!(da[1] = a[1]*0.1))
da[1] = 0.01;
da[2] = a[2]*0.25;
da[3] = 0.01;
nEval = simplexMin(&result, a, da, alo, ahi, NULL, n_dimen, -DBL_MAX, tolerance, fitFunction,
(verbosity>0?report:NULL), nEvalMax, nPassMax,
12, 3, 1.0, simplexFlags);
fitData = trealloc(fitData, sizeof(*fitData)*nData);
residualData = trealloc(residualData, sizeof(*residualData)*nData);
for (i=result=0; i<nData; i++)
result += sqr(residualData[i] = yData[i]-(fitData[i]=a[0]+a[1]*sin(PIx2*a[2]*xData[i]+a[3])));
rmsResidual = sqrt(result/nData);
if (verbosity>1) {
fprintf(stderr, "RMS deviation: %.15e\n", rmsResidual);
fprintf(stderr, "(RMS deviation)/(largest value): %.15e\n", rmsResidual/MAX(fabs(yMin), fabs(yMax)));
}
if (verbosity>0) {
fprintf(stderr, "coefficients of fit to the form y = a0 + a1*sin(2*PI*a2*x+a3), a = \n");
for (i=0; i<4; i++)
fprintf(stderr, "%.8e ", a[i]);
fprintf(stderr, "\n");
}
if (!SDDS_StartPage(&OutputTable, nData) ||
!SDDS_SetColumn(&OutputTable, SDDS_SET_BY_INDEX, xData, nData, xIndex) ||
!SDDS_SetColumn(&OutputTable, SDDS_SET_BY_INDEX, fitData, nData, fitIndex) ||
!SDDS_SetParameters(&OutputTable, SDDS_PASS_BY_VALUE|SDDS_SET_BY_NAME,
"sinefitConstant", a[0], "sinefitFactor", a[1], "sinefitFrequency", a[2],
"sinefitPhase", a[3], "sinefitRmsResidual", rmsResidual, NULL) ||
(fullOutput &&
(!SDDS_SetColumn(&OutputTable, SDDS_SET_BY_INDEX, yData, nData, yIndex) ||
!SDDS_SetColumn(&OutputTable, SDDS_SET_BY_INDEX, residualData, nData, residualIndex) )) ||
!SDDS_WritePage(&OutputTable))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
return 0;
}
int main(int argc, char **argv)
{
double *xData, *yData, *xError, *yError, *derivative, *derivativeError, *derivativePosition;
char *input, *output, *xName, *xErrorName, **yName, **yErrorName, **yOutputName, **yOutputErrorName, *ptr;
char **yOutputUnits, **yExcludeName;
char *mainTemplate[3] = {NULL, NULL, NULL};
char *errorTemplate[3] = {NULL, NULL, NULL};
long i, iArg, yNames, yExcludeNames, rows;
int32_t interval;
SDDS_DATASET SDDSin, SDDSout;
SCANNED_ARG *scanned;
unsigned long flags, pipeFlags;
long SGLeft, SGRight, SGOrder, SGDerivOrder, intervalGiven, yErrorsSeen;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&scanned, argc, argv);
if (argc==1)
bomb(NULL, USAGE);
input = output = xName = xErrorName = NULL;
yName = yErrorName = yExcludeName = NULL;
derivative = derivativeError = derivativePosition = yError = yData = xData = xError = NULL;
yNames = yExcludeNames = 0;
pipeFlags = 0;
interval = 2;
SGOrder = -1;
SGDerivOrder = 1;
intervalGiven = 0;
yErrorsSeen = 0;
for (iArg=1; iArg<argc; iArg++) {
if (scanned[iArg].arg_type==OPTION) {
/* process options here */
switch (match_string(scanned[iArg].list[0], option, CLO_OPTIONS, 0)) {
case CLO_DIFFERENTIATE:
if (scanned[iArg].n_items!=2 && scanned[iArg].n_items!=3)
SDDS_Bomb("invalid -differentiate syntax");
yName = SDDS_Realloc(yName, sizeof(*yName)*(yNames+1));
yErrorName = SDDS_Realloc(yErrorName, sizeof(*yErrorName)*(yNames+1));
yName[yNames] = scanned[iArg].list[1];
if (scanned[iArg].n_items==3) {
yErrorsSeen = 1;
yErrorName[yNames] = scanned[iArg].list[2];
} else
yErrorName[yNames] = NULL;
yNames++;
break;
case CLO_EXCLUDE:
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -exclude syntax");
moveToStringArray(&yExcludeName, &yExcludeNames, scanned[iArg].list+1, scanned[iArg].n_items-1);
break;
case CLO_VERSUS:
if (xName)
SDDS_Bomb("give -versus only once");
if (scanned[iArg].n_items!=2)
SDDS_Bomb("invalid -versus syntax");
xName = scanned[iArg].list[1];
xErrorName = NULL;
break;
case CLO_MAINTEMPLATE:
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -mainTemplate syntax");
scanned[iArg].n_items--;
if (!scanItemList(&flags, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"name", SDDS_STRING, mainTemplate+0, 1, 0,
"description", SDDS_STRING, mainTemplate+1, 1, 0,
"symbol", SDDS_STRING, mainTemplate+2, 1, 0,
NULL))
SDDS_Bomb("invalid -mainTemplate syntax");
break;
case CLO_ERRORTEMPLATE:
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -errorTemplate syntax");
scanned[iArg].n_items--;
if (!scanItemList(&flags, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"name", SDDS_STRING, errorTemplate+0, 1, 0,
"description", SDDS_STRING, errorTemplate+1, 1, 0,
"symbol", SDDS_STRING, errorTemplate+2, 1, 0,
NULL))
SDDS_Bomb("invalid -errorTemplate syntax");
break;
case CLO_PIPE:
if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case CLO_INTERVAL:
if (scanned[iArg].n_items!=2 || sscanf(scanned[iArg].list[1], "%" SCNd32, &interval)!=1 ||
interval<=0)
SDDS_Bomb("invalid -interval syntax/value");
intervalGiven = 1;
break;
case CLO_SAVITZKYGOLAY:
if ((scanned[iArg].n_items!=4 && scanned[iArg].n_items!=5) ||
sscanf(scanned[iArg].list[1], "%ld", &SGLeft)!=1 ||
sscanf(scanned[iArg].list[2], "%ld", &SGRight)!=1 ||
sscanf(scanned[iArg].list[3], "%ld", &SGOrder)!=1 ||
(scanned[iArg].n_items==5 &&
sscanf(scanned[iArg].list[4], "%ld", &SGDerivOrder)!=1) ||
SGLeft<0 || SGRight<0 || (SGLeft+SGRight)<SGOrder ||
SGOrder<0 || SGDerivOrder<0)
SDDS_Bomb("invalid -SavitzkyGolay syntax/values");
break;
default:
fprintf(stderr, "invalid option seen: %s\n", scanned[iArg].list[0]);
exit(1);
break;
}
}
else {
if (!input)
input = scanned[iArg].list[0];
else if (!output)
output = scanned[iArg].list[0];
else
SDDS_Bomb("too many filenames");
}
}
if (intervalGiven && SGOrder>=0)
SDDS_Bomb("-interval and -SavitzkyGolay options are incompatible");
if (SGOrder>=0 && (xErrorName || yErrorsSeen))
SDDS_Bomb("Savitzky-Golay method does not support errors in data");
processFilenames("sddsderiv", &input, &output, pipeFlags, 0, NULL);
if (!yNames)
SDDS_Bomb("-differentiate option must be given at least once");
if (!checkErrorNames(yErrorName, yNames))
SDDS_Bomb("either all -differentiate quantities must have errors, or none");
if (!SDDS_InitializeInput(&SDDSin, input))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!(ptr=SDDS_FindColumn(&SDDSin, FIND_NUMERIC_TYPE, xName, NULL))) {
fprintf(stderr, "error: column %s doesn't exist\n", xName);
exit(1);
}
free(xName);
xName = ptr;
if (xErrorName) {
if (!(ptr=SDDS_FindColumn(&SDDSin, FIND_NUMERIC_TYPE, xErrorName, NULL))) {
fprintf(stderr, "error: column %s doesn't exist\n", xErrorName);
exit(1);
}
else {
free(xErrorName);
xErrorName = ptr;
}
}
if (!(yNames=expandColumnPairNames(&SDDSin, &yName, &yErrorName, yNames,
yExcludeName, yExcludeNames, FIND_NUMERIC_TYPE, 0))) {
fprintf(stderr, "error: no quantities to differentiate found in file\n");
exit(1);
}
setupOutputFile(&SDDSout, &SDDSin, output, &yOutputName, &yOutputErrorName, &yOutputUnits,
xName, xErrorName, yName, yErrorName, yNames, mainTemplate, errorTemplate,
interval, SGOrder>=0?SGDerivOrder:1);
while (SDDS_ReadPage(&SDDSin)>0) {
if ((rows = SDDS_CountRowsOfInterest(&SDDSin))<2)
SDDS_Bomb("Can't compute derivatives: too little data.");
derivative = SDDS_Realloc(derivative, sizeof(*derivative)*rows);
derivativeError = SDDS_Realloc(derivativeError, sizeof(*derivativeError)*rows);
derivativePosition = SDDS_Realloc(derivativePosition, sizeof(*derivativePosition)*rows);
if (!SDDS_StartPage(&SDDSout, rows) || !SDDS_CopyParameters(&SDDSout, &SDDSin))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
xError = NULL;
if (!(xData = SDDS_GetColumnInDoubles(&SDDSin, xName)) ||
(xErrorName && !(xError=SDDS_GetColumnInDoubles(&SDDSin, xErrorName))))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (i=0; i<yNames; i++) {
yError = NULL;
if (!(yData = SDDS_GetColumnInDoubles(&SDDSin, yName[i])) ||
(yErrorName && yErrorName[i] && !(yError=SDDS_GetColumnInDoubles(&SDDSin, yErrorName[i]))))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (SGOrder>=0)
takeSGDerivative(xData, yData, rows, derivative, derivativePosition,
SGLeft, SGRight, SGOrder, SGDerivOrder);
else
takeDerivative(xData, yData, yError, rows, derivative, derivativeError,
derivativePosition, interval);
if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, derivative, rows, yOutputName[i]) ||
(yOutputErrorName && yOutputErrorName[i] &&
!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, derivativeError, rows, yOutputErrorName[i])))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (yData) free(yData);
if (yError) free(yError);
yData = yError = NULL;
}
if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, derivativePosition, rows, xName) ||
(xErrorName && !SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, xError, rows, xErrorName)))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!SDDS_WritePage(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (xData) free(xData);
if (xError) free(xError);
xData = xError = NULL;
}
if (!SDDS_Terminate(&SDDSin) || !SDDS_Terminate(&SDDSout)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (derivative) free(derivative);
if (derivativeError) free(derivativeError);
if (derivativePosition) free(derivativePosition);
return(0);
}
int main(int argc, char **argv)
{
STAT_DEFINITION *stat;
long stats;
STAT_REQUEST *request;
long requests, count;
SCANNED_ARG *scanned;
SDDS_DATASET inData, outData;
int32_t power;
long i_arg, code, iStat, rows, tmpFileUsed, iColumn, row, posColIndex;
long noWarnings, maxSourceColumns;
char *input, *output, *positionColumn, **posColumnName;
double **inputData, *outputData, value1, value2, topLimit, bottomLimit;
unsigned long pipeFlags, scanFlags, majorOrderFlag;
char s[100];
double *statWorkArray;
double quartilePoint[2] = {25.0, 75.0 }, quartileResult[2];
double decilePoint[2] = {10.0, 90.0 }, decileResult[2];
double percent;
short columnMajorOrder=-1;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&scanned, argc, argv);
if (argc<2) {
bomb("too few arguments", USAGE);
}
posColumnName = NULL;
input = output = positionColumn = NULL;
stat = NULL;
request = NULL;
stats = requests = pipeFlags = 0;
noWarnings = 0;
outputData = NULL;
statWorkArray = NULL;
for (i_arg=1; i_arg<argc; i_arg++) {
scanFlags = 0;
if (scanned[i_arg].arg_type==OPTION) {
/* process options here */
switch (code=match_string(scanned[i_arg].list[0], option, N_OPTIONS, 0)) {
case SET_MAXIMUM:
case SET_MINIMUM:
case SET_MEAN:
case SET_MEDIAN:
case SET_STANDARDDEVIATION:
case SET_RMS:
case SET_SIGMA:
case SET_MAD:
case SET_COUNT:
case SET_DRANGE:
case SET_QRANGE:
case SET_SMALLEST:
case SET_LARGEST:
case SET_SPREAD:
if (scanned[i_arg].n_items<3) {
fprintf(stderr, "error: invalid -%s syntax\n", option[code]);
exit(1);
}
if (!scanItemList(&scanFlags, scanned[i_arg].list, &scanned[i_arg].n_items,
SCANITEMLIST_UNKNOWN_VALUE_OK|SCANITEMLIST_REMOVE_USED_ITEMS|
SCANITEMLIST_IGNORE_VALUELESS,
"positionColumn", SDDS_STRING, &positionColumn, 1, POSITIONCOLUMN_GIVEN,
"toplimit", SDDS_DOUBLE, &topLimit, 1, TOPLIMIT_GIVEN,
"bottomlimit", SDDS_DOUBLE, &bottomLimit, 1, BOTTOMLIMIT_GIVEN,
NULL)) {
sprintf(s, "invalid -%s syntax", scanned[i_arg].list[0]);
SDDS_Bomb(s);
}
requests = addStatRequests(&request, requests, scanned[i_arg].list+1, scanned[i_arg].n_items-1,
code, scanFlags);
request[requests-1].topLimit = topLimit;
request[requests-1].bottomLimit = bottomLimit;
if (positionColumn) {
if (code==SET_MAXIMUM || code==SET_MINIMUM || code==SET_LARGEST || code==SET_SMALLEST)
SDDS_CopyString(&request[requests-1].positionColumn, positionColumn);
free(positionColumn);
positionColumn = NULL;
}
break;
case SET_PERCENTILE:
if (scanned[i_arg].n_items<3) {
fprintf(stderr, "error: invalid -%s syntax\n", option[code]);
exit(1);
}
if (!scanItemList(&scanFlags, scanned[i_arg].list, &scanned[i_arg].n_items,
SCANITEMLIST_UNKNOWN_VALUE_OK|SCANITEMLIST_REMOVE_USED_ITEMS|
SCANITEMLIST_IGNORE_VALUELESS,
"value", SDDS_DOUBLE, &percent, 1, PERCENT_GIVEN,
"toplimit", SDDS_DOUBLE, &topLimit, 1, TOPLIMIT_GIVEN,
"bottomlimit", SDDS_DOUBLE, &bottomLimit, 1, BOTTOMLIMIT_GIVEN,
NULL) ||
!(scanFlags&PERCENT_GIVEN) || percent<=0 || percent>=100)
SDDS_Bomb("invalid -percentile syntax");
requests = addStatRequests(&request, requests, scanned[i_arg].list+1, scanned[i_arg].n_items-1,
code, scanFlags);
request[requests-1].percent = percent;
request[requests-1].topLimit = topLimit;
request[requests-1].bottomLimit = bottomLimit;
break;
case SET_SUM:
if (scanned[i_arg].n_items<3) {
fprintf(stderr, "error: invalid -%s syntax\n", option[code]);
exit(1);
}
power = 1;
if (!scanItemList(&scanFlags, scanned[i_arg].list, &scanned[i_arg].n_items,
SCANITEMLIST_UNKNOWN_VALUE_OK|SCANITEMLIST_REMOVE_USED_ITEMS|
SCANITEMLIST_IGNORE_VALUELESS,
"power", SDDS_LONG, &power, 1, 0,
"toplimit", SDDS_DOUBLE, &topLimit, 1, TOPLIMIT_GIVEN,
"bottomlimit", SDDS_DOUBLE, &bottomLimit, 1, BOTTOMLIMIT_GIVEN,
NULL))
SDDS_Bomb("invalid -sum syntax");
requests = addStatRequests(&request, requests, scanned[i_arg].list+1, scanned[i_arg].n_items-1,
code, scanFlags);
request[requests-1].sumPower = power;
request[requests-1].topLimit = topLimit;
request[requests-1].bottomLimit = bottomLimit;
break;
case SET_PIPE:
if (!processPipeOption(scanned[i_arg].list+1, scanned[i_arg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case SET_NOWARNINGS:
noWarnings = 1;
break;
case SET_MAJOR_ORDER:
majorOrderFlag=0;
scanned[i_arg].n_items --;
if (scanned[i_arg].n_items>0 &&
(!scanItemList(&majorOrderFlag, scanned[i_arg].list+1, &scanned[i_arg].n_items, 0,
"row", -1, NULL, 0, SDDS_ROW_MAJOR_ORDER,
"column", -1, NULL, 0, SDDS_COLUMN_MAJOR_ORDER,
NULL)))
SDDS_Bomb("invalid -majorOrder syntax/values");
if (majorOrderFlag&SDDS_COLUMN_MAJOR_ORDER)
columnMajorOrder=1;
else if (majorOrderFlag&SDDS_ROW_MAJOR_ORDER)
columnMajorOrder=0;
break;
default:
fprintf(stderr, "error: unknown option '%s' given\n", scanned[i_arg].list[0]);
exit(1);
break;
}
}
else {
/* argument is filename */
if (!input)
input = scanned[i_arg].list[0];
else if (!output)
output = scanned[i_arg].list[0];
else
SDDS_Bomb("too many filenames seen");
}
}
processFilenames("sddsrowstats", &input, &output, pipeFlags, noWarnings, &tmpFileUsed);
if (!requests)
SDDS_Bomb("no statistics requested");
if (!SDDS_InitializeInput(&inData, input))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!(stat=compileStatDefinitions(&inData, request, requests, &stats, noWarnings))) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (stats<0)
SDDS_Bomb("No valid statistics requests.");
for (iStat=maxSourceColumns=0; iStat<stats; iStat++) {
if (stat[iStat].sourceColumns>maxSourceColumns)
maxSourceColumns = stat[iStat].sourceColumns;
}
if (!(statWorkArray=malloc(sizeof(*statWorkArray)*maxSourceColumns)))
SDDS_Bomb("allocation failure (statWorkArray)");
if (!setupOutputFile(&outData, output, &inData, stat, stats)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
inputData = NULL;
while ((code=SDDS_ReadPage(&inData))>0) {
if (!SDDS_CopyPage(&outData, &inData)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if ((rows = SDDS_CountRowsOfInterest(&inData))) {
if (!(outputData = (double*)malloc(sizeof(*outputData)*rows)))
SDDS_Bomb("memory allocation failure");
if (!(posColumnName = (char**)malloc(sizeof(*posColumnName)*rows)))
SDDS_Bomb("memory allocation failure");
for (iStat=0; iStat<stats; iStat++) {
if (!(inputData = (double**)malloc(sizeof(*inputData)*stat[iStat].sourceColumns)))
SDDS_Bomb("memory allocation failure");
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (!(inputData[iColumn] = SDDS_GetColumnInDoubles(&inData, stat[iStat].sourceColumn[iColumn])))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
for (row=0; row<rows; row++)
outputData[row] = DBL_MAX;
switch (stat[iStat].optionCode) {
case SET_MINIMUM:
for (row=0; row<rows; row++) {
value1 = DBL_MAX;
posColIndex = 0;
posColumnName[row] = NULL;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
if (inputData[iColumn][row]<value1) {
value1 = inputData[iColumn][row];
posColIndex = iColumn;
}
}
outputData[row] = value1;
if (stat[iStat].positionColumn)
posColumnName[row] = stat[iStat].sourceColumn[posColIndex];
}
break;
case SET_MAXIMUM:
for (row=0; row<rows; row++) {
posColIndex = 0;
value1 = -DBL_MAX;
posColumnName[row] = NULL;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
if (inputData[iColumn][row]>value1) {
posColIndex = iColumn;
value1 = inputData[iColumn][row];
}
}
outputData[row] = value1;
if (stat[iStat].positionColumn)
posColumnName[row] = stat[iStat].sourceColumn[posColIndex];
}
break;
case SET_MEAN:
for (row=0; row<rows; row++) {
value1 = 0;
count = 0;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
value1 += inputData[iColumn][row];
count ++;
}
if (count)
outputData[row] = value1/count;
}
break;
case SET_MEDIAN:
for (row=0; row<rows; row++) {
for (iColumn=count=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
statWorkArray[count] = inputData[iColumn][row];
count++;
}
if (count)
compute_median(outputData+row, statWorkArray, count);
}
break;
case SET_STANDARDDEVIATION:
for (row=0; row<rows; row++) {
value1 = 0;
value2 = 0;
count = 0;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
value1 += inputData[iColumn][row];
value2 += inputData[iColumn][row]*inputData[iColumn][row];
count ++;
}
if (count>1) {
if ((value1 = value2/count - sqr(value1/count))<=0)
outputData[row] = 0;
else
outputData[row] = sqrt(value1*count/(count-1.0));
}
}
break;
case SET_SIGMA:
for (row=0; row<rows; row++) {
value1 = 0;
value2 = 0;
count = 0;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
value1 += inputData[iColumn][row];
value2 += inputData[iColumn][row]*inputData[iColumn][row];
count ++;
}
if (count>1) {
if ((value1 = value2/count - sqr(value1/count))<=0)
outputData[row] = 0;
else
outputData[row] = sqrt(value1/(count-1.0));
}
}
break;
case SET_RMS:
for (row=0; row<rows; row++) {
value1 = 0;
count = 0;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
value1 += sqr(inputData[iColumn][row]);
count ++;
}
if (count)
outputData[row] = sqrt(value1/count);
}
break;
case SET_SUM:
for (row=0; row<rows; row++) {
value1 = 0;
count = 0;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
value1 += ipow(inputData[iColumn][row], stat[iStat].sumPower);
count ++;
}
if (count)
outputData[row] = value1;
}
break;
case SET_COUNT:
for (row=0; row<rows; row++) {
count = 0;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
count++;
}
outputData[row] = count;
}
break;
case SET_MAD:
for (row=0; row<rows; row++) {
for (iColumn=count=value1=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
statWorkArray[count] = inputData[iColumn][row];
count++;
}
if (count)
computeMoments(NULL, NULL, NULL, &outputData[row],
statWorkArray, count);
}
break;
case SET_DRANGE:
for (row=0; row<rows; row++) {
for (iColumn=count=value1=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
statWorkArray[count] = inputData[iColumn][row];
count++;
}
if (count &&
compute_percentiles(decileResult, decilePoint, 2,
statWorkArray, count))
outputData[row] = decileResult[1] - decileResult[0];
}
break;
case SET_QRANGE:
for (row=0; row<rows; row++) {
for (iColumn=count=value1=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
statWorkArray[count] = inputData[iColumn][row];
count++;
}
if (count &&
compute_percentiles(quartileResult, quartilePoint, 2,
statWorkArray, count))
outputData[row] = quartileResult[1] - quartileResult[0];
}
break;
case SET_SMALLEST:
for (row=0; row<rows; row++) {
value1 = DBL_MAX;
posColIndex = 0;
posColumnName[row] = NULL;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
if ((value2=fabs(inputData[iColumn][row]))<value1) {
posColIndex = iColumn;
value1 = value2;
}
}
outputData[row] = value1;
if (stat[iStat].positionColumn)
posColumnName[row] = stat[iStat].sourceColumn[posColIndex];
}
break;
case SET_LARGEST:
for (row=0; row<rows; row++) {
value1 = 0;
posColIndex = 0;
posColumnName[row] = NULL;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
if ((value2=fabs(inputData[iColumn][row]))>value1) {
posColIndex = iColumn;
value1 = value2;
}
}
outputData[row] = value1;
if (stat[iStat].positionColumn)
posColumnName[row] = stat[iStat].sourceColumn[posColIndex];
}
break;
case SET_SPREAD:
for (row=0; row<rows; row++) {
value1 = DBL_MAX; /* min */
value2 = -DBL_MAX; /* max */
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
if (inputData[iColumn][row]<value1)
value1 = inputData[iColumn][row];
if (inputData[iColumn][row]>value2)
value2 = inputData[iColumn][row];
}
outputData[row] = value2-value1;
}
break;
case SET_PERCENTILE:
for (row=0; row<rows; row++) {
for (iColumn=count=value1=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
statWorkArray[count] = inputData[iColumn][row];
count++;
}
outputData[row] = HUGE_VAL;
if (count)
compute_percentiles(&outputData[row], &stat[iStat].percent, 1, statWorkArray, count);
}
break;
default:
SDDS_Bomb("invalid statistic code (accumulation loop)");
break;
}
if (!SDDS_SetColumn(&outData, SDDS_SET_BY_INDEX, outputData, rows, stat[iStat].resultIndex))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (stat[iStat].positionColumn) {
if (!SDDS_SetColumn(&outData, SDDS_SET_BY_INDEX, posColumnName, rows,
stat[iStat].positionColumnIndex))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++)
free(inputData[iColumn]);
free(inputData);
inputData = NULL;
}
free(outputData);
outputData = NULL;
free(posColumnName);
posColumnName = NULL;
}
if (!SDDS_WritePage(&outData))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
free_scanargs(&scanned, argc);
for (iStat=0; iStat<stats; iStat++) {
if (stat[iStat].positionColumn) free(stat[iStat].positionColumn);
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++)
free(stat[iStat].sourceColumn[iColumn]);
free(stat[iStat].sourceColumn);
}
free(request);
free(stat);
if (statWorkArray) free(statWorkArray);
if (!SDDS_Terminate(&inData) || !SDDS_Terminate(&outData)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (tmpFileUsed && !replaceFileAndBackUp(input, output))
exit(1);
return 0;
}
