void VideoWriter_ufmf::addFrame(StampedImage stampedImg) { bool skipFrame = false; bool haveNewMedianImage = false; currentImage_ = stampedImg; // On first call - setup output file, background modeling, start compressors, ... if (isFirst_) { checkImageFormat(stampedImg); // Set output file and write header setupOutputFile(stampedImg); writeHeader(); // Set initial bg median image - just use current image. bgMedianImage_ = stampedImg.image; bgMembershipImage_.create(stampedImg.image.rows, stampedImg.image.cols,CV_8UC1); cv::add(bgMedianImage_, backgroundThreshold_, bgUpperBoundImage_); cv::subtract(bgMedianImage_, backgroundThreshold_, bgLowerBoundImage_); // Start background model and frame compressors startBackgroundModeling(); startCompressors(); writeKeyFrame(); isFirst_ = false; } // Process frames or skip if (frameCount_%frameSkip_==0) { // Add frame to background image queue if it is empty bgImageQueuePtr_ -> acquireLock(); if (bgImageQueuePtr_ -> empty()) { bgImageQueuePtr_ -> push(stampedImg); bgImageQueuePtr_ -> signalNotEmpty(); } bgImageQueuePtr_ -> releaseLock(); // Get median image if available medianMatQueuePtr_ -> acquireLock(); if (!(medianMatQueuePtr_ -> empty())) { bgMedianImage_ = medianMatQueuePtr_ -> front(); medianMatQueuePtr_ -> pop(); haveNewMedianImage = true; } medianMatQueuePtr_ -> releaseLock(); // When new median image available re-calculate thresholds if (haveNewMedianImage) { cv::add(bgMedianImage_, backgroundThreshold_, bgUpperBoundImage_); cv::subtract(bgMedianImage_, backgroundThreshold_, bgLowerBoundImage_); bgUpdateCount_++; bgModelTimeStamp_ = currentImage_.timeStamp; bgModelFrameCount_ = currentImage_.frameCount; writeKeyFrame(); } // Create compressed frame and set its data using the current frame CompressedFrame_ufmf compressedFrame(boxLength_); compressedFrame.dilateEnabled(dilateState_); compressedFrame.setDilateWindowSize(dilateWindowSize_); if (!(framesWaitQueuePtr_ -> empty())) { // Take pre-allocated compressed frame if available compressedFrame = framesWaitQueuePtr_ -> front(); framesWaitQueuePtr_ -> pop(); } compressedFrame.setData( currentImage_, bgLowerBoundImage_, bgUpperBoundImage_, bgUpdateCount_ ); framesToDoQueuePtr_ -> acquireLock(); unsigned int framesToDoQueueSize = framesToDoQueuePtr_ -> size(); if (framesToDoQueueSize < FRAMES_TODO_MAX_QUEUE_SIZE) { // Insert new (uncalculated) compressed frame into "to do" queue. framesToDoQueuePtr_ -> push(compressedFrame); framesToDoQueuePtr_ -> wakeOne(); } else { skipFrame = true; } framesToDoQueuePtr_ -> releaseLock(); if (skipFrame) { // Queue is full - skip frame framesSkippedIndexListPtr_ -> acquireLock(); framesSkippedIndexListPtr_ -> push_back(stampedImg.frameCount); framesSkippedIndexListPtr_ -> releaseLock(); skipReported_ = true; } } // if (frameCount_%frameSkip_==0) // Remove frames from compressed frames "finished" set and write to disk //framesFinishedSetSize = clearFinishedFrames(); clearFinishedFrames(); frameCount_++; // Cull framesWaitQueue if it starts to grow too large unsigned int framesWaitQueueSize = framesWaitQueuePtr_ -> size(); if ( framesWaitQueueSize > FRAMES_WAIT_MAX_QUEUE_SIZE ) { unsigned int numToCull = framesWaitQueueSize - FRAMES_WAIT_MAX_QUEUE_SIZE/2; for (unsigned int i=0; i<numToCull; i++) { framesWaitQueuePtr_ -> pop(); } } // Report skipped frame if ((skipFrame) && (!skipReported_)) { std::cout << "warning: logging overflow - skipped frame -" << std::endl; unsigned int errorId = ERROR_FRAMES_TODO_MAX_QUEUE_SIZE; QString errorMsg("logger framesToDoQueue has exceeded the maximum allowed size"); emit imageLoggingError(errorId, errorMsg); skipReported_ = true; } }
int main(int argc, char **argv) { double tolerance, result; long nEval; int32_t nEvalMax = 5000, nPassMax = 25; double a[4], da[4]; double alo[4], ahi[4]; long n_dimen = 4, zeroes; SDDS_DATASET InputTable, OutputTable; SCANNED_ARG *s_arg; long i_arg, i, clue, fullOutput; char *input, *output, *xName, *yName; int32_t xIndex, yIndex, fitIndex, residualIndex; long retval; double *fitData, *residualData, rmsResidual; unsigned long guessFlags, dummyFlags, pipeFlags; double constantGuess, factorGuess, freqGuess, phaseGuess; double firstZero, lastZero; unsigned long simplexFlags = 0; SDDS_RegisterProgramName(argv[0]); argc = scanargs(&s_arg, argc, argv); if (argc<2 || argc>(2+N_OPTIONS)) bomb(NULL, USAGE); input = output = NULL; tolerance = 1e-6; verbosity = fullOutput = 0; clue = -1; xName = yName = NULL; guessFlags = 0; pipeFlags = 0; for (i_arg=1; i_arg<argc; i_arg++) { if (s_arg[i_arg].arg_type==OPTION) { switch (match_string(s_arg[i_arg].list[0], option, N_OPTIONS, 0)) { case SET_TOLERANCE: if (s_arg[i_arg].n_items!=2 || sscanf(s_arg[i_arg].list[1], "%lf", &tolerance)!=1) SDDS_Bomb("incorrect -tolerance syntax"); break; case SET_VERBOSITY: if (s_arg[i_arg].n_items!=2 || sscanf(s_arg[i_arg].list[1], "%ld", &verbosity)!=1) SDDS_Bomb("incorrect -verbosity syntax"); break; case SET_GUESS: if (s_arg[i_arg].n_items<2) SDDS_Bomb("incorrect -guess syntax"); s_arg[i_arg].n_items -= 1; if (!scanItemList(&guessFlags, s_arg[i_arg].list+1, &s_arg[i_arg].n_items, 0, "constant", SDDS_DOUBLE, &constantGuess, 1, GUESS_CONSTANT_GIVEN, "factor", SDDS_DOUBLE, &factorGuess, 1, GUESS_FACTOR_GIVEN, "frequency", SDDS_DOUBLE, &freqGuess, 1, GUESS_FREQ_GIVEN, "phase", SDDS_DOUBLE, &phaseGuess, 1, GUESS_PHASE_GIVEN, NULL)) SDDS_Bomb("invalid -guess syntax"); break; case SET_COLUMNS: if (s_arg[i_arg].n_items!=3) SDDS_Bomb("invalid -columns syntax"); xName = s_arg[i_arg].list[1]; yName = s_arg[i_arg].list[2]; break; case SET_FULLOUTPUT: fullOutput = 1; break; case SET_LIMITS: if (s_arg[i_arg].n_items<2) SDDS_Bomb("incorrect -limits syntax"); s_arg[i_arg].n_items -= 1; if (!scanItemList(&dummyFlags, s_arg[i_arg].list+1, &s_arg[i_arg].n_items, 0, "evaluations", SDDS_LONG, &nEvalMax, 1, 0, "passes", SDDS_LONG, &nPassMax, 1, 0, NULL) || nEvalMax<=0 || nPassMax<=0) SDDS_Bomb("invalid -limits syntax"); break; case SET_PIPE: if (!processPipeOption(s_arg[i_arg].list+1, s_arg[i_arg].n_items-1, &pipeFlags)) SDDS_Bomb("invalid -pipe syntax"); break; default: fprintf(stderr, "error: unknown/ambiguous option: %s\n", s_arg[i_arg].list[0]); exit(1); break; } } else { if (input==NULL) input = s_arg[i_arg].list[0]; else if (output==NULL) output = s_arg[i_arg].list[0]; else SDDS_Bomb("too many filenames"); } } processFilenames("sddssinefit", &input, &output, pipeFlags, 0, NULL); if (!xName || !yName) SDDS_Bomb("-columns option must be given"); if (!SDDS_InitializeInput(&InputTable, input) || SDDS_GetColumnIndex(&InputTable, xName)<0 || SDDS_GetColumnIndex(&InputTable, yName)<0) SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors); setupOutputFile(&OutputTable, &xIndex, &yIndex, &fitIndex, &residualIndex, output, fullOutput, &InputTable, xName, yName); fitData = residualData = NULL; alo[0] = - (ahi[0] = DBL_MAX); alo[1] = alo[2] = 0; ahi[1] = ahi[2] = DBL_MAX; alo[3] = - (ahi[3] = PIx2); firstZero = lastZero = 0; while ((retval=SDDS_ReadPage(&InputTable))>0) { if (!(xData = SDDS_GetColumnInDoubles(&InputTable, xName)) || !(yData = SDDS_GetColumnInDoubles(&InputTable, yName))) SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors); if ((nData = SDDS_CountRowsOfInterest(&InputTable))<4) continue; find_min_max(&yMin, &yMax, yData, nData); find_min_max(&xMin, &xMax, xData, nData); zeroes = 0; for (i=1; i<nData; i++) if (yData[i]*yData[i-1]<=0) { i++; if (!zeroes) firstZero = (xData[i]+xData[i-1])/2; else lastZero = (xData[i]+xData[i-1])/2; zeroes ++; } a[0] = (yMin+yMax)/2; a[1] = (yMax-yMin)/2; if (!zeroes) a[2] = 2/fabs(xMax-xMin); else a[2] = zeroes/(2*fabs(lastZero-firstZero)); a[3] = 0; if (guessFlags&GUESS_CONSTANT_GIVEN) a[0] = constantGuess; if (guessFlags&GUESS_FACTOR_GIVEN) a[1] = factorGuess; if (guessFlags&GUESS_FREQ_GIVEN) a[2] = freqGuess; if (guessFlags&GUESS_PHASE_GIVEN) a[3] = phaseGuess; alo[1] = a[1]/2; if (!(da[0] = a[0]*0.1)) da[0] = 0.01; if (!(da[1] = a[1]*0.1)) da[1] = 0.01; da[2] = a[2]*0.25; da[3] = 0.01; nEval = simplexMin(&result, a, da, alo, ahi, NULL, n_dimen, -DBL_MAX, tolerance, fitFunction, (verbosity>0?report:NULL), nEvalMax, nPassMax, 12, 3, 1.0, simplexFlags); fitData = trealloc(fitData, sizeof(*fitData)*nData); residualData = trealloc(residualData, sizeof(*residualData)*nData); for (i=result=0; i<nData; i++) result += sqr(residualData[i] = yData[i]-(fitData[i]=a[0]+a[1]*sin(PIx2*a[2]*xData[i]+a[3]))); rmsResidual = sqrt(result/nData); if (verbosity>1) { fprintf(stderr, "RMS deviation: %.15e\n", rmsResidual); fprintf(stderr, "(RMS deviation)/(largest value): %.15e\n", rmsResidual/MAX(fabs(yMin), fabs(yMax))); } if (verbosity>0) { fprintf(stderr, "coefficients of fit to the form y = a0 + a1*sin(2*PI*a2*x+a3), a = \n"); for (i=0; i<4; i++) fprintf(stderr, "%.8e ", a[i]); fprintf(stderr, "\n"); } if (!SDDS_StartPage(&OutputTable, nData) || !SDDS_SetColumn(&OutputTable, SDDS_SET_BY_INDEX, xData, nData, xIndex) || !SDDS_SetColumn(&OutputTable, SDDS_SET_BY_INDEX, fitData, nData, fitIndex) || !SDDS_SetParameters(&OutputTable, SDDS_PASS_BY_VALUE|SDDS_SET_BY_NAME, "sinefitConstant", a[0], "sinefitFactor", a[1], "sinefitFrequency", a[2], "sinefitPhase", a[3], "sinefitRmsResidual", rmsResidual, NULL) || (fullOutput && (!SDDS_SetColumn(&OutputTable, SDDS_SET_BY_INDEX, yData, nData, yIndex) || !SDDS_SetColumn(&OutputTable, SDDS_SET_BY_INDEX, residualData, nData, residualIndex) )) || !SDDS_WritePage(&OutputTable)) SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors); } return 0; }
int main(int argc, char **argv) { double *xData, *yData, *xError, *yError, *derivative, *derivativeError, *derivativePosition; char *input, *output, *xName, *xErrorName, **yName, **yErrorName, **yOutputName, **yOutputErrorName, *ptr; char **yOutputUnits, **yExcludeName; char *mainTemplate[3] = {NULL, NULL, NULL}; char *errorTemplate[3] = {NULL, NULL, NULL}; long i, iArg, yNames, yExcludeNames, rows; int32_t interval; SDDS_DATASET SDDSin, SDDSout; SCANNED_ARG *scanned; unsigned long flags, pipeFlags; long SGLeft, SGRight, SGOrder, SGDerivOrder, intervalGiven, yErrorsSeen; SDDS_RegisterProgramName(argv[0]); argc = scanargs(&scanned, argc, argv); if (argc==1) bomb(NULL, USAGE); input = output = xName = xErrorName = NULL; yName = yErrorName = yExcludeName = NULL; derivative = derivativeError = derivativePosition = yError = yData = xData = xError = NULL; yNames = yExcludeNames = 0; pipeFlags = 0; interval = 2; SGOrder = -1; SGDerivOrder = 1; intervalGiven = 0; yErrorsSeen = 0; for (iArg=1; iArg<argc; iArg++) { if (scanned[iArg].arg_type==OPTION) { /* process options here */ switch (match_string(scanned[iArg].list[0], option, CLO_OPTIONS, 0)) { case CLO_DIFFERENTIATE: if (scanned[iArg].n_items!=2 && scanned[iArg].n_items!=3) SDDS_Bomb("invalid -differentiate syntax"); yName = SDDS_Realloc(yName, sizeof(*yName)*(yNames+1)); yErrorName = SDDS_Realloc(yErrorName, sizeof(*yErrorName)*(yNames+1)); yName[yNames] = scanned[iArg].list[1]; if (scanned[iArg].n_items==3) { yErrorsSeen = 1; yErrorName[yNames] = scanned[iArg].list[2]; } else yErrorName[yNames] = NULL; yNames++; break; case CLO_EXCLUDE: if (scanned[iArg].n_items<2) SDDS_Bomb("invalid -exclude syntax"); moveToStringArray(&yExcludeName, &yExcludeNames, scanned[iArg].list+1, scanned[iArg].n_items-1); break; case CLO_VERSUS: if (xName) SDDS_Bomb("give -versus only once"); if (scanned[iArg].n_items!=2) SDDS_Bomb("invalid -versus syntax"); xName = scanned[iArg].list[1]; xErrorName = NULL; break; case CLO_MAINTEMPLATE: if (scanned[iArg].n_items<2) SDDS_Bomb("invalid -mainTemplate syntax"); scanned[iArg].n_items--; if (!scanItemList(&flags, scanned[iArg].list+1, &scanned[iArg].n_items, 0, "name", SDDS_STRING, mainTemplate+0, 1, 0, "description", SDDS_STRING, mainTemplate+1, 1, 0, "symbol", SDDS_STRING, mainTemplate+2, 1, 0, NULL)) SDDS_Bomb("invalid -mainTemplate syntax"); break; case CLO_ERRORTEMPLATE: if (scanned[iArg].n_items<2) SDDS_Bomb("invalid -errorTemplate syntax"); scanned[iArg].n_items--; if (!scanItemList(&flags, scanned[iArg].list+1, &scanned[iArg].n_items, 0, "name", SDDS_STRING, errorTemplate+0, 1, 0, "description", SDDS_STRING, errorTemplate+1, 1, 0, "symbol", SDDS_STRING, errorTemplate+2, 1, 0, NULL)) SDDS_Bomb("invalid -errorTemplate syntax"); break; case CLO_PIPE: if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags)) SDDS_Bomb("invalid -pipe syntax"); break; case CLO_INTERVAL: if (scanned[iArg].n_items!=2 || sscanf(scanned[iArg].list[1], "%" SCNd32, &interval)!=1 || interval<=0) SDDS_Bomb("invalid -interval syntax/value"); intervalGiven = 1; break; case CLO_SAVITZKYGOLAY: if ((scanned[iArg].n_items!=4 && scanned[iArg].n_items!=5) || sscanf(scanned[iArg].list[1], "%ld", &SGLeft)!=1 || sscanf(scanned[iArg].list[2], "%ld", &SGRight)!=1 || sscanf(scanned[iArg].list[3], "%ld", &SGOrder)!=1 || (scanned[iArg].n_items==5 && sscanf(scanned[iArg].list[4], "%ld", &SGDerivOrder)!=1) || SGLeft<0 || SGRight<0 || (SGLeft+SGRight)<SGOrder || SGOrder<0 || SGDerivOrder<0) SDDS_Bomb("invalid -SavitzkyGolay syntax/values"); break; default: fprintf(stderr, "invalid option seen: %s\n", scanned[iArg].list[0]); exit(1); break; } } else { if (!input) input = scanned[iArg].list[0]; else if (!output) output = scanned[iArg].list[0]; else SDDS_Bomb("too many filenames"); } } if (intervalGiven && SGOrder>=0) SDDS_Bomb("-interval and -SavitzkyGolay options are incompatible"); if (SGOrder>=0 && (xErrorName || yErrorsSeen)) SDDS_Bomb("Savitzky-Golay method does not support errors in data"); processFilenames("sddsderiv", &input, &output, pipeFlags, 0, NULL); if (!yNames) SDDS_Bomb("-differentiate option must be given at least once"); if (!checkErrorNames(yErrorName, yNames)) SDDS_Bomb("either all -differentiate quantities must have errors, or none"); if (!SDDS_InitializeInput(&SDDSin, input)) SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors); if (!(ptr=SDDS_FindColumn(&SDDSin, FIND_NUMERIC_TYPE, xName, NULL))) { fprintf(stderr, "error: column %s doesn't exist\n", xName); exit(1); } free(xName); xName = ptr; if (xErrorName) { if (!(ptr=SDDS_FindColumn(&SDDSin, FIND_NUMERIC_TYPE, xErrorName, NULL))) { fprintf(stderr, "error: column %s doesn't exist\n", xErrorName); exit(1); } else { free(xErrorName); xErrorName = ptr; } } if (!(yNames=expandColumnPairNames(&SDDSin, &yName, &yErrorName, yNames, yExcludeName, yExcludeNames, FIND_NUMERIC_TYPE, 0))) { fprintf(stderr, "error: no quantities to differentiate found in file\n"); exit(1); } setupOutputFile(&SDDSout, &SDDSin, output, &yOutputName, &yOutputErrorName, &yOutputUnits, xName, xErrorName, yName, yErrorName, yNames, mainTemplate, errorTemplate, interval, SGOrder>=0?SGDerivOrder:1); while (SDDS_ReadPage(&SDDSin)>0) { if ((rows = SDDS_CountRowsOfInterest(&SDDSin))<2) SDDS_Bomb("Can't compute derivatives: too little data."); derivative = SDDS_Realloc(derivative, sizeof(*derivative)*rows); derivativeError = SDDS_Realloc(derivativeError, sizeof(*derivativeError)*rows); derivativePosition = SDDS_Realloc(derivativePosition, sizeof(*derivativePosition)*rows); if (!SDDS_StartPage(&SDDSout, rows) || !SDDS_CopyParameters(&SDDSout, &SDDSin)) SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors); xError = NULL; if (!(xData = SDDS_GetColumnInDoubles(&SDDSin, xName)) || (xErrorName && !(xError=SDDS_GetColumnInDoubles(&SDDSin, xErrorName)))) SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors); for (i=0; i<yNames; i++) { yError = NULL; if (!(yData = SDDS_GetColumnInDoubles(&SDDSin, yName[i])) || (yErrorName && yErrorName[i] && !(yError=SDDS_GetColumnInDoubles(&SDDSin, yErrorName[i])))) SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors); if (SGOrder>=0) takeSGDerivative(xData, yData, rows, derivative, derivativePosition, SGLeft, SGRight, SGOrder, SGDerivOrder); else takeDerivative(xData, yData, yError, rows, derivative, derivativeError, derivativePosition, interval); if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, derivative, rows, yOutputName[i]) || (yOutputErrorName && yOutputErrorName[i] && !SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, derivativeError, rows, yOutputErrorName[i]))) SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors); if (yData) free(yData); if (yError) free(yError); yData = yError = NULL; } if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, derivativePosition, rows, xName) || (xErrorName && !SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, xError, rows, xErrorName))) SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors); if (!SDDS_WritePage(&SDDSout)) SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors); if (xData) free(xData); if (xError) free(xError); xData = xError = NULL; } if (!SDDS_Terminate(&SDDSin) || !SDDS_Terminate(&SDDSout)) { SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors); exit(1); } if (derivative) free(derivative); if (derivativeError) free(derivativeError); if (derivativePosition) free(derivativePosition); return(0); }
int main(int argc, char **argv) { STAT_DEFINITION *stat; long stats; STAT_REQUEST *request; long requests, count; SCANNED_ARG *scanned; SDDS_DATASET inData, outData; int32_t power; long i_arg, code, iStat, rows, tmpFileUsed, iColumn, row, posColIndex; long noWarnings, maxSourceColumns; char *input, *output, *positionColumn, **posColumnName; double **inputData, *outputData, value1, value2, topLimit, bottomLimit; unsigned long pipeFlags, scanFlags, majorOrderFlag; char s[100]; double *statWorkArray; double quartilePoint[2] = {25.0, 75.0 }, quartileResult[2]; double decilePoint[2] = {10.0, 90.0 }, decileResult[2]; double percent; short columnMajorOrder=-1; SDDS_RegisterProgramName(argv[0]); argc = scanargs(&scanned, argc, argv); if (argc<2) { bomb("too few arguments", USAGE); } posColumnName = NULL; input = output = positionColumn = NULL; stat = NULL; request = NULL; stats = requests = pipeFlags = 0; noWarnings = 0; outputData = NULL; statWorkArray = NULL; for (i_arg=1; i_arg<argc; i_arg++) { scanFlags = 0; if (scanned[i_arg].arg_type==OPTION) { /* process options here */ switch (code=match_string(scanned[i_arg].list[0], option, N_OPTIONS, 0)) { case SET_MAXIMUM: case SET_MINIMUM: case SET_MEAN: case SET_MEDIAN: case SET_STANDARDDEVIATION: case SET_RMS: case SET_SIGMA: case SET_MAD: case SET_COUNT: case SET_DRANGE: case SET_QRANGE: case SET_SMALLEST: case SET_LARGEST: case SET_SPREAD: if (scanned[i_arg].n_items<3) { fprintf(stderr, "error: invalid -%s syntax\n", option[code]); exit(1); } if (!scanItemList(&scanFlags, scanned[i_arg].list, &scanned[i_arg].n_items, SCANITEMLIST_UNKNOWN_VALUE_OK|SCANITEMLIST_REMOVE_USED_ITEMS| SCANITEMLIST_IGNORE_VALUELESS, "positionColumn", SDDS_STRING, &positionColumn, 1, POSITIONCOLUMN_GIVEN, "toplimit", SDDS_DOUBLE, &topLimit, 1, TOPLIMIT_GIVEN, "bottomlimit", SDDS_DOUBLE, &bottomLimit, 1, BOTTOMLIMIT_GIVEN, NULL)) { sprintf(s, "invalid -%s syntax", scanned[i_arg].list[0]); SDDS_Bomb(s); } requests = addStatRequests(&request, requests, scanned[i_arg].list+1, scanned[i_arg].n_items-1, code, scanFlags); request[requests-1].topLimit = topLimit; request[requests-1].bottomLimit = bottomLimit; if (positionColumn) { if (code==SET_MAXIMUM || code==SET_MINIMUM || code==SET_LARGEST || code==SET_SMALLEST) SDDS_CopyString(&request[requests-1].positionColumn, positionColumn); free(positionColumn); positionColumn = NULL; } break; case SET_PERCENTILE: if (scanned[i_arg].n_items<3) { fprintf(stderr, "error: invalid -%s syntax\n", option[code]); exit(1); } if (!scanItemList(&scanFlags, scanned[i_arg].list, &scanned[i_arg].n_items, SCANITEMLIST_UNKNOWN_VALUE_OK|SCANITEMLIST_REMOVE_USED_ITEMS| SCANITEMLIST_IGNORE_VALUELESS, "value", SDDS_DOUBLE, &percent, 1, PERCENT_GIVEN, "toplimit", SDDS_DOUBLE, &topLimit, 1, TOPLIMIT_GIVEN, "bottomlimit", SDDS_DOUBLE, &bottomLimit, 1, BOTTOMLIMIT_GIVEN, NULL) || !(scanFlags&PERCENT_GIVEN) || percent<=0 || percent>=100) SDDS_Bomb("invalid -percentile syntax"); requests = addStatRequests(&request, requests, scanned[i_arg].list+1, scanned[i_arg].n_items-1, code, scanFlags); request[requests-1].percent = percent; request[requests-1].topLimit = topLimit; request[requests-1].bottomLimit = bottomLimit; break; case SET_SUM: if (scanned[i_arg].n_items<3) { fprintf(stderr, "error: invalid -%s syntax\n", option[code]); exit(1); } power = 1; if (!scanItemList(&scanFlags, scanned[i_arg].list, &scanned[i_arg].n_items, SCANITEMLIST_UNKNOWN_VALUE_OK|SCANITEMLIST_REMOVE_USED_ITEMS| SCANITEMLIST_IGNORE_VALUELESS, "power", SDDS_LONG, &power, 1, 0, "toplimit", SDDS_DOUBLE, &topLimit, 1, TOPLIMIT_GIVEN, "bottomlimit", SDDS_DOUBLE, &bottomLimit, 1, BOTTOMLIMIT_GIVEN, NULL)) SDDS_Bomb("invalid -sum syntax"); requests = addStatRequests(&request, requests, scanned[i_arg].list+1, scanned[i_arg].n_items-1, code, scanFlags); request[requests-1].sumPower = power; request[requests-1].topLimit = topLimit; request[requests-1].bottomLimit = bottomLimit; break; case SET_PIPE: if (!processPipeOption(scanned[i_arg].list+1, scanned[i_arg].n_items-1, &pipeFlags)) SDDS_Bomb("invalid -pipe syntax"); break; case SET_NOWARNINGS: noWarnings = 1; break; case SET_MAJOR_ORDER: majorOrderFlag=0; scanned[i_arg].n_items --; if (scanned[i_arg].n_items>0 && (!scanItemList(&majorOrderFlag, scanned[i_arg].list+1, &scanned[i_arg].n_items, 0, "row", -1, NULL, 0, SDDS_ROW_MAJOR_ORDER, "column", -1, NULL, 0, SDDS_COLUMN_MAJOR_ORDER, NULL))) SDDS_Bomb("invalid -majorOrder syntax/values"); if (majorOrderFlag&SDDS_COLUMN_MAJOR_ORDER) columnMajorOrder=1; else if (majorOrderFlag&SDDS_ROW_MAJOR_ORDER) columnMajorOrder=0; break; default: fprintf(stderr, "error: unknown option '%s' given\n", scanned[i_arg].list[0]); exit(1); break; } } else { /* argument is filename */ if (!input) input = scanned[i_arg].list[0]; else if (!output) output = scanned[i_arg].list[0]; else SDDS_Bomb("too many filenames seen"); } } processFilenames("sddsrowstats", &input, &output, pipeFlags, noWarnings, &tmpFileUsed); if (!requests) SDDS_Bomb("no statistics requested"); if (!SDDS_InitializeInput(&inData, input)) SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors); if (!(stat=compileStatDefinitions(&inData, request, requests, &stats, noWarnings))) { SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors); exit(1); } if (stats<0) SDDS_Bomb("No valid statistics requests."); for (iStat=maxSourceColumns=0; iStat<stats; iStat++) { if (stat[iStat].sourceColumns>maxSourceColumns) maxSourceColumns = stat[iStat].sourceColumns; } if (!(statWorkArray=malloc(sizeof(*statWorkArray)*maxSourceColumns))) SDDS_Bomb("allocation failure (statWorkArray)"); if (!setupOutputFile(&outData, output, &inData, stat, stats)) { SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors); exit(1); } inputData = NULL; while ((code=SDDS_ReadPage(&inData))>0) { if (!SDDS_CopyPage(&outData, &inData)) { SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors); exit(1); } if ((rows = SDDS_CountRowsOfInterest(&inData))) { if (!(outputData = (double*)malloc(sizeof(*outputData)*rows))) SDDS_Bomb("memory allocation failure"); if (!(posColumnName = (char**)malloc(sizeof(*posColumnName)*rows))) SDDS_Bomb("memory allocation failure"); for (iStat=0; iStat<stats; iStat++) { if (!(inputData = (double**)malloc(sizeof(*inputData)*stat[iStat].sourceColumns))) SDDS_Bomb("memory allocation failure"); for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (!(inputData[iColumn] = SDDS_GetColumnInDoubles(&inData, stat[iStat].sourceColumn[iColumn]))) SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors); } for (row=0; row<rows; row++) outputData[row] = DBL_MAX; switch (stat[iStat].optionCode) { case SET_MINIMUM: for (row=0; row<rows; row++) { value1 = DBL_MAX; posColIndex = 0; posColumnName[row] = NULL; for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; if (inputData[iColumn][row]<value1) { value1 = inputData[iColumn][row]; posColIndex = iColumn; } } outputData[row] = value1; if (stat[iStat].positionColumn) posColumnName[row] = stat[iStat].sourceColumn[posColIndex]; } break; case SET_MAXIMUM: for (row=0; row<rows; row++) { posColIndex = 0; value1 = -DBL_MAX; posColumnName[row] = NULL; for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; if (inputData[iColumn][row]>value1) { posColIndex = iColumn; value1 = inputData[iColumn][row]; } } outputData[row] = value1; if (stat[iStat].positionColumn) posColumnName[row] = stat[iStat].sourceColumn[posColIndex]; } break; case SET_MEAN: for (row=0; row<rows; row++) { value1 = 0; count = 0; for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; value1 += inputData[iColumn][row]; count ++; } if (count) outputData[row] = value1/count; } break; case SET_MEDIAN: for (row=0; row<rows; row++) { for (iColumn=count=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; statWorkArray[count] = inputData[iColumn][row]; count++; } if (count) compute_median(outputData+row, statWorkArray, count); } break; case SET_STANDARDDEVIATION: for (row=0; row<rows; row++) { value1 = 0; value2 = 0; count = 0; for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; value1 += inputData[iColumn][row]; value2 += inputData[iColumn][row]*inputData[iColumn][row]; count ++; } if (count>1) { if ((value1 = value2/count - sqr(value1/count))<=0) outputData[row] = 0; else outputData[row] = sqrt(value1*count/(count-1.0)); } } break; case SET_SIGMA: for (row=0; row<rows; row++) { value1 = 0; value2 = 0; count = 0; for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; value1 += inputData[iColumn][row]; value2 += inputData[iColumn][row]*inputData[iColumn][row]; count ++; } if (count>1) { if ((value1 = value2/count - sqr(value1/count))<=0) outputData[row] = 0; else outputData[row] = sqrt(value1/(count-1.0)); } } break; case SET_RMS: for (row=0; row<rows; row++) { value1 = 0; count = 0; for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; value1 += sqr(inputData[iColumn][row]); count ++; } if (count) outputData[row] = sqrt(value1/count); } break; case SET_SUM: for (row=0; row<rows; row++) { value1 = 0; count = 0; for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; value1 += ipow(inputData[iColumn][row], stat[iStat].sumPower); count ++; } if (count) outputData[row] = value1; } break; case SET_COUNT: for (row=0; row<rows; row++) { count = 0; for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; count++; } outputData[row] = count; } break; case SET_MAD: for (row=0; row<rows; row++) { for (iColumn=count=value1=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; statWorkArray[count] = inputData[iColumn][row]; count++; } if (count) computeMoments(NULL, NULL, NULL, &outputData[row], statWorkArray, count); } break; case SET_DRANGE: for (row=0; row<rows; row++) { for (iColumn=count=value1=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; statWorkArray[count] = inputData[iColumn][row]; count++; } if (count && compute_percentiles(decileResult, decilePoint, 2, statWorkArray, count)) outputData[row] = decileResult[1] - decileResult[0]; } break; case SET_QRANGE: for (row=0; row<rows; row++) { for (iColumn=count=value1=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; statWorkArray[count] = inputData[iColumn][row]; count++; } if (count && compute_percentiles(quartileResult, quartilePoint, 2, statWorkArray, count)) outputData[row] = quartileResult[1] - quartileResult[0]; } break; case SET_SMALLEST: for (row=0; row<rows; row++) { value1 = DBL_MAX; posColIndex = 0; posColumnName[row] = NULL; for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; if ((value2=fabs(inputData[iColumn][row]))<value1) { posColIndex = iColumn; value1 = value2; } } outputData[row] = value1; if (stat[iStat].positionColumn) posColumnName[row] = stat[iStat].sourceColumn[posColIndex]; } break; case SET_LARGEST: for (row=0; row<rows; row++) { value1 = 0; posColIndex = 0; posColumnName[row] = NULL; for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; if ((value2=fabs(inputData[iColumn][row]))>value1) { posColIndex = iColumn; value1 = value2; } } outputData[row] = value1; if (stat[iStat].positionColumn) posColumnName[row] = stat[iStat].sourceColumn[posColIndex]; } break; case SET_SPREAD: for (row=0; row<rows; row++) { value1 = DBL_MAX; /* min */ value2 = -DBL_MAX; /* max */ for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; if (inputData[iColumn][row]<value1) value1 = inputData[iColumn][row]; if (inputData[iColumn][row]>value2) value2 = inputData[iColumn][row]; } outputData[row] = value2-value1; } break; case SET_PERCENTILE: for (row=0; row<rows; row++) { for (iColumn=count=value1=0; iColumn<stat[iStat].sourceColumns; iColumn++) { if (stat[iStat].flags&TOPLIMIT_GIVEN && inputData[iColumn][row]>stat[iStat].topLimit) continue; if (stat[iStat].flags&BOTTOMLIMIT_GIVEN && inputData[iColumn][row]<stat[iStat].bottomLimit) continue; statWorkArray[count] = inputData[iColumn][row]; count++; } outputData[row] = HUGE_VAL; if (count) compute_percentiles(&outputData[row], &stat[iStat].percent, 1, statWorkArray, count); } break; default: SDDS_Bomb("invalid statistic code (accumulation loop)"); break; } if (!SDDS_SetColumn(&outData, SDDS_SET_BY_INDEX, outputData, rows, stat[iStat].resultIndex)) SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors); if (stat[iStat].positionColumn) { if (!SDDS_SetColumn(&outData, SDDS_SET_BY_INDEX, posColumnName, rows, stat[iStat].positionColumnIndex)) SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors); } for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) free(inputData[iColumn]); free(inputData); inputData = NULL; } free(outputData); outputData = NULL; free(posColumnName); posColumnName = NULL; } if (!SDDS_WritePage(&outData)) SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors); } free_scanargs(&scanned, argc); for (iStat=0; iStat<stats; iStat++) { if (stat[iStat].positionColumn) free(stat[iStat].positionColumn); for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) free(stat[iStat].sourceColumn[iColumn]); free(stat[iStat].sourceColumn); } free(request); free(stat); if (statWorkArray) free(statWorkArray); if (!SDDS_Terminate(&inData) || !SDDS_Terminate(&outData)) { SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors); exit(1); } if (tmpFileUsed && !replaceFileAndBackUp(input, output)) exit(1); return 0; }