int main(int argc, char **argv)
{
int iArg;
char *indepQuantity, **depenQuantity, *fileValuesQuantity, *fileValuesFile, **exclude;
long depenQuantities, monotonicity, excludes;
char *input, *output;
long i, j, rows, readCode, order, valuesReadCode, fillIn, row;
long sequencePoints, combineDuplicates, branch;
int32_t *rowFlag;
double sequenceStart, sequenceEnd;
double sequenceSpacing;
unsigned long flags, interpCode, printFlags, forceMonotonic;
SCANNED_ARG *scanned;
SDDS_DATASET SDDSin, SDDSout, SDDSvalues;
OUTRANGE_CONTROL aboveRange, belowRange;
double *atValue;
long atValues, interpPoints, doNotRead, parallelPages;
double *indepValue, **depenValue, *interpPoint, **outputData;
unsigned long pipeFlags;
FILE *fpPrint;
short interpShort=0, interpShortOrder=-1, *shortValue=NULL;
long nextPos;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&scanned, argc, argv);
if (argc<3 || argc>(3+CLO_OPTIONS))
bomb(NULL, USAGE);
atValue = NULL;
atValues = fillIn = 0;
output = input = NULL;
combineDuplicates = branch = sequencePoints = parallelPages = 0;
indepQuantity = NULL;
depenQuantity = exclude = NULL;
depenQuantities = excludes = 0;
aboveRange.flags = belowRange.flags = OUTRANGE_SATURATE;
order = 1;
readCode = interpPoints = 0;
fileValuesFile = fileValuesQuantity = NULL;
sequenceStart = sequenceEnd = sequenceSpacing = 0;
printFlags = pipeFlags = 0;
forceMonotonic = 0;
indepValue = interpPoint = NULL;
depenValue = outputData = NULL;
for (iArg=1; iArg<argc; iArg++) {
if (scanned[iArg].arg_type==OPTION) {
/* process options here */
switch (match_string(scanned[iArg].list[0], option, CLO_OPTIONS, 0)) {
case CLO_ORDER:
if (scanned[iArg].n_items!=2 || sscanf(scanned[iArg].list[1], "%ld", &order)!=1 ||
order<1)
SDDS_Bomb("invalid -order syntax/value");
break;
case CLO_ATVALUES:
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -atValues syntax");
if (atValue)
SDDS_Bomb("give -atValues only once");
atValue = tmalloc(sizeof(*atValue)*(atValues=scanned[iArg].n_items-1));
for (i=0; i<atValues; i++)
if (sscanf(scanned[iArg].list[i+1], "%lf", &atValue[i])!=1)
SDDS_Bomb("invalid -atValues value");
break;
case CLO_INTERP_SHORT:
if (scanned[iArg].n_items==2) {
if (sscanf(scanned[iArg].list[1], "%hd", &interpShortOrder)!=1)
SDDS_Bomb("invalid -interpShort value");
}
interpShort = 1;
break;
case CLO_SEQUENCE:
if ((scanned[iArg].n_items!=2 && scanned[iArg].n_items!=4) ||
sscanf(scanned[iArg].list[1], "%ld", &sequencePoints)!=1 || sequencePoints<2)
SDDS_Bomb("invalid -sequence syntax/value");
if (scanned[iArg].n_items==4 &&
(sscanf(scanned[iArg].list[2], "%lf", &sequenceStart)!=1 ||
sscanf(scanned[iArg].list[3], "%lf", &sequenceEnd)!=1))
SDDS_Bomb("invalid -sequence syntax/value");
if (sequenceSpacing)
SDDS_Bomb("give only one of -sequence and -equispaced");
break;
case CLO_EQUISPACED:
if ((scanned[iArg].n_items!=2 && scanned[iArg].n_items!=4) ||
sscanf(scanned[iArg].list[1], "%lf", &sequenceSpacing)!=1 || sequenceSpacing<=0)
SDDS_Bomb("invalid -equispaced syntax/value");
if (scanned[iArg].n_items==4 &&
(sscanf(scanned[iArg].list[2], "%lf", &sequenceStart)!=1 ||
sscanf(scanned[iArg].list[3], "%lf", &sequenceEnd)!=1))
SDDS_Bomb("invalid -equispaced syntax/values");
if (sequencePoints)
SDDS_Bomb("give only one of -sequence and -equispaced");
break;
case CLO_COLUMNS:
if (indepQuantity)
SDDS_Bomb("only one -columns option may be given");
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -columns syntax");
indepQuantity = scanned[iArg].list[1];
if (scanned[iArg].n_items>=2) {
depenQuantity = tmalloc(sizeof(*depenQuantity)*(depenQuantities=scanned[iArg].n_items-2));
for (i=0; i<depenQuantities; i++)
depenQuantity[i] = scanned[iArg].list[i+2];
}
break;
case CLO_PRINTOUT:
if ((scanned[iArg].n_items-=1)>=1) {
if (!scanItemList(&printFlags, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"bare", -1, NULL, 0, BARE_PRINTOUT,
"stdout", -1, NULL, 0, STDOUT_PRINTOUT,
NULL))
SDDS_Bomb("invalid -printout syntax");
}
if (!(printFlags&BARE_PRINTOUT))
printFlags |= NORMAL_PRINTOUT;
break;
case CLO_FILEVALUES:
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -fileValues syntax");
fileValuesFile = scanned[iArg].list[1];
scanned[iArg].n_items -= 2;
if (!scanItemList(&flags, scanned[iArg].list+2, &scanned[iArg].n_items, 0,
"column", SDDS_STRING, &fileValuesQuantity, 1, 0,
"parallelpages", -1, NULL, 0, FILEVALUES_PARALLEL_PAGES,
NULL))
SDDS_Bomb("invalid -fileValues syntax");
if (flags&FILEVALUES_PARALLEL_PAGES)
parallelPages = 1;
break;
case CLO_COMBINEDUPLICATES:
SDDS_Bomb("-combineDuplicates option not implemented yet--send email to [email protected]");
combineDuplicates = 1;
break;
case CLO_BRANCH:
SDDS_Bomb("-branch option not implemented yet--send email to [email protected]");
if (scanned[iArg].n_items!=2 || sscanf(scanned[iArg].list[1], "%ld", &branch)!=1 ||
branch<1)
SDDS_Bomb("invalid -branch syntax/value");
break;
case CLO_BELOWRANGE:
if ((scanned[iArg].n_items-=1)<1 ||
!scanItemList(&belowRange.flags, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"value", SDDS_DOUBLE, &belowRange.value, 1, OUTRANGE_VALUE,
"skip", -1, NULL, 0, OUTRANGE_SKIP,
"saturate", -1, NULL, 0, OUTRANGE_SATURATE,
"extrapolate", -1, NULL, 0, OUTRANGE_EXTRAPOLATE,
"wrap", -1, NULL, 0, OUTRANGE_WRAP,
"abort", -1, NULL, 0, OUTRANGE_ABORT,
"warn", -1, NULL, 0, OUTRANGE_WARN,
NULL))
SDDS_Bomb("invalid -belowRange syntax/value");
if ((i=bitsSet(belowRange.flags&
(OUTRANGE_VALUE|OUTRANGE_SKIP|OUTRANGE_SATURATE|OUTRANGE_EXTRAPOLATE|OUTRANGE_WRAP|OUTRANGE_ABORT)))>1)
SDDS_Bomb("incompatible keywords given for -belowRange");
if (i!=1)
belowRange.flags |= OUTRANGE_SATURATE;
break;
case CLO_ABOVERANGE:
if ((scanned[iArg].n_items-=1)<1 ||
!scanItemList(&aboveRange.flags, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"value", SDDS_DOUBLE, &aboveRange.value, 1, OUTRANGE_VALUE,
"skip", -1, NULL, 0, OUTRANGE_SKIP,
"saturate", -1, NULL, 0, OUTRANGE_SATURATE,
"extrapolate", -1, NULL, 0, OUTRANGE_EXTRAPOLATE,
"wrap", -1, NULL, 0, OUTRANGE_WRAP,
"abort", -1, NULL, 0, OUTRANGE_ABORT,
"warn", -1, NULL, 0, OUTRANGE_WARN,
NULL))
SDDS_Bomb("invalid -aboveRange syntax/value");
if ((i=bitsSet(aboveRange.flags&
(OUTRANGE_VALUE|OUTRANGE_SKIP|OUTRANGE_SATURATE|OUTRANGE_EXTRAPOLATE|OUTRANGE_WRAP|OUTRANGE_ABORT)))>1)
SDDS_Bomb("incompatible keywords given for -aboveRange");
if (i!=1)
aboveRange.flags |= OUTRANGE_SATURATE;
break;
case CLO_PIPE:
if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case CLO_EXCLUDE:
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -exclude syntax");
moveToStringArray(&exclude, &excludes, scanned[iArg].list+1, scanned[iArg].n_items-1);
break;
case CLO_FORCEMONOTONIC:
if ((scanned[iArg].n_items-=1)>0) {
if (!scanItemList(&forceMonotonic, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"increasing", -1, NULL, 0, FORCE_INCREASING,
"decreasing", -1, NULL, 0, FORCE_DECREASING,
NULL) || bitsSet(forceMonotonic)!=1)
SDDS_Bomb("invalid -forceMonotonic syntax/value");
}
else
forceMonotonic = FORCE_MONOTONIC;
break;
case CLO_FILLIN:
fillIn = 1;
break;
default:
fprintf(stderr, "error: unknown/ambiguous option: %s\n",
scanned[iArg].list[0]);
exit(1);
break;
}
}
else {
if (!input)
input = scanned[iArg].list[0];
else if (!output)
output = scanned[iArg].list[0];
else
SDDS_Bomb("too many filenames seen");
}
}
processFilenames("sddsinterp", &input, &output, pipeFlags, 0, NULL);
fpPrint = stderr;
if (printFlags&STDOUT_PRINTOUT)
fpPrint = stdout;
if (!indepQuantity)
SDDS_Bomb("supply the independent quantity name with the -columns option");
if ((atValues?1:0)+(fileValuesFile?1:0)+(sequencePoints?1:0)+fillIn+(sequenceSpacing>0?1:0) != 1)
SDDS_Bomb("you must give one and only one of -atValues, -fileValues, -sequence, -equispaced, and -fillIn");
if (!SDDS_InitializeInput(&SDDSin, input))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
excludes = appendToStringArray(&exclude, excludes, indepQuantity);
if (!depenQuantities)
depenQuantities = appendToStringArray(&depenQuantity, depenQuantities, "*");
if ((depenQuantities=expandColumnPairNames(&SDDSin, &depenQuantity, NULL, depenQuantities,
exclude, excludes, FIND_NUMERIC_TYPE, 0))<=0) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
SDDS_Bomb("no dependent quantities selected for interpolation");
}
if (fileValuesFile && !SDDS_InitializeInput(&SDDSvalues, fileValuesFile))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!SDDS_InitializeOutput(&SDDSout, SDDS_BINARY, 0, NULL, "sddsinterp output", output))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!SDDS_TransferColumnDefinition(&SDDSout, &SDDSin, indepQuantity, NULL))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
/*
if (fileValuesQuantity && strcmp(fileValuesQuantity, indepQuantity)!=0 &&
!SDDS_TransferColumnDefinition(&SDDSout, &SDDSvalues, fileValuesQuantity, NULL)) {
fprintf(stderr, "problem creating -fileValues column %s\n", fileValuesQuantity);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
*/
if (SDDS_DefineParameter(&SDDSout, "InterpDataPage", NULL, NULL,
"Page of interpolation data file used to create this page",
NULL, SDDS_LONG, NULL)<0 ||
SDDS_DefineParameter(&SDDSout, "InterpPointsPage", NULL, NULL,
"Page of interpolation points file used to create this page",
NULL, SDDS_LONG, NULL)<0)
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (i=0; i<depenQuantities; i++)
if (!SDDS_TransferColumnDefinition(&SDDSout, &SDDSin, depenQuantity[i], NULL)) {
fprintf(stderr, "problem creating interpolated-output column %s\n", depenQuantity[i]);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_TransferAllParameterDefinitions(&SDDSout, &SDDSin, SDDS_TRANSFER_KEEPOLD) ||
!SDDS_WriteLayout(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
doNotRead = 0;
interpPoint = NULL;
outputData = tmalloc(sizeof(*outputData)*(depenQuantities));
depenValue = tmalloc(sizeof(*depenValue)*(depenQuantities));
rowFlag = NULL;
valuesReadCode = 0;
while (doNotRead || (readCode=SDDS_ReadPage(&SDDSin))>0) {
rows = SDDS_CountRowsOfInterest(&SDDSin);
if (!(indepValue = SDDS_GetColumnInDoubles(&SDDSin, indepQuantity)))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (atValues) {
interpPoint = atValue;
interpPoints = atValues;
}
else if (fileValuesFile) {
if (interpPoint)
free(interpPoint);
if ((valuesReadCode=SDDS_ReadPage(&SDDSvalues))==0)
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
else if (valuesReadCode==-1) {
if (parallelPages) {
fprintf(stderr, "warning: file %s ends before file %s\n", fileValuesFile, input);
break;
}
else {
/* "rewind" the values file */
if (!SDDS_Terminate(&SDDSvalues) ||
!SDDS_InitializeInput(&SDDSvalues, fileValuesFile))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if ((valuesReadCode=SDDS_ReadPage(&SDDSvalues))<1) {
fprintf(stderr, "error: unable to (re)read file %s\n", fileValuesFile);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
/* read the next page of the interpolation data file */
if ((readCode=SDDS_ReadPage(&SDDSin))<1) {
if (readCode==-1)
break;
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
rows = SDDS_CountRowsOfInterest(&SDDSin);
if (indepValue)
free(indepValue);
if (!(indepValue = SDDS_GetColumnInDoubles(&SDDSin, indepQuantity)))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
if (!parallelPages)
doNotRead = 1;
interpPoints = SDDS_CountRowsOfInterest(&SDDSvalues);
interpPoint = SDDS_GetColumnInDoubles(&SDDSvalues, fileValuesQuantity);
if (SDDS_NumberOfErrors())
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
else if (sequencePoints || sequenceSpacing) {
if (interpPoint)
free(interpPoint);
interpPoints = sequencePoints;
if (!(interpPoint = makeSequence(&interpPoints, sequenceStart, sequenceEnd, sequenceSpacing,
indepValue, rows)))
exit(1);
} else {
/* fillIn interpolation */
if (interpPoint)
free(interpPoint);
if (!(interpPoint = makeFillInSequence(indepValue, rows, &interpPoints)))
exit(1);
}
for (i=0; i<depenQuantities; i++)
outputData[i] = tmalloc(sizeof(*outputData[i])*interpPoints);
rowFlag = trealloc(rowFlag, sizeof(*rowFlag)*interpPoints);
for (j=0; j<interpPoints; j++)
rowFlag[j] = 1;
for (i=0; i<depenQuantities; i++) {
if (!(depenValue[i] = SDDS_GetColumnInDoubles(&SDDSin, depenQuantity[i])))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (forceMonotonic)
rows = forceMonotonicity(indepValue, depenValue, depenQuantities, rows, forceMonotonic);
else if (combineDuplicates)
rows = combineDuplicatePoints(indepValue, depenValue, depenQuantities, rows, 0.0);
if ((monotonicity=checkMonotonicity(indepValue, rows))==0)
SDDS_Bomb("independent data values do not change monotonically or repeated independent values exist");
if (interpShort)
shortValue = malloc(sizeof(*shortValue)*rows);
for (i=0; i<depenQuantities; i++) {
if (interpShort) {
for (row=0; row<rows; row++) {
shortValue[row] = (short)depenValue[i][row];
}
}
for (j=0; j<interpPoints; j++) {
if (!interpShort) {
outputData[i][j] = interpolate(depenValue[i], indepValue, rows, interpPoint[j], &belowRange,
&aboveRange, order, &interpCode, monotonicity);
} else {
outputData[i][j] = (double)interp_short(shortValue, indepValue, rows, interpPoint[j], 0, -1,
&interpCode, &nextPos);
}
if (interpCode) {
if (interpCode&OUTRANGE_ABORT) {
fprintf(stderr, "error: value %e is out of range for column %s\n",
interpPoint[j], depenQuantity[i]);
exit(1);
}
if (interpCode&OUTRANGE_WARN)
fprintf(stderr, "warning: value %e is out of range for column %s\n",
interpPoint[j], depenQuantity[i]);
if (interpCode&OUTRANGE_SKIP)
rowFlag[j] = 0;
}
}
}
if (interpShort)
free(shortValue);
if (!SDDS_StartPage(&SDDSout, interpPoints) ||
!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_SET_BY_NAME, interpPoint, interpPoints, indepQuantity))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!SDDS_SetParameters(&SDDSout, SDDS_BY_NAME|SDDS_PASS_BY_VALUE,
"InterpDataPage", readCode,
"InterpPointsPage", valuesReadCode, NULL) ||
!SDDS_CopyParameters(&SDDSout, &SDDSin))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (i=0; i<depenQuantities; i++)
if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_SET_BY_NAME, outputData[i], interpPoints, depenQuantity[i]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!SDDS_AssertRowFlags(&SDDSout, SDDS_FLAG_ARRAY, rowFlag, rows) || !SDDS_WritePage(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (printFlags&BARE_PRINTOUT) {
for (j=0; j<interpPoints; j++)
if (rowFlag[j]) {
fprintf(fpPrint, "%21.15e ", interpPoint[j]);
for (i=0; i<depenQuantities; i++)
fprintf(fpPrint, "%21.15e ", outputData[i][j]);
fputc('\n', fpPrint);
}
}
else if (printFlags&NORMAL_PRINTOUT) {
for (j=0; j<interpPoints; j++)
if (rowFlag[j]) {
fprintf(fpPrint, "%s=%21.15e ", indepQuantity, interpPoint[j]);
for (i=0; i<depenQuantities; i++)
fprintf(fpPrint, "%s=%21.15e ", depenQuantity[i], outputData[i][j]);
fputc('\n', fpPrint);
}
}
if (indepValue) free(indepValue);
indepValue = NULL;
for (i=0; i<depenQuantities; i++) {
if (outputData[i]) free(outputData[i]);
outputData[i] = NULL;
if (depenValue[i]) free(depenValue[i]);
depenValue[i] = NULL;
}
if (fileValuesFile) {
if (interpPoint) free(interpPoint);
interpPoint = NULL;
}
if (rowFlag) free(rowFlag);
rowFlag = NULL;
}
if (!SDDS_Terminate(&SDDSin)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (!SDDS_Terminate(&SDDSout)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (fileValuesFile) {
if (!SDDS_Terminate(&SDDSvalues)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
}
return 0;
}
int main(int argc, char **argv)
{
int iArg;
char **outputColumn, **difColumn;
char *indepColumn, **depenColumn, **exclude;
long depenColumns, excludes;
char *input, *output;
long i, rows, readCode, optionCode;
unsigned long flags, pipeFlags;
SCANNED_ARG *scanned;
SDDS_DATASET SDDSin, SDDSout;
double *timeData, *inputData, *outputData;
FILTER_STAGE *filterStage;
long filterStages, totalFilters;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&scanned, argc, argv);
if (argc<3 || argc>(3+N_OPTIONS))
bomb(NULL, USAGE);
output = input = NULL;
flags = pipeFlags = 0;
indepColumn = NULL;
depenColumn = exclude = NULL;
depenColumns = excludes = 0;
if (!(filterStage = (FILTER_STAGE*)calloc(1, sizeof(*filterStage))))
SDDS_Bomb("allocation failure");
filterStage->filter = NULL;
filterStage->filters = 0;
filterStages = 1;
totalFilters = 0;
for (iArg=1; iArg<argc; iArg++) {
if (scanned[iArg].arg_type==OPTION) {
/* process options here */
switch (optionCode=match_string(scanned[iArg].list[0], option, N_OPTIONS, 0)) {
case SET_PIPE:
if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case SET_COLUMNS:
if (indepColumn)
SDDS_Bomb("only one -columns option may be given");
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -columns syntax");
indepColumn = scanned[iArg].list[1];
if (scanned[iArg].n_items>=2) {
depenColumn = tmalloc(sizeof(*depenColumn)*(depenColumns=scanned[iArg].n_items-2));
for (i=0; i<depenColumns; i++)
depenColumn[i] = scanned[iArg].list[i+2];
}
break;
case SET_THRESHOLD:
case SET_HIGHPASS:
case SET_LOWPASS:
case SET_NOTCH:
case SET_BANDPASS:
case SET_FILTERFILE:
case SET_CLIPFREQ:
addFilter(filterStage+filterStages-1, optionCode, scanned+iArg);
totalFilters++;
break;
case SET_CASCADE:
if (filterStages==0)
SDDS_Bomb("-cascade option precedes all filter definitions");
if (!(filterStage = SDDS_Realloc(filterStage, (filterStages+1)*sizeof(*filterStage))))
SDDS_Bomb("allocation failure");
filterStage[filterStages].filter = NULL;
filterStage[filterStages].filters = 0;
filterStages++;
break;
case SET_NEWCOLUMNS:
flags |= FL_NEWCOLUMNS;
break;
case SET_DIFFERENCECOLUMNS:
flags |= FL_DIFCOLUMNS;
break;
case SET_EXCLUDE:
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -exclude syntax");
moveToStringArray(&exclude, &excludes, scanned[iArg].list+1, scanned[iArg].n_items-1);
break;
default:
fprintf(stderr, "error: unknown/ambiguous option: %s (%s)\n",
scanned[iArg].list[0], argv[0]);
exit(1);
break;
}
}
else {
if (!input)
input = scanned[iArg].list[0];
else if (!output)
output = scanned[iArg].list[0];
else
SDDS_Bomb("too many filenames seen");
}
}
processFilenames("sddsfdfilter", &input, &output, pipeFlags, 0, NULL);
if (!totalFilters)
fputs("warning: no filters specified (sddsfdfilter)\n", stderr);
if (!indepColumn)
SDDS_Bomb("supply the independent column name with the -columns option");
if (!SDDS_InitializeInput(&SDDSin, input))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (SDDS_CheckColumn(&SDDSin, indepColumn, NULL, SDDS_ANY_NUMERIC_TYPE, stderr)!=SDDS_CHECK_OKAY)
exit(1);
excludes = appendToStringArray(&exclude, excludes, indepColumn);
if (!depenColumns)
depenColumns = appendToStringArray(&depenColumn, depenColumns, "*");
if ((depenColumns=expandColumnPairNames(&SDDSin, &depenColumn, NULL, depenColumns, exclude, excludes,
FIND_NUMERIC_TYPE, 0))<=0) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
SDDS_Bomb("No quantities selected to filter");
}
if (!SDDS_InitializeCopy(&SDDSout, &SDDSin, output, "w"))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (flags&FL_NEWCOLUMNS) {
outputColumn = tmalloc(sizeof(*outputColumn)*depenColumns);
for (i=0; i<depenColumns; i++) {
outputColumn[i] = tmalloc(sizeof(**outputColumn)*(strlen(depenColumn[i])+1+strlen("Filtered")));
sprintf(outputColumn[i], "%sFiltered", depenColumn[i]);
if (!SDDS_TransferColumnDefinition(&SDDSout, &SDDSin, depenColumn[i], outputColumn[i]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
else
outputColumn=depenColumn;
difColumn = NULL;
if (flags&FL_DIFCOLUMNS) {
difColumn = tmalloc(sizeof(*difColumn)*depenColumns);
for (i=0; i<depenColumns; i++) {
difColumn[i] = tmalloc(sizeof(**difColumn)*(strlen(depenColumn[i])+1+strlen("Difference")));
sprintf(difColumn[i], "%sDifference", depenColumn[i]);
if (!SDDS_TransferColumnDefinition(&SDDSout, &SDDSin, depenColumn[i], difColumn[i]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
if (!SDDS_WriteLayout(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
outputData = NULL;
while ((readCode=SDDS_ReadPage(&SDDSin))>0) {
if (!SDDS_CopyPage(&SDDSout, &SDDSin))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if ((rows = SDDS_CountRowsOfInterest(&SDDSin))<0)
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (rows) {
if (!(timeData = SDDS_GetColumnInDoubles(&SDDSin, indepColumn)))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!(outputData = SDDS_Realloc(outputData, sizeof(*outputData)*rows)))
SDDS_Bomb("allocation failure");
for (i=0; i<depenColumns; i++) {
if (!(inputData = SDDS_GetColumnInDoubles(&SDDSin, depenColumn[i])))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!applyFilters(outputData, inputData, timeData, rows, filterStage, filterStages))
exit(1);
if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, outputData, rows, outputColumn[i]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (flags&FL_DIFCOLUMNS) {
long j;
for (j=0; j<rows; j++)
outputData[j] = inputData[j] - outputData[j];
if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, outputData, rows, difColumn[i]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
free(inputData);
}
free(timeData);
}
if (!SDDS_WritePage(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
free(outputData);
for(i=0;i<depenColumns;i++) {
free(depenColumn[i]);
if (flags&FL_NEWCOLUMNS)
free(outputColumn[i]);
if (flags&FL_DIFCOLUMNS)
free(difColumn[i]);
}
for(i=0;i<excludes;i++)
free(exclude[i]);
free(indepColumn);
if (flags&FL_NEWCOLUMNS)
free(outputColumn);
free(depenColumn);
if (flags&FL_DIFCOLUMNS)
free(difColumn);
free(exclude);
for(i=0;i<filterStages;i++) {
long j;
for(j=0;j<filterStage[i].filters;j++) {
switch (filterStage[i].filter[j].filterType) {
case SET_FILTERFILE :
free( ((FILE_FILTER*) (filterStage[i].filter[j].filter))->freqData);
free( ((FILE_FILTER*) (filterStage[i].filter[j].filter))->magData);
free( ((FILE_FILTER*) (filterStage[i].filter[j].filter))->imagData);
free( ((FILE_FILTER*) (filterStage[i].filter[j].filter))->realData);
break;
default :
break;
}
}
}
if (!SDDS_Terminate(&SDDSout) || !SDDS_Terminate(&SDDSin))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
return 0;
}
int main(int argc, char **argv)
{
double *xData, *yData, *xError, *yError, *derivative, *derivativeError, *derivativePosition;
char *input, *output, *xName, *xErrorName, **yName, **yErrorName, **yOutputName, **yOutputErrorName, *ptr;
char **yOutputUnits, **yExcludeName;
char *mainTemplate[3] = {NULL, NULL, NULL};
char *errorTemplate[3] = {NULL, NULL, NULL};
long i, iArg, yNames, yExcludeNames, rows;
int32_t interval;
SDDS_DATASET SDDSin, SDDSout;
SCANNED_ARG *scanned;
unsigned long flags, pipeFlags;
long SGLeft, SGRight, SGOrder, SGDerivOrder, intervalGiven, yErrorsSeen;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&scanned, argc, argv);
if (argc==1)
bomb(NULL, USAGE);
input = output = xName = xErrorName = NULL;
yName = yErrorName = yExcludeName = NULL;
derivative = derivativeError = derivativePosition = yError = yData = xData = xError = NULL;
yNames = yExcludeNames = 0;
pipeFlags = 0;
interval = 2;
SGOrder = -1;
SGDerivOrder = 1;
intervalGiven = 0;
yErrorsSeen = 0;
for (iArg=1; iArg<argc; iArg++) {
if (scanned[iArg].arg_type==OPTION) {
/* process options here */
switch (match_string(scanned[iArg].list[0], option, CLO_OPTIONS, 0)) {
case CLO_DIFFERENTIATE:
if (scanned[iArg].n_items!=2 && scanned[iArg].n_items!=3)
SDDS_Bomb("invalid -differentiate syntax");
yName = SDDS_Realloc(yName, sizeof(*yName)*(yNames+1));
yErrorName = SDDS_Realloc(yErrorName, sizeof(*yErrorName)*(yNames+1));
yName[yNames] = scanned[iArg].list[1];
if (scanned[iArg].n_items==3) {
yErrorsSeen = 1;
yErrorName[yNames] = scanned[iArg].list[2];
} else
yErrorName[yNames] = NULL;
yNames++;
break;
case CLO_EXCLUDE:
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -exclude syntax");
moveToStringArray(&yExcludeName, &yExcludeNames, scanned[iArg].list+1, scanned[iArg].n_items-1);
break;
case CLO_VERSUS:
if (xName)
SDDS_Bomb("give -versus only once");
if (scanned[iArg].n_items!=2)
SDDS_Bomb("invalid -versus syntax");
xName = scanned[iArg].list[1];
xErrorName = NULL;
break;
case CLO_MAINTEMPLATE:
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -mainTemplate syntax");
scanned[iArg].n_items--;
if (!scanItemList(&flags, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"name", SDDS_STRING, mainTemplate+0, 1, 0,
"description", SDDS_STRING, mainTemplate+1, 1, 0,
"symbol", SDDS_STRING, mainTemplate+2, 1, 0,
NULL))
SDDS_Bomb("invalid -mainTemplate syntax");
break;
case CLO_ERRORTEMPLATE:
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -errorTemplate syntax");
scanned[iArg].n_items--;
if (!scanItemList(&flags, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"name", SDDS_STRING, errorTemplate+0, 1, 0,
"description", SDDS_STRING, errorTemplate+1, 1, 0,
"symbol", SDDS_STRING, errorTemplate+2, 1, 0,
NULL))
SDDS_Bomb("invalid -errorTemplate syntax");
break;
case CLO_PIPE:
if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case CLO_INTERVAL:
if (scanned[iArg].n_items!=2 || sscanf(scanned[iArg].list[1], "%" SCNd32, &interval)!=1 ||
interval<=0)
SDDS_Bomb("invalid -interval syntax/value");
intervalGiven = 1;
break;
case CLO_SAVITZKYGOLAY:
if ((scanned[iArg].n_items!=4 && scanned[iArg].n_items!=5) ||
sscanf(scanned[iArg].list[1], "%ld", &SGLeft)!=1 ||
sscanf(scanned[iArg].list[2], "%ld", &SGRight)!=1 ||
sscanf(scanned[iArg].list[3], "%ld", &SGOrder)!=1 ||
(scanned[iArg].n_items==5 &&
sscanf(scanned[iArg].list[4], "%ld", &SGDerivOrder)!=1) ||
SGLeft<0 || SGRight<0 || (SGLeft+SGRight)<SGOrder ||
SGOrder<0 || SGDerivOrder<0)
SDDS_Bomb("invalid -SavitzkyGolay syntax/values");
break;
default:
fprintf(stderr, "invalid option seen: %s\n", scanned[iArg].list[0]);
exit(1);
break;
}
}
else {
if (!input)
input = scanned[iArg].list[0];
else if (!output)
output = scanned[iArg].list[0];
else
SDDS_Bomb("too many filenames");
}
}
if (intervalGiven && SGOrder>=0)
SDDS_Bomb("-interval and -SavitzkyGolay options are incompatible");
if (SGOrder>=0 && (xErrorName || yErrorsSeen))
SDDS_Bomb("Savitzky-Golay method does not support errors in data");
processFilenames("sddsderiv", &input, &output, pipeFlags, 0, NULL);
if (!yNames)
SDDS_Bomb("-differentiate option must be given at least once");
if (!checkErrorNames(yErrorName, yNames))
SDDS_Bomb("either all -differentiate quantities must have errors, or none");
if (!SDDS_InitializeInput(&SDDSin, input))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!(ptr=SDDS_FindColumn(&SDDSin, FIND_NUMERIC_TYPE, xName, NULL))) {
fprintf(stderr, "error: column %s doesn't exist\n", xName);
exit(1);
}
free(xName);
xName = ptr;
if (xErrorName) {
if (!(ptr=SDDS_FindColumn(&SDDSin, FIND_NUMERIC_TYPE, xErrorName, NULL))) {
fprintf(stderr, "error: column %s doesn't exist\n", xErrorName);
exit(1);
}
else {
free(xErrorName);
xErrorName = ptr;
}
}
if (!(yNames=expandColumnPairNames(&SDDSin, &yName, &yErrorName, yNames,
yExcludeName, yExcludeNames, FIND_NUMERIC_TYPE, 0))) {
fprintf(stderr, "error: no quantities to differentiate found in file\n");
exit(1);
}
setupOutputFile(&SDDSout, &SDDSin, output, &yOutputName, &yOutputErrorName, &yOutputUnits,
xName, xErrorName, yName, yErrorName, yNames, mainTemplate, errorTemplate,
interval, SGOrder>=0?SGDerivOrder:1);
while (SDDS_ReadPage(&SDDSin)>0) {
if ((rows = SDDS_CountRowsOfInterest(&SDDSin))<2)
SDDS_Bomb("Can't compute derivatives: too little data.");
derivative = SDDS_Realloc(derivative, sizeof(*derivative)*rows);
derivativeError = SDDS_Realloc(derivativeError, sizeof(*derivativeError)*rows);
derivativePosition = SDDS_Realloc(derivativePosition, sizeof(*derivativePosition)*rows);
if (!SDDS_StartPage(&SDDSout, rows) || !SDDS_CopyParameters(&SDDSout, &SDDSin))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
xError = NULL;
if (!(xData = SDDS_GetColumnInDoubles(&SDDSin, xName)) ||
(xErrorName && !(xError=SDDS_GetColumnInDoubles(&SDDSin, xErrorName))))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (i=0; i<yNames; i++) {
yError = NULL;
if (!(yData = SDDS_GetColumnInDoubles(&SDDSin, yName[i])) ||
(yErrorName && yErrorName[i] && !(yError=SDDS_GetColumnInDoubles(&SDDSin, yErrorName[i]))))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (SGOrder>=0)
takeSGDerivative(xData, yData, rows, derivative, derivativePosition,
SGLeft, SGRight, SGOrder, SGDerivOrder);
else
takeDerivative(xData, yData, yError, rows, derivative, derivativeError,
derivativePosition, interval);
if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, derivative, rows, yOutputName[i]) ||
(yOutputErrorName && yOutputErrorName[i] &&
!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, derivativeError, rows, yOutputErrorName[i])))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (yData) free(yData);
if (yError) free(yError);
yData = yError = NULL;
}
if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, derivativePosition, rows, xName) ||
(xErrorName && !SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, xError, rows, xErrorName)))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!SDDS_WritePage(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (xData) free(xData);
if (xError) free(xError);
xData = xError = NULL;
}
if (!SDDS_Terminate(&SDDSin) || !SDDS_Terminate(&SDDSout)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (derivative) free(derivative);
if (derivativeError) free(derivativeError);
if (derivativePosition) free(derivativePosition);
return(0);
}
int main(int argc, char **argv)
{
int iArg;
char **column, **excludeColumn, *withOnly;
long columns, excludeColumns;
char *input, *output;
SCANNED_ARG *scanned;
SDDS_DATASET SDDSin, SDDSout;
long i, j, row, rows, count, readCode, rankOrder, iName1, iName2;
int32_t outlierStDevPasses;
double **data, correlation, significance, outlierStDevLimit;
double **rank;
short **accept;
char s[SDDS_MAXLINE];
unsigned long pipeFlags, dummyFlags, majorOrderFlag;
short columnMajorOrder=-1;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&scanned, argc, argv);
if (argc<2)
bomb(NULL, USAGE);
output = input = withOnly = NULL;
columns = excludeColumns = 0;
column = excludeColumn = NULL;
pipeFlags = 0;
rankOrder = 0;
outlierStDevPasses = 0;
outlierStDevLimit = 1.;
rank = NULL;
accept = NULL;
for (iArg=1; iArg<argc; iArg++) {
if (scanned[iArg].arg_type==OPTION) {
/* process options here */
switch (match_string(scanned[iArg].list[0], option, N_OPTIONS, 0)) {
case SET_MAJOR_ORDER:
majorOrderFlag=0;
scanned[iArg].n_items--;
if (scanned[iArg].n_items>0 &&
(!scanItemList(&majorOrderFlag, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"row", -1, NULL, 0, SDDS_ROW_MAJOR_ORDER,
"column", -1, NULL, 0, SDDS_COLUMN_MAJOR_ORDER,
NULL)))
SDDS_Bomb("invalid -majorOrder syntax/values");
if (majorOrderFlag&SDDS_COLUMN_MAJOR_ORDER)
columnMajorOrder=1;
else if (majorOrderFlag&SDDS_ROW_MAJOR_ORDER)
columnMajorOrder=0;
break;
case SET_COLUMNS:
if (columns)
SDDS_Bomb("only one -columns option may be given");
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -columns syntax");
column = tmalloc(sizeof(*column)*(columns=scanned[iArg].n_items-1));
for (i=0; i<columns; i++)
column[i] = scanned[iArg].list[i+1];
break;
case SET_EXCLUDE:
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -excludeColumns syntax");
moveToStringArray(&excludeColumn, &excludeColumns, scanned[iArg].list+1, scanned[iArg].n_items-1);
break;
case SET_WITHONLY:
if (withOnly)
SDDS_Bomb("only one -withOnly option may be given");
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -withOnly syntax");
withOnly = scanned[iArg].list[1];
break;
case SET_PIPE:
if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case SET_RANKORDER:
rankOrder = 1;
break;
case SET_STDEVOUTLIER:
scanned[iArg].n_items--;
outlierStDevPasses = 1;
outlierStDevLimit = 1;
if (!scanItemList(&dummyFlags, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"limit", SDDS_DOUBLE, &outlierStDevLimit, 1, 0,
"passes", SDDS_LONG, &outlierStDevPasses, 1, 0,
NULL) ||
outlierStDevPasses<=0 || outlierStDevLimit<=0)
SDDS_Bomb("invalid -stdevOutlier syntax/values");
break;
default:
fprintf(stderr, "error: unknown/ambiguous option: %s\n",
scanned[iArg].list[0]);
exit(1);
break;
}
}
else {
if (!input)
input = scanned[iArg].list[0];
else if (!output)
output = scanned[iArg].list[0];
else
SDDS_Bomb("too many filenames seen");
}
}
processFilenames("sddscorrelate", &input, &output, pipeFlags, 0, NULL);
if (!SDDS_InitializeInput(&SDDSin, input))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!columns)
columns = appendToStringArray(&column, columns, "*");
if (withOnly)
columns = appendToStringArray(&column, columns, withOnly);
if ((columns=expandColumnPairNames(&SDDSin, &column, NULL, columns,
excludeColumn, excludeColumns, FIND_NUMERIC_TYPE, 0))<=0) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
SDDS_Bomb("no columns selected for correlation analysis");
}
if (!SDDS_InitializeOutput(&SDDSout, SDDS_BINARY, 0, NULL, "sddscorrelate output", output) ||
SDDS_DefineColumn(&SDDSout, "Correlate1Name", NULL, NULL, "Name of correlated quantity 1", NULL, SDDS_STRING, 0)<0 ||
SDDS_DefineColumn(&SDDSout, "Correlate2Name", NULL, NULL, "Name of correlated quantity 2", NULL, SDDS_STRING, 0)<0 ||
SDDS_DefineColumn(&SDDSout, "CorrelatePair", NULL, NULL, "Names of correlated quantities", NULL, SDDS_STRING, 0)<0 ||
SDDS_DefineColumn(&SDDSout, "CorrelationCoefficient", "r", NULL, "Linear correlation coefficient",
NULL, SDDS_DOUBLE, 0)<0 ||
SDDS_DefineColumn(&SDDSout, "CorrelationSignificance", "P$br$n", NULL, "Linear correlation coefficient significance",
NULL, SDDS_DOUBLE, 0)<0 ||
SDDS_DefineColumn(&SDDSout, "CorrelationPoints", NULL, NULL, "Number of points used for correlation",
NULL, SDDS_LONG, 0)<0 ||
SDDS_DefineParameter(&SDDSout, "CorrelatedRows", NULL, NULL, "Number of data rows in correlation analysis",
NULL, SDDS_LONG, NULL)<0 ||
SDDS_DefineParameter(&SDDSout, "sddscorrelateInputFile", NULL, NULL, "Data file processed by sddscorrelate",
NULL, SDDS_STRING, input?input:"stdin")<0 ||
SDDS_DefineParameter(&SDDSout, "sddscorrelateMode", NULL, NULL, NULL, NULL, SDDS_STRING,
rankOrder?"Rank-Order (Spearman)":"Linear (Pearson)")<0 ||
SDDS_DefineParameter1(&SDDSout, "sddscorrelateStDevOutlierPasses", NULL, NULL,
"Number of passes of standard-deviation outlier elimination applied",
NULL, SDDS_LONG, &outlierStDevPasses)<0 ||
SDDS_DefineParameter1(&SDDSout, "sddscorrelateStDevOutlierLimit", NULL, NULL,
"Standard-deviation outlier limit applied",
NULL, SDDS_DOUBLE, &outlierStDevLimit)<0)
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (columnMajorOrder!=-1)
SDDSout.layout.data_mode.column_major = columnMajorOrder;
else
SDDSout.layout.data_mode.column_major = SDDSin.layout.data_mode.column_major;
if (!SDDS_WriteLayout(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!(data = (double**)malloc(sizeof(*data)*columns)) ||
(rankOrder && !(rank = (double**)malloc(sizeof(*rank)*columns))) ||
!(accept = (short**)malloc(sizeof(*accept)*columns)))
SDDS_Bomb("allocation failure");
while ((readCode=SDDS_ReadPage(&SDDSin))>0) {
if ((rows = SDDS_CountRowsOfInterest(&SDDSin))<3)
continue;
if (!SDDS_StartPage(&SDDSout, columns*(columns-1)/2) ||
!SDDS_SetParameters(&SDDSout, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE,
"CorrelatedRows", rows, NULL))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (i=0; i<columns; i++) {
if (!(data[i] = SDDS_GetColumnInDoubles(&SDDSin, column[i])))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (rankOrder)
rank[i] = findRank(data[i], rows);
if (outlierStDevPasses) {
if (!(accept[i] = (short*)malloc(sizeof(**accept)*rows)))
SDDS_Bomb("allocation failure");
markStDevOutliers(data[i], outlierStDevLimit, outlierStDevPasses,
accept[i], rows);
}
else
accept[i] = NULL;
}
for (i=row=0; i<columns; i++) {
for (j=i+1; j<columns; j++) {
iName1 = i;
iName2 = j;
if (withOnly) {
if (strcmp(withOnly, column[i])==0) {
iName1 = j;
iName2 = i;
}
else if (strcmp(withOnly, column[j])==0) {
iName1 = i;
iName2 = j;
}
else
continue;
}
correlation
= linearCorrelationCoefficient(rankOrder?rank[i]:data[i],
rankOrder?rank[j]:data[j],
accept[i], accept[j], rows, &count);
significance = linearCorrelationSignificance(correlation, count);
sprintf(s, "%s.%s", column[iName1], column[iName2]);
if (!SDDS_SetRowValues(&SDDSout, SDDS_SET_BY_INDEX|SDDS_PASS_BY_VALUE, row++,
0, column[iName1],
1, column[iName2],
2, s, 3, correlation, 4, significance,
5, count, -1))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
for (i=0; i<columns; i++) {
free(data[i]);
if (rankOrder)
free(rank[i]);
if (accept[i])
free(accept[i]);
}
if (!SDDS_WritePage(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_Terminate(&SDDSin)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (!SDDS_Terminate(&SDDSout)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
return 0;
}
int main(int argc, char **argv)
{
char *indepQuantity, **depenQuantity, **exclude, **depenQuantityPair;
long depenQuantities, excludes;
char *input, *output;
long iArg, i, j, rows, readCode, noWarnings, items;
long rowsToUse;
unsigned long flags, pairFlags, tmpFlags, pipeFlags;
SCANNED_ARG *scArg;
SDDS_DATASET SDDSin, SDDSout;
double *tdata, *data, t0, dt;
double fracRMSChangeLimit, fracFreqAccuracyLimit;
int32_t frequenciesDesired, maxFrequencies, freqCycleLimit;
short truncate;
double *frequency, *amplitude=NULL, *phase=NULL, *significance=NULL, *phase1=NULL, *amplitude1=NULL, *significance1=NULL;
long frequenciesFound;
long *frequencyIndex, *amplitudeIndex, *phaseIndex, *significanceIndex, pairs;
long *amplitudeIndex1, *phaseIndex1, *significanceIndex1;
SDDS_RegisterProgramName(argv[0]);
#ifdef DEBUG
if (1) {
long code;
double x, y;
x = 1.1;
code = OneDFunctionOptimize(&y, &x, 0.07, -4, 4,
trialFn, 50, 1e-6, 0, 1);
fprintf(stderr, "code: %ld x=%e, y=%e\n", code, x, y);
x = .9;
code = OneDFunctionOptimize(&y, &x, 0.15, -4, 4,
trialFn, 50, 1e-6, 0, 1);
fprintf(stderr, "code: %ld x=%e, y=%e\n", code, x, y);
x = .999;
code = OneDFunctionOptimize(&y, &x, 0.11, -4, 4,
trialFn, 50, 1e-6, 0, 1);
fprintf(stderr, "code: %ld x=%e, y=%e\n", code, x, y);
exit(0);
}
#endif
argc = scanargs(&scArg, argc, argv);
if (argc<3) {
fprintf(stderr, "%s%s", USAGE1, USAGE2);
exit(1);
}
output = input = NULL;
flags = pipeFlags = excludes = truncate = pairFlags= 0;
indepQuantity = NULL;
depenQuantity = exclude = depenQuantityPair=NULL;
depenQuantities = 0;
noWarnings = 0;
fracRMSChangeLimit = 0.0;
fracFreqAccuracyLimit = 0.00001;
frequenciesDesired = 1;
maxFrequencies = 4;
freqCycleLimit = 100;
pairs=0;
for (iArg=1; iArg<argc; iArg++) {
if (scArg[iArg].arg_type==OPTION) {
/* process options here */
switch (match_string(scArg[iArg].list[0], option, N_OPTIONS, 0)) {
case SET_TRUNCATE:
truncate = 1;
break;
case SET_PAIR:
if (depenQuantities)
SDDS_Bomb("Invalid -pair option, the depent-quantity is provided by -column option already.");
if (scArg[iArg].n_items!=3)
SDDS_Bomb("invalid -pair syntax");
depenQuantity = SDDS_Realloc(depenQuantity, sizeof(*depenQuantity)*(pairs+1));
depenQuantityPair = SDDS_Realloc(depenQuantityPair, sizeof(*depenQuantityPair)*(pairs+1));
depenQuantity[pairs] = scArg[iArg].list[1];
depenQuantityPair[pairs] = scArg[iArg].list[2];
pairs++;
break;
case SET_COLUMN:
if (indepQuantity)
SDDS_Bomb("only one -column option may be given");
if (scArg[iArg].n_items<2)
SDDS_Bomb("invalid -column syntax");
indepQuantity = scArg[iArg].list[1];
if (scArg[iArg].n_items>=2) {
if (pairs)
SDDS_Bomb("Invalid -column syntax, the depent-quantity is provided by -pair option already.");
depenQuantity = tmalloc(sizeof(*depenQuantity)*(depenQuantities=scArg[iArg].n_items-2));
for (i=0; i<depenQuantities; i++)
depenQuantity[i] = scArg[iArg].list[i+2];
}
break;
case SET_PIPE:
if (!processPipeOption(scArg[iArg].list+1, scArg[iArg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case SET_EXCLUDE:
if (scArg[iArg].n_items<2)
SDDS_Bomb("invalid -exclude syntax");
moveToStringArray(&exclude, &excludes, scArg[iArg].list+1, scArg[iArg].n_items-1);
break;
case SET_NOWARNINGS:
noWarnings = 1;
break;
case SET_TERM_SEARCH:
items = scArg[iArg].n_items-1;
flags &= ~(NAFF_RMS_CHANGE_LIMIT|NAFF_FREQS_DESIRED|NAFF_MAX_FREQUENCIES);
fracRMSChangeLimit = 0;
frequenciesDesired = 0;
maxFrequencies = 10;
if (!scanItemList(&tmpFlags, scArg[iArg].list+1, &items, 0,
"changelimit", SDDS_DOUBLE, &fracRMSChangeLimit, 1,
NAFF_RMS_CHANGE_LIMIT,
"maxfrequencies", SDDS_LONG, &maxFrequencies, 1,
NAFF_MAX_FREQUENCIES,
"frequencies", SDDS_LONG, &frequenciesDesired, 1,
NAFF_FREQS_DESIRED,
NULL) ||
(tmpFlags&NAFF_RMS_CHANGE_LIMIT && tmpFlags&NAFF_FREQS_DESIRED) ||
maxFrequencies<1)
SDDS_Bomb("invalid -terminateSearch syntax");
flags |= tmpFlags;
if (frequenciesDesired)
maxFrequencies = frequenciesDesired;
break;
case SET_ITERATE_FREQ:
items = scArg[iArg].n_items - 1;
flags &= ~(NAFF_FREQ_CYCLE_LIMIT|NAFF_FREQ_ACCURACY_LIMIT);
if (!scanItemList(&tmpFlags, scArg[iArg].list+1, &items, 0,
"cyclelimit", SDDS_LONG, &freqCycleLimit, 1, NAFF_FREQ_CYCLE_LIMIT,
"accuracylimit", SDDS_DOUBLE, &fracFreqAccuracyLimit,
1, NAFF_FREQ_ACCURACY_LIMIT,
NULL) || !bitsSet(tmpFlags) || freqCycleLimit<2)
SDDS_Bomb("invalid -iterateFrequency syntax");
flags |= tmpFlags;
break;
default:
fprintf(stderr, "error: unknown/ambiguous option: %s\n",
scArg[iArg].list[0]);
exit(1);
break;
}
}
else {
if (!input)
input = scArg[iArg].list[0];
else if (!output)
output = scArg[iArg].list[0];
else
SDDS_Bomb("too many filenames seen");
}
}
processFilenames("sddsnaff", &input, &output, pipeFlags, 0, NULL);
if (!indepQuantity)
SDDS_Bomb("supply the independent quantity name with the -columns option");
if (!SDDS_InitializeInput(&SDDSin, input))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (SDDS_CheckColumn(&SDDSin, indepQuantity, NULL, SDDS_ANY_NUMERIC_TYPE, stderr)
!=SDDS_CHECK_OKAY)
exit(1);
excludes = appendToStringArray(&exclude, excludes, indepQuantity);
if (pairs) {
pairFlags = flags | NAFF_FREQ_FOUND;
depenQuantities = pairs;
}
if (!depenQuantities)
depenQuantities = appendToStringArray(&depenQuantity, depenQuantities, "*");
if (!pairs) {
if ((depenQuantities=expandColumnPairNames(&SDDSin, &depenQuantity, NULL,
depenQuantities, exclude, excludes,
FIND_NUMERIC_TYPE, 0))<=0) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
SDDS_Bomb("No quantities selected to fft");
}
}
if (!SetupNAFFOutput(&SDDSout, output, &SDDSin, indepQuantity,
depenQuantities, depenQuantity,
&frequencyIndex, &litudeIndex, &phaseIndex,
&significanceIndex, depenQuantityPair,&litudeIndex1,
&phaseIndex1,&significanceIndex1))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!(frequency = SDDS_Malloc(sizeof(*frequency)*maxFrequencies)) ||
!(amplitude = SDDS_Malloc(sizeof(*amplitude)*maxFrequencies)) ||
!(phase = SDDS_Malloc(sizeof(*phase )*maxFrequencies)) ||
!(significance= SDDS_Malloc(sizeof(*significance)*maxFrequencies)))
SDDS_Bomb("memory allocation failure");
if (pairs) {
if (!(amplitude1 = SDDS_Malloc(sizeof(*amplitude1)*maxFrequencies)) ||
!(phase1 = SDDS_Malloc(sizeof(*phase1 )*maxFrequencies)) ||
!(significance1 = SDDS_Malloc(sizeof(*significance1)*maxFrequencies)))
SDDS_Bomb("memory allocation failure");
}
while ((readCode=SDDS_ReadPage(&SDDSin))>0) {
if ((rows = SDDS_CountRowsOfInterest(&SDDSin))<0)
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (rows) {
long primeRows;
rowsToUse = rows;
primeRows = greatestProductOfSmallPrimes(rows);
if (rows!=primeRows) {
if (truncate)
rowsToUse = greatestProductOfSmallPrimes(rows);
else if (largest_prime_factor(rows)>100 && !noWarnings)
fputs("Warning: number of points has large prime factors.\nThis could take a very long time.\nConsider using the -truncate option.\n", stderr);
}
if (!SDDS_StartPage(&SDDSout, maxFrequencies) ||
!SDDS_CopyParameters(&SDDSout, &SDDSin))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!(tdata = SDDS_GetColumnInDoubles(&SDDSin, indepQuantity)))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (i=1; i<rowsToUse; i++)
if (tdata[i]<=tdata[i-1])
SDDS_Bomb("independent data is not monotonically increasing");
dt = (tdata[rowsToUse-1]-tdata[0])/(rowsToUse-1.0),
t0 = tdata[0];
free(tdata);
for (i=0; i<depenQuantities; i++) {
if (!(data = SDDS_GetColumnInDoubles(&SDDSin, depenQuantity[i])))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (j=0; j<maxFrequencies; j++)
frequency[j] = amplitude[j] = phase[j] = significance[j] = -1;
frequenciesFound = PerformNAFF(frequency, amplitude, phase, significance,
t0, dt,
data, rowsToUse, flags,
fracRMSChangeLimit, maxFrequencies,
freqCycleLimit, fracFreqAccuracyLimit, 0, 0);
#ifdef DEBUG
fprintf(stderr, "Column %s: ", depenQuantity[i]);
fprintf(stderr, "f=%10.3e a=%10.3e p=%10.3e s=%10.3e\n",
frequency[0], amplitude[0], phase[0], significance[0]);
#endif
free(data);
data = NULL;
if (pairs) {
if (!(data = SDDS_GetColumnInDoubles(&SDDSin, depenQuantityPair[i])))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
frequenciesFound = PerformNAFF(frequency, amplitude1, phase1, significance1,
t0, dt,
data, rowsToUse, pairFlags,
fracRMSChangeLimit, maxFrequencies,
freqCycleLimit, fracFreqAccuracyLimit, 0, 0);
for (j=0; j<maxFrequencies; j++)
if (frequency[j]!=-1)
frequency[j] = adjustFrequencyHalfPlane(frequency[j], phase[j], phase1[j], dt);
free(data);
data = NULL;
}
if (!SDDS_SetColumn(&SDDSout, SDDS_SET_BY_INDEX,
frequency, maxFrequencies, frequencyIndex[i]) ||
!SDDS_SetColumn(&SDDSout, SDDS_SET_BY_INDEX,
amplitude, maxFrequencies, amplitudeIndex[i]) ||
!SDDS_SetColumn(&SDDSout, SDDS_SET_BY_INDEX,
phase, maxFrequencies, phaseIndex[i]) ||
!SDDS_SetColumn(&SDDSout, SDDS_SET_BY_INDEX,
significance, maxFrequencies, significanceIndex[i]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (pairs) {
if (!SDDS_SetColumn(&SDDSout, SDDS_SET_BY_INDEX,
amplitude1, maxFrequencies, amplitudeIndex1[i]) ||
!SDDS_SetColumn(&SDDSout, SDDS_SET_BY_INDEX,
phase1, maxFrequencies, phaseIndex1[i]) ||
!SDDS_SetColumn(&SDDSout, SDDS_SET_BY_INDEX,
significance1, maxFrequencies, significanceIndex1[i]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
}
else {
if (!SDDS_StartPage(&SDDSout, 0) || !SDDS_CopyParameters(&SDDSout, &SDDSin))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_WritePage(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_Terminate(&SDDSin)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (!SDDS_Terminate(&SDDSout)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
free(frequency);
free(amplitude);
free(phase);
free(significance);
if (pairs) {
free(amplitude1);
free(phase1);
free(significance1);
free(amplitudeIndex1);
free(phaseIndex1);
free(significanceIndex1);
free(depenQuantityPair);
}
free(depenQuantity);
free(frequencyIndex);
free(amplitudeIndex);
free(phaseIndex);
free(significanceIndex);
return 0;
}
long expandColumnPairNames(SDDS_DATASET *SDDSin, char ***name, char ***errorName, long names, char **excludeName,
long excludeNames, long typeMode, long typeValue)
{
long i, j, names1, errorNames1, names2;
char **name1, **errorName1, **name2, **errorName2;
if (!names || !*name)
return 0;
name2 = errorName1 = errorName2 = NULL;
names2 = errorNames1 = 0;
for (i=0; i<names; i++) {
switch (typeMode) {
case FIND_ANY_TYPE:
case FIND_NUMERIC_TYPE:
case FIND_INTEGER_TYPE:
case FIND_FLOATING_TYPE:
names1 = SDDS_MatchColumns(SDDSin, excludeNames?NULL:&name1, SDDS_MATCH_STRING, typeMode,
(*name)[i], SDDS_0_PREVIOUS|SDDS_OR);
break;
case FIND_SPECIFIED_TYPE:
if (!SDDS_VALID_TYPE(typeValue))
SDDS_Bomb("invalid type value in expandColumnPairNames");
names1 = SDDS_MatchColumns(SDDSin, excludeNames?NULL:&name1, SDDS_MATCH_STRING, typeMode, typeValue,
(*name)[i], SDDS_0_PREVIOUS|SDDS_OR);
break;
default:
SDDS_Bomb("invalid typeMode in expandColumnPairNames");
exit(1);
break;
}
if (names1==0)
continue;
if (names1==-1) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
SDDS_Bomb("unable to perform column name match in expandColumnPairNames");
}
if (excludeNames) {
for (j=0; j<excludeNames; j++)
if (!(names1=SDDS_MatchColumns(SDDSin, (j==excludeNames-1?&name1:0),
SDDS_MATCH_STRING, FIND_ANY_TYPE,
excludeName[j], SDDS_NEGATE_MATCH|SDDS_AND)))
break;
}
if (!names1)
continue;
moveToStringArray(&name2, &names2, name1, names1);
if (errorName && *errorName && (*errorName)[i]) {
if (strstr((*errorName)[i], "%s")) {
if (!(errorName1 = (char**)malloc(sizeof(*errorName1)*names1)))
SDDS_Bomb("allocation failure in expandColumnPairNames");
errorNames1 = names1;
for (j=0; j<names1; j++) {
if (!(errorName1[j] = (char*)malloc(sizeof(**errorName1)*
strlen(name1[j])+strlen((*errorName)[i])+1)))
SDDS_Bomb("allocation failure in expandColumnPairNames");
replace_stringn(errorName1[j], (*errorName)[i], "%s", name1[j], 1);
if (SDDS_CheckColumn(SDDSin, errorName1[j], NULL, SDDS_ANY_NUMERIC_TYPE, stderr)!=SDDS_CHECK_OKAY)
exit(1);
}
}
else if (names1==1) {
errorNames1 = 1;
if (!(errorName1 = (char**)malloc(sizeof(*errorName1))))
SDDS_Bomb("allocation failure in expandColumnPairNames");
if (!SDDS_CopyString(errorName1, (*errorName)[i]))
return 0;
}
else
SDDS_Bomb("%s template must be employed with error names when primary name has wildcards");
names2 -= names1;
moveToStringArray(&errorName2, &names2, errorName1, errorNames1);
free(errorName1);
free(name1);
}
}
if (names2==0)
return 0;
SDDS_FreeStringArray(*name, names);
*name = name2;
if (errorName) {
if (*errorName)
SDDS_FreeStringArray(*errorName, names);
*errorName = errorName2;
}
return names2;
}