long setupOutputFile(SDDS_DATASET *outData, char *output, SDDS_DATASET *inData,
STAT_DEFINITION *stat, long stats)
{
long column;
char s[SDDS_MAXLINE];
if (!SDDS_InitializeCopy(outData, inData, output, "w"))
return 0;
for (column=0; column<stats; column++) {
if (!SDDS_TransferColumnDefinition(outData, inData, stat[column].sourceColumn[0],
stat[column].resultColumn)) {
sprintf(s, "Problem transferring definition of column %s to %s\n",
stat[column].sourceColumn[0], stat[column].resultColumn);
SDDS_SetError(s);
return 0;
}
if ((stat[column].resultIndex
= SDDS_GetColumnIndex(outData, stat[column].resultColumn))<0) {
sprintf(s, "Problem creating column %s", stat[column].resultColumn);
SDDS_SetError(s);
return 0;
}
if (stat[column].positionColumn) {
if (!SDDS_DefineSimpleColumn(outData, stat[column].positionColumn, NULL, SDDS_STRING)) {
sprintf(s, "Problem define column %s\n", stat[column].positionColumn);
SDDS_SetError(s);
return 0;
}
if ((stat[column].positionColumnIndex
= SDDS_GetColumnIndex(outData, stat[column].positionColumn))<0) {
sprintf(s, "Problem creating column %s", stat[column].positionColumn);
SDDS_SetError(s);
return 0;
}
}
if (!SDDS_ChangeColumnInformation
(outData, "description", "", SDDS_SET_BY_NAME, stat[column].resultColumn) ||
!SDDS_ChangeColumnInformation
(outData, "symbol", "", SDDS_SET_BY_NAME, stat[column].resultColumn) ||
!SDDS_ChangeColumnInformation
(outData, "type", "double", SDDS_SET_BY_NAME|SDDS_PASS_BY_STRING, stat[column].resultColumn)) {
sprintf(s, "Problem changing attributes of new column %s", stat[column].resultColumn);
SDDS_SetError(s);
return 0;
}
}
if (!SDDS_WriteLayout(outData))
return 0;
return 1;
}
int main(int argc, char **argv)
{
SDDS_DATASET SDDSnew, SDDSold;
long i, j, iArg;
SCANNED_ARG *scArg;
char *input, *output, *columnName;
long mode, matchCode, rows, rowsMinus1, tmpfile_used;
double gapAmount, *columnData, gapFactor;
char *matchPattern;
long matchPatternAfter = 0;
double changeAmount, changeBase;
long retval, newStart, rowLimit, breakNext;
int32_t dataType, overlap=0;
unsigned long flags, pipeFlags, changeFlags;
char **stringData;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&scArg, argc, argv);
if (argc<2) {
fprintf(stderr, "%s", USAGE);
return(1);
}
columnData = NULL;
stringData = NULL;
input = output = columnName = NULL;
mode = -1;
pipeFlags = flags = 0;
gapAmount = changeAmount = rowLimit = gapFactor = 0;
matchPattern = NULL;
for (iArg=1; iArg<argc; iArg++) {
if (scArg[iArg].arg_type==OPTION) {
switch (matchCode=match_string(scArg[iArg].list[0], option, N_OPTIONS, 0)) {
case SET_GAPIN:
if ((scArg[iArg].n_items-=2)<0 ||
!scanItemList(&flags, scArg[iArg].list+2, &scArg[iArg].n_items, 0,
"amount", SDDS_DOUBLE, &gapAmount, 1, GAPIN_AMOUNT,
"factor", SDDS_DOUBLE, &gapFactor, 1, GAPIN_FACTOR,
NULL) ||
(flags&GAPIN_AMOUNT && gapAmount<=0) ||
(flags&GAPIN_FACTOR && gapFactor<=0)) {
fprintf(stderr, "Error: invalid -gapin syntax/values\n");
return(1);
}
columnName = scArg[iArg].list[1];
mode = matchCode;
break;
case SET_INCREASEOF: case SET_DECREASEOF:
if (scArg[iArg].n_items!=2) {
fprintf(stderr, "Error: invalid option syntax---specify column-name with -increaseof and -decreaseof\n");
return(1);
}
columnName = scArg[iArg].list[1];
mode = matchCode;
break;
case SET_CHANGEOF:
if ((scArg[iArg].n_items-=2)<0 ||
!scanItemList(&changeFlags, scArg[iArg].list+2, &scArg[iArg].n_items, 0,
"amount", SDDS_DOUBLE, &changeAmount, 1, CHANGEOF_AMOUNT,
"base", SDDS_DOUBLE, &changeBase, 1, CHANGEOF_BASE,
NULL) ||
(changeFlags&CHANGEOF_AMOUNT && changeAmount<=0)) {
fprintf(stderr, "Error: invalid -changeof syntax/values\n");
return(1);
}
columnName = scArg[iArg].list[1];
mode = matchCode;
break;
case SET_ROWLIMIT:
if (scArg[iArg].n_items<2) {
fprintf(stderr, "Error: invalid -rowlimit syntax\n");
return(1);
}
if (sscanf(scArg[iArg].list[1], "%ld", &rowLimit)!=1 ||
rowLimit<=0) {
fprintf(stderr, "Error: invalid -rowlimit syntax\n");
return(1);
}
if (scArg[iArg].n_items>2) {
scArg[iArg].n_items-=2;
if (!scanItemList(&flags, scArg[iArg].list+2, &scArg[iArg].n_items, 0,
"overlap", SDDS_LONG, &overlap, NULL) ||
overlap<0) {
fprintf(stderr, "Error: invalid overlap given in -rowlimit syntax\n");
return(1);
}
}
mode = matchCode;
break;
case SET_PIPE:
if (!processPipeOption(scArg[iArg].list+1, scArg[iArg].n_items-1, &pipeFlags)) {
fprintf(stderr, "Error: invalid -pipe syntax\n");
return(1);
}
break;
case SET_MATCHTO:
if ((scArg[iArg].n_items!=3 && scArg[iArg].n_items!=4) ||
strlen(columnName=scArg[iArg].list[1])==0 ||
strlen(matchPattern=scArg[iArg].list[2])==0) {
fprintf(stderr, "Error: invalid -matchTo syntax\n");
return(1);
}
if (scArg[iArg].n_items==4) {
if (strncmp(scArg[iArg].list[3], "after", strlen(scArg[iArg].list[3]))==0)
matchPatternAfter = 1;
else {
fprintf(stderr, "Error: invalid -matchTo syntax\n");
return(1);
}
}
mode = matchCode;
break;
default:
fprintf(stderr, "Error: unknown switch: %s\n", scArg[iArg].list[0]);
fprintf(stderr, "%s", USAGE);
return(1);
}
}
else {
if (input==NULL)
input = scArg[iArg].list[0];
else if (output==NULL)
output = scArg[iArg].list[0];
else {
fprintf(stderr, "Error: too many filenames given\n");
return(1);
}
}
}
processFilenames("sddsbreak", &input, &output, pipeFlags, 0, &tmpfile_used);
if (mode==-1) {
fprintf(stderr, "Error: no break mode specified\n");
return(1);
}
if (!SDDS_InitializeInput(&SDDSold, input) ||
!SDDS_InitializeCopy(&SDDSnew, &SDDSold, output, "w")) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
SDDSnew.layout.data_mode.no_row_counts = 0;
if (!SDDS_WriteLayout(&SDDSnew)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
if (mode!=SET_ROWLIMIT) {
if (SDDS_GetColumnInformation(&SDDSold, "type", &dataType, SDDS_BY_NAME, columnName)!=SDDS_LONG) {
SDDS_SetError("problem getting type information on given column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
if (mode==SET_MATCHTO) {
if (!(dataType==SDDS_STRING)) {
fprintf(stderr, "Error: given column does not contain string data\n");
return(1);
}
} else if (!SDDS_NUMERIC_TYPE(dataType)) {
if (!(mode==SET_CHANGEOF && !(changeFlags&CHANGEOF_AMOUNT) && !(changeFlags&CHANGEOF_BASE))) {
fprintf(stderr, "Error: given column does not contain numeric data\n");
return(1);
}
}
}
while ((retval=SDDS_ReadPage(&SDDSold))>0) {
if ((rows = SDDS_CountRowsOfInterest(&SDDSold))<0) {
SDDS_SetError("Problem getting number of rows of tabular data");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
rowsMinus1 = rows-1;
if (!SDDS_StartPage(&SDDSnew, rows) ||
!SDDS_CopyParameters(&SDDSnew, &SDDSold) ||
!SDDS_CopyArrays(&SDDSnew, &SDDSold)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
if (rows==0) {
if (!SDDS_WritePage(&SDDSnew)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
continue;
}
switch (mode) {
case SET_GAPIN:
if (!(columnData=SDDS_GetColumnInDoubles(&SDDSold, columnName))) {
SDDS_SetError("unable to read specified column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
if (!gapAmount && rows>1) {
double *gap;
gap = tmalloc(sizeof(*gap)*rows);
for (i=1; i<rows; i++)
gap[i-1] = fabs(columnData[i]-columnData[i-1]);
if (!compute_average(&gapAmount, gap, rows-1)) {
fprintf(stderr, "Error: unable to determine default gap amount--couldn't find median gap\n");
return(1);
}
gapAmount *= (gapFactor?gapFactor:2);
free(gap);
}
newStart = 0;
for (i=1; i<=rows; i++) {
if (i!=rows && fabs(columnData[i]-columnData[i-1])<gapAmount)
continue;
if (!SDDS_SetRowFlags(&SDDSold, 0) ||
!SDDS_AssertRowFlags(&SDDSold, SDDS_INDEX_LIMITS, newStart, i-1, 1) ||
!SDDS_CopyRowsOfInterest(&SDDSnew, &SDDSold) || !SDDS_WritePage(&SDDSnew)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
newStart = i;
}
free(columnData);
break;
case SET_INCREASEOF:
if (!(columnData=SDDS_GetColumnInDoubles(&SDDSold, columnName))) {
SDDS_SetError("unable to read specified column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
newStart = 0;
for (i=1; i<=rows; i++) {
if (i!=rows && columnData[i]<=columnData[i-1])
continue;
if (!SDDS_SetRowFlags(&SDDSold, 0) ||
!SDDS_AssertRowFlags(&SDDSold, SDDS_INDEX_LIMITS, newStart, i-1, 1) ||
!SDDS_CopyRowsOfInterest(&SDDSnew, &SDDSold) || !SDDS_WritePage(&SDDSnew)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
newStart = i;
}
free(columnData);
break;
case SET_DECREASEOF:
if (!(columnData=SDDS_GetColumnInDoubles(&SDDSold, columnName))) {
SDDS_SetError("unable to read specified column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
newStart = 0;
for (i=1; i<=rows; i++) {
if (i!=rows && columnData[i]>=columnData[i-1])
continue;
if (!SDDS_SetRowFlags(&SDDSold, 0) ||
!SDDS_AssertRowFlags(&SDDSold, SDDS_INDEX_LIMITS, newStart, i-1, 1) ||
!SDDS_CopyRowsOfInterest(&SDDSnew, &SDDSold) || !SDDS_WritePage(&SDDSnew)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
newStart = i;
}
free(columnData);
break;
case SET_CHANGEOF:
if (dataType!=SDDS_STRING) {
if (!(columnData=SDDS_GetColumnInDoubles(&SDDSold, columnName))) {
SDDS_SetError("unable to read specified column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
} else {
if (!(stringData=SDDS_GetColumn(&SDDSold, columnName))) {
SDDS_SetError("unable to read specified column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
}
newStart = 0;
if (dataType==SDDS_STRING || !changeAmount) {
for (i=1; i<=rows; i++) {
if (i!=rows &&
((dataType==SDDS_STRING && strcmp(stringData[i], stringData[i-1])==0) ||
(dataType!=SDDS_STRING && columnData[i]==columnData[i-1])))
continue;
if (!SDDS_SetRowFlags(&SDDSold, 0) ||
!SDDS_AssertRowFlags(&SDDSold, SDDS_INDEX_LIMITS, newStart, i-1, 1) ||
!SDDS_CopyRowsOfInterest(&SDDSnew, &SDDSold) || !SDDS_WritePage(&SDDSnew)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
newStart = i;
}
}
else {
long region, lastRegion;
region = lastRegion = 0;
if (!(changeFlags&CHANGEOF_BASE) && rows>=1)
changeBase = columnData[0];
if (rows>1)
lastRegion = (columnData[0]-changeBase)/changeAmount;
#ifdef DEBUG
fprintf(stderr, "change base=%e, lastRegion=%ld\n", changeBase, lastRegion);
fprintf(stderr, "start value = %e\n", columnData[0]);
#endif
newStart = 0;
for (i=1; i<=rows; i++) {
if (i!=rows)
region = (columnData[i]-changeBase)/changeAmount;
if (i!=rows && region==lastRegion)
continue;
#ifdef DEBUG
fprintf(stderr, "split after %e, before %e, region = %d\n",
columnData[i-1], columnData[i], region);
#endif
if (!SDDS_SetRowFlags(&SDDSold, 0) ||
!SDDS_AssertRowFlags(&SDDSold, SDDS_INDEX_LIMITS, newStart, i-1, 1) ||
!SDDS_CopyRowsOfInterest(&SDDSnew, &SDDSold) || !SDDS_WritePage(&SDDSnew)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
newStart = i;
lastRegion = region;
#ifdef DEBUG
fprintf(stderr, "start value = %e\n", columnData[i]);
#endif
}
}
if (dataType!=SDDS_STRING)
free(columnData);
else
SDDS_FreeStringArray(stringData, rows);
break;
case SET_MATCHTO:
if (!(stringData=SDDS_GetColumn(&SDDSold, columnName))) {
SDDS_SetError("unable to read specified column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
newStart = 0;
breakNext = 0;
for (i=1; i<=rows; i++) {
if (i!=rows && !breakNext) {
if (wild_match(stringData[i], matchPattern)) {
if (matchPatternAfter) {
breakNext = 1;
continue;
}
} else
continue;
}
if (!SDDS_SetRowFlags(&SDDSold, 0) ||
!SDDS_AssertRowFlags(&SDDSold, SDDS_INDEX_LIMITS, newStart, i-1, 1) ||
!SDDS_CopyRowsOfInterest(&SDDSnew, &SDDSold) || !SDDS_WritePage(&SDDSnew)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
breakNext = 0;
newStart = i;
}
SDDS_FreeStringArray(stringData, rows);
break;
case SET_ROWLIMIT:
for (i=0; i<rows; i+=rowLimit-overlap) {
if ((j=i+rowLimit-1)>=rows)
j = rows-1;
if (!SDDS_SetRowFlags(&SDDSold, 0) ||
!SDDS_AssertRowFlags(&SDDSold, SDDS_INDEX_LIMITS, i, j, 1) ||
!SDDS_CopyRowsOfInterest(&SDDSnew, &SDDSold) || !SDDS_WritePage(&SDDSnew)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
if (j==rows-1)
break;
}
break;
default:
fprintf(stderr, "Error: unknown break mode code seen---this can't happen\n");
return(1);
}
}
if (retval==0) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
if (!SDDS_Terminate(&SDDSold)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
return(1);
}
if (tmpfile_used && !replaceFileAndBackUp(input, output)) {
return(1);
}
return(0);
}
int main(int argc, char **argv)
{
SDDS_DATASET inSet, outSet;
SCANNED_ARG *s_arg;
long i_arg, pageReturned, rows, row;
int32_t *rowFlag;
char *input, *output, *columnName, *par_thresholdName;
double *data;
unsigned long pipeFlags, flags;
double threshold, ezoneFraction, changeThreshold;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&s_arg, argc, argv);
if (argc<2 || argc>(2+N_OPTIONS))
bomb(NULL, USAGE);
flags = pipeFlags = 0;
input = output = NULL;
columnName = NULL;
ezoneFraction = changeThreshold = 0;
rowFlag = NULL;
par_thresholdName=NULL;
for (i_arg=1; i_arg<argc; i_arg++) {
if (s_arg[i_arg].arg_type==OPTION) {
switch (match_string(s_arg[i_arg].list[0], option, N_OPTIONS, 0)) {
case CLO_THRESHOLD:
if (s_arg[i_arg].n_items==2) {
if (s_arg[i_arg].list[1][0]=='@' ) {
SDDS_CopyString(&par_thresholdName, s_arg[i_arg].list[1]+1);
flags |= PAR_THRESHOLD;
} else {
if (sscanf(s_arg[i_arg].list[1], "%lf", &threshold)!=1)
SDDS_Bomb("incorrect -threshold syntax");
flags |= THRESHOLD;
}
} else
SDDS_Bomb("incorrect -threshold syntax");
break;
case CLO_FIVEPOINT:
flags |= FIVEPOINT;
break;
case CLO_CHANGETHRESHOLD:
if (s_arg[i_arg].n_items!=2 ||
sscanf(s_arg[i_arg].list[1], "%lf", &changeThreshold)!=1 || changeThreshold<=0)
SDDS_Bomb("incorrect -changeThreshold syntax or values");
flags |= CHANGETHRES;
break;
case CLO_PIPE:
if (!processPipeOption(s_arg[i_arg].list+1, s_arg[i_arg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case CLO_COLUMN:
if (s_arg[i_arg].n_items!=2)
SDDS_Bomb("invalid -column syntax");
columnName = s_arg[i_arg].list[1];
break;
case CLO_EXCLUSIONZONE:
if (s_arg[i_arg].n_items!=2 || sscanf(s_arg[i_arg].list[1], "%lf", &ezoneFraction)!=1 ||
ezoneFraction<=0)
SDDS_Bomb("invalid -exclusionZone syntax or value");
flags |= EZONEFRAC;
break;
default:
fprintf(stderr, "error: unknown/ambiguous option: %s\n", s_arg[i_arg].list[0]);
exit(1);
break;
}
}
else {
if (input==NULL)
input = s_arg[i_arg].list[0];
else if (output==NULL)
output = s_arg[i_arg].list[0];
else
SDDS_Bomb("too many filenames");
}
}
processFilenames("sddspeakfind", &input, &output, pipeFlags, 0, NULL);
if (!columnName)
SDDS_Bomb("-column option must be given");
if (!SDDS_InitializeInput(&inSet, input))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!SDDS_FindColumn(&inSet, FIND_NUMERIC_TYPE, columnName, NULL))
SDDS_Bomb("the given column is nonexistent or nonnumeric");
if (!SDDS_InitializeCopy(&outSet, &inSet, output, "w") || !SDDS_WriteLayout(&outSet))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
while ((pageReturned=SDDS_ReadPage(&inSet))>0) {
if (!SDDS_CopyPage(&outSet, &inSet)) {
SDDS_SetError("Problem copying data for output file");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if ((rows=SDDS_CountRowsOfInterest(&outSet))>1) {
if (!(data = SDDS_GetColumnInDoubles(&inSet, columnName)))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
rowFlag = SDDS_Realloc(rowFlag, sizeof(*rowFlag)*rows);
for (row=0; row<rows; row++)
rowFlag[row] = 0;
markPeaks(data, rowFlag, rows, flags&FIVEPOINT);
if (flags&PAR_THRESHOLD) {
if (!SDDS_GetParameter(&inSet,par_thresholdName,&threshold))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (flags&THRESHOLD || flags&PAR_THRESHOLD ) {
for (row=0; row<rows; row++)
if (rowFlag[row] && data[row]<threshold)
rowFlag[row] = 0;
}
if (flags&CHANGETHRES)
unmarkFlatPeaks(data, rowFlag, rows, changeThreshold, flags&FIVEPOINT);
if (flags&EZONEFRAC)
unmarkExcludedPeaks(data, rowFlag, rows, ezoneFraction);
if (!SDDS_AssertRowFlags(&outSet, SDDS_FLAG_ARRAY, rowFlag, rows))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
free(data);
}
if (!SDDS_WritePage(&outSet))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_Terminate(&inSet) || !SDDS_Terminate(&outSet)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
free_scanargs(&s_arg,argc);
if (par_thresholdName) free(par_thresholdName);
if (rowFlag) free(rowFlag);
return(0);
}
int main(int argc, char **argv)
{
SCANNED_ARG *scanned;
SDDS_TABLE inputPage, outputPage, residualPage;
char *inputfile, *outputfile;
char **column, **excludeColumn;
int32_t columns;
long excludeColumns;
char *indColumnName;
long verbose;
long iArg, i, j, ipage;
double *indVar, *indVarOrig;
char *indVarUnits;
char **intColumn, **slopeColumn, **slopeSigmaColumn, **interceptSigmaColumn;
char *Units,*slopeUnits;
double *depVar, *depVarOrig;
long order;
double *coef, *coefsigma, *weight, *diff, *diffOrig, chi;
long iCol, iRow;
long rows, rowsOrig;
double rmsResidual;
double slopeSigma, interceptSigma;
char **sigmaColumn, **chiSquaredColumn;
long *sigmaColumnExists;
long doSlopeSigma, generateSigma, doPreliminaryFit;
long validSigmas;
double sigmaSum, averageSigma;
long ascii;
char *residualFile;
unsigned long pipeFlags;
long tmpfile_used, noWarnings;
double xMin, xMax;
indVar = indVarOrig = depVar = depVarOrig = coef = coefsigma = weight = diff = NULL;
intColumn = slopeColumn = slopeSigmaColumn = interceptSigmaColumn = sigmaColumn = chiSquaredColumn = NULL;
slopeUnits = NULL;
sigmaColumnExists = NULL;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&scanned, argc, argv);
if (argc == 1)
bomb(NULL, USAGE);
inputfile = outputfile = NULL;
columns = excludeColumns = 0;
column = excludeColumn = NULL;
indColumnName = NULL;
verbose = 0;
doSlopeSigma = 0;
generateSigma = 0;
doPreliminaryFit = 0;
ascii = 0;
pipeFlags = 0;
tmpfile_used=0;
noWarnings=0;
residualFile = NULL;
xMin = xMax = 0;
for (iArg = 1; iArg<argc; iArg++) {
if (scanned[iArg].arg_type == OPTION) {
delete_chars(scanned[iArg].list[0], "_");
switch (match_string(scanned[iArg].list[0], commandline_option, COMMANDLINE_OPTIONS, UNIQUE_MATCH)) {
case CLO_INDEPENDENT_COLUMN:
if (!(indColumnName = scanned[iArg].list[1]))
SDDS_Bomb("no string given for option -independentVariable");
break;
case CLO_COLUMNS:
if (columns)
SDDS_Bomb("only one -columns option may be given");
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -columns syntax");
column = tmalloc(sizeof(*column)*(columns = scanned[iArg].n_items-1));
for (i = 0; i<columns; i++)
column[i] = scanned[iArg].list[i+1];
break;
case CLO_EXCLUDE:
if (excludeColumns)
SDDS_Bomb("only one -excludecolumns option may be given");
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -excludecolumns syntax");
excludeColumn = tmalloc(sizeof(*excludeColumn)*(excludeColumns = scanned[iArg].n_items-1));
for (i = 0; i<excludeColumns; i++)
excludeColumn[i] = scanned[iArg].list[i+1];
break;
case CLO_VERBOSE:
verbose = 1;
break;
case CLO_ASCII:
ascii = 1;
break;
case CLO_PIPE:
if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case CLO_SIGMA:
doSlopeSigma = 1;
if (scanned[iArg].n_items > 1 ) {
switch (match_string(scanned[iArg].list[1], sigma_option, SIGMA_OPTIONS, UNIQUE_MATCH)) {
case SIGMA_GENERATE:
generateSigma = 1;
break;
default:
SDDS_Bomb("unrecognized sigma option given");
break;
}
}
break;
case CLO_RESIDUAL:
if (!(residualFile=scanned[iArg].list[1])){
fprintf(stderr,"No file specified in -residual option.\n");
exit(1);
}
break;
case CLO_RANGE:
if (scanned[iArg].n_items!=3 ||
1!=sscanf(scanned[iArg].list[1], "%lf", &xMin) ||
1!=sscanf(scanned[iArg].list[2], "%lf", &xMax) ||
xMin>=xMax)
SDDS_Bomb("incorrect -range syntax");
break;
default:
SDDS_Bomb("unrecognized option given");
break;
}
}
else {
if (!inputfile)
inputfile = scanned[iArg].list[0];
else if (!outputfile)
outputfile = scanned[iArg].list[0];
else
SDDS_Bomb("too many filenames given");
}
}
if (residualFile && outputfile) {
if (!strcmp( residualFile, outputfile)) {
fprintf( stderr, "Residual file can't be the same as the output file.\n");
exit(1);
}
}
processFilenames("sddsslopes", &inputfile, &outputfile, pipeFlags, noWarnings, &tmpfile_used);
if (!indColumnName) {
fprintf( stderr, "independentVariable not given\n");
exit(1);
}
if (!excludeColumns) {
excludeColumn = defaultExcludedColumn;
excludeColumns = DEFAULT_EXCLUDED_COLUMNS;
}
if (verbose)
fprintf(stderr,"Reading file %s.\n",inputfile);
if ( !SDDS_InitializeInput( &inputPage, inputfile) )
SDDS_PrintErrors( stderr, SDDS_EXIT_PrintErrors|SDDS_VERBOSE_PrintErrors);
while (0 < (ipage=SDDS_ReadTable( &inputPage))) {
if (verbose) {
fprintf(stderr, "working on page %ld\n", ipage);
}
rows = SDDS_CountRowsOfInterest(&inputPage);
rowsOrig = rows;
/*************************************\
* make array of independent variable
\*************************************/
if (ipage==1) {
indVar = (double*) malloc( sizeof(*indVar) * rows);
}
else {
indVar = (double*) realloc( indVar, sizeof(*indVar) * rows);
}
if (ipage==1) {
if (!SDDS_FindColumn(&inputPage, FIND_NUMERIC_TYPE, indColumnName, NULL)){
fprintf(stderr,"Something wrong with column %s.\n", indColumnName);
SDDS_CheckColumn(&inputPage, indColumnName, NULL, SDDS_ANY_NUMERIC_TYPE, stderr);
exit(1);
}
}
/* filter out the specified range in independent variable */
if (xMin!=xMax) {
if (!(indVarOrig = SDDS_GetColumnInDoubles( &inputPage, indColumnName)))
SDDS_PrintErrors( stderr, SDDS_EXIT_PrintErrors|SDDS_VERBOSE_PrintErrors);
for (i=j=0; i<rowsOrig; i++) {
if (indVarOrig[i]<=xMax && indVarOrig[i]>=xMin) {
indVar[j] = indVarOrig[i];
j++;
}
}
rows = j;
}
else {
if (!(indVar = SDDS_GetColumnInDoubles( &inputPage, indColumnName)))
SDDS_PrintErrors( stderr, SDDS_EXIT_PrintErrors|SDDS_VERBOSE_PrintErrors);
}
if ( ipage == 1 ) {
if (!SDDS_GetColumnInformation(&inputPage, "units", &indVarUnits, SDDS_GET_BY_NAME, indColumnName))
SDDS_PrintErrors( stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!indVarUnits) {
indVarUnits = (char *) malloc(sizeof(*indVarUnits));
indVarUnits[0] = 0;
}
}
/************************************\
* initialize residual file
\************************************/
if (residualFile) {
if ( ipage == 1 ) {
if(!SDDS_InitializeOutput(&residualPage,ascii?SDDS_ASCII:SDDS_BINARY,1,
"Residual of 2-term fit",NULL,outputfile) ||
!SDDS_InitializeCopy(&residualPage, &inputPage, residualFile, "w") ||
!SDDS_WriteLayout(&residualPage) )
SDDS_PrintErrors(stderr,SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_CopyPage(&residualPage,&inputPage))
SDDS_PrintErrors(stderr,SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
/************************************\
* get columns of interest. use set_multicolumn_flags to simply
* return new values for array column.
\*************************************/
if (!set_multicolumn_flags(&inputPage, &column, &columns, excludeColumn, excludeColumns)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
/************************************\
* make column names for the output
\*************************************/
if (ipage==1) {
intColumn = (char**) malloc((sizeof(char*)*columns));
slopeColumn = (char**) malloc((sizeof(char*)*columns));
if (doSlopeSigma) {
slopeSigmaColumn = (char**) malloc((sizeof(char*)*columns));
interceptSigmaColumn = (char**) malloc((sizeof(char*)*columns));
chiSquaredColumn = (char**) malloc((sizeof(char*)*columns));
}
for (i=0; i<columns; i++) {
intColumn[i] = (char*) malloc((sizeof(char)*(strlen(column[i])+strlen("Intercept")+1)));
strcat(strcpy(intColumn[i], column[i]), "Intercept");
slopeColumn[i] = (char*) malloc((sizeof(char)*(strlen(column[i])+strlen("Slope")+1)));
strcat(strcpy(slopeColumn[i], column[i]), "Slope");
if (doSlopeSigma) {
slopeSigmaColumn[i] = (char*) malloc((sizeof(char)*(strlen(column[i])+strlen("SlopeSigma")+1)));
strcat(strcpy(slopeSigmaColumn[i], column[i]), "SlopeSigma");
interceptSigmaColumn[i] = (char*) malloc((sizeof(char)*(strlen(column[i])+strlen("InterceptSigma")+1)));
strcat(strcpy(interceptSigmaColumn[i], column[i]), "InterceptSigma");
chiSquaredColumn[i] = (char*) malloc((sizeof(char)*(strlen(column[i])+strlen("ChiSquared")+1)));
strcat(strcpy(chiSquaredColumn[i], column[i]), "ChiSquared");
}
}
}
/************************************\
* Write layout for output file
\*************************************/
if (ipage==1) {
if (verbose)
fprintf(stderr,"Opening file %s.\n",outputfile);
if(!SDDS_InitializeOutput(&outputPage,ascii?SDDS_ASCII:SDDS_BINARY,1,
"2-term fit",NULL,outputfile) ||
0>SDDS_DefineParameter(&outputPage, "InputFile", "InputFile", NULL,
"InputFile", NULL, SDDS_STRING, 0) ||
0>SDDS_DefineColumn(&outputPage, "IndependentVariable", NULL, NULL, NULL, NULL, SDDS_STRING,0) )
SDDS_PrintErrors(stderr,SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (iCol=0; iCol<columns; iCol++) {
if (!SDDS_GetColumnInformation(&inputPage, "units", &Units, SDDS_GET_BY_NAME, column[iCol]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!Units) {
Units = (char*) malloc(sizeof(*Units));
Units[0] = 0;
}
if (0>SDDS_DefineColumn(&outputPage, intColumn[iCol], NULL, Units, NULL, NULL, SDDS_DOUBLE,0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
/* units for slopes columns */
if (strlen(indVarUnits) && strlen(Units) ) {
slopeUnits = (char*)malloc(sizeof(*slopeUnits)*(strlen(Units)+strlen(indVarUnits)+2));
strcat( strcat( strcpy(slopeUnits, Units), "/"), indVarUnits);
}
if (strlen(indVarUnits) && !strlen(Units) ) {
slopeUnits = (char*)malloc(sizeof(*slopeUnits)*(strlen(indVarUnits)+2));
strcat( strcpy( slopeUnits, "1/"), indVarUnits);
}
if (!strlen(indVarUnits) && strlen(Units) ) {
slopeUnits = (char*)malloc(sizeof(*slopeUnits)*(strlen(Units)+2));
strcpy( slopeUnits, indVarUnits);
}
if (!strlen(indVarUnits) && !strlen(Units) ) {
slopeUnits = (char*)malloc(sizeof(*slopeUnits));
strcpy( slopeUnits, "");
}
if (0>SDDS_DefineColumn(&outputPage, slopeColumn[iCol], NULL, slopeUnits, NULL, NULL, SDDS_DOUBLE,0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (doSlopeSigma) {
if (0>SDDS_DefineColumn(&outputPage, interceptSigmaColumn[iCol], NULL, Units,
NULL, NULL, SDDS_DOUBLE,0) ||
0>SDDS_DefineColumn(&outputPage, slopeSigmaColumn[iCol], NULL, slopeUnits,
NULL, NULL, SDDS_DOUBLE,0) ||
0>SDDS_DefineColumn(&outputPage, chiSquaredColumn[iCol], NULL, NULL,
NULL, NULL, SDDS_DOUBLE,0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
free(slopeUnits);
}
if ( !SDDS_WriteLayout(&outputPage))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if ( !SDDS_StartTable(&outputPage,1) ||
!SDDS_SetParameters(&outputPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE,
"InputFile",inputfile?inputfile:"pipe",NULL) ||
!SDDS_SetRowValues(&outputPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE, 0,
"IndependentVariable", indColumnName, NULL) )
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
/* determine which included columns has a Sigma column defined in the input file */
if ( ipage == 1 ) {
sigmaColumn = (char **) malloc( sizeof(*sigmaColumn)*columns);
sigmaColumnExists = (long *) malloc(columns*sizeof(*sigmaColumnExists));
for (iCol=0; iCol<columns; iCol++) {
sigmaColumn[iCol] = (char *) malloc( sizeof(**sigmaColumn) *
(strlen(column[iCol]) + strlen("Sigma") + 1) );
strcat( strcpy(sigmaColumn[iCol], column[iCol]), "Sigma");
switch(SDDS_CheckColumn(&inputPage, sigmaColumn[iCol], NULL, SDDS_DOUBLE, NULL)) {
case SDDS_CHECK_WRONGUNITS:
case SDDS_CHECK_OKAY:
sigmaColumnExists[iCol] = 1;
break;
default:
/* try other possible spelling */
strcat( strcpy(sigmaColumn[iCol] ,"Sigma"), column[iCol]);
switch(SDDS_CheckColumn(&inputPage, sigmaColumn[iCol], NULL, SDDS_DOUBLE, NULL)) {
case SDDS_CHECK_WRONGUNITS:
case SDDS_CHECK_OKAY:
sigmaColumnExists[iCol] = 1;
break;
default:
sigmaColumnExists[iCol] = 0;
}
break;
}
}
}
if ( ipage == 1 ) {
weight = (double*)malloc(sizeof(*weight)*rows);
diff = (double*)malloc(sizeof(*diff)*rows);
order=1;
coef = (double*)malloc(sizeof(*coef)*(order+1));
coefsigma = (double*)malloc(sizeof(*coefsigma)*(order+1));
}
else {
weight = (double*)realloc( weight, sizeof(*weight)*rows);
diff = (double*)realloc( diff, sizeof(*diff)*rows);
order=1;
coef = (double*)realloc( coef, sizeof(*coef)*(order+1));
coefsigma = (double*)realloc( coefsigma, sizeof(*coefsigma)*(order+1));
}
if (ipage==1) {
depVar = (double*) malloc( sizeof(*depVar) * rows);
}
else {
depVar = (double*) realloc( depVar, sizeof(*depVar) * rows);
}
for (iCol=0; iCol<columns; iCol++) {
if (verbose)
fprintf(stderr,"Doing column %s.\n", column[iCol]);
/* filter out the specified range in independent variable */
if (xMin!=xMax) {
if (!(depVarOrig = (double*) SDDS_GetColumnInDoubles(&inputPage, column[iCol])))
SDDS_PrintErrors(stderr, SDDS_EXIT_PrintErrors|SDDS_VERBOSE_PrintErrors);
for (i=j=0; i<rowsOrig; i++) {
if ( xMin <= indVarOrig[i] && indVarOrig[i] <= xMax ) {
depVar[j] = depVarOrig[i];
j++;
}
}
}
else {
if (!(depVar=SDDS_GetColumnInDoubles(&inputPage, column[iCol])))
SDDS_PrintErrors(stderr, SDDS_EXIT_PrintErrors|SDDS_VERBOSE_PrintErrors);
}
/*********************
three possibilities:
1) don't do or write slope errors. (doSlopeSigma=0)
do one lsf call with all weights = 1
2) calculated slope errors from sigma columns in the input file.
(doSlopeSigma=1 && generateSigma=0 && sigmaColumnExists[iCol]=1 )
do one lsf call with weights from sigma columns
3) calculate slope errors from generated sigma from a preliminary fit.
(doSlopeSigma=1 && (generateSigma=1 || sigmaColumnExists[iCol]=NULL)
do preliminary fit to generate sigma
*********************/
for (iRow=0; iRow<rows; iRow++)
weight[iRow] = 1;
if (doSlopeSigma) {
/*********************
check validity of sigma column values
*********************/
if( !generateSigma && sigmaColumnExists[iCol]) {
if (verbose)
fprintf(stderr,"\tUsing column %s for sigma.\n",sigmaColumn[iCol]);
if(!(weight=SDDS_GetColumnInDoubles(&inputPage, sigmaColumn[iCol])))
SDDS_PrintErrors(stderr, SDDS_EXIT_PrintErrors|SDDS_VERBOSE_PrintErrors);
/* check for zero weight values which will give lsfn problems */
validSigmas = rows;
sigmaSum = 0;
for (iRow=0; iRow<rows; iRow++) {
sigmaSum += weight[iRow];
if(!weight[iRow]) {
validSigmas--;
/*
fprintf(stderr,"Warning: %s of row number %ld is zero. Using average sigma.\n",sigmaColumn[iCol],iRow); */
}
}
if (!validSigmas) {
fprintf(stderr,"Warning: All sigmas are zero.\n");
doPreliminaryFit = 1;
}
else if (validSigmas!=rows) {
/* fix some sigmas */
averageSigma = sigmaSum/ validSigmas;
fprintf(stderr, "Warning: replacing %ld invalid sigmas with average (%e)\n",
rows-validSigmas, averageSigma);
for (iRow=0; iRow<rows; iRow++) {
if(!weight[iRow]) {
weight[iRow] = averageSigma;
}
}
}
}
else {
doPreliminaryFit = 1;
}
}
if (doPreliminaryFit) {
if (verbose)
fprintf(stderr,"\tGenerating sigmas from rms residual of a preliminary fit.\n");
if (!(lsfn(indVar, depVar, weight, rows, order, coef, coefsigma, &chi, diff))){
fprintf(stderr,"Problem with call to lsfn\n.");
exit(1);
}
rmsResidual = 0;
/* calculate rms residual */
for (iRow=0; iRow<rows; iRow++) {
rmsResidual += sqr(diff[iRow]);
}
rmsResidual = sqrt(rmsResidual/(rows));
for (iRow=0; iRow<rows; iRow++) {
weight[iRow] = rmsResidual;
}
}
if (!(lsfn(indVar, depVar, weight, rows, order, coef, coefsigma, &chi, diff))) {
fprintf(stderr,"Problem with call to lsfn\n.");
exit(1);
}
if (!SDDS_SetRowValues(&outputPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE, 0,
intColumn[iCol], coef[0],
slopeColumn[iCol], coef[1], NULL))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (doSlopeSigma) {
interceptSigma = coefsigma[0];
slopeSigma = coefsigma[1];
if (!SDDS_SetRowValues(&outputPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE, 0,
chiSquaredColumn[iCol], chi,
interceptSigmaColumn[iCol], interceptSigma,
slopeSigmaColumn[iCol], slopeSigma, NULL))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (residualFile) {
if (xMin!=xMax) {
/* calculate the residuals for the whole column explicitly since there
are points outside the range of which the lsf call did not calculate
the difference. */
diffOrig = (double*) malloc( rowsOrig * sizeof(double) );
for (i=0; i<rowsOrig; i++) {
diffOrig[i] = depVarOrig[i] - coef[0] - coef[1] * indVarOrig[i];
}
if( !SDDS_SetColumnFromDoubles(&residualPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_REFERENCE,
diffOrig,rowsOrig,column[iCol]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
else {
if( !SDDS_SetColumnFromDoubles(&residualPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_REFERENCE,
diff,rows,column[iCol]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
}
if (residualFile) {
if (!SDDS_WriteTable(&residualPage))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_WriteTable(&outputPage))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (residualFile) {
if (!SDDS_Terminate(&residualPage))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if(!SDDS_Terminate(&inputPage) || !SDDS_Terminate(&outputPage))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (tmpfile_used && !replaceFileAndBackUp(inputfile, outputfile))
exit(1);
return(0);
}
int main(int argc, char **argv)
{
SCANNED_ARG *scanned;
SDDS_TABLE inputPage, *copiedPage, outputPage;
long copiedPages;
char *inputfile, *outputfile;
char **column, **excludeColumn=NULL;
int32_t columns;
long excludeColumns;
char *indParameterName;
char **copyColumn;
int32_t copyColumns;
long verbose;
long slopeErrors;
long iArg,i;
double *indVar;
char *indVarUnits;
char **intColumn,**slopeColumn,**slopeSigmaColumn;
char *Units,*slopeUnits;
double *depVar;
long order;
double *coef,*coefsigma,*weight,*diff,chi;
long iCol,iRow,iPage;
long rows;
double *slope, slope2, slopeAve, slopeSigma;
unsigned long pipeFlags, majorOrderFlag;
long tmpfile_used, noWarnings;
long generateIndex;
short columnMajorOrder=-1;
copiedPage = NULL;
slopeSigmaColumn = NULL;
slopeUnits = Units = indVarUnits = NULL;
rows = 0;
slope = NULL;
slope2 = 0;
coef = coefsigma = weight = diff = slope = NULL;
argc = scanargs(&scanned, argc, argv);
if (argc == 1)
bomb(NULL, USAGE);
inputfile = outputfile = NULL;
columns = excludeColumns = 0;
column = excludeColumn = NULL;
indParameterName = NULL;
verbose = 0;
slopeErrors = 0;
pipeFlags = 0;
tmpfile_used=0;
noWarnings=0;
for (iArg = 1; iArg<argc; iArg++) {
if (scanned[iArg].arg_type == OPTION) {
delete_chars(scanned[iArg].list[0], "_");
switch (match_string(scanned[iArg].list[0], commandline_option, COMMANDLINE_OPTIONS, UNIQUE_MATCH)) {
case CLO_MAJOR_ORDER:
majorOrderFlag=0;
scanned[iArg].n_items--;
if (scanned[iArg].n_items>0 &&
(!scanItemList(&majorOrderFlag, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"row", -1, NULL, 0, SDDS_ROW_MAJOR_ORDER,
"column", -1, NULL, 0, SDDS_COLUMN_MAJOR_ORDER,
NULL)))
SDDS_Bomb("invalid -majorOrder syntax/values");
if (majorOrderFlag&SDDS_COLUMN_MAJOR_ORDER)
columnMajorOrder=1;
else if (majorOrderFlag&SDDS_ROW_MAJOR_ORDER)
columnMajorOrder=0;
break;
case CLO_INDEPENDENT_PARAMETER:
if (!(indParameterName = scanned[iArg].list[1]))
SDDS_Bomb("no string given for option -independentVariable");
break;
case CLO_COLUMNS:
if (columns)
SDDS_Bomb("only one -columns option may be given");
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -columns syntax");
column = tmalloc(sizeof(*column)*(columns = scanned[iArg].n_items-1));
for (i = 0; i<columns; i++)
column[i] = scanned[iArg].list[i+1];
break;
case CLO_EXCLUDE:
if (excludeColumns)
SDDS_Bomb("only one -excludecolumns option may be given");
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -excludecolumns syntax");
excludeColumn = tmalloc(sizeof(*excludeColumn)*(excludeColumns = scanned[iArg].n_items-1));
for (i = 0; i<excludeColumns; i++)
excludeColumn[i] = scanned[iArg].list[i+1];
break;
case CLO_VERBOSE:
verbose = 1;
break;
case CLO_PIPE:
if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case CLO_SLOPE_ERRORS:
slopeErrors = 1;
break;
default:
SDDS_Bomb("unrecognized option given");
break;
}
}
else {
if (!inputfile)
inputfile = scanned[iArg].list[0];
else if (!outputfile)
outputfile = scanned[iArg].list[0];
else
SDDS_Bomb("too many filenames given");
}
}
processFilenames("sddsvslopes", &inputfile, &outputfile, pipeFlags, noWarnings, &tmpfile_used);
if (!indParameterName)
SDDS_Bomb("independentVariable not given");
if (!excludeColumns) {
excludeColumn = defaultExcludedColumn;
excludeColumns = DEFAULT_EXCLUDED_COLUMNS;
}
if (verbose)
fprintf(stderr,"Reading file %s.\n",inputfile);
SDDS_InitializeInput(&inputPage, inputfile);
copiedPages = 0;
while (SDDS_ReadTable(&inputPage)>0) {
if (!copiedPages) {
copiedPage = (SDDS_TABLE*)malloc(sizeof(SDDS_TABLE));
rows = SDDS_CountRowsOfInterest(&inputPage);
}
else {
copiedPage = (SDDS_TABLE*)realloc(copiedPage, (copiedPages+1)*sizeof(SDDS_TABLE));
}
if (!SDDS_InitializeCopy(&copiedPage[copiedPages], &inputPage, NULL, "m"))
SDDS_PrintErrors(stderr, SDDS_EXIT_PrintErrors|SDDS_VERBOSE_PrintErrors);
if (!SDDS_CopyTable(&copiedPage[copiedPages], &inputPage))
SDDS_PrintErrors(stderr, SDDS_EXIT_PrintErrors|SDDS_VERBOSE_PrintErrors);
copiedPages++;
}
if (copiedPages<2) {
fprintf(stderr,"Insufficient data (i.e. number of data pages) to fit a straight line.\n");
exit(1);
}
switch(SDDS_CheckColumn(&inputPage, "Rootname", NULL, SDDS_STRING, NULL)) {
case SDDS_CHECK_WRONGUNITS:
case SDDS_CHECK_OKAY:
break;
default:
fprintf(stderr,"Something wrong with column %s.\n", "Rootname");
exit(1);
}
switch(SDDS_CheckColumn(&inputPage, "Index", NULL, SDDS_LONG, NULL)) {
case SDDS_CHECK_WRONGUNITS:
case SDDS_CHECK_OKAY:
generateIndex = 0;
break;
case SDDS_CHECK_NONEXISTENT:
generateIndex = 1;
break;
default:
fprintf(stderr,"Something wrong with column %s.\n", "Rootname");
exit(1);
}
/****************\
* make array of independent variable
\**************/
indVar = (double*)malloc(sizeof(*indVar)*copiedPages);
switch(SDDS_CheckParameter(&inputPage, indParameterName, NULL, SDDS_DOUBLE, NULL)) {
case SDDS_CHECK_WRONGUNITS:
case SDDS_CHECK_OKAY:
break;
default:
fprintf(stderr,"Something wrong with parameter %s.\n", indParameterName);
exit(1);
}
for (iPage = 0; iPage<copiedPages; iPage++) {
if (!SDDS_GetParameter(&copiedPage[iPage],indParameterName,&indVar[iPage]))
SDDS_PrintErrors(stderr, SDDS_EXIT_PrintErrors|SDDS_VERBOSE_PrintErrors);
}
if (!SDDS_GetParameterInformation(&inputPage, "units", &indVarUnits, SDDS_GET_BY_NAME, indParameterName))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!indVarUnits) {
indVarUnits = (char *) malloc(sizeof(*indVarUnits));
indVarUnits[0] = 0;
}
/************************************\
* get columns of interest. use set_multicolumn_flags to simply
* return new values for array column.
\*************************************/
if (!set_multicolumn_flags(&inputPage, &column, &columns, excludeColumn, excludeColumns)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
/************************************\
* make column names for the output
\*************************************/
intColumn = (char**)malloc((sizeof(char*)*columns));
slopeColumn = (char**)malloc((sizeof(char*)*columns));
if (slopeErrors)
slopeSigmaColumn = (char**)malloc((sizeof(char*)*columns));
for (i=0; i<columns; i++) {
intColumn[i] = (char*)malloc((sizeof(char)*(strlen(column[i])+strlen("Intercept")+1)));
strcat(strcpy(intColumn[i], column[i]), "Intercept");
slopeColumn[i] = (char*)malloc((sizeof(char)*(strlen(column[i])+strlen("Slope")+1)));
strcat(strcpy(slopeColumn[i], column[i]), "Slope");
if (slopeErrors) {
slopeSigmaColumn[i] = (char*)malloc((sizeof(char)*(strlen(column[i])+strlen("SlopeSigma")+1)));
strcat(strcpy(slopeSigmaColumn[i], column[i]), "SlopeSigma");
}
}
/************************************\
* Write layout for output file
\*************************************/
if (verbose)
fprintf(stderr,"Opening file %s.\n",outputfile);
if(!SDDS_InitializeOutput(&outputPage,SDDS_BINARY,1,
"lsf of sddsvexperiment",NULL,outputfile) ||
0>SDDS_DefineParameter(&outputPage, "InputFile", "InputFile", NULL,
"InputFile", NULL, SDDS_STRING, 0) ||
0>SDDS_DefineParameter(&outputPage, "IndependentVariable", "IndependentVariable", NULL,
"IndependentVariable", NULL, SDDS_STRING, 0) ||
(0>SDDS_DefineColumn(&outputPage,"Index",NULL,NULL,"Rootname index",NULL,SDDS_LONG,0))||
(0>SDDS_DefineColumn(&outputPage,"Rootname",NULL,NULL,NULL,NULL,SDDS_STRING,0)))
SDDS_PrintErrors(stderr,SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (columnMajorOrder!=-1)
outputPage.layout.data_mode.column_major = columnMajorOrder;
else
outputPage.layout.data_mode.column_major = inputPage.layout.data_mode.column_major;
for (iCol=0; iCol<columns; iCol++) {
if (!SDDS_GetColumnInformation(&inputPage, "units", &Units, SDDS_GET_BY_NAME,column[iCol]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!Units) {
Units = (char*) malloc(sizeof(*Units));
Units[0] = 0;
}
if (0>SDDS_DefineColumn(&outputPage, intColumn[iCol], NULL, Units, NULL, NULL, SDDS_DOUBLE,0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
/* units for slopes columns */
if (strlen(indVarUnits) && strlen(Units) ) {
slopeUnits = (char*)malloc(sizeof(*slopeUnits)*(strlen(Units)+strlen(indVarUnits)+2));
strcat( strcat( strcpy(slopeUnits, Units), "/"), indVarUnits);
}
if (strlen(indVarUnits) && !strlen(Units) ) {
slopeUnits = (char*)malloc(sizeof(*slopeUnits)*(strlen(indVarUnits)+2));
strcat( strcpy( slopeUnits, "1/"), indVarUnits);
}
if (!strlen(indVarUnits) && strlen(Units) ) {
slopeUnits = (char*)malloc(sizeof(*slopeUnits)*(strlen(Units)+2));
strcpy( slopeUnits, indVarUnits);
}
if (!strlen(indVarUnits) && !strlen(Units) ) {
slopeUnits = (char*)malloc(sizeof(*slopeUnits));
strcpy( slopeUnits, "");
}
if (0>SDDS_DefineColumn(&outputPage, slopeColumn[iCol], NULL, slopeUnits, NULL, NULL, SDDS_DOUBLE,0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (slopeErrors) {
if (0>SDDS_DefineColumn(&outputPage, slopeSigmaColumn[iCol], NULL, slopeUnits, NULL, NULL, SDDS_DOUBLE,0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
free(slopeUnits);
}
if(!SDDS_WriteLayout(&outputPage) ||
!SDDS_StartTable(&outputPage,rows) ||
!SDDS_SetParameters(&outputPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE,
"InputFile",inputfile?inputfile:"pipe",NULL) ||
!SDDS_SetParameters(&outputPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE, 0,
"IndependentVariable", indParameterName, NULL) )
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
/************************************\
* Copy columns to output file (usually columns Index and Rootname)
\*************************************/
copyColumns = DEFAULT_COPY_COLUMNS;
copyColumn = defaultCopyColumn;
if (!set_multicolumn_flags(&inputPage, ©Column, ©Columns, NULL, 0)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
exit(1);
}
if(!SDDS_CopyColumns(&outputPage, &inputPage))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
depVar = (double*)malloc(sizeof(*depVar)*copiedPages);
weight = (double*)malloc(sizeof(*weight)*copiedPages);
diff = (double*)malloc(sizeof(*weight)*copiedPages);
order=1;
coef = (double*)malloc(sizeof(*coef)*(order+1));
coefsigma = (double*)malloc(sizeof(*coefsigma)*(order+1));
if(slopeErrors)
slope = (double*)malloc(sizeof(*slope)*copiedPages);
for (iCol=0; iCol<columns; iCol++) {
for (iPage=0; iPage<copiedPages; iPage++)
weight[iPage]=1;
if (verbose)
fprintf(stderr,"Doing column %s.\n", column[iCol]);
for (iRow=0; iRow<rows; iRow++) {
for (iPage=0; iPage<copiedPages; iPage++) {
if (!SDDS_GetValue(&copiedPage[iPage], column[iCol], iRow, &depVar[iPage]))
SDDS_PrintErrors(stderr, SDDS_EXIT_PrintErrors|SDDS_VERBOSE_PrintErrors);
}
if (!(lsfn(indVar, depVar, weight, copiedPages, order, coef, coefsigma, &chi, diff))){
fprintf(stderr,"Problem with call to lsfn\n.");
exit(1);
}
if (generateIndex) {
if (!SDDS_SetRowValues(&outputPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE, iRow,
"Index",iRow,
intColumn[iCol], coef[0],
slopeColumn[iCol], coef[1], NULL))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
else {
if (!SDDS_SetRowValues(&outputPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE, iRow,
intColumn[iCol], coef[0],
slopeColumn[iCol], coef[1], NULL))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if(slopeErrors){
/* recalculate the slope with a subset of points */
slopeAve= slope2 = 0;
for (iPage=0; iPage<copiedPages; iPage++) {
weight[iPage] = 1e10;
if(iPage)
weight[iPage-1] = 1;
if (!(lsfn(indVar, depVar, weight, copiedPages, order, coef, coefsigma, &chi, diff))){
fprintf(stderr,"Problem with call to lsfn\n.");
exit(1);
}
slope[iPage] = coef[1];
slopeAve += slope[iPage];
slope2 += sqr(slope[iPage]);
}
slopeSigma = sqrt(slope2/copiedPages - sqr(slopeAve/copiedPages));
if (!SDDS_SetRowValues(&outputPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE, iRow,
slopeSigmaColumn[iCol], slopeSigma, NULL))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
}
if( !SDDS_WriteTable(&outputPage)|| SDDS_Terminate(&inputPage) )
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (iPage=0; iPage<copiedPages; iPage++) {
if( !SDDS_Terminate(&copiedPage[iPage]) )
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if( SDDS_Terminate(&outputPage) )
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (tmpfile_used && !replaceFileAndBackUp(inputfile, outputfile))
exit(1);
SDDS_FreeStringArray(column, columns);
free(column);
SDDS_FreeStringArray(intColumn, columns);
SDDS_FreeStringArray(slopeColumn, columns);
free(intColumn);
free(slopeColumn);
if (slopeErrors) {
SDDS_FreeStringArray(slopeSigmaColumn, columns);
free(slopeSigmaColumn);
}
free(copiedPage);
free(indVar);
free(depVar);
if (weight)
free(weight);
if (diff)
free(diff);
if (slope)
free(slope);
if (coef) free(coef);
if (coefsigma) free(coefsigma);
if (Units) free(Units);
if (indVarUnits) free(indVarUnits);
free_scanargs(&scanned, argc);
return(0);
}
int main(int argc, char **argv)
{
int iArg;
char **outputColumn, **difColumn;
char *indepColumn, **depenColumn, **exclude;
long depenColumns, excludes;
char *input, *output;
long i, rows, readCode, optionCode;
unsigned long flags, pipeFlags;
SCANNED_ARG *scanned;
SDDS_DATASET SDDSin, SDDSout;
double *timeData, *inputData, *outputData;
FILTER_STAGE *filterStage;
long filterStages, totalFilters;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&scanned, argc, argv);
if (argc<3 || argc>(3+N_OPTIONS))
bomb(NULL, USAGE);
output = input = NULL;
flags = pipeFlags = 0;
indepColumn = NULL;
depenColumn = exclude = NULL;
depenColumns = excludes = 0;
if (!(filterStage = (FILTER_STAGE*)calloc(1, sizeof(*filterStage))))
SDDS_Bomb("allocation failure");
filterStage->filter = NULL;
filterStage->filters = 0;
filterStages = 1;
totalFilters = 0;
for (iArg=1; iArg<argc; iArg++) {
if (scanned[iArg].arg_type==OPTION) {
/* process options here */
switch (optionCode=match_string(scanned[iArg].list[0], option, N_OPTIONS, 0)) {
case SET_PIPE:
if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case SET_COLUMNS:
if (indepColumn)
SDDS_Bomb("only one -columns option may be given");
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -columns syntax");
indepColumn = scanned[iArg].list[1];
if (scanned[iArg].n_items>=2) {
depenColumn = tmalloc(sizeof(*depenColumn)*(depenColumns=scanned[iArg].n_items-2));
for (i=0; i<depenColumns; i++)
depenColumn[i] = scanned[iArg].list[i+2];
}
break;
case SET_THRESHOLD:
case SET_HIGHPASS:
case SET_LOWPASS:
case SET_NOTCH:
case SET_BANDPASS:
case SET_FILTERFILE:
case SET_CLIPFREQ:
addFilter(filterStage+filterStages-1, optionCode, scanned+iArg);
totalFilters++;
break;
case SET_CASCADE:
if (filterStages==0)
SDDS_Bomb("-cascade option precedes all filter definitions");
if (!(filterStage = SDDS_Realloc(filterStage, (filterStages+1)*sizeof(*filterStage))))
SDDS_Bomb("allocation failure");
filterStage[filterStages].filter = NULL;
filterStage[filterStages].filters = 0;
filterStages++;
break;
case SET_NEWCOLUMNS:
flags |= FL_NEWCOLUMNS;
break;
case SET_DIFFERENCECOLUMNS:
flags |= FL_DIFCOLUMNS;
break;
case SET_EXCLUDE:
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -exclude syntax");
moveToStringArray(&exclude, &excludes, scanned[iArg].list+1, scanned[iArg].n_items-1);
break;
default:
fprintf(stderr, "error: unknown/ambiguous option: %s (%s)\n",
scanned[iArg].list[0], argv[0]);
exit(1);
break;
}
}
else {
if (!input)
input = scanned[iArg].list[0];
else if (!output)
output = scanned[iArg].list[0];
else
SDDS_Bomb("too many filenames seen");
}
}
processFilenames("sddsfdfilter", &input, &output, pipeFlags, 0, NULL);
if (!totalFilters)
fputs("warning: no filters specified (sddsfdfilter)\n", stderr);
if (!indepColumn)
SDDS_Bomb("supply the independent column name with the -columns option");
if (!SDDS_InitializeInput(&SDDSin, input))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (SDDS_CheckColumn(&SDDSin, indepColumn, NULL, SDDS_ANY_NUMERIC_TYPE, stderr)!=SDDS_CHECK_OKAY)
exit(1);
excludes = appendToStringArray(&exclude, excludes, indepColumn);
if (!depenColumns)
depenColumns = appendToStringArray(&depenColumn, depenColumns, "*");
if ((depenColumns=expandColumnPairNames(&SDDSin, &depenColumn, NULL, depenColumns, exclude, excludes,
FIND_NUMERIC_TYPE, 0))<=0) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
SDDS_Bomb("No quantities selected to filter");
}
if (!SDDS_InitializeCopy(&SDDSout, &SDDSin, output, "w"))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (flags&FL_NEWCOLUMNS) {
outputColumn = tmalloc(sizeof(*outputColumn)*depenColumns);
for (i=0; i<depenColumns; i++) {
outputColumn[i] = tmalloc(sizeof(**outputColumn)*(strlen(depenColumn[i])+1+strlen("Filtered")));
sprintf(outputColumn[i], "%sFiltered", depenColumn[i]);
if (!SDDS_TransferColumnDefinition(&SDDSout, &SDDSin, depenColumn[i], outputColumn[i]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
else
outputColumn=depenColumn;
difColumn = NULL;
if (flags&FL_DIFCOLUMNS) {
difColumn = tmalloc(sizeof(*difColumn)*depenColumns);
for (i=0; i<depenColumns; i++) {
difColumn[i] = tmalloc(sizeof(**difColumn)*(strlen(depenColumn[i])+1+strlen("Difference")));
sprintf(difColumn[i], "%sDifference", depenColumn[i]);
if (!SDDS_TransferColumnDefinition(&SDDSout, &SDDSin, depenColumn[i], difColumn[i]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
if (!SDDS_WriteLayout(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
outputData = NULL;
while ((readCode=SDDS_ReadPage(&SDDSin))>0) {
if (!SDDS_CopyPage(&SDDSout, &SDDSin))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if ((rows = SDDS_CountRowsOfInterest(&SDDSin))<0)
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (rows) {
if (!(timeData = SDDS_GetColumnInDoubles(&SDDSin, indepColumn)))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!(outputData = SDDS_Realloc(outputData, sizeof(*outputData)*rows)))
SDDS_Bomb("allocation failure");
for (i=0; i<depenColumns; i++) {
if (!(inputData = SDDS_GetColumnInDoubles(&SDDSin, depenColumn[i])))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!applyFilters(outputData, inputData, timeData, rows, filterStage, filterStages))
exit(1);
if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, outputData, rows, outputColumn[i]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (flags&FL_DIFCOLUMNS) {
long j;
for (j=0; j<rows; j++)
outputData[j] = inputData[j] - outputData[j];
if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, outputData, rows, difColumn[i]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
free(inputData);
}
free(timeData);
}
if (!SDDS_WritePage(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
free(outputData);
for(i=0;i<depenColumns;i++) {
free(depenColumn[i]);
if (flags&FL_NEWCOLUMNS)
free(outputColumn[i]);
if (flags&FL_DIFCOLUMNS)
free(difColumn[i]);
}
for(i=0;i<excludes;i++)
free(exclude[i]);
free(indepColumn);
if (flags&FL_NEWCOLUMNS)
free(outputColumn);
free(depenColumn);
if (flags&FL_DIFCOLUMNS)
free(difColumn);
free(exclude);
for(i=0;i<filterStages;i++) {
long j;
for(j=0;j<filterStage[i].filters;j++) {
switch (filterStage[i].filter[j].filterType) {
case SET_FILTERFILE :
free( ((FILE_FILTER*) (filterStage[i].filter[j].filter))->freqData);
free( ((FILE_FILTER*) (filterStage[i].filter[j].filter))->magData);
free( ((FILE_FILTER*) (filterStage[i].filter[j].filter))->imagData);
free( ((FILE_FILTER*) (filterStage[i].filter[j].filter))->realData);
break;
default :
break;
}
}
}
if (!SDDS_Terminate(&SDDSout) || !SDDS_Terminate(&SDDSin))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
return 0;
}
int main(int argc, char **argv)
{
POLYNOMIAL *poly;
long nPoly, iPoly, row, rows, iInput;
int iArg;
char *input, *output;
unsigned long pipeFlags;
SCANNED_ARG *scanned;
SDDS_DATASET SDDSin, SDDSout;
double *outputData ;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&scanned, argc, argv);
if (argc<3)
bomb(NULL, USAGE);
outputData = NULL;
input = output = NULL;
pipeFlags = 0;
poly = NULL;
nPoly = 0;
for (iArg=1; iArg<argc; iArg++) {
if (scanned[iArg].arg_type==OPTION) {
/* process options here */
switch (match_string(scanned[iArg].list[0], option, CLO_OPTIONS, 0)) {
case CLO_EVALUATE:
if (!(poly = SDDS_Realloc(poly, sizeof(*poly)*(nPoly+1))) ||
!(poly[nPoly].inputColumn
=SDDS_Malloc(sizeof(*(poly[nPoly].inputColumn))*5)) ||
!(poly[nPoly].powerColumn
=SDDS_Malloc(sizeof(*(poly[nPoly].powerColumn))*5)))
SDDS_Bomb("memory allocation failure");
scanned[iArg].n_items -= 1;
if (!scanItemList(&poly[nPoly].flags, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"filename", SDDS_STRING, &(poly[nPoly].filename), 1, POLY_FILE_SEEN,
"output", SDDS_STRING, &(poly[nPoly].outputColumn), 1, POLY_OUTPUT_SEEN,
"coefficients", SDDS_STRING, &(poly[nPoly].coefColumn), 1, POLY_COEF_SEEN,
"input0", SDDS_STRING, poly[nPoly].inputColumn+0, 1, POLY_IN0_SEEN,
"power0", SDDS_STRING, poly[nPoly].powerColumn+0, 1, POLY_OUT0_SEEN,
"input1", SDDS_STRING, poly[nPoly].inputColumn+1, 1, POLY_IN1_SEEN,
"power1", SDDS_STRING, poly[nPoly].powerColumn+1, 1, POLY_OUT1_SEEN,
"input2", SDDS_STRING, poly[nPoly].inputColumn+2, 1, POLY_IN2_SEEN,
"power2", SDDS_STRING, poly[nPoly].powerColumn+2, 1, POLY_OUT2_SEEN,
"input3", SDDS_STRING, poly[nPoly].inputColumn+3, 1, POLY_IN3_SEEN,
"power3", SDDS_STRING, poly[nPoly].powerColumn+3, 1, POLY_OUT3_SEEN,
"input4", SDDS_STRING, poly[nPoly].inputColumn+4, 1, POLY_IN4_SEEN,
"power4", SDDS_STRING, poly[nPoly].powerColumn+4, 1, POLY_OUT4_SEEN,
NULL) ||
!(poly[nPoly].flags&POLY_FILE_SEEN) || !(poly[nPoly].flags&POLY_OUTPUT_SEEN) ||
!(poly[nPoly].flags&POLY_COEF_SEEN) || !(poly[nPoly].flags&POLY_IN0_SEEN) ||
!(poly[nPoly].flags&POLY_OUT0_SEEN))
SDDS_Bomb("invalid -evaluate syntax");
nPoly++;
break;
case CLO_PIPE:
if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
default:
fprintf(stderr, "error: unknown/ambiguous option: %s\n",
scanned[iArg].list[0]);
exit(1);
break;
}
}
else {
if (!input)
input = scanned[iArg].list[0];
else if (!output)
output = scanned[iArg].list[0];
else
SDDS_Bomb("too many filenames seen");
}
}
processFilenames("sddspoly", &input, &output, pipeFlags, 0, NULL);
if (nPoly==0)
SDDS_Bomb("give at least one -evaluate option");
if (!SDDS_InitializeInput(&SDDSin, input))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!SDDS_InitializeCopy(&SDDSout, &SDDSin, output, "w"))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (iPoly=0; iPoly<nPoly; iPoly++)
initializePolynomial(&poly[iPoly], &SDDSin, &SDDSout);
if (!SDDS_WriteLayout(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
while (SDDS_ReadPage(&SDDSin)>0) {
rows = SDDS_CountRowsOfInterest(&SDDSin);
if (!SDDS_CopyPage(&SDDSout, &SDDSin))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!(outputData = SDDS_Realloc(outputData, sizeof(*outputData)*rows)))
SDDS_Bomb("memory allocation failure");
for (iPoly=0; iPoly<nPoly; iPoly++) {
for (iInput=0; iInput<poly[iPoly].nInputs; iInput++) {
if (!(poly[iPoly].inputData[iInput]=
SDDS_GetColumnInDoubles(&SDDSin, poly[iPoly].inputColumn[iInput])))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
for (row=0; row<rows; row++) {
for (iInput=0; iInput<poly[iPoly].nInputs; iInput++)
poly[iPoly].input[iInput] = poly[iPoly].inputData[iInput][row];
outputData[row] = evaluatePoly(poly[iPoly].coef, poly[iPoly].power,
poly[iPoly].nTerms,
poly[iPoly].input, poly[iPoly].nInputs);
}
if (!SDDS_SetColumn(&SDDSout, SDDS_SET_BY_NAME,
outputData, rows, poly[iPoly].outputColumn))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_WritePage(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
free(outputData);
if (!SDDS_Terminate(&SDDSin)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (!SDDS_Terminate(&SDDSout)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
free_scanargs(&scanned,argc);
FreePolynormialMemory(poly,nPoly);
free(poly);
return 0;
}
int main(int argc, char **argv)
{
SDDS_DATASET SDDSin, SDDSout;
long i_arg, tmpfileUsed;
SCANNED_ARG *s_arg;
char *input, *output, *description_text, *description_contents;
unsigned long pipeFlags;
long pageNumber, nonNative = 0;
char *outputEndianess = NULL;
char buffer[40];
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&s_arg, argc, argv);
if (argc<2)
bomb(NULL, USAGE);
input = output = description_text = description_contents = NULL;
pipeFlags = 0;
for (i_arg=1; i_arg<argc; i_arg++) {
if (s_arg[i_arg].arg_type==OPTION) {
delete_chars(s_arg[i_arg].list[0], "_");
switch (match_string(s_arg[i_arg].list[0], option, N_OPTIONS, 0)) {
case SET_PIPE:
if (!processPipeOption(s_arg[i_arg].list+1, s_arg[i_arg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case NONNATIVE:
nonNative = 1;
break;
default:
fprintf(stderr, "Error (%s): unknown switch: %s\n", argv[0], s_arg[i_arg].list[0]);
exit(1);
break;
}
}
else {
if (input==NULL)
input = s_arg[i_arg].list[0];
else if (output==NULL)
output = s_arg[i_arg].list[0];
else
SDDS_Bomb("too many filenames");
}
}
processFilenames("sddsendian", &input, &output, pipeFlags, 0, &tmpfileUsed);
outputEndianess = getenv("SDDS_OUTPUT_ENDIANESS");
if (outputEndianess) {
putenv("SDDS_OUTPUT_ENDIANESS=");
}
if (!SDDS_InitializeInput(&SDDSin, input)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (!description_text)
SDDS_GetDescription(&SDDSin, &description_text, &description_contents);
if (!SDDS_InitializeCopy(&SDDSout, &SDDSin, output, "w") ||
!SDDS_SetDataMode(&SDDSout, nonNative?SDDS_BINARY:-SDDS_BINARY) ||
!SDDS_WriteLayout(&SDDSout)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (nonNative) {
while ((pageNumber=SDDS_ReadNonNativePage(&SDDSin))>=0) {
if (!SDDS_CopyPage(&SDDSout, &SDDSin) ||
!SDDS_WritePage(&SDDSout)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
}
} else {
while ((pageNumber=SDDS_ReadPage(&SDDSin))>=0) {
if (!SDDS_CopyPage(&SDDSout, &SDDSin) ||
!SDDS_WriteNonNativeBinaryPage(&SDDSout)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
}
}
if (!SDDS_Terminate(&SDDSin) || !SDDS_Terminate(&SDDSout)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (tmpfileUsed && !replaceFileAndBackUp(input, output))
exit(1);
if (outputEndianess) {
sprintf(buffer, "SDDS_OUTPUT_ENDIANESS=%s", outputEndianess);
putenv(buffer);
}
return(0);
}
int main(int argc, char **argv)
{
SDDS_DATASET SDDS_1, SDDS_2, SDDS_output;
long i, j, i_arg, rows1, rows2, reuse, reusePage, i1, i2;
SCANNED_ARG *s_arg;
char s[200], *ptr;
char **match_column, **equate_column;
long match_columns, equate_columns;
char *input1, *input2, *output;
long tmpfile_used, retval1, retval2;
long warnings, invert;
unsigned long pipeFlags;
KEYED_EQUIVALENT **keyGroup=NULL;
long keyGroups=0;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&s_arg, argc, argv);
if (argc<3)
bomb(NULL, USAGE);
input1 = input2 = output = NULL;
match_column = equate_column = NULL;
match_columns = equate_columns = reuse = reusePage = 0;
tmpfile_used = invert = 0;
warnings = 1;
pipeFlags = 0;
for (i_arg=1; i_arg<argc; i_arg++) {
if (s_arg[i_arg].arg_type==OPTION) {
delete_chars(s_arg[i_arg].list[0], "_");
switch (match_string(s_arg[i_arg].list[0], option, N_OPTIONS, 0)) {
case SET_MATCH_COLUMN:
if (s_arg[i_arg].n_items!=2)
SDDS_Bomb("invalid -match syntax");
if (match_columns!=0)
SDDS_Bomb("only one -match option may be given");
match_column = tmalloc(sizeof(*match_column)*2);
if ((ptr=strchr(s_arg[i_arg].list[1], '=')))
*ptr++ = 0;
else
ptr = s_arg[i_arg].list[1];
match_column[0] = s_arg[i_arg].list[1];
match_column[1] = ptr;
match_columns = 1;
break;
case SET_EQUATE_COLUMN:
if (s_arg[i_arg].n_items!=2)
SDDS_Bomb("invalid -equate syntax");
if (equate_columns!=0)
SDDS_Bomb("only one -equate option may be given");
equate_column = tmalloc(sizeof(*equate_column)*2);
if ((ptr=strchr(s_arg[i_arg].list[1], '=')))
*ptr++ = 0;
else
ptr = s_arg[i_arg].list[1];
equate_column[0] = s_arg[i_arg].list[1];
equate_column[1] = ptr;
equate_columns = 1;
break;
case SET_REUSE:
if (s_arg[i_arg].n_items==1)
reuse = 1;
else {
char *reuseOptions[2] = {"rows", "page"};
for (i=1; i<s_arg[i_arg].n_items; i++) {
switch (match_string(s_arg[i_arg].list[i], reuseOptions, 2, 0)) {
case 0:
reuse = 1;
break;
case 1:
reusePage = 1;
break;
default:
SDDS_Bomb("unknown reuse keyword");
break;
}
}
}
break;
case SET_INVERT:
invert = 1;
break;
case SET_NOWARNINGS:
warnings = 0;
break;
case SET_PIPE:
if (!processPipeOption(s_arg[i_arg].list+1, s_arg[i_arg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
default:
fprintf(stderr, "error: unknown switch: %s\n", s_arg[i_arg].list[0]);
SDDS_Bomb(NULL);
break;
}
}
else {
if (input1==NULL)
input1 = s_arg[i_arg].list[0];
else if (input2==NULL)
input2 = s_arg[i_arg].list[0];
else if (output==NULL)
output = s_arg[i_arg].list[0];
else
SDDS_Bomb("too many filenames");
}
}
if (pipeFlags&USE_STDIN && input1) {
if (output)
SDDS_Bomb("too many filenames (sddsxref)");
output = input2;
input2 = input1;
input1 = NULL;
}
processFilenames("sddsselect", &input1, &output, pipeFlags, !warnings, &tmpfile_used);
if (!input2)
SDDS_Bomb("second input file not specified (sddsxref)");
if (equate_columns && match_columns)
SDDS_Bomb("only one of -equate or -match may be given");
if (!equate_columns && !match_columns)
SDDS_Bomb("one of -equate or -match must be given");
if (!SDDS_InitializeInput(&SDDS_1, input1)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (!SDDS_InitializeInput(&SDDS_2, input2)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (match_columns) {
if ((j=SDDS_GetColumnIndex(&SDDS_1, match_column[0]))<0 || SDDS_GetColumnType(&SDDS_1, j)!=SDDS_STRING) {
sprintf(s, "error: column %s not found or not string type in file %s", match_column[0], input1?input1:"stdin");
SDDS_SetError(s);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if ((j=SDDS_GetColumnIndex(&SDDS_2, match_column[1]))<0 || SDDS_GetColumnType(&SDDS_2, j)!=SDDS_STRING) {
sprintf(s, "error: column %s not found or not string type in file %s", match_column[1], input2);
SDDS_SetError(s);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
if (equate_columns) {
if ((j=SDDS_GetColumnIndex(&SDDS_1, equate_column[0]))<0 || !SDDS_NUMERIC_TYPE(SDDS_GetColumnType(&SDDS_1, j))) {
sprintf(s, "error: column %s not found or not numeric type in file %s", equate_column[0], input1?input1:"stdin");
SDDS_SetError(s);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if ((j=SDDS_GetColumnIndex(&SDDS_2, equate_column[1]))<0 || !SDDS_NUMERIC_TYPE(SDDS_GetColumnType(&SDDS_2, j))) {
sprintf(s, "error: column %s not found or not numeric type in file %s", equate_column[1], input2);
SDDS_SetError(s);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
if (output && pipeFlags&USE_STDOUT)
SDDS_Bomb("too many filenames with -pipe option");
if (!output && !(pipeFlags&USE_STDOUT)) {
if (warnings)
fprintf(stderr, "warning: existing file %s will be replaced (sddsselect)\n", input1?input1:"stdin");
tmpfile_used = 1;
cp_str(&output, tmpname(NULL));
}
if (!SDDS_InitializeCopy(&SDDS_output, &SDDS_1, output, "w")) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (!SDDS_WriteLayout(&SDDS_output))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
while ((retval1=SDDS_ReadPage(&SDDS_1))>0) {
if (!reusePage) {
if ((retval2=SDDS_ReadPage(&SDDS_2))<=0) {
if (warnings)
fprintf(stderr, "warning: <input2> ends before <input1>\n");
if (invert) {
/* nothing to match, so everything would normally be thrown out */
if (!SDDS_CopyPage(&SDDS_output, &SDDS_1) || !SDDS_WritePage(&SDDS_output))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
continue;
}
else
/* nothing to match, so everything thrown out */
break;
}
}
else {
if (retval1==1 && (retval2=SDDS_ReadPage(&SDDS_2))<=0)
SDDS_Bomb("<input2> has no data");
SDDS_SetRowFlags(&SDDS_2, 1);
}
rows1 = SDDS_CountRowsOfInterest(&SDDS_1);
rows2 = SDDS_CountRowsOfInterest(&SDDS_2);
if (!SDDS_StartPage(&SDDS_output, rows1)) {
SDDS_SetError("Problem starting output page");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_CopyParameters(&SDDS_output, &SDDS_2) || !SDDS_CopyArrays(&SDDS_output, &SDDS_2)) {
SDDS_SetError("Problem copying parameter or array data from second input file");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_CopyParameters(&SDDS_output, &SDDS_1) || !SDDS_CopyArrays(&SDDS_output, &SDDS_1)) {
SDDS_SetError("Problem copying parameter or array data from first input file");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (match_columns) {
char **string1, **string2;
long matched;
string2 = NULL;
if (!(string1 = SDDS_GetColumn(&SDDS_1, match_column[0]))) {
fprintf(stderr, "Error: problem getting column %s from file %s\n",
match_column[0], input1?input1:"stdin");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (rows2 && !(string2 = SDDS_GetColumn(&SDDS_2, match_column[1]))) {
fprintf(stderr, "Error: problem getting column %s from file %s\n",
match_column[1], input2);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (rows2)
keyGroup = MakeSortedKeyGroups(&keyGroups, SDDS_STRING, string2, rows2);
for (i1=0; i1<rows1; i1++) {
if (!SDDS_CopyRowDirect(&SDDS_output, i1, &SDDS_1, i1)) {
sprintf(s, "Problem copying row %ld of first data set", i1);
SDDS_SetError(s);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
matched = 0;
if (rows2 &&(i2 = FindMatchingKeyGroup(keyGroup, keyGroups, SDDS_STRING, string1+i1, reuse))>=0) {
matched = 1;
}
if ((!matched && !invert) || (matched && invert)) {
if (!SDDS_AssertRowFlags(&SDDS_output, SDDS_INDEX_LIMITS, i1, i1, 0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
if (string1) {
for (i=0;i<rows1;i++)
free(string1[i]);
free(string1);
string1 = NULL;
}
if (string2) {
for (i=0;i<rows2;i++)
free(string2[i]);
free(string2);
string2 = NULL;
}
for (i=0;i<keyGroups;i++) {
if (keyGroup[i]) {
if (keyGroup[i]->equivalent)
free(keyGroup[i]->equivalent);
free(keyGroup[i]);
keyGroup[i] = NULL;
}
}
if (keyGroups) {
free(keyGroup);
keyGroup = NULL;
keyGroups = 0;
}
}
else if (equate_columns) {
double *value1, *value2;
long equated;
value2 = NULL;
if (!(value1 = SDDS_GetColumnInDoubles(&SDDS_1, equate_column[0]))) {
fprintf(stderr, "Error: problem getting column %s from file %s\n",
equate_column[0], input1?input1:"stdin");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (rows2 && !(value2 = SDDS_GetColumnInDoubles(&SDDS_2, equate_column[1]))) {
fprintf(stderr, "Error: problem getting column %s from file %s\n",
equate_column[1], input2);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (rows2)
keyGroup = MakeSortedKeyGroups(&keyGroups, SDDS_DOUBLE, value2, rows2);
for (i1=0; i1<rows1; i1++) {
if (!SDDS_CopyRowDirect(&SDDS_output, i1, &SDDS_1, i1)) {
sprintf(s, "Problem copying row %ld of first data set", i1);
SDDS_SetError(s);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
equated = 0;
if (rows2 && (i2 = FindMatchingKeyGroup(keyGroup, keyGroups, SDDS_DOUBLE, value1+i1, reuse))>=0) {
equated = 1;
}
if ((!equated && !invert) || (equated && invert)) {
if (!SDDS_AssertRowFlags(&SDDS_output, SDDS_INDEX_LIMITS, i1, i1, 0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
if (value1)
free(value1);
value1 = NULL;
if (rows2 && value2)
free(value2);
value2 = NULL;
for (i=0;i<keyGroups;i++) {
if (keyGroup[i]) {
if (keyGroup[i]->equivalent)
free(keyGroup[i]->equivalent);
free(keyGroup[i]);
keyGroup[i] = NULL;
}
}
if (keyGroups) {
free(keyGroup);
keyGroup = NULL;
keyGroups = 0;
}
}
if (!SDDS_WritePage(&SDDS_output)) {
SDDS_SetError("Problem writing data to output file");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
if (!SDDS_Terminate(&SDDS_1) || !SDDS_Terminate(&SDDS_2) || !SDDS_Terminate(&SDDS_output)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (tmpfile_used && !replaceFileAndBackUp(input1, output))
exit(1);
free_scanargs(&s_arg,argc);
if (match_columns)
free(match_column);
return(0);
}
