Exemple #1
0
int main(int argc, char **argv)
{
  int iArg;
  char *indepQuantity, **depenQuantity, *fileValuesQuantity, *fileValuesFile, **exclude;
  long depenQuantities, monotonicity, excludes;
  char *input, *output;
  long i, j, rows, readCode, order, valuesReadCode, fillIn, row;
  long sequencePoints, combineDuplicates, branch;
  int32_t *rowFlag;
  double sequenceStart, sequenceEnd;
  double sequenceSpacing;
  unsigned long flags, interpCode, printFlags, forceMonotonic;
  SCANNED_ARG *scanned;
  SDDS_DATASET SDDSin, SDDSout, SDDSvalues;
  OUTRANGE_CONTROL aboveRange, belowRange;
  double *atValue;
  long atValues, interpPoints, doNotRead, parallelPages;
  double *indepValue, **depenValue, *interpPoint, **outputData;
  unsigned long pipeFlags;
  FILE *fpPrint;
  short interpShort=0, interpShortOrder=-1, *shortValue=NULL;
  long nextPos;

  SDDS_RegisterProgramName(argv[0]);
  argc = scanargs(&scanned, argc, argv); 
  if (argc<3 || argc>(3+CLO_OPTIONS)) 
    bomb(NULL, USAGE);

  atValue = NULL;
  atValues = fillIn = 0;
  output = input = NULL;
  combineDuplicates = branch = sequencePoints = parallelPages = 0;
  indepQuantity = NULL;
  depenQuantity = exclude = NULL;
  depenQuantities = excludes = 0;
  aboveRange.flags = belowRange.flags = OUTRANGE_SATURATE;
  order = 1;
  readCode = interpPoints = 0;
  fileValuesFile = fileValuesQuantity = NULL;
  sequenceStart = sequenceEnd = sequenceSpacing = 0;
  printFlags = pipeFlags = 0;
  forceMonotonic = 0;
  indepValue = interpPoint = NULL;
  depenValue = outputData = NULL;

  for (iArg=1; iArg<argc; iArg++) {
    if (scanned[iArg].arg_type==OPTION) {
      /* process options here */
      switch (match_string(scanned[iArg].list[0], option, CLO_OPTIONS, 0)) {
      case CLO_ORDER:
        if (scanned[iArg].n_items!=2 || sscanf(scanned[iArg].list[1], "%ld", &order)!=1 ||
            order<1)
          SDDS_Bomb("invalid -order syntax/value");
        break;
      case CLO_ATVALUES:
        if (scanned[iArg].n_items<2)
          SDDS_Bomb("invalid -atValues syntax");
        if (atValue)
          SDDS_Bomb("give -atValues only once");
        atValue = tmalloc(sizeof(*atValue)*(atValues=scanned[iArg].n_items-1));
        for (i=0; i<atValues; i++)
          if (sscanf(scanned[iArg].list[i+1], "%lf", &atValue[i])!=1)
            SDDS_Bomb("invalid -atValues value");
        break;
      case CLO_INTERP_SHORT:
        if (scanned[iArg].n_items==2) {
          if (sscanf(scanned[iArg].list[1], "%hd", &interpShortOrder)!=1)
            SDDS_Bomb("invalid -interpShort value");
        }
        interpShort = 1;
        break;
      case CLO_SEQUENCE:
        if ((scanned[iArg].n_items!=2 && scanned[iArg].n_items!=4) ||
            sscanf(scanned[iArg].list[1], "%ld", &sequencePoints)!=1 || sequencePoints<2)
          SDDS_Bomb("invalid -sequence syntax/value");
        if (scanned[iArg].n_items==4 &&
            (sscanf(scanned[iArg].list[2], "%lf", &sequenceStart)!=1 ||
             sscanf(scanned[iArg].list[3], "%lf", &sequenceEnd)!=1))
          SDDS_Bomb("invalid -sequence syntax/value");
        if (sequenceSpacing)
          SDDS_Bomb("give only one of -sequence and -equispaced");
        break;
      case CLO_EQUISPACED:
        if ((scanned[iArg].n_items!=2 && scanned[iArg].n_items!=4) ||
            sscanf(scanned[iArg].list[1], "%lf", &sequenceSpacing)!=1 || sequenceSpacing<=0)
          SDDS_Bomb("invalid -equispaced syntax/value");
        if (scanned[iArg].n_items==4 &&
            (sscanf(scanned[iArg].list[2], "%lf", &sequenceStart)!=1 ||
             sscanf(scanned[iArg].list[3], "%lf", &sequenceEnd)!=1))
          SDDS_Bomb("invalid -equispaced syntax/values");
        if (sequencePoints)
          SDDS_Bomb("give only one of -sequence and -equispaced");
        break;
      case CLO_COLUMNS:
        if (indepQuantity)
          SDDS_Bomb("only one -columns option may be given");
        if (scanned[iArg].n_items<2)
          SDDS_Bomb("invalid -columns syntax");
        indepQuantity = scanned[iArg].list[1];
        if (scanned[iArg].n_items>=2) {
          depenQuantity = tmalloc(sizeof(*depenQuantity)*(depenQuantities=scanned[iArg].n_items-2));
          for (i=0; i<depenQuantities; i++)
            depenQuantity[i] = scanned[iArg].list[i+2];
        }
        break;
      case CLO_PRINTOUT:
        if ((scanned[iArg].n_items-=1)>=1) {
          if (!scanItemList(&printFlags, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
                            "bare", -1, NULL, 0, BARE_PRINTOUT,
                            "stdout", -1, NULL, 0, STDOUT_PRINTOUT,
                            NULL)) 
            SDDS_Bomb("invalid -printout syntax");
        }
        if (!(printFlags&BARE_PRINTOUT))
          printFlags |= NORMAL_PRINTOUT;
        break;
      case CLO_FILEVALUES:
        if (scanned[iArg].n_items<2)
          SDDS_Bomb("invalid -fileValues syntax");
        fileValuesFile = scanned[iArg].list[1];
        scanned[iArg].n_items -= 2;
        if (!scanItemList(&flags, scanned[iArg].list+2, &scanned[iArg].n_items, 0,
                          "column", SDDS_STRING, &fileValuesQuantity, 1, 0,
                          "parallelpages", -1, NULL, 0, FILEVALUES_PARALLEL_PAGES,
                          NULL))
          SDDS_Bomb("invalid -fileValues syntax");
        if (flags&FILEVALUES_PARALLEL_PAGES)
          parallelPages = 1;
        break;
      case CLO_COMBINEDUPLICATES:
        SDDS_Bomb("-combineDuplicates option not implemented yet--send email to [email protected]");
        combineDuplicates = 1;
        break;
      case CLO_BRANCH:
        SDDS_Bomb("-branch option not implemented yet--send email to [email protected]");
        if (scanned[iArg].n_items!=2 || sscanf(scanned[iArg].list[1], "%ld", &branch)!=1 ||
            branch<1)
          SDDS_Bomb("invalid -branch syntax/value");
        break;
      case CLO_BELOWRANGE:
        if ((scanned[iArg].n_items-=1)<1 ||
            !scanItemList(&belowRange.flags, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
                          "value", SDDS_DOUBLE, &belowRange.value, 1, OUTRANGE_VALUE,
                          "skip", -1, NULL, 0, OUTRANGE_SKIP,
                          "saturate", -1, NULL, 0, OUTRANGE_SATURATE,
                          "extrapolate", -1, NULL, 0, OUTRANGE_EXTRAPOLATE,
                          "wrap", -1, NULL, 0, OUTRANGE_WRAP,
                          "abort", -1, NULL, 0, OUTRANGE_ABORT,
                          "warn", -1, NULL, 0, OUTRANGE_WARN,
                          NULL))
          SDDS_Bomb("invalid -belowRange syntax/value");
        if ((i=bitsSet(belowRange.flags&
                       (OUTRANGE_VALUE|OUTRANGE_SKIP|OUTRANGE_SATURATE|OUTRANGE_EXTRAPOLATE|OUTRANGE_WRAP|OUTRANGE_ABORT)))>1)
          SDDS_Bomb("incompatible keywords given for -belowRange");
        if (i!=1)
          belowRange.flags |= OUTRANGE_SATURATE;
        break;
      case CLO_ABOVERANGE:
        if ((scanned[iArg].n_items-=1)<1 ||
            !scanItemList(&aboveRange.flags, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
                          "value", SDDS_DOUBLE, &aboveRange.value, 1, OUTRANGE_VALUE,
                          "skip", -1, NULL, 0, OUTRANGE_SKIP,
                          "saturate", -1, NULL, 0, OUTRANGE_SATURATE,
                          "extrapolate", -1, NULL, 0, OUTRANGE_EXTRAPOLATE,
                          "wrap", -1, NULL, 0, OUTRANGE_WRAP,
                          "abort", -1, NULL, 0, OUTRANGE_ABORT,
                          "warn", -1, NULL, 0, OUTRANGE_WARN,
                          NULL))
          SDDS_Bomb("invalid -aboveRange syntax/value");
        if ((i=bitsSet(aboveRange.flags&
                       (OUTRANGE_VALUE|OUTRANGE_SKIP|OUTRANGE_SATURATE|OUTRANGE_EXTRAPOLATE|OUTRANGE_WRAP|OUTRANGE_ABORT)))>1)
          SDDS_Bomb("incompatible keywords given for -aboveRange");
        if (i!=1)
          aboveRange.flags |= OUTRANGE_SATURATE;
        break;
      case CLO_PIPE:
        if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags))
          SDDS_Bomb("invalid -pipe syntax");
        break;
      case CLO_EXCLUDE:
        if (scanned[iArg].n_items<2)
          SDDS_Bomb("invalid -exclude syntax");
        moveToStringArray(&exclude, &excludes, scanned[iArg].list+1, scanned[iArg].n_items-1);
        break;
      case CLO_FORCEMONOTONIC:
        if ((scanned[iArg].n_items-=1)>0) {
          if (!scanItemList(&forceMonotonic, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
                            "increasing", -1, NULL, 0, FORCE_INCREASING,
                            "decreasing", -1, NULL, 0, FORCE_DECREASING,
                            NULL) || bitsSet(forceMonotonic)!=1)
            SDDS_Bomb("invalid -forceMonotonic syntax/value");
        }
        else 
          forceMonotonic = FORCE_MONOTONIC;
        break;
      case CLO_FILLIN:
        fillIn = 1;
        break;
      default:
        fprintf(stderr, "error: unknown/ambiguous option: %s\n", 
                scanned[iArg].list[0]);
        exit(1);
        break;
      }
    }
    else {
      if (!input)
        input = scanned[iArg].list[0];
      else if (!output)
        output = scanned[iArg].list[0];
      else
        SDDS_Bomb("too many filenames seen");
    }
  }

  processFilenames("sddsinterp", &input, &output, pipeFlags, 0, NULL);

  fpPrint = stderr;
  if (printFlags&STDOUT_PRINTOUT)
    fpPrint = stdout;

  if (!indepQuantity)
    SDDS_Bomb("supply the independent quantity name with the -columns option");

  if ((atValues?1:0)+(fileValuesFile?1:0)+(sequencePoints?1:0)+fillIn+(sequenceSpacing>0?1:0) != 1)
    SDDS_Bomb("you must give one and only one of -atValues, -fileValues, -sequence, -equispaced, and -fillIn");

  if (!SDDS_InitializeInput(&SDDSin, input))
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);

  excludes = appendToStringArray(&exclude, excludes, indepQuantity);
  if (!depenQuantities)
    depenQuantities = appendToStringArray(&depenQuantity, depenQuantities, "*"); 

  if ((depenQuantities=expandColumnPairNames(&SDDSin, &depenQuantity, NULL, depenQuantities, 
                                             exclude, excludes, FIND_NUMERIC_TYPE, 0))<=0) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
    SDDS_Bomb("no dependent quantities selected for interpolation");
  }

  if (fileValuesFile && !SDDS_InitializeInput(&SDDSvalues, fileValuesFile))
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);

  if (!SDDS_InitializeOutput(&SDDSout, SDDS_BINARY, 0, NULL, "sddsinterp output", output))
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
  if (!SDDS_TransferColumnDefinition(&SDDSout, &SDDSin, indepQuantity, NULL))
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
  /*
    if (fileValuesQuantity && strcmp(fileValuesQuantity, indepQuantity)!=0 &&
    !SDDS_TransferColumnDefinition(&SDDSout, &SDDSvalues, fileValuesQuantity, NULL)) {
    fprintf(stderr, "problem creating -fileValues column %s\n", fileValuesQuantity);
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
    }
    */
  if (SDDS_DefineParameter(&SDDSout, "InterpDataPage", NULL, NULL, 
                           "Page of interpolation data file used to create this page",
                           NULL, SDDS_LONG, NULL)<0 ||
      SDDS_DefineParameter(&SDDSout, "InterpPointsPage", NULL, NULL, 
                           "Page of interpolation points file used to create this page",
                           NULL, SDDS_LONG, NULL)<0)
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
  for (i=0; i<depenQuantities; i++) 
    if (!SDDS_TransferColumnDefinition(&SDDSout, &SDDSin, depenQuantity[i], NULL)) {
      fprintf(stderr, "problem creating interpolated-output column %s\n", depenQuantity[i]);
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
    }
  if (!SDDS_TransferAllParameterDefinitions(&SDDSout, &SDDSin, SDDS_TRANSFER_KEEPOLD) ||
      !SDDS_WriteLayout(&SDDSout))
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);

  doNotRead = 0;
  interpPoint = NULL;
  outputData = tmalloc(sizeof(*outputData)*(depenQuantities));
  depenValue = tmalloc(sizeof(*depenValue)*(depenQuantities));
  rowFlag = NULL;
  valuesReadCode = 0;

  while (doNotRead || (readCode=SDDS_ReadPage(&SDDSin))>0) {
    rows = SDDS_CountRowsOfInterest(&SDDSin);
    if (!(indepValue = SDDS_GetColumnInDoubles(&SDDSin, indepQuantity)))
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
    if (atValues) {
      interpPoint = atValue;
      interpPoints = atValues;
    }
    else if (fileValuesFile) {
      if (interpPoint)
        free(interpPoint);
      if ((valuesReadCode=SDDS_ReadPage(&SDDSvalues))==0)
        SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
      else if (valuesReadCode==-1) {
        if (parallelPages) {
          fprintf(stderr, "warning: file %s ends before file %s\n", fileValuesFile, input);
          break;
        }
        else {
          /* "rewind" the values file */
          if (!SDDS_Terminate(&SDDSvalues) ||
              !SDDS_InitializeInput(&SDDSvalues, fileValuesFile))
            SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
          if ((valuesReadCode=SDDS_ReadPage(&SDDSvalues))<1) {
            fprintf(stderr, "error: unable to (re)read file %s\n", fileValuesFile);
            SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
            exit(1);
          }
          /* read the next page of the interpolation data file */
          if ((readCode=SDDS_ReadPage(&SDDSin))<1) {
            if (readCode==-1)
              break;
            SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
          }
          rows = SDDS_CountRowsOfInterest(&SDDSin);
          if (indepValue)
            free(indepValue);
          if (!(indepValue = SDDS_GetColumnInDoubles(&SDDSin, indepQuantity)))
            SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
        }
      }
      
      if (!parallelPages)
        doNotRead = 1;
      interpPoints = SDDS_CountRowsOfInterest(&SDDSvalues);
      interpPoint = SDDS_GetColumnInDoubles(&SDDSvalues, fileValuesQuantity);
      if (SDDS_NumberOfErrors())
        SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
    }
    else if (sequencePoints || sequenceSpacing) {
      if (interpPoint)
        free(interpPoint);
      interpPoints = sequencePoints;
      if (!(interpPoint = makeSequence(&interpPoints, sequenceStart, sequenceEnd, sequenceSpacing, 
                                       indepValue, rows)))
        exit(1);
    } else {
      /* fillIn interpolation */
      if (interpPoint)
        free(interpPoint);
      if (!(interpPoint = makeFillInSequence(indepValue, rows, &interpPoints)))
        exit(1);
    }
    
    for (i=0; i<depenQuantities; i++)
      outputData[i] = tmalloc(sizeof(*outputData[i])*interpPoints);
    rowFlag = trealloc(rowFlag, sizeof(*rowFlag)*interpPoints);
    for (j=0; j<interpPoints; j++)
      rowFlag[j] = 1;
    for (i=0; i<depenQuantities; i++) {
      if (!(depenValue[i] = SDDS_GetColumnInDoubles(&SDDSin, depenQuantity[i])))
        SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
    }
    if (forceMonotonic)
      rows = forceMonotonicity(indepValue, depenValue, depenQuantities, rows, forceMonotonic);
    else if (combineDuplicates)
      rows = combineDuplicatePoints(indepValue, depenValue, depenQuantities, rows, 0.0);
    if ((monotonicity=checkMonotonicity(indepValue, rows))==0)
      SDDS_Bomb("independent data values do not change monotonically or repeated independent values exist");
    if (interpShort)
      shortValue = malloc(sizeof(*shortValue)*rows);
    
    for (i=0; i<depenQuantities; i++) {
      if (interpShort) {
        for (row=0; row<rows; row++) {
          shortValue[row] = (short)depenValue[i][row];
        }
      }
      for (j=0; j<interpPoints; j++) {
        if (!interpShort) {
          outputData[i][j] = interpolate(depenValue[i], indepValue, rows, interpPoint[j], &belowRange,
                                         &aboveRange, order, &interpCode, monotonicity);
        } else {
          outputData[i][j] = (double)interp_short(shortValue, indepValue, rows, interpPoint[j], 0, -1,
                                                  &interpCode, &nextPos);
        }
        if (interpCode) {
          if (interpCode&OUTRANGE_ABORT) {
            fprintf(stderr, "error: value %e is out of range for column %s\n",
                    interpPoint[j], depenQuantity[i]);
            exit(1);
          }
          if (interpCode&OUTRANGE_WARN)
            fprintf(stderr, "warning: value %e is out of range for column %s\n",
                    interpPoint[j], depenQuantity[i]);
          if (interpCode&OUTRANGE_SKIP) 
            rowFlag[j] = 0;
        }
      }
    }
    if (interpShort)
      free(shortValue);
    if (!SDDS_StartPage(&SDDSout, interpPoints) || 
        !SDDS_SetColumnFromDoubles(&SDDSout, SDDS_SET_BY_NAME, interpPoint, interpPoints, indepQuantity))
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
    if (!SDDS_SetParameters(&SDDSout, SDDS_BY_NAME|SDDS_PASS_BY_VALUE,
                            "InterpDataPage", readCode,
                            "InterpPointsPage", valuesReadCode, NULL) ||
        !SDDS_CopyParameters(&SDDSout, &SDDSin))
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
    for (i=0; i<depenQuantities; i++)
      if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_SET_BY_NAME, outputData[i], interpPoints, depenQuantity[i]))
        SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
    if (!SDDS_AssertRowFlags(&SDDSout, SDDS_FLAG_ARRAY, rowFlag, rows) || !SDDS_WritePage(&SDDSout))
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
    if (printFlags&BARE_PRINTOUT) {
      for (j=0; j<interpPoints; j++)
        if (rowFlag[j]) {
          fprintf(fpPrint, "%21.15e ", interpPoint[j]);
          for (i=0; i<depenQuantities; i++)
            fprintf(fpPrint, "%21.15e ", outputData[i][j]);
          fputc('\n', fpPrint);
        }
    }
    else if (printFlags&NORMAL_PRINTOUT) {
      for (j=0; j<interpPoints; j++)
        if (rowFlag[j]) {
          fprintf(fpPrint, "%s=%21.15e ", indepQuantity, interpPoint[j]);
          for (i=0; i<depenQuantities; i++)
            fprintf(fpPrint, "%s=%21.15e ", depenQuantity[i], outputData[i][j]);
          fputc('\n', fpPrint);
        }
    }
    if (indepValue) free(indepValue);
    indepValue = NULL;
    for (i=0; i<depenQuantities; i++) {
      if (outputData[i]) free(outputData[i]);
      outputData[i] = NULL;
      if (depenValue[i]) free(depenValue[i]);
      depenValue[i] = NULL;
    }
    if (fileValuesFile) {
      if (interpPoint) free(interpPoint);
      interpPoint = NULL;
    }
    if (rowFlag) free(rowFlag);
    rowFlag = NULL;
  }

  if (!SDDS_Terminate(&SDDSin)) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    exit(1);
  }
  if (!SDDS_Terminate(&SDDSout)) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    exit(1);
  }
  if (fileValuesFile) {
    if (!SDDS_Terminate(&SDDSvalues)) {
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
      exit(1);
    }
  }

  return 0;
}
Exemple #2
0
int main(int argc, char **argv)
{
  int iArg;
  char **column, **excludeColumn, *withOnly;
  long columns, excludeColumns;
  char *input, *output;
  SCANNED_ARG *scanned;
  SDDS_DATASET SDDSin, SDDSout;
  long i, j, row, rows, count, readCode, rankOrder, iName1, iName2;
  int32_t outlierStDevPasses;
  double **data, correlation, significance, outlierStDevLimit;
  double **rank;
  short **accept;
  char s[SDDS_MAXLINE];
  unsigned long pipeFlags, dummyFlags, majorOrderFlag;
  short columnMajorOrder=-1;
  
  SDDS_RegisterProgramName(argv[0]);
  argc = scanargs(&scanned, argc, argv); 
  if (argc<2)
    bomb(NULL, USAGE);
  
  output = input = withOnly = NULL;
  columns = excludeColumns = 0;
  column = excludeColumn = NULL;
  pipeFlags = 0;
  rankOrder = 0;
  outlierStDevPasses = 0;
  outlierStDevLimit  = 1.;
  rank = NULL;
  accept = NULL;
  
  for (iArg=1; iArg<argc; iArg++) {
    if (scanned[iArg].arg_type==OPTION) {
      /* process options here */
      switch (match_string(scanned[iArg].list[0], option, N_OPTIONS, 0)) {
      case SET_MAJOR_ORDER:
        majorOrderFlag=0;
        scanned[iArg].n_items--;
        if (scanned[iArg].n_items>0 &&
            (!scanItemList(&majorOrderFlag, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
			   "row", -1, NULL, 0, SDDS_ROW_MAJOR_ORDER,
                           "column", -1, NULL, 0, SDDS_COLUMN_MAJOR_ORDER,
                           NULL)))
          SDDS_Bomb("invalid -majorOrder syntax/values");
        if (majorOrderFlag&SDDS_COLUMN_MAJOR_ORDER)
          columnMajorOrder=1;
        else if (majorOrderFlag&SDDS_ROW_MAJOR_ORDER)
          columnMajorOrder=0;
        break;
      case SET_COLUMNS:
	if (columns)
	  SDDS_Bomb("only one -columns option may be given");
	if (scanned[iArg].n_items<2)
	  SDDS_Bomb("invalid -columns syntax");
	column = tmalloc(sizeof(*column)*(columns=scanned[iArg].n_items-1));
	for (i=0; i<columns; i++)
	  column[i] = scanned[iArg].list[i+1];
	break;
      case SET_EXCLUDE:
	if (scanned[iArg].n_items<2)
	  SDDS_Bomb("invalid -excludeColumns syntax");
	moveToStringArray(&excludeColumn, &excludeColumns, scanned[iArg].list+1, scanned[iArg].n_items-1);
	break;
      case SET_WITHONLY:
	if (withOnly)
	  SDDS_Bomb("only one -withOnly option may be given");
	if (scanned[iArg].n_items<2)
	  SDDS_Bomb("invalid -withOnly syntax");
	withOnly = scanned[iArg].list[1];
	break;
      case SET_PIPE:
	if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags))
	  SDDS_Bomb("invalid -pipe syntax");
	break;
      case SET_RANKORDER:
	rankOrder = 1;
	break;
      case SET_STDEVOUTLIER:
	scanned[iArg].n_items--;
	outlierStDevPasses = 1;
	outlierStDevLimit = 1;
	if (!scanItemList(&dummyFlags, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
			  "limit", SDDS_DOUBLE, &outlierStDevLimit, 1, 0,
			  "passes", SDDS_LONG, &outlierStDevPasses, 1, 0,
			  NULL) || 
	    outlierStDevPasses<=0 || outlierStDevLimit<=0)
	  SDDS_Bomb("invalid -stdevOutlier syntax/values");
	break;
      default:
	fprintf(stderr, "error: unknown/ambiguous option: %s\n", 
		scanned[iArg].list[0]);
	exit(1);
	break;
      }
    }
    else {
      if (!input)
	input = scanned[iArg].list[0];
      else if (!output)
	output = scanned[iArg].list[0];
      else
	SDDS_Bomb("too many filenames seen");
    }
  }
  
  processFilenames("sddscorrelate", &input, &output, pipeFlags, 0, NULL);
  
  if (!SDDS_InitializeInput(&SDDSin, input))
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
  
  if (!columns)
    columns = appendToStringArray(&column, columns, "*"); 
  if (withOnly)
    columns = appendToStringArray(&column, columns, withOnly);
  
  if ((columns=expandColumnPairNames(&SDDSin, &column, NULL, columns, 
				     excludeColumn, excludeColumns, FIND_NUMERIC_TYPE, 0))<=0) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
    SDDS_Bomb("no columns selected for correlation analysis");
  }
  
  if (!SDDS_InitializeOutput(&SDDSout, SDDS_BINARY, 0, NULL, "sddscorrelate output", output) ||
      SDDS_DefineColumn(&SDDSout, "Correlate1Name", NULL, NULL, "Name of correlated quantity 1", NULL, SDDS_STRING, 0)<0 ||
      SDDS_DefineColumn(&SDDSout, "Correlate2Name", NULL, NULL, "Name of correlated quantity 2", NULL, SDDS_STRING, 0)<0 ||
      SDDS_DefineColumn(&SDDSout, "CorrelatePair", NULL, NULL, "Names of correlated quantities", NULL, SDDS_STRING, 0)<0 ||
      SDDS_DefineColumn(&SDDSout, "CorrelationCoefficient", "r", NULL, "Linear correlation coefficient", 
			NULL, SDDS_DOUBLE, 0)<0 ||
      SDDS_DefineColumn(&SDDSout, "CorrelationSignificance", "P$br$n", NULL, "Linear correlation coefficient significance", 
			NULL, SDDS_DOUBLE, 0)<0 ||
      SDDS_DefineColumn(&SDDSout, "CorrelationPoints", NULL, NULL, "Number of points used for correlation",
			NULL, SDDS_LONG, 0)<0 ||
      SDDS_DefineParameter(&SDDSout, "CorrelatedRows", NULL, NULL, "Number of data rows in correlation analysis", 
			   NULL, SDDS_LONG, NULL)<0 ||
      SDDS_DefineParameter(&SDDSout, "sddscorrelateInputFile", NULL, NULL, "Data file processed by sddscorrelate", 
			   NULL, SDDS_STRING, input?input:"stdin")<0 ||
      SDDS_DefineParameter(&SDDSout, "sddscorrelateMode", NULL, NULL, NULL, NULL, SDDS_STRING,
			   rankOrder?"Rank-Order (Spearman)":"Linear (Pearson)")<0 ||
      SDDS_DefineParameter1(&SDDSout, "sddscorrelateStDevOutlierPasses", NULL, NULL, 
			    "Number of passes of standard-deviation outlier elimination applied",
			    NULL, SDDS_LONG, &outlierStDevPasses)<0 ||
      SDDS_DefineParameter1(&SDDSout, "sddscorrelateStDevOutlierLimit", NULL, NULL, 
			    "Standard-deviation outlier limit applied",
			    NULL, SDDS_DOUBLE, &outlierStDevLimit)<0)
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);

  if (columnMajorOrder!=-1)
    SDDSout.layout.data_mode.column_major = columnMajorOrder;
  else
    SDDSout.layout.data_mode.column_major = SDDSin.layout.data_mode.column_major;
  
  if (!SDDS_WriteLayout(&SDDSout))
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
  
  if (!(data = (double**)malloc(sizeof(*data)*columns)) ||
      (rankOrder && !(rank = (double**)malloc(sizeof(*rank)*columns))) ||
      !(accept = (short**)malloc(sizeof(*accept)*columns)))
    SDDS_Bomb("allocation failure");
  while ((readCode=SDDS_ReadPage(&SDDSin))>0) {
    if ((rows = SDDS_CountRowsOfInterest(&SDDSin))<3)
      continue;
    if (!SDDS_StartPage(&SDDSout, columns*(columns-1)/2) ||
	!SDDS_SetParameters(&SDDSout, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE, 
			    "CorrelatedRows", rows, NULL))
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
    for (i=0; i<columns; i++) {
      if (!(data[i] = SDDS_GetColumnInDoubles(&SDDSin, column[i])))
	SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
      if (rankOrder)
	rank[i] = findRank(data[i], rows);
      if (outlierStDevPasses) {
	if (!(accept[i] = (short*)malloc(sizeof(**accept)*rows)))
	  SDDS_Bomb("allocation failure");
	markStDevOutliers(data[i], outlierStDevLimit, outlierStDevPasses,
			  accept[i], rows);
      }
      else
	accept[i] = NULL;
    }
    for (i=row=0; i<columns; i++) {
      for (j=i+1; j<columns; j++) {
	iName1 = i;
	iName2 = j;
	if (withOnly) {
	  if (strcmp(withOnly, column[i])==0) {
	    iName1 = j;
	    iName2 = i;
	  }
	  else if (strcmp(withOnly, column[j])==0) {
	    iName1 = i;
	    iName2 = j;
	  }
	  else 
	    continue;
	}
	correlation 
	  = linearCorrelationCoefficient(rankOrder?rank[i]:data[i], 
					 rankOrder?rank[j]:data[j], 
					 accept[i], accept[j], rows, &count);
	significance = linearCorrelationSignificance(correlation, count);
	sprintf(s, "%s.%s", column[iName1], column[iName2]);
                if (!SDDS_SetRowValues(&SDDSout, SDDS_SET_BY_INDEX|SDDS_PASS_BY_VALUE, row++,
                                       0, column[iName1], 
                                       1, column[iName2], 
                                       2, s, 3, correlation, 4, significance, 
                                       5, count, -1))
		  SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
      }
    }
    for (i=0; i<columns; i++) {
      free(data[i]);
      if (rankOrder)
	free(rank[i]);
      if (accept[i])
	free(accept[i]);
    }
    if (!SDDS_WritePage(&SDDSout))
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
  }
  
  if (!SDDS_Terminate(&SDDSin)) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    exit(1);
  }
  if (!SDDS_Terminate(&SDDSout)) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    exit(1);
  }
  
  return 0;
}
Exemple #3
0
int main(int argc, char **argv)
{
    int iArg;
    char **outputColumn, **difColumn;
    char *indepColumn, **depenColumn, **exclude;
    long depenColumns, excludes;
    char *input, *output;
    long i, rows, readCode, optionCode;
    unsigned long flags, pipeFlags;
    SCANNED_ARG *scanned;
    SDDS_DATASET SDDSin, SDDSout;
    double *timeData, *inputData, *outputData;
    FILTER_STAGE *filterStage;
    long filterStages, totalFilters;

    SDDS_RegisterProgramName(argv[0]);
    argc = scanargs(&scanned, argc, argv); 
    if (argc<3 || argc>(3+N_OPTIONS)) 
        bomb(NULL, USAGE);

    output = input = NULL;
    flags = pipeFlags = 0;
    indepColumn = NULL;
    depenColumn = exclude = NULL;
    depenColumns = excludes = 0;
    if (!(filterStage = (FILTER_STAGE*)calloc(1, sizeof(*filterStage))))
        SDDS_Bomb("allocation failure");
    filterStage->filter = NULL;
    filterStage->filters = 0;
    filterStages = 1;
    totalFilters = 0;

    for (iArg=1; iArg<argc; iArg++) {
        if (scanned[iArg].arg_type==OPTION) {
            /* process options here */
            switch (optionCode=match_string(scanned[iArg].list[0], option, N_OPTIONS, 0)) {
              case SET_PIPE:
                if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags))
                    SDDS_Bomb("invalid -pipe syntax");
                break;
              case SET_COLUMNS:
                if (indepColumn)
                    SDDS_Bomb("only one -columns option may be given");
                if (scanned[iArg].n_items<2)
                    SDDS_Bomb("invalid -columns syntax");
                indepColumn = scanned[iArg].list[1];
                if (scanned[iArg].n_items>=2) {
                    depenColumn = tmalloc(sizeof(*depenColumn)*(depenColumns=scanned[iArg].n_items-2));
                    for (i=0; i<depenColumns; i++)
                        depenColumn[i] = scanned[iArg].list[i+2];
                    }
                break;
              case SET_THRESHOLD:
              case SET_HIGHPASS:
              case SET_LOWPASS:
              case SET_NOTCH:
              case SET_BANDPASS:
              case SET_FILTERFILE:
              case SET_CLIPFREQ:
                addFilter(filterStage+filterStages-1, optionCode, scanned+iArg);
                totalFilters++;
                break;
              case SET_CASCADE:
                if (filterStages==0)
                    SDDS_Bomb("-cascade option precedes all filter definitions");
                if (!(filterStage = SDDS_Realloc(filterStage, (filterStages+1)*sizeof(*filterStage))))
                    SDDS_Bomb("allocation failure");
                filterStage[filterStages].filter = NULL;
                filterStage[filterStages].filters = 0;
                filterStages++;
                break;
              case SET_NEWCOLUMNS:
                flags |= FL_NEWCOLUMNS;
                break;
              case SET_DIFFERENCECOLUMNS:
                flags |= FL_DIFCOLUMNS;
                break;
              case SET_EXCLUDE:
                if (scanned[iArg].n_items<2)
                    SDDS_Bomb("invalid -exclude syntax");
                moveToStringArray(&exclude, &excludes, scanned[iArg].list+1, scanned[iArg].n_items-1);
                break;
              default:
                fprintf(stderr, "error: unknown/ambiguous option: %s (%s)\n", 
                        scanned[iArg].list[0], argv[0]);
                exit(1);
                break;
                }
            }
        else {
            if (!input)
                input = scanned[iArg].list[0];
            else if (!output)
                output = scanned[iArg].list[0];
            else
                SDDS_Bomb("too many filenames seen");
            }
        }

    processFilenames("sddsfdfilter", &input, &output, pipeFlags, 0, NULL);

    if (!totalFilters)
        fputs("warning: no filters specified (sddsfdfilter)\n", stderr);

    if (!indepColumn)
        SDDS_Bomb("supply the independent column name with the -columns option");
    
    if (!SDDS_InitializeInput(&SDDSin, input))
        SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);

    if (SDDS_CheckColumn(&SDDSin, indepColumn, NULL, SDDS_ANY_NUMERIC_TYPE, stderr)!=SDDS_CHECK_OKAY) 
        exit(1);

    excludes = appendToStringArray(&exclude, excludes, indepColumn); 
    if (!depenColumns)
        depenColumns = appendToStringArray(&depenColumn, depenColumns, "*"); 

    if ((depenColumns=expandColumnPairNames(&SDDSin, &depenColumn, NULL, depenColumns, exclude, excludes,
                                            FIND_NUMERIC_TYPE, 0))<=0) {
        SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
        SDDS_Bomb("No quantities selected to filter");
        }

    if (!SDDS_InitializeCopy(&SDDSout, &SDDSin, output, "w"))
        SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);

       
    if (flags&FL_NEWCOLUMNS) {
        outputColumn = tmalloc(sizeof(*outputColumn)*depenColumns);
        for (i=0; i<depenColumns; i++) {
            outputColumn[i] = tmalloc(sizeof(**outputColumn)*(strlen(depenColumn[i])+1+strlen("Filtered")));
            sprintf(outputColumn[i], "%sFiltered", depenColumn[i]);
            if (!SDDS_TransferColumnDefinition(&SDDSout, &SDDSin, depenColumn[i], outputColumn[i]))
                SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
            }
        }
    else
        outputColumn=depenColumn;
        
    difColumn = NULL;
    if (flags&FL_DIFCOLUMNS) {
        difColumn = tmalloc(sizeof(*difColumn)*depenColumns);
        for (i=0; i<depenColumns; i++) {
            difColumn[i] = tmalloc(sizeof(**difColumn)*(strlen(depenColumn[i])+1+strlen("Difference")));
            sprintf(difColumn[i], "%sDifference", depenColumn[i]);
            if (!SDDS_TransferColumnDefinition(&SDDSout, &SDDSin, depenColumn[i], difColumn[i]))
                SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
            }
        }

    if (!SDDS_WriteLayout(&SDDSout))
        SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
        
    outputData = NULL;
    while ((readCode=SDDS_ReadPage(&SDDSin))>0) {
        if (!SDDS_CopyPage(&SDDSout, &SDDSin))
            SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
        if ((rows = SDDS_CountRowsOfInterest(&SDDSin))<0)
            SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
	 
        if (rows) {
            if (!(timeData = SDDS_GetColumnInDoubles(&SDDSin, indepColumn)))
                SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
            if (!(outputData = SDDS_Realloc(outputData, sizeof(*outputData)*rows)))
                SDDS_Bomb("allocation failure");
            for (i=0; i<depenColumns; i++) {
                if (!(inputData = SDDS_GetColumnInDoubles(&SDDSin, depenColumn[i])))
                    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
                if (!applyFilters(outputData, inputData, timeData, rows, filterStage, filterStages))
                    exit(1);
                if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, outputData, rows, outputColumn[i]))
                    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
                if (flags&FL_DIFCOLUMNS) {
                    long j;
                    for (j=0; j<rows; j++)
                        outputData[j] = inputData[j] - outputData[j];
                    if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_BY_NAME, outputData, rows, difColumn[i]))
                        SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
                    }
                free(inputData);
                }
            free(timeData);
            }
        if (!SDDS_WritePage(&SDDSout))
            SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
        }
	
    free(outputData);	
    for(i=0;i<depenColumns;i++) {
       free(depenColumn[i]);
       if (flags&FL_NEWCOLUMNS)
          free(outputColumn[i]);
       if (flags&FL_DIFCOLUMNS) 
          free(difColumn[i]);
    }   
    for(i=0;i<excludes;i++) 
       free(exclude[i]);
    
    free(indepColumn);
    if (flags&FL_NEWCOLUMNS)
       free(outputColumn);
    free(depenColumn);
    if (flags&FL_DIFCOLUMNS) 
       free(difColumn);
    free(exclude);
    for(i=0;i<filterStages;i++) {
       long j;
       for(j=0;j<filterStage[i].filters;j++)  {
          switch (filterStage[i].filter[j].filterType) {
	  case SET_FILTERFILE :
	     free( ((FILE_FILTER*) (filterStage[i].filter[j].filter))->freqData);
	     free( ((FILE_FILTER*) (filterStage[i].filter[j].filter))->magData);
	     free( ((FILE_FILTER*) (filterStage[i].filter[j].filter))->imagData);
	     free( ((FILE_FILTER*) (filterStage[i].filter[j].filter))->realData);
	     break;
	  default :
	     break;
	  }
       }	           
    } 	   
	    
    if (!SDDS_Terminate(&SDDSout) || !SDDS_Terminate(&SDDSin))
            SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);	
    return 0;
    }
Exemple #4
0
int main(int argc, char **argv)
{
  char *indepQuantity, **depenQuantity, **exclude, **depenQuantityPair;
  long depenQuantities, excludes;
  char *input, *output;
  long iArg, i, j, rows, readCode, noWarnings, items;
  long rowsToUse;
  unsigned long flags, pairFlags, tmpFlags, pipeFlags;
  SCANNED_ARG *scArg;
  SDDS_DATASET SDDSin, SDDSout;
  double *tdata, *data, t0, dt;
  double fracRMSChangeLimit, fracFreqAccuracyLimit;
  int32_t frequenciesDesired, maxFrequencies, freqCycleLimit;
  short truncate;
  double *frequency, *amplitude=NULL, *phase=NULL, *significance=NULL, *phase1=NULL, *amplitude1=NULL, *significance1=NULL;
  long frequenciesFound;
  long *frequencyIndex, *amplitudeIndex, *phaseIndex, *significanceIndex, pairs;
  long *amplitudeIndex1, *phaseIndex1, *significanceIndex1;

  SDDS_RegisterProgramName(argv[0]);

#ifdef DEBUG
  if (1) {
    long code;
    double x, y; 
    x = 1.1;
    code = OneDFunctionOptimize(&y, &x, 0.07, -4, 4, 
				trialFn, 50, 1e-6, 0, 1);
    fprintf(stderr, "code: %ld   x=%e,  y=%e\n", code, x, y);
    
    x = .9;
    code = OneDFunctionOptimize(&y, &x, 0.15, -4, 4, 
				trialFn, 50, 1e-6, 0, 1);
    fprintf(stderr, "code: %ld   x=%e,  y=%e\n", code, x, y);
    
    x = .999;
    code = OneDFunctionOptimize(&y, &x, 0.11, -4, 4, 
				trialFn, 50, 1e-6, 0, 1);
    fprintf(stderr, "code: %ld   x=%e,  y=%e\n", code, x, y);
    exit(0);
  }
#endif

  argc = scanargs(&scArg, argc, argv); 
  if (argc<3) {
    fprintf(stderr, "%s%s", USAGE1, USAGE2);
    exit(1);
  }
  output = input = NULL;
  flags = pipeFlags = excludes = truncate = pairFlags= 0;
  indepQuantity = NULL;
  depenQuantity = exclude = depenQuantityPair=NULL;
  depenQuantities = 0;
  noWarnings = 0;
  fracRMSChangeLimit = 0.0;
  fracFreqAccuracyLimit = 0.00001;
  frequenciesDesired = 1;
  maxFrequencies = 4;
  freqCycleLimit = 100;
  pairs=0;
  
  for (iArg=1; iArg<argc; iArg++) {
    if (scArg[iArg].arg_type==OPTION) {
      /* process options here */
      switch (match_string(scArg[iArg].list[0], option, N_OPTIONS, 0)) {
      case SET_TRUNCATE:
	truncate = 1;
	break;
      case SET_PAIR:
        if (depenQuantities)
          SDDS_Bomb("Invalid -pair option, the depent-quantity is provided by -column option already.");
        if (scArg[iArg].n_items!=3) 
          SDDS_Bomb("invalid -pair syntax");
        depenQuantity = SDDS_Realloc(depenQuantity, sizeof(*depenQuantity)*(pairs+1));
        depenQuantityPair = SDDS_Realloc(depenQuantityPair, sizeof(*depenQuantityPair)*(pairs+1));
        depenQuantity[pairs] = scArg[iArg].list[1];
        depenQuantityPair[pairs] = scArg[iArg].list[2];
        pairs++;
        break;
      case SET_COLUMN:
	if (indepQuantity)
	  SDDS_Bomb("only one -column option may be given");
	if (scArg[iArg].n_items<2)
	  SDDS_Bomb("invalid -column syntax");
	indepQuantity = scArg[iArg].list[1];
	if (scArg[iArg].n_items>=2) {
          if (pairs)
            SDDS_Bomb("Invalid -column syntax, the depent-quantity is provided by -pair option already.");
	  depenQuantity = tmalloc(sizeof(*depenQuantity)*(depenQuantities=scArg[iArg].n_items-2));
	  for (i=0; i<depenQuantities; i++)
	    depenQuantity[i] = scArg[iArg].list[i+2];
	}
	break;
      case SET_PIPE:
	if (!processPipeOption(scArg[iArg].list+1, scArg[iArg].n_items-1, &pipeFlags))
	  SDDS_Bomb("invalid -pipe syntax");
	break;
      case SET_EXCLUDE:
	if (scArg[iArg].n_items<2)
	  SDDS_Bomb("invalid -exclude syntax");
	moveToStringArray(&exclude, &excludes, scArg[iArg].list+1, scArg[iArg].n_items-1);
	break;
      case SET_NOWARNINGS:
	noWarnings = 1;
	break;
      case SET_TERM_SEARCH:
	items = scArg[iArg].n_items-1;
	flags &= ~(NAFF_RMS_CHANGE_LIMIT|NAFF_FREQS_DESIRED|NAFF_MAX_FREQUENCIES);
	fracRMSChangeLimit = 0;
	frequenciesDesired = 0;
	maxFrequencies = 10;
	if (!scanItemList(&tmpFlags, scArg[iArg].list+1, &items, 0,
			  "changelimit", SDDS_DOUBLE, &fracRMSChangeLimit, 1, 
			  NAFF_RMS_CHANGE_LIMIT,
			  "maxfrequencies", SDDS_LONG, &maxFrequencies, 1, 
			  NAFF_MAX_FREQUENCIES,
			  "frequencies", SDDS_LONG, &frequenciesDesired, 1, 
			  NAFF_FREQS_DESIRED,
			  NULL) ||
	    (tmpFlags&NAFF_RMS_CHANGE_LIMIT && tmpFlags&NAFF_FREQS_DESIRED) ||
	    maxFrequencies<1)
	  SDDS_Bomb("invalid -terminateSearch syntax");
	flags |= tmpFlags;
	if (frequenciesDesired)
	  maxFrequencies = frequenciesDesired;
        break;
      case SET_ITERATE_FREQ:
	items = scArg[iArg].n_items - 1;
	flags &= ~(NAFF_FREQ_CYCLE_LIMIT|NAFF_FREQ_ACCURACY_LIMIT);
	if (!scanItemList(&tmpFlags, scArg[iArg].list+1, &items, 0,
			  "cyclelimit", SDDS_LONG, &freqCycleLimit, 1, NAFF_FREQ_CYCLE_LIMIT,
			  "accuracylimit", SDDS_DOUBLE, &fracFreqAccuracyLimit, 
			  1, NAFF_FREQ_ACCURACY_LIMIT,
			  NULL) || !bitsSet(tmpFlags) || freqCycleLimit<2)
	  SDDS_Bomb("invalid -iterateFrequency syntax");
	flags |= tmpFlags;
        break;
      default:
	fprintf(stderr, "error: unknown/ambiguous option: %s\n", 
		scArg[iArg].list[0]);
	exit(1);
	break;
      }
    }
    else {
      if (!input)
	input = scArg[iArg].list[0];
      else if (!output)
	output = scArg[iArg].list[0];
      else
	SDDS_Bomb("too many filenames seen");
    }
  }
  
  processFilenames("sddsnaff", &input, &output, pipeFlags, 0, NULL);
  
  if (!indepQuantity)
    SDDS_Bomb("supply the independent quantity name with the -columns option");
  
  if (!SDDS_InitializeInput(&SDDSin, input))
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
  
  if (SDDS_CheckColumn(&SDDSin, indepQuantity, NULL, SDDS_ANY_NUMERIC_TYPE, stderr)
      !=SDDS_CHECK_OKAY) 
    exit(1);
  
  excludes = appendToStringArray(&exclude, excludes, indepQuantity);
  if (pairs) {
    pairFlags = flags | NAFF_FREQ_FOUND;
    depenQuantities = pairs;
  }
  if (!depenQuantities)
    depenQuantities = appendToStringArray(&depenQuantity, depenQuantities, "*"); 
  if (!pairs) {
    if ((depenQuantities=expandColumnPairNames(&SDDSin, &depenQuantity, NULL, 
                                               depenQuantities, exclude, excludes, 
                                               FIND_NUMERIC_TYPE, 0))<=0) {
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
      SDDS_Bomb("No quantities selected to fft");
    }
  }
  
  if (!SetupNAFFOutput(&SDDSout, output, &SDDSin, indepQuantity,
		       depenQuantities, depenQuantity,
		       &frequencyIndex, &amplitudeIndex, &phaseIndex,
                       &significanceIndex, depenQuantityPair,&amplitudeIndex1,
                       &phaseIndex1,&significanceIndex1)) 
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
  
  if (!(frequency = SDDS_Malloc(sizeof(*frequency)*maxFrequencies)) ||
      !(amplitude = SDDS_Malloc(sizeof(*amplitude)*maxFrequencies)) ||
      !(phase     = SDDS_Malloc(sizeof(*phase    )*maxFrequencies)) ||
      !(significance= SDDS_Malloc(sizeof(*significance)*maxFrequencies))) 
    SDDS_Bomb("memory allocation failure");
  if (pairs) {
    if (!(amplitude1 = SDDS_Malloc(sizeof(*amplitude1)*maxFrequencies)) ||
        !(phase1     = SDDS_Malloc(sizeof(*phase1    )*maxFrequencies)) ||
        !(significance1 = SDDS_Malloc(sizeof(*significance1)*maxFrequencies)))
      SDDS_Bomb("memory allocation failure");
  }
  while ((readCode=SDDS_ReadPage(&SDDSin))>0) {
    if ((rows = SDDS_CountRowsOfInterest(&SDDSin))<0) 
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
    if (rows) {
      long primeRows;
      rowsToUse = rows;
      primeRows = greatestProductOfSmallPrimes(rows);
      if (rows!=primeRows) {
	if (truncate)
	  rowsToUse = greatestProductOfSmallPrimes(rows);
	else if (largest_prime_factor(rows)>100 && !noWarnings)
	  fputs("Warning: number of points has large prime factors.\nThis could take a very long time.\nConsider using the -truncate option.\n", stderr);
      }
      if (!SDDS_StartPage(&SDDSout, maxFrequencies) ||
	  !SDDS_CopyParameters(&SDDSout, &SDDSin))
	SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
      if (!(tdata = SDDS_GetColumnInDoubles(&SDDSin, indepQuantity)))
	SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
      for (i=1; i<rowsToUse; i++)
	if (tdata[i]<=tdata[i-1])
	  SDDS_Bomb("independent data is not monotonically increasing");
      dt = (tdata[rowsToUse-1]-tdata[0])/(rowsToUse-1.0),
      t0 = tdata[0];
      free(tdata);
      for (i=0; i<depenQuantities; i++) {
	if (!(data = SDDS_GetColumnInDoubles(&SDDSin, depenQuantity[i])))
	  SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
        for (j=0; j<maxFrequencies; j++)
          frequency[j] = amplitude[j] = phase[j] = significance[j] = -1;
	frequenciesFound = PerformNAFF(frequency, amplitude, phase, significance,
                                       t0, dt,
				       data, rowsToUse, flags,
				       fracRMSChangeLimit, maxFrequencies,
				       freqCycleLimit, fracFreqAccuracyLimit, 0, 0);
#ifdef DEBUG
        fprintf(stderr, "Column %s: ", depenQuantity[i]);
        fprintf(stderr, "f=%10.3e  a=%10.3e  p=%10.3e  s=%10.3e\n",
                frequency[0], amplitude[0], phase[0], significance[0]);
#endif
	free(data);
	data = NULL;
        if (pairs) {
          if (!(data = SDDS_GetColumnInDoubles(&SDDSin, depenQuantityPair[i])))
            SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
          frequenciesFound = PerformNAFF(frequency, amplitude1, phase1, significance1,
                                         t0, dt,
                                         data, rowsToUse, pairFlags,
                                         fracRMSChangeLimit, maxFrequencies,
                                         freqCycleLimit, fracFreqAccuracyLimit, 0, 0);
          
          for (j=0; j<maxFrequencies; j++)
            if (frequency[j]!=-1)
              frequency[j] = adjustFrequencyHalfPlane(frequency[j], phase[j], phase1[j], dt); 
          free(data);
          data = NULL;
        } 
	if (!SDDS_SetColumn(&SDDSout, SDDS_SET_BY_INDEX,
			    frequency, maxFrequencies, frequencyIndex[i]) ||
	    !SDDS_SetColumn(&SDDSout, SDDS_SET_BY_INDEX,
			    amplitude, maxFrequencies, amplitudeIndex[i]) ||
	    !SDDS_SetColumn(&SDDSout, SDDS_SET_BY_INDEX,
			    phase, maxFrequencies, phaseIndex[i]) ||
	    !SDDS_SetColumn(&SDDSout, SDDS_SET_BY_INDEX,
			    significance, maxFrequencies, significanceIndex[i]))
	  SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
        if (pairs) {
          if (!SDDS_SetColumn(&SDDSout, SDDS_SET_BY_INDEX,
                              amplitude1, maxFrequencies, amplitudeIndex1[i]) ||
              !SDDS_SetColumn(&SDDSout, SDDS_SET_BY_INDEX,
                              phase1, maxFrequencies, phaseIndex1[i]) ||
              !SDDS_SetColumn(&SDDSout, SDDS_SET_BY_INDEX,
                              significance1, maxFrequencies, significanceIndex1[i]))
            SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
        }
      }
    }
    else  {
      if (!SDDS_StartPage(&SDDSout, 0) || !SDDS_CopyParameters(&SDDSout, &SDDSin))
	SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
    }
    if (!SDDS_WritePage(&SDDSout))
      SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
  }
  
  if (!SDDS_Terminate(&SDDSin)) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    exit(1);
  }
  if (!SDDS_Terminate(&SDDSout)) {
    SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
    exit(1);
  }
  free(frequency);
  free(amplitude);
  free(phase);
  free(significance);
  if (pairs) {
    free(amplitude1);
    free(phase1);
    free(significance1);
    free(amplitudeIndex1);
    free(phaseIndex1);
    free(significanceIndex1);
    free(depenQuantityPair);
  }
  free(depenQuantity);
  free(frequencyIndex);
  free(amplitudeIndex);
  free(phaseIndex);
  free(significanceIndex);
  return 0;
}