int main(int argc, char **argv)
{
SDDS_DATASET inSet, outSet;
SCANNED_ARG *s_arg;
long i_arg, pageReturned, rows, row;
int32_t *rowFlag;
char *input, *output, *columnName, *par_thresholdName;
double *data;
unsigned long pipeFlags, flags;
double threshold, ezoneFraction, changeThreshold;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&s_arg, argc, argv);
if (argc<2 || argc>(2+N_OPTIONS))
bomb(NULL, USAGE);
flags = pipeFlags = 0;
input = output = NULL;
columnName = NULL;
ezoneFraction = changeThreshold = 0;
rowFlag = NULL;
par_thresholdName=NULL;
for (i_arg=1; i_arg<argc; i_arg++) {
if (s_arg[i_arg].arg_type==OPTION) {
switch (match_string(s_arg[i_arg].list[0], option, N_OPTIONS, 0)) {
case CLO_THRESHOLD:
if (s_arg[i_arg].n_items==2) {
if (s_arg[i_arg].list[1][0]=='@' ) {
SDDS_CopyString(&par_thresholdName, s_arg[i_arg].list[1]+1);
flags |= PAR_THRESHOLD;
} else {
if (sscanf(s_arg[i_arg].list[1], "%lf", &threshold)!=1)
SDDS_Bomb("incorrect -threshold syntax");
flags |= THRESHOLD;
}
} else
SDDS_Bomb("incorrect -threshold syntax");
break;
case CLO_FIVEPOINT:
flags |= FIVEPOINT;
break;
case CLO_CHANGETHRESHOLD:
if (s_arg[i_arg].n_items!=2 ||
sscanf(s_arg[i_arg].list[1], "%lf", &changeThreshold)!=1 || changeThreshold<=0)
SDDS_Bomb("incorrect -changeThreshold syntax or values");
flags |= CHANGETHRES;
break;
case CLO_PIPE:
if (!processPipeOption(s_arg[i_arg].list+1, s_arg[i_arg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case CLO_COLUMN:
if (s_arg[i_arg].n_items!=2)
SDDS_Bomb("invalid -column syntax");
columnName = s_arg[i_arg].list[1];
break;
case CLO_EXCLUSIONZONE:
if (s_arg[i_arg].n_items!=2 || sscanf(s_arg[i_arg].list[1], "%lf", &ezoneFraction)!=1 ||
ezoneFraction<=0)
SDDS_Bomb("invalid -exclusionZone syntax or value");
flags |= EZONEFRAC;
break;
default:
fprintf(stderr, "error: unknown/ambiguous option: %s\n", s_arg[i_arg].list[0]);
exit(1);
break;
}
}
else {
if (input==NULL)
input = s_arg[i_arg].list[0];
else if (output==NULL)
output = s_arg[i_arg].list[0];
else
SDDS_Bomb("too many filenames");
}
}
processFilenames("sddspeakfind", &input, &output, pipeFlags, 0, NULL);
if (!columnName)
SDDS_Bomb("-column option must be given");
if (!SDDS_InitializeInput(&inSet, input))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!SDDS_FindColumn(&inSet, FIND_NUMERIC_TYPE, columnName, NULL))
SDDS_Bomb("the given column is nonexistent or nonnumeric");
if (!SDDS_InitializeCopy(&outSet, &inSet, output, "w") || !SDDS_WriteLayout(&outSet))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
while ((pageReturned=SDDS_ReadPage(&inSet))>0) {
if (!SDDS_CopyPage(&outSet, &inSet)) {
SDDS_SetError("Problem copying data for output file");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if ((rows=SDDS_CountRowsOfInterest(&outSet))>1) {
if (!(data = SDDS_GetColumnInDoubles(&inSet, columnName)))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
rowFlag = SDDS_Realloc(rowFlag, sizeof(*rowFlag)*rows);
for (row=0; row<rows; row++)
rowFlag[row] = 0;
markPeaks(data, rowFlag, rows, flags&FIVEPOINT);
if (flags&PAR_THRESHOLD) {
if (!SDDS_GetParameter(&inSet,par_thresholdName,&threshold))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (flags&THRESHOLD || flags&PAR_THRESHOLD ) {
for (row=0; row<rows; row++)
if (rowFlag[row] && data[row]<threshold)
rowFlag[row] = 0;
}
if (flags&CHANGETHRES)
unmarkFlatPeaks(data, rowFlag, rows, changeThreshold, flags&FIVEPOINT);
if (flags&EZONEFRAC)
unmarkExcludedPeaks(data, rowFlag, rows, ezoneFraction);
if (!SDDS_AssertRowFlags(&outSet, SDDS_FLAG_ARRAY, rowFlag, rows))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
free(data);
}
if (!SDDS_WritePage(&outSet))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_Terminate(&inSet) || !SDDS_Terminate(&outSet)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
free_scanargs(&s_arg,argc);
if (par_thresholdName) free(par_thresholdName);
if (rowFlag) free(rowFlag);
return(0);
}
int main(int argc, char **argv)
{
FILE *fileID;
COLUMN_DATA_STRUCTURES *columnValues;
PARAMETER_DATA_STRUCTURES *parameterValues;
SDDS_DATASET SDDS_dataset;
SCANNED_ARG *s_arg;
long i, j, k, n, i_arg;
int32_t rows;
int32_t maxRows=10000, initRows=10000, row;
long par, col, page, size, readline=1, fillin=0;
int32_t ptrSize=0;
char *input, *output, s[1024], *ptr, *ptr2, data[10240],temp[10240];
unsigned long pipeFlags=0,majorOrderFlag;
long noWarnings=0, tmpfile_used=0, columnOrder=0, whitespace=1;
short shortValue;
int32_t longValue;
float floatValue;
double doubleValue;
char stringValue[SDDS_MAXLINE];
char characterValue;
char buffer[124], buffer2[124];
long *parameterIndex, *columnIndex;
long binary=0, noRowCount=0, inputBinary=0, count=0;
char separator;
char commentCharacters[20];
short checkComment=0;
short commentFound;
long parameters=0, columns=0;
long skiplines=0;
short abort=0, recover=1, columnMajorOrder=0;
input = output = NULL;
separator = ' ';
columnValues = NULL;
parameterValues = NULL;
parameterIndex = columnIndex = NULL;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&s_arg, argc, argv);
if (argc<3)
bomb(NULL, USAGE);
for (i_arg=1; i_arg<argc; i_arg++) {
if (s_arg[i_arg].arg_type==OPTION) {
switch (match_string(s_arg[i_arg].list[0], option, N_OPTIONS, 0)) {
case SET_MAJOR_ORDER:
majorOrderFlag=0;
s_arg[i_arg].n_items -=1;
if (s_arg[i_arg].n_items>0 &&
(!scanItemList(&majorOrderFlag, s_arg[i_arg].list+1, &s_arg[i_arg].n_items, 0,
"row", -1, NULL, 0, SDDS_ROW_MAJOR_ORDER,
"column", -1, NULL, 0, SDDS_COLUMN_MAJOR_ORDER,
NULL)))
SDDS_Bomb("invalid -majorOrder syntax/values");
if (majorOrderFlag&SDDS_COLUMN_MAJOR_ORDER)
columnMajorOrder=1;
else if (majorOrderFlag&SDDS_ROW_MAJOR_ORDER)
columnMajorOrder=0;
break;
case SET_OUTPUTMODE:
if (s_arg[i_arg].n_items!=2)
SDDS_Bomb("invalid -outputMode syntax");
switch (match_string(s_arg[i_arg].list[1], mode_name, MODES, 0)) {
case ASCII_MODE:
binary = 0;
break;
case BINARY_MODE:
binary = 1;
break;
default:
SDDS_Bomb("invalid -outputMode syntax");
break;
}
break;
case SET_INPUTMODE:
if (s_arg[i_arg].n_items!=2)
SDDS_Bomb("invalid -inputMode syntax");
switch (match_string(s_arg[i_arg].list[1], mode_name, MODES, 0)) {
case ASCII_MODE:
inputBinary = 0;
break;
case BINARY_MODE:
inputBinary = 1;
break;
default:
SDDS_Bomb("invalid -inputMode syntax");
break;
}
break;
case SET_SEPARATOR:
if (s_arg[i_arg].n_items!=2)
SDDS_Bomb("invalid -separator syntax");
separator = s_arg[i_arg].list[1][0];
whitespace = 0;
break;
case SET_COMMENT:
if (s_arg[i_arg].n_items!=2)
SDDS_Bomb("invalid -commentCharacters syntax");
sprintf(commentCharacters, "%s", s_arg[i_arg].list[1]);
checkComment=1;
break;
case SET_FILLIN:
fillin=1;
break;
case SET_NOROWCOUNT:
if (s_arg[i_arg].n_items!=1)
SDDS_Bomb("invalid -noRowCount syntax");
noRowCount = 1;
break;
case SET_ORDER:
if (s_arg[i_arg].n_items!=2)
SDDS_Bomb("invalid -order syntax");
switch (match_string(s_arg[i_arg].list[1], order_names, ORDERS, 0)) {
case ROW_ORDER:
columnOrder = 0;
break;
case COLUMN_ORDER:
columnOrder = 1;
break;
default:
SDDS_Bomb("invalid -order syntax");
break;
}
break;
case SET_PARAMETER:
if (s_arg[i_arg].n_items<3)
SDDS_Bomb("invalid -parameter syntax");
count=1;
parameters++;
parameterValues = trealloc(parameterValues, sizeof(*parameterValues)*(parameters));
SDDS_CopyString(¶meterValues[parameters-1].name, s_arg[i_arg].list[1]);
parameterValues[parameters-1].units = NULL;
parameterValues[parameters-1].description = NULL;
parameterValues[parameters-1].symbol = NULL;
switch (match_string(s_arg[i_arg].list[2], type_name, DATATYPES, 0)) {
case TYPE_SHORT:
parameterValues[parameters-1].type = SDDS_SHORT;
break;
case TYPE_LONG:
parameterValues[parameters-1].type = SDDS_LONG;
break;
case TYPE_FLOAT:
parameterValues[parameters-1].type = SDDS_FLOAT;
break;
case TYPE_DOUBLE:
parameterValues[parameters-1].type = SDDS_DOUBLE;
break;
case TYPE_STRING:
parameterValues[parameters-1].type = SDDS_STRING;
break;
case TYPE_CHARACTER:
parameterValues[parameters-1].type = SDDS_CHARACTER;
break;
default:
SDDS_Bomb("invalid -parameter type");
break;
}
for (i=3;i<s_arg[i_arg].n_items;i++) {
if (!(ptr=strchr(s_arg[i_arg].list[i], '=')))
SDDS_Bomb("invalid -parameter syntax");
*ptr++ = 0;
switch (match_string(s_arg[i_arg].list[i], header_elements, HEADERELEMENTS, 0)) {
case HEADER_UNITS:
SDDS_CopyString(¶meterValues[parameters-1].units, ptr);
break;
case HEADER_DESCRIPTION:
SDDS_CopyString(¶meterValues[parameters-1].description, ptr);
break;
case HEADER_SYMBOL:
SDDS_CopyString(¶meterValues[parameters-1].symbol, ptr);
break;
case HEADER_COUNT:
if (sscanf(ptr, "%ld", &count)!=1 ||
count<=0)
SDDS_Bomb("invalid parameter count value");
sprintf(buffer, "%s", parameterValues[parameters-1].name);
sprintf(buffer2, "%s1", buffer);
free(parameterValues[parameters-1].name);
SDDS_CopyString(¶meterValues[parameters-1].name, buffer2);
break;
default:
SDDS_Bomb("invalid -parameter syntax");
break;
}
}
for (i=2;i<=count;i++) {
parameters++;
parameterValues = trealloc(parameterValues, sizeof(*parameterValues)*(parameters));
sprintf(buffer2, "%s%ld", buffer, i);
SDDS_CopyString(¶meterValues[parameters-1].name, buffer2);
parameterValues[parameters-1].units = NULL;
parameterValues[parameters-1].description = NULL;
parameterValues[parameters-1].symbol = NULL;
parameterValues[parameters-1].type = parameterValues[parameters-2].type;
if (parameterValues[parameters-2].units)
SDDS_CopyString(¶meterValues[parameters-1].units, parameterValues[parameters-2].units);
if (parameterValues[parameters-2].description)
SDDS_CopyString(¶meterValues[parameters-1].description, parameterValues[parameters-2].description);
if (parameterValues[parameters-2].symbol)
SDDS_CopyString(¶meterValues[parameters-1].symbol, parameterValues[parameters-2].symbol);
}
break;
case SET_COLUMN:
if (s_arg[i_arg].n_items<3)
SDDS_Bomb("invalid -column syntax");
count=1;
columns++;
columnValues = trealloc(columnValues, sizeof(*columnValues)*(columns));
SDDS_CopyString(&columnValues[columns-1].name, s_arg[i_arg].list[1]);
columnValues[columns-1].elements = 0;
columnValues[columns-1].values = NULL;
columnValues[columns-1].stringValues = NULL;
columnValues[columns-1].units = NULL;
columnValues[columns-1].description = NULL;
columnValues[columns-1].symbol = NULL;
columnValues[columns-1].skip = 0;
switch (match_string(s_arg[i_arg].list[2], type_name, DATATYPES, 0)) {
case TYPE_SHORT:
columnValues[columns-1].type = SDDS_SHORT;
break;
case TYPE_LONG:
columnValues[columns-1].type = SDDS_LONG;
break;
case TYPE_FLOAT:
columnValues[columns-1].type = SDDS_FLOAT;
break;
case TYPE_DOUBLE:
columnValues[columns-1].type = SDDS_DOUBLE;
break;
case TYPE_STRING:
columnValues[columns-1].type = SDDS_STRING;
break;
case TYPE_CHARACTER:
columnValues[columns-1].type = SDDS_CHARACTER;
break;
default:
SDDS_Bomb("invalid -column type");
break;
}
for (i=3;i<s_arg[i_arg].n_items;i++) {
if (!(ptr=strchr(s_arg[i_arg].list[i], '=')))
SDDS_Bomb("invalid -column syntax");
*ptr++ = 0;
switch (match_string(s_arg[i_arg].list[i], header_elements, HEADERELEMENTS, 0)) {
case HEADER_UNITS:
SDDS_CopyString(&columnValues[columns-1].units, ptr);
break;
case HEADER_DESCRIPTION:
SDDS_CopyString(&columnValues[columns-1].description, ptr);
break;
case HEADER_SYMBOL:
SDDS_CopyString(&columnValues[columns-1].symbol, ptr);
break;
case HEADER_COUNT:
if (sscanf(ptr, "%ld", &count)!=1 ||
count<=0)
SDDS_Bomb("invalid column count value");
sprintf(buffer, "%s", columnValues[columns-1].name);
sprintf(buffer2, "%s1", buffer);
free(columnValues[columns-1].name);
SDDS_CopyString(&columnValues[columns-1].name, buffer2);
break;
default:
SDDS_Bomb("invalid -column syntax");
break;
}
}
for (i=2;i<=count;i++) {
columns++;
columnValues = trealloc(columnValues, sizeof(*columnValues)*(columns));
sprintf(buffer2, "%s%ld", buffer, i);
SDDS_CopyString(&columnValues[columns-1].name, buffer2);
columnValues[columns-1].elements = 0;
columnValues[columns-1].values = NULL;
columnValues[columns-1].stringValues = NULL;
columnValues[columns-1].units = NULL;
columnValues[columns-1].description = NULL;
columnValues[columns-1].symbol = NULL;
columnValues[columns-1].type = columnValues[columns-2].type;
if (columnValues[columns-2].units)
SDDS_CopyString(&columnValues[columns-1].units, columnValues[columns-2].units);
if (columnValues[columns-2].description)
SDDS_CopyString(&columnValues[columns-1].description, columnValues[columns-2].description);
if (columnValues[columns-2].symbol)
SDDS_CopyString(&columnValues[columns-1].symbol, columnValues[columns-2].symbol);
}
break;
case SET_SKIPCOLUMN:
if (s_arg[i_arg].n_items!=2)
SDDS_Bomb("invalid -skipcolumn syntax");
count=1;
columns++;
columnValues = trealloc(columnValues, sizeof(*columnValues)*(columns));
columnValues[columns-1].name = NULL;
columnValues[columns-1].elements = 0;
columnValues[columns-1].values = NULL;
columnValues[columns-1].stringValues = NULL;
columnValues[columns-1].units = NULL;
columnValues[columns-1].description = NULL;
columnValues[columns-1].symbol = NULL;
columnValues[columns-1].skip = 1;
switch (match_string(s_arg[i_arg].list[1], type_name, DATATYPES, 0)) {
case TYPE_SHORT:
columnValues[columns-1].type = SDDS_SHORT;
break;
case TYPE_LONG:
columnValues[columns-1].type = SDDS_LONG;
break;
case TYPE_FLOAT:
columnValues[columns-1].type = SDDS_FLOAT;
break;
case TYPE_DOUBLE:
columnValues[columns-1].type = SDDS_DOUBLE;
break;
case TYPE_STRING:
columnValues[columns-1].type = SDDS_STRING;
break;
case TYPE_CHARACTER:
columnValues[columns-1].type = SDDS_CHARACTER;
break;
default:
SDDS_Bomb("invalid -skipcolumn type");
break;
}
break;
case SET_PIPE:
if (!processPipeOption(s_arg[i_arg].list+1, s_arg[i_arg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case SET_NOWARNINGS:
if (s_arg[i_arg].n_items!=1)
SDDS_Bomb("invalid -nowarnings syntax");
noWarnings = 1;
break;
case SET_SKIPLINES:
if (s_arg[i_arg].n_items!=2 ||
sscanf(s_arg[i_arg].list[1], "%ld", &skiplines)!=1 ||
skiplines<=0)
SDDS_Bomb("invalid -skiplines syntax");
break;
default:
fprintf(stderr, "error: unknown switch: %s\n", s_arg[i_arg].list[0]);
exit(1);
break;
}
} else {
if (input == NULL) {
input = s_arg[i_arg].list[0];
} else if (output == NULL) {
output = s_arg[i_arg].list[0];
} else {
fprintf(stderr, "too many filenames");
exit(1);
}
}
}
processFilenames("plaindata2sdds", &input, &output, pipeFlags, noWarnings, &tmpfile_used);
if (!columns && !parameters)
SDDS_Bomb("you must specify one of the -column or the -parameter options");
if (skiplines && inputBinary)
SDDS_Bomb("-skiplines does not work with binary input files");
if (!input) {
if (inputBinary) {
#if defined(_WIN32)
if (_setmode(_fileno(stdin), _O_BINARY) == -1) {
fprintf(stderr, "error: unable to set stdin to binary mode\n");
exit(1);
}
#endif
}
fileID = stdin;
} else {
if (!fexists(input)) {
fprintf(stderr, "input file not found\n");
exit(1);
}
if (inputBinary) {
fileID = fopen(input, "rb");
} else {
fileID = fopen(input, "r");
}
}
if (fileID == NULL) {
fprintf(stderr, "unable to open input file for reading\n");
exit(1);
}
if (!SDDS_InitializeOutput(&SDDS_dataset, binary?SDDS_BINARY:SDDS_ASCII,
1, NULL, NULL, output))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
SDDS_dataset.layout.data_mode.column_major = columnMajorOrder;
if (parameters) {
parameterIndex = tmalloc(sizeof(*parameterIndex)*parameters);
}
if (columns) {
columnIndex = tmalloc(sizeof(*columnIndex)*columns);
}
for (i=0; i<parameters; i++) {
if ((parameterIndex[i] =
SDDS_DefineParameter(&SDDS_dataset, parameterValues[i].name, parameterValues[i].symbol,
parameterValues[i].units, parameterValues[i].description, NULL,
parameterValues[i].type, 0))<0) {
sprintf(s, "Problem defining parameter %s.", parameterValues[i].name);
SDDS_SetError(s);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
for (i=0; i<columns; i++) {
if (columnValues[i].skip)
continue;
if ((columnIndex[i] =
SDDS_DefineColumn(&SDDS_dataset, columnValues[i].name, columnValues[i].symbol, \
columnValues[i].units, columnValues[i].description, NULL,
columnValues[i].type, 0))<0) {
sprintf(s, "Problem defining column %s.", columnValues[i].name);
SDDS_SetError(s);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
if (!SDDS_WriteLayout(&SDDS_dataset))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!SDDS_StartPage(&SDDS_dataset, initRows))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
row = par = col = page = 0;
while (inputBinary) {
row = par = col = 0;
if (fread(&rows,sizeof(rows),1,fileID) != 1) {
if (page == 0) {
SDDS_SetError("Unable to read number of rows");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
} else {
if (!SDDS_Terminate(&SDDS_dataset))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (k=0;k<columns;k++) {
if (columnValues[k].type == SDDS_STRING) {
SDDS_FreeStringArray(columnValues[k].stringValues,columnValues[k].elements);
} else {
free(columnValues[k].values);
}
}
return 0;
}
}
page++;
for (par=0;par<parameters;par++) {
switch (parameterValues[par].type) {
case SDDS_SHORT:
if (fread(&shortValue,2,1,fileID) != 1) {
SDDS_SetError("Unable to read short parameter");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_SetParameters(&SDDS_dataset,SDDS_SET_BY_INDEX|SDDS_PASS_BY_VALUE,par,shortValue,-1))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
break;
case SDDS_LONG:
if (fread(&longValue,4,1,fileID) != 1) {
SDDS_SetError("Unable to read long parameter");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_SetParameters(&SDDS_dataset,SDDS_SET_BY_INDEX|SDDS_PASS_BY_VALUE,par,longValue,-1))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
break;
case SDDS_FLOAT:
if (fread(&floatValue,4,1,fileID) != 1) {
SDDS_SetError("Unable to read float parameter");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_SetParameters(&SDDS_dataset,SDDS_SET_BY_INDEX|SDDS_PASS_BY_VALUE,par,floatValue,-1))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
break;
case SDDS_DOUBLE:
if (fread(&doubleValue,8,1,fileID) != 1) {
SDDS_SetError("Unable to read double parameter");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_SetParameters(&SDDS_dataset,SDDS_SET_BY_INDEX|SDDS_PASS_BY_VALUE,par,doubleValue,-1))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
break;
case SDDS_STRING:
if (fread(&size,4,1,fileID) != 1) {
SDDS_SetError("Unable to read string parameter");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (size>SDDS_MAXLINE-1)
SDDS_Bomb("String is too long");
if (size > 0) {
if (fread(&stringValue,size,1,fileID) != 1) {
SDDS_SetError("Unable to read string parameter");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
stringValue[size] = 0;
} else {
strcpy(stringValue,"");
}
if (!SDDS_SetParameters(&SDDS_dataset,SDDS_SET_BY_INDEX|SDDS_PASS_BY_VALUE,par,stringValue,-1))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
break;
case SDDS_CHARACTER:
if (fread(&characterValue,1,1,fileID) != 1) {
SDDS_SetError("Unable to read character parameter");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_SetParameters(&SDDS_dataset,SDDS_SET_BY_INDEX|SDDS_PASS_BY_VALUE,par,characterValue,-1))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
break;
}
}
for (i=0;i<columns;i++) {
if (rows > columnValues[i].elements) {
if (columnValues[i].type == SDDS_STRING) {
columnValues[i].stringValues = AllocateColumnStringData(columnValues[i].stringValues, rows, columnValues[i].elements);
} else {
columnValues[i].values = AllocateColumnData(columnValues[i].type, columnValues[i].values, rows);
}
columnValues[i].elements = rows;
}
}
if (columnOrder) {
for (col=0;col<columns;col++) {
switch (columnValues[col].type) {
case SDDS_SHORT:
for (i=0;i<rows;i++) {
if (fread((short*)(columnValues[col].values)+i,2,1,fileID) != 1) {
SDDS_SetError("Unable to read short column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
break;
case SDDS_LONG:
for (i=0;i<rows;i++) {
if (fread((int32_t*)(columnValues[col].values)+i,4,1,fileID) != 1) {
SDDS_SetError("Unable to read long column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
break;
case SDDS_FLOAT:
for (i=0;i<rows;i++) {
if (fread((float*)(columnValues[col].values)+i,4,1,fileID) != 1) {
SDDS_SetError("Unable to read float column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
break;
case SDDS_DOUBLE:
for (i=0;i<rows;i++) {
if (fread((double*)(columnValues[col].values)+i,8,1,fileID) != 1) {
SDDS_SetError("Unable to read double double");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
break;
case SDDS_STRING:
for (i=0;i<rows;i++) {
if (fread(&size,4,1,fileID) != 1) {
SDDS_SetError("Unable to read string column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (size>SDDS_MAXLINE-1)
SDDS_Bomb("String is too long");
columnValues[col].stringValues[i] = malloc(size + 1);
if (size > 0) {
if (fread(columnValues[col].stringValues[i],size,1,fileID) != 1) {
SDDS_SetError("Unable to read string column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
columnValues[col].stringValues[i][size] = 0;
} else {
strcpy(columnValues[col].stringValues[i],"");
}
}
break;
case SDDS_CHARACTER:
for (i=0;i<rows;i++) {
if (fread((char*)(columnValues[col].values)+i,1,1,fileID) != 1) {
SDDS_SetError("Unable to read character column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
break;
}
}
} else {
for (i=0;i<rows;i++) {
for (col=0;col<columns;col++) {
switch (columnValues[col].type) {
case SDDS_SHORT:
if (fread((short*)(columnValues[col].values)+i,2,1,fileID) != 1) {
SDDS_SetError("Unable to read short column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
break;
case SDDS_LONG:
if (fread((int32_t*)(columnValues[col].values)+i,4,1,fileID) != 1) {
SDDS_SetError("Unable to read long column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
break;
case SDDS_FLOAT:
if (fread((float*)(columnValues[col].values)+i,4,1,fileID) != 1) {
SDDS_SetError("Unable to read float column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
break;
case SDDS_DOUBLE:
if (fread(((double*)(columnValues[col].values)+i),8,1,fileID) != 1) {
SDDS_SetError("Unable to read double column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
break;
case SDDS_STRING:
if (fread(&size,4,1,fileID) != 1) {
SDDS_SetError("Unable to read string column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (size>SDDS_MAXLINE-1)
SDDS_Bomb("String is too long");
columnValues[col].stringValues[i] = malloc(size + 1);
if (size > 0) {
if (fread(columnValues[col].stringValues[i],size,1,fileID) != 1) {
SDDS_SetError("Unable to read string column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
columnValues[col].stringValues[i][size] = 0;
} else {
strcpy(columnValues[col].stringValues[i],"");
}
break;
case SDDS_CHARACTER:
if (fread(((char*)(columnValues[col].values)+i),1,1,fileID) != 1) {
SDDS_SetError("Unable to read character column");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
break;
}
}
}
}
if (rows > maxRows) {
if (!SDDS_LengthenTable(&SDDS_dataset, rows - maxRows)) {
SDDS_SetError("Unable to lengthen table");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
maxRows = rows;
}
j=n;
for (i=0;i<columns;i++) {
if (columnValues[i].skip)
continue;
if (columnValues[i].type == SDDS_STRING) {
SetColumnData(columnValues[i].type, &SDDS_dataset, columnValues[i].stringValues, rows, n);
} else {
SetColumnData(columnValues[i].type, &SDDS_dataset, columnValues[i].values, rows, n);
}
n++;
}
if (!SDDS_WritePage(&SDDS_dataset)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
maxRows = 10000;
if (!SDDS_StartPage(&SDDS_dataset, initRows))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
row = par = col = n = 0;
rows = -1;
ptr = NULL;
ptr = SDDS_Malloc(sizeof(*ptr)*(ptrSize=2048));
/* ptr2 = NULL;
ptr2 = SDDS_Malloc(sizeof(*ptr)*(ptrSize=2048));*/
ptr[0] = 0;
while (1) {
if (readline) {
while (skiplines > 0) {
fgets(ptr, ptrSize, fileID);
skiplines--;
}
if (!fgetsSkipCommentsResize(&ptr,&ptrSize,fileID, '!'))
break;
commentFound=0;
if (checkComment) {
for (i=0; i<strlen(commentCharacters); i++) {
if (ptr[0] == commentCharacters[i]) {
commentFound=1;
}
}
}
if (commentFound == 1) {
continue;
}
if (ptr[strlen(ptr)-1] == '\n')
ptr[strlen(ptr)-1] = 0;
strcpy(temp,ptr);
/*skip empty lines*/
if (getToken(temp,data,10240,separator,whitespace)<0)
continue;
} else {
readline = 1;
}
if (par < parameters) {
switch (parameterValues[par].type) {
case SDDS_SHORT:
if (sscanf(ptr,"%hd",&shortValue) != 1) {
SDDS_SetError("Invalid short parameter");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_SetParameters(&SDDS_dataset,SDDS_SET_BY_INDEX|SDDS_PASS_BY_VALUE,par,shortValue,-1))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
break;
case SDDS_LONG:
if (sscanf(ptr,"%" SCNd32 ,&longValue) != 1) {
SDDS_SetError("Invalid long parameter");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_SetParameters(&SDDS_dataset,SDDS_SET_BY_INDEX|SDDS_PASS_BY_VALUE,par,longValue,-1))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
break;
case SDDS_FLOAT:
ConvertDNotationToENotation(ptr);
if (sscanf(ptr,"%f",&floatValue) != 1) {
SDDS_SetError("Invalid float parameter");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_SetParameters(&SDDS_dataset,SDDS_SET_BY_INDEX|SDDS_PASS_BY_VALUE,par,floatValue,-1))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
break;
case SDDS_DOUBLE:
ConvertDNotationToENotation(ptr);
if (sscanf(ptr,"%lf",&doubleValue) != 1) {
SDDS_SetError("Invalid double parameter");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_SetParameters(&SDDS_dataset,SDDS_SET_BY_INDEX|SDDS_PASS_BY_VALUE,par,doubleValue,-1))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
break;
case SDDS_STRING:
SDDS_GetToken(ptr,stringValue,SDDS_MAXLINE);
SDDS_InterpretEscapes(stringValue);
if (!SDDS_SetParameters(&SDDS_dataset,SDDS_SET_BY_INDEX|SDDS_PASS_BY_VALUE,par,stringValue,-1))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
break;
case SDDS_CHARACTER:
SDDS_InterpretEscapes(ptr);
characterValue = ptr[0];
if (!SDDS_SetParameters(&SDDS_dataset,SDDS_SET_BY_INDEX|SDDS_PASS_BY_VALUE,par,characterValue,-1))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
break;
}
par++;
} else if ((rows == -1) && (!noRowCount)) {
if (sscanf(ptr,"%ld",&rows) != 1) {
SDDS_SetError("Invalid row count");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
} else if ((columns > 0) && ((row < rows) || (noRowCount))) {
if (columnOrder) {
if (noRowCount) {
cp_str(&ptr2,ptr);
rows = 0;
while (getToken(ptr2,data,10240,separator,whitespace) >= 0) {
rows++;
}
free(ptr2);
}
if (rows > columnValues[col].elements) {
if (columnValues[col].type == SDDS_STRING) {
columnValues[col].stringValues = AllocateColumnStringData(columnValues[col].stringValues, rows, columnValues[col].elements);
} else {
columnValues[col].values = AllocateColumnData(columnValues[col].type, columnValues[col].values, rows);
}
columnValues[col].elements = rows;
}
switch (columnValues[col].type) {
case SDDS_SHORT:
for (row=0;row<rows;row++) {
if (getToken(ptr,data,10240,separator,whitespace) < 0) {
SDDS_SetError("Invalid short column element");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (sscanf(data,"%hd",((short*)(columnValues[col].values)+row)) != 1) {
SDDS_SetError("Invalid short column element");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
break;
case SDDS_LONG:
for (row=0;row<rows;row++) {
if (getToken(ptr,data,10240,separator,whitespace) < 0) {
SDDS_SetError("Invalid long column element");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (sscanf(data,"%" SCNd32,((int32_t*)(columnValues[col].values)+row)) != 1) {
SDDS_SetError("Invalid long column element");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
break;
case SDDS_FLOAT:
for (row=0;row<rows;row++) {
if (getToken(ptr,data,10240,separator,whitespace) < 0) {
SDDS_SetError("Invalid float column element");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
ConvertDNotationToENotation(data);
if (sscanf(data,"%f",((float*)(columnValues[col].values)+row)) != 1) {
SDDS_SetError("Invalid float column element");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
break;
case SDDS_DOUBLE:
for (row=0;row<rows;row++) {
if (getToken(ptr,data,10240,separator,whitespace) < 0) {
SDDS_SetError("Invalid double column element");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
ConvertDNotationToENotation(data);
if (sscanf(data,"%lf",((double*)(columnValues[col].values)+row)) != 1) {
SDDS_SetError("Invalid double column element");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
break;
case SDDS_STRING:
for (row=0;row<rows;row++) {
if (getToken(ptr,data,10240,separator,whitespace) < 0) {
SDDS_SetError("Invalid string column element");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
SDDS_InterpretEscapes(data);
columnValues[col].stringValues[row] = malloc(strlen(data) + 1);
strcpy(columnValues[col].stringValues[row],data);
}
break;
case SDDS_CHARACTER:
for (row=0;row<rows;row++) {
if (getToken(ptr,data,10240,separator,whitespace) < 0) {
SDDS_SetError("Invalid character column element");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
SDDS_InterpretEscapes(data);
*((char*)(columnValues[col].values)+row) = data[0];
}
break;
}
if (rows > maxRows) {
if (!SDDS_LengthenTable(&SDDS_dataset, rows - maxRows)) {
SDDS_SetError("Unable to lengthen table");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
maxRows = rows;
}
if (columnValues[col].skip == 0) {
if (columnValues[col].type == SDDS_STRING) {
SetColumnData(columnValues[col].type, &SDDS_dataset, columnValues[col].stringValues, rows, col);
} else {
SetColumnData(columnValues[col].type, &SDDS_dataset, columnValues[col].values, rows, col);
}
n++;
}
col++;
row = 0;
} else {
if (noRowCount) {
if (row == 0) {
rows = 3;
} else if (row == rows - 1) {
rows = rows + 3;
for (i=0;i<columns;i++) {
if (rows > columnValues[i].elements) {
if (columnValues[i].type == SDDS_STRING) {
columnValues[i].stringValues = AllocateColumnStringData(columnValues[i].stringValues, rows, columnValues[i].elements);
} else {
columnValues[i].values = AllocateColumnData(columnValues[i].type, columnValues[i].values, rows);
}
}
columnValues[i].elements = rows;
}
}
}
if (row == 0)
for (i=0;i<columns;i++) {
if (rows > columnValues[i].elements) {
if (columnValues[i].type == SDDS_STRING) {
columnValues[i].stringValues = AllocateColumnStringData(columnValues[i].stringValues, rows, columnValues[i].elements);
} else {
columnValues[i].values = AllocateColumnData(columnValues[i].type, columnValues[i].values, rows);
}
}
columnValues[i].elements = rows;
}
if (noRowCount) {
cp_str(&ptr2,ptr);
i = 0;
while (getToken(ptr2,data,10240,separator,whitespace) >= 0)
i++;
free(ptr2);
if ((i != columns) && (parameters>0 && i==1)) {
if (row > 0) {
if (row > maxRows) {
if (!SDDS_LengthenTable(&SDDS_dataset, row - maxRows)) {
SDDS_SetError("Unable to lengthen table");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
maxRows = row;
}
n=0;
for (j=0;j<columns;j++) {
if (columnValues[j].skip)
continue;
if (columnValues[j].type == SDDS_STRING) {
SetColumnData(columnValues[j].type, &SDDS_dataset, columnValues[j].stringValues, row, n);
} else {
SetColumnData(columnValues[j].type, &SDDS_dataset, columnValues[j].values, row, n);
}
n++;
}
if (!SDDS_WritePage(&SDDS_dataset))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
maxRows = 10000;
if (!SDDS_StartPage(&SDDS_dataset, initRows))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
row = par = col = 0;
rows = -1;
}
readline = 0;
continue;
}
}
for (i=0;i<columns;i++) {
if (getToken(ptr,data,10240,separator,whitespace) < 0) {
if (!fillin) {
SDDS_SetError("Invalid column element");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
} else {
switch (columnValues[i].type) {
case SDDS_SHORT:
case SDDS_LONG:
case SDDS_FLOAT:
case SDDS_DOUBLE:
data[0]='0';
data[1]='\0';
break;
case SDDS_STRING:
case SDDS_CHARACTER:
data[0]='\0';
break;
}
}
}
switch (columnValues[i].type) {
case SDDS_SHORT:
if (sscanf(data,"%hd",((short*)(columnValues[i].values)+row)) != 1) {
if (recover) {
abort=1;
row--;
} else {
SDDS_SetError("Invalid short column element");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
break;
case SDDS_LONG:
if (sscanf(data,"%" SCNd32,((int32_t*)(columnValues[i].values)+row)) != 1) {
if (recover) {
abort=1;
row--;
} else {
SDDS_SetError("Invalid long column element");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
break;
case SDDS_FLOAT:
ConvertDNotationToENotation(data);
if (sscanf(data,"%f",((float*)(columnValues[i].values)+row)) != 1) {
if (recover) {
abort=1;
row--;
} else {
SDDS_SetError("Invalid float column element");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
break;
case SDDS_DOUBLE:
ConvertDNotationToENotation(data);
if (sscanf(data,"%lf",((double*)(columnValues[i].values)+row)) != 1) {
if (recover) {
abort=1;
row--;
} else {
SDDS_SetError("Invalid double column element");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
break;
case SDDS_STRING:
SDDS_InterpretEscapes(data);
columnValues[i].stringValues[row] = malloc(strlen(data) + 1);
strcpy(columnValues[i].stringValues[row],data);
break;
case SDDS_CHARACTER:
SDDS_InterpretEscapes(data);
*((char*)(columnValues[i].values)+row) = data[0];
break;
}
if (recover && abort) {
break;
}
}
row++;
if ((row == rows) && (!noRowCount)) {
if (rows > maxRows) {
if (!SDDS_LengthenTable(&SDDS_dataset, rows - maxRows)) {
SDDS_SetError("Unable to lengthen table");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
maxRows = rows;
}
n=0;
for (i=0;i<columns;i++) {
if (columnValues[i].skip)
continue;
if (columnValues[i].type == SDDS_STRING) {
SetColumnData(columnValues[i].type, &SDDS_dataset, columnValues[i].stringValues, rows, n);
} else {
SetColumnData(columnValues[i].type, &SDDS_dataset, columnValues[i].values, rows, n);
}
n++;
}
}
}
}
if ((par == parameters) &&
(((!noRowCount) && (rows != -1)) || (noRowCount)) &&
(((columnOrder) && (col == columns)) || ((columns > 0) && (row == rows)) || (columns == 0))) {
if (!SDDS_WritePage(&SDDS_dataset)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
maxRows = 10000;
if (!SDDS_StartPage(&SDDS_dataset, initRows))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
row = par = col = 0;
rows = -1;
}
ptr[0] = 0;
}
if (noRowCount) {
if (row > 0) {
if (row > maxRows) {
if (!SDDS_LengthenTable(&SDDS_dataset, row - maxRows)) {
SDDS_SetError("Unable to lengthen table");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
maxRows = row;
}
n=0;
for (j=0;j<columns;j++) {
if (columnValues[j].skip)
continue;
if (columnValues[j].type == SDDS_STRING) {
SetColumnData(columnValues[j].type, &SDDS_dataset, columnValues[j].stringValues, row, n);
} else {
SetColumnData(columnValues[j].type, &SDDS_dataset, columnValues[j].values, row, n);
}
n++;
}
if (!SDDS_WritePage(&SDDS_dataset)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
maxRows = 10000;
}
}
for (i=0;i<columns;i++) {
if (columnValues[i].type == SDDS_STRING) {
for (j=0;j<columnValues[i].elements;j++) {
free(columnValues[i].stringValues[j]);
}
free(columnValues[i].stringValues);
} else {
free(columnValues[i].values);
}
if (columnValues[i].name) free(columnValues[i].name);
if (columnValues[i].units) free(columnValues[i].units);
if (columnValues[i].description) free(columnValues[i].description);
if (columnValues[i].symbol) free(columnValues[i].symbol);
}
for (i=0;i<parameters;i++) {
free(parameterValues[i].name);
if (parameterValues[i].units) free(parameterValues[i].units);
if (parameterValues[i].description) free(parameterValues[i].description);
if (parameterValues[i].symbol) free(parameterValues[i].symbol);
}
if (columnValues) free(columnValues);
if (parameterValues) free(parameterValues);
if (columnIndex) free(columnIndex);
if (parameterIndex) free(parameterIndex);
if (ptr) free(ptr);
if (!SDDS_Terminate(&SDDS_dataset))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
free_scanargs(&s_arg,argc);
return 0;
}
int main(int argc, char **argv)
{
SCANNED_ARG *scanned;
SDDS_TABLE inputPage, *copiedPage, outputPage;
long copiedPages;
char *inputfile, *outputfile;
char **column, **excludeColumn=NULL;
int32_t columns;
long excludeColumns;
char *indParameterName;
char **copyColumn;
int32_t copyColumns;
long verbose;
long slopeErrors;
long iArg,i;
double *indVar;
char *indVarUnits;
char **intColumn,**slopeColumn,**slopeSigmaColumn;
char *Units,*slopeUnits;
double *depVar;
long order;
double *coef,*coefsigma,*weight,*diff,chi;
long iCol,iRow,iPage;
long rows;
double *slope, slope2, slopeAve, slopeSigma;
unsigned long pipeFlags, majorOrderFlag;
long tmpfile_used, noWarnings;
long generateIndex;
short columnMajorOrder=-1;
copiedPage = NULL;
slopeSigmaColumn = NULL;
slopeUnits = Units = indVarUnits = NULL;
rows = 0;
slope = NULL;
slope2 = 0;
coef = coefsigma = weight = diff = slope = NULL;
argc = scanargs(&scanned, argc, argv);
if (argc == 1)
bomb(NULL, USAGE);
inputfile = outputfile = NULL;
columns = excludeColumns = 0;
column = excludeColumn = NULL;
indParameterName = NULL;
verbose = 0;
slopeErrors = 0;
pipeFlags = 0;
tmpfile_used=0;
noWarnings=0;
for (iArg = 1; iArg<argc; iArg++) {
if (scanned[iArg].arg_type == OPTION) {
delete_chars(scanned[iArg].list[0], "_");
switch (match_string(scanned[iArg].list[0], commandline_option, COMMANDLINE_OPTIONS, UNIQUE_MATCH)) {
case CLO_MAJOR_ORDER:
majorOrderFlag=0;
scanned[iArg].n_items--;
if (scanned[iArg].n_items>0 &&
(!scanItemList(&majorOrderFlag, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"row", -1, NULL, 0, SDDS_ROW_MAJOR_ORDER,
"column", -1, NULL, 0, SDDS_COLUMN_MAJOR_ORDER,
NULL)))
SDDS_Bomb("invalid -majorOrder syntax/values");
if (majorOrderFlag&SDDS_COLUMN_MAJOR_ORDER)
columnMajorOrder=1;
else if (majorOrderFlag&SDDS_ROW_MAJOR_ORDER)
columnMajorOrder=0;
break;
case CLO_INDEPENDENT_PARAMETER:
if (!(indParameterName = scanned[iArg].list[1]))
SDDS_Bomb("no string given for option -independentVariable");
break;
case CLO_COLUMNS:
if (columns)
SDDS_Bomb("only one -columns option may be given");
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -columns syntax");
column = tmalloc(sizeof(*column)*(columns = scanned[iArg].n_items-1));
for (i = 0; i<columns; i++)
column[i] = scanned[iArg].list[i+1];
break;
case CLO_EXCLUDE:
if (excludeColumns)
SDDS_Bomb("only one -excludecolumns option may be given");
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -excludecolumns syntax");
excludeColumn = tmalloc(sizeof(*excludeColumn)*(excludeColumns = scanned[iArg].n_items-1));
for (i = 0; i<excludeColumns; i++)
excludeColumn[i] = scanned[iArg].list[i+1];
break;
case CLO_VERBOSE:
verbose = 1;
break;
case CLO_PIPE:
if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case CLO_SLOPE_ERRORS:
slopeErrors = 1;
break;
default:
SDDS_Bomb("unrecognized option given");
break;
}
}
else {
if (!inputfile)
inputfile = scanned[iArg].list[0];
else if (!outputfile)
outputfile = scanned[iArg].list[0];
else
SDDS_Bomb("too many filenames given");
}
}
processFilenames("sddsvslopes", &inputfile, &outputfile, pipeFlags, noWarnings, &tmpfile_used);
if (!indParameterName)
SDDS_Bomb("independentVariable not given");
if (!excludeColumns) {
excludeColumn = defaultExcludedColumn;
excludeColumns = DEFAULT_EXCLUDED_COLUMNS;
}
if (verbose)
fprintf(stderr,"Reading file %s.\n",inputfile);
SDDS_InitializeInput(&inputPage, inputfile);
copiedPages = 0;
while (SDDS_ReadTable(&inputPage)>0) {
if (!copiedPages) {
copiedPage = (SDDS_TABLE*)malloc(sizeof(SDDS_TABLE));
rows = SDDS_CountRowsOfInterest(&inputPage);
}
else {
copiedPage = (SDDS_TABLE*)realloc(copiedPage, (copiedPages+1)*sizeof(SDDS_TABLE));
}
if (!SDDS_InitializeCopy(&copiedPage[copiedPages], &inputPage, NULL, "m"))
SDDS_PrintErrors(stderr, SDDS_EXIT_PrintErrors|SDDS_VERBOSE_PrintErrors);
if (!SDDS_CopyTable(&copiedPage[copiedPages], &inputPage))
SDDS_PrintErrors(stderr, SDDS_EXIT_PrintErrors|SDDS_VERBOSE_PrintErrors);
copiedPages++;
}
if (copiedPages<2) {
fprintf(stderr,"Insufficient data (i.e. number of data pages) to fit a straight line.\n");
exit(1);
}
switch(SDDS_CheckColumn(&inputPage, "Rootname", NULL, SDDS_STRING, NULL)) {
case SDDS_CHECK_WRONGUNITS:
case SDDS_CHECK_OKAY:
break;
default:
fprintf(stderr,"Something wrong with column %s.\n", "Rootname");
exit(1);
}
switch(SDDS_CheckColumn(&inputPage, "Index", NULL, SDDS_LONG, NULL)) {
case SDDS_CHECK_WRONGUNITS:
case SDDS_CHECK_OKAY:
generateIndex = 0;
break;
case SDDS_CHECK_NONEXISTENT:
generateIndex = 1;
break;
default:
fprintf(stderr,"Something wrong with column %s.\n", "Rootname");
exit(1);
}
/****************\
* make array of independent variable
\**************/
indVar = (double*)malloc(sizeof(*indVar)*copiedPages);
switch(SDDS_CheckParameter(&inputPage, indParameterName, NULL, SDDS_DOUBLE, NULL)) {
case SDDS_CHECK_WRONGUNITS:
case SDDS_CHECK_OKAY:
break;
default:
fprintf(stderr,"Something wrong with parameter %s.\n", indParameterName);
exit(1);
}
for (iPage = 0; iPage<copiedPages; iPage++) {
if (!SDDS_GetParameter(&copiedPage[iPage],indParameterName,&indVar[iPage]))
SDDS_PrintErrors(stderr, SDDS_EXIT_PrintErrors|SDDS_VERBOSE_PrintErrors);
}
if (!SDDS_GetParameterInformation(&inputPage, "units", &indVarUnits, SDDS_GET_BY_NAME, indParameterName))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!indVarUnits) {
indVarUnits = (char *) malloc(sizeof(*indVarUnits));
indVarUnits[0] = 0;
}
/************************************\
* get columns of interest. use set_multicolumn_flags to simply
* return new values for array column.
\*************************************/
if (!set_multicolumn_flags(&inputPage, &column, &columns, excludeColumn, excludeColumns)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
/************************************\
* make column names for the output
\*************************************/
intColumn = (char**)malloc((sizeof(char*)*columns));
slopeColumn = (char**)malloc((sizeof(char*)*columns));
if (slopeErrors)
slopeSigmaColumn = (char**)malloc((sizeof(char*)*columns));
for (i=0; i<columns; i++) {
intColumn[i] = (char*)malloc((sizeof(char)*(strlen(column[i])+strlen("Intercept")+1)));
strcat(strcpy(intColumn[i], column[i]), "Intercept");
slopeColumn[i] = (char*)malloc((sizeof(char)*(strlen(column[i])+strlen("Slope")+1)));
strcat(strcpy(slopeColumn[i], column[i]), "Slope");
if (slopeErrors) {
slopeSigmaColumn[i] = (char*)malloc((sizeof(char)*(strlen(column[i])+strlen("SlopeSigma")+1)));
strcat(strcpy(slopeSigmaColumn[i], column[i]), "SlopeSigma");
}
}
/************************************\
* Write layout for output file
\*************************************/
if (verbose)
fprintf(stderr,"Opening file %s.\n",outputfile);
if(!SDDS_InitializeOutput(&outputPage,SDDS_BINARY,1,
"lsf of sddsvexperiment",NULL,outputfile) ||
0>SDDS_DefineParameter(&outputPage, "InputFile", "InputFile", NULL,
"InputFile", NULL, SDDS_STRING, 0) ||
0>SDDS_DefineParameter(&outputPage, "IndependentVariable", "IndependentVariable", NULL,
"IndependentVariable", NULL, SDDS_STRING, 0) ||
(0>SDDS_DefineColumn(&outputPage,"Index",NULL,NULL,"Rootname index",NULL,SDDS_LONG,0))||
(0>SDDS_DefineColumn(&outputPage,"Rootname",NULL,NULL,NULL,NULL,SDDS_STRING,0)))
SDDS_PrintErrors(stderr,SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (columnMajorOrder!=-1)
outputPage.layout.data_mode.column_major = columnMajorOrder;
else
outputPage.layout.data_mode.column_major = inputPage.layout.data_mode.column_major;
for (iCol=0; iCol<columns; iCol++) {
if (!SDDS_GetColumnInformation(&inputPage, "units", &Units, SDDS_GET_BY_NAME,column[iCol]))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!Units) {
Units = (char*) malloc(sizeof(*Units));
Units[0] = 0;
}
if (0>SDDS_DefineColumn(&outputPage, intColumn[iCol], NULL, Units, NULL, NULL, SDDS_DOUBLE,0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
/* units for slopes columns */
if (strlen(indVarUnits) && strlen(Units) ) {
slopeUnits = (char*)malloc(sizeof(*slopeUnits)*(strlen(Units)+strlen(indVarUnits)+2));
strcat( strcat( strcpy(slopeUnits, Units), "/"), indVarUnits);
}
if (strlen(indVarUnits) && !strlen(Units) ) {
slopeUnits = (char*)malloc(sizeof(*slopeUnits)*(strlen(indVarUnits)+2));
strcat( strcpy( slopeUnits, "1/"), indVarUnits);
}
if (!strlen(indVarUnits) && strlen(Units) ) {
slopeUnits = (char*)malloc(sizeof(*slopeUnits)*(strlen(Units)+2));
strcpy( slopeUnits, indVarUnits);
}
if (!strlen(indVarUnits) && !strlen(Units) ) {
slopeUnits = (char*)malloc(sizeof(*slopeUnits));
strcpy( slopeUnits, "");
}
if (0>SDDS_DefineColumn(&outputPage, slopeColumn[iCol], NULL, slopeUnits, NULL, NULL, SDDS_DOUBLE,0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (slopeErrors) {
if (0>SDDS_DefineColumn(&outputPage, slopeSigmaColumn[iCol], NULL, slopeUnits, NULL, NULL, SDDS_DOUBLE,0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
free(slopeUnits);
}
if(!SDDS_WriteLayout(&outputPage) ||
!SDDS_StartTable(&outputPage,rows) ||
!SDDS_SetParameters(&outputPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE,
"InputFile",inputfile?inputfile:"pipe",NULL) ||
!SDDS_SetParameters(&outputPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE, 0,
"IndependentVariable", indParameterName, NULL) )
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
/************************************\
* Copy columns to output file (usually columns Index and Rootname)
\*************************************/
copyColumns = DEFAULT_COPY_COLUMNS;
copyColumn = defaultCopyColumn;
if (!set_multicolumn_flags(&inputPage, ©Column, ©Columns, NULL, 0)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
exit(1);
}
if(!SDDS_CopyColumns(&outputPage, &inputPage))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
depVar = (double*)malloc(sizeof(*depVar)*copiedPages);
weight = (double*)malloc(sizeof(*weight)*copiedPages);
diff = (double*)malloc(sizeof(*weight)*copiedPages);
order=1;
coef = (double*)malloc(sizeof(*coef)*(order+1));
coefsigma = (double*)malloc(sizeof(*coefsigma)*(order+1));
if(slopeErrors)
slope = (double*)malloc(sizeof(*slope)*copiedPages);
for (iCol=0; iCol<columns; iCol++) {
for (iPage=0; iPage<copiedPages; iPage++)
weight[iPage]=1;
if (verbose)
fprintf(stderr,"Doing column %s.\n", column[iCol]);
for (iRow=0; iRow<rows; iRow++) {
for (iPage=0; iPage<copiedPages; iPage++) {
if (!SDDS_GetValue(&copiedPage[iPage], column[iCol], iRow, &depVar[iPage]))
SDDS_PrintErrors(stderr, SDDS_EXIT_PrintErrors|SDDS_VERBOSE_PrintErrors);
}
if (!(lsfn(indVar, depVar, weight, copiedPages, order, coef, coefsigma, &chi, diff))){
fprintf(stderr,"Problem with call to lsfn\n.");
exit(1);
}
if (generateIndex) {
if (!SDDS_SetRowValues(&outputPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE, iRow,
"Index",iRow,
intColumn[iCol], coef[0],
slopeColumn[iCol], coef[1], NULL))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
else {
if (!SDDS_SetRowValues(&outputPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE, iRow,
intColumn[iCol], coef[0],
slopeColumn[iCol], coef[1], NULL))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if(slopeErrors){
/* recalculate the slope with a subset of points */
slopeAve= slope2 = 0;
for (iPage=0; iPage<copiedPages; iPage++) {
weight[iPage] = 1e10;
if(iPage)
weight[iPage-1] = 1;
if (!(lsfn(indVar, depVar, weight, copiedPages, order, coef, coefsigma, &chi, diff))){
fprintf(stderr,"Problem with call to lsfn\n.");
exit(1);
}
slope[iPage] = coef[1];
slopeAve += slope[iPage];
slope2 += sqr(slope[iPage]);
}
slopeSigma = sqrt(slope2/copiedPages - sqr(slopeAve/copiedPages));
if (!SDDS_SetRowValues(&outputPage, SDDS_SET_BY_NAME|SDDS_PASS_BY_VALUE, iRow,
slopeSigmaColumn[iCol], slopeSigma, NULL))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
}
if( !SDDS_WriteTable(&outputPage)|| SDDS_Terminate(&inputPage) )
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (iPage=0; iPage<copiedPages; iPage++) {
if( !SDDS_Terminate(&copiedPage[iPage]) )
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if( SDDS_Terminate(&outputPage) )
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (tmpfile_used && !replaceFileAndBackUp(inputfile, outputfile))
exit(1);
SDDS_FreeStringArray(column, columns);
free(column);
SDDS_FreeStringArray(intColumn, columns);
SDDS_FreeStringArray(slopeColumn, columns);
free(intColumn);
free(slopeColumn);
if (slopeErrors) {
SDDS_FreeStringArray(slopeSigmaColumn, columns);
free(slopeSigmaColumn);
}
free(copiedPage);
free(indVar);
free(depVar);
if (weight)
free(weight);
if (diff)
free(diff);
if (slope)
free(slope);
if (coef) free(coef);
if (coefsigma) free(coefsigma);
if (Units) free(Units);
if (indVarUnits) free(indVarUnits);
free_scanargs(&scanned, argc);
return(0);
}
int main(int argc, char **argv)
{
int iArg;
char *input, *output,*meanPar, *sigmaPar, *maxPar, *minPar;
long i, j, mainInputOpened, haltonID=0, requireInput=0;
unsigned long pipeFlags;
SCANNED_ARG *scanned;
SDDS_DATASET SDDSin, SDDSout, *SDDSptr;
long randomNumberSeed = 0;
SEQ_REQUEST *seqRequest;
long samples, values, seqRequests, randomizationGroups=0;
double *sample, *IVValue, *CDFValue;
char msgBuffer[1000];
RANDOMIZED_ORDER *randomizationData = NULL;
long verbose, optimalHalton=0;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&scanned, argc, argv);
if (argc<2) {
fprintf(stderr, "%s%s%s\n", USAGE1, USAGE2, USAGE3);
return(1);
}
seqRequest = NULL;
seqRequests = 0;
output = input = NULL;
pipeFlags = 0;
samples = values = 0;
sample = IVValue = CDFValue = NULL;
verbose = 0;
maxPar = minPar = meanPar = sigmaPar = NULL;
for (iArg=1; iArg<argc; iArg++) {
if (scanned[iArg].arg_type==OPTION) {
/* process options here */
switch (match_string(scanned[iArg].list[0], option, CLO_OPTIONS, 0)) {
case CLO_COLUMNS:
if (scanned[iArg].n_items<3)
SDDS_Bomb("invalid -columns syntax");
if (!(seqRequest = SDDS_Realloc(seqRequest, sizeof(*seqRequest)*(seqRequests+1))))
SDDS_Bomb("memory allocation failure");
scanned[iArg].n_items -= 1;
memset(seqRequest+seqRequests, 0, sizeof(*seqRequest));
/*remove following pointer initialization because memset already initialize them */
seqRequest[seqRequests].randomizationGroup = -1;
seqRequest[seqRequests].factor = 1;
seqRequest[seqRequests].offset = 0;
if (!scanItemList(&seqRequest[seqRequests].flags,
scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"datafile", SDDS_STRING,
&seqRequest[seqRequests].dataFileName, 1, SEQ_DATAFILE,
"independentvariable", SDDS_STRING,
&seqRequest[seqRequests].indepName, 1, SEQ_INDEPNAME,
"cdf", SDDS_STRING,
&seqRequest[seqRequests].CDFName, 1, SEQ_CDFNAME,
"df", SDDS_STRING,
&seqRequest[seqRequests].DFName, 1, SEQ_DFNAME,
"output", SDDS_STRING,
&seqRequest[seqRequests].outputName, 1, SEQ_OUTPUTNAME,
"units", SDDS_STRING,
&seqRequest[seqRequests].units, 1, SEQ_UNITSGIVEN,
"haltonradix", SDDS_LONG,
&seqRequest[seqRequests].haltonRadix, 1, SEQ_HALTONRADIX,
"haltonoffset", SDDS_LONG,
&seqRequest[seqRequests].haltonOffset, 1, SEQ_HALTONOFFSET,
"randomize", -1, NULL, 0, SEQ_RANDOMIZE,
"group", SDDS_LONG,
&seqRequest[seqRequests].randomizationGroup, 1, SEQ_RANDOMGROUP,
"factor", SDDS_DOUBLE,
&seqRequest[seqRequests].factor, 1, 0,
"offset", SDDS_DOUBLE,
&seqRequest[seqRequests].offset, 1, 0,
NULL) ||
bitsSet(seqRequest[seqRequests].flags&(SEQ_INDEPNAME+SEQ_CDFNAME+SEQ_DFNAME))!=2)
SDDS_Bomb("invalid -columns syntax");
if (seqRequest[seqRequests].flags&SEQ_RANDOMGROUP &&
seqRequest[seqRequests].randomizationGroup<=0)
SDDS_Bomb("use a positive integer for the randomization group ID");
if (seqRequest[seqRequests].flags&SEQ_CDFNAME &&
seqRequest[seqRequests].flags&SEQ_DFNAME)
SDDS_Bomb("give df or cdf for -columns, not both");
if (seqRequest[seqRequests].flags&SEQ_HALTONRADIX &&
!is_prime(seqRequest[seqRequests].haltonRadix))
SDDS_Bomb("halton radix must be a prime number");
seqRequests ++;
scanned[iArg].n_items += 1;
break;
case CLO_GAUSSIAN:
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -gaussian syntax");
if (!(seqRequest = SDDS_Realloc(seqRequest, sizeof(*seqRequest)*(seqRequests+1))))
SDDS_Bomb("memory allocation failure");
memset(seqRequest+seqRequests, 0, sizeof(*seqRequest));
scanned[iArg].n_items -= 1;
seqRequest[seqRequests].randomizationGroup = -1;
seqRequest[seqRequests].mean = 0;
seqRequest[seqRequests].sigma = 1;
if (!scanItemList(&seqRequest[seqRequests].flags,
scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"columnName", SDDS_STRING, &seqRequest[seqRequests].outputName, 1, SEQ_OUTPUTNAME,
"meanValue", SDDS_STRING, &meanPar, 1, 0,
"sigmaValue", SDDS_STRING, &sigmaPar, 1, 0,
"units", SDDS_STRING, &seqRequest[seqRequests].units, 1, SEQ_UNITSGIVEN,
NULL))
SDDS_Bomb("invalid -gaussian syntax");
seqRequest[seqRequests].flags |= SEQ_DIRECT_GAUSSIAN;
if (!(seqRequest[seqRequests].flags&SEQ_OUTPUTNAME) || !(seqRequest[seqRequests].outputName))
SDDS_Bomb("columnName is not provided for gaussian distribution/");
if (meanPar) {
if (wild_match(meanPar, "@*"))
SDDS_CopyString(&seqRequest[seqRequests].meanPar, meanPar+1);
else if (!get_double(&seqRequest[seqRequests].mean, meanPar))
SDDS_Bomb("Invalid value given for mean value of -gaussian distribution.");
free(meanPar);
meanPar = NULL;
}
if (sigmaPar) {
if (wild_match(sigmaPar, "@*"))
SDDS_CopyString(&seqRequest[seqRequests].sigmaPar, sigmaPar+1);
else if (!get_double(&seqRequest[seqRequests].sigma, sigmaPar))
SDDS_Bomb("Invalid value given for sigma value of -gaussian distribution.");
free(sigmaPar);
sigmaPar = NULL;
}
seqRequests ++;
scanned[iArg].n_items += 1;
break;
case CLO_UNIFORM:
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -uniform syntax");
if (!(seqRequest = SDDS_Realloc(seqRequest, sizeof(*seqRequest)*(seqRequests+1))))
SDDS_Bomb("memory allocation failure");
memset(seqRequest+seqRequests, 0, sizeof(*seqRequest));
scanned[iArg].n_items -= 1;
memset(seqRequest+seqRequests, 0, sizeof(*seqRequest));
seqRequest[seqRequests].randomizationGroup = -1;
seqRequest[seqRequests].min = 0;
seqRequest[seqRequests].max = 1;
if (!scanItemList(&seqRequest[seqRequests].flags,
scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"columnName", SDDS_STRING, &seqRequest[seqRequests].outputName, 1, SEQ_OUTPUTNAME,
"minimumValue", SDDS_STRING, &minPar, 1, 0,
"maximumValue", SDDS_STRING, &maxPar, 1, 0,
"units", SDDS_STRING, &seqRequest[seqRequests].units, 1, SEQ_UNITSGIVEN,
NULL))
SDDS_Bomb("invalid -uniform syntax");
seqRequest[seqRequests].flags |= SEQ_DIRECT_UNIFORM;
if (!(seqRequest[seqRequests].flags&SEQ_OUTPUTNAME) || !(seqRequest[seqRequests].outputName))
SDDS_Bomb("columnName is not provided for uniform distribution/");
if (minPar) {
if (wild_match(minPar, "@*"))
SDDS_CopyString(&seqRequest[seqRequests].minPar, minPar+1);
else if (!get_double(&seqRequest[seqRequests].min, minPar))
SDDS_Bomb("Invalid value given for minimum value of -uniform distribution.");
free(minPar);
minPar = NULL;
}
if (maxPar) {
if (wild_match(maxPar, "@*"))
SDDS_CopyString(&seqRequest[seqRequests].maxPar, maxPar+1);
else if (!get_double(&seqRequest[seqRequests].max, maxPar))
SDDS_Bomb("Invalid value given for maximum value of -uniform distribution.");
free(maxPar);
maxPar = NULL;
}
seqRequests ++;
scanned[iArg].n_items += 1;
break;
case CLO_POISSON:
if (scanned[iArg].n_items<2)
SDDS_Bomb("invalid -poisson syntax");
if (!(seqRequest = SDDS_Realloc(seqRequest, sizeof(*seqRequest)*(seqRequests+1))))
SDDS_Bomb("memory allocation failure");
memset(seqRequest+seqRequests, 0, sizeof(*seqRequest));
scanned[iArg].n_items -= 1;
memset(seqRequest+seqRequests, 0, sizeof(*seqRequest));
seqRequest[seqRequests].randomizationGroup = -1;
seqRequest[seqRequests].mean = 1;
if (!scanItemList(&seqRequest[seqRequests].flags,
scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"columnName", SDDS_STRING, &seqRequest[seqRequests].outputName, 1, SEQ_OUTPUTNAME,
"meanValue", SDDS_STRING, &meanPar, 1, 0,
"units", SDDS_STRING, &seqRequest[seqRequests].units, 1, SEQ_UNITSGIVEN,
NULL))
SDDS_Bomb("invalid -poisson syntax");
seqRequest[seqRequests].flags |= SEQ_DIRECT_POISSON;
if (!(seqRequest[seqRequests].flags&SEQ_OUTPUTNAME) || !(seqRequest[seqRequests].outputName))
SDDS_Bomb("columnName is not provided for uniform distribution/");
if (meanPar) {
if (wild_match(meanPar, "@*"))
SDDS_CopyString(&seqRequest[seqRequests].meanPar, meanPar+1);
else if (!get_double(&seqRequest[seqRequests].mean, meanPar))
SDDS_Bomb("Invalid value given for mean value of -poisson distribution.");
free(meanPar);
meanPar = NULL;
}
seqRequests ++;
scanned[iArg].n_items += 1;
break;
case CLO_SAMPLES:
if (scanned[iArg].n_items!=2 ||
sscanf(scanned[iArg].list[1], "%ld", &samples)!=1 ||
samples<=0)
SDDS_Bomb("invalid -samples syntax");
break;
case CLO_SEED:
if (scanned[iArg].n_items!=2 ||
sscanf(scanned[iArg].list[1], "%ld", &randomNumberSeed)!=1 ||
randomNumberSeed<=0)
SDDS_Bomb("invalid -seed syntax");
break;
case CLO_PIPE:
if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case CLO_VERBOSE:
verbose = 1;
break;
case CLO_OPTIMAL_HALTON:
optimalHalton = 1;
break;
default:
fprintf(stderr, "error: unknown/ambiguous option: %s\n",
scanned[iArg].list[0]);
exit(1);
break;
}
}
else {
if (!input)
input = scanned[iArg].list[0];
else if (!output)
output = scanned[iArg].list[0];
else
SDDS_Bomb("too many filenames seen");
}
}
if (!seqRequests)
SDDS_Bomb("give one or more -columns options");
if (samples<1)
SDDS_Bomb("-samples option not given");
for (i=0; i<seqRequests; i++) {
if (!(seqRequest[i].flags&
(SEQ_DATAFILE|SEQ_DIRECT_GAUSSIAN|SEQ_DIRECT_UNIFORM|SEQ_DIRECT_POISSON)))
break;
}
if (i==seqRequests) {
/* all columns options have either their own input files or else use
* one of the "direct" distributions. Hence, we don't expect an input
* file.
*/
if (!input)
pipeFlags |= USE_STDIN; /* not really, but fakes out processFilenames */
if (input && !output) {
output = input;
input = NULL;
pipeFlags |= USE_STDIN;
if (fexists(output)) {
sprintf(msgBuffer, "%s exists already (sddssampledist)", output);
SDDS_Bomb(msgBuffer);
}
}
}
processFilenames("sddssampledist", &input, &output, pipeFlags, 0, NULL);
if (!SDDS_InitializeOutput(&SDDSout, SDDS_BINARY, 0, NULL, NULL, output))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (verbose)
fprintf(stderr, "Initialized output file %s\n", output);
/* open and check input files */
for (i=mainInputOpened=0; i<seqRequests; i++) {
if (seqRequest[i].flags&SEQ_DIRECT_GAUSSIAN) {
if (seqRequest[i].meanPar || seqRequest[i].sigmaPar) {
if (!mainInputOpened) {
if (!SDDS_InitializeInput(&SDDSin, input) ||
!SDDS_TransferAllParameterDefinitions(&SDDSout, &SDDSin, 0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
mainInputOpened = 1;
}
requireInput = 1;
SDDSptr = &SDDSin;
if ((seqRequest[i].meanPar &&
SDDS_CheckParameter(SDDSptr, seqRequest[i].meanPar, NULL, SDDS_ANY_NUMERIC_TYPE, stderr)!=SDDS_CHECK_OK) ||
(seqRequest[i].sigmaPar &&
SDDS_CheckParameter(SDDSptr, seqRequest[i].sigmaPar, NULL, SDDS_ANY_NUMERIC_TYPE, stderr)!=SDDS_CHECK_OK)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
}
if (!SDDS_DefineSimpleColumn(&SDDSout, seqRequest[i].outputName, NULL, SDDS_DOUBLE))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
} else if (seqRequest[i].flags&SEQ_DIRECT_UNIFORM) {
if (seqRequest[i].minPar || seqRequest[i].maxPar) {
if (!mainInputOpened) {
if (!SDDS_InitializeInput(&SDDSin, input) ||
!SDDS_TransferAllParameterDefinitions(&SDDSout, &SDDSin, 0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
mainInputOpened = 1;
}
requireInput = 1;
SDDSptr = &SDDSin;
if ((seqRequest[i].minPar &&
SDDS_CheckParameter(SDDSptr, seqRequest[i].minPar, NULL, SDDS_ANY_NUMERIC_TYPE, stderr)!=SDDS_CHECK_OK) ||
(seqRequest[i].maxPar &&
SDDS_CheckParameter(SDDSptr, seqRequest[i].maxPar, NULL, SDDS_ANY_NUMERIC_TYPE, stderr)!=SDDS_CHECK_OK)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
}
if (!SDDS_DefineSimpleColumn(&SDDSout, seqRequest[i].outputName, NULL, SDDS_DOUBLE))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
} else if (seqRequest[i].flags&SEQ_DIRECT_POISSON) {
if (seqRequest[i].meanPar) {
if (!mainInputOpened) {
if (!SDDS_InitializeInput(&SDDSin, input) ||
!SDDS_TransferAllParameterDefinitions(&SDDSout, &SDDSin, 0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
mainInputOpened = 1;
}
requireInput = 1;
SDDSptr = &SDDSin;
if ( SDDS_CheckParameter(SDDSptr, seqRequest[i].meanPar, NULL, SDDS_ANY_NUMERIC_TYPE, stderr)!=SDDS_CHECK_OK) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
}
if (!SDDS_DefineSimpleColumn(&SDDSout, seqRequest[i].outputName, NULL, SDDS_LONG))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
} else {
if (seqRequest[i].flags&SEQ_RANDOMIZE) {
long newGroupID=0;
/* define randomization groups */
if (seqRequest[i].flags&SEQ_RANDOMGROUP) {
newGroupID = seqRequest[i].randomizationGroup;
for (j=0; j<randomizationGroups; j++)
if (randomizationData[j].group==newGroupID) {
newGroupID = 0;
break;
}
} else {
seqRequest[i].randomizationGroup = newGroupID = -(i+1);
}
if (newGroupID!=0) {
if (!(randomizationData =
SDDS_Realloc(randomizationData,
sizeof(*randomizationData)*(randomizationGroups+1))))
SDDS_Bomb("memory allocation failure");
randomizationData[randomizationGroups].group = newGroupID;
randomizationData[randomizationGroups].order = NULL;
randomizationGroups ++;
}
}
if (seqRequest[i].flags&SEQ_DATAFILE) {
if (!SDDS_InitializeInput(&seqRequest[i].SDDSin,
seqRequest[i].dataFileName))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
SDDSptr = &seqRequest[i].SDDSin;
} else {
if (!mainInputOpened) {
if (!SDDS_InitializeInput(&SDDSin, input) ||
!SDDS_TransferAllParameterDefinitions(&SDDSout, &SDDSin, 0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
mainInputOpened = 1;
}
requireInput = 1;
SDDSptr = &SDDSin;
}
if (SDDS_CheckColumn(SDDSptr,
seqRequest[i].indepName, NULL, SDDS_ANY_NUMERIC_TYPE,
stderr)!=SDDS_CHECK_OK ||
((seqRequest[i].flags&SEQ_CDFNAME) &&
SDDS_CheckColumn(SDDSptr, seqRequest[i].CDFName, NULL, SDDS_ANY_NUMERIC_TYPE,
stderr)!=SDDS_CHECK_OK) ||
((seqRequest[i].flags&SEQ_DFNAME) &&
SDDS_CheckColumn(SDDSptr, seqRequest[i].DFName, NULL, SDDS_ANY_NUMERIC_TYPE,
stderr)!=SDDS_CHECK_OK) ||
!SDDS_TransferColumnDefinition(&SDDSout, SDDSptr,
seqRequest[i].indepName,
seqRequest[i].outputName)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
}
if (seqRequest[i].flags&SEQ_UNITSGIVEN &&
!SDDS_ChangeColumnInformation(&SDDSout, "units", seqRequest[i].units,
SDDS_SET_BY_NAME,
seqRequest[i].outputName))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (verbose)
fprintf(stderr, "Initialized input files\n");
if (!SDDS_WriteLayout(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (randomNumberSeed==0) {
randomNumberSeed = (long)time((time_t *) NULL);
randomNumberSeed = 2*(randomNumberSeed/2) + 1;
#if defined(_WIN32)
random_1(-labs(randomNumberSeed));
#else
random_1(-FABS(randomNumberSeed));
#endif
} else
random_1(-randomNumberSeed);
if (!((sample = calloc(sizeof(*sample), samples))))
SDDS_Bomb("memory allocation failure");
while (1) {
if (verbose)
fprintf(stderr, "Beginning page loop\n");
if (input && SDDS_ReadPage(&SDDSin)<=0)
break;
for (i=0; i<seqRequests; i++) {
if (seqRequest[i].flags&SEQ_DATAFILE &&
SDDS_ReadPage(&seqRequest[i].SDDSin)<=0)
break;
}
if (i!=seqRequests)
break;
if (!SDDS_StartPage(&SDDSout, samples) ||
(input && !SDDS_CopyParameters(&SDDSout, &SDDSin)))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (verbose)
fprintf(stderr, "Defining randomization tables\n");
/* define randomization tables */
for (i=0; i<randomizationGroups; i++) {
if (!(randomizationData[i].order=
SDDS_Malloc(sizeof(*randomizationData[i].order)*samples)))
SDDS_Bomb("memory allocation failure");
for (j=0; j<samples; j++)
randomizationData[i].order[j] = j;
randomizeOrder((char*)randomizationData[i].order,
sizeof(*randomizationData[i].order), samples, 0,
random_1);
}
if (verbose)
fprintf(stderr, "Beginning loop over sequence requests\n");
for (i=0; i<seqRequests; i++) {
if (verbose)
fprintf(stderr, "Processing sequence request %ld\n", i);
if (seqRequest[i].flags&SEQ_DIRECT_GAUSSIAN) {
if ((seqRequest[i].meanPar && !SDDS_GetParameterAsDouble(&SDDSin, seqRequest[i].meanPar, &seqRequest[i].mean)) ||
(seqRequest[i].sigmaPar && !SDDS_GetParameterAsDouble(&SDDSin, seqRequest[i].sigmaPar, &seqRequest[i].sigma)))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (j=0; j<samples; j++)
sample[j] = gauss_rn_lim(seqRequest[i].mean, seqRequest[i].sigma, -1, random_1);
} else if (seqRequest[i].flags&SEQ_DIRECT_UNIFORM) {
if ((seqRequest[i].minPar && !SDDS_GetParameterAsDouble(&SDDSin, seqRequest[i].minPar, &seqRequest[i].min)) ||
(seqRequest[i].maxPar && !SDDS_GetParameterAsDouble(&SDDSin, seqRequest[i].maxPar, &seqRequest[i].max)))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (j=0; j<samples; j++)
sample[j] = seqRequest[i].min + (seqRequest[i].max - seqRequest[i].min) * random_1(1);
} else if (seqRequest[i].flags&SEQ_DIRECT_POISSON) {
double *pos_x, *pos_cdf, CDF;
long pos_points, code;
pos_x = pos_cdf = NULL;
if ((seqRequest[i].meanPar && !SDDS_GetParameterAsDouble(&SDDSin, seqRequest[i].meanPar, &seqRequest[i].mean)))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
pos_points = CreatePoissonDistributionTable(&pos_x, &pos_cdf, seqRequest[i].mean);
for (j=0; j<samples; j++) {
CDF = random_1(1);
sample[j]= (int)(interp(pos_x, pos_cdf, pos_points, CDF, 0, 1, &code));
/* fprintf(stderr, "%d, cdf=%f, sample=%f\n", j, CDF, sample[j]);*/
}
free(pos_x);
free(pos_cdf);
} else {
if (input && !(seqRequest[i].flags&SEQ_DATAFILE))
SDDSptr = &SDDSin;
else
SDDSptr = &seqRequest[i].SDDSin;
if ((values = SDDS_CountRowsOfInterest(SDDSptr))) {
if (!(IVValue
= SDDS_GetColumnInDoubles(SDDSptr, seqRequest[i].indepName)) ||
(seqRequest[i].flags&SEQ_CDFNAME &&
!(CDFValue
= SDDS_GetColumnInDoubles(SDDSptr, seqRequest[i].CDFName))) ||
(seqRequest[i].flags&SEQ_DFNAME &&
!(CDFValue
= SDDS_GetColumnInDoubles(SDDSptr, seqRequest[i].DFName))))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
} else {
sprintf(msgBuffer, "empty page for file %s\n",
seqRequest[i].flags&SEQ_DATAFILE?
seqRequest[i].dataFileName:input);
SDDS_Bomb(msgBuffer);
}
if (verbose)
fprintf(stderr, "Checking and converting CDF/DF values\n");
/* check/convert CDF/DF values */
for (j=1; j<values; j++) {
if (IVValue[j-1]>IVValue[j]) {
sprintf(msgBuffer, "random variate values not monotonically increasing for %s",
seqRequest[i].flags&SEQ_DATAFILE?
seqRequest[i].dataFileName:input);
SDDS_Bomb(msgBuffer);
}
if (seqRequest[i].flags&SEQ_DFNAME)
/* convert DF to CDF */
CDFValue[j] += CDFValue[j-1];
if (CDFValue[j] < CDFValue[j-1]) {
sprintf(msgBuffer, "CDF values decreasing for %s",
seqRequest[i].flags&SEQ_DATAFILE?
seqRequest[i].dataFileName:input);
SDDS_Bomb(msgBuffer);
}
}
if (verbose)
fprintf(stderr, "Normalizing CDF\n");
/* normalize the CDF */
if (CDFValue[values-1]<=0) {
sprintf(msgBuffer, "CDF not valid for %s\n",
seqRequest[i].dataFileName);
SDDS_Bomb(msgBuffer);
}
for (j=0; j<values; j++)
CDFValue[j] /= CDFValue[values-1];
if (seqRequest[i].flags&SEQ_HALTONRADIX) {
if (verbose)
fprintf(stderr, "Starting halton sequence, offset=%" PRId32 "\n", seqRequest[i].haltonOffset);
if (!optimalHalton)
haltonID = startHaltonSequence(&seqRequest[i].haltonRadix, 0.5);
else
haltonID = startModHaltonSequence(&seqRequest[i].haltonRadix, 0);
while (seqRequest[i].haltonOffset-- >0) {
if (!optimalHalton)
nextHaltonSequencePoint(haltonID);
else
nextModHaltonSequencePoint(haltonID);
}
}
if (verbose)
fprintf(stderr, "Generating samples\n");
for (j=0; j<samples; j++) {
double CDF;
long code;
while (1) {
if (seqRequest[i].flags&SEQ_HALTONRADIX) {
if (!optimalHalton)
CDF = nextHaltonSequencePoint(haltonID);
else
CDF = nextModHaltonSequencePoint(haltonID);
}
else
CDF = random_1(1);
if (CDF<=CDFValue[values-1] && CDF>=CDFValue[0])
break;
}
sample[j]
= seqRequest[i].factor*interp(IVValue, CDFValue, values, CDF, 0, 1, &code)
+ seqRequest[i].offset;
}
if (seqRequest[i].flags&SEQ_RANDOMIZE) {
long k, l;
double *sample1;
if (verbose)
fprintf(stderr, "Randomizing order of values\n");
if (!(sample1 = malloc(sizeof(*sample1)*samples)))
SDDS_Bomb("memory allocation failure");
for (l=0; l<randomizationGroups; l++)
if (randomizationData[l].group==seqRequest[i].randomizationGroup)
break;
if (l==randomizationGroups)
SDDS_Bomb("problem with construction of randomization groups!");
for (k=0; k<samples; k++)
sample1[k] = sample[randomizationData[l].order[k]];
free(sample);
sample = sample1;
}
free(IVValue);
free(CDFValue);
}
if (verbose)
fprintf(stderr, "Setting SDDS column values\n");
if (!SDDS_SetColumnFromDoubles(&SDDSout, SDDS_SET_BY_NAME,
sample, samples,
seqRequest[i].outputName?
seqRequest[i].outputName:
seqRequest[i].indepName))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (verbose)
fprintf(stderr, "Writing data page\n");
if (!SDDS_WritePage(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!requireInput)
break;
}
if (verbose)
fprintf(stderr, "Exited read loop\n");
free(sample);
if ((input && !SDDS_Terminate(&SDDSin)) || !SDDS_Terminate(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (i=0; i<seqRequests; i++) {
if (seqRequest[i].dataFileName) free(seqRequest[i].dataFileName);
if (seqRequest[i].indepName) free(seqRequest[i].indepName);
if (seqRequest[i].outputName) free(seqRequest[i].outputName);
if (seqRequest[i].DFName) free(seqRequest[i].DFName);
if (seqRequest[i].CDFName) free(seqRequest[i].CDFName);
if (seqRequest[i].units) free(seqRequest[i].units);
if (seqRequest[i].meanPar) free(seqRequest[i].meanPar);
if (seqRequest[i].sigmaPar) free(seqRequest[i].sigmaPar);
if (seqRequest[i].minPar) free(seqRequest[i].minPar);
if (seqRequest[i].maxPar) free(seqRequest[i].maxPar);
if (seqRequest[i].flags&SEQ_DATAFILE && !SDDS_Terminate(&(seqRequest[i].SDDSin)))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
free(seqRequest);
for (i=0; i<randomizationGroups; i++)
free(randomizationData[i].order);
if (randomizationData) free(randomizationData);
free_scanargs(&scanned, argc);
return(0);
}
int main(int argc, char **argv)
{
POLYNOMIAL *poly;
long nPoly, iPoly, row, rows, iInput;
int iArg;
char *input, *output;
unsigned long pipeFlags;
SCANNED_ARG *scanned;
SDDS_DATASET SDDSin, SDDSout;
double *outputData ;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&scanned, argc, argv);
if (argc<3)
bomb(NULL, USAGE);
outputData = NULL;
input = output = NULL;
pipeFlags = 0;
poly = NULL;
nPoly = 0;
for (iArg=1; iArg<argc; iArg++) {
if (scanned[iArg].arg_type==OPTION) {
/* process options here */
switch (match_string(scanned[iArg].list[0], option, CLO_OPTIONS, 0)) {
case CLO_EVALUATE:
if (!(poly = SDDS_Realloc(poly, sizeof(*poly)*(nPoly+1))) ||
!(poly[nPoly].inputColumn
=SDDS_Malloc(sizeof(*(poly[nPoly].inputColumn))*5)) ||
!(poly[nPoly].powerColumn
=SDDS_Malloc(sizeof(*(poly[nPoly].powerColumn))*5)))
SDDS_Bomb("memory allocation failure");
scanned[iArg].n_items -= 1;
if (!scanItemList(&poly[nPoly].flags, scanned[iArg].list+1, &scanned[iArg].n_items, 0,
"filename", SDDS_STRING, &(poly[nPoly].filename), 1, POLY_FILE_SEEN,
"output", SDDS_STRING, &(poly[nPoly].outputColumn), 1, POLY_OUTPUT_SEEN,
"coefficients", SDDS_STRING, &(poly[nPoly].coefColumn), 1, POLY_COEF_SEEN,
"input0", SDDS_STRING, poly[nPoly].inputColumn+0, 1, POLY_IN0_SEEN,
"power0", SDDS_STRING, poly[nPoly].powerColumn+0, 1, POLY_OUT0_SEEN,
"input1", SDDS_STRING, poly[nPoly].inputColumn+1, 1, POLY_IN1_SEEN,
"power1", SDDS_STRING, poly[nPoly].powerColumn+1, 1, POLY_OUT1_SEEN,
"input2", SDDS_STRING, poly[nPoly].inputColumn+2, 1, POLY_IN2_SEEN,
"power2", SDDS_STRING, poly[nPoly].powerColumn+2, 1, POLY_OUT2_SEEN,
"input3", SDDS_STRING, poly[nPoly].inputColumn+3, 1, POLY_IN3_SEEN,
"power3", SDDS_STRING, poly[nPoly].powerColumn+3, 1, POLY_OUT3_SEEN,
"input4", SDDS_STRING, poly[nPoly].inputColumn+4, 1, POLY_IN4_SEEN,
"power4", SDDS_STRING, poly[nPoly].powerColumn+4, 1, POLY_OUT4_SEEN,
NULL) ||
!(poly[nPoly].flags&POLY_FILE_SEEN) || !(poly[nPoly].flags&POLY_OUTPUT_SEEN) ||
!(poly[nPoly].flags&POLY_COEF_SEEN) || !(poly[nPoly].flags&POLY_IN0_SEEN) ||
!(poly[nPoly].flags&POLY_OUT0_SEEN))
SDDS_Bomb("invalid -evaluate syntax");
nPoly++;
break;
case CLO_PIPE:
if (!processPipeOption(scanned[iArg].list+1, scanned[iArg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
default:
fprintf(stderr, "error: unknown/ambiguous option: %s\n",
scanned[iArg].list[0]);
exit(1);
break;
}
}
else {
if (!input)
input = scanned[iArg].list[0];
else if (!output)
output = scanned[iArg].list[0];
else
SDDS_Bomb("too many filenames seen");
}
}
processFilenames("sddspoly", &input, &output, pipeFlags, 0, NULL);
if (nPoly==0)
SDDS_Bomb("give at least one -evaluate option");
if (!SDDS_InitializeInput(&SDDSin, input))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!SDDS_InitializeCopy(&SDDSout, &SDDSin, output, "w"))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
for (iPoly=0; iPoly<nPoly; iPoly++)
initializePolynomial(&poly[iPoly], &SDDSin, &SDDSout);
if (!SDDS_WriteLayout(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
while (SDDS_ReadPage(&SDDSin)>0) {
rows = SDDS_CountRowsOfInterest(&SDDSin);
if (!SDDS_CopyPage(&SDDSout, &SDDSin))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!(outputData = SDDS_Realloc(outputData, sizeof(*outputData)*rows)))
SDDS_Bomb("memory allocation failure");
for (iPoly=0; iPoly<nPoly; iPoly++) {
for (iInput=0; iInput<poly[iPoly].nInputs; iInput++) {
if (!(poly[iPoly].inputData[iInput]=
SDDS_GetColumnInDoubles(&SDDSin, poly[iPoly].inputColumn[iInput])))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
for (row=0; row<rows; row++) {
for (iInput=0; iInput<poly[iPoly].nInputs; iInput++)
poly[iPoly].input[iInput] = poly[iPoly].inputData[iInput][row];
outputData[row] = evaluatePoly(poly[iPoly].coef, poly[iPoly].power,
poly[iPoly].nTerms,
poly[iPoly].input, poly[iPoly].nInputs);
}
if (!SDDS_SetColumn(&SDDSout, SDDS_SET_BY_NAME,
outputData, rows, poly[iPoly].outputColumn))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_WritePage(&SDDSout))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
free(outputData);
if (!SDDS_Terminate(&SDDSin)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (!SDDS_Terminate(&SDDSout)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
free_scanargs(&scanned,argc);
FreePolynormialMemory(poly,nPoly);
free(poly);
return 0;
}
int main(int argc, char **argv)
{
SDDS_DATASET SDDS_1, SDDS_2, SDDS_output;
long i, j, i_arg, rows1, rows2, reuse, reusePage, i1, i2;
SCANNED_ARG *s_arg;
char s[200], *ptr;
char **match_column, **equate_column;
long match_columns, equate_columns;
char *input1, *input2, *output;
long tmpfile_used, retval1, retval2;
long warnings, invert;
unsigned long pipeFlags;
KEYED_EQUIVALENT **keyGroup=NULL;
long keyGroups=0;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&s_arg, argc, argv);
if (argc<3)
bomb(NULL, USAGE);
input1 = input2 = output = NULL;
match_column = equate_column = NULL;
match_columns = equate_columns = reuse = reusePage = 0;
tmpfile_used = invert = 0;
warnings = 1;
pipeFlags = 0;
for (i_arg=1; i_arg<argc; i_arg++) {
if (s_arg[i_arg].arg_type==OPTION) {
delete_chars(s_arg[i_arg].list[0], "_");
switch (match_string(s_arg[i_arg].list[0], option, N_OPTIONS, 0)) {
case SET_MATCH_COLUMN:
if (s_arg[i_arg].n_items!=2)
SDDS_Bomb("invalid -match syntax");
if (match_columns!=0)
SDDS_Bomb("only one -match option may be given");
match_column = tmalloc(sizeof(*match_column)*2);
if ((ptr=strchr(s_arg[i_arg].list[1], '=')))
*ptr++ = 0;
else
ptr = s_arg[i_arg].list[1];
match_column[0] = s_arg[i_arg].list[1];
match_column[1] = ptr;
match_columns = 1;
break;
case SET_EQUATE_COLUMN:
if (s_arg[i_arg].n_items!=2)
SDDS_Bomb("invalid -equate syntax");
if (equate_columns!=0)
SDDS_Bomb("only one -equate option may be given");
equate_column = tmalloc(sizeof(*equate_column)*2);
if ((ptr=strchr(s_arg[i_arg].list[1], '=')))
*ptr++ = 0;
else
ptr = s_arg[i_arg].list[1];
equate_column[0] = s_arg[i_arg].list[1];
equate_column[1] = ptr;
equate_columns = 1;
break;
case SET_REUSE:
if (s_arg[i_arg].n_items==1)
reuse = 1;
else {
char *reuseOptions[2] = {"rows", "page"};
for (i=1; i<s_arg[i_arg].n_items; i++) {
switch (match_string(s_arg[i_arg].list[i], reuseOptions, 2, 0)) {
case 0:
reuse = 1;
break;
case 1:
reusePage = 1;
break;
default:
SDDS_Bomb("unknown reuse keyword");
break;
}
}
}
break;
case SET_INVERT:
invert = 1;
break;
case SET_NOWARNINGS:
warnings = 0;
break;
case SET_PIPE:
if (!processPipeOption(s_arg[i_arg].list+1, s_arg[i_arg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
default:
fprintf(stderr, "error: unknown switch: %s\n", s_arg[i_arg].list[0]);
SDDS_Bomb(NULL);
break;
}
}
else {
if (input1==NULL)
input1 = s_arg[i_arg].list[0];
else if (input2==NULL)
input2 = s_arg[i_arg].list[0];
else if (output==NULL)
output = s_arg[i_arg].list[0];
else
SDDS_Bomb("too many filenames");
}
}
if (pipeFlags&USE_STDIN && input1) {
if (output)
SDDS_Bomb("too many filenames (sddsxref)");
output = input2;
input2 = input1;
input1 = NULL;
}
processFilenames("sddsselect", &input1, &output, pipeFlags, !warnings, &tmpfile_used);
if (!input2)
SDDS_Bomb("second input file not specified (sddsxref)");
if (equate_columns && match_columns)
SDDS_Bomb("only one of -equate or -match may be given");
if (!equate_columns && !match_columns)
SDDS_Bomb("one of -equate or -match must be given");
if (!SDDS_InitializeInput(&SDDS_1, input1)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (!SDDS_InitializeInput(&SDDS_2, input2)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (match_columns) {
if ((j=SDDS_GetColumnIndex(&SDDS_1, match_column[0]))<0 || SDDS_GetColumnType(&SDDS_1, j)!=SDDS_STRING) {
sprintf(s, "error: column %s not found or not string type in file %s", match_column[0], input1?input1:"stdin");
SDDS_SetError(s);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if ((j=SDDS_GetColumnIndex(&SDDS_2, match_column[1]))<0 || SDDS_GetColumnType(&SDDS_2, j)!=SDDS_STRING) {
sprintf(s, "error: column %s not found or not string type in file %s", match_column[1], input2);
SDDS_SetError(s);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
if (equate_columns) {
if ((j=SDDS_GetColumnIndex(&SDDS_1, equate_column[0]))<0 || !SDDS_NUMERIC_TYPE(SDDS_GetColumnType(&SDDS_1, j))) {
sprintf(s, "error: column %s not found or not numeric type in file %s", equate_column[0], input1?input1:"stdin");
SDDS_SetError(s);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if ((j=SDDS_GetColumnIndex(&SDDS_2, equate_column[1]))<0 || !SDDS_NUMERIC_TYPE(SDDS_GetColumnType(&SDDS_2, j))) {
sprintf(s, "error: column %s not found or not numeric type in file %s", equate_column[1], input2);
SDDS_SetError(s);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
if (output && pipeFlags&USE_STDOUT)
SDDS_Bomb("too many filenames with -pipe option");
if (!output && !(pipeFlags&USE_STDOUT)) {
if (warnings)
fprintf(stderr, "warning: existing file %s will be replaced (sddsselect)\n", input1?input1:"stdin");
tmpfile_used = 1;
cp_str(&output, tmpname(NULL));
}
if (!SDDS_InitializeCopy(&SDDS_output, &SDDS_1, output, "w")) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (!SDDS_WriteLayout(&SDDS_output))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
while ((retval1=SDDS_ReadPage(&SDDS_1))>0) {
if (!reusePage) {
if ((retval2=SDDS_ReadPage(&SDDS_2))<=0) {
if (warnings)
fprintf(stderr, "warning: <input2> ends before <input1>\n");
if (invert) {
/* nothing to match, so everything would normally be thrown out */
if (!SDDS_CopyPage(&SDDS_output, &SDDS_1) || !SDDS_WritePage(&SDDS_output))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
continue;
}
else
/* nothing to match, so everything thrown out */
break;
}
}
else {
if (retval1==1 && (retval2=SDDS_ReadPage(&SDDS_2))<=0)
SDDS_Bomb("<input2> has no data");
SDDS_SetRowFlags(&SDDS_2, 1);
}
rows1 = SDDS_CountRowsOfInterest(&SDDS_1);
rows2 = SDDS_CountRowsOfInterest(&SDDS_2);
if (!SDDS_StartPage(&SDDS_output, rows1)) {
SDDS_SetError("Problem starting output page");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_CopyParameters(&SDDS_output, &SDDS_2) || !SDDS_CopyArrays(&SDDS_output, &SDDS_2)) {
SDDS_SetError("Problem copying parameter or array data from second input file");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (!SDDS_CopyParameters(&SDDS_output, &SDDS_1) || !SDDS_CopyArrays(&SDDS_output, &SDDS_1)) {
SDDS_SetError("Problem copying parameter or array data from first input file");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (match_columns) {
char **string1, **string2;
long matched;
string2 = NULL;
if (!(string1 = SDDS_GetColumn(&SDDS_1, match_column[0]))) {
fprintf(stderr, "Error: problem getting column %s from file %s\n",
match_column[0], input1?input1:"stdin");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (rows2 && !(string2 = SDDS_GetColumn(&SDDS_2, match_column[1]))) {
fprintf(stderr, "Error: problem getting column %s from file %s\n",
match_column[1], input2);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (rows2)
keyGroup = MakeSortedKeyGroups(&keyGroups, SDDS_STRING, string2, rows2);
for (i1=0; i1<rows1; i1++) {
if (!SDDS_CopyRowDirect(&SDDS_output, i1, &SDDS_1, i1)) {
sprintf(s, "Problem copying row %ld of first data set", i1);
SDDS_SetError(s);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
matched = 0;
if (rows2 &&(i2 = FindMatchingKeyGroup(keyGroup, keyGroups, SDDS_STRING, string1+i1, reuse))>=0) {
matched = 1;
}
if ((!matched && !invert) || (matched && invert)) {
if (!SDDS_AssertRowFlags(&SDDS_output, SDDS_INDEX_LIMITS, i1, i1, 0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
if (string1) {
for (i=0;i<rows1;i++)
free(string1[i]);
free(string1);
string1 = NULL;
}
if (string2) {
for (i=0;i<rows2;i++)
free(string2[i]);
free(string2);
string2 = NULL;
}
for (i=0;i<keyGroups;i++) {
if (keyGroup[i]) {
if (keyGroup[i]->equivalent)
free(keyGroup[i]->equivalent);
free(keyGroup[i]);
keyGroup[i] = NULL;
}
}
if (keyGroups) {
free(keyGroup);
keyGroup = NULL;
keyGroups = 0;
}
}
else if (equate_columns) {
double *value1, *value2;
long equated;
value2 = NULL;
if (!(value1 = SDDS_GetColumnInDoubles(&SDDS_1, equate_column[0]))) {
fprintf(stderr, "Error: problem getting column %s from file %s\n",
equate_column[0], input1?input1:"stdin");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (rows2 && !(value2 = SDDS_GetColumnInDoubles(&SDDS_2, equate_column[1]))) {
fprintf(stderr, "Error: problem getting column %s from file %s\n",
equate_column[1], input2);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
if (rows2)
keyGroup = MakeSortedKeyGroups(&keyGroups, SDDS_DOUBLE, value2, rows2);
for (i1=0; i1<rows1; i1++) {
if (!SDDS_CopyRowDirect(&SDDS_output, i1, &SDDS_1, i1)) {
sprintf(s, "Problem copying row %ld of first data set", i1);
SDDS_SetError(s);
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
equated = 0;
if (rows2 && (i2 = FindMatchingKeyGroup(keyGroup, keyGroups, SDDS_DOUBLE, value1+i1, reuse))>=0) {
equated = 1;
}
if ((!equated && !invert) || (equated && invert)) {
if (!SDDS_AssertRowFlags(&SDDS_output, SDDS_INDEX_LIMITS, i1, i1, 0))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
if (value1)
free(value1);
value1 = NULL;
if (rows2 && value2)
free(value2);
value2 = NULL;
for (i=0;i<keyGroups;i++) {
if (keyGroup[i]) {
if (keyGroup[i]->equivalent)
free(keyGroup[i]->equivalent);
free(keyGroup[i]);
keyGroup[i] = NULL;
}
}
if (keyGroups) {
free(keyGroup);
keyGroup = NULL;
keyGroups = 0;
}
}
if (!SDDS_WritePage(&SDDS_output)) {
SDDS_SetError("Problem writing data to output file");
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
}
if (!SDDS_Terminate(&SDDS_1) || !SDDS_Terminate(&SDDS_2) || !SDDS_Terminate(&SDDS_output)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (tmpfile_used && !replaceFileAndBackUp(input1, output))
exit(1);
free_scanargs(&s_arg,argc);
if (match_columns)
free(match_column);
return(0);
}
int main(int argc, char **argv)
{
STAT_DEFINITION *stat;
long stats;
STAT_REQUEST *request;
long requests, count;
SCANNED_ARG *scanned;
SDDS_DATASET inData, outData;
int32_t power;
long i_arg, code, iStat, rows, tmpFileUsed, iColumn, row, posColIndex;
long noWarnings, maxSourceColumns;
char *input, *output, *positionColumn, **posColumnName;
double **inputData, *outputData, value1, value2, topLimit, bottomLimit;
unsigned long pipeFlags, scanFlags, majorOrderFlag;
char s[100];
double *statWorkArray;
double quartilePoint[2] = {25.0, 75.0 }, quartileResult[2];
double decilePoint[2] = {10.0, 90.0 }, decileResult[2];
double percent;
short columnMajorOrder=-1;
SDDS_RegisterProgramName(argv[0]);
argc = scanargs(&scanned, argc, argv);
if (argc<2) {
bomb("too few arguments", USAGE);
}
posColumnName = NULL;
input = output = positionColumn = NULL;
stat = NULL;
request = NULL;
stats = requests = pipeFlags = 0;
noWarnings = 0;
outputData = NULL;
statWorkArray = NULL;
for (i_arg=1; i_arg<argc; i_arg++) {
scanFlags = 0;
if (scanned[i_arg].arg_type==OPTION) {
/* process options here */
switch (code=match_string(scanned[i_arg].list[0], option, N_OPTIONS, 0)) {
case SET_MAXIMUM:
case SET_MINIMUM:
case SET_MEAN:
case SET_MEDIAN:
case SET_STANDARDDEVIATION:
case SET_RMS:
case SET_SIGMA:
case SET_MAD:
case SET_COUNT:
case SET_DRANGE:
case SET_QRANGE:
case SET_SMALLEST:
case SET_LARGEST:
case SET_SPREAD:
if (scanned[i_arg].n_items<3) {
fprintf(stderr, "error: invalid -%s syntax\n", option[code]);
exit(1);
}
if (!scanItemList(&scanFlags, scanned[i_arg].list, &scanned[i_arg].n_items,
SCANITEMLIST_UNKNOWN_VALUE_OK|SCANITEMLIST_REMOVE_USED_ITEMS|
SCANITEMLIST_IGNORE_VALUELESS,
"positionColumn", SDDS_STRING, &positionColumn, 1, POSITIONCOLUMN_GIVEN,
"toplimit", SDDS_DOUBLE, &topLimit, 1, TOPLIMIT_GIVEN,
"bottomlimit", SDDS_DOUBLE, &bottomLimit, 1, BOTTOMLIMIT_GIVEN,
NULL)) {
sprintf(s, "invalid -%s syntax", scanned[i_arg].list[0]);
SDDS_Bomb(s);
}
requests = addStatRequests(&request, requests, scanned[i_arg].list+1, scanned[i_arg].n_items-1,
code, scanFlags);
request[requests-1].topLimit = topLimit;
request[requests-1].bottomLimit = bottomLimit;
if (positionColumn) {
if (code==SET_MAXIMUM || code==SET_MINIMUM || code==SET_LARGEST || code==SET_SMALLEST)
SDDS_CopyString(&request[requests-1].positionColumn, positionColumn);
free(positionColumn);
positionColumn = NULL;
}
break;
case SET_PERCENTILE:
if (scanned[i_arg].n_items<3) {
fprintf(stderr, "error: invalid -%s syntax\n", option[code]);
exit(1);
}
if (!scanItemList(&scanFlags, scanned[i_arg].list, &scanned[i_arg].n_items,
SCANITEMLIST_UNKNOWN_VALUE_OK|SCANITEMLIST_REMOVE_USED_ITEMS|
SCANITEMLIST_IGNORE_VALUELESS,
"value", SDDS_DOUBLE, &percent, 1, PERCENT_GIVEN,
"toplimit", SDDS_DOUBLE, &topLimit, 1, TOPLIMIT_GIVEN,
"bottomlimit", SDDS_DOUBLE, &bottomLimit, 1, BOTTOMLIMIT_GIVEN,
NULL) ||
!(scanFlags&PERCENT_GIVEN) || percent<=0 || percent>=100)
SDDS_Bomb("invalid -percentile syntax");
requests = addStatRequests(&request, requests, scanned[i_arg].list+1, scanned[i_arg].n_items-1,
code, scanFlags);
request[requests-1].percent = percent;
request[requests-1].topLimit = topLimit;
request[requests-1].bottomLimit = bottomLimit;
break;
case SET_SUM:
if (scanned[i_arg].n_items<3) {
fprintf(stderr, "error: invalid -%s syntax\n", option[code]);
exit(1);
}
power = 1;
if (!scanItemList(&scanFlags, scanned[i_arg].list, &scanned[i_arg].n_items,
SCANITEMLIST_UNKNOWN_VALUE_OK|SCANITEMLIST_REMOVE_USED_ITEMS|
SCANITEMLIST_IGNORE_VALUELESS,
"power", SDDS_LONG, &power, 1, 0,
"toplimit", SDDS_DOUBLE, &topLimit, 1, TOPLIMIT_GIVEN,
"bottomlimit", SDDS_DOUBLE, &bottomLimit, 1, BOTTOMLIMIT_GIVEN,
NULL))
SDDS_Bomb("invalid -sum syntax");
requests = addStatRequests(&request, requests, scanned[i_arg].list+1, scanned[i_arg].n_items-1,
code, scanFlags);
request[requests-1].sumPower = power;
request[requests-1].topLimit = topLimit;
request[requests-1].bottomLimit = bottomLimit;
break;
case SET_PIPE:
if (!processPipeOption(scanned[i_arg].list+1, scanned[i_arg].n_items-1, &pipeFlags))
SDDS_Bomb("invalid -pipe syntax");
break;
case SET_NOWARNINGS:
noWarnings = 1;
break;
case SET_MAJOR_ORDER:
majorOrderFlag=0;
scanned[i_arg].n_items --;
if (scanned[i_arg].n_items>0 &&
(!scanItemList(&majorOrderFlag, scanned[i_arg].list+1, &scanned[i_arg].n_items, 0,
"row", -1, NULL, 0, SDDS_ROW_MAJOR_ORDER,
"column", -1, NULL, 0, SDDS_COLUMN_MAJOR_ORDER,
NULL)))
SDDS_Bomb("invalid -majorOrder syntax/values");
if (majorOrderFlag&SDDS_COLUMN_MAJOR_ORDER)
columnMajorOrder=1;
else if (majorOrderFlag&SDDS_ROW_MAJOR_ORDER)
columnMajorOrder=0;
break;
default:
fprintf(stderr, "error: unknown option '%s' given\n", scanned[i_arg].list[0]);
exit(1);
break;
}
}
else {
/* argument is filename */
if (!input)
input = scanned[i_arg].list[0];
else if (!output)
output = scanned[i_arg].list[0];
else
SDDS_Bomb("too many filenames seen");
}
}
processFilenames("sddsrowstats", &input, &output, pipeFlags, noWarnings, &tmpFileUsed);
if (!requests)
SDDS_Bomb("no statistics requested");
if (!SDDS_InitializeInput(&inData, input))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (!(stat=compileStatDefinitions(&inData, request, requests, &stats, noWarnings))) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (stats<0)
SDDS_Bomb("No valid statistics requests.");
for (iStat=maxSourceColumns=0; iStat<stats; iStat++) {
if (stat[iStat].sourceColumns>maxSourceColumns)
maxSourceColumns = stat[iStat].sourceColumns;
}
if (!(statWorkArray=malloc(sizeof(*statWorkArray)*maxSourceColumns)))
SDDS_Bomb("allocation failure (statWorkArray)");
if (!setupOutputFile(&outData, output, &inData, stat, stats)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
inputData = NULL;
while ((code=SDDS_ReadPage(&inData))>0) {
if (!SDDS_CopyPage(&outData, &inData)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if ((rows = SDDS_CountRowsOfInterest(&inData))) {
if (!(outputData = (double*)malloc(sizeof(*outputData)*rows)))
SDDS_Bomb("memory allocation failure");
if (!(posColumnName = (char**)malloc(sizeof(*posColumnName)*rows)))
SDDS_Bomb("memory allocation failure");
for (iStat=0; iStat<stats; iStat++) {
if (!(inputData = (double**)malloc(sizeof(*inputData)*stat[iStat].sourceColumns)))
SDDS_Bomb("memory allocation failure");
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (!(inputData[iColumn] = SDDS_GetColumnInDoubles(&inData, stat[iStat].sourceColumn[iColumn])))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
for (row=0; row<rows; row++)
outputData[row] = DBL_MAX;
switch (stat[iStat].optionCode) {
case SET_MINIMUM:
for (row=0; row<rows; row++) {
value1 = DBL_MAX;
posColIndex = 0;
posColumnName[row] = NULL;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
if (inputData[iColumn][row]<value1) {
value1 = inputData[iColumn][row];
posColIndex = iColumn;
}
}
outputData[row] = value1;
if (stat[iStat].positionColumn)
posColumnName[row] = stat[iStat].sourceColumn[posColIndex];
}
break;
case SET_MAXIMUM:
for (row=0; row<rows; row++) {
posColIndex = 0;
value1 = -DBL_MAX;
posColumnName[row] = NULL;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
if (inputData[iColumn][row]>value1) {
posColIndex = iColumn;
value1 = inputData[iColumn][row];
}
}
outputData[row] = value1;
if (stat[iStat].positionColumn)
posColumnName[row] = stat[iStat].sourceColumn[posColIndex];
}
break;
case SET_MEAN:
for (row=0; row<rows; row++) {
value1 = 0;
count = 0;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
value1 += inputData[iColumn][row];
count ++;
}
if (count)
outputData[row] = value1/count;
}
break;
case SET_MEDIAN:
for (row=0; row<rows; row++) {
for (iColumn=count=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
statWorkArray[count] = inputData[iColumn][row];
count++;
}
if (count)
compute_median(outputData+row, statWorkArray, count);
}
break;
case SET_STANDARDDEVIATION:
for (row=0; row<rows; row++) {
value1 = 0;
value2 = 0;
count = 0;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
value1 += inputData[iColumn][row];
value2 += inputData[iColumn][row]*inputData[iColumn][row];
count ++;
}
if (count>1) {
if ((value1 = value2/count - sqr(value1/count))<=0)
outputData[row] = 0;
else
outputData[row] = sqrt(value1*count/(count-1.0));
}
}
break;
case SET_SIGMA:
for (row=0; row<rows; row++) {
value1 = 0;
value2 = 0;
count = 0;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
value1 += inputData[iColumn][row];
value2 += inputData[iColumn][row]*inputData[iColumn][row];
count ++;
}
if (count>1) {
if ((value1 = value2/count - sqr(value1/count))<=0)
outputData[row] = 0;
else
outputData[row] = sqrt(value1/(count-1.0));
}
}
break;
case SET_RMS:
for (row=0; row<rows; row++) {
value1 = 0;
count = 0;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
value1 += sqr(inputData[iColumn][row]);
count ++;
}
if (count)
outputData[row] = sqrt(value1/count);
}
break;
case SET_SUM:
for (row=0; row<rows; row++) {
value1 = 0;
count = 0;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
value1 += ipow(inputData[iColumn][row], stat[iStat].sumPower);
count ++;
}
if (count)
outputData[row] = value1;
}
break;
case SET_COUNT:
for (row=0; row<rows; row++) {
count = 0;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
count++;
}
outputData[row] = count;
}
break;
case SET_MAD:
for (row=0; row<rows; row++) {
for (iColumn=count=value1=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
statWorkArray[count] = inputData[iColumn][row];
count++;
}
if (count)
computeMoments(NULL, NULL, NULL, &outputData[row],
statWorkArray, count);
}
break;
case SET_DRANGE:
for (row=0; row<rows; row++) {
for (iColumn=count=value1=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
statWorkArray[count] = inputData[iColumn][row];
count++;
}
if (count &&
compute_percentiles(decileResult, decilePoint, 2,
statWorkArray, count))
outputData[row] = decileResult[1] - decileResult[0];
}
break;
case SET_QRANGE:
for (row=0; row<rows; row++) {
for (iColumn=count=value1=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
statWorkArray[count] = inputData[iColumn][row];
count++;
}
if (count &&
compute_percentiles(quartileResult, quartilePoint, 2,
statWorkArray, count))
outputData[row] = quartileResult[1] - quartileResult[0];
}
break;
case SET_SMALLEST:
for (row=0; row<rows; row++) {
value1 = DBL_MAX;
posColIndex = 0;
posColumnName[row] = NULL;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
if ((value2=fabs(inputData[iColumn][row]))<value1) {
posColIndex = iColumn;
value1 = value2;
}
}
outputData[row] = value1;
if (stat[iStat].positionColumn)
posColumnName[row] = stat[iStat].sourceColumn[posColIndex];
}
break;
case SET_LARGEST:
for (row=0; row<rows; row++) {
value1 = 0;
posColIndex = 0;
posColumnName[row] = NULL;
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
if ((value2=fabs(inputData[iColumn][row]))>value1) {
posColIndex = iColumn;
value1 = value2;
}
}
outputData[row] = value1;
if (stat[iStat].positionColumn)
posColumnName[row] = stat[iStat].sourceColumn[posColIndex];
}
break;
case SET_SPREAD:
for (row=0; row<rows; row++) {
value1 = DBL_MAX; /* min */
value2 = -DBL_MAX; /* max */
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
if (inputData[iColumn][row]<value1)
value1 = inputData[iColumn][row];
if (inputData[iColumn][row]>value2)
value2 = inputData[iColumn][row];
}
outputData[row] = value2-value1;
}
break;
case SET_PERCENTILE:
for (row=0; row<rows; row++) {
for (iColumn=count=value1=0; iColumn<stat[iStat].sourceColumns; iColumn++) {
if (stat[iStat].flags&TOPLIMIT_GIVEN &&
inputData[iColumn][row]>stat[iStat].topLimit)
continue;
if (stat[iStat].flags&BOTTOMLIMIT_GIVEN &&
inputData[iColumn][row]<stat[iStat].bottomLimit)
continue;
statWorkArray[count] = inputData[iColumn][row];
count++;
}
outputData[row] = HUGE_VAL;
if (count)
compute_percentiles(&outputData[row], &stat[iStat].percent, 1, statWorkArray, count);
}
break;
default:
SDDS_Bomb("invalid statistic code (accumulation loop)");
break;
}
if (!SDDS_SetColumn(&outData, SDDS_SET_BY_INDEX, outputData, rows, stat[iStat].resultIndex))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
if (stat[iStat].positionColumn) {
if (!SDDS_SetColumn(&outData, SDDS_SET_BY_INDEX, posColumnName, rows,
stat[iStat].positionColumnIndex))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++)
free(inputData[iColumn]);
free(inputData);
inputData = NULL;
}
free(outputData);
outputData = NULL;
free(posColumnName);
posColumnName = NULL;
}
if (!SDDS_WritePage(&outData))
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors|SDDS_EXIT_PrintErrors);
}
free_scanargs(&scanned, argc);
for (iStat=0; iStat<stats; iStat++) {
if (stat[iStat].positionColumn) free(stat[iStat].positionColumn);
for (iColumn=0; iColumn<stat[iStat].sourceColumns; iColumn++)
free(stat[iStat].sourceColumn[iColumn]);
free(stat[iStat].sourceColumn);
}
free(request);
free(stat);
if (statWorkArray) free(statWorkArray);
if (!SDDS_Terminate(&inData) || !SDDS_Terminate(&outData)) {
SDDS_PrintErrors(stderr, SDDS_VERBOSE_PrintErrors);
exit(1);
}
if (tmpFileUsed && !replaceFileAndBackUp(input, output))
exit(1);
return 0;
}
