bool seek_time(const stdString &index_name,
const stdString &channel_name,
const stdString &start_txt)
{
epicsTime start;
if (!string2epicsTime(start_txt, start))
{
fprintf(stderr, "Cannot convert '%s' to time stamp\n",
start_txt.c_str());
return false;
}
IndexFile index(3);
index.open(index_name, true);
stdString directory;
AutoPtr<RTree> tree(index.getTree(channel_name, directory));
if (tree)
{
RTree::Datablock block;
RTree::Node node(tree->getM(), true);
int idx;
if (tree->searchDatablock(start, node, idx, block))
{
stdString s, e;
printf("Found block %s - %s\n",
epicsTimeTxt(node.record[idx].start, s),
epicsTimeTxt(node.record[idx].end, e));
}
else
printf("Nothing found\n");
}
else
fprintf(stderr, "Cannot find channel '%s'\n", channel_name.c_str());
return true;
}
void list_names(const stdString &index_name, bool channel_detail)
{
IndexFile index(3);
IndexFile::NameIterator names;
epicsTime stime, etime, t0, t1;
stdString directory, start, end;
bool ok;
size_t channels = 0, blocks = 0, values = 0;
index.open(index_name, true);
if (channel_detail)
printf("# Name\tType\tCount\tStart\tEnd\tBlocks\tValues\n");
for (ok = index.getFirstChannel(names);
ok;
ok = index.getNextChannel(names))
{
AutoPtr<RTree> tree(index.getTree(names.getName(), directory));
tree->getInterval(stime, etime);
if (channels == 0)
{
t0 = stime;
t1 = etime;
}
else
{
if (t0 > stime)
t0 = stime;
if (t1 < etime)
t1 = etime;
}
size_t chan_blocks, chan_values;
DbrType type;
DbrCount count;
count_channel_values(tree, directory, type, count, chan_blocks, chan_values);
++channels;
blocks += chan_blocks;
values += chan_values;
if (channel_detail)
printf("%s\t%d\t%d\t%s\t%s\t%zu\t%zu\n",
names.getName().c_str(),
(int) type, (int) count,
epicsTimeTxt(stime, start),
epicsTimeTxt(etime, end),
chan_blocks, chan_values);
}
printf("%zu channels, %zu values, %s - %s\n",
channels, values,
epicsTimeTxt(t0, start), epicsTimeTxt(t1, end));
}
// List channel names, maybe with start/end info.
void list_channels(Index &index, stdVector<stdString> names, bool info)
{
epicsTime start, end;
stdString s, e;
AutoPtr<RTree> tree;
size_t i;
for (i=0; i<names.size(); ++i)
{
if (info)
{
stdString directory;
tree = index.getTree(names[i], directory);
if (!tree)
throw GenericException(__FILE__, __LINE__,
"Cannot locate channel '%s'",
names[i].c_str());
tree->getInterval(start, end);
printf("%s\t%s\t%s\n", names[i].c_str(),
epicsTimeTxt(start, s), epicsTimeTxt(end, e));
}
else
printf("%s\n", names[i].c_str());
}
}
// Return the data for all the names[], start .. end etc.
// as get_values() is supposed to return them.
//
// Returns raw values if interpol <= 0.0.
// Returns 0 on error.
xmlrpc_value *get_sheet_data(xmlrpc_env *env,
int key,
const stdVector<stdString> names,
const epicsTime &start, const epicsTime &end,
long count,
ReaderFactory::How how, double delta)
{
try
{
#ifdef LOGFILE
stdString txt;
LOG_MSG("get_sheet_data\n");
LOG_MSG("Start : %s\n", epicsTimeTxt(start, txt));
LOG_MSG("End : %s\n", epicsTimeTxt(end, txt));
LOG_MSG("Method: %s\n", ReaderFactory::toString(how, delta));
#endif
AutoPtr<Index> index(open_index(env, key));
if (env->fault_occurred)
return 0;
AutoPtr<SpreadsheetReader> sheet(new SpreadsheetReader(*index, how, delta));
AutoXmlRpcValue results(xmlrpc_build_value(env, "()"));
if (env->fault_occurred)
return 0;
long name_count = names.size();
AutoArrayPtr<AutoXmlRpcValue> meta (new AutoXmlRpcValue[name_count]);
AutoArrayPtr<AutoXmlRpcValue> values (new AutoXmlRpcValue[name_count]);
AutoArrayPtr<xmlrpc_int32> xml_type (new xmlrpc_int32[name_count]);
AutoArrayPtr<xmlrpc_int32> xml_count(new xmlrpc_int32[name_count]);
AutoArrayPtr<size_t> ch_vals (new size_t[name_count]);
bool ok = sheet->find(names, &start);
long i;
// Per-channel meta-info.
for (i=0; i<name_count; ++i)
{
#ifdef LOGFILE
LOG_MSG("Handling '%s'\n", names[i].c_str());
#endif
ch_vals[i] = 0;
values[i] = xmlrpc_build_value(env, "()");
if (env->fault_occurred)
return 0;
if (sheet->found(i))
{ // Fix meta/type/count based on first value
meta[i] = encode_ctrl_info(env, &sheet->getInfo(i));
dbr_type_to_xml_type(sheet->getType(i), sheet->getCount(i),
xml_type[i], xml_count[i]);
#if 0
LOG_MSG("Ch %lu: type, count = %d, %d\n", i, (int)xml_type[i], (int)xml_count[i]);
#endif
}
else
{ // Channel exists, but has no data
meta[i] = encode_ctrl_info(env, 0);
xml_type[i] = XML_ENUM;
xml_count[i] = 1;
}
}
// Collect values
long num_vals = 0;
while (ok && num_vals < count && sheet->getTime() < end)
{
for (i=0; i<name_count; ++i)
{
if (sheet->get(i))
{
++ch_vals[i];
encode_value(env,
sheet->getType(i), sheet->getCount(i),
sheet->getTime(), sheet->get(i),
xml_type[i], xml_count[i], values[i]);
}
else
{ // Encode as no value, but one of them ;-)
// to avoid confusion w/ viewers that like to see some data in any case.
encode_value(env, 0, 1, sheet->getTime(), 0,
xml_type[i], xml_count[i], values[i]);
}
}
++num_vals;
ok = sheet->next();
}
// Assemble result = { name, meta, type, count, values }
for (i=0; i<name_count; ++i)
{
AutoXmlRpcValue result(
xmlrpc_build_value(env, "{s:s,s:V,s:i,s:i,s:V}",
"name", names[i].c_str(),
"meta", meta[i].get(),
"type", xml_type[i],
"count", xml_count[i],
"values", values[i].get()));
// Add to result array
xmlrpc_array_append_item(env, results, result);
#ifdef LOGFILE
LOG_MSG("Ch %lu: %zu values\n", i, ch_vals[i]);
#endif
}
#ifdef LOGFILE
LOG_MSG("%ld values total\n", num_vals);
#endif
return results.release();
}
catch (GenericException &e)
{
#ifdef LOGFILE
LOG_MSG("Error:\n%s\n", e.what());
#endif
xmlrpc_env_set_fault_formatted(env, ARCH_DAT_DATA_ERROR,
"%s", e.what());
}
return 0;
}
// Return the data for all the names[], start .. end etc.
// as get_values() is supposed to return them.
// Returns 0 on error.
xmlrpc_value *get_channel_data(xmlrpc_env *env,
int key,
const stdVector<stdString> names,
const epicsTime &start, const epicsTime &end,
long count,
ReaderFactory::How how, double delta)
{
AutoXmlRpcValue results;
try
{
#ifdef LOGFILE
stdString txt;
LOG_MSG("get_channel_data\n");
LOG_MSG("Method: %s\n", ReaderFactory::toString(how, delta));
LOG_MSG("Start: %s\n", epicsTimeTxt(start, txt));
LOG_MSG("End : %s\n", epicsTimeTxt(end, txt));
#endif
AutoPtr<Index> index(open_index(env, key));
if (env->fault_occurred)
return 0;
AutoPtr<DataReader> reader(ReaderFactory::create(*index, how, delta));
results = xmlrpc_build_value(env, "()");
if (env->fault_occurred)
return 0;
long i, name_count = names.size();
for (i=0; i<name_count; ++i)
{
#ifdef LOGFILE
LOG_MSG("Handling '%s'\n", names[i].c_str());
#endif
long num_vals = 0;
AutoXmlRpcValue values(xmlrpc_build_value(env, "()"));
if (env->fault_occurred)
return 0;
const RawValue::Data *data = reader->find(names[i], &start);
AutoXmlRpcValue meta;
xmlrpc_int32 xml_type, xml_count;
if (data == 0)
{ // No exception from file error etc., just no data.
meta = encode_ctrl_info(env, 0);
xml_type = XML_ENUM;
xml_count = 1;
}
else
{
// Fix meta/type/count based on first value
meta = encode_ctrl_info(env, &reader->getInfo());
dbr_type_to_xml_type(reader->getType(), reader->getCount(),
xml_type, xml_count);
while (num_vals < count
&& data
&& RawValue::getTime(data) < end)
{
encode_value(env, reader->getType(), reader->getCount(),
RawValue::getTime(data), data,
xml_type, xml_count, values);
++num_vals;
data = reader->next();
}
}
// Assemble result = { name, meta, type, count, values }
AutoXmlRpcValue result(xmlrpc_build_value(
env, "{s:s,s:V,s:i,s:i,s:V}",
"name", names[i].c_str(),
"meta", (xmlrpc_value *)meta,
"type", xml_type,
"count", xml_count,
"values", (xmlrpc_value *)values));
// Add to result array
xmlrpc_array_append_item(env, results, result);
#ifdef LOGFILE
LOG_MSG("%ld values\n", num_vals);
#endif
} // for ( .. name .. )
}
catch (GenericException &e)
{
#ifdef LOGFILE
LOG_MSG("Error:\n%s\n", e.what());
#endif
xmlrpc_env_set_fault_formatted(env, ARCH_DAT_DATA_ERROR,
"%s", e.what());
return 0;
}
return results.release();
}
int main(int argc, const char *argv[])
{
try
{
CmdArgParser parser(argc, argv);
parser.setHeader("Archive Export version " ARCH_VERSION_TXT ", "
EPICS_VERSION_STRING
", built " __DATE__ ", " __TIME__ "\n\n");
parser.setArgumentsInfo("<index file> {channel}");
CmdArgFlag be_verbose (parser, "verbose", "Verbose mode");
CmdArgString pattern (parser, "match", "<reg. exp.>",
"Channel name pattern");
CmdArgFlag do_list (parser, "list", "List all channels");
CmdArgFlag do_info (parser, "info", "Time-range info on channels");
CmdArgString start_time (parser, "start", "<time>",
"Format: \"mm/dd/yyyy[ hh:mm:ss[.nano-secs]]\"");
CmdArgString end_time (parser, "end", "<time>", "(exclusive)");
CmdArgFlag status_text(parser, "text",
"Include text column for status/severity (default)");
CmdArgFlag no_status_text(parser, "no_text",
"Exclude text column for status/severity");
CmdArgString output (parser,
"output", "<file>", "Output to file");
CmdArgDouble plotbin (parser,
"plotbin", "<seconds>",
"Bin the raw data for plotting");
CmdArgDouble average (parser,
"average", "<seconds>", "average values");
CmdArgDouble linear (parser,
"linear", "<seconds>",
"Interpolate values linearly");
CmdArgString format_txt (parser,
"format", "<decimal|engineering|exponential>",
"Use specific format for numbers");
CmdArgInt prec (parser,
"precision", "<int>", "Precision of numbers");
CmdArgFlag GNUPlot (parser,
"gnuplot", "Generate GNUPlot command file");
CmdArgFlag image (parser,
"Gnuplot", "Generate GNUPlot output for Image");
CmdArgFlag raw_time (parser, "raw_time",
"Include columns for EPICS time stamp");
CmdArgFlag millisecs (parser, "millisecs",
"Truncate time to millisecs in spreadsheet dump.");
// defaults
prec.set(-1);
if (! parser.parse())
return -1;
if (parser.getArguments().size() < 1)
{
parser.usage();
return -1;
}
precision = prec;
if (!strncmp(format_txt.get().c_str(), "d", 1))
format = RawValue::DECIMAL;
else if (!strncmp(format_txt.get().c_str(), "en", 2))
format = RawValue::ENGINEERING;
else if (!strncmp(format_txt.get().c_str(), "ex", 2))
format = RawValue::EXPONENTIAL;
else if (format_txt.get().length() > 0)
{
fprintf(stderr, "Unknown format string '%s'\n", format_txt.get().c_str());
return -1;
}
verbose = be_verbose;
only_millisecs = millisecs;
// Start/end time
AutoPtr<epicsTime> start, end;
stdString txt;
if (start_time.get().length() > 0)
{
start = new epicsTime;
if (!string2epicsTime(start_time.get(), *start))
{
fprintf(stderr, "Parse error for start time '%s'\n",
start_time.get().c_str());
start = 0;
parser.usage();
return -1;
}
if (verbose)
printf("Using start time %s\n", epicsTimeTxt(*start, txt));
}
if (end_time.get().length() > 0)
{
end = new epicsTime();
if (!string2epicsTime(end_time.get(), *end))
{
fprintf(stderr, "Parse error for end time '%s'\n",
end_time.get().c_str());
end = 0;
parser.usage();
return -1;
}
if (verbose)
printf("Using end time %s\n", epicsTimeTxt(*end, txt));
}
if (start && end && *start > *end)
{ // Could simply swap start and end, but assume the user is
// confused and should rethink the request.
fprintf(stderr, "start time is greater than end time.\n");
return -1;
}
// Index name
stdString index_name = parser.getArgument(0);
// Channel names
stdVector<stdString> names;
if (parser.getArguments().size() > 1)
{
if (! pattern.get().empty())
{
fputs("Pattern from '-m' switch is ignored\n"
"since a list of channels was also provided.\n", stderr);
}
// first argument was directory file name, skip that:
for (size_t i=1; i<parser.getArguments().size(); ++i)
names.push_back(parser.getArgument(i));
}
if ((GNUPlot || image) && output.get().length() == 0)
{
fprintf(stderr, "The -gnuplot/Gnuplot options require "
"an -output file\n");
return -1;
}
// How?
ReaderFactory::How how = ReaderFactory::Raw;
double delta = 0.0;
if (double(plotbin) > 0.0)
{
how = ReaderFactory::Plotbin;
delta = double(plotbin);
}
else if (double(average) > 0.0)
{
how = ReaderFactory::Average;
delta = double(average);
}
else if (double(linear) > 0.0)
{
how = ReaderFactory::Linear;
delta = double(linear);
}
// Open index
AutoIndex index;
index.open(index_name.c_str());
if (verbose)
printf("Opened index '%s'\n", index_name.c_str());
if (do_info && names.size()<=0 && pattern.get().length()<=0)
do_list.set(); // otherwise it'd be a NOP
if (names.size() <= 0 &&
(do_list || pattern.get().length() > 0))
get_names_for_pattern(index, names, pattern);
if (do_info)
list_channels(index, names, true);
else if (do_list)
list_channels(index, names, false);
else if (names.size() > 0)
{
if (GNUPlot || image)
dump_gnuplot(index, names, start, end,
how, delta, output, image);
else
dump_spreadsheet(index, names, start, end,
raw_time, !no_status_text, how, delta,
output);
}
index.close();
}
catch (GenericException &e)
{
fprintf(stderr, "Error:\n%s\n", e.what());
return -1;
}
return 0;
}
int main(int argc, const char *argv[])
{
CmdArgParser parser(argc, argv);
parser.setHeader("Archive Data Tool version " ARCH_VERSION_TXT ", "
EPICS_VERSION_STRING
", built " __DATE__ ", " __TIME__ "\n\n"
);
parser.setArgumentsInfo("<index-file>");
CmdArgFlag help (parser, "help", "Show help");
CmdArgInt verbosity (parser, "verbose", "<level>", "Show more info");
CmdArgFlag info (parser, "info", "Simple archive info");
CmdArgFlag list_index (parser, "list", "List channel name info");
CmdArgString copy_index (parser, "copy", "<new index>", "Copy channels");
CmdArgString start_time (parser, "start", "<time>",
"Format: \"mm/dd/yyyy[ hh:mm:ss[.nano-secs]]\"");
CmdArgString end_time (parser, "end", "<time>", "(exclusive)");
CmdArgDouble file_limit (parser, "file_limit", "<MB>", "File Size Limit");
CmdArgString basename (parser, "basename", "<string>", "Basename for new data files");
CmdArgFlag enforce_off (parser, "append_off", "Enforce a final 'Archive_Off' value when copying data");
CmdArgString dir2index (parser, "dir2index", "<dir. file>",
"Convert old directory file to index");
CmdArgString index2dir (parser, "index2dir", "<dir. file>",
"Convert index to old directory file");
CmdArgInt RTreeM (parser, "M", "<1-100>", "RTree M value");
CmdArgFlag dump_blocks (parser, "blocks", "List channel's data blocks");
CmdArgFlag all_blocks (parser, "Blocks", "List all data blocks");
CmdArgString dotindex (parser, "dotindex", "<dot filename>",
"Dump contents of RTree index into dot file");
CmdArgString channel_name(parser, "channel", "<name>", "Channel name");
CmdArgFlag hashinfo (parser, "hashinfo", "Show Hash table info");
CmdArgString seek_test (parser, "seek", "<time>", "Perform seek test");
CmdArgFlag test (parser, "test", "Perform some consistency tests");
try
{
// Defaults
RTreeM.set(50);
file_limit.set(100.0);
// Get Arguments
if (! parser.parse())
return -1;
if (help || parser.getArguments().size() != 1)
{
parser.usage();
return -1;
}
// Consistency checks
if ((dump_blocks ||
dotindex.get().length() > 0 ||
seek_test.get().length() > 0)
&& channel_name.get().length() <= 0)
{
fprintf(stderr,
"Options 'blocks' and 'dotindex' require 'channel'.\n");
return -1;
}
verbose = verbosity;
stdString index_name = parser.getArgument(0);
DataWriter::file_size_limit = (unsigned long)(file_limit*1024*1024);
if (file_limit < 10.0)
fprintf(stderr, "file_limit values under 10.0 MB are not useful\n");
// Start/end time
epicsTime *start = 0, *end = 0;
stdString txt;
if (start_time.get().length() > 0)
{
start = new epicsTime;
string2epicsTime(start_time.get(), *start);
if (verbose > 1)
printf("Using start time %s\n", epicsTimeTxt(*start, txt));
}
if (end_time.get().length() > 0)
{
end = new epicsTime();
string2epicsTime(end_time.get(), *end);
if (verbose > 1)
printf("Using end time %s\n", epicsTimeTxt(*end, txt));
}
// Base name
if (basename.get().length() > 0)
DataWriter::data_file_name_base = basename.get();
if (enforce_off)
do_enforce_off = true;
// What's requested?
if (info)
list_names(index_name, false);
else if (list_index)
list_names(index_name, true);
else if (copy_index.get().length() > 0)
{
copy(index_name, copy_index, RTreeM, start, end, channel_name);
return 0;
}
else if (hashinfo)
show_hash_info(index_name);
else if (dir2index.get().length() > 0)
{
convert_dir_index(RTreeM, dir2index, index_name);
return 0;
}
else if (index2dir.get().length() > 0)
{
convert_index_dir(index_name, index2dir);
return 0;
}
else if (all_blocks)
{
dump_all_datablocks(index_name);
return 0;
}
else if (dump_blocks)
{
dump_datablocks(index_name, channel_name);
return 0;
}
else if (dotindex.get().length() > 0)
{
dot_index(index_name, channel_name, dotindex);
return 0;
}
else if (seek_test.get().length() > 0)
{
seek_time(index_name, channel_name, seek_test);
return 0;
}
else if (test)
{
return check(index_name) ? 0 : -1;
}
else
{
parser.usage();
return -1;
}
}
catch (GenericException &e)
{
fprintf(stderr, "Error:\n%s\n", e.what());
}
return 0;
}
unsigned long dump_datablocks_for_channel(IndexFile &index,
const stdString &channel_name,
unsigned long &direct_count,
unsigned long &chained_count)
{
DataFile *datafile;
AutoPtr<DataHeader> header;
direct_count = chained_count = 0;
stdString directory;
AutoPtr<RTree> tree(index.getTree(channel_name, directory));
if (! tree)
return 0;
RTree::Datablock block;
RTree::Node node(tree->getM(), true);
stdString start, end;
int idx;
bool ok;
if (verbose > 1)
printf("RTree M for channel '%s': %d\n", channel_name.c_str(),
tree->getM());
if (verbose > 2)
printf("Datablocks for channel '%s':\n", channel_name.c_str());
for (ok = tree->getFirstDatablock(node, idx, block);
ok;
ok = tree->getNextDatablock(node, idx, block))
{
++direct_count;
if (verbose > 2)
printf("'%s' @ 0x%lX: Indexed range %s - %s\n",
block.data_filename.c_str(),
(unsigned long)block.data_offset,
epicsTimeTxt(node.record[idx].start, start),
epicsTimeTxt(node.record[idx].end, end));
if (verbose > 3)
{
datafile = DataFile::reference(directory,
block.data_filename,
false);
header = datafile->getHeader(block.data_offset);
datafile->release();
if (header)
{
header->show(stdout, true);
header = 0;
}
else
printf("Cannot read header in data file.\n");
}
bool first_hidden_block = true;
while (tree->getNextChainedBlock(block))
{
if (first_hidden_block && verbose > 2)
{
first_hidden_block = false;
printf("Hidden blocks with smaller time range:\n");
}
++chained_count;
if (verbose > 2)
{
printf("--- '%s' @ 0x%lX\n",
block.data_filename.c_str(),
(unsigned long)block.data_offset);
if (verbose > 3)
{
datafile = DataFile::reference(directory,
block.data_filename,
false);
header = datafile->getHeader(block.data_offset);
if (header)
{
header->show(stdout, false);
header = 0;
}
else
printf("Cannot read header in data file.\n");
datafile->release();
}
}
}
printf("\n");
}
return direct_count + chained_count;
}
void convert_index_dir(const stdString &index_name, const stdString &dir_name)
{
IndexFile::NameIterator names;
IndexFile index(50);
stdString index_directory;
RTree::Datablock block;
stdString start_file, end_file;
FileOffset start_offset, end_offset;
epicsTime start_time, end_time;
bool ok;
index.open(index_name.c_str(), true);
OldDirectoryFile dir;
if (!dir.open(dir_name, true))
{
fprintf(stderr, "Cannot create '%s'\n", dir_name.c_str());
return;
}
for (ok=index.getFirstChannel(names); ok; ok=index.getNextChannel(names))
{
if (verbose)
printf("Channel '%s':\n", names.getName().c_str());
AutoPtr<const RTree> tree(index.getTree(names.getName(),
index_directory));
RTree::Node node(tree->getM(), false);
int i;
if (!tree->getFirstDatablock(node, i, block))
{
printf("No first datablock for channel '%s'\n",
names.getName().c_str());
continue;
}
start_file = block.data_filename;
start_offset = block.data_offset;
if (!tree->getLastDatablock(node, i, block))
{
printf("No last datablock for channel '%s'\n",
names.getName().c_str());
continue;
}
end_file = block.data_filename;
end_offset = block.data_offset;
// Check by reading first+last buffer & getting times
bool times_ok = false;
AutoPtr<DataHeader> header(
get_dataheader(index_directory, start_file, start_offset));
if (header)
{
start_time = header->data.begin_time;
header = get_dataheader(index_directory, end_file, end_offset);
if (header)
{
end_time = header->data.end_time;
header = 0;
times_ok = true;
}
}
if (times_ok)
{
stdString start, end;
if (verbose)
printf("%s - %s\n",
epicsTimeTxt(start_time, start),
epicsTimeTxt(end_time, end));
OldDirectoryFileIterator dfi;
dfi = dir.find(names.getName());
if (!dfi.isValid())
dfi = dir.add(names.getName());
dfi.entry.setFirst(start_file, start_offset);
dfi.entry.setLast(end_file, end_offset);
dfi.entry.data.create_time = start_time;
dfi.entry.data.first_save_time = start_time;
dfi.entry.data.last_save_time = end_time;
dfi.save();
}
}
}
