int main(int argc, char **argv)
{
char **argptr = argv+1;
if (!*argptr) usage();
std::string storename = *argptr++;
if (!*argptr) usage();
int uid = atoi(*argptr++);
if (uid <= 0) usage();
set_log_prefix(string_printf("%d %d ", getpid(), uid));
{
std::string arglist;
for (int i = 0; i < argc; i++) {
if (i) arglist += " ";
arglist += std::string("'")+argv[i]+"'";
}
log_f("export START: %s", arglist.c_str());
}
std::vector<std::string> channel_full_names;
while (*argptr) {
channel_full_names.push_back(*argptr++);
}
if (!channel_full_names.size()) usage();
Range times = Range::all();
FilesystemKVS store(storename.c_str());
for (unsigned i = 0; i < channel_full_names.size(); i++) {
std::string &channel_full_name = channel_full_names[i];
if (i) printf("\f");
printf("Time\t%s\n", channel_full_name.c_str());
Channel ch(store, uid, channel_full_name);
Channel::Locker locker(ch);
ch.read_tiles_in_range(times, dump_samples, TileIndex::lowest_level());
}
return 0;
}
int main(int argc, char *argv[])
{
oph_metadb_db_row *db_table = NULL;
set_debug_level(LOG_DEBUG);
set_log_prefix(OPH_IO_SERVER_PREFIX);
int ch;
unsigned short int instance = 0;
unsigned short int help = 0;
while ((ch = getopt(argc, argv, "i:h"))!=-1)
{
switch (ch)
{
case 'h':
help = 1;
break;
case 'i':
instance = (unsigned short int)strtol(optarg, NULL, 10);
break;
}
}
if (help)
{
pmesg(LOG_ERROR,__FILE__,__LINE__,"Execute MetaDB Reader: oph_metadb_client -i <instance_number>\n");
exit(0);
}
if(instance == 0){
pmesg(LOG_WARNING,__FILE__,__LINE__,"Using default (first) instance in configuration file\n");
}
HASHTBL *conf_db = NULL;
//Load params from conf files
if(oph_server_conf_load(instance, &conf_db)){
pmesg(LOG_ERROR,__FILE__,__LINE__,"Error while loading configuration file\n");
//logging(LOG_ERROR,__FILE__,__LINE__,"Error while loading configuration file\n");
return -1;
}
char *dir = 0;
if(oph_server_conf_get_param(conf_db, OPH_SERVER_CONF_DIR, &dir)){
pmesg(LOG_WARNING,__FILE__,__LINE__,"Unable to get server dir param\n");
dir = OPH_IO_SERVER_PREFIX;
}
oph_metadb_set_data_prefix(dir);
if(oph_metadb_load_schema (&db_table, 0)){
printf("Unable to load MetaDB\n");
oph_metadb_unload_schema (db_table);
oph_server_conf_unload(&conf_db);
return -1;
}
int i = 0;
int thread_number = 10;
pthread_t tid[thread_number];
int n = 0;
for (i = 0 ; i < thread_number ; i++)
{
pmesg(LOG_DEBUG,__FILE__,__LINE__,"Creating new thread\n");
if ( (n = pthread_create(&tid[i], NULL, &test_metadb, (void *) db_table)) != 0){
pmesg(LOG_ERROR,__FILE__,__LINE__,"Error creating thread\n");
}
}
for(i = 0; i < thread_number; i++)
{
pmesg(LOG_DEBUG,__FILE__,__LINE__,"Joining thread\n");
if ( (n = pthread_join(tid[i], NULL)) != 0){
pmesg(LOG_WARNING,__FILE__,__LINE__,"Error joining thread\n");
}
}
oph_server_conf_unload(&conf_db);
oph_metadb_unload_schema (db_table);
return 0;
}
int main (int argc, char *argv[]) {
//Initialize environment
int size, myrank, res = -1;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &size);
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
if (!myrank) {
fprintf(stdout,"%s%s",OPH_VERSION,OPH_DISCLAIMER);
}
if (argc != 2) {
if (!myrank) fprintf(stdout, "USAGE: ./oph_analytics_framework \"operator=value;param=value;...\"\n");
res = 0;
}
if (!strcmp(argv[1],"-v")) {
res = 0;
}
if (!strcmp(argv[1],"-x")) {
if (!myrank) fprintf(stdout,"%s",OPH_WARRANTY);
res = 0;
}
if (!strcmp(argv[1],"-z")) {
if (!myrank) fprintf(stdout,"%s",OPH_CONDITIONS);
res = 0;
}
if (res)
{
struct timeval start_time, end_time, total_time;
#ifdef OPH_PARALLEL_LOCATION
char log_prefix[OPH_COMMON_BUFFER_LEN];
snprintf(log_prefix,OPH_COMMON_BUFFER_LEN, OPH_FRAMEWORK_LOG_PATH_PREFIX, OPH_PARALLEL_LOCATION);
set_log_prefix(log_prefix);
pmesg(LOG_DEBUG, __FILE__, __LINE__, "Set logging directory to '%s'\n",log_prefix);
#endif
if(!myrank)
gettimeofday(&start_time, NULL);
char task_string[OPH_COMMON_BUFFER_LEN];
strncpy(task_string,argv[1],OPH_COMMON_BUFFER_LEN);
if((res = oph_af_execute_framework(task_string, size, myrank))) {
pmesg(LOG_ERROR, __FILE__, __LINE__, "Framework execution failed! ERROR: %d\n",res);
}
MPI_Barrier(MPI_COMM_WORLD);
if(!myrank) {
gettimeofday(&end_time, NULL);
timeval_subtract(&total_time, &end_time, &start_time);
printf("Proc %d: Total execution:\t Time %d,%06d sec\n", myrank, (int)total_time.tv_sec, (int)total_time.tv_usec);
}
}
MPI_Finalize();
return 0;
}
int main(int argc, char **argv)
{
long long begin_time = millitime();
char **argptr = argv+1;
if (!*argptr) usage();
std::string storename = *argptr++;
if (!*argptr) usage();
int uid = atoi(*argptr++);
set_log_prefix(string_printf("%d %d ", getpid(), uid));
if (!*argptr) usage();
std::string full_channel_name = *argptr++;
#if FFT_SUPPORT
bool writing_fft = false;
size_t fftpos = full_channel_name.rfind(".DFT");
if (fftpos != std::string::npos) {
full_channel_name = full_channel_name.substr(0, fftpos);
writing_fft = true;
}
#endif /* FFT_SUPPORT */
if (!*argptr) usage();
int tile_level = atoi(*argptr++);
if (!*argptr) usage();
long long tile_offset = atoll(*argptr++);
if (*argptr) usage();
// Desired level and offset
// Translation between tile request and tilestore:
// tile: level 0 is 512 samples in 512 seconds
// store: level 0 is 65536 samples in 1 second
// for tile level 0, we want to get store level 14, which is 65536 samples in 16384 seconds
// Levels differ by 9 between client and server
TileIndex client_tile_index = TileIndex(tile_level+9, tile_offset);
{
std::string arglist;
for (int i = 0; i < argc; i++) {
if (i) arglist += " ";
arglist += std::string("'")+argv[i]+"'";
}
log_f("gettile START: %s (time %.9f-%.9f)",
arglist.c_str(), client_tile_index.start_time(), client_tile_index.end_time());
}
FilesystemKVS store(storename.c_str());
// 5th ancestor
TileIndex requested_index = client_tile_index.parent().parent().parent().parent().parent();
std::vector<DataSample<double> > double_samples;
std::vector<DataSample<std::string> > string_samples;
std::vector<DataSample<std::string> > comments;
bool doubles_binned, strings_binned, comments_binned;
// TODO: If writing FFT, ***get more data***
// TODO: Use min_time_required and max_time_required, get max-res data
read_tile_samples(store, uid, full_channel_name, requested_index, client_tile_index, double_samples, doubles_binned);
#if FFT_SUPPORT
if (writing_fft) {
std::vector<std::vector<double> > fft, shifted;
int num_values;
windowed_fft(double_samples, requested_index, fft);
present_fft(fft, shifted, num_values);
// JSON tile to send back to the client includes some of the same
// information as a non-DFT tile
Json::Value tile(Json::objectValue);
tile["level"] = Json::Value(tile_level);
// See discussion below for reason to cast tile_offset
// from long long to double
tile["offset"] = Json::Value((double)tile_offset);
tile["num_values"] = Json::Value(num_values);
tile["dft"] = Json::Value(Json::arrayValue);
for (unsigned window_id = 0; window_id < shifted.size(); window_id++) {
Json::Value window(Json::arrayValue);
for (unsigned i = 0; i < shifted[window_id].size(); i++)
window.append(shifted[window_id][i]);
tile["dft"].append(window);
}
std::cout << Json::FastWriter().write(tile) << std::endl;
return 0;
}
#endif /* FFT_SUPPORT */
read_tile_samples(store, uid, full_channel_name, requested_index, client_tile_index, string_samples, strings_binned);
read_tile_samples(store, uid, full_channel_name+"._comment", requested_index, client_tile_index, comments, comments_binned);
string_samples.insert(string_samples.end(), comments.begin(), comments.end());
std::sort(string_samples.begin(), string_samples.end(), DataSample<std::string>::time_lessthan);
std::map<double, DataSample<double> > double_sample_map;
for (unsigned i = 0; i < double_samples.size(); i++) {
double_sample_map[double_samples[i].time] = double_samples[i]; // TODO: combine if two samples at same time?
}
std::set<double> has_string;
for (unsigned i = 0; i < string_samples.size(); i++) {
has_string.insert(string_samples[i].time);
}
std::vector<GraphSample> graph_samples;
bool has_fifth_col = string_samples.size()>0;
for (unsigned i = 0; i < string_samples.size(); i++) {
if (double_sample_map.find(string_samples[i].time) != double_sample_map.end()) {
GraphSample gs(double_sample_map[string_samples[i].time]);
gs.has_comment = true;
gs.comment = string_samples[i].value;
graph_samples.push_back(gs);
} else {
graph_samples.push_back(GraphSample(string_samples[i]));
}
}
for (unsigned i = 0; i < double_samples.size(); i++) {
if (has_string.find(double_samples[i].time) == has_string.end()) {
graph_samples.push_back(GraphSample(double_samples[i]));
}
}
std::sort(graph_samples.begin(), graph_samples.end());
double bin_width = client_tile_index.duration() / 512.0;
double line_break_threshold = bin_width * 4.0;
if (!doubles_binned && double_samples.size() > 1) {
// Find the median distance between samples
std::vector<double> spacing(double_samples.size()-1);
for (size_t i = 0; i < double_samples.size()-1; i++) {
spacing[i] = double_samples[i+1].time - double_samples[i].time;
}
std::sort(spacing.begin(), spacing.end());
double median_spacing = spacing[spacing.size()/2];
// Set line_break_threshold to larger of 4*median_spacing and 4*bin_width
line_break_threshold = std::max(line_break_threshold, median_spacing * 4);
}
if (graph_samples.size()) {
log_f("gettile: outputting %zd samples", graph_samples.size());
Json::Value tile(Json::objectValue);
tile["level"] = Json::Value(tile_level);
// An aside about offset type and precision:
// JSONCPP doesn't have a long long type; to preserve full resolution we need to convert to double here. As Javascript itself
// will read this as a double-precision value, we're not introducing a problem.
// For a detailed discussion, see https://sites.google.com/a/bodytrack.org/wiki/website/tile-coordinates-and-numeric-precision
// Irritatingly, JSONCPP wants to add ".0" to the end of floating-point numbers that don't need it. This is inconsistent
// with Javascript itself and simply introduces extra bytes to the representation
tile["offset"] = Json::Value((double)tile_offset);
tile["fields"] = Json::Value(Json::arrayValue);
tile["fields"].append(Json::Value("time"));
tile["fields"].append(Json::Value("mean"));
tile["fields"].append(Json::Value("stddev"));
tile["fields"].append(Json::Value("count"));
if (has_fifth_col) tile["fields"].append(Json::Value("comment"));
Json::Value data(Json::arrayValue);
double previous_sample_time = client_tile_index.start_time();
bool previous_had_value = true;
for (unsigned i = 0; i < graph_samples.size(); i++) {
// TODO: improve linebreak calculations:
// 1) observe channel specs line break size from database (expressed in time; some observations have long time periods and others short)
// 2) insert breaks at beginning or end of tile if needed
// 3) should client be the one to decide where line breaks are (if we give it the threshold?)
if (graph_samples[i].time - previous_sample_time > line_break_threshold ||
!graph_samples[i].has_value || !previous_had_value) {
// Insert line break, which has value -1e+308
Json::Value sample = Json::Value(Json::arrayValue);
sample.append(Json::Value(0.5*(graph_samples[i].time+previous_sample_time)));
sample.append(Json::Value(-1e308));
sample.append(Json::Value(0));
sample.append(Json::Value(0));
if (has_fifth_col) sample.append(Json::Value()); // NULL
data.append(sample);
}
previous_sample_time = graph_samples[i].time;
previous_had_value = graph_samples[i].has_value;
{
Json::Value sample = Json::Value(Json::arrayValue);
sample.append(Json::Value(graph_samples[i].time));
sample.append(Json::Value(graph_samples[i].has_value ? graph_samples[i].value : 0.0));
// TODO: fix datastore so we never see NAN crop up here!
sample.append(Json::Value(isnan(graph_samples[i].stddev) ? 0 : graph_samples[i].stddev));
sample.append(Json::Value(graph_samples[i].weight));
if (has_fifth_col) {
sample.append(graph_samples[i].has_comment ? Json::Value(graph_samples[i].comment) : Json::Value());
}
data.append(sample);
}
}
if (client_tile_index.end_time() - previous_sample_time > line_break_threshold ||
!previous_had_value) {
// Insert line break, which has value -1e+308
Json::Value sample = Json::Value(Json::arrayValue);
sample.append(Json::Value(0.5*(previous_sample_time + client_tile_index.end_time())));
sample.append(Json::Value(-1e308));
sample.append(Json::Value(0));
sample.append(Json::Value(0));
if (has_fifth_col) sample.append(Json::Value()); // NULL
data.append(sample);
}
tile["data"] = data;
// only include the sample_width field if we actually binned
if (doubles_binned) {
tile["sample_width"] = bin_width;
}
printf("%s\n", rtrim(Json::FastWriter().write(tile)).c_str());
} else {
log_f("gettile: no samples");
printf("{}");
}
log_f("gettile: finished in %lld msec", millitime() - begin_time);
return 0;
}
