void rw_new() {
// clean up from any previous run
rm_hashdb_dir(temp_dir);
make_dir_if_not_there(temp_dir);
// create each db
lmdb_hash_store_t hash_store(temp_dir, RW_NEW, 512, 0);
lmdb_name_store_t name_store(temp_dir, RW_NEW);
lmdb_source_store_t source_store(temp_dir, RW_NEW);
}
void rw_modify_source_store() {
// open db
lmdb_source_store_t source_store(temp_dir, RW_MODIFY);
// checks while size is zero
lmdb_source_it_data_t it_data;
TEST_EQ(source_store.size(), 0);
it_data = source_store.find_first();
TEST_EQ(it_data.is_valid, false);
// add data0 ID 0
lmdb_source_data_t data0("r2", "fn3", 4, "hash5");
TEST_EQ(source_store.add(0, data0), true);
TEST_EQ(source_store.add(0, data0), false);
// add data ID 1
lmdb_source_data_t data;
data.repository_name = "rn";
data.filename = "fn";
data.filesize = 1;
TEST_EQ(source_store.add(1, data), true);
TEST_EQ(source_store.add(1, data), false);
// changing file size zeroes out filesize and file hash
data.filesize = 20;
TEST_EQ(source_store.add(1, data), true);
TEST_EQ(source_store.add(1, data), false);
// check data0 ID 0
lmdb_source_data_t data0b = source_store.find(0);
TEST_EQ(data0b.repository_name, "r2");
TEST_EQ(data0b.filename, "fn3");
TEST_EQ(data0b.filesize, 4);
TEST_EQ(data0b.binary_hash, "hash5");
// check data ID 1
lmdb_source_data_t data1b = source_store.find(1);
TEST_EQ(data1b.repository_name, "rn");
TEST_EQ(data1b.filename, "fn");
TEST_EQ(data1b.filesize, 0);
TEST_EQ(data1b.binary_hash, lmdb_helper::hex_to_binary_hash("00"));
// check iterator
it_data = source_store.find_first();
TEST_EQ(it_data.source_data.repository_name, "r2");
TEST_EQ(it_data.is_valid, true);
it_data = source_store.find_next(it_data.source_lookup_index);
TEST_EQ(it_data.source_data.repository_name, "rn");
it_data = source_store.find_next(it_data.source_lookup_index);
TEST_EQ(it_data.is_valid, false);
TEST_EQ(source_store.size(), 2);
}
void read_only_source_store() {
// open db
lmdb_source_store_t source_store(temp_dir, READ_ONLY);
lmdb_source_data_t data;
lmdb_source_it_data_t it_data;
it_data = source_store.find_first();
TEST_EQ(it_data.source_data.repository_name, "r2");
TEST_EQ(it_data.is_valid, true);
it_data = source_store.find_next(it_data.source_lookup_index);
TEST_EQ(it_data.source_data.repository_name, "rn");
TEST_EQ(it_data.is_valid, true);
it_data = source_store.find_next(it_data.source_lookup_index);
TEST_EQ(it_data.is_valid, false);
}
int main(int argc, char** argv) {
try {
std::ostringstream oss;
oss << "Usage: " << argv[0] << " ACTION [additional options]";
po::options_description desc(oss.str());
desc.add_options()
("help,h", "produce help message")
("version,v", "print libklio version and exit")
("action,a", po::value<std::string>(), "Valid actions are: create, check, sync, upgrade")
("storefile,s", po::value<std::string>(), "the data store to use")
("sourcestore,r", po::value<std::string>(), "the data store to use as source for synchronization")
;
po::positional_options_description p;
p.add("action", 1);
p.add("storefile", 1);
po::variables_map vm;
po::store(po::command_line_parser(argc, argv).
options(desc).positional(p).run(), vm);
po::notify(vm);
// Begin processing of command line parameters.
std::string action;
std::string storefile;
if (vm.count("help")) {
std::cout << desc << std::endl;
return 0;
}
if (vm.count("version")) {
klio::VersionInfo::Ptr vi(new klio::VersionInfo());
std::cout << "klio library version " << vi->getVersion() << std::endl;
return 0;
}
if (!vm.count("storefile")) {
std::cerr << "You must specify a store to work on." << std::endl;
return 1;
} else {
storefile = vm["storefile"].as<std::string>();
}
if (!vm.count("action")) {
std::cerr << "You must specify an action." << std::endl;
return 1;
} else {
action = (vm["action"].as<std::string>());
}
klio::StoreFactory::Ptr factory(new klio::StoreFactory());
bfs::path db(storefile);
/**
* CREATE action
*/
if (boost::iequals(action, std::string("create"))) {
if (bfs::exists(db)) {
std::cerr << "File " << db << " already exists, exiting." << std::endl;
return 2;
}
try {
std::cout << "Attempting to create " << db << std::endl;
klio::Store::Ptr store(factory->create_sqlite3_store(db));
std::cout << "Initialized store: " << store->str() << std::endl;
} catch (klio::StoreException const& ex) {
std::cout << "Failed to create: " << ex.what() << std::endl;
return 1;
}
/**
* CHECK action
*/
} else if (boost::iequals(action, std::string("check"))) {
if (!bfs::exists(db)) {
std::cerr << "File " << db << " does not exist, exiting." << std::endl;
return 2;
}
try {
std::cout << "Attempting to open " << db << std::endl;
klio::Store::Ptr store(factory->open_sqlite3_store(db));
std::cout << "opened store: " << store->str() << std::endl;
std::vector<klio::Sensor::uuid_t> uuids = store->get_sensor_uuids();
std::vector<klio::Sensor::uuid_t>::iterator it;
klio::timestamp_t all_sensors_last_timestamp = 0;
// First pass: get the latest timestamps of all sensors
std::cout << "First pass: get the latest timestamps of all sensors" << db << std::endl;
for (it = uuids.begin(); it < uuids.end(); it++) {
klio::Sensor::Ptr loadedSensor(store->get_sensor(*it));
klio::reading_t last_reading = store->get_last_reading(loadedSensor);
all_sensors_last_timestamp = std::max(all_sensors_last_timestamp, last_reading.first);
}
std::cout << std::setw(5) << "stat";
std::cout << std::setw(8) << "# val";
std::cout << std::setw(8) << "dt med";
std::cout << std::setw(8) << "dt min";
std::cout << std::setw(8) << "dt max";
std::cout << std::setw(8) << "avg(W)";
std::cout << std::setw(8) << "min(W)";
std::cout << std::setw(8) << "max(W)";
std::cout << std::setw(21) << "first timestamp";
std::cout << std::setw(21) << "last timestamp";
std::cout << '\t' << "type";
std::cout << std::setw(8) << "unit";
std::cout << '\t' << "sensor name / description" << std::endl;
for (it = uuids.begin(); it < uuids.end(); it++) {
klio::Sensor::Ptr loadedSensor(store->get_sensor(*it));
klio::readings_t_Ptr readings;
readings = store->get_all_readings(loadedSensor);
// loop over all readings and calculate metrics
uint64_t num_readings = 0;
klio::timestamp_t first_timestamp = 0;
klio::timestamp_t last_timestamp = 0;
uint32_t min_timestamp_interval = std::numeric_limits<uint32_t>::max();
uint32_t max_timestamp_interval = std::numeric_limits<uint32_t>::min();
double min_value = std::numeric_limits<double>::max();
double max_value = std::numeric_limits<double>::min();
double aggregated_value = 0;
std::vector<uint32_t> all_intervals;
for (klio::readings_it_t it = readings->begin();
it != readings->end(); it++) {
num_readings++;
klio::timestamp_t ts1 = (*it).first;
double val1 = (*it).second;
min_value = std::min(min_value, val1);
max_value = std::max(max_value, val1);
aggregated_value += val1;
if (last_timestamp == 0) {
// this is the first value we read, initialize state
first_timestamp = ts1;
last_timestamp = ts1;
} else {
// everything set up, this is just another value
uint32_t interval = ts1 - last_timestamp;
all_intervals.push_back(interval);
min_timestamp_interval = std::min(min_timestamp_interval, interval);
max_timestamp_interval = std::max(max_timestamp_interval, interval);
// Update state variables
last_timestamp = ts1;
}
}
size_t middle_element_idx = all_intervals.size() / 2;
uint32_t mean_interval = 0;
if (middle_element_idx != 0) {
std::nth_element(all_intervals.begin(),
all_intervals.begin() + middle_element_idx,
all_intervals.end());
mean_interval = all_intervals[middle_element_idx];
}
// compose output line
std::ostringstream stat_oss;
if (last_timestamp + (60*60) < all_sensors_last_timestamp) {
// no value in the last hour - mark as late
stat_oss << "L"; // mark as late
} else {
stat_oss << "C"; // mark as current
}
std::cout << std::setfill(' ') << std::setw(5) << stat_oss.str();
std::cout << std::setw(8) << num_readings;
std::cout << std::setw(8) << mean_interval;
// print mean of intervals
if (min_timestamp_interval == std::numeric_limits<uint32_t>::max() ||
max_timestamp_interval == std::numeric_limits<uint32_t>::min()) {
std::cout << std::setw(8) << "n/a";
std::cout << std::setw(8) << "n/a";
} else {
std::cout << std::setw(8) << min_timestamp_interval;
std::cout << std::setw(8) << max_timestamp_interval;
}
// value stats
if (num_readings == 0) {
std::cout << std::setw(8) << "n/a";
} else {
std::cout << std::setw(8) << (uint32_t) aggregated_value / num_readings;
}
std::cout << std::setw(8) << (uint32_t) min_value;
std::cout << std::setw(8) << (uint32_t) max_value;
// Time conversion foo
klio::LocalTime::Ptr lt(new klio::LocalTime("."));
boost::local_time::local_time_facet* output_facet
= new boost::local_time::local_time_facet();
output_facet->format("%Y.%m.%d-%H:%M:%S");
std::ostringstream oss;
oss.imbue(std::locale(std::locale::classic(), output_facet));
boost::local_time::local_date_time first_timestamp_datetime =
lt->get_local_time(loadedSensor, first_timestamp);
oss.str("");
oss << first_timestamp_datetime;
std::cout << std::setw(21) << oss.str();
boost::local_time::local_date_time last_timestamp_datetime =
lt->get_local_time(loadedSensor, last_timestamp);
oss.str("");
oss << last_timestamp_datetime;
std::cout << std::setw(21) << oss.str();
// sensor data
std::cout << '\t' << loadedSensor->device_type()->name();
// unit
std::cout << '\t' << loadedSensor->unit();
// name and description
std::cout << '\t' << loadedSensor->name();
std::cout << '\t' << loadedSensor->description();
// for debugging only: external_id
//std::cout << '\t' << '#' << loadedSensor->external_id();
std::cout << std::endl;
}
} catch (klio::StoreException const& ex) {
std::cout << "Failed to check: " << ex.what() << std::endl;
return 1;
}
/**
* SYNC action
*/
} else if (boost::iequals(action, std::string("sync"))) {
std::string sourcestore;
if (!vm.count("sourcestore")) {
std::cerr << "You must specify a source store for synchronization." << std::endl;
return 1;
} else {
sourcestore = vm["sourcestore"].as<std::string>();
}
bfs::path source_path(sourcestore);
if (!bfs::exists(source_path)) {
std::cerr << "File " << source_path << " does not exist, exiting." << std::endl;
return 2;
}
bfs::path target_path(storefile);
if (!bfs::exists(target_path)) {
std::cerr << "File " << target_path << " does not exist, exiting." << std::endl;
return 2;
}
try {
std::cout << "Attempting to open source store " << source_path << std::endl;
klio::Store::Ptr source_store(factory->open_sqlite3_store(source_path));
std::cout << "Opened source store: " << source_store->str() << std::endl;
std::cout << "Attempting to open target store " << target_path << std::endl;
klio::Store::Ptr target_store(factory->open_sqlite3_store(target_path));
std::cout << "Opened target store: " << target_store->str() << std::endl;
std::cout << "Synchronizing stores..." << std::endl;
target_store->sync(source_store);
std::cout << "Stores Synchronized." << std::endl;
source_store->close();
target_store->close();
} catch (klio::StoreException const& ex) {
std::cout << "Failed to synchronize stores. " << ex.what() << std::endl;
return 1;
}
/**
* UPGRADE action
*/
} else if (boost::iequals(action, std::string("upgrade"))) {
try {
klio::VersionInfo::Ptr info = klio::VersionInfo::Ptr(new klio::VersionInfo());
std::cout << "Attempting to upgrade " << db << " to version " << info->getVersion() << std::endl;
klio::SQLite3Store::Ptr store(factory->create_sqlite3_store(db, false));
store->upgrade();
std::cout << "Store upgraded." << std::endl;
} catch (klio::StoreException const& ex) {
std::cout << "Failed to upgrade: " << ex.what() << std::endl;
return 1;
}
/**
* unknown command
*/
} else {
std::cerr << "Unknown command " << action << std::endl;
return 1;
}
} catch (std::exception& e) {
std::cerr << "error: " << e.what() << std::endl;
return 1;
} catch (...) {
std::cerr << "Exception of unknown type!" << std::endl;
return 1;
}
return 0;
}
