void LinkBuilder::build(const hdf5::node::Node &parent) const
{
hdf5::node::Group link_parent(parent);
std::string link_name = node().attribute("name").str_data();
pni::io::nexus::Path link_target(pni::io::nexus::Path::from_string(node().attribute("target").str_data()));
if(link_target.has_filename())
{
//create an external link
boost::filesystem::path file_path = link_target.filename();
hdf5::Path target_path(link_target);
hdf5::node::link(file_path,target_path,link_parent,link_name);
}
else
{
//create a soft link
hdf5::Path target_path(link_target);
hdf5::node::link(target_path,link_parent,link_name);
}
}
virtual int copy_file_to_hash
(const entry_t& source, const path_t& target, hash_t& hash) {
entry_t& entry = target_entry_;
string_t target_path(target.chars());
const char_t* chars = 0;
int err = 1;
if ((append_hash_name_to_target_path_) && (chars = hash.name())) {
target_path.append(&target.extension_separator(), 1);
target_path.append(hash_name_prefix_);
target_path.append(chars);
target_path.append(hash_name_suffix_);
}
if ((entry.exists(chars = target_path.chars()))) {
if ((write_overwrite != write_) && (write_append != write_)) {
errf("target file \"%s\" already exists\n", chars);
} else {
fs::entry_type type = fs::entry_type_none;
switch (type = entry.type()) {
case fs::entry_type_file:
err = copy_file_to_file_hash(source, entry, hash);
break;
default:
break;
}
}
} else {
if (!(err = make_directory(entry, target))) {
entry.set_path(chars);
err = copy_file_to_file_hash(source, entry, hash);
} else {
errf("failed to make directory \"%s\"\n", target.directory().chars());
}
}
if (!(err) && (!(to_same != to_) || !(target_modified_))) {
if ((entry.set_times_to_set(source))) {
if ((entry.set_times_set())) {
} else {
}
}
}
return err;
}
void ObjectCollection::SaveToStreams()
{
typedef std::pair<String, Array<TypedCompoundPointer>> FileEntry;
typedef std::unordered_map<String, Array<TypedCompoundPointer>> FileMap;
FileMap map;
for (TypedCompoundPointer& p : mObjects)
{
String* loc = p.GetCompoundMember<String>("!location");
gAssert(loc != nullptr);
FileMap::iterator i = map.find(*loc);
if (i == map.end())
map.insert(FileEntry(*loc, Array<TypedCompoundPointer>(&p, 1)));
else
i->second.Append(p);
}
for (const FileEntry& entry : map)
{
Path target_path(entry.first);
StreamDevice* d = gDevices.FindDevice(target_path.GetDeviceName());
gAssert(d != nullptr);
Stream* s = d->CreateStream(target_path, smWrite);
gAssert(s != nullptr);
ObjectWriter wr(*s);
std::cout << "Opened stream to " << target_path << std::endl;
for (const TypedCompoundPointer& object : entry.second)
{
const String* name = object.GetCompoundMember<String>("!name");
const String* loc = object.GetCompoundMember<String>("!location");
std::cout << " Writing: " << (loc != nullptr ? *loc : String("<noloc>")) << " : " << (name != nullptr ? *name : String("<noname>")) << std::endl;
wr.WriteTypedCompoundPointer(object);
}
d->CloseStream(s);
}
}
// Note that, blob name is case sensitive.
void download_block_blob(azure::storage::cloud_blob_container &container,
const utility::string_t &blob_name, const utility::string_t &target_file_name,
utility::size64_t downloading_bytes_each_time)
{
auto blob = container.get_block_blob_reference(blob_name);
if (!blob.exists()) {
ucerr << U("Block blob \"") << blob_name << U("\" doesn't exist.\n");
return;
}
boost::filesystem::path target_path(target_file_name);
if (boost::filesystem::exists(target_path))
{
ucout << U("Warning! File \"") << target_file_name
<< U("\" already exists, and it will be overwrite.") << std::endl;
}
else
{
target_path.remove_filename();
if (!boost::filesystem::exists(target_path))
{
ucout << U("Creating directory \"") << target_path.string<utility::string_t>()
<< U("\"...") << std::endl;
if (!boost::filesystem::create_directories(target_path))
{
ucerr << U("Failed to create directory \"")
<< target_path.string<utility::string_t>() << U("\"!\n");
return;
}
}
}
// Get blob's size(in bytes)
auto blob_size = blob.properties().size();
if (0 == blob_size) {
blob.download_to_file(target_file_name);
return;
}
assert(downloading_bytes_each_time > 0);
if (downloading_bytes_each_time == 0 || downloading_bytes_each_time > blob_size) {
downloading_bytes_each_time = blob_size;
}
std::ofstream out_file(target_file_name, std::ofstream::binary);
if (!out_file) {
ucout << U("Opening file \"") << target_file_name << U("\" failed.\n");
return;
}
ucout << U("Downloading file to: ") << target_file_name << U("...");
boost::progress_display prog(blob_size);
utility::size64_t offset = 0;
utility::size64_t bytes_read;
while (offset < blob_size) {
// Create a memory buffer
concurrency::streams::container_buffer<std::vector<uint8_t>> buffer;
// Read data
concurrency::streams::ostream tmp_out_stream(buffer);
blob.download_range_to_stream(tmp_out_stream, offset, downloading_bytes_each_time);
bytes_read = buffer.collection().size();
// Write out...
out_file.write(reinterpret_cast<char *>(buffer.collection().data()), bytes_read);
//out_file.flush();
// Upgrade the progress bar and other states
prog += bytes_read;
offset += bytes_read;
}
// The last step...
blob.download_block_list();
}
int main(int argc, char** argv) {
try {
std::ostringstream oss;
oss << "Usage: " << argv[0] << " ACTION [additional options]";
po::options_description desc(oss.str());
desc.add_options()
("help,h", "produce help message")
("version,v", "print libklio version and exit")
("action,a", po::value<std::string>(), "Valid actions are: create, check, sync, upgrade")
("storefile,s", po::value<std::string>(), "the data store to use")
("sourcestore,r", po::value<std::string>(), "the data store to use as source for synchronization")
;
po::positional_options_description p;
p.add("action", 1);
p.add("storefile", 1);
po::variables_map vm;
po::store(po::command_line_parser(argc, argv).
options(desc).positional(p).run(), vm);
po::notify(vm);
// Begin processing of command line parameters.
std::string action;
std::string storefile;
if (vm.count("help")) {
std::cout << desc << std::endl;
return 0;
}
if (vm.count("version")) {
klio::VersionInfo::Ptr vi(new klio::VersionInfo());
std::cout << "klio library version " << vi->getVersion() << std::endl;
return 0;
}
if (!vm.count("storefile")) {
std::cerr << "You must specify a store to work on." << std::endl;
return 1;
} else {
storefile = vm["storefile"].as<std::string>();
}
if (!vm.count("action")) {
std::cerr << "You must specify an action." << std::endl;
return 1;
} else {
action = (vm["action"].as<std::string>());
}
klio::StoreFactory::Ptr factory(new klio::StoreFactory());
bfs::path db(storefile);
/**
* CREATE action
*/
if (boost::iequals(action, std::string("create"))) {
if (bfs::exists(db)) {
std::cerr << "File " << db << " already exists, exiting." << std::endl;
return 2;
}
try {
std::cout << "Attempting to create " << db << std::endl;
klio::Store::Ptr store(factory->create_sqlite3_store(db));
std::cout << "Initialized store: " << store->str() << std::endl;
} catch (klio::StoreException const& ex) {
std::cout << "Failed to create: " << ex.what() << std::endl;
return 1;
}
/**
* CHECK action
*/
} else if (boost::iequals(action, std::string("check"))) {
if (!bfs::exists(db)) {
std::cerr << "File " << db << " does not exist, exiting." << std::endl;
return 2;
}
try {
std::cout << "Attempting to open " << db << std::endl;
klio::Store::Ptr store(factory->open_sqlite3_store(db));
std::cout << "opened store: " << store->str() << std::endl;
std::vector<klio::Sensor::uuid_t> uuids = store->get_sensor_uuids();
std::vector<klio::Sensor::uuid_t>::iterator it;
klio::timestamp_t all_sensors_last_timestamp = 0;
// First pass: get the latest timestamps of all sensors
std::cout << "First pass: get the latest timestamps of all sensors" << db << std::endl;
for (it = uuids.begin(); it < uuids.end(); it++) {
klio::Sensor::Ptr loadedSensor(store->get_sensor(*it));
klio::reading_t last_reading = store->get_last_reading(loadedSensor);
all_sensors_last_timestamp = std::max(all_sensors_last_timestamp, last_reading.first);
}
std::cout << std::setw(5) << "stat";
std::cout << std::setw(8) << "# val";
std::cout << std::setw(8) << "dt med";
std::cout << std::setw(8) << "dt min";
std::cout << std::setw(8) << "dt max";
std::cout << std::setw(8) << "avg(W)";
std::cout << std::setw(8) << "min(W)";
std::cout << std::setw(8) << "max(W)";
std::cout << std::setw(21) << "first timestamp";
std::cout << std::setw(21) << "last timestamp";
std::cout << '\t' << "type";
std::cout << std::setw(8) << "unit";
std::cout << '\t' << "sensor name / description" << std::endl;
for (it = uuids.begin(); it < uuids.end(); it++) {
klio::Sensor::Ptr loadedSensor(store->get_sensor(*it));
klio::readings_t_Ptr readings;
readings = store->get_all_readings(loadedSensor);
// loop over all readings and calculate metrics
uint64_t num_readings = 0;
klio::timestamp_t first_timestamp = 0;
klio::timestamp_t last_timestamp = 0;
uint32_t min_timestamp_interval = std::numeric_limits<uint32_t>::max();
uint32_t max_timestamp_interval = std::numeric_limits<uint32_t>::min();
double min_value = std::numeric_limits<double>::max();
double max_value = std::numeric_limits<double>::min();
double aggregated_value = 0;
std::vector<uint32_t> all_intervals;
for (klio::readings_it_t it = readings->begin();
it != readings->end(); it++) {
num_readings++;
klio::timestamp_t ts1 = (*it).first;
double val1 = (*it).second;
min_value = std::min(min_value, val1);
max_value = std::max(max_value, val1);
aggregated_value += val1;
if (last_timestamp == 0) {
// this is the first value we read, initialize state
first_timestamp = ts1;
last_timestamp = ts1;
} else {
// everything set up, this is just another value
uint32_t interval = ts1 - last_timestamp;
all_intervals.push_back(interval);
min_timestamp_interval = std::min(min_timestamp_interval, interval);
max_timestamp_interval = std::max(max_timestamp_interval, interval);
// Update state variables
last_timestamp = ts1;
}
}
size_t middle_element_idx = all_intervals.size() / 2;
uint32_t mean_interval = 0;
if (middle_element_idx != 0) {
std::nth_element(all_intervals.begin(),
all_intervals.begin() + middle_element_idx,
all_intervals.end());
mean_interval = all_intervals[middle_element_idx];
}
// compose output line
std::ostringstream stat_oss;
if (last_timestamp + (60*60) < all_sensors_last_timestamp) {
// no value in the last hour - mark as late
stat_oss << "L"; // mark as late
} else {
stat_oss << "C"; // mark as current
}
std::cout << std::setfill(' ') << std::setw(5) << stat_oss.str();
std::cout << std::setw(8) << num_readings;
std::cout << std::setw(8) << mean_interval;
// print mean of intervals
if (min_timestamp_interval == std::numeric_limits<uint32_t>::max() ||
max_timestamp_interval == std::numeric_limits<uint32_t>::min()) {
std::cout << std::setw(8) << "n/a";
std::cout << std::setw(8) << "n/a";
} else {
std::cout << std::setw(8) << min_timestamp_interval;
std::cout << std::setw(8) << max_timestamp_interval;
}
// value stats
if (num_readings == 0) {
std::cout << std::setw(8) << "n/a";
} else {
std::cout << std::setw(8) << (uint32_t) aggregated_value / num_readings;
}
std::cout << std::setw(8) << (uint32_t) min_value;
std::cout << std::setw(8) << (uint32_t) max_value;
// Time conversion foo
klio::LocalTime::Ptr lt(new klio::LocalTime("."));
boost::local_time::local_time_facet* output_facet
= new boost::local_time::local_time_facet();
output_facet->format("%Y.%m.%d-%H:%M:%S");
std::ostringstream oss;
oss.imbue(std::locale(std::locale::classic(), output_facet));
boost::local_time::local_date_time first_timestamp_datetime =
lt->get_local_time(loadedSensor, first_timestamp);
oss.str("");
oss << first_timestamp_datetime;
std::cout << std::setw(21) << oss.str();
boost::local_time::local_date_time last_timestamp_datetime =
lt->get_local_time(loadedSensor, last_timestamp);
oss.str("");
oss << last_timestamp_datetime;
std::cout << std::setw(21) << oss.str();
// sensor data
std::cout << '\t' << loadedSensor->device_type()->name();
// unit
std::cout << '\t' << loadedSensor->unit();
// name and description
std::cout << '\t' << loadedSensor->name();
std::cout << '\t' << loadedSensor->description();
// for debugging only: external_id
//std::cout << '\t' << '#' << loadedSensor->external_id();
std::cout << std::endl;
}
} catch (klio::StoreException const& ex) {
std::cout << "Failed to check: " << ex.what() << std::endl;
return 1;
}
/**
* SYNC action
*/
} else if (boost::iequals(action, std::string("sync"))) {
std::string sourcestore;
if (!vm.count("sourcestore")) {
std::cerr << "You must specify a source store for synchronization." << std::endl;
return 1;
} else {
sourcestore = vm["sourcestore"].as<std::string>();
}
bfs::path source_path(sourcestore);
if (!bfs::exists(source_path)) {
std::cerr << "File " << source_path << " does not exist, exiting." << std::endl;
return 2;
}
bfs::path target_path(storefile);
if (!bfs::exists(target_path)) {
std::cerr << "File " << target_path << " does not exist, exiting." << std::endl;
return 2;
}
try {
std::cout << "Attempting to open source store " << source_path << std::endl;
klio::Store::Ptr source_store(factory->open_sqlite3_store(source_path));
std::cout << "Opened source store: " << source_store->str() << std::endl;
std::cout << "Attempting to open target store " << target_path << std::endl;
klio::Store::Ptr target_store(factory->open_sqlite3_store(target_path));
std::cout << "Opened target store: " << target_store->str() << std::endl;
std::cout << "Synchronizing stores..." << std::endl;
target_store->sync(source_store);
std::cout << "Stores Synchronized." << std::endl;
source_store->close();
target_store->close();
} catch (klio::StoreException const& ex) {
std::cout << "Failed to synchronize stores. " << ex.what() << std::endl;
return 1;
}
/**
* UPGRADE action
*/
} else if (boost::iequals(action, std::string("upgrade"))) {
try {
klio::VersionInfo::Ptr info = klio::VersionInfo::Ptr(new klio::VersionInfo());
std::cout << "Attempting to upgrade " << db << " to version " << info->getVersion() << std::endl;
klio::SQLite3Store::Ptr store(factory->create_sqlite3_store(db, false));
store->upgrade();
std::cout << "Store upgraded." << std::endl;
} catch (klio::StoreException const& ex) {
std::cout << "Failed to upgrade: " << ex.what() << std::endl;
return 1;
}
/**
* unknown command
*/
} else {
std::cerr << "Unknown command " << action << std::endl;
return 1;
}
} catch (std::exception& e) {
std::cerr << "error: " << e.what() << std::endl;
return 1;
} catch (...) {
std::cerr << "Exception of unknown type!" << std::endl;
return 1;
}
return 0;
}
