void LinkBuilder::build(const hdf5::node::Node &parent) const { hdf5::node::Group link_parent(parent); std::string link_name = node().attribute("name").str_data(); pni::io::nexus::Path link_target(pni::io::nexus::Path::from_string(node().attribute("target").str_data())); if(link_target.has_filename()) { //create an external link boost::filesystem::path file_path = link_target.filename(); hdf5::Path target_path(link_target); hdf5::node::link(file_path,target_path,link_parent,link_name); } else { //create a soft link hdf5::Path target_path(link_target); hdf5::node::link(target_path,link_parent,link_name); } }
virtual int copy_file_to_hash (const entry_t& source, const path_t& target, hash_t& hash) { entry_t& entry = target_entry_; string_t target_path(target.chars()); const char_t* chars = 0; int err = 1; if ((append_hash_name_to_target_path_) && (chars = hash.name())) { target_path.append(&target.extension_separator(), 1); target_path.append(hash_name_prefix_); target_path.append(chars); target_path.append(hash_name_suffix_); } if ((entry.exists(chars = target_path.chars()))) { if ((write_overwrite != write_) && (write_append != write_)) { errf("target file \"%s\" already exists\n", chars); } else { fs::entry_type type = fs::entry_type_none; switch (type = entry.type()) { case fs::entry_type_file: err = copy_file_to_file_hash(source, entry, hash); break; default: break; } } } else { if (!(err = make_directory(entry, target))) { entry.set_path(chars); err = copy_file_to_file_hash(source, entry, hash); } else { errf("failed to make directory \"%s\"\n", target.directory().chars()); } } if (!(err) && (!(to_same != to_) || !(target_modified_))) { if ((entry.set_times_to_set(source))) { if ((entry.set_times_set())) { } else { } } } return err; }
void ObjectCollection::SaveToStreams() { typedef std::pair<String, Array<TypedCompoundPointer>> FileEntry; typedef std::unordered_map<String, Array<TypedCompoundPointer>> FileMap; FileMap map; for (TypedCompoundPointer& p : mObjects) { String* loc = p.GetCompoundMember<String>("!location"); gAssert(loc != nullptr); FileMap::iterator i = map.find(*loc); if (i == map.end()) map.insert(FileEntry(*loc, Array<TypedCompoundPointer>(&p, 1))); else i->second.Append(p); } for (const FileEntry& entry : map) { Path target_path(entry.first); StreamDevice* d = gDevices.FindDevice(target_path.GetDeviceName()); gAssert(d != nullptr); Stream* s = d->CreateStream(target_path, smWrite); gAssert(s != nullptr); ObjectWriter wr(*s); std::cout << "Opened stream to " << target_path << std::endl; for (const TypedCompoundPointer& object : entry.second) { const String* name = object.GetCompoundMember<String>("!name"); const String* loc = object.GetCompoundMember<String>("!location"); std::cout << " Writing: " << (loc != nullptr ? *loc : String("<noloc>")) << " : " << (name != nullptr ? *name : String("<noname>")) << std::endl; wr.WriteTypedCompoundPointer(object); } d->CloseStream(s); } }
// Note that, blob name is case sensitive. void download_block_blob(azure::storage::cloud_blob_container &container, const utility::string_t &blob_name, const utility::string_t &target_file_name, utility::size64_t downloading_bytes_each_time) { auto blob = container.get_block_blob_reference(blob_name); if (!blob.exists()) { ucerr << U("Block blob \"") << blob_name << U("\" doesn't exist.\n"); return; } boost::filesystem::path target_path(target_file_name); if (boost::filesystem::exists(target_path)) { ucout << U("Warning! File \"") << target_file_name << U("\" already exists, and it will be overwrite.") << std::endl; } else { target_path.remove_filename(); if (!boost::filesystem::exists(target_path)) { ucout << U("Creating directory \"") << target_path.string<utility::string_t>() << U("\"...") << std::endl; if (!boost::filesystem::create_directories(target_path)) { ucerr << U("Failed to create directory \"") << target_path.string<utility::string_t>() << U("\"!\n"); return; } } } // Get blob's size(in bytes) auto blob_size = blob.properties().size(); if (0 == blob_size) { blob.download_to_file(target_file_name); return; } assert(downloading_bytes_each_time > 0); if (downloading_bytes_each_time == 0 || downloading_bytes_each_time > blob_size) { downloading_bytes_each_time = blob_size; } std::ofstream out_file(target_file_name, std::ofstream::binary); if (!out_file) { ucout << U("Opening file \"") << target_file_name << U("\" failed.\n"); return; } ucout << U("Downloading file to: ") << target_file_name << U("..."); boost::progress_display prog(blob_size); utility::size64_t offset = 0; utility::size64_t bytes_read; while (offset < blob_size) { // Create a memory buffer concurrency::streams::container_buffer<std::vector<uint8_t>> buffer; // Read data concurrency::streams::ostream tmp_out_stream(buffer); blob.download_range_to_stream(tmp_out_stream, offset, downloading_bytes_each_time); bytes_read = buffer.collection().size(); // Write out... out_file.write(reinterpret_cast<char *>(buffer.collection().data()), bytes_read); //out_file.flush(); // Upgrade the progress bar and other states prog += bytes_read; offset += bytes_read; } // The last step... blob.download_block_list(); }
int main(int argc, char** argv) { try { std::ostringstream oss; oss << "Usage: " << argv[0] << " ACTION [additional options]"; po::options_description desc(oss.str()); desc.add_options() ("help,h", "produce help message") ("version,v", "print libklio version and exit") ("action,a", po::value<std::string>(), "Valid actions are: create, check, sync, upgrade") ("storefile,s", po::value<std::string>(), "the data store to use") ("sourcestore,r", po::value<std::string>(), "the data store to use as source for synchronization") ; po::positional_options_description p; p.add("action", 1); p.add("storefile", 1); po::variables_map vm; po::store(po::command_line_parser(argc, argv). options(desc).positional(p).run(), vm); po::notify(vm); // Begin processing of command line parameters. std::string action; std::string storefile; if (vm.count("help")) { std::cout << desc << std::endl; return 0; } if (vm.count("version")) { klio::VersionInfo::Ptr vi(new klio::VersionInfo()); std::cout << "klio library version " << vi->getVersion() << std::endl; return 0; } if (!vm.count("storefile")) { std::cerr << "You must specify a store to work on." << std::endl; return 1; } else { storefile = vm["storefile"].as<std::string>(); } if (!vm.count("action")) { std::cerr << "You must specify an action." << std::endl; return 1; } else { action = (vm["action"].as<std::string>()); } klio::StoreFactory::Ptr factory(new klio::StoreFactory()); bfs::path db(storefile); /** * CREATE action */ if (boost::iequals(action, std::string("create"))) { if (bfs::exists(db)) { std::cerr << "File " << db << " already exists, exiting." << std::endl; return 2; } try { std::cout << "Attempting to create " << db << std::endl; klio::Store::Ptr store(factory->create_sqlite3_store(db)); std::cout << "Initialized store: " << store->str() << std::endl; } catch (klio::StoreException const& ex) { std::cout << "Failed to create: " << ex.what() << std::endl; return 1; } /** * CHECK action */ } else if (boost::iequals(action, std::string("check"))) { if (!bfs::exists(db)) { std::cerr << "File " << db << " does not exist, exiting." << std::endl; return 2; } try { std::cout << "Attempting to open " << db << std::endl; klio::Store::Ptr store(factory->open_sqlite3_store(db)); std::cout << "opened store: " << store->str() << std::endl; std::vector<klio::Sensor::uuid_t> uuids = store->get_sensor_uuids(); std::vector<klio::Sensor::uuid_t>::iterator it; klio::timestamp_t all_sensors_last_timestamp = 0; // First pass: get the latest timestamps of all sensors std::cout << "First pass: get the latest timestamps of all sensors" << db << std::endl; for (it = uuids.begin(); it < uuids.end(); it++) { klio::Sensor::Ptr loadedSensor(store->get_sensor(*it)); klio::reading_t last_reading = store->get_last_reading(loadedSensor); all_sensors_last_timestamp = std::max(all_sensors_last_timestamp, last_reading.first); } std::cout << std::setw(5) << "stat"; std::cout << std::setw(8) << "# val"; std::cout << std::setw(8) << "dt med"; std::cout << std::setw(8) << "dt min"; std::cout << std::setw(8) << "dt max"; std::cout << std::setw(8) << "avg(W)"; std::cout << std::setw(8) << "min(W)"; std::cout << std::setw(8) << "max(W)"; std::cout << std::setw(21) << "first timestamp"; std::cout << std::setw(21) << "last timestamp"; std::cout << '\t' << "type"; std::cout << std::setw(8) << "unit"; std::cout << '\t' << "sensor name / description" << std::endl; for (it = uuids.begin(); it < uuids.end(); it++) { klio::Sensor::Ptr loadedSensor(store->get_sensor(*it)); klio::readings_t_Ptr readings; readings = store->get_all_readings(loadedSensor); // loop over all readings and calculate metrics uint64_t num_readings = 0; klio::timestamp_t first_timestamp = 0; klio::timestamp_t last_timestamp = 0; uint32_t min_timestamp_interval = std::numeric_limits<uint32_t>::max(); uint32_t max_timestamp_interval = std::numeric_limits<uint32_t>::min(); double min_value = std::numeric_limits<double>::max(); double max_value = std::numeric_limits<double>::min(); double aggregated_value = 0; std::vector<uint32_t> all_intervals; for (klio::readings_it_t it = readings->begin(); it != readings->end(); it++) { num_readings++; klio::timestamp_t ts1 = (*it).first; double val1 = (*it).second; min_value = std::min(min_value, val1); max_value = std::max(max_value, val1); aggregated_value += val1; if (last_timestamp == 0) { // this is the first value we read, initialize state first_timestamp = ts1; last_timestamp = ts1; } else { // everything set up, this is just another value uint32_t interval = ts1 - last_timestamp; all_intervals.push_back(interval); min_timestamp_interval = std::min(min_timestamp_interval, interval); max_timestamp_interval = std::max(max_timestamp_interval, interval); // Update state variables last_timestamp = ts1; } } size_t middle_element_idx = all_intervals.size() / 2; uint32_t mean_interval = 0; if (middle_element_idx != 0) { std::nth_element(all_intervals.begin(), all_intervals.begin() + middle_element_idx, all_intervals.end()); mean_interval = all_intervals[middle_element_idx]; } // compose output line std::ostringstream stat_oss; if (last_timestamp + (60*60) < all_sensors_last_timestamp) { // no value in the last hour - mark as late stat_oss << "L"; // mark as late } else { stat_oss << "C"; // mark as current } std::cout << std::setfill(' ') << std::setw(5) << stat_oss.str(); std::cout << std::setw(8) << num_readings; std::cout << std::setw(8) << mean_interval; // print mean of intervals if (min_timestamp_interval == std::numeric_limits<uint32_t>::max() || max_timestamp_interval == std::numeric_limits<uint32_t>::min()) { std::cout << std::setw(8) << "n/a"; std::cout << std::setw(8) << "n/a"; } else { std::cout << std::setw(8) << min_timestamp_interval; std::cout << std::setw(8) << max_timestamp_interval; } // value stats if (num_readings == 0) { std::cout << std::setw(8) << "n/a"; } else { std::cout << std::setw(8) << (uint32_t) aggregated_value / num_readings; } std::cout << std::setw(8) << (uint32_t) min_value; std::cout << std::setw(8) << (uint32_t) max_value; // Time conversion foo klio::LocalTime::Ptr lt(new klio::LocalTime(".")); boost::local_time::local_time_facet* output_facet = new boost::local_time::local_time_facet(); output_facet->format("%Y.%m.%d-%H:%M:%S"); std::ostringstream oss; oss.imbue(std::locale(std::locale::classic(), output_facet)); boost::local_time::local_date_time first_timestamp_datetime = lt->get_local_time(loadedSensor, first_timestamp); oss.str(""); oss << first_timestamp_datetime; std::cout << std::setw(21) << oss.str(); boost::local_time::local_date_time last_timestamp_datetime = lt->get_local_time(loadedSensor, last_timestamp); oss.str(""); oss << last_timestamp_datetime; std::cout << std::setw(21) << oss.str(); // sensor data std::cout << '\t' << loadedSensor->device_type()->name(); // unit std::cout << '\t' << loadedSensor->unit(); // name and description std::cout << '\t' << loadedSensor->name(); std::cout << '\t' << loadedSensor->description(); // for debugging only: external_id //std::cout << '\t' << '#' << loadedSensor->external_id(); std::cout << std::endl; } } catch (klio::StoreException const& ex) { std::cout << "Failed to check: " << ex.what() << std::endl; return 1; } /** * SYNC action */ } else if (boost::iequals(action, std::string("sync"))) { std::string sourcestore; if (!vm.count("sourcestore")) { std::cerr << "You must specify a source store for synchronization." << std::endl; return 1; } else { sourcestore = vm["sourcestore"].as<std::string>(); } bfs::path source_path(sourcestore); if (!bfs::exists(source_path)) { std::cerr << "File " << source_path << " does not exist, exiting." << std::endl; return 2; } bfs::path target_path(storefile); if (!bfs::exists(target_path)) { std::cerr << "File " << target_path << " does not exist, exiting." << std::endl; return 2; } try { std::cout << "Attempting to open source store " << source_path << std::endl; klio::Store::Ptr source_store(factory->open_sqlite3_store(source_path)); std::cout << "Opened source store: " << source_store->str() << std::endl; std::cout << "Attempting to open target store " << target_path << std::endl; klio::Store::Ptr target_store(factory->open_sqlite3_store(target_path)); std::cout << "Opened target store: " << target_store->str() << std::endl; std::cout << "Synchronizing stores..." << std::endl; target_store->sync(source_store); std::cout << "Stores Synchronized." << std::endl; source_store->close(); target_store->close(); } catch (klio::StoreException const& ex) { std::cout << "Failed to synchronize stores. " << ex.what() << std::endl; return 1; } /** * UPGRADE action */ } else if (boost::iequals(action, std::string("upgrade"))) { try { klio::VersionInfo::Ptr info = klio::VersionInfo::Ptr(new klio::VersionInfo()); std::cout << "Attempting to upgrade " << db << " to version " << info->getVersion() << std::endl; klio::SQLite3Store::Ptr store(factory->create_sqlite3_store(db, false)); store->upgrade(); std::cout << "Store upgraded." << std::endl; } catch (klio::StoreException const& ex) { std::cout << "Failed to upgrade: " << ex.what() << std::endl; return 1; } /** * unknown command */ } else { std::cerr << "Unknown command " << action << std::endl; return 1; } } catch (std::exception& e) { std::cerr << "error: " << e.what() << std::endl; return 1; } catch (...) { std::cerr << "Exception of unknown type!" << std::endl; return 1; } return 0; }