explicit LMDBLabelCursor(MDB_txn* mdb_txn, MDB_cursor* mdb_cursor)
: LMDBCursor(mdb_txn, mdb_cursor) {
inlabel_samples = 4;
srand (time(NULL));
//std::cout << "inlabel_samples " << inlabel_samples ;
GetLabelsKeysMap(&labels_keys_, &labels_);
SeekToFirst();
}
explicit LMDBShuffleCursor(MDB_txn* mdb_txn, MDB_cursor* mdb_cursor)
: LMDBCursor(mdb_txn, mdb_cursor) {
srand (time(NULL));
GetKeys(&keys_);
keys_permutation_.resize(keys_.size());
for (int i = 0; i < keys_.size(); ++i) {
keys_permutation_[i] = i;
}
SeekToFirst();
}
std::ostream& operator<<(std::ostream& _out, BlockChain const& _bc)
{
string cmp = toBigEndianString(_bc.m_lastBlockHash);
auto it = _bc.m_detailsDB->NewIterator(_bc.m_readOptions);
for (it->SeekToFirst(); it->Valid(); it->Next())
if (it->key().ToString() != "best")
{
BlockDetails d(RLP(it->value().ToString()));
_out << asHex(it->key().ToString()) << ": " << d.number << " @ " << d.parent << (cmp == it->key().ToString() ? " BEST" : "") << std::endl;
}
delete it;
return _out;
}
osquery::Status DBHandle::Scan(const std::string& domain,
std::vector<std::string>& results) {
auto cfh = getHandleForColumnFamily(domain);
if (cfh == nullptr) {
return Status(1, "Could not get column family for " + domain);
}
auto it = getDB()->NewIterator(rocksdb::ReadOptions(), cfh);
if (it == nullptr) {
return Status(1, "Could not get iterator for " + domain);
}
for (it->SeekToFirst(); it->Valid(); it->Next()) {
results.push_back(it->key().ToString());
}
delete it;
return Status(0, "OK");
}
size_t LevelDB::size(bool thorough, DbSnapshot* aSnapshot /*nullptr*/) throw(DbException) {
if (!thorough && lastSize > 0)
return lastSize;
// leveldb doesn't support any easy way to do this
size_t ret = 0;
leveldb::ReadOptions options;
options.fill_cache = false;
if (aSnapshot)
options.snapshot = static_cast<LevelSnapshot*>(aSnapshot)->snapshot;
auto it = unique_ptr<leveldb::Iterator>(db->NewIterator(options));
for (it->SeekToFirst(); it->Valid(); it->Next()) {
checkDbError(it->status());
ret++;
}
lastSize = ret;
return ret;
}
void LevelDB::remove_if(std::function<bool(void* aKey, size_t key_len, void* aValue, size_t valueLen)> f, DbSnapshot* aSnapshot /*nullptr*/) throw(DbException) {
leveldb::WriteBatch wb;
leveldb::ReadOptions options;
options.fill_cache = false;
options.verify_checksums = false; // it will stop iterating when a checksum mismatch is found otherwise
if (aSnapshot)
options.snapshot = static_cast<LevelSnapshot*>(aSnapshot)->snapshot;
{
auto it = unique_ptr<leveldb::Iterator>(db->NewIterator(options));
for (it->SeekToFirst(); it->Valid(); it->Next()) {
checkDbError(it->status());
if (f((void*)it->key().data(), it->key().size(), (void*)it->value().data(), it->value().size())) {
wb.Delete(it->key());
}
}
}
DBACTION(db->Write(writeoptions, &wb));
}
explicit LMDBCursor(MDB_txn* mdb_txn, MDB_cursor* mdb_cursor)
: mdb_txn_(mdb_txn), mdb_cursor_(mdb_cursor), valid_(false) {
SeekToFirst();
}
explicit LevelDBCursor(leveldb::Iterator* iter)
: iter_(iter) { SeekToFirst(); }
void Circuit::load() {
unsigned long element_id;
unsigned long version = 0;
std::istringstream in(std::ios_base::binary);
m_database = new KeyValueStorage(m_savedir, "circuit");
m_virtual_database = new KeyValueStorage(m_savedir, "circuit_virtual");
std::ifstream input_elements_func((m_savedir + DIR_DELIM + elements_func_file).c_str(), std::ios_base::binary);
if(input_elements_func.good()) {
input_elements_func.read(reinterpret_cast<char*>(&version), sizeof(version));
m_circuit_elements_states.deSerialize(input_elements_func);
}
#if USE_LEVELDB
// Filling list with empty virtual elements
auto virtual_it = m_virtual_database->new_iterator();
std::map <unsigned long, std::list <CircuitElementVirtual>::iterator> id_to_virtual_pointer;
for(virtual_it->SeekToFirst(); virtual_it->Valid(); virtual_it->Next()) {
element_id = stoi(virtual_it->key().ToString());
id_to_virtual_pointer[element_id] =
m_virtual_elements.insert(m_virtual_elements.begin(), CircuitElementVirtual(element_id));
if(element_id + 1 > m_max_virtual_id) {
m_max_virtual_id = element_id + 1;
}
}
// Filling list with empty elements
auto it = m_database->new_iterator();
std::map <unsigned long, std::list <CircuitElement>::iterator> id_to_pointer;
for(it->SeekToFirst(); it->Valid(); it->Next()) {
element_id = stoi(it->key().ToString());
id_to_pointer[element_id] =
m_elements.insert(m_elements.begin(), CircuitElement(element_id));
if(element_id + 1 > m_max_id) {
m_max_id = element_id + 1;
}
}
// Loading states of elements
std::ifstream input_elements_states((m_savedir + DIR_DELIM + elements_states_file).c_str());
if(input_elements_states.good()) {
for(unsigned long i = 0; i < m_elements.size(); ++i) {
input_elements_states.read(reinterpret_cast<char*>(&element_id), sizeof(element_id));
if(id_to_pointer.find(element_id) != id_to_pointer.end()) {
id_to_pointer[element_id]->deSerializeState(input_elements_states);
} else {
throw SerializationError(static_cast<std::string>("File \"")
+ elements_states_file + "\" seems to be corrupted.");
}
}
}
// Loading elements data
for(it->SeekToFirst(); it->Valid(); it->Next()) {
in.str(it->value().ToString());
element_id = stoi(it->key().ToString());
std::list <CircuitElement>::iterator current_element = id_to_pointer[element_id];
current_element->deSerialize(in, id_to_virtual_pointer);
current_element->setFunc(m_circuit_elements_states.getFunc(current_element->getFuncId()), current_element->getFuncId());
m_pos_to_iterator[current_element->getPos()] = current_element;
}
delete it;
// Loading virtual elements data
for(virtual_it->SeekToFirst(); virtual_it->Valid(); virtual_it->Next()) {
in.str(virtual_it->value().ToString());
element_id = stoi(virtual_it->key().ToString());
std::list <CircuitElementVirtual>::iterator current_element = id_to_virtual_pointer[element_id];
current_element->deSerialize(in, current_element, id_to_pointer);
}
delete virtual_it;
#endif
}
explicit Base64Cursor( string base64_cursor)
: is_key_pos_map_ready_(false){
base64_cursor_.open(base64_cursor);
SeekToFirst();
}
int main(int argc, char* argv[]) {
if (argc != 5 && argc != 7 && argc != 8) {
std::cout
<< "Usage: leveldb_kmedoids dbpath K intravariance-out-file "
<< "kmedoids-out-file [start end] [concurrency]\n"
<< "Runs K-Medoid on the key range [start, end].\n"
<< "Prints intermediate centroids per iteration to the out files, "
<< "as well as within-cluster total distances (\"intravariances\")."
<< std::endl;
return 1;
}
int concurrency = std::thread::hardware_concurrency();
if (argc == 8) concurrency = std::atoi(argv[7]);
std::cout << "Using concurrency " << concurrency << std::endl;
int K = std::atoi(argv[2]);
leveldb::DB* db;
CHECK_OK(leveldb::DB::Open({}, argv[1], &db));
std::cout << "Database open" << std::endl;
leveldb::DB* work_db;
leveldb::Options options;
options.create_if_missing = true;
CHECK_OK(leveldb::DB::Open(options, "/tmp/kmedoids-work", &work_db));
std::cout << "Work database open" << std::endl;
std::string start_base, end_base;
leveldb::Slice start, end;
if (argc > 5) {
start = argv[5];
end = argv[6];
} else {
auto it = iter(db);
it->SeekToFirst();
CHECK(it->Valid());
start_base = it->key().ToString();
it->SeekToLast();
CHECK(it->Valid());
end_base = it->key().ToString();
start = start_base;
end = end_base;
}
std::cout << "Running K-medoids on range [" << start.ToString()
<< ", " << end.ToString()
<< "]" << std::endl;
std::ofstream ivar_out(argv[3]);
std::ofstream cent_out(argv[4]);
RunKMedoids(start, end, K, db, work_db,
concurrency, ivar_out, cent_out);
delete work_db;
CHECK_OK(leveldb::DestroyDB("/tmp/kmedoids-work", {}));
std::cout << "Closing databases" << std::endl;
delete db;
}
