void
checkKmers(DnaString const & kmer,
TVertexDescriptor const & starting_vertex,
TVertexDescriptor const & source_vertex,
TGraph const & graph,
std::vector<VertexLabels> & vertex_vector,
boost::unordered_set<TVertexDescriptor> const & free_nodes,
boost::unordered_map< std::pair<TVertexDescriptor, TVertexDescriptor>, boost::dynamic_bitset<> > & edge_ids,
boost::dynamic_bitset<> const & id_bits,
TKmerMap & kmer_map,
std::size_t const & kmer_size
)
{
if (id_bits.none())
return;
if (length(kmer) == kmer_size)
{
KmerLabels new_kmer_label =
{
starting_vertex,
source_vertex,
id_bits
};
if (kmer_map.count(kmer) == 0)
{
std::vector<KmerLabels> new_vector(1, new_kmer_label);
kmer_map[kmer] = new_vector;
}
else
{
kmer_map[kmer].push_back(new_kmer_label);
}
return;
}
for (Iterator<TGraph, OutEdgeIterator>::Type out_edge_iterator (graph, source_vertex) ; !atEnd(out_edge_iterator) ; ++out_edge_iterator)
{
DnaString new_kmer(kmer);
TVertexDescriptor const & target_vertex = targetVertex(out_edge_iterator);
boost::dynamic_bitset<> new_id_bits(id_bits);
if (free_nodes.count(target_vertex) == 0)
{
seqan::appendValue(new_kmer, vertex_vector[target_vertex].dna);
std::pair<TVertexDescriptor, TVertexDescriptor> edge_pair(source_vertex, target_vertex);
if (edge_ids.count(edge_pair) == 1)
{
new_id_bits = id_bits & edge_ids[edge_pair];
}
}
checkKmers(new_kmer, starting_vertex, target_vertex, graph, vertex_vector, free_nodes, edge_ids, new_id_bits, kmer_map, kmer_size);
}
}
void
check_kmers_simple(DnaString const & kmer,
TGraph const & graph,
TVertexDescriptor const & source_vertex,
std::vector<VertexLabels> & vertex_vector,
boost::unordered_set<TVertexDescriptor> const & free_nodes,
boost::unordered_map< std::pair<TVertexDescriptor, TVertexDescriptor>, boost::dynamic_bitset<> > & edge_ids,
boost::dynamic_bitset<> const & id_bits,
TKmerMapSimple & kmer_map,
std::size_t const & kmer_size
)
{
if (id_bits.none())
return;
if (length(kmer) == kmer_size)
{
if (id_bits.all())
return;
if (kmer_map.count(kmer) == 0)
{
kmer_map[kmer] = id_bits;
}
else
{
kmer_map[kmer] |= id_bits;
}
return;
}
for (Iterator<TGraph, OutEdgeIterator>::Type out_edge_iterator (graph, source_vertex) ; !atEnd(out_edge_iterator) ; ++out_edge_iterator)
{
DnaString new_kmer(kmer);
TVertexDescriptor const & target_vertex = targetVertex(out_edge_iterator);
// std::cout << source_vertex << " -> " << target_vertex << std::endl;
boost::dynamic_bitset<> new_id_bits(id_bits);
if (free_nodes.count(target_vertex) == 0)
{
seqan::appendValue(new_kmer, vertex_vector[target_vertex].dna);
std::pair<TVertexDescriptor, TVertexDescriptor> edge_pair(source_vertex, target_vertex);
if (edge_ids.count(edge_pair) == 1)
{
new_id_bits = id_bits & edge_ids[edge_pair];
}
}
check_kmers_simple(new_kmer, graph, target_vertex, vertex_vector, free_nodes, edge_ids, new_id_bits, kmer_map, kmer_size);
}
}
void hopscotch_map_sanity_checks() {
ASSERT_TRUE(cm2.begin() == cm2.end());
for (size_t i = 0;i < NINS; ++i) {
cm2[17 * i] = i;
um2[17 * i] = i;
}
for (size_t i = 0;i < NINS; ++i) {
assert(cm2[17 * i] == i);
assert(um2[17 * i] == i);
}
assert(cm2.size() == NINS);
assert(um2.size() == NINS);
for (size_t i = 0;i < NINS; i+=2) {
cm2.erase(17*i);
um2.erase(17*i);
}
for (size_t i = 0;i < NINS; i+=2) {
assert(cm2.count(17*i) == i % 2);
assert(um2.count(17*i) == i % 2);
if (cm2.count(17*i)) {
assert(cm2.find(17*i)->second == i);
}
}
assert(cm2.size() == NINS / 2);
assert(um2.size() == NINS / 2);
typedef graphlab::hopscotch_map<uint32_t, uint32_t>::value_type vpair;
{
size_t cnt = 0;
foreach(vpair &v, cm2) {
ASSERT_EQ(v.second, um2[v.first]);
++cnt;
}
ASSERT_EQ(cnt, NINS / 2);
}
int DeviceCassandraController::updateDevice(
const string &devicename, const string &username,
boost::unordered_map<string, string> ¶ms) {
if (params.size() <= 0)
return 0;
string errorCode = "";
int result;
// Get the current device.
boost::shared_ptr<Device> device = getDevice(devicename, username, errorCode);
if (!errorCode.empty()) {
return boost::lexical_cast<int>(errorCode);
}
// Check if devicename needs to be changed.
if (params.count("new_devicename")) {
// Delete the current device from db.
result = deleteDevice(devicename, username);
if (result != 0) {
return result;
}
}
// Update data in the current device.
if (params.count("new_devicename")) {
device->setDevicename(params["new_devicename"]);
}
if (params.count("type")) {
device->setType(params["type"]);
}
if (params.count("metadata")) {
device->setContentMeta(params["metadata"]);
device->setContentTimestamp(time(NULL));
}
if (params.count("dir_ip")) {
device->setDirIp(params["dir_ip"]);
}
if (params.count("dir_port")) {
unsigned short portValue = -1;
if (!ControllerHelper::isNullOREmptyString(params["dir_port"]))
portValue = boost::lexical_cast<unsigned short>(params["dir_port"]);
device->setDirPort(portValue);
}
if (params.count("ip")) {
device->setIp(params["ip"]);
}
if (params.count("port")) {
unsigned short portValue = -1;
if (!ControllerHelper::isNullOREmptyString(params["port"]))
portValue = boost::lexical_cast<unsigned short>(params["port"]);
device->setPort(portValue);
}
if (params.count("os")) {
device->setOs(params["os"]);
}
if (params.count("description")) {
device->setDescription(params["description"]);
}
if (params.count("public_folder")) {
device->setPublicFolder(params["public_folder"]);
}
if (params.count("private_folder")) {
device->setPrivateFolder(params["private_folder"]);
}
if (params.count("is_indexed")) {
bool is_indexed = false;
if (boost::iequals(params["is_indexed"], "true"))
is_indexed = true;
device->setSearchable(is_indexed);
}
// Update lastseen.
device->setLastSeen(time(NULL));
// Add the new device.
result = addDevice(device);
return result;
}
bool buildTable(Kompex::SQLiteStatement * stmt,
int32_t &name_id,
boost::unordered_map<std::string,int32_t> &table_names,
std::string const &sql_table_name,
std::vector<Tile*> list_tiles,
osmscout::Database &map,
osmscout::TypeSet const &typeSet,
bool allow_duplicate_nodes,
bool allow_duplicate_ways,
bool allow_duplicate_areas)
{
// spec area search parameter
osmscout::AreaSearchParameter area_search_param;
area_search_param.SetUseLowZoomOptimization(false);
// create the sql statement
std::string stmt_insert = "INSERT INTO "+sql_table_name;
stmt_insert +=
"(id,node_offsets,way_offsets,area_offsets) VALUES("
"@id,@node_offsets,@way_offsets,@area_offsets);";
try {
stmt->BeginTransaction();
stmt->Sql(stmt_insert);
}
catch(Kompex::SQLiteException &exception) {
qDebug() << "ERROR: SQLite exception with insert statement:"
<< QString::fromStdString(exception.GetString());
return false;
}
// osmscout may return nearby results again so we make
// sure offsets are only included once if requested
std::set<osmscout::FileOffset> set_node_offsets;
std::set<osmscout::FileOffset> set_way_offsets;
std::set<osmscout::FileOffset> set_area_offsets;
// container for blob memory we delete after
// committing the sql transaction
std::vector<char*> list_blobs;
// keep track of the number of transactions and
// commit after a certain limit
size_t transaction_limit=5000;
size_t transaction_count=0;
for(size_t i=0; i < list_tiles.size(); i++) {
// for each tile
// get objects from osmscout
GeoBoundingBox const &bbox = list_tiles[i]->bbox;
std::vector<osmscout::NodeRef> listNodes;
std::vector<osmscout::WayRef> listWays;
std::vector<osmscout::AreaRef> listAreas;
map.GetObjects(bbox.minLon,bbox.minLat,
bbox.maxLon,bbox.maxLat,
typeSet,
listNodes,
listWays,
listAreas);
// merge all of the object refs into one list
// of MapObjects so its easier to manage
std::vector<MapObject> list_map_objects;
list_map_objects.reserve(listNodes.size()+listWays.size()+listAreas.size());
for(size_t j=0; j < listNodes.size(); j++) {
osmscout::NodeRef &nodeRef = listNodes[j];
if(nodeRef->GetName().empty()) {
continue;
}
if(!allow_duplicate_nodes) {
if(set_node_offsets.count(nodeRef->GetFileOffset()) != 0) {
continue;
}
set_node_offsets.insert(nodeRef->GetFileOffset());
}
MapObject map_object;
map_object.name = nodeRef->GetName();
map_object.offset = nodeRef->GetFileOffset();
map_object.type = osmscout::refNode;
list_map_objects.push_back(map_object);
}
for(size_t j=0; j < listWays.size(); j++) {
osmscout::WayRef &wayRef = listWays[j];
if(wayRef->GetName().empty()) {
continue;
}
if(!allow_duplicate_ways) {
if(set_way_offsets.count(wayRef->GetFileOffset()) != 0) {
continue;
}
set_way_offsets.insert(wayRef->GetFileOffset());
}
MapObject map_object;
map_object.name = wayRef->GetName();
map_object.offset = wayRef->GetFileOffset();
map_object.type = osmscout::refWay;
list_map_objects.push_back(map_object);
}
for(size_t j=0; j < listAreas.size(); j++) {
osmscout::AreaRef &areaRef = listAreas[j];
if(areaRef->rings.front().GetName().empty()) {
continue;
}
if(!allow_duplicate_areas) {
if(set_area_offsets.count(areaRef->GetFileOffset()) != 0) {
continue;
}
set_area_offsets.insert(areaRef->GetFileOffset());
}
MapObject map_object;
map_object.name = areaRef->rings.front().GetName();
map_object.offset = areaRef->GetFileOffset();
map_object.type = osmscout::refArea;
list_map_objects.push_back(map_object);
}
// create structs to sort file offsets by name lookup
// [name_lookup_id] [list_offsets]
boost::unordered_map<int32_t,OffsetGroup> entry_admin_regions;
for(size_t j=0; j < list_map_objects.size(); j++) {
MapObject &map_object = list_map_objects[j];
// add name_lookup up to table_names
std::string name_lookup = convNameToLookup(map_object.name);
int32_t name_lookup_id;
// check if this lookup string already exists
if(table_names.count(name_lookup) == 0) {
// insert lookup string
std::pair<std::string,int32_t> data;
data.first = name_lookup;
data.second = name_id;
table_names.insert(data);
name_lookup_id = name_id;
name_id++;
}
else {
name_lookup_id = table_names.find(name_lookup)->second;
}
// check if this lookup string already exists
if(entry_admin_regions.count(name_lookup_id) == 0) {
// insert new list of offsets
std::pair<int32_t,OffsetGroup> data;
data.first = name_lookup_id;
entry_admin_regions.insert(data);
}
// add offset to list
boost::unordered_map<int32_t,OffsetGroup>::iterator it;
it = entry_admin_regions.find(name_lookup_id);
if(map_object.type == osmscout::refNode) {
it->second.node_offsets.push_back(map_object.offset);
}
else if(map_object.type == osmscout::refWay) {
it->second.way_offsets.push_back(map_object.offset);
}
else if(map_object.type == osmscout::refArea) {
it->second.area_offsets.push_back(map_object.offset);
}
}
//qDebug() << list_tiles[i]->id << ":" << entry_admin_regions.size();
// write information for this tile to the database
boost::unordered_map<int32_t,OffsetGroup>::iterator it;
for(it = entry_admin_regions.begin();
it != entry_admin_regions.end(); ++it) {
// convert offsets to byte arrays
char * data_node_offsets = NULL; size_t sz_node_offsets=0;
char * data_way_offsets = NULL; size_t sz_way_offsets=0;
char * data_area_offsets = NULL; size_t sz_area_offsets=0;
OffsetGroup &g = it->second;
// // ### debug start
// if(g_table_nameid_count.count(it->first) == 0) {
// std::pair<int64_t,int64_t> debugdata;
// debugdata.first = it->first;
// debugdata.second =
// g.node_offsets.size()+
// g.way_offsets.size()+
// g.area_offsets.size();
// g_table_nameid_count.insert(debugdata);
// }
// else {
// std::map<int64_t,int64_t>::iterator d_it;
// d_it = g_table_nameid_count.find(it->first);
// d_it->second +=
// g.node_offsets.size()+
// g.way_offsets.size()+
// g.area_offsets.size();
// }
// // ### debug end
if(!(g.node_offsets.empty())) {
sz_node_offsets = sizeof(osmscout::FileOffset)*g.node_offsets.size();
data_node_offsets = new char[sz_node_offsets];
memcpy(data_node_offsets,&(g.node_offsets[0]),sz_node_offsets);
list_blobs.push_back(data_node_offsets);
}
if(!(g.way_offsets.empty())) {
sz_way_offsets = sizeof(osmscout::FileOffset)*g.way_offsets.size();
data_way_offsets = new char[sz_way_offsets];
memcpy(data_way_offsets,&(g.way_offsets[0]),sz_way_offsets);
list_blobs.push_back(data_way_offsets);
}
if(!(g.area_offsets.empty())) {
sz_area_offsets = sizeof(osmscout::FileOffset)*g.area_offsets.size();
data_area_offsets = new char[sz_area_offsets];
memcpy(data_area_offsets,&(g.area_offsets[0]),sz_area_offsets);
list_blobs.push_back(data_area_offsets);
}
if(data_node_offsets == NULL &&
data_way_offsets == NULL &&
data_area_offsets == NULL) {
continue;
}
// prepare sql
try {
// cat name_id and tile_id into one id
// get mask for bitlength
int32_t mask = pow(2,g_sz_bits_tile_id)-1;
int64_t id = it->first; // name_id
id = id << g_sz_bits_tile_id; // bitshift
id |= (list_tiles[i]->id & mask); // 24 most sig bits of tile_id
stmt->BindInt64(1,id);
if(data_node_offsets) {
stmt->BindBlob(2,data_node_offsets,sz_node_offsets);
}
else {
stmt->BindNull(2);
}
if(data_way_offsets) {
stmt->BindBlob(3,data_way_offsets,sz_way_offsets);
}
else {
stmt->BindNull(3);
}
if(data_area_offsets) {
stmt->BindBlob(4,data_area_offsets,sz_area_offsets);
}
else {
stmt->BindNull(4);
}
stmt->Execute();
stmt->Reset();
if(transaction_count > transaction_limit) {
stmt->FreeQuery();
stmt->CommitTransaction();
stmt->BeginTransaction();
stmt->Sql(stmt_insert);
transaction_count=0;
}
transaction_count++;
}
catch(Kompex::SQLiteException &exception) {
qDebug() << "ERROR: SQLite exception writing tile data:"
<< QString::fromStdString(exception.GetString());
qDebug() << "ERROR: id:" << list_tiles[i]->id;
qDebug() << "ERROR: name_lookup_id:" << it->first;
qDebug() << "ERROR:" << sz_node_offsets << sz_way_offsets << sz_area_offsets;
return false;
}
}
// debug
// qDebug() << i << "/" << list_tiles.size();
}
stmt->FreeQuery();
stmt->CommitTransaction();
// free up blob memory
for(size_t i=0; i < list_blobs.size(); i++) {
delete[] list_blobs[i];
}
return true;
}
void
addExonToGraph (TGraph & g,
unsigned short const & bit_id,
unsigned short const & bit_n,
CharString const & sequence,
std::map<VertexLabels, TVertexDescriptor> & vertex_label_map,
std::vector<VertexLabels> & vertex_vector,
boost::unordered_map<std::pair<TVertexDescriptor, TVertexDescriptor>, boost::dynamic_bitset<> > & edge_ids
)
{
VertexLabels prev_vertex_label = {0, Dna('A')};
TVertexDescriptor prev = vertex_label_map[prev_vertex_label];
for (int pos = 0 ; ; ++pos)
{
if (pos >= (int) length(sequence))
{
VertexLabels final_vertex_label = {
-1, // level
Dna('A'), // Dna, initialized here but we should never use it!
};
TVertexDescriptor target_vertex = vertex_label_map[final_vertex_label];
std::pair<TVertexDescriptor, TVertexDescriptor> my_pair (prev, target_vertex);
addBitsetEdge(edge_ids[my_pair], bit_id, bit_n, edge_ids.count(my_pair));
if (!findEdge(g, prev, target_vertex))
addEdge(g, prev, target_vertex);
break;
}
if (sequence[pos] == '-')
continue;
VertexLabels new_vertex_label =
{
pos+1, // level
Dna(sequence[pos]), // dna
};
TVertexDescriptor target_vertex;
if (vertex_label_map.count(new_vertex_label) == 1)
{
target_vertex = vertex_label_map[new_vertex_label];
std::pair<TVertexDescriptor, TVertexDescriptor> my_pair (prev, target_vertex);
addBitsetEdge(edge_ids[my_pair], bit_id, bit_n, edge_ids.count(my_pair));
if (!findEdge(g, prev, target_vertex))
addEdge(g, prev, target_vertex);
}
else
{
target_vertex = addVertex(g);
std::pair<TVertexDescriptor, TVertexDescriptor> my_pair (prev, target_vertex);
addBitsetEdge(edge_ids[my_pair], bit_id, bit_n, edge_ids.count(my_pair));
addEdge(g, prev, target_vertex);
vertex_label_map[new_vertex_label] = target_vertex;
vertex_vector.push_back(new_vertex_label);
}
prev = target_vertex;
}
}
