示例#1
0
void
checkKmers(DnaString const & kmer,
           TVertexDescriptor const & starting_vertex,
           TVertexDescriptor const & source_vertex,
           TGraph const & graph,
           std::vector<VertexLabels> & vertex_vector,
           boost::unordered_set<TVertexDescriptor> const & free_nodes,
           boost::unordered_map< std::pair<TVertexDescriptor, TVertexDescriptor>, boost::dynamic_bitset<> > & edge_ids,
           boost::dynamic_bitset<> const & id_bits,
           TKmerMap & kmer_map,
           std::size_t const & kmer_size
          )
{
  if (id_bits.none())
    return;

  if (length(kmer) == kmer_size)
  {
    KmerLabels new_kmer_label =
    {
      starting_vertex,
      source_vertex,
      id_bits
    };

    if (kmer_map.count(kmer) == 0)
    {
      std::vector<KmerLabels> new_vector(1, new_kmer_label);
      kmer_map[kmer] = new_vector;
    }
    else
    {
      kmer_map[kmer].push_back(new_kmer_label);
    }

    return;
  }

  for (Iterator<TGraph, OutEdgeIterator>::Type out_edge_iterator (graph, source_vertex) ; !atEnd(out_edge_iterator) ; ++out_edge_iterator)
  {
    DnaString new_kmer(kmer);
    TVertexDescriptor const & target_vertex = targetVertex(out_edge_iterator);
    boost::dynamic_bitset<> new_id_bits(id_bits);

    if (free_nodes.count(target_vertex) == 0)
    {
      seqan::appendValue(new_kmer, vertex_vector[target_vertex].dna);
      std::pair<TVertexDescriptor, TVertexDescriptor> edge_pair(source_vertex, target_vertex);
      
      if (edge_ids.count(edge_pair) == 1)
      {
        new_id_bits = id_bits & edge_ids[edge_pair];
      }
    }

    checkKmers(new_kmer, starting_vertex, target_vertex, graph, vertex_vector, free_nodes, edge_ids, new_id_bits, kmer_map, kmer_size);
  }
}
示例#2
0
void
check_kmers_simple(DnaString const & kmer,
                   TGraph const & graph,
                   TVertexDescriptor const & source_vertex,
                   std::vector<VertexLabels> & vertex_vector,
                   boost::unordered_set<TVertexDescriptor> const & free_nodes,
                   boost::unordered_map< std::pair<TVertexDescriptor, TVertexDescriptor>, boost::dynamic_bitset<> > & edge_ids,
                   boost::dynamic_bitset<> const & id_bits,
                   TKmerMapSimple & kmer_map,
                   std::size_t const & kmer_size
                  )
{
  if (id_bits.none())
    return;

  if (length(kmer) == kmer_size)
  {
    if (id_bits.all())
      return;

    if (kmer_map.count(kmer) == 0)
    {
      kmer_map[kmer] = id_bits;
    }
    else
    {
      kmer_map[kmer] |= id_bits;
    }

    return;
  }

  for (Iterator<TGraph, OutEdgeIterator>::Type out_edge_iterator (graph, source_vertex) ; !atEnd(out_edge_iterator) ; ++out_edge_iterator)
  {
    DnaString new_kmer(kmer);
    TVertexDescriptor const & target_vertex = targetVertex(out_edge_iterator);

    // std::cout << source_vertex << " -> " << target_vertex << std::endl;

    boost::dynamic_bitset<> new_id_bits(id_bits);

    if (free_nodes.count(target_vertex) == 0)
    {
      seqan::appendValue(new_kmer, vertex_vector[target_vertex].dna);
      std::pair<TVertexDescriptor, TVertexDescriptor> edge_pair(source_vertex, target_vertex);
      
      if (edge_ids.count(edge_pair) == 1)
      {
        new_id_bits = id_bits & edge_ids[edge_pair];
      }
    }

    check_kmers_simple(new_kmer, graph, target_vertex, vertex_vector, free_nodes, edge_ids, new_id_bits, kmer_map, kmer_size);
  }
}
示例#3
0
void hopscotch_map_sanity_checks() {
  ASSERT_TRUE(cm2.begin() == cm2.end());
  for (size_t i = 0;i < NINS; ++i) {
    cm2[17 * i] = i;
    um2[17 * i] = i;
  }

  for (size_t i = 0;i < NINS; ++i) {
    assert(cm2[17 * i] == i);
    assert(um2[17 * i] == i);
  }
  assert(cm2.size() == NINS);
  assert(um2.size() == NINS);

  for (size_t i = 0;i < NINS; i+=2) {
    cm2.erase(17*i);
    um2.erase(17*i);
  }
  for (size_t i = 0;i < NINS; i+=2) {
    assert(cm2.count(17*i) == i % 2);
    assert(um2.count(17*i) == i % 2);
    if (cm2.count(17*i)) {
      assert(cm2.find(17*i)->second == i);
    }
  }

  assert(cm2.size() == NINS / 2);
  assert(um2.size() == NINS / 2);

  typedef graphlab::hopscotch_map<uint32_t, uint32_t>::value_type vpair;
  {
    size_t cnt = 0;
    foreach(vpair &v, cm2) {
      ASSERT_EQ(v.second, um2[v.first]);
      ++cnt;
    }
    ASSERT_EQ(cnt, NINS / 2);
  }
int DeviceCassandraController::updateDevice(
    const string &devicename, const string &username,
    boost::unordered_map<string, string> &params) {
  if (params.size() <= 0)
    return 0;

  string errorCode = "";
  int result;

  // Get the current device.
  boost::shared_ptr<Device> device = getDevice(devicename, username, errorCode);
  if (!errorCode.empty()) {
    return boost::lexical_cast<int>(errorCode);
  }

  // Check if devicename needs to be changed.
  if (params.count("new_devicename")) {
    // Delete the current device from db.
    result = deleteDevice(devicename, username);
    if (result != 0) {
      return result;
    }
  }

  // Update data in the current device.
  if (params.count("new_devicename")) {
    device->setDevicename(params["new_devicename"]);
  }
  if (params.count("type")) {
    device->setType(params["type"]);
  }
  if (params.count("metadata")) {
    device->setContentMeta(params["metadata"]);
    device->setContentTimestamp(time(NULL));
  }
  if (params.count("dir_ip")) {
    device->setDirIp(params["dir_ip"]);
  }
  if (params.count("dir_port")) {
    unsigned short portValue = -1;
    if (!ControllerHelper::isNullOREmptyString(params["dir_port"]))
      portValue = boost::lexical_cast<unsigned short>(params["dir_port"]);
    device->setDirPort(portValue);
  }
  if (params.count("ip")) {
    device->setIp(params["ip"]);
  }
  if (params.count("port")) {
    unsigned short portValue = -1;
    if (!ControllerHelper::isNullOREmptyString(params["port"]))
      portValue = boost::lexical_cast<unsigned short>(params["port"]);
    device->setPort(portValue);
  }
  if (params.count("os")) {
    device->setOs(params["os"]);
  }
  if (params.count("description")) {
    device->setDescription(params["description"]);
  }
  if (params.count("public_folder")) {
    device->setPublicFolder(params["public_folder"]);
  }
  if (params.count("private_folder")) {
    device->setPrivateFolder(params["private_folder"]);
  }
  if (params.count("is_indexed")) {
    bool is_indexed = false;
    if (boost::iequals(params["is_indexed"], "true"))
      is_indexed = true;
    device->setSearchable(is_indexed);
  }

  // Update lastseen.
  device->setLastSeen(time(NULL));

  // Add the new device.
  result = addDevice(device);
  return result;
}
示例#5
0
bool buildTable(Kompex::SQLiteStatement * stmt,
                int32_t &name_id,
                boost::unordered_map<std::string,int32_t> &table_names,
                std::string const &sql_table_name,
                std::vector<Tile*> list_tiles,
                osmscout::Database &map,
                osmscout::TypeSet const &typeSet,
                bool allow_duplicate_nodes,
                bool allow_duplicate_ways,
                bool allow_duplicate_areas)
{
    // spec area search parameter
    osmscout::AreaSearchParameter area_search_param;
    area_search_param.SetUseLowZoomOptimization(false);

    // create the sql statement
    std::string stmt_insert = "INSERT INTO "+sql_table_name;
    stmt_insert +=
            "(id,node_offsets,way_offsets,area_offsets) VALUES("
            "@id,@node_offsets,@way_offsets,@area_offsets);";

    try   {
        stmt->BeginTransaction();
        stmt->Sql(stmt_insert);
    }
    catch(Kompex::SQLiteException &exception)   {
        qDebug() << "ERROR: SQLite exception with insert statement:"
                 << QString::fromStdString(exception.GetString());
        return false;
    }

    // osmscout may return nearby results again so we make
    // sure offsets are only included once if requested
    std::set<osmscout::FileOffset> set_node_offsets;
    std::set<osmscout::FileOffset> set_way_offsets;
    std::set<osmscout::FileOffset> set_area_offsets;

    // container for blob memory we delete after
    // committing the sql transaction
    std::vector<char*> list_blobs;

    // keep track of the number of transactions and
    // commit after a certain limit
    size_t transaction_limit=5000;
    size_t transaction_count=0;

    for(size_t i=0; i < list_tiles.size(); i++)   {
        // for each tile

        // get objects from osmscout
        GeoBoundingBox const &bbox = list_tiles[i]->bbox;
        std::vector<osmscout::NodeRef> listNodes;
        std::vector<osmscout::WayRef>  listWays;
        std::vector<osmscout::AreaRef> listAreas;
        map.GetObjects(bbox.minLon,bbox.minLat,
                       bbox.maxLon,bbox.maxLat,
                       typeSet,
                       listNodes,
                       listWays,
                       listAreas);

        // merge all of the object refs into one list
        // of MapObjects so its easier to manage
        std::vector<MapObject> list_map_objects;
        list_map_objects.reserve(listNodes.size()+listWays.size()+listAreas.size());

        for(size_t j=0; j < listNodes.size(); j++)   {
            osmscout::NodeRef &nodeRef = listNodes[j];

            if(nodeRef->GetName().empty())   {
                continue;
            }
            if(!allow_duplicate_nodes)   {
                if(set_node_offsets.count(nodeRef->GetFileOffset()) != 0)   {
                    continue;
                }
                set_node_offsets.insert(nodeRef->GetFileOffset());
            }
            MapObject map_object;
            map_object.name     = nodeRef->GetName();
            map_object.offset   = nodeRef->GetFileOffset();
            map_object.type     = osmscout::refNode;
            list_map_objects.push_back(map_object);
        }
        for(size_t j=0; j < listWays.size(); j++)   {
            osmscout::WayRef &wayRef = listWays[j];

            if(wayRef->GetName().empty())   {
                continue;
            }
            if(!allow_duplicate_ways)   {
                if(set_way_offsets.count(wayRef->GetFileOffset()) != 0)   {
                    continue;
                }
                set_way_offsets.insert(wayRef->GetFileOffset());
            }
            MapObject map_object;
            map_object.name     = wayRef->GetName();
            map_object.offset   = wayRef->GetFileOffset();
            map_object.type     = osmscout::refWay;
            list_map_objects.push_back(map_object);
        }
        for(size_t j=0; j < listAreas.size(); j++)   {
            osmscout::AreaRef &areaRef = listAreas[j];

            if(areaRef->rings.front().GetName().empty())   {
                continue;
            }
            if(!allow_duplicate_areas)   {
                if(set_area_offsets.count(areaRef->GetFileOffset()) != 0)   {
                    continue;
                }
                set_area_offsets.insert(areaRef->GetFileOffset());
            }
            MapObject map_object;
            map_object.name     = areaRef->rings.front().GetName();
            map_object.offset   = areaRef->GetFileOffset();
            map_object.type     = osmscout::refArea;
            list_map_objects.push_back(map_object);
        }

        // create structs to sort file offsets by name lookup
        // [name_lookup_id] [list_offsets]
        boost::unordered_map<int32_t,OffsetGroup> entry_admin_regions;

        for(size_t j=0; j < list_map_objects.size(); j++)   {

            MapObject &map_object = list_map_objects[j];

            // add name_lookup up to table_names
            std::string name_lookup = convNameToLookup(map_object.name);
            int32_t name_lookup_id;

            // check if this lookup string already exists
            if(table_names.count(name_lookup) == 0)   {
                // insert lookup string
                std::pair<std::string,int32_t> data;
                data.first  = name_lookup;
                data.second = name_id;
                table_names.insert(data);

                name_lookup_id = name_id;
                name_id++;
            }
            else   {
                name_lookup_id = table_names.find(name_lookup)->second;
            }

            // check if this lookup string already exists
            if(entry_admin_regions.count(name_lookup_id) == 0)   {
                // insert new list of offsets
                std::pair<int32_t,OffsetGroup> data;
                data.first = name_lookup_id;
                entry_admin_regions.insert(data);
            }

            // add offset to list
            boost::unordered_map<int32_t,OffsetGroup>::iterator it;
            it = entry_admin_regions.find(name_lookup_id);
            if(map_object.type == osmscout::refNode)   {
                it->second.node_offsets.push_back(map_object.offset);
            }
            else if(map_object.type == osmscout::refWay)   {
                it->second.way_offsets.push_back(map_object.offset);
            }
            else if(map_object.type == osmscout::refArea)   {
                it->second.area_offsets.push_back(map_object.offset);
            }
        }

        //qDebug() << list_tiles[i]->id << ":" << entry_admin_regions.size();

        // write information for this tile to the database
        boost::unordered_map<int32_t,OffsetGroup>::iterator it;
        for(it  = entry_admin_regions.begin();
            it != entry_admin_regions.end(); ++it)   {
            // convert offsets to byte arrays
            char * data_node_offsets = NULL; size_t sz_node_offsets=0;
            char * data_way_offsets  = NULL; size_t sz_way_offsets=0;
            char * data_area_offsets = NULL; size_t sz_area_offsets=0;

            OffsetGroup &g = it->second;

//            // ### debug start
//            if(g_table_nameid_count.count(it->first) == 0)   {
//                std::pair<int64_t,int64_t> debugdata;
//                debugdata.first = it->first;
//                debugdata.second =
//                        g.node_offsets.size()+
//                        g.way_offsets.size()+
//                        g.area_offsets.size();
//                g_table_nameid_count.insert(debugdata);
//            }
//            else   {
//                std::map<int64_t,int64_t>::iterator d_it;
//                d_it = g_table_nameid_count.find(it->first);
//                d_it->second +=
//                        g.node_offsets.size()+
//                        g.way_offsets.size()+
//                        g.area_offsets.size();
//            }
//            // ### debug end

            if(!(g.node_offsets.empty()))   {
                sz_node_offsets = sizeof(osmscout::FileOffset)*g.node_offsets.size();
                data_node_offsets = new char[sz_node_offsets];
                memcpy(data_node_offsets,&(g.node_offsets[0]),sz_node_offsets);
                list_blobs.push_back(data_node_offsets);
            }
            if(!(g.way_offsets.empty()))   {
                sz_way_offsets = sizeof(osmscout::FileOffset)*g.way_offsets.size();
                data_way_offsets = new char[sz_way_offsets];
                memcpy(data_way_offsets,&(g.way_offsets[0]),sz_way_offsets);
                list_blobs.push_back(data_way_offsets);
            }
            if(!(g.area_offsets.empty()))   {
                sz_area_offsets = sizeof(osmscout::FileOffset)*g.area_offsets.size();
                data_area_offsets = new char[sz_area_offsets];
                memcpy(data_area_offsets,&(g.area_offsets[0]),sz_area_offsets);
                list_blobs.push_back(data_area_offsets);
            }

            if(data_node_offsets == NULL &&
               data_way_offsets  == NULL &&
               data_area_offsets == NULL)   {
                continue;
            }

            // prepare sql
            try   {

                // cat name_id and tile_id into one id
                // get mask for bitlength
                int32_t mask = pow(2,g_sz_bits_tile_id)-1;

                int64_t id = it->first;             // name_id
                id = id << g_sz_bits_tile_id;       // bitshift
                id |= (list_tiles[i]->id & mask);   // 24 most sig bits of tile_id

                stmt->BindInt64(1,id);

                if(data_node_offsets)   {
                    stmt->BindBlob(2,data_node_offsets,sz_node_offsets);
                }
                else   {
                    stmt->BindNull(2);
                }
                if(data_way_offsets)   {
                    stmt->BindBlob(3,data_way_offsets,sz_way_offsets);
                }
                else   {
                    stmt->BindNull(3);
                }
                if(data_area_offsets)   {
                    stmt->BindBlob(4,data_area_offsets,sz_area_offsets);
                }
                else   {
                    stmt->BindNull(4);
                }

                stmt->Execute();
                stmt->Reset();

                if(transaction_count > transaction_limit)   {
                    stmt->FreeQuery();
                    stmt->CommitTransaction();

                    stmt->BeginTransaction();
                    stmt->Sql(stmt_insert);
                    transaction_count=0;
                }
                transaction_count++;
            }
            catch(Kompex::SQLiteException &exception)   {
                qDebug() << "ERROR: SQLite exception writing tile data:"
                         << QString::fromStdString(exception.GetString());
                qDebug() << "ERROR: id:" << list_tiles[i]->id;
                qDebug() << "ERROR: name_lookup_id:" << it->first;
                qDebug() << "ERROR:" << sz_node_offsets << sz_way_offsets << sz_area_offsets;
                return false;
            }
        }
        // debug
//        qDebug() << i << "/" << list_tiles.size();
    }

    stmt->FreeQuery();
    stmt->CommitTransaction();

    // free up blob memory
    for(size_t i=0; i < list_blobs.size(); i++)   {
        delete[] list_blobs[i];
    }

    return true;
}
示例#6
0
void
addExonToGraph (TGraph & g,
                unsigned short const & bit_id,
                unsigned short const & bit_n,
                CharString const & sequence,
                std::map<VertexLabels, TVertexDescriptor> & vertex_label_map,
                std::vector<VertexLabels> & vertex_vector,
                boost::unordered_map<std::pair<TVertexDescriptor, TVertexDescriptor>, boost::dynamic_bitset<> > & edge_ids
               )
{
  VertexLabels prev_vertex_label = {0, Dna('A')};
  TVertexDescriptor prev = vertex_label_map[prev_vertex_label];
  
  for (int pos = 0 ; ; ++pos)
  {
    if (pos >= (int) length(sequence))
    {
      VertexLabels final_vertex_label = {
        -1, // level
        Dna('A'), // Dna, initialized here but we should never use it!
      };

      TVertexDescriptor target_vertex = vertex_label_map[final_vertex_label];
      std::pair<TVertexDescriptor, TVertexDescriptor> my_pair (prev, target_vertex);
      addBitsetEdge(edge_ids[my_pair], bit_id, bit_n, edge_ids.count(my_pair));

      if (!findEdge(g, prev, target_vertex))
        addEdge(g, prev, target_vertex);

      break;
    }

    if (sequence[pos] == '-')
      continue;

    VertexLabels new_vertex_label =
    {
      pos+1,               // level
      Dna(sequence[pos]),  // dna
    };

    TVertexDescriptor target_vertex;

    if (vertex_label_map.count(new_vertex_label) == 1)
    {
      target_vertex = vertex_label_map[new_vertex_label];
      std::pair<TVertexDescriptor, TVertexDescriptor> my_pair (prev, target_vertex);
      addBitsetEdge(edge_ids[my_pair], bit_id, bit_n, edge_ids.count(my_pair));

      if (!findEdge(g, prev, target_vertex))
        addEdge(g, prev, target_vertex);
    }
    else
    {
      target_vertex = addVertex(g);
      std::pair<TVertexDescriptor, TVertexDescriptor> my_pair (prev, target_vertex);
      addBitsetEdge(edge_ids[my_pair], bit_id, bit_n, edge_ids.count(my_pair));
      addEdge(g, prev, target_vertex);
      vertex_label_map[new_vertex_label] = target_vertex;
      vertex_vector.push_back(new_vertex_label);
    }
    prev = target_vertex;
  }
}