Пример #1
0
  bool RawNode::Read(const TypeConfig& typeConfig,
                     FileScanner& scanner)
  {
    if (!scanner.ReadNumber(id)) {
      return false;
    }

    TypeId typeId;

    if (!scanner.ReadTypeId(typeId,
                            typeConfig.GetNodeTypeIdBytes())) {
      return false;
    }

    TypeInfoRef type=typeConfig.GetNodeTypeInfo(typeId);

    featureValueBuffer.SetType(type);

    if (!type->GetIgnore()) {
      if (!featureValueBuffer.Read(scanner)) {
        return false;
      }
    }

    if (!scanner.ReadCoord(coords)) {
      return false;
    }

    return !scanner.HasError();
  }
Пример #2
0
  bool Way::ReadIds(FileScanner& scanner)
  {
    ids.resize(nodes.size());

    Id minId;

    scanner.ReadNumber(minId);

    if (minId>0) {
      size_t idCurrent=0;

      while (idCurrent<ids.size()) {
        uint8_t bitset;
        size_t  bitmask=1;

        scanner.Read(bitset);

        for (size_t i=0; i<8 && idCurrent<ids.size(); i++) {
          if (bitset & bitmask) {
            scanner.ReadNumber(ids[idCurrent]);

            ids[idCurrent]+=minId;
          }
          else {
            ids[idCurrent]=0;
          }

          bitmask*=2;
          idCurrent++;
        }
      }
    }

    return !scanner.HasError();
  }
Пример #3
0
  bool Way::ReadOptimized(const TypeConfig& typeConfig,
                          FileScanner& scanner)
  {
    if (!scanner.GetPos(fileOffset)) {
      return false;
    }

    TypeId typeId;

    scanner.ReadTypeId(typeId,
                       typeConfig.GetWayTypeIdBytes());

    TypeInfoRef type=typeConfig.GetWayTypeInfo(typeId);

    featureValueBuffer.SetType(type);

    if (!featureValueBuffer.Read(scanner)) {
      return false;
    }

    if (!scanner.Read(nodes)) {
      return false;
    }

    return !scanner.HasError();
  }
  bool OptimizeAreasLowZoomGenerator::GetAreas(const TypeConfig& typeConfig,
                                               const ImportParameter& parameter,
                                               Progress& progress,
                                               FileScanner& scanner,
                                               const TypeInfoSet& types,
                                               std::vector<std::list<AreaRef> >& areas,
                                               TypeInfoSet& loadedTypes)
  {
    uint32_t    areaCount=0;
    size_t      collectedAreasCount=0;

    loadedTypes=types;

    progress.SetAction("Collecting area data to optimize");

    scanner.GotoBegin();

    scanner.Read(areaCount);

    for (uint32_t a=1; a<=areaCount; a++) {
      AreaRef area=std::make_shared<Area>();

      progress.SetProgress(a,areaCount);

      area->Read(typeConfig,
                 scanner);

      if (loadedTypes.IsSet(area->GetType())) {
        areas[area->GetType()->GetIndex()].push_back(area);

        collectedAreasCount++;

        while (collectedAreasCount>parameter.GetOptimizationMaxWayCount() &&
               loadedTypes.Size()>1) {
          TypeInfoRef victimType;

          for (auto &type : loadedTypes) {
            if (areas[type->GetIndex()].size()>0 &&
                (!victimType ||
                 areas[type->GetIndex()].size()<areas[victimType->GetIndex()].size())) {
              victimType=type;
            }
          }

          assert(victimType);

          collectedAreasCount-=areas[victimType->GetIndex()].size();
          areas[victimType->GetIndex()].clear();
          loadedTypes.Remove(victimType);
        }
      }
    }

    progress.Info("Collected "+NumberToString(collectedAreasCount)+" areas for "+NumberToString(loadedTypes.Size())+" types");

    return !scanner.HasError();
  }
Пример #5
0
  bool Relation::Read(FileScanner& scanner)
  {
    uint32_t roleCount;

    scanner.Read(id);

    if (!attributes.Read(scanner)) {
      return false;
    }

    scanner.ReadNumber(roleCount);
    if (scanner.HasError()) {
      return false;
    }

    roles.resize(roleCount);
    for (size_t i=0; i<roleCount; i++) {
      uint32_t nodesCount;

      if (!roles[i].attributes.Read(scanner)) {
        return false;
      }

      scanner.Read(roles[i].role);

      scanner.ReadNumber(nodesCount);

      if (nodesCount>0) {
        roles[i].nodes.resize(nodesCount);

        Id       minId;
        uint32_t minLat;
        uint32_t minLon;

        scanner.Read(minId);
        scanner.Read(minLat);
        scanner.Read(minLon);

        for (size_t j=0; j<nodesCount; j++) {
          Id       id;
          uint32_t latValue;
          uint32_t lonValue;

          scanner.ReadNumber(id);
          scanner.ReadNumber(latValue);
          scanner.ReadNumber(lonValue);

          roles[i].nodes[j].id=minId+id;
          roles[i].nodes[j].lat=(minLat+latValue)/conversionFactor-90.0;
          roles[i].nodes[j].lon=(minLon+lonValue)/conversionFactor-180.0;
        }
      }
    }

    return !scanner.HasError();
  }
Пример #6
0
  /**
   * Reads the TurnRestriction data from the given FileScanner
   *
   * @throws IOException
   */
  void TurnRestriction::Read(FileScanner& scanner)
  {
    uint32_t typeValue;

    scanner.ReadNumber(typeValue);
    this->type=(Type)typeValue;

    scanner.ReadNumber(from);
    scanner.ReadNumber(via);
    scanner.ReadNumber(to);
  }
Пример #7
0
static void runScan(void)
{
    loadMusic();

    if ("" != gMusicData->startdir)
    {
        FileScanner *fscan = new FileScanner();
        fscan->SearchDir(gMusicData->startdir);
        gMusicData->reloadMusic();
        delete fscan;
    }
}
Пример #8
0
static void runScan(void)
{
    // maybe we haven't loaded the music yet in which case we wont have a valid music dir set
    if (gMusicData->musicDir.isEmpty())
    {
        QString startdir = gCoreContext->GetSetting("MusicLocation");
        startdir = QDir::cleanPath(startdir);
        if (!startdir.isEmpty() && !startdir.endsWith("/"))
            startdir += "/";

        gMusicData->musicDir = startdir;
    }

    // if we still don't have a valid start dir warn the user and give up
    if (gMusicData->musicDir.isEmpty())
    {
        ShowOkPopup(QObject::tr("You need to tell me where to find your music on the "
                                "'General Settings' page of MythMusic's settings pages."));
       return;
    }

    if (!QFile::exists(gMusicData->musicDir))
    {
        ShowOkPopup(QObject::tr("Can't find your music directory. Have you set it correctly on the "
                                "'General Settings' page of MythMusic's settings pages?"));
       return;
    }

    LOG(VB_GENERAL, LOG_INFO, QString("Scanning '%1' for music files").arg(gMusicData->musicDir));

    FileScanner *fscan = new FileScanner();
    fscan->SearchDir(gMusicData->musicDir);

    // save anything that may have changed
    if (gMusicData->all_music && gMusicData->all_music->cleanOutThreads())
        gMusicData->all_music->save();

    if (gMusicData->all_playlists && gMusicData->all_playlists->cleanOutThreads())
    {
        gMusicData->all_playlists->save();
        int x = gMusicData->all_playlists->getPending();
        SavePending(x);
    }

    // force a complete reload of the tracks and playlists
    gPlayer->stop(true);
    delete gMusicData;

    gMusicData = new MusicData;
    loadMusic();

    delete fscan;
}
Пример #9
0
  bool LocationIndex::VisitLocationAddressEntries(FileScanner& scanner,
                                                  const AdminRegion& region,
                                                  const Location& location,
                                                  AddressVisitor& visitor,
                                                  bool& stopped) const
  {
    uint32_t addressCount;

    if (!scanner.SetPos(location.addressesOffset)) {
      return false;
    }

    if (!scanner.ReadNumber(addressCount)) {
      return false;
    }

    ObjectFileRefStreamReader objectFileRefReader(scanner);

    for (size_t i=0; i<addressCount; i++) {
      Address address;

      if (!scanner.GetPos(address.addressOffset)) {
        return false;
      }

      address.locationOffset=location.locationOffset;
      address.regionOffset=location.regionOffset;

      if (!scanner.Read(address.name)) {
        return false;
      }

      if (!objectFileRefReader.Read(address.object)) {
        return false;
      }

      if (!visitor.Visit(region,
                         location,
                         address)) {
        stopped=true;

        break;
      }
    }

    return !scanner.HasError();
  }
Пример #10
0
  bool LocationIndex::VisitAdminRegions(AdminRegionVisitor& visitor) const
  {
    FileScanner scanner;

    if (!scanner.Open(AppendFileToDir(path,
                                      FILENAME_LOCATION_IDX),
                      FileScanner::LowMemRandom,
                      false)) {
      log.Error() << "Cannot open file '" << scanner.GetFilename() << "'!";
      return false;
    }

    if (!scanner.SetPos(indexOffset)) {
      return false;
    }

    uint32_t regionCount;

    if (!scanner.ReadNumber(regionCount)) {
      return false;
    }

    for (size_t i=0; i<regionCount; i++) {
      AdminRegionVisitor::Action action;
      FileOffset                 nextChildOffset;

      if (!scanner.ReadFileOffset(nextChildOffset)) {
        return false;
      }

      action=VisitRegionEntries(scanner,
                                visitor);

      if (action==AdminRegionVisitor::error) {
        return false;
      }
      else if (action==AdminRegionVisitor::stop) {
        return true;
      }
      else if (action==AdminRegionVisitor::skipChildren) {
        if (i+1<regionCount) {
          if (!scanner.SetPos(nextChildOffset)) {
            return false;
          }
        }
      }
    }

    return !scanner.HasError() && scanner.Close();
  }
Пример #11
0
  bool LocationIndex::VisitAdminRegionLocations(const AdminRegion& region,
                                                LocationVisitor& visitor,
                                                bool recursive) const
  {
    FileScanner scanner;
    bool        stopped=false;

    if (!scanner.Open(AppendFileToDir(path,
                                      FILENAME_LOCATION_IDX),
                      FileScanner::LowMemRandom,
                      true)) {
      log.Error() << "Cannot open file '" << scanner.GetFilename() << "'!";
      return false;
    }

    if (!scanner.SetPos(indexOffset)) {
      return false;
    }

    if (!scanner.SetPos(region.regionOffset)) {
      return false;
    }

    if (!VisitRegionLocationEntries(scanner,
                                    visitor,
                                    recursive,
                                    stopped)) {
      return false;
    }

    return !scanner.HasError() && scanner.Close();
  }
Пример #12
0
  bool LocationIndex::ResolveAdminRegionHierachie(const AdminRegionRef& adminRegion,
                                                  std::map<FileOffset,AdminRegionRef >& refs) const
  {
    FileScanner scanner;

    if (!scanner.Open(AppendFileToDir(path,
                                      FILENAME_LOCATION_IDX),
                      FileScanner::LowMemRandom,
                      true)) {
      log.Error() << "Cannot open file '" << scanner.GetFilename() << "'!";
      return false;
    }

    if (!scanner.SetPos(indexOffset)) {
      return false;
    }

    std::list<FileOffset> offsets;

    refs[adminRegion->regionOffset]=adminRegion;

    if (adminRegion->parentRegionOffset!=0) {
      offsets.push_back(adminRegion->parentRegionOffset);
    }

    while (!offsets.empty()) {
      std::list<FileOffset> newOffsets;

      for (const auto& offset : offsets) {
        if (refs.find(offset)!=refs.end()) {
          continue;
        }

        if (!scanner.SetPos(offset)) {
          return false;
        }

        AdminRegion adminRegion;

        if (!LoadAdminRegion(scanner,
                             adminRegion)) {
          return false;
        }

        refs[adminRegion.regionOffset]=std::make_shared<AdminRegion>(adminRegion);

        if (adminRegion.parentRegionOffset!=0) {
          newOffsets.push_back(adminRegion.parentRegionOffset);
        }

      }
      offsets.clear();

      std::swap(offsets,
                newOffsets);
    }

    return !scanner.HasError() && scanner.Close();
  }
Пример #13
0
static void MusicCallback(void *data, QString &selection)
{
    (void) data;

    QString sel = selection.toLower();
    if (sel == "music_create_playlist")
        startDatabaseTree();
    else if (sel == "music_play")
        startPlayback();
    else if (sel == "music_rip")
    {
        startRipper();
    }
    else if (sel == "music_import")
    {
        startImport();
    }
    else if (sel == "settings_scan")
    {
        if ("" != gMusicData->startdir)
        {
            loadMusic();
            FileScanner *fscan = new FileScanner();
            fscan->SearchDir(gMusicData->startdir);
            gMusicData->reloadMusic();
            delete fscan;
        }
    }
    else if (sel == "music_set_general")
    {
        MusicGeneralSettings settings;
        settings.exec();
    }
    else if (sel == "music_set_player")
    {
        MusicPlayerSettings settings;
        settings.exec();
    }
    else if (sel == "music_set_ripper")
    {
        MusicRipperSettings settings;
        settings.exec();
    }
}
Пример #14
0
  bool DebugDatabase::ResolveReferences(const std::string& mapName,
                                        RefType fileType,
                                        const std::set<ObjectOSMRef>& ids,
                                        const std::set<ObjectFileRef>& fileOffsets,
                                        std::map<ObjectOSMRef,ObjectFileRef>& idFileOffsetMap,
                                        std::map<ObjectFileRef,ObjectOSMRef>& fileOffsetIdMap)
  {
    FileScanner scanner;
    uint32_t    entryCount;
    std::string filename=AppendFileToDir(path,mapName);

    if (!scanner.Open(filename,FileScanner::LowMemRandom,false)) {
      std::cerr << "Cannot open file '" << scanner.GetFilename() << "'!" << std::endl;
      return false;
    }

    if (!scanner.Read(entryCount)) {
      return false;
    }

    for (size_t i=1; i<=entryCount; i++) {
      Id         id;
      uint8_t    typeByte;
      OSMRefType osmType;
      FileOffset fileOffset;

      if (!scanner.Read(id)) {
        return false;
      }

      if (!scanner.Read(typeByte)) {
        return false;
      }

      osmType=(OSMRefType)typeByte;

      if (!scanner.ReadFileOffset(fileOffset)) {
        return false;
      }

      ObjectOSMRef  osmRef(id,osmType);
      ObjectFileRef fileRef(fileOffset,fileType);

      if (ids.find(osmRef)!=ids.end() ||
          fileOffsets.find(fileRef)!=fileOffsets.end()) {
        idFileOffsetMap.insert(std::make_pair(osmRef,fileRef));
        fileOffsetIdMap.insert(std::make_pair(fileRef,osmRef));
      }
    }

    return scanner.Close();
  }
Пример #15
0
  bool LocationIndex::LoadAdminRegion(FileScanner& scanner,
                                      AdminRegion& region) const
  {
    uint32_t aliasCount;

    if (!scanner.GetPos(region.regionOffset)) {
      return false;
    }

    if (!scanner.ReadFileOffset(region.dataOffset)) {
      return false;
    }

    if (!scanner.ReadFileOffset(region.parentRegionOffset)) {
      return false;
    }

    if (!scanner.Read(region.name)) {
      return false;
    }

    if (!Read(scanner,
              region.object)) {
      return false;
    }

    if (!scanner.ReadNumber(aliasCount)) {
      return false;
    }

    region.aliases.clear();

    if (aliasCount>0) {
      region.aliases.resize(aliasCount);

      for (size_t i=0; i<aliasCount; i++) {
        if (!scanner.Read(region.aliases[i].name)) {
          return false;
        }

        if (!scanner.ReadFileOffset(region.aliases[i].objectOffset,
                                    bytesForNodeFileOffset)) {
          return false;
        }
      }
    }

    return !scanner.HasError();
  }
Пример #16
0
  bool LocationIndex::Read(FileScanner& scanner,
                           ObjectFileRef& object) const
  {
    uint8_t    type;
    FileOffset fileOffset;

    if (!scanner.Read(type)) {
      return false;
    }

    switch (type) {
    case refNode:
      if (!scanner.ReadFileOffset(fileOffset,
                                  bytesForNodeFileOffset)) {
        return false;
      }
      break;
    case refArea:
      if (!scanner.ReadFileOffset(fileOffset,
                                  bytesForAreaFileOffset)) {
        return false;
      }
      break;
    case refWay:
      if (!scanner.ReadFileOffset(fileOffset,
                                  bytesForWayFileOffset)) {
        return false;
      }
      break;
    default:
      return false;
    }

    object.Set(fileOffset,
               (RefType)type);

    return true;
  }
Пример #17
0
  /**
   * Load the object variant data from the given file.
   *
   * @param typeConfig
   *    TypeConfig instance
   * @param filename
   *    Name of the file containing the object variant data
   * @return
   *    True on success, else false
   */
  bool ObjectVariantDataFile::Load(const TypeConfig& typeConfig,
                                   const std::string& filename)
  {
    FileScanner scanner;

    data.clear();

    isLoaded=false;
    this->filename=filename;

    try {
      scanner.Open(filename,
                   FileScanner::Sequential,true);

      uint32_t dataCount;

      scanner.Read(dataCount);

      data.resize(dataCount);

      for (size_t i=0; i<dataCount; i++) {
        data[i].Read(typeConfig,
                     scanner);
      }

      scanner.Close();

      isLoaded=true;
    }
    catch (IOException& e) {
      log.Error() << e.GetDescription();
      scanner.CloseFailsafe();
      return false;
    }

    return true;
  }
Пример #18
0
  bool Intersection::Read(FileScanner& scanner)
  {
    try {
      scanner.ReadNumber(nodeId);

      uint32_t objectCount;

      scanner.ReadNumber(objectCount);

      objects.resize(objectCount);

      ObjectFileRefStreamReader objectFileRefReader(scanner);

      for (size_t i=0; i<objectCount; i++) {
        objectFileRefReader.Read(objects[i]);
      }
    }
    catch (IOException& e) {
      log.Error() << e.GetDescription();
      return false;
    }

    return true;
  }
Пример #19
0
  bool Database::Open(const std::string& path)
  {
    assert(!path.empty());

    this->path=path;

    typeConfig=std::make_shared<TypeConfig>();

    if (!typeConfig->LoadFromDataFile(path)) {
      log.Error() << "Cannot load 'types.dat'!";
      return false;
    }

    FileScanner scanner;

    try {
      scanner.Open(AppendFileToDir(path,"bounding.dat"),
                   FileScanner::Normal,
                   false);

      scanner.ReadBox(boundingBox);

      log.Debug() << "BoundingBox: " << boundingBox.GetDisplayText();

      scanner.Close();
    }
    catch (IOException& e) {
      log.Error() << e.GetDescription();
      scanner.CloseFailsafe();
      return false;
    }

    isOpen=true;

    return true;
  }
Пример #20
0
  bool Node::Read(const TypeConfig& typeConfig,
                  FileScanner& scanner)
  {

    if (!scanner.GetPos(fileOffset)) {
      return false;
    }

    uint32_t tmpType;

    scanner.ReadNumber(tmpType);

    TypeInfoRef type=typeConfig.GetTypeInfo((TypeId)tmpType);

    featureValueBuffer.SetType(type);

    if (!featureValueBuffer.Read(scanner)) {
      return false;
    }

    scanner.ReadCoord(coords);

    return !scanner.HasError();
  }
Пример #21
0
  bool Database::Open(const std::string& path)
  {
    assert(!path.empty());

    this->path=path;

    typeConfig=new TypeConfig();

    if (!typeConfig->LoadFromDataFile(path)) {
      log.Error() << "Cannot load 'types.dat'!";
      return false;
    }

    FileScanner scanner;
    std::string file=AppendFileToDir(path,"bounding.dat");

    if (!scanner.Open(file,FileScanner::Normal,true)) {
      log.Error() << "Cannot open '" << scanner.GetFilename() << "'";
      return false;
    }

    if (!scanner.ReadBox(boundingBox)) {
      log.Error() << "Error while reading '" << scanner.GetFilename() << "'";
    }

    log.Debug() << "BoundingBox: " << boundingBox.GetDisplayText();

    if (scanner.HasError() || !scanner.Close()) {
      log.Error() << "Cannot while reading/closing '" << scanner.GetFilename() << "'";
      return false;
    }

    isOpen=true;

    return true;
  }
Пример #22
0
static void loadMusic()
{
    // only do this once
    if (gMusicData->initialized)
        return;

    MythScreenStack *popupStack = GetMythMainWindow()->GetStack("popup stack");
    QString message = QObject::tr("Loading Music. Please wait ...");

    MythUIBusyDialog *busy = new MythUIBusyDialog(message, popupStack,
                                                  "musicscanbusydialog");
    if (busy->Create())
        popupStack->AddScreen(busy, false);
    else
        busy = NULL;

    srand(time(NULL));

    CheckFreeDBServerFile();

    MSqlQuery count_query(MSqlQuery::InitCon());

    bool musicdata_exists = false;
    if (count_query.exec("SELECT COUNT(*) FROM music_songs;"))
    {
        if(count_query.next() &&
            0 != count_query.value(0).toInt())
        {
            musicdata_exists = true;
        }
    }

    //  Load all available info about songs (once!)
    QString startdir = gCoreContext->GetSetting("MusicLocation");
    startdir = QDir::cleanPath(startdir);
    if (!startdir.endsWith("/"))
        startdir += "/";

    Metadata::SetStartdir(startdir);

    Decoder::SetLocationFormatUseTags();

    // Only search music files if a directory was specified & there
    // is no data in the database yet (first run).  Otherwise, user
    // can choose "Setup" option from the menu to force it.
    if (!startdir.isEmpty() && !musicdata_exists)
    {
        FileScanner *fscan = new FileScanner();
        fscan->SearchDir(startdir);
        delete fscan;
    }

    QString paths = gCoreContext->GetSetting("TreeLevels");

    // Set the various track formatting modes
    Metadata::setArtistAndTrackFormats();

    AllMusic *all_music = new AllMusic(paths, startdir);

    //  Load all playlists into RAM (once!)
    PlaylistContainer *all_playlists = new PlaylistContainer(
            all_music, gCoreContext->GetHostName());

    gMusicData->paths = paths;
    gMusicData->startdir = startdir;
    gMusicData->all_playlists = all_playlists;
    gMusicData->all_music = all_music;
    gMusicData->initialized = true;

    while (!gMusicData->all_playlists->doneLoading() || !gMusicData->all_music->doneLoading())
    {
        qApp->processEvents();
        usleep(50000);
    }
    gMusicData->all_playlists->postLoad();

    gPlayer->constructPlaylist();

    if (busy)
        busy->Close();

}
Пример #23
0
 /**
  * Reads the data from the given FileScanner
  *
  * @throws IOException
  */
 void RawCoord::Read(const TypeConfig& /*typeConfig*/,
                    FileScanner& scanner)
 {
   scanner.ReadNumber(id);
   scanner.ReadCoord(coord);
 }
Пример #24
0
  bool AreaAreaIndexGenerator::Import(const TypeConfigRef& typeConfig,
                                      const ImportParameter& parameter,
                                      Progress& progress)
  {
    FileScanner               scanner;
    size_t                    areas=0;         // Number of areas found
    size_t                    areasConsumed=0; // Number of areas consumed
    std::vector<double>       cellWidth;
    std::vector<double>       cellHeight;
    std::map<Pixel,AreaLeaf>  leafs;
    std::map<Pixel,AreaLeaf>  newAreaLeafs;

    cellWidth.resize(parameter.GetAreaAreaIndexMaxMag()+1);
    cellHeight.resize(parameter.GetAreaAreaIndexMaxMag()+1);

    for (size_t i=0; i<cellWidth.size(); i++) {
      cellWidth[i]=360.0/pow(2.0,(int)i);
    }

    for (size_t i=0; i<cellHeight.size(); i++) {
      cellHeight[i]=180.0/pow(2.0,(int)i);
    }

    //
    // Writing index file
    //

    progress.SetAction("Generating 'areaarea.idx'");

    FileWriter writer;
    FileOffset topLevelOffset=0;
    FileOffset topLevelOffsetOffset; // Offset of the toplevel entry

    if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                     "areaarea.idx"))) {
      progress.Error("Cannot create 'areaarea.idx'");
      return false;
    }

    if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                         "areas.dat"),
                         FileScanner::Sequential,
                         parameter.GetWayDataMemoryMaped())) {
      progress.Error("Cannot open 'areas.dat'");
      return false;
    }

    writer.WriteNumber((uint32_t)parameter.GetAreaAreaIndexMaxMag()); // MaxMag

    if (!writer.GetPos(topLevelOffsetOffset)) {
      progress.Error("Cannot read current file position");
      return false;
    }

    if (!writer.WriteFileOffset(topLevelOffset)) {
      progress.Error("Cannot write top level entry offset");
      return false;
    }

    int l=parameter.GetAreaAreaIndexMaxMag();

    while (l>=0) {
      size_t areaLevelEntries=0;

      progress.Info(std::string("Storing level ")+NumberToString(l)+"...");

      newAreaLeafs.clear();

      SetOffsetOfChildren(leafs,newAreaLeafs);

      leafs=newAreaLeafs;

      // Areas

      if (areas==0 ||
          (areas>0 && areas>areasConsumed)) {
        uint32_t areaCount=0;

        progress.Info(std::string("Scanning areas.dat for areas of index level ")+NumberToString(l)+"...");

        if (!scanner.GotoBegin()) {
          progress.Error("Cannot go to begin of way file");
        }

        if (!scanner.Read(areaCount)) {
          progress.Error("Error while reading number of data entries in file");
          return false;
        }

        areas=0;
        for (uint32_t a=1; a<=areaCount; a++) {
          progress.SetProgress(a,areaCount);

          FileOffset offset;
          Area       area;

          scanner.GetPos(offset);

          if (!area.Read(typeConfig,
                         scanner)) {
            progress.Error(std::string("Error while reading data entry ")+
                           NumberToString(a)+" of "+
                           NumberToString(areaCount)+
                           " in file '"+
                           scanner.GetFilename()+"'");
            return false;
          }

          areas++;

          double minLon;
          double maxLon;
          double minLat;
          double maxLat;

          area.GetBoundingBox(minLon,maxLon,minLat,maxLat);

          //
          // Calculate highest level where the bounding box completely
          // fits in the cell size and assign area to the tiles that
          // hold the geometric center of the tile.
          //

          int level=parameter.GetAreaAreaIndexMaxMag();
          while (level>=0) {
            if (maxLon-minLon<=cellWidth[level] &&
                maxLat-minLat<=cellHeight[level]) {
              break;
            }

            level--;
          }

          if (level==l) {
            //
            // Renormated coordinate space (everything is >=0)
            //

            minLon+=180;
            maxLon+=180;
            minLat+=90;
            maxLat+=90;

            //
            // Calculate minimum and maximum tile ids that are covered
            // by the area
            //
            uint32_t minyc=(uint32_t)floor(minLat/cellHeight[level]);
            uint32_t maxyc=(uint32_t)ceil(maxLat/cellHeight[level]);
            uint32_t minxc=(uint32_t)floor(minLon/cellWidth[level]);
            uint32_t maxxc=(uint32_t)ceil(maxLon/cellWidth[level]);

            Entry entry;

            entry.type=area.GetType()->GetId();
            entry.offset=offset;

            // Add this area to the tile where the center of the area lies in.
            leafs[Pixel((minxc+maxxc)/2,(minyc+maxyc)/2)].areas.push_back(entry);
            areaLevelEntries++;

            areasConsumed++;
          }
        }
      }

      progress.Debug(std::string("Writing ")+NumberToString(leafs.size())+" leafs ("+
                     NumberToString(areaLevelEntries)+") "+
                     "to index of level "+NumberToString(l)+"...");

      // Remember the offset of one cell in level '0'
      if (l==0) {
        if (!writer.GetPos(topLevelOffset)) {
          progress.Error("Cannot read top level entry offset");
          return false;
        }
      }

      /*
      uint32_t minX=std::numeric_limits<uint32_t>::max();
      uint32_t minY=std::numeric_limits<uint32_t>::max();
      uint32_t maxX=std::numeric_limits<uint32_t>::min();
      uint32_t maxY=std::numeric_limits<uint32_t>::min();

      std::map<TypeId,size_t> useMap;

      for (std::map<Pixel,AreaLeaf>::const_iterator leaf=leafs.begin();
           leaf!=leafs.end();
           ++leaf) {
        minX=std::min(minX,leaf->first.x);
        maxX=std::max(maxX,leaf->first.x);
        minY=std::min(minY,leaf->first.y);
        maxY=std::max(maxY,leaf->first.y);

        for (std::list<Entry>::const_iterator entry=leaf->second.areas.begin();
             entry!=leaf->second.areas.end();
             entry++) {
          std::map<TypeId,size_t>::iterator u=useMap.find(entry->type);

          if (u==useMap.end()) {
            useMap[entry->type]=1;
          }
          else {
            u->second++;
          }
        }
      }*/

      /*
      std::cout << "[" << minX << "-" << maxX << "]x[" << minY << "-" << maxY << "] => " << leafs.size() << "/" << (maxX-minX+1)*(maxY-minY+1) << " " << (int)BytesNeededToAddressFileData(leafs.size()) << " " << ByteSizeToString(BytesNeededToAddressFileData(leafs.size())*(maxX-minX+1)*(maxY-minY+1)) << std::endl;

      for (std::map<TypeId,size_t>::const_iterator u=useMap.begin();
          u!=useMap.end();
          ++u) {
        std::cout << "* " << u->first << " " << typeConfig.GetTypeInfo(u->first).GetName() << " " << u->second << std::endl;
      }*/

      if (!WriteIndexLevel(parameter,
                           writer,
                           (int)l,
                           leafs)) {
        return false;
      }

      l--;
    }

    writer.SetPos(topLevelOffsetOffset);
    writer.WriteFileOffset(topLevelOffset);

    return !writer.HasError() && writer.Close();
  }
Пример #25
0
  /**
   * Scans all areas. If an areas is of one of the given merge types, index all node ids
   * into the given NodeUseMap for all outer rings of the given area.
   */
  bool MergeAreasGenerator::ScanAreaNodeIds(Progress& progress,
                                            const TypeConfig& typeConfig,
                                            FileScanner& scanner,
                                            const TypeInfoSet& mergeTypes,
                                            std::unordered_set<Id>& nodeUseMap)
  {
    uint32_t areaCount=0;

    progress.SetAction("Scanning for nodes joining areas from '"+scanner.GetFilename()+"'");

    scanner.GotoBegin();

    scanner.Read(areaCount);

    uint8_t type;
    Id      id;
    Area    data;

    std::unordered_set<Id> usedOnceSet;

    for (uint32_t current=1; current<=areaCount; current++) {
      progress.SetProgress(current,areaCount);

      scanner.Read(type);
      scanner.Read(id);

      data.ReadImport(typeConfig,
                      scanner);

      if (!mergeTypes.IsSet(data.GetType())) {
        continue;
      }

      // We insert every node id only once per area, because we want to
      // find nodes that are shared by *different* areas.

      std::unordered_set<Id> nodeIds;

      for (const auto& ring: data.rings) {
        if (!ring.IsOuterRing()) {
          continue;
        }

        for (const auto node : ring.nodes) {
          Id id=node.GetId();

          if (nodeIds.find(id)==nodeIds.end()) {
            auto entry=usedOnceSet.find(id);

            if (entry!=usedOnceSet.end()) {
              nodeUseMap.insert(id);
            }
            else {
              usedOnceSet.insert(id);
            }
            nodeIds.insert(id);
          }
        }
      }
    }

    return true;
  }
Пример #26
0
  bool MergeAreasGenerator::Import(const TypeConfigRef& typeConfig,
                                   const ImportParameter& parameter,
                                   Progress& progress)
  {
    TypeInfoSet                    mergeTypes;
    FileScanner                    scanner;
    FileWriter                     writer;
    uint32_t                       areasWritten=0;

    for (const auto& type : typeConfig->GetTypes()) {
      if (type->CanBeArea() &&
          type->GetMergeAreas()) {
        mergeTypes.Set(type);
      }
    }

    std::unordered_set<Id> nodeUseMap;

    try {
      scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                   MergeAreaDataGenerator::AREAS_TMP),
                   FileScanner::Sequential,
                   parameter.GetRawWayDataMemoryMaped());

      if (!ScanAreaNodeIds(progress,
                           *typeConfig,
                           scanner,
                           mergeTypes,
                           nodeUseMap)) {
        return false;
      }

      uint32_t nodeCount=nodeUseMap.size();

      progress.Info("Found "+NumberToString(nodeCount)+" nodes as possible connection points for areas");

      /* ------ */

      writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                  AREAS2_TMP));

      writer.Write(areasWritten);

      while (true) {
        TypeInfoSet                loadedTypes;
        std::vector<AreaMergeData> mergeJob(typeConfig->GetTypeCount());

        //
        // Load type data
        //

        progress.SetAction("Collecting area data by type");

        if (!GetAreas(parameter,
                      progress,
                      *typeConfig,
                      mergeTypes,
                      loadedTypes,
                      nodeUseMap,
                      scanner,
                      writer,
                      mergeJob,
                      areasWritten)) {
          return false;
        }

        // Merge

        progress.SetAction("Merging areas");

        for (const auto& type : loadedTypes) {
          if (!mergeJob[type->GetIndex()].areas.empty()) {
            progress.Info("Merging areas of type "+type->GetName());
            MergeAreas(progress,
                       nodeUseMap,
                       mergeJob[type->GetIndex()]);
            progress.Info("Reduced areas of '"+type->GetName()+"' from "+NumberToString(mergeJob[type->GetIndex()].areaCount)+" to "+NumberToString(mergeJob[type->GetIndex()].areaCount-mergeJob[type->GetIndex()].mergedAway.size()));

            mergeJob[type->GetIndex()].areas.clear();
          }
        }

        // Store back merge result

        if (!loadedTypes.Empty()) {
          if (!WriteMergeResult(progress,
                                *typeConfig,
                                scanner,
                                writer,
                                loadedTypes,
                                mergeJob,
                                areasWritten)) {
            return false;
          }

          mergeTypes.Remove(loadedTypes);
        }


        if (mergeTypes.Empty()) {
          break;
        }
      }

      scanner.Close();

      writer.GotoBegin();
      writer.Write(areasWritten);
      writer.Close();
    }
    catch (IOException& e) {
      progress.Error(e.GetDescription());

      scanner.CloseFailsafe();
      writer.CloseFailsafe();

      return false;
    }

    return true;
  }
Пример #27
0
  bool MergeAreasGenerator::WriteMergeResult(Progress& progress,
                                             const TypeConfig& typeConfig,
                                             FileScanner& scanner,
                                             FileWriter& writer,
                                             const TypeInfoSet& loadedTypes,
                                             std::vector<AreaMergeData>& mergeJob,
                                             uint32_t& areasWritten)
  {
    uint32_t                               areaCount=0;
    std::unordered_map<FileOffset,AreaRef> merges;
    std::unordered_set<FileOffset>         ignores;

    for (const auto& type : loadedTypes) {
      for (const auto& area : mergeJob[type->GetIndex()].merges) {
        merges[area->GetFileOffset()]=area;
      }

      ignores.insert(mergeJob[type->GetIndex()].mergedAway.begin(),
                     mergeJob[type->GetIndex()].mergedAway.end());
    }

    scanner.GotoBegin();

    scanner.Read(areaCount);

    for (uint32_t a=1; a<=areaCount; a++) {
      uint8_t type;
      Id      id;
      AreaRef area=std::make_shared<Area>();

      progress.SetProgress(a,areaCount);

      scanner.Read(type);
      scanner.Read(id);

      area->ReadImport(typeConfig,
                       scanner);

      if (loadedTypes.IsSet(area->GetType())) {
        if (ignores.find(area->GetFileOffset())!=ignores.end()) {
          continue;
        }

        writer.Write(type);
        writer.Write(id);

        const auto& merge=merges.find(area->GetFileOffset()) ;

        if (merge!=merges.end()) {
          area=merge->second;
        }

        area->WriteImport(typeConfig,
                          writer);

        areasWritten++;
      }
    }

    return true;
  }
Пример #28
0
  /**
   * Load areas which has a one for the types given by types. If at leats one node
   * in one of the outer rings of the areas is marked in nodeUseMap as "used at least twice",
   * index it into the areas map.
   *
   * If the number of indexed areas is bigger than parameter.GetRawWayBlockSize() types are
   * dropped form areas until the number is again below the lmit.
   */
  bool MergeAreasGenerator::GetAreas(const ImportParameter& parameter,
                                     Progress& progress,
                                     const TypeConfig& typeConfig,
                                     const TypeInfoSet& candidateTypes,
                                     TypeInfoSet& loadedTypes,
                                     const std::unordered_set<Id>& nodeUseMap,
                                     FileScanner& scanner,
                                     FileWriter& writer,
                                     std::vector<AreaMergeData>& mergeJob,
                                     uint32_t& areasWritten)
  {
    bool        firstCall=areasWritten==0; // We are called for the first time
    uint32_t    areaCount=0;
    size_t      collectedAreasCount=0;
    size_t      typesWithAreas=0;

    for (auto& data : mergeJob) {
      data.areaCount=0;
    }

    loadedTypes=candidateTypes;

    scanner.GotoBegin();

    scanner.Read(areaCount);

    for (uint32_t a=1; a<=areaCount; a++) {
      uint8_t type;
      Id      id;
      AreaRef area=std::make_shared<Area>();

      progress.SetProgress(a,areaCount);

      scanner.Read(type);
      scanner.Read(id);

      area->ReadImport(typeConfig,
                       scanner);

      mergeJob[area->GetType()->GetIndex()].areaCount++;

      // This is an area of a type that does not get merged,
      // we directly store it in the target file.
      if (!loadedTypes.IsSet(area->GetType())) {
        if (firstCall) {
          writer.Write(type);
          writer.Write(id);

          area->WriteImport(typeConfig,
                            writer);

          areasWritten++;
        }

        continue;
      }

      bool isMergeCandidate=false;

      for (const auto& ring: area->rings) {
        if (!ring.IsOuterRing()) {
          continue;
        }

        for (const auto node : ring.nodes) {
          if (nodeUseMap.find(node.GetId())!=nodeUseMap.end()) {
            isMergeCandidate=true;
            break;
          }
        }

        if (isMergeCandidate) {
          break;
        }
      }

      if (!isMergeCandidate) {
        continue;
      }

      if (mergeJob[area->GetType()->GetIndex()].areas.empty()) {
        typesWithAreas++;
      }

      mergeJob[area->GetType()->GetIndex()].areas.push_back(area);

      collectedAreasCount++;

      while (collectedAreasCount>parameter.GetRawWayBlockSize() &&
             typesWithAreas>1) {
        TypeInfoRef victimType;

        // Find the type with the smallest amount of ways loaded
        for (auto &type : loadedTypes) {
          if (!mergeJob[type->GetIndex()].areas.empty() &&
              (!victimType ||
               mergeJob[type->GetIndex()].areas.size()<mergeJob[victimType->GetIndex()].areas.size())) {
            victimType=type;
          }
        }

        // If there is more then one type of way, we always must find a "victim" type.
        assert(victimType);

        // Correct the statistics
        collectedAreasCount-=mergeJob[victimType->GetIndex()].areas.size();
        // Clear already loaded data of th victim type
        mergeJob[victimType->GetIndex()].areas.clear();

        typesWithAreas--;
        loadedTypes.Remove(victimType);
      }
    }

    progress.SetAction("Collected "+NumberToString(collectedAreasCount)+" areas for "+NumberToString(loadedTypes.Size())+" types");

    return true;
  }
Пример #29
0
  bool NodeDataGenerator::Import(const TypeConfigRef& typeConfig,
                                 const ImportParameter& parameter,
                                 Progress& progress)
  {
    uint32_t rawNodeCount=0;
    uint32_t nodesReadCount=0;
    uint32_t nodesWrittenCount=0;

    //
    // Iterator over all raw nodes, hcekc they type, and convert them from raw nodes
    // to nodes if the type is interesting (!=typeIgnore).
    //
    // Count the bounding box by the way...
    //

    progress.SetAction("Generating nodes.tmp");

    FileScanner scanner;
    FileWriter  writer;

    if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                      "rawnodes.dat"),
                      FileScanner::Sequential,
                      parameter.GetRawNodeDataMemoryMaped())) {
      progress.Error("Cannot open 'rawnodes.dat'");
      return false;
    }

    if (!scanner.Read(rawNodeCount)) {
      progress.Error("Error while reading number of data entries in file");
      return false;
    }

    if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                     "nodes.tmp"))) {
      progress.Error("Cannot create 'nodes.tmp'");
      return false;
    }

    writer.Write(nodesWrittenCount);

    for (uint32_t n=1; n<=rawNodeCount; n++) {
      progress.SetProgress(n,rawNodeCount);

      RawNode rawNode;
      Node    node;

      if (!rawNode.Read(typeConfig,
                        scanner)) {
        progress.Error(std::string("Error while reading data entry ")+
                       NumberToString(n)+" of "+
                       NumberToString(rawNodeCount)+
                       " in file '"+
                       scanner.GetFilename()+"'");
        return false;
      }

      nodesReadCount++;

      if (!rawNode.GetType()->GetIgnore()) {
        node.SetFeatures(rawNode.GetFeatureValueBuffer());
        node.SetCoords(rawNode.GetCoords());

        FileOffset fileOffset;

        if (!writer.GetPos(fileOffset)) {
          progress.Error(std::string("Error while reading current fileOffset in file '")+
                         writer.GetFilename()+"'");
          return false;
        }

        writer.Write(rawNode.GetId());
        node.Write(typeConfig,
                   writer);

        nodesWrittenCount++;
      }
    }

    if (!scanner.Close()) {
      return false;
    }

    writer.SetPos(0);
    writer.Write(nodesWrittenCount);

    if (!writer.Close()) {
      return false;
    }

    progress.Info(std::string("Read "+NumberToString(nodesReadCount)+" nodes, wrote "+NumberToString(nodesWrittenCount)+" nodes"));

    return true;
  }
Пример #30
0
  bool OptimizeAreaWayIdsGenerator::ScanAreaIds(const ImportParameter& parameter,
                                                Progress& progress,
                                                const TypeConfig& typeConfig,
                                                NodeUseMap& nodeUseMap)
  {
    FileScanner scanner;
    uint32_t    dataCount=0;

    progress.SetAction("Scanning ids from 'areas2.tmp'");

    if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                      "areas2.tmp"),
                      FileScanner::Sequential,
                      parameter.GetAreaDataMemoryMaped())) {
      progress.Error(std::string("Cannot open '")+scanner.GetFilename()+"'");
      return false;
    }

    if (!scanner.Read(dataCount)) {
      progress.Error("Error while reading number of data entries in file");
      return false;
    }

    for (uint32_t current=1; current<=dataCount; current++) {
      uint8_t type;
      Id      id;
      Area    data;

      progress.SetProgress(current,dataCount);

      if (!scanner.Read(type) ||
          !scanner.Read(id) ||
          !data.ReadImport(typeConfig,
                           scanner)) {
        progress.Error(std::string("Error while reading data entry ")+
                       NumberToString(current)+" of "+
                       NumberToString(dataCount)+
                       " in file '"+
                       scanner.GetFilename()+"'");

        return false;
      }

      for (const auto& ring: data.rings) {
        std::unordered_set<Id> nodeIds;

        if (!ring.GetType()->CanRoute()) {
          continue;
        }

        for (const auto id: ring.ids) {
          if (nodeIds.find(id)==nodeIds.end()) {
            nodeUseMap.SetNodeUsed(id);

            nodeIds.insert(id);
          }
        }
      }
    }

    if (!scanner.Close()) {
      progress.Error(std::string("Error while closing file '")+
                     scanner.GetFilename()+"'");
      return false;
    }

    return true;
  }