bool OptimizeAreasLowZoomGenerator::Import(const TypeConfigRef& typeConfig,
                                             const ImportParameter& parameter,
                                             Progress& progress)
  {
    FileOffset          indexOffset=0;
    FileWriter          writer;
    Magnification       magnification; // Magnification, we optimize for
    TypeInfoSet         areaTypes;     // Types we optimize
    std::list<TypeData> areaTypesData;

    GetAreaTypesToOptimize(*typeConfig,
                           areaTypes);

    try {
      writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                  OptimizeAreasLowZoom::FILE_AREASOPT_DAT));

      //
      // Write header
      //

      writer.WriteFileOffset(indexOffset);

      if (!HandleAreas(parameter,
                       progress,
                       *typeConfig,
                       writer,
                       areaTypes,
                       areaTypesData)) {
        progress.Error("Error while optimizing areas");
        return false;
      }

      // Position of the index
      indexOffset=writer.GetPos();

      if (!WriteHeader(writer,
                       areaTypesData,
                       (uint32_t)parameter.GetOptimizationMaxMag())) {
        progress.Error("Cannot write file header");
        return false;
      }

      writer.GotoBegin();
      writer.WriteFileOffset(indexOffset);

      writer.Close();
    }
    catch (IOException& e) {
      progress.Error(e.GetDescription());

      writer.CloseFailsafe();

      return false;
    }

    return true;
  }
Beispiel #2
0
  bool Preprocess::Initialize(const ImportParameter& parameter)
  {
    coordPageCount=0;
    currentPageId=std::numeric_limits<PageId>::max();

    nodeCount=0;
    wayCount=0;
    areaCount=0;
    relationCount=0;
    coastlineCount=0;

    lastNodeId=std::numeric_limits<OSMId>::min();
    lastWayId=std::numeric_limits<OSMId>::min();
    lastRelationId=std::numeric_limits<OSMId>::min();

    nodeSortingError=false;
    waySortingError=false;
    relationSortingError=false;

    nodeWriter.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                    "rawnodes.dat"));
    nodeWriter.Write(nodeCount);

    wayWriter.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                   "rawways.dat"));
    wayWriter.Write(wayCount+areaCount);

    relationWriter.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                        "rawrels.dat"));
    relationWriter.Write(relationCount);

    coastlineWriter.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                        "rawcoastline.dat"));
    coastlineWriter.Write(coastlineCount);

    coordWriter.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                     "coord.dat"));

    FileOffset offset=0;

    coordWriter.Write(coordPageSize);
    coordWriter.Write(offset);
    coordWriter.FlushCurrentBlockWithZeros(coordPageSize*2*sizeof(uint32_t));

    coordPageCount++;

    lats.resize(coordPageSize);
    lons.resize(coordPageSize);
    isSet.resize(coordPageSize);

    return !nodeWriter.HasError() &&
           !wayWriter.HasError() &&
           !relationWriter.HasError() &&
           !coastlineWriter.HasError() &&
           !coordWriter.HasError();
  }
  bool OptimizeAreasLowZoomGenerator::GetAreas(const TypeConfig& typeConfig,
                                               const ImportParameter& parameter,
                                               Progress& progress,
                                               FileScanner& scanner,
                                               const TypeInfoSet& types,
                                               std::vector<std::list<AreaRef> >& areas,
                                               TypeInfoSet& loadedTypes)
  {
    uint32_t    areaCount=0;
    size_t      collectedAreasCount=0;

    loadedTypes=types;

    progress.SetAction("Collecting area data to optimize");

    scanner.GotoBegin();

    scanner.Read(areaCount);

    for (uint32_t a=1; a<=areaCount; a++) {
      AreaRef area=std::make_shared<Area>();

      progress.SetProgress(a,areaCount);

      area->Read(typeConfig,
                 scanner);

      if (loadedTypes.IsSet(area->GetType())) {
        areas[area->GetType()->GetIndex()].push_back(area);

        collectedAreasCount++;

        while (collectedAreasCount>parameter.GetOptimizationMaxWayCount() &&
               loadedTypes.Size()>1) {
          TypeInfoRef victimType;

          for (auto &type : loadedTypes) {
            if (areas[type->GetIndex()].size()>0 &&
                (!victimType ||
                 areas[type->GetIndex()].size()<areas[victimType->GetIndex()].size())) {
              victimType=type;
            }
          }

          assert(victimType);

          collectedAreasCount-=areas[victimType->GetIndex()].size();
          areas[victimType->GetIndex()].clear();
          loadedTypes.Remove(victimType);
        }
      }
    }

    progress.Info("Collected "+NumberToString(collectedAreasCount)+" areas for "+NumberToString(loadedTypes.Size())+" types");

    return !scanner.HasError();
  }
Beispiel #4
0
  static bool ExecuteModules(std::list<ImportModule*>& modules,
                            const ImportParameter& parameter,
                            Progress& progress,
                            const TypeConfigRef& typeConfig)
  {
    StopClock overAllTimer;
    size_t    currentStep=1;

    for (const auto& module : modules) {
      if (currentStep>=parameter.GetStartStep() &&
          currentStep<=parameter.GetEndStep()) {
        StopClock timer;
        bool      success;

        progress.SetStep(std::string("Step #")+
                         NumberToString(currentStep)+
                         " - "+
                         module->GetDescription());

        success=module->Import(typeConfig,
                               parameter,
                               progress);

        timer.Stop();

        progress.Info(std::string("=> ")+timer.ResultString()+" second(s)");

        if (!success) {
          progress.Error(std::string("Error while executing step '")+module->GetDescription()+"'!");
          return false;
        }
      }

      currentStep++;
    }

    overAllTimer.Stop();
    progress.Info(std::string("=> ")+overAllTimer.ResultString()+" second(s)");

    return true;
  }
Beispiel #5
0
  bool AreaWayIndexGenerator::FitsIndexCriteria(const ImportParameter& parameter,
                                                Progress& progress,
                                                const TypeInfo& typeInfo,
                                                const TypeData& typeData,
                                                const CoordCountMap& cellFillCount)
  {
    if (typeData.indexCells==0) {
      return true;
    }

    size_t entryCount=0;
    size_t max=0;

    for (CoordCountMap::const_iterator cell=cellFillCount.begin();
         cell!=cellFillCount.end();
         ++cell) {
      entryCount+=cell->second;
      max=std::max(max,cell->second);
    }

    // Average number of entries per tile cell
    double average=entryCount*1.0/cellFillCount.size();

    // If the fill rate of the index is too low, we use this index level anyway
    if (typeData.indexCells/(1.0*typeData.cellXCount*typeData.cellYCount)<=
        parameter.GetAreaWayIndexMinFillRate()) {
      progress.Warning(typeInfo.GetName()+" ("+NumberToString(typeInfo.GetId())+") is not well distributed");
      return true;
    }

    // If average fill size and max fill size for tile cells
    // is within limits, store it now.
    if (max<=parameter.GetAreaWayIndexCellSizeMax() &&
        average<=parameter.GetAreaWayIndexCellSizeAverage()) {
      return true;
    }

    return false;
  }
Beispiel #6
0
  bool Preprocess::ProcessFiles(const TypeConfigRef& typeConfig,
                                const ImportParameter& parameter,
                                Progress& progress,
                                Callback& callback)
  {
    for (const auto& filename : parameter.GetMapfiles()) {
      if (filename.length()>=4 &&
          filename.substr(filename.length()-4)==".osm")  {

#if defined(HAVE_LIB_XML)
        PreprocessOSM preprocess(callback);

        if (!preprocess.Import(typeConfig,
                               parameter,
                               progress,
                               filename)) {
          return false;
        }
#else
        progress.Error("Support for the OSM file format is not enabled!");
        return false;
#endif
      }
      else if (filename.length()>=4 &&
            filename.substr(filename.length()-4)==".pbf") {

#if defined(HAVE_LIB_PROTOBUF)
        PreprocessPBF preprocess(callback);

        if (!preprocess.Import(typeConfig,
                               parameter,
                               progress,
                               filename)) {
          return false;
        }
#else
        progress.Error("Support for the PBF file format is not enabled!");
        return false;
#endif
      }
      else {
        progress.Error("Sorry, this file type is not yet supported!");
        return false;
      }
    }

    return true;
  }
  bool AreaAreaIndexGenerator::WriteIndexLevel(const ImportParameter& parameter,
                                               FileWriter& writer,
                                               int level,
                                               std::map<Pixel,AreaLeaf>& leafs)
  {
    for (std::map<Pixel,AreaLeaf>::iterator leaf=leafs.begin();
         leaf!=leafs.end();
         ++leaf) {
      writer.GetPos(leaf->second.offset);

      assert(!leaf->second.areas.empty() ||
             leaf->second.children[0]!=0 ||
             leaf->second.children[1]!=0 ||
             leaf->second.children[2]!=0 ||
             leaf->second.children[3]!=0);

      if (level<(int)parameter.GetAreaAreaIndexMaxMag()) {
        for (size_t c=0; c<4; c++) {
          if (leaf->second.children[c]==0) {
            writer.WriteNumber(0);
          }
          else {
            writer.WriteNumber(leaf->second.children[c]);
          }
        }
      }

      FileOffset lastOffset;

      // Areas
      writer.WriteNumber((uint32_t)leaf->second.areas.size());

      lastOffset=0;
      for (std::list<Entry>::const_iterator entry=leaf->second.areas.begin();
           entry!=leaf->second.areas.end();
           entry++) {
        writer.WriteNumber(entry->type);
        // Since objects are inserted in file position order, we do not need
        // to sort objects by file offset at this place
        writer.WriteNumber(entry->offset-lastOffset);

        lastOffset=entry->offset;
      }
    }

    return !writer.HasError();
  }
Beispiel #8
0
  bool WayLocationProcessorFilter::BeforeProcessingStart(const ImportParameter& parameter,
                                                         Progress& progress,
                                                         const TypeConfig& typeConfig)
  {
    overallDataCount=0;

    nameReader=new NameFeatureValueReader(typeConfig);
    locationReader=new LocationFeatureValueReader(typeConfig);

    try {
      writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                  SortWayDataGenerator::WAYADDRESS_DAT));

      writer.Write(overallDataCount);
    }
    catch (IOException& e) {
      progress.Error(e.GetDescription());

      return false;
    }

    return true;
  }
Beispiel #9
0
  bool Preprocess::Import(const ImportParameter& parameter,
                          Progress& progress,
                          const TypeConfig& typeConfig)
  {
    if (parameter.GetMapfile().length()>=4 &&
        parameter.GetMapfile().substr(parameter.GetMapfile().length()-4)==".osm")  {

#if defined(HAVE_LIB_XML)
      PreprocessOSM preprocess;

      return preprocess.Import(parameter,
                               progress,
                               typeConfig);
#else
      progress.Error("Support for the OSM file format is not enabled!");
#endif
    }

    if (parameter.GetMapfile().length()>=4 &&
             parameter.GetMapfile().substr(parameter.GetMapfile().length()-4)==".pbf") {

#if defined(HAVE_LIB_PROTOBUF)
      PreprocessPBF preprocess;

      return preprocess.Import(parameter,
                               progress,
                               typeConfig);
#else
      progress.Error("Support for the PBF file format is not enabled!");
      return false;
#endif
    }

    progress.Error("Sorry, this file type is not yet supported!");
    return false;
  }
  bool OptimizeAreasLowZoomGenerator::HandleAreas(const ImportParameter& parameter,
                                                  Progress& progress,
                                                  const TypeConfig& typeConfig,
                                                  FileWriter& writer,
                                                  const TypeInfoSet& types,
                                                  std::list<TypeData>& typesData)
  {
    FileScanner scanner;
    // Everything smaller than 2mm should get dropped. Width, height and DPI come from the Nexus 4
    double dpi=320.0;
    double pixel=2.0/* mm */ * dpi / 25.4 /* inch */;

    progress.Info("Minimum visible size in pixel: "+NumberToString((unsigned long)pixel));

    try {
      scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                   AreaDataFile::AREAS_DAT),
                   FileScanner::Sequential,
                   parameter.GetWayDataMemoryMaped());

      TypeInfoSet                      typesToProcess(types);
      std::vector<std::list<AreaRef> > allAreas(typeConfig.GetTypeCount());

      while (true) {
        //
        // Load type data
        //

        TypeInfoSet loadedTypes;

        if (!GetAreas(typeConfig,
                      parameter,
                      progress,
                      scanner,
                      typesToProcess,
                      allAreas,
                      loadedTypes)) {
          return false;
        }

        typesToProcess.Remove(loadedTypes);

        for (const auto& type : loadedTypes) {
          progress.SetAction("Optimizing type "+ type->GetName());

          for (uint32_t level=parameter.GetOptimizationMinMag();
               level<=parameter.GetOptimizationMaxMag();
               level++) {
            Magnification      magnification; // Magnification, we optimize for
            std::list<AreaRef> optimizedAreas;

            magnification.SetLevel(level);

            OptimizeAreas(allAreas[type->GetIndex()],
                          optimizedAreas,
                          1280,768,
                          dpi,
                          pixel,
                          magnification,
                          parameter.GetOptimizationWayMethod());

            if (optimizedAreas.empty()) {
              progress.Debug("Empty optimization result for level "+NumberToString(level)+", no index generated");

              TypeData typeData;

              typeData.type=type;
              typeData.optLevel=level;

              typesData.push_back(typeData);

              continue;
            }

            progress.Info("Optimized from "+NumberToString(allAreas[type->GetIndex()].size())+" to "+NumberToString(optimizedAreas.size())+" areas");

            /*
            size_t optAreas=optimizedAreas.size();
            size_t optRoles=0;
            size_t optNodes=0;

            for (std::list<AreaRef>::const_iterator a=optimizedAreas.begin();
                a!=optimizedAreas.end();
                ++a) {
              AreaRef area=*a;

              optRoles+=area->rings.size();

              for (size_t r=0; r<area->rings.size(); r++) {
                optNodes+=area->rings[r].nodes.size();
              }
            }*/

            /*
            std::cout << "Areas: " << origAreas << " => " << optAreas << std::endl;
            std::cout << "Roles: " << origRoles << " => " << optRoles << std::endl;
            std::cout << "Nodes: " << origNodes << " => " << optNodes << std::endl;*/

            TypeData typeData;

            typeData.type=type;
            typeData.optLevel=level;

            GetAreaIndexLevel(parameter,
                              optimizedAreas,
                              typeData);

            //std::cout << "Resulting index level: " << typeData.indexLevel << ", " << typeData.indexCells << ", " << typeData.indexEntries << std::endl;

            FileOffsetFileOffsetMap offsets;

            WriteAreas(typeConfig,
                       writer,
                       optimizedAreas,
                       offsets);

            if (!WriteAreaBitmap(progress,
                                 writer,
                                 optimizedAreas,
                                 offsets,
                                 typeData)) {
              return false;
            }

            typesData.push_back(typeData);
          }

          allAreas[type->GetIndex()].clear();
        }

        if (typesToProcess.Empty()) {
          break;
        }
      }

      scanner.Close();
    }
    catch (IOException& e) {
      progress.Error(e.GetDescription());
      return false;
    }

    return true;
  }
  bool AreaAreaIndexGenerator::Import(const TypeConfigRef& typeConfig,
                                      const ImportParameter& parameter,
                                      Progress& progress)
  {
    FileScanner               scanner;
    size_t                    areas=0;         // Number of areas found
    size_t                    areasConsumed=0; // Number of areas consumed
    std::vector<double>       cellWidth;
    std::vector<double>       cellHeight;
    std::map<Pixel,AreaLeaf>  leafs;
    std::map<Pixel,AreaLeaf>  newAreaLeafs;

    cellWidth.resize(parameter.GetAreaAreaIndexMaxMag()+1);
    cellHeight.resize(parameter.GetAreaAreaIndexMaxMag()+1);

    for (size_t i=0; i<cellWidth.size(); i++) {
      cellWidth[i]=360.0/pow(2.0,(int)i);
    }

    for (size_t i=0; i<cellHeight.size(); i++) {
      cellHeight[i]=180.0/pow(2.0,(int)i);
    }

    //
    // Writing index file
    //

    progress.SetAction("Generating 'areaarea.idx'");

    FileWriter writer;
    FileOffset topLevelOffset=0;
    FileOffset topLevelOffsetOffset; // Offset of the toplevel entry

    if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                     "areaarea.idx"))) {
      progress.Error("Cannot create 'areaarea.idx'");
      return false;
    }

    if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                         "areas.dat"),
                         FileScanner::Sequential,
                         parameter.GetWayDataMemoryMaped())) {
      progress.Error("Cannot open 'areas.dat'");
      return false;
    }

    writer.WriteNumber((uint32_t)parameter.GetAreaAreaIndexMaxMag()); // MaxMag

    if (!writer.GetPos(topLevelOffsetOffset)) {
      progress.Error("Cannot read current file position");
      return false;
    }

    if (!writer.WriteFileOffset(topLevelOffset)) {
      progress.Error("Cannot write top level entry offset");
      return false;
    }

    int l=parameter.GetAreaAreaIndexMaxMag();

    while (l>=0) {
      size_t areaLevelEntries=0;

      progress.Info(std::string("Storing level ")+NumberToString(l)+"...");

      newAreaLeafs.clear();

      SetOffsetOfChildren(leafs,newAreaLeafs);

      leafs=newAreaLeafs;

      // Areas

      if (areas==0 ||
          (areas>0 && areas>areasConsumed)) {
        uint32_t areaCount=0;

        progress.Info(std::string("Scanning areas.dat for areas of index level ")+NumberToString(l)+"...");

        if (!scanner.GotoBegin()) {
          progress.Error("Cannot go to begin of way file");
        }

        if (!scanner.Read(areaCount)) {
          progress.Error("Error while reading number of data entries in file");
          return false;
        }

        areas=0;
        for (uint32_t a=1; a<=areaCount; a++) {
          progress.SetProgress(a,areaCount);

          FileOffset offset;
          Area       area;

          scanner.GetPos(offset);

          if (!area.Read(typeConfig,
                         scanner)) {
            progress.Error(std::string("Error while reading data entry ")+
                           NumberToString(a)+" of "+
                           NumberToString(areaCount)+
                           " in file '"+
                           scanner.GetFilename()+"'");
            return false;
          }

          areas++;

          double minLon;
          double maxLon;
          double minLat;
          double maxLat;

          area.GetBoundingBox(minLon,maxLon,minLat,maxLat);

          //
          // Calculate highest level where the bounding box completely
          // fits in the cell size and assign area to the tiles that
          // hold the geometric center of the tile.
          //

          int level=parameter.GetAreaAreaIndexMaxMag();
          while (level>=0) {
            if (maxLon-minLon<=cellWidth[level] &&
                maxLat-minLat<=cellHeight[level]) {
              break;
            }

            level--;
          }

          if (level==l) {
            //
            // Renormated coordinate space (everything is >=0)
            //

            minLon+=180;
            maxLon+=180;
            minLat+=90;
            maxLat+=90;

            //
            // Calculate minimum and maximum tile ids that are covered
            // by the area
            //
            uint32_t minyc=(uint32_t)floor(minLat/cellHeight[level]);
            uint32_t maxyc=(uint32_t)ceil(maxLat/cellHeight[level]);
            uint32_t minxc=(uint32_t)floor(minLon/cellWidth[level]);
            uint32_t maxxc=(uint32_t)ceil(maxLon/cellWidth[level]);

            Entry entry;

            entry.type=area.GetType()->GetId();
            entry.offset=offset;

            // Add this area to the tile where the center of the area lies in.
            leafs[Pixel((minxc+maxxc)/2,(minyc+maxyc)/2)].areas.push_back(entry);
            areaLevelEntries++;

            areasConsumed++;
          }
        }
      }

      progress.Debug(std::string("Writing ")+NumberToString(leafs.size())+" leafs ("+
                     NumberToString(areaLevelEntries)+") "+
                     "to index of level "+NumberToString(l)+"...");

      // Remember the offset of one cell in level '0'
      if (l==0) {
        if (!writer.GetPos(topLevelOffset)) {
          progress.Error("Cannot read top level entry offset");
          return false;
        }
      }

      /*
      uint32_t minX=std::numeric_limits<uint32_t>::max();
      uint32_t minY=std::numeric_limits<uint32_t>::max();
      uint32_t maxX=std::numeric_limits<uint32_t>::min();
      uint32_t maxY=std::numeric_limits<uint32_t>::min();

      std::map<TypeId,size_t> useMap;

      for (std::map<Pixel,AreaLeaf>::const_iterator leaf=leafs.begin();
           leaf!=leafs.end();
           ++leaf) {
        minX=std::min(minX,leaf->first.x);
        maxX=std::max(maxX,leaf->first.x);
        minY=std::min(minY,leaf->first.y);
        maxY=std::max(maxY,leaf->first.y);

        for (std::list<Entry>::const_iterator entry=leaf->second.areas.begin();
             entry!=leaf->second.areas.end();
             entry++) {
          std::map<TypeId,size_t>::iterator u=useMap.find(entry->type);

          if (u==useMap.end()) {
            useMap[entry->type]=1;
          }
          else {
            u->second++;
          }
        }
      }*/

      /*
      std::cout << "[" << minX << "-" << maxX << "]x[" << minY << "-" << maxY << "] => " << leafs.size() << "/" << (maxX-minX+1)*(maxY-minY+1) << " " << (int)BytesNeededToAddressFileData(leafs.size()) << " " << ByteSizeToString(BytesNeededToAddressFileData(leafs.size())*(maxX-minX+1)*(maxY-minY+1)) << std::endl;

      for (std::map<TypeId,size_t>::const_iterator u=useMap.begin();
          u!=useMap.end();
          ++u) {
        std::cout << "* " << u->first << " " << typeConfig.GetTypeInfo(u->first).GetName() << " " << u->second << std::endl;
      }*/

      if (!WriteIndexLevel(parameter,
                           writer,
                           (int)l,
                           leafs)) {
        return false;
      }

      l--;
    }

    writer.SetPos(topLevelOffsetOffset);
    writer.WriteFileOffset(topLevelOffset);

    return !writer.HasError() && writer.Close();
  }
Beispiel #12
0
  bool MergeAreasGenerator::Import(const TypeConfigRef& typeConfig,
                                   const ImportParameter& parameter,
                                   Progress& progress)
  {
    TypeInfoSet                    mergeTypes;
    FileScanner                    scanner;
    FileWriter                     writer;
    uint32_t                       areasWritten=0;

    for (const auto& type : typeConfig->GetTypes()) {
      if (type->CanBeArea() &&
          type->GetMergeAreas()) {
        mergeTypes.Set(type);
      }
    }

    std::unordered_set<Id> nodeUseMap;

    try {
      scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                   MergeAreaDataGenerator::AREAS_TMP),
                   FileScanner::Sequential,
                   parameter.GetRawWayDataMemoryMaped());

      if (!ScanAreaNodeIds(progress,
                           *typeConfig,
                           scanner,
                           mergeTypes,
                           nodeUseMap)) {
        return false;
      }

      uint32_t nodeCount=nodeUseMap.size();

      progress.Info("Found "+NumberToString(nodeCount)+" nodes as possible connection points for areas");

      /* ------ */

      writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                  AREAS2_TMP));

      writer.Write(areasWritten);

      while (true) {
        TypeInfoSet                loadedTypes;
        std::vector<AreaMergeData> mergeJob(typeConfig->GetTypeCount());

        //
        // Load type data
        //

        progress.SetAction("Collecting area data by type");

        if (!GetAreas(parameter,
                      progress,
                      *typeConfig,
                      mergeTypes,
                      loadedTypes,
                      nodeUseMap,
                      scanner,
                      writer,
                      mergeJob,
                      areasWritten)) {
          return false;
        }

        // Merge

        progress.SetAction("Merging areas");

        for (const auto& type : loadedTypes) {
          if (!mergeJob[type->GetIndex()].areas.empty()) {
            progress.Info("Merging areas of type "+type->GetName());
            MergeAreas(progress,
                       nodeUseMap,
                       mergeJob[type->GetIndex()]);
            progress.Info("Reduced areas of '"+type->GetName()+"' from "+NumberToString(mergeJob[type->GetIndex()].areaCount)+" to "+NumberToString(mergeJob[type->GetIndex()].areaCount-mergeJob[type->GetIndex()].mergedAway.size()));

            mergeJob[type->GetIndex()].areas.clear();
          }
        }

        // Store back merge result

        if (!loadedTypes.Empty()) {
          if (!WriteMergeResult(progress,
                                *typeConfig,
                                scanner,
                                writer,
                                loadedTypes,
                                mergeJob,
                                areasWritten)) {
            return false;
          }

          mergeTypes.Remove(loadedTypes);
        }


        if (mergeTypes.Empty()) {
          break;
        }
      }

      scanner.Close();

      writer.GotoBegin();
      writer.Write(areasWritten);
      writer.Close();
    }
    catch (IOException& e) {
      progress.Error(e.GetDescription());

      scanner.CloseFailsafe();
      writer.CloseFailsafe();

      return false;
    }

    return true;
  }
Beispiel #13
0
  /**
   * Load areas which has a one for the types given by types. If at leats one node
   * in one of the outer rings of the areas is marked in nodeUseMap as "used at least twice",
   * index it into the areas map.
   *
   * If the number of indexed areas is bigger than parameter.GetRawWayBlockSize() types are
   * dropped form areas until the number is again below the lmit.
   */
  bool MergeAreasGenerator::GetAreas(const ImportParameter& parameter,
                                     Progress& progress,
                                     const TypeConfig& typeConfig,
                                     const TypeInfoSet& candidateTypes,
                                     TypeInfoSet& loadedTypes,
                                     const std::unordered_set<Id>& nodeUseMap,
                                     FileScanner& scanner,
                                     FileWriter& writer,
                                     std::vector<AreaMergeData>& mergeJob,
                                     uint32_t& areasWritten)
  {
    bool        firstCall=areasWritten==0; // We are called for the first time
    uint32_t    areaCount=0;
    size_t      collectedAreasCount=0;
    size_t      typesWithAreas=0;

    for (auto& data : mergeJob) {
      data.areaCount=0;
    }

    loadedTypes=candidateTypes;

    scanner.GotoBegin();

    scanner.Read(areaCount);

    for (uint32_t a=1; a<=areaCount; a++) {
      uint8_t type;
      Id      id;
      AreaRef area=std::make_shared<Area>();

      progress.SetProgress(a,areaCount);

      scanner.Read(type);
      scanner.Read(id);

      area->ReadImport(typeConfig,
                       scanner);

      mergeJob[area->GetType()->GetIndex()].areaCount++;

      // This is an area of a type that does not get merged,
      // we directly store it in the target file.
      if (!loadedTypes.IsSet(area->GetType())) {
        if (firstCall) {
          writer.Write(type);
          writer.Write(id);

          area->WriteImport(typeConfig,
                            writer);

          areasWritten++;
        }

        continue;
      }

      bool isMergeCandidate=false;

      for (const auto& ring: area->rings) {
        if (!ring.IsOuterRing()) {
          continue;
        }

        for (const auto node : ring.nodes) {
          if (nodeUseMap.find(node.GetId())!=nodeUseMap.end()) {
            isMergeCandidate=true;
            break;
          }
        }

        if (isMergeCandidate) {
          break;
        }
      }

      if (!isMergeCandidate) {
        continue;
      }

      if (mergeJob[area->GetType()->GetIndex()].areas.empty()) {
        typesWithAreas++;
      }

      mergeJob[area->GetType()->GetIndex()].areas.push_back(area);

      collectedAreasCount++;

      while (collectedAreasCount>parameter.GetRawWayBlockSize() &&
             typesWithAreas>1) {
        TypeInfoRef victimType;

        // Find the type with the smallest amount of ways loaded
        for (auto &type : loadedTypes) {
          if (!mergeJob[type->GetIndex()].areas.empty() &&
              (!victimType ||
               mergeJob[type->GetIndex()].areas.size()<mergeJob[victimType->GetIndex()].areas.size())) {
            victimType=type;
          }
        }

        // If there is more then one type of way, we always must find a "victim" type.
        assert(victimType);

        // Correct the statistics
        collectedAreasCount-=mergeJob[victimType->GetIndex()].areas.size();
        // Clear already loaded data of th victim type
        mergeJob[victimType->GetIndex()].areas.clear();

        typesWithAreas--;
        loadedTypes.Remove(victimType);
      }
    }

    progress.SetAction("Collected "+NumberToString(collectedAreasCount)+" areas for "+NumberToString(loadedTypes.Size())+" types");

    return true;
  }
  bool OptimizeAreaWayIdsGenerator::ScanAreaIds(const ImportParameter& parameter,
                                                Progress& progress,
                                                const TypeConfig& typeConfig,
                                                NodeUseMap& nodeUseMap)
  {
    FileScanner scanner;
    uint32_t    dataCount=0;

    progress.SetAction("Scanning ids from 'areas2.tmp'");

    if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                      "areas2.tmp"),
                      FileScanner::Sequential,
                      parameter.GetAreaDataMemoryMaped())) {
      progress.Error(std::string("Cannot open '")+scanner.GetFilename()+"'");
      return false;
    }

    if (!scanner.Read(dataCount)) {
      progress.Error("Error while reading number of data entries in file");
      return false;
    }

    for (uint32_t current=1; current<=dataCount; current++) {
      uint8_t type;
      Id      id;
      Area    data;

      progress.SetProgress(current,dataCount);

      if (!scanner.Read(type) ||
          !scanner.Read(id) ||
          !data.ReadImport(typeConfig,
                           scanner)) {
        progress.Error(std::string("Error while reading data entry ")+
                       NumberToString(current)+" of "+
                       NumberToString(dataCount)+
                       " in file '"+
                       scanner.GetFilename()+"'");

        return false;
      }

      for (const auto& ring: data.rings) {
        std::unordered_set<Id> nodeIds;

        if (!ring.GetType()->CanRoute()) {
          continue;
        }

        for (const auto id: ring.ids) {
          if (nodeIds.find(id)==nodeIds.end()) {
            nodeUseMap.SetNodeUsed(id);

            nodeIds.insert(id);
          }
        }
      }
    }

    if (!scanner.Close()) {
      progress.Error(std::string("Error while closing file '")+
                     scanner.GetFilename()+"'");
      return false;
    }

    return true;
  }
  bool OptimizeAreaWayIdsGenerator::CopyWays(const ImportParameter& parameter,
                                             Progress& progress,
                                             const TypeConfig& typeConfig,
                                             NodeUseMap& nodeUseMap)
  {
    FileScanner scanner;
    FileWriter  writer;
    uint32_t    dataCount=0;

    progress.SetAction("Copy data from 'wayway.tmp' to 'ways.tmp'");

    if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                      "wayway.tmp"),
                                      FileScanner::Sequential,
                                      parameter.GetWayDataMemoryMaped())) {
      progress.Error(std::string("Cannot open '")+scanner.GetFilename()+"'");
      return false;
    }

    if (!scanner.Read(dataCount)) {
      progress.Error("Error while reading number of data entries in file");
      return false;
    }

    if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                     "ways.tmp"))) {
      progress.Error(std::string("Cannot create '")+writer.GetFilename()+"'");
      return false;
    }

    writer.Write(dataCount);

    for (uint32_t current=1; current<=dataCount; current++) {
      uint8_t type;
      Id      id;
      Way     data;

      progress.SetProgress(current,dataCount);

      if (!scanner.Read(type) ||
          !scanner.Read(id) ||
          !data.Read(typeConfig,
                     scanner)) {
        progress.Error(std::string("Error while reading data entry ")+
                       NumberToString(current)+" of "+
                       NumberToString(dataCount)+
                       " in file '"+
                       scanner.GetFilename()+"'");

        return false;
      }

      for (auto& id : data.ids) {
        if (!nodeUseMap.IsNodeUsedAtLeastTwice(id)) {
          id=0;
        }
      }

      if (!writer.Write(type) ||
          !writer.Write(id) ||
          !data.Write(typeConfig,
                      writer)) {
        progress.Error(std::string("Error while writing data entry to file '")+
                       writer.GetFilename()+"'");

        return false;
      }
    }

    if (!scanner.Close()) {
      progress.Error(std::string("Error while closing file '")+
                     scanner.GetFilename()+"'");
      return false;
    }

    if (!writer.Close()) {
      progress.Error(std::string("Error while closing file '")+
                     writer.GetFilename()+"'");
      return false;
    }

    return true;
  }
  bool OptimizeAreaWayIdsGenerator::ScanWayIds(const ImportParameter& parameter,
                                               Progress& progress,
                                               const TypeConfig& typeConfig,
                                               NodeUseMap& nodeUseMap)
  {
    FileScanner scanner;
    uint32_t    dataCount=0;

    progress.SetAction("Scanning ids from 'wayway.tmp'");

    if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                      "wayway.tmp"),
                                      FileScanner::Sequential,
                                      parameter.GetWayDataMemoryMaped())) {
      progress.Error(std::string("Cannot open '")+scanner.GetFilename()+"'");
      return false;
    }

    if (!scanner.Read(dataCount)) {
      progress.Error("Error while reading number of data entries in file");
      return false;
    }

    for (uint32_t current=1; current<=dataCount; current++) {
      uint8_t type;
      Id      id;
      Way     data;

      progress.SetProgress(current,dataCount);

      if (!scanner.Read(type) ||
          !scanner.Read(id) ||
          !data.Read(typeConfig,
                     scanner)) {
        progress.Error(std::string("Error while reading data entry ")+
                       NumberToString(current)+" of "+
                       NumberToString(dataCount)+
                       " in file '"+
                       scanner.GetFilename()+"'");

        return false;
      }

      if (!data.GetType()->CanRoute()) {
        continue;
      }

      std::unordered_set<Id> nodeIds;

      for (const auto& id : data.ids) {
        if (nodeIds.find(id)==nodeIds.end()) {
          nodeUseMap.SetNodeUsed(id);

          nodeIds.insert(id);
        }
      }

      // If we have a circular way, we "fake" a double usage,
      // to make sure, that the node id of the first node
      // is not dropped later on, and we cannot detect
      // circular ways anymore
      if (data.ids.front()==data.ids.back()) {
        nodeUseMap.SetNodeUsed(data.ids.back());
      }
    }

    if (!scanner.Close()) {
      progress.Error(std::string("Error while closing file '")+
                     scanner.GetFilename()+"'");
      return false;
    }

    return true;
  }
  bool AreaWayIndexGenerator::Import(const TypeConfigRef& typeConfig,
                                     const ImportParameter& parameter,
                                     Progress& progress)
  {
    FileScanner           wayScanner;
    FileWriter            writer;
    std::vector<TypeData> wayTypeData;
    size_t                maxLevel;

    progress.Info("Minimum magnification: "+NumberToString(parameter.GetAreaWayMinMag()));

    //
    // Scanning distribution
    //

    progress.SetAction("Scanning level distribution of way types");

    if (!CalculateDistribution(typeConfig,
                               parameter,
                               progress,
                               wayTypeData,
                               maxLevel)) {
      return false;
    }

    // Calculate number of types which have data

    uint32_t indexEntries=0;

    for (const auto& type : typeConfig->GetWayTypes())
    {
      if (wayTypeData[type->GetIndex()].HasEntries()) {
        indexEntries++;
      }
    }

    //
    // Writing index file
    //

    progress.SetAction("Generating 'areaway.idx'");

    if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                     "areaway.idx"))) {
      progress.Error("Cannot create 'areaway.idx'");
      return false;
    }

    writer.Write(indexEntries);

    for (const auto &type : typeConfig->GetWayTypes()) {
      size_t i=type->GetIndex();

      if (wayTypeData[i].HasEntries()) {
        uint8_t    dataOffsetBytes=0;
        FileOffset bitmapOffset=0;

        writer.WriteTypeId(type->GetWayId(),
                           typeConfig->GetWayTypeIdBytes());

        writer.GetPos(wayTypeData[i].indexOffset);

        writer.WriteFileOffset(bitmapOffset);
        writer.Write(dataOffsetBytes);
        writer.WriteNumber(wayTypeData[i].indexLevel);
        writer.WriteNumber(wayTypeData[i].cellXStart);
        writer.WriteNumber(wayTypeData[i].cellXEnd);
        writer.WriteNumber(wayTypeData[i].cellYStart);
        writer.WriteNumber(wayTypeData[i].cellYEnd);
      }
    }

    if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                         "ways.dat"),
                         FileScanner::Sequential,
                         parameter.GetWayDataMemoryMaped())) {
      progress.Error("Cannot open 'ways.dat'");
      return false;
    }

    for (size_t l=parameter.GetAreaWayMinMag(); l<=maxLevel; l++) {
      TypeInfoSet indexTypes(*typeConfig);
      uint32_t    wayCount;
      double      cellWidth=360.0/pow(2.0,(int)l);
      double      cellHeight=180.0/pow(2.0,(int)l);

      wayScanner.GotoBegin();

      for (const auto &type : typeConfig->GetWayTypes()) {
        if (wayTypeData[type->GetIndex()].HasEntries() &&
            wayTypeData[type->GetIndex()].indexLevel==l) {
          indexTypes.Set(type);
        }
      }

      if (indexTypes.Empty()) {
        continue;
      }

      progress.Info("Scanning ways for index level "+NumberToString(l));

      std::vector<CoordOffsetsMap> typeCellOffsets(typeConfig->GetTypeCount());

      if (!wayScanner.Read(wayCount)) {
        progress.Error("Error while reading number of data entries in file");
        return false;
      }

      Way way;

      for (uint32_t w=1; w<=wayCount; w++) {
        progress.SetProgress(w,wayCount);

        FileOffset offset;

        wayScanner.GetPos(offset);

        if (!way.Read(*typeConfig,
                      wayScanner)) {
          progress.Error(std::string("Error while reading data entry ")+
                         NumberToString(w)+" of "+
                         NumberToString(wayCount)+
                         " in file '"+
                         wayScanner.GetFilename()+"'");
          return false;
        }

        if (!indexTypes.IsSet(way.GetType())) {
          continue;
        }

        GeoBox boundingBox;

        way.GetBoundingBox(boundingBox);

        //
        // Calculate minimum and maximum tile ids that are covered
        // by the way
        // Renormalized coordinate space (everything is >=0)
        //
        uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellWidth);
        uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellWidth);
        uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellHeight);
        uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellHeight);

        for (uint32_t y=minyc; y<=maxyc; y++) {
          for (uint32_t x=minxc; x<=maxxc; x++) {
            typeCellOffsets[way.GetType()->GetIndex()][Pixel(x,y)].push_back(offset);
          }
        }
      }

      for (const auto &type : indexTypes) {
        size_t index=type->GetIndex();

        if (!WriteBitmap(progress,
                         writer,
                         *typeConfig->GetTypeInfo(index),
                         wayTypeData[index],
                         typeCellOffsets[index])) {
          return false;
        }
      }
    }

    return !writer.HasError() && writer.Close();
  }
Beispiel #18
0
  bool AreaWayIndexGenerator::Import(const ImportParameter& parameter,
                                     Progress& progress,
                                     const TypeConfig& typeConfig)
  {
    FileScanner           wayScanner;
    FileWriter            writer;
    std::set<TypeId>      remainingWayTypes;
    std::vector<TypeData> wayTypeData;
    size_t                level;
    size_t                maxLevel=0;

    wayTypeData.resize(typeConfig.GetTypes().size());

    if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                         "ways.dat"),
                         FileScanner::Sequential,
                         parameter.GetWayDataMemoryMaped())) {
      progress.Error("Cannot open 'ways.dat'");
      return false;
    }

    //
    // Scanning distribution
    //

    progress.SetAction("Scanning level distribution of way types");

    for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
      if (typeConfig.GetTypeInfo(i).CanBeWay() &&
          !typeConfig.GetTypeInfo(i).GetIgnore()) {
        remainingWayTypes.insert(i);
      }
    }

    level=parameter.GetAreaWayMinMag();
    while (!remainingWayTypes.empty()) {
      uint32_t                   wayCount=0;
      std::set<TypeId>           currentWayTypes(remainingWayTypes);
      double                     cellWidth=360.0/pow(2.0,(int)level);
      double                     cellHeight=180.0/pow(2.0,(int)level);
      std::vector<CoordCountMap> cellFillCount(typeConfig.GetTypes().size());

      progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingWayTypes.size())+" types remaining)");

      wayScanner.GotoBegin();

      if (!wayScanner.Read(wayCount)) {
        progress.Error("Error while reading number of data entries in file");
        return false;
      }

      Way way;

      for (uint32_t w=1; w<=wayCount; w++) {
        progress.SetProgress(w,wayCount);

        if (!way.Read(wayScanner)) {
          progress.Error(std::string("Error while reading data entry ")+
                         NumberToString(w)+" of "+
                         NumberToString(wayCount)+
                         " in file '"+
                         wayScanner.GetFilename()+"'");
          return false;
        }

        // Count number of entries per current type and coordinate
        if (currentWayTypes.find(way.GetType())==currentWayTypes.end()) {
          continue;
        }

        double minLon;
        double maxLon;
        double minLat;
        double maxLat;

        way.GetBoundingBox(minLon,maxLon,minLat,maxLat);

        //
        // Calculate minimum and maximum tile ids that are covered
        // by the way
        // Renormated coordinate space (everything is >=0)
        //
        uint32_t minxc=(uint32_t)floor((minLon+180.0)/cellWidth);
        uint32_t maxxc=(uint32_t)floor((maxLon+180.0)/cellWidth);
        uint32_t minyc=(uint32_t)floor((minLat+90.0)/cellHeight);
        uint32_t maxyc=(uint32_t)floor((maxLat+90.0)/cellHeight);

        for (uint32_t y=minyc; y<=maxyc; y++) {
          for (uint32_t x=minxc; x<=maxxc; x++) {
            cellFillCount[way.GetType()][Pixel(x,y)]++;
          }
        }
      }

      // Check if cell fill for current type is in defined limits
      for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
        if (currentWayTypes.find(i)!=currentWayTypes.end()) {
          CalculateStatistics(level,wayTypeData[i],cellFillCount[i]);

          if (!FitsIndexCriteria(parameter,
                                 progress,
                                 typeConfig.GetTypeInfo(i),
                                 wayTypeData[i],
                                 cellFillCount[i])) {
            currentWayTypes.erase(i);
          }
        }
      }

      for (std::set<TypeId>::const_iterator cwt=currentWayTypes.begin();
           cwt!=currentWayTypes.end();
           cwt++) {
        maxLevel=std::max(maxLevel,level);

        progress.Info("Type "+typeConfig.GetTypeInfo(*cwt).GetName()+"(" + NumberToString(*cwt)+"), "+NumberToString(wayTypeData[*cwt].indexCells)+" cells, "+NumberToString(wayTypeData[*cwt].indexEntries)+" objects");

        remainingWayTypes.erase(*cwt);
      }

      level++;
    }

    //
    // Writing index file
    //

    progress.SetAction("Generating 'areaway.idx'");

    if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                     "areaway.idx"))) {
      progress.Error("Cannot create 'areaway.idx'");
      return false;
    }

    uint32_t indexEntries=0;

    for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
    {
      if (typeConfig.GetTypeInfo(i).CanBeWay() &&
          wayTypeData[i].HasEntries()) {
        indexEntries++;
      }
    }

    writer.Write(indexEntries);

    for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
    {
      if (typeConfig.GetTypeInfo(i).CanBeWay() &&
          wayTypeData[i].HasEntries()) {
        uint8_t    dataOffsetBytes=0;
        FileOffset bitmapOffset=0;

        writer.WriteNumber(typeConfig.GetTypeInfo(i).GetId());

        writer.GetPos(wayTypeData[i].indexOffset);

        writer.WriteFileOffset(bitmapOffset);

        if (wayTypeData[i].HasEntries()) {
          writer.Write(dataOffsetBytes);
          writer.WriteNumber(wayTypeData[i].indexLevel);
          writer.WriteNumber(wayTypeData[i].cellXStart);
          writer.WriteNumber(wayTypeData[i].cellXEnd);
          writer.WriteNumber(wayTypeData[i].cellYStart);
          writer.WriteNumber(wayTypeData[i].cellYEnd);
        }
      }
    }

    for (size_t l=parameter.GetAreaWayMinMag(); l<=maxLevel; l++) {
      std::set<TypeId> indexTypes;
      uint32_t         wayCount;
      double           cellWidth=360.0/pow(2.0,(int)l);
      double           cellHeight=180.0/pow(2.0,(int)l);

      for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
        if (typeConfig.GetTypeInfo(i).CanBeWay() &&
            wayTypeData[i].HasEntries() &&
            wayTypeData[i].indexLevel==l) {
          indexTypes.insert(i);
        }
      }

      if (indexTypes.empty()) {
        continue;
      }

      progress.Info("Scanning ways for index level "+NumberToString(l));

      std::vector<CoordOffsetsMap> typeCellOffsets(typeConfig.GetTypes().size());

      wayScanner.GotoBegin();

      if (!wayScanner.Read(wayCount)) {
        progress.Error("Error while reading number of data entries in file");
        return false;
      }

      Way way;

      for (uint32_t w=1; w<=wayCount; w++) {
        progress.SetProgress(w,wayCount);

        FileOffset offset;

        wayScanner.GetPos(offset);

        if (!way.Read(wayScanner)) {
          progress.Error(std::string("Error while reading data entry ")+
                         NumberToString(w)+" of "+
                         NumberToString(wayCount)+
                         " in file '"+
                         wayScanner.GetFilename()+"'");
          return false;
        }

        if (indexTypes.find(way.GetType())==indexTypes.end()) {
          continue;
        }

        double minLon;
        double maxLon;
        double minLat;
        double maxLat;

        way.GetBoundingBox(minLon,maxLon,minLat,maxLat);

        //
        // Calculate minimum and maximum tile ids that are covered
        // by the way
        // Renormated coordinate space (everything is >=0)
        //
        uint32_t minxc=(uint32_t)floor((minLon+180.0)/cellWidth);
        uint32_t maxxc=(uint32_t)floor((maxLon+180.0)/cellWidth);
        uint32_t minyc=(uint32_t)floor((minLat+90.0)/cellHeight);
        uint32_t maxyc=(uint32_t)floor((maxLat+90.0)/cellHeight);

        for (uint32_t y=minyc; y<=maxyc; y++) {
          for (uint32_t x=minxc; x<=maxxc; x++) {
            typeCellOffsets[way.GetType()][Pixel(x,y)].push_back(offset);
          }
        }
      }

      for (std::set<TypeId>::const_iterator type=indexTypes.begin();
           type!=indexTypes.end();
           ++type) {
        if (!WriteBitmap(progress,
                         writer,
                         typeConfig.GetTypeInfo(*type),
                         wayTypeData[*type],
                         typeCellOffsets[*type])) {
          return false;
        }
      }
    }

    return !writer.HasError() && writer.Close();
  }
  void OptimizeAreasLowZoomGenerator::GetAreaIndexLevel(const ImportParameter& parameter,
                                                        const std::list<AreaRef>& areas,
                                                        TypeData& typeData)
  {
    size_t level=5;//parameter.GetOptimizationMinMag();

    while (true) {
      std::map<Pixel,size_t> cellFillCount;

      for (const auto& area : areas) {
        GeoBox  boundingBox;

        area->GetBoundingBox(boundingBox);

        //
        // Calculate minimum and maximum tile ids that are covered
        // by the way
        // Renormated coordinate space (everything is >=0)
        //
        uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellDimension[level].width);
        uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellDimension[level].width);
        uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellDimension[level].height);
        uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellDimension[level].height);

        for (uint32_t y=minyc; y<=maxyc; y++) {
          for (uint32_t x=minxc; x<=maxxc; x++) {
            cellFillCount[Pixel(x,y)]++;
          }
        }
      }

      // Check if cell fill for current type is in defined limits
      size_t entryCount=0;
      size_t max=0;

      for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount.begin();
           cell!=cellFillCount.end();
           ++cell) {
        entryCount+=cell->second;
        max=std::max(max,cell->second);
      }


      double average=entryCount*1.0/cellFillCount.size();

      if (!(max>parameter.GetOptimizationCellSizeMax() ||
           average>parameter.GetOptimizationCellSizeAverage())) {
        typeData.indexLevel=(uint32_t)level;
        typeData.indexCells=cellFillCount.size();
        typeData.indexEntries=0;

        if (!cellFillCount.empty()) {
          typeData.cellXStart=cellFillCount.begin()->first.x;
          typeData.cellYStart=cellFillCount.begin()->first.y;

          typeData.cellXEnd=typeData.cellXStart;
          typeData.cellYEnd=typeData.cellYStart;

          for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount.begin();
               cell!=cellFillCount.end();
               ++cell) {
            typeData.indexEntries+=cell->second;

            typeData.cellXStart=std::min(typeData.cellXStart,cell->first.x);
            typeData.cellXEnd=std::max(typeData.cellXEnd,cell->first.x);

            typeData.cellYStart=std::min(typeData.cellYStart,cell->first.y);
            typeData.cellYEnd=std::max(typeData.cellYEnd,cell->first.y);
          }
        }

        typeData.cellXCount=typeData.cellXEnd-typeData.cellXStart+1;
        typeData.cellYCount=typeData.cellYEnd-typeData.cellYStart+1;

        return;
      }

      level++;
    }
  }
  bool NodeDataGenerator::Import(const TypeConfigRef& typeConfig,
                                 const ImportParameter& parameter,
                                 Progress& progress)
  {
    uint32_t rawNodeCount=0;
    uint32_t nodesReadCount=0;
    uint32_t nodesWrittenCount=0;

    //
    // Iterator over all raw nodes, hcekc they type, and convert them from raw nodes
    // to nodes if the type is interesting (!=typeIgnore).
    //
    // Count the bounding box by the way...
    //

    progress.SetAction("Generating nodes.tmp");

    FileScanner scanner;
    FileWriter  writer;

    if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                      "rawnodes.dat"),
                      FileScanner::Sequential,
                      parameter.GetRawNodeDataMemoryMaped())) {
      progress.Error("Cannot open 'rawnodes.dat'");
      return false;
    }

    if (!scanner.Read(rawNodeCount)) {
      progress.Error("Error while reading number of data entries in file");
      return false;
    }

    if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                     "nodes.tmp"))) {
      progress.Error("Cannot create 'nodes.tmp'");
      return false;
    }

    writer.Write(nodesWrittenCount);

    for (uint32_t n=1; n<=rawNodeCount; n++) {
      progress.SetProgress(n,rawNodeCount);

      RawNode rawNode;
      Node    node;

      if (!rawNode.Read(typeConfig,
                        scanner)) {
        progress.Error(std::string("Error while reading data entry ")+
                       NumberToString(n)+" of "+
                       NumberToString(rawNodeCount)+
                       " in file '"+
                       scanner.GetFilename()+"'");
        return false;
      }

      nodesReadCount++;

      if (!rawNode.GetType()->GetIgnore()) {
        node.SetFeatures(rawNode.GetFeatureValueBuffer());
        node.SetCoords(rawNode.GetCoords());

        FileOffset fileOffset;

        if (!writer.GetPos(fileOffset)) {
          progress.Error(std::string("Error while reading current fileOffset in file '")+
                         writer.GetFilename()+"'");
          return false;
        }

        writer.Write(rawNode.GetId());
        node.Write(typeConfig,
                   writer);

        nodesWrittenCount++;
      }
    }

    if (!scanner.Close()) {
      return false;
    }

    writer.SetPos(0);
    writer.Write(nodesWrittenCount);

    if (!writer.Close()) {
      return false;
    }

    progress.Info(std::string("Read "+NumberToString(nodesReadCount)+" nodes, wrote "+NumberToString(nodesWrittenCount)+" nodes"));

    return true;
  }
Beispiel #21
0
  bool AreaNodeIndexGenerator::Import(const ImportParameter& parameter,
                                      Progress& progress,
                                      const TypeConfig& typeConfig)
  {
    FileScanner           nodeScanner;
    FileWriter            writer;
    std::set<TypeId>      remainingNodeTypes; //! Set of types we still must process
    std::vector<TypeData> nodeTypeData;
    size_t                level;
    size_t                maxLevel=0;

    nodeTypeData.resize(typeConfig.GetTypes().size());

    if (!nodeScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                         "nodes.dat"),
                          FileScanner::Sequential,
                          true)) {
      progress.Error("Cannot open 'nodes.dat'");
      return false;
    }

    //
    // Scanning distribution
    //

    progress.SetAction("Scanning level distribution of node types");

    // Initially we must process all types that represents nodes and that should
    // not be ignored
    for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
      if (typeConfig.GetTypeInfo(i).CanBeNode() &&
          !typeConfig.GetTypeInfo(i).GetIgnore()) {
        remainingNodeTypes.insert(i);
      }
    }

    level=parameter.GetAreaNodeMinMag();
    while (!remainingNodeTypes.empty()) {
      uint32_t         nodeCount=0;
      std::set<TypeId> currentNodeTypes(remainingNodeTypes);
      double           cellWidth=360.0/pow(2.0,(int)level);
      double           cellHeight=180.0/pow(2.0,(int)level);
      std::vector<std::map<Pixel,size_t> > cellFillCount;

      cellFillCount.resize(typeConfig.GetTypes().size());

      progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingNodeTypes.size())+" types still to process)");

      nodeScanner.GotoBegin();

      if (!nodeScanner.Read(nodeCount)) {
        progress.Error("Error while reading number of data entries in file");
        return false;
      }

      for (uint32_t n=1; n<=nodeCount; n++) {
        progress.SetProgress(n,nodeCount);

        FileOffset offset;
        Node       node;

        nodeScanner.GetPos(offset);

        if (!node.Read(nodeScanner)) {
          progress.Error(std::string("Error while reading data entry ")+
                         NumberToString(n)+" of "+
                         NumberToString(nodeCount)+
                         " in file '"+
                         nodeScanner.GetFilename()+"'");
          return false;
        }

        // If we still need to handle this type,
        // count number of entries per type and tile cell
        if (currentNodeTypes.find(node.GetType())!=currentNodeTypes.end()) {
          uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth);
          uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight);

          cellFillCount[node.GetType()][Pixel(xc,yc)]++;
        }
      }

      // Check statistics for each type
      // If statistics are within goal limits, use this level
      // for this type (else try again with the next higher level)
      for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
        if (currentNodeTypes.find(i)!=currentNodeTypes.end()) {
          size_t entryCount=0;
          size_t max=0;

          nodeTypeData[i].indexLevel=(uint32_t)level;
          nodeTypeData[i].indexCells=cellFillCount[i].size();
          nodeTypeData[i].indexEntries=0;

          if (!cellFillCount[i].empty()) {
            nodeTypeData[i].cellXStart=cellFillCount[i].begin()->first.x;
            nodeTypeData[i].cellYStart=cellFillCount[i].begin()->first.y;

            nodeTypeData[i].cellXEnd=nodeTypeData[i].cellXStart;
            nodeTypeData[i].cellYEnd=nodeTypeData[i].cellYStart;

            for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin();
                 cell!=cellFillCount[i].end();
                 ++cell) {
              nodeTypeData[i].indexEntries+=cell->second;

              nodeTypeData[i].cellXStart=std::min(nodeTypeData[i].cellXStart,cell->first.x);
              nodeTypeData[i].cellXEnd=std::max(nodeTypeData[i].cellXEnd,cell->first.x);

              nodeTypeData[i].cellYStart=std::min(nodeTypeData[i].cellYStart,cell->first.y);
              nodeTypeData[i].cellYEnd=std::max(nodeTypeData[i].cellYEnd,cell->first.y);
            }
          }

          nodeTypeData[i].cellXCount=nodeTypeData[i].cellXEnd-nodeTypeData[i].cellXStart+1;
          nodeTypeData[i].cellYCount=nodeTypeData[i].cellYEnd-nodeTypeData[i].cellYStart+1;

          // Count absolute number of entries
          for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin();
               cell!=cellFillCount[i].end();
               ++cell) {
            entryCount+=cell->second;
            max=std::max(max,cell->second);
          }

          // Average number of entries per tile cell
          double average=entryCount*1.0/cellFillCount[i].size();

          // If we do not have any entries, we store it now
          if (cellFillCount[i].empty()) {
            continue;
          }

          // If the fill rate of the index is too low, we use this index level anyway
          if (nodeTypeData[i].indexCells/(1.0*nodeTypeData[i].cellXCount*nodeTypeData[i].cellYCount)<=
              parameter.GetAreaNodeIndexMinFillRate()) {
            progress.Warning(typeConfig.GetTypeInfo(i).GetName()+" ("+NumberToString(i)+") is not well distributed");
            continue;
          }

          // If average fill size and max fill size for tile cells
          // is within limits, store it now.
          if (max<=parameter.GetAreaNodeIndexCellSizeMax() &&
              average<=parameter.GetAreaNodeIndexCellSizeAverage()) {
            continue;
          }

          // else, we remove it from the list and try again with an higher
          // level.
          currentNodeTypes.erase(i);
        }
      }

      // Now process all types for this limit, that are within the limits
      for (std::set<TypeId>::const_iterator cnt=currentNodeTypes.begin();
           cnt!=currentNodeTypes.end();
           cnt++) {
        maxLevel=std::max(maxLevel,level);

        progress.Info("Type "+typeConfig.GetTypeInfo(*cnt).GetName()+"(" + NumberToString(*cnt)+"), "+NumberToString(nodeTypeData[*cnt].indexCells)+" cells, "+NumberToString(nodeTypeData[*cnt].indexEntries)+" objects");

        remainingNodeTypes.erase(*cnt);
      }

      level++;
    }

    //
    // Writing index file
    //

    progress.SetAction("Generating 'areanode.idx'");

    if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                     "areanode.idx"))) {
      progress.Error("Cannot create 'areanode.idx'");
      return false;
    }

    uint32_t indexEntries=0;

    // Count number of types in index
    for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
    {
      if (typeConfig.GetTypeInfo(i).CanBeNode() &&
          nodeTypeData[i].HasEntries()) {
        indexEntries++;
      }
    }

    writer.Write(indexEntries);

    // Store index data for each type
    for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
    {
      if (typeConfig.GetTypeInfo(i).CanBeNode() &&
          nodeTypeData[i].HasEntries()) {
        FileOffset bitmapOffset=0;
        uint8_t    dataOffsetBytes=0;

        writer.WriteNumber(typeConfig.GetTypeInfo(i).GetId());

        writer.GetPos(nodeTypeData[i].indexOffset);

        writer.WriteFileOffset(bitmapOffset);
        writer.Write(dataOffsetBytes);

        writer.WriteNumber(nodeTypeData[i].indexLevel);
        writer.WriteNumber(nodeTypeData[i].cellXStart);
        writer.WriteNumber(nodeTypeData[i].cellXEnd);
        writer.WriteNumber(nodeTypeData[i].cellYStart);
        writer.WriteNumber(nodeTypeData[i].cellYEnd);
      }
    }

    // Now store index bitmap for each type in increasing level order (why?)
    for (size_t l=0; l<=maxLevel; l++) {
      std::set<TypeId> indexTypes;
      uint32_t         nodeCount;
      double           cellWidth=360.0/pow(2.0,(int)l);
      double           cellHeight=180.0/pow(2.0,(int)l);

      for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
        if (typeConfig.GetTypeInfo(i).CanBeNode() &&
            nodeTypeData[i].HasEntries() &&
            nodeTypeData[i].indexLevel==l) {
          indexTypes.insert(i);
        }
      }

      if (indexTypes.empty()) {
        continue;
      }

      progress.Info("Scanning nodes for index level "+NumberToString(l));

      std::vector<std::map<Pixel,std::list<FileOffset> > > typeCellOffsets;

      typeCellOffsets.resize(typeConfig.GetTypes().size());

      nodeScanner.GotoBegin();

      if (!nodeScanner.Read(nodeCount)) {
        progress.Error("Error while reading number of data entries in file");
        return false;
      }

      //
      // Collect all offsets
      //
      for (uint32_t n=1; n<=nodeCount; n++) {
        progress.SetProgress(n,nodeCount);

        FileOffset offset;
        Node       node;

        nodeScanner.GetPos(offset);

        if (!node.Read(nodeScanner)) {
          progress.Error(std::string("Error while reading data entry ")+
                         NumberToString(n)+" of "+
                         NumberToString(nodeCount)+
                         " in file '"+
                         nodeScanner.GetFilename()+"'");
          return false;
        }

        if (indexTypes.find(node.GetType())!=indexTypes.end()) {
          uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth);
          uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight);

          typeCellOffsets[node.GetType()][Pixel(xc,yc)].push_back(offset);
        }
      }

      //
      // Write bitmap
      //
      for (std::set<TypeId>::const_iterator type=indexTypes.begin();
           type!=indexTypes.end();
           ++type) {
        size_t indexEntries=0;
        size_t dataSize=0;
        char   buffer[10];

        for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin();
             cell!=typeCellOffsets[*type].end();
             ++cell) {
          indexEntries+=cell->second.size();

          dataSize+=EncodeNumber(cell->second.size(),buffer);

          FileOffset previousOffset=0;
          for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
               offset!=cell->second.end();
               ++offset) {
            FileOffset data=*offset-previousOffset;

            dataSize+=EncodeNumber(data,buffer);

            previousOffset=*offset;
          }
        }

        // "+1" because we add +1 to every offset, to generate offset > 0
        uint8_t dataOffsetBytes=BytesNeededToAddressFileData(dataSize);

        progress.Info("Writing map for "+
                      typeConfig.GetTypeInfo(*type).GetName()+", "+
                      NumberToString(typeCellOffsets[*type].size())+" cells, "+
                      NumberToString(indexEntries)+" entries, "+
                      ByteSizeToString(1.0*dataOffsetBytes*nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount));

        FileOffset bitmapOffset;

        if (!writer.GetPos(bitmapOffset)) {
          progress.Error("Cannot get type index start position in file");
          return false;
        }

        assert(nodeTypeData[*type].indexOffset!=0);

        if (!writer.SetPos(nodeTypeData[*type].indexOffset)) {
          progress.Error("Cannot go to type index offset in file");
          return false;
        }

        writer.WriteFileOffset(bitmapOffset);
        writer.Write(dataOffsetBytes);

        if (!writer.SetPos(bitmapOffset)) {
          progress.Error("Cannot go to type index start position in file");
          return false;
        }

        // Write the bitmap with offsets for each cell
        // We prefill with zero and only overwrite cells that have data
        // So zero means "no data for this cell"
        for (size_t i=0; i<nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount; i++) {
          FileOffset cellOffset=0;

          writer.WriteFileOffset(cellOffset,
                                 dataOffsetBytes);
        }

        FileOffset dataStartOffset;

        if (!writer.GetPos(dataStartOffset)) {
          progress.Error("Cannot get start of data section after bitmap");
          return false;
        }

        // Now write the list of offsets of objects for every cell with content
        for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin();
             cell!=typeCellOffsets[*type].end();
             ++cell) {
          FileOffset bitmapCellOffset=bitmapOffset+
                                      ((cell->first.y-nodeTypeData[*type].cellYStart)*nodeTypeData[*type].cellXCount+
                                       cell->first.x-nodeTypeData[*type].cellXStart)*dataOffsetBytes;
          FileOffset previousOffset=0;
          FileOffset cellOffset;

          if (!writer.GetPos(cellOffset)) {
            progress.Error("Cannot get cell start position in file");
            return false;
          }

          if (!writer.SetPos(bitmapCellOffset)) {
            progress.Error("Cannot go to cell start position in file");
            return false;
          }

          writer.WriteFileOffset(cellOffset-dataStartOffset+1,
                                 dataOffsetBytes);

          if (!writer.SetPos(cellOffset)) {
            progress.Error("Cannot go back to cell start position in file");
            return false;
          }

          writer.WriteNumber((uint32_t)cell->second.size());

          for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
               offset!=cell->second.end();
               ++offset) {
            writer.WriteNumber((FileOffset)(*offset-previousOffset));

            previousOffset=*offset;
          }
        }
      }
    }

    return !writer.HasError() && writer.Close();
  }
Beispiel #22
0
  bool Import(const ImportParameter& parameter,
              Progress& progress)
  {
    // TODO: verify parameter

    TypeConfigRef            typeConfig(new TypeConfig());
    std::list<ImportModule*> modules;

    if (parameter.GetAreaWayMinMag()<=parameter.GetOptimizationMaxMag()) {
      progress.Error("Area way index minimum magnification is <= than optimization max magnification");
    }

    progress.SetStep("Loading type config");

    if (!typeConfig->LoadFromOSTFile(parameter.GetTypefile())) {
      progress.Error("Cannot load type configuration!");
      return false;
    }

    progress.Info("Number of types: "+NumberToString(typeConfig->GetTypes().size()));
    progress.Info("Number of node types: "+NumberToString(typeConfig->GetNodeTypes().size())+" "+NumberToString(typeConfig->GetNodeTypeIdBytes())+" byte(s)");
    progress.Info("Number of way types: "+NumberToString(typeConfig->GetWayTypes().size())+" "+NumberToString(typeConfig->GetWayTypeIdBytes())+" byte(s)");
    progress.Info("Number of area types: "+NumberToString(typeConfig->GetAreaTypes().size())+" "+NumberToString(typeConfig->GetAreaTypeIdBytes())+" byte(s)");

    typeConfig->RegisterNameTag("name",0);
    typeConfig->RegisterNameTag("place_name",1);

    /*
    typeConfig->RegisterNameAltTag("name:ru",0);
    typeConfig->RegisterNameAltTag("place_name:ru",1);
    */

    /* 1 */
    modules.push_back(new TypeDataGenerator());

    /* 2 */
    modules.push_back(new Preprocess());

    /* 3 */
    modules.push_back(new RawNodeIndexGenerator(AppendFileToDir(parameter.GetDestinationDirectory(),
                                                                "rawnodes.dat"),
                                                AppendFileToDir(parameter.GetDestinationDirectory(),
                                                                "rawnode.idx")));
    /* 4 */
    modules.push_back(new RawWayIndexGenerator(AppendFileToDir(parameter.GetDestinationDirectory(),
                                                               "rawways.dat"),
                                               AppendFileToDir(parameter.GetDestinationDirectory(),
                                                               "rawway.idx")));
    /* 5 */
    modules.push_back(new RawRelationIndexGenerator(AppendFileToDir(parameter.GetDestinationDirectory(),
                                                                    "rawrels.dat"),
                                                    AppendFileToDir(parameter.GetDestinationDirectory(),
                                                                    "rawrel.idx")));
    /* 6 */
    modules.push_back(new RelAreaDataGenerator());

    /* 7 */
    modules.push_back(new WayAreaDataGenerator());

    /* 8 */
    modules.push_back(new MergeAreaDataGenerator());

    /* 9 */
    modules.push_back(new MergeAreasGenerator());

    /* 10 */
    modules.push_back(new WayWayDataGenerator());

    /* 11 */
    modules.push_back(new OptimizeAreaWayIdsGenerator());

    /* 12 */
    modules.push_back(new NodeDataGenerator());

    /* 13 */
    modules.push_back(new SortNodeDataGenerator());

    /* 14 */
    modules.push_back(new SortWayDataGenerator());

    /* 15 */
    modules.push_back(new AreaNodeIndexGenerator());

    /* 16 */
    modules.push_back(new AreaWayIndexGenerator());

    /* 17 */
    modules.push_back(new AreaAreaIndexGenerator());

    /* 18 */
    modules.push_back(new WaterIndexGenerator());

    /* 19 */
    modules.push_back(new OptimizeAreasLowZoomGenerator());

    /* 20 */
    modules.push_back(new OptimizeWaysLowZoomGenerator());

    /* 21 */
    modules.push_back(new LocationIndexGenerator());

    /* 22 */
    modules.push_back(new RouteDataGenerator());

    /* 23 */
    modules.push_back(new NumericIndexGenerator<Id,Intersection>(std::string("Generating '")+RoutingService::FILENAME_INTERSECTIONS_IDX+"'",
                                                                 AppendFileToDir(parameter.GetDestinationDirectory(),
                                                                                 RoutingService::FILENAME_INTERSECTIONS_DAT),
                                                                 AppendFileToDir(parameter.GetDestinationDirectory(),
                                                                                 RoutingService::FILENAME_INTERSECTIONS_IDX)));

#if defined(OSMSCOUT_IMPORT_HAVE_LIB_MARISA)
    /* 24 */
    modules.push_back(new TextIndexGenerator());
#endif

    bool result=ExecuteModules(modules,
                               parameter,
                               progress,
                               typeConfig);

    for (const auto& module : modules) {
      delete module;
    }

    return result;
  }
  bool AreaWayIndexGenerator::CalculateDistribution(const TypeConfigRef& typeConfig,
                                                    const ImportParameter& parameter,
                                                    Progress& progress,
                                                    std::vector<TypeData>& wayTypeData,
                                                    size_t& maxLevel) const
  {
    FileScanner wayScanner;
    TypeInfoSet remainingWayTypes;
    size_t      level;

    maxLevel=0;
    wayTypeData.resize(typeConfig->GetTypeCount());

    if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                         "ways.dat"),
                         FileScanner::Sequential,
                         parameter.GetWayDataMemoryMaped())) {
      progress.Error("Cannot open 'ways.dat'");
      return false;
    }

    remainingWayTypes.Set(typeConfig->GetWayTypes());

    level=parameter.GetAreaWayMinMag();
    while (!remainingWayTypes.Empty()) {
      uint32_t                   wayCount=0;
      TypeInfoSet                currentWayTypes(remainingWayTypes);
      double                     cellWidth=360.0/pow(2.0,(int)level);
      double                     cellHeight=180.0/pow(2.0,(int)level);
      std::vector<CoordCountMap> cellFillCount(typeConfig->GetTypeCount());

      progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingWayTypes.Size())+" types remaining)");

      wayScanner.GotoBegin();

      if (!wayScanner.Read(wayCount)) {
        progress.Error("Error while reading number of data entries in file");
        return false;
      }

      Way way;

      for (uint32_t w=1; w<=wayCount; w++) {
        progress.SetProgress(w,wayCount);

        if (!way.Read(*typeConfig,
                      wayScanner)) {
          progress.Error(std::string("Error while reading data entry ")+
                         NumberToString(w)+" of "+
                         NumberToString(wayCount)+
                         " in file '"+
                         wayScanner.GetFilename()+"'");
          return false;
        }

        // Count number of entries per current type and coordinate
        if (!currentWayTypes.IsSet(way.GetType())) {
          continue;
        }

        GeoBox boundingBox;

        way.GetBoundingBox(boundingBox);

        //
        // Calculate minimum and maximum tile ids that are covered
        // by the way
        // Renormalized coordinate space (everything is >=0)
        //
        uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellWidth);
        uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellWidth);
        uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellHeight);
        uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellHeight);

        for (uint32_t y=minyc; y<=maxyc; y++) {
          for (uint32_t x=minxc; x<=maxxc; x++) {
            cellFillCount[way.GetType()->GetIndex()][Pixel(x,y)]++;
          }
        }
      }

      // Check if cell fill for current type is in defined limits
      for (auto &type : currentWayTypes) {
        size_t i=type->GetIndex();

        CalculateStatistics(level,
                            wayTypeData[i],
                            cellFillCount[i]);

        if (!FitsIndexCriteria(parameter,
                               progress,
                               *typeConfig->GetTypeInfo(i),
                               wayTypeData[i],
                               cellFillCount[i])) {
          currentWayTypes.Remove(type);
        }
      }

      for (const auto &type : currentWayTypes) {
        maxLevel=std::max(maxLevel,level);

        progress.Info("Type "+type->GetName()+", "+NumberToString(wayTypeData[type->GetIndex()].indexCells)+" cells, "+NumberToString(wayTypeData[type->GetIndex()].indexEntries)+" objects");

        remainingWayTypes.Remove(type);
      }

      level++;
    }

    return !wayScanner.HasError() && wayScanner.Close();
  }