Esempio n. 1
0
  bool Import(const ImportParameter& parameter,
              Progress& progress)
  {
    // TODO: verify parameter

    TypeConfigRef            typeConfig(new TypeConfig());
    std::list<ImportModule*> modules;

    if (parameter.GetAreaWayMinMag()<=parameter.GetOptimizationMaxMag()) {
      progress.Error("Area way index minimum magnification is <= than optimization max magnification");
    }

    progress.SetStep("Loading type config");

    if (!typeConfig->LoadFromOSTFile(parameter.GetTypefile())) {
      progress.Error("Cannot load type configuration!");
      return false;
    }

    progress.Info("Number of types: "+NumberToString(typeConfig->GetTypes().size()));
    progress.Info("Number of node types: "+NumberToString(typeConfig->GetNodeTypes().size())+" "+NumberToString(typeConfig->GetNodeTypeIdBytes())+" byte(s)");
    progress.Info("Number of way types: "+NumberToString(typeConfig->GetWayTypes().size())+" "+NumberToString(typeConfig->GetWayTypeIdBytes())+" byte(s)");
    progress.Info("Number of area types: "+NumberToString(typeConfig->GetAreaTypes().size())+" "+NumberToString(typeConfig->GetAreaTypeIdBytes())+" byte(s)");

    typeConfig->RegisterNameTag("name",0);
    typeConfig->RegisterNameTag("place_name",1);

    /*
    typeConfig->RegisterNameAltTag("name:ru",0);
    typeConfig->RegisterNameAltTag("place_name:ru",1);
    */

    /* 1 */
    modules.push_back(new TypeDataGenerator());

    /* 2 */
    modules.push_back(new Preprocess());

    /* 3 */
    modules.push_back(new RawNodeIndexGenerator(AppendFileToDir(parameter.GetDestinationDirectory(),
                                                                "rawnodes.dat"),
                                                AppendFileToDir(parameter.GetDestinationDirectory(),
                                                                "rawnode.idx")));
    /* 4 */
    modules.push_back(new RawWayIndexGenerator(AppendFileToDir(parameter.GetDestinationDirectory(),
                                                               "rawways.dat"),
                                               AppendFileToDir(parameter.GetDestinationDirectory(),
                                                               "rawway.idx")));
    /* 5 */
    modules.push_back(new RawRelationIndexGenerator(AppendFileToDir(parameter.GetDestinationDirectory(),
                                                                    "rawrels.dat"),
                                                    AppendFileToDir(parameter.GetDestinationDirectory(),
                                                                    "rawrel.idx")));
    /* 6 */
    modules.push_back(new RelAreaDataGenerator());

    /* 7 */
    modules.push_back(new WayAreaDataGenerator());

    /* 8 */
    modules.push_back(new MergeAreaDataGenerator());

    /* 9 */
    modules.push_back(new MergeAreasGenerator());

    /* 10 */
    modules.push_back(new WayWayDataGenerator());

    /* 11 */
    modules.push_back(new OptimizeAreaWayIdsGenerator());

    /* 12 */
    modules.push_back(new NodeDataGenerator());

    /* 13 */
    modules.push_back(new SortNodeDataGenerator());

    /* 14 */
    modules.push_back(new SortWayDataGenerator());

    /* 15 */
    modules.push_back(new AreaNodeIndexGenerator());

    /* 16 */
    modules.push_back(new AreaWayIndexGenerator());

    /* 17 */
    modules.push_back(new AreaAreaIndexGenerator());

    /* 18 */
    modules.push_back(new WaterIndexGenerator());

    /* 19 */
    modules.push_back(new OptimizeAreasLowZoomGenerator());

    /* 20 */
    modules.push_back(new OptimizeWaysLowZoomGenerator());

    /* 21 */
    modules.push_back(new LocationIndexGenerator());

    /* 22 */
    modules.push_back(new RouteDataGenerator());

    /* 23 */
    modules.push_back(new NumericIndexGenerator<Id,Intersection>(std::string("Generating '")+RoutingService::FILENAME_INTERSECTIONS_IDX+"'",
                                                                 AppendFileToDir(parameter.GetDestinationDirectory(),
                                                                                 RoutingService::FILENAME_INTERSECTIONS_DAT),
                                                                 AppendFileToDir(parameter.GetDestinationDirectory(),
                                                                                 RoutingService::FILENAME_INTERSECTIONS_IDX)));

#if defined(OSMSCOUT_IMPORT_HAVE_LIB_MARISA)
    /* 24 */
    modules.push_back(new TextIndexGenerator());
#endif

    bool result=ExecuteModules(modules,
                               parameter,
                               progress,
                               typeConfig);

    for (const auto& module : modules) {
      delete module;
    }

    return result;
  }
  bool AreaWayIndexGenerator::Import(const TypeConfigRef& typeConfig,
                                     const ImportParameter& parameter,
                                     Progress& progress)
  {
    FileScanner           wayScanner;
    FileWriter            writer;
    std::vector<TypeData> wayTypeData;
    size_t                maxLevel;

    progress.Info("Minimum magnification: "+NumberToString(parameter.GetAreaWayMinMag()));

    //
    // Scanning distribution
    //

    progress.SetAction("Scanning level distribution of way types");

    if (!CalculateDistribution(typeConfig,
                               parameter,
                               progress,
                               wayTypeData,
                               maxLevel)) {
      return false;
    }

    // Calculate number of types which have data

    uint32_t indexEntries=0;

    for (const auto& type : typeConfig->GetWayTypes())
    {
      if (wayTypeData[type->GetIndex()].HasEntries()) {
        indexEntries++;
      }
    }

    //
    // Writing index file
    //

    progress.SetAction("Generating 'areaway.idx'");

    if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                     "areaway.idx"))) {
      progress.Error("Cannot create 'areaway.idx'");
      return false;
    }

    writer.Write(indexEntries);

    for (const auto &type : typeConfig->GetWayTypes()) {
      size_t i=type->GetIndex();

      if (wayTypeData[i].HasEntries()) {
        uint8_t    dataOffsetBytes=0;
        FileOffset bitmapOffset=0;

        writer.WriteTypeId(type->GetWayId(),
                           typeConfig->GetWayTypeIdBytes());

        writer.GetPos(wayTypeData[i].indexOffset);

        writer.WriteFileOffset(bitmapOffset);
        writer.Write(dataOffsetBytes);
        writer.WriteNumber(wayTypeData[i].indexLevel);
        writer.WriteNumber(wayTypeData[i].cellXStart);
        writer.WriteNumber(wayTypeData[i].cellXEnd);
        writer.WriteNumber(wayTypeData[i].cellYStart);
        writer.WriteNumber(wayTypeData[i].cellYEnd);
      }
    }

    if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                         "ways.dat"),
                         FileScanner::Sequential,
                         parameter.GetWayDataMemoryMaped())) {
      progress.Error("Cannot open 'ways.dat'");
      return false;
    }

    for (size_t l=parameter.GetAreaWayMinMag(); l<=maxLevel; l++) {
      TypeInfoSet indexTypes(*typeConfig);
      uint32_t    wayCount;
      double      cellWidth=360.0/pow(2.0,(int)l);
      double      cellHeight=180.0/pow(2.0,(int)l);

      wayScanner.GotoBegin();

      for (const auto &type : typeConfig->GetWayTypes()) {
        if (wayTypeData[type->GetIndex()].HasEntries() &&
            wayTypeData[type->GetIndex()].indexLevel==l) {
          indexTypes.Set(type);
        }
      }

      if (indexTypes.Empty()) {
        continue;
      }

      progress.Info("Scanning ways for index level "+NumberToString(l));

      std::vector<CoordOffsetsMap> typeCellOffsets(typeConfig->GetTypeCount());

      if (!wayScanner.Read(wayCount)) {
        progress.Error("Error while reading number of data entries in file");
        return false;
      }

      Way way;

      for (uint32_t w=1; w<=wayCount; w++) {
        progress.SetProgress(w,wayCount);

        FileOffset offset;

        wayScanner.GetPos(offset);

        if (!way.Read(*typeConfig,
                      wayScanner)) {
          progress.Error(std::string("Error while reading data entry ")+
                         NumberToString(w)+" of "+
                         NumberToString(wayCount)+
                         " in file '"+
                         wayScanner.GetFilename()+"'");
          return false;
        }

        if (!indexTypes.IsSet(way.GetType())) {
          continue;
        }

        GeoBox boundingBox;

        way.GetBoundingBox(boundingBox);

        //
        // Calculate minimum and maximum tile ids that are covered
        // by the way
        // Renormalized coordinate space (everything is >=0)
        //
        uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellWidth);
        uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellWidth);
        uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellHeight);
        uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellHeight);

        for (uint32_t y=minyc; y<=maxyc; y++) {
          for (uint32_t x=minxc; x<=maxxc; x++) {
            typeCellOffsets[way.GetType()->GetIndex()][Pixel(x,y)].push_back(offset);
          }
        }
      }

      for (const auto &type : indexTypes) {
        size_t index=type->GetIndex();

        if (!WriteBitmap(progress,
                         writer,
                         *typeConfig->GetTypeInfo(index),
                         wayTypeData[index],
                         typeCellOffsets[index])) {
          return false;
        }
      }
    }

    return !writer.HasError() && writer.Close();
  }
Esempio n. 3
0
  bool AreaWayIndexGenerator::Import(const ImportParameter& parameter,
                                     Progress& progress,
                                     const TypeConfig& typeConfig)
  {
    FileScanner           wayScanner;
    FileWriter            writer;
    std::set<TypeId>      remainingWayTypes;
    std::vector<TypeData> wayTypeData;
    size_t                level;
    size_t                maxLevel=0;

    wayTypeData.resize(typeConfig.GetTypes().size());

    if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                         "ways.dat"),
                         FileScanner::Sequential,
                         parameter.GetWayDataMemoryMaped())) {
      progress.Error("Cannot open 'ways.dat'");
      return false;
    }

    //
    // Scanning distribution
    //

    progress.SetAction("Scanning level distribution of way types");

    for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
      if (typeConfig.GetTypeInfo(i).CanBeWay() &&
          !typeConfig.GetTypeInfo(i).GetIgnore()) {
        remainingWayTypes.insert(i);
      }
    }

    level=parameter.GetAreaWayMinMag();
    while (!remainingWayTypes.empty()) {
      uint32_t                   wayCount=0;
      std::set<TypeId>           currentWayTypes(remainingWayTypes);
      double                     cellWidth=360.0/pow(2.0,(int)level);
      double                     cellHeight=180.0/pow(2.0,(int)level);
      std::vector<CoordCountMap> cellFillCount(typeConfig.GetTypes().size());

      progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingWayTypes.size())+" types remaining)");

      wayScanner.GotoBegin();

      if (!wayScanner.Read(wayCount)) {
        progress.Error("Error while reading number of data entries in file");
        return false;
      }

      Way way;

      for (uint32_t w=1; w<=wayCount; w++) {
        progress.SetProgress(w,wayCount);

        if (!way.Read(wayScanner)) {
          progress.Error(std::string("Error while reading data entry ")+
                         NumberToString(w)+" of "+
                         NumberToString(wayCount)+
                         " in file '"+
                         wayScanner.GetFilename()+"'");
          return false;
        }

        // Count number of entries per current type and coordinate
        if (currentWayTypes.find(way.GetType())==currentWayTypes.end()) {
          continue;
        }

        double minLon;
        double maxLon;
        double minLat;
        double maxLat;

        way.GetBoundingBox(minLon,maxLon,minLat,maxLat);

        //
        // Calculate minimum and maximum tile ids that are covered
        // by the way
        // Renormated coordinate space (everything is >=0)
        //
        uint32_t minxc=(uint32_t)floor((minLon+180.0)/cellWidth);
        uint32_t maxxc=(uint32_t)floor((maxLon+180.0)/cellWidth);
        uint32_t minyc=(uint32_t)floor((minLat+90.0)/cellHeight);
        uint32_t maxyc=(uint32_t)floor((maxLat+90.0)/cellHeight);

        for (uint32_t y=minyc; y<=maxyc; y++) {
          for (uint32_t x=minxc; x<=maxxc; x++) {
            cellFillCount[way.GetType()][Pixel(x,y)]++;
          }
        }
      }

      // Check if cell fill for current type is in defined limits
      for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
        if (currentWayTypes.find(i)!=currentWayTypes.end()) {
          CalculateStatistics(level,wayTypeData[i],cellFillCount[i]);

          if (!FitsIndexCriteria(parameter,
                                 progress,
                                 typeConfig.GetTypeInfo(i),
                                 wayTypeData[i],
                                 cellFillCount[i])) {
            currentWayTypes.erase(i);
          }
        }
      }

      for (std::set<TypeId>::const_iterator cwt=currentWayTypes.begin();
           cwt!=currentWayTypes.end();
           cwt++) {
        maxLevel=std::max(maxLevel,level);

        progress.Info("Type "+typeConfig.GetTypeInfo(*cwt).GetName()+"(" + NumberToString(*cwt)+"), "+NumberToString(wayTypeData[*cwt].indexCells)+" cells, "+NumberToString(wayTypeData[*cwt].indexEntries)+" objects");

        remainingWayTypes.erase(*cwt);
      }

      level++;
    }

    //
    // Writing index file
    //

    progress.SetAction("Generating 'areaway.idx'");

    if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                     "areaway.idx"))) {
      progress.Error("Cannot create 'areaway.idx'");
      return false;
    }

    uint32_t indexEntries=0;

    for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
    {
      if (typeConfig.GetTypeInfo(i).CanBeWay() &&
          wayTypeData[i].HasEntries()) {
        indexEntries++;
      }
    }

    writer.Write(indexEntries);

    for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
    {
      if (typeConfig.GetTypeInfo(i).CanBeWay() &&
          wayTypeData[i].HasEntries()) {
        uint8_t    dataOffsetBytes=0;
        FileOffset bitmapOffset=0;

        writer.WriteNumber(typeConfig.GetTypeInfo(i).GetId());

        writer.GetPos(wayTypeData[i].indexOffset);

        writer.WriteFileOffset(bitmapOffset);

        if (wayTypeData[i].HasEntries()) {
          writer.Write(dataOffsetBytes);
          writer.WriteNumber(wayTypeData[i].indexLevel);
          writer.WriteNumber(wayTypeData[i].cellXStart);
          writer.WriteNumber(wayTypeData[i].cellXEnd);
          writer.WriteNumber(wayTypeData[i].cellYStart);
          writer.WriteNumber(wayTypeData[i].cellYEnd);
        }
      }
    }

    for (size_t l=parameter.GetAreaWayMinMag(); l<=maxLevel; l++) {
      std::set<TypeId> indexTypes;
      uint32_t         wayCount;
      double           cellWidth=360.0/pow(2.0,(int)l);
      double           cellHeight=180.0/pow(2.0,(int)l);

      for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
        if (typeConfig.GetTypeInfo(i).CanBeWay() &&
            wayTypeData[i].HasEntries() &&
            wayTypeData[i].indexLevel==l) {
          indexTypes.insert(i);
        }
      }

      if (indexTypes.empty()) {
        continue;
      }

      progress.Info("Scanning ways for index level "+NumberToString(l));

      std::vector<CoordOffsetsMap> typeCellOffsets(typeConfig.GetTypes().size());

      wayScanner.GotoBegin();

      if (!wayScanner.Read(wayCount)) {
        progress.Error("Error while reading number of data entries in file");
        return false;
      }

      Way way;

      for (uint32_t w=1; w<=wayCount; w++) {
        progress.SetProgress(w,wayCount);

        FileOffset offset;

        wayScanner.GetPos(offset);

        if (!way.Read(wayScanner)) {
          progress.Error(std::string("Error while reading data entry ")+
                         NumberToString(w)+" of "+
                         NumberToString(wayCount)+
                         " in file '"+
                         wayScanner.GetFilename()+"'");
          return false;
        }

        if (indexTypes.find(way.GetType())==indexTypes.end()) {
          continue;
        }

        double minLon;
        double maxLon;
        double minLat;
        double maxLat;

        way.GetBoundingBox(minLon,maxLon,minLat,maxLat);

        //
        // Calculate minimum and maximum tile ids that are covered
        // by the way
        // Renormated coordinate space (everything is >=0)
        //
        uint32_t minxc=(uint32_t)floor((minLon+180.0)/cellWidth);
        uint32_t maxxc=(uint32_t)floor((maxLon+180.0)/cellWidth);
        uint32_t minyc=(uint32_t)floor((minLat+90.0)/cellHeight);
        uint32_t maxyc=(uint32_t)floor((maxLat+90.0)/cellHeight);

        for (uint32_t y=minyc; y<=maxyc; y++) {
          for (uint32_t x=minxc; x<=maxxc; x++) {
            typeCellOffsets[way.GetType()][Pixel(x,y)].push_back(offset);
          }
        }
      }

      for (std::set<TypeId>::const_iterator type=indexTypes.begin();
           type!=indexTypes.end();
           ++type) {
        if (!WriteBitmap(progress,
                         writer,
                         typeConfig.GetTypeInfo(*type),
                         wayTypeData[*type],
                         typeCellOffsets[*type])) {
          return false;
        }
      }
    }

    return !writer.HasError() && writer.Close();
  }
  bool AreaWayIndexGenerator::CalculateDistribution(const TypeConfigRef& typeConfig,
                                                    const ImportParameter& parameter,
                                                    Progress& progress,
                                                    std::vector<TypeData>& wayTypeData,
                                                    size_t& maxLevel) const
  {
    FileScanner wayScanner;
    TypeInfoSet remainingWayTypes;
    size_t      level;

    maxLevel=0;
    wayTypeData.resize(typeConfig->GetTypeCount());

    if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                         "ways.dat"),
                         FileScanner::Sequential,
                         parameter.GetWayDataMemoryMaped())) {
      progress.Error("Cannot open 'ways.dat'");
      return false;
    }

    remainingWayTypes.Set(typeConfig->GetWayTypes());

    level=parameter.GetAreaWayMinMag();
    while (!remainingWayTypes.Empty()) {
      uint32_t                   wayCount=0;
      TypeInfoSet                currentWayTypes(remainingWayTypes);
      double                     cellWidth=360.0/pow(2.0,(int)level);
      double                     cellHeight=180.0/pow(2.0,(int)level);
      std::vector<CoordCountMap> cellFillCount(typeConfig->GetTypeCount());

      progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingWayTypes.Size())+" types remaining)");

      wayScanner.GotoBegin();

      if (!wayScanner.Read(wayCount)) {
        progress.Error("Error while reading number of data entries in file");
        return false;
      }

      Way way;

      for (uint32_t w=1; w<=wayCount; w++) {
        progress.SetProgress(w,wayCount);

        if (!way.Read(*typeConfig,
                      wayScanner)) {
          progress.Error(std::string("Error while reading data entry ")+
                         NumberToString(w)+" of "+
                         NumberToString(wayCount)+
                         " in file '"+
                         wayScanner.GetFilename()+"'");
          return false;
        }

        // Count number of entries per current type and coordinate
        if (!currentWayTypes.IsSet(way.GetType())) {
          continue;
        }

        GeoBox boundingBox;

        way.GetBoundingBox(boundingBox);

        //
        // Calculate minimum and maximum tile ids that are covered
        // by the way
        // Renormalized coordinate space (everything is >=0)
        //
        uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellWidth);
        uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellWidth);
        uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellHeight);
        uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellHeight);

        for (uint32_t y=minyc; y<=maxyc; y++) {
          for (uint32_t x=minxc; x<=maxxc; x++) {
            cellFillCount[way.GetType()->GetIndex()][Pixel(x,y)]++;
          }
        }
      }

      // Check if cell fill for current type is in defined limits
      for (auto &type : currentWayTypes) {
        size_t i=type->GetIndex();

        CalculateStatistics(level,
                            wayTypeData[i],
                            cellFillCount[i]);

        if (!FitsIndexCriteria(parameter,
                               progress,
                               *typeConfig->GetTypeInfo(i),
                               wayTypeData[i],
                               cellFillCount[i])) {
          currentWayTypes.Remove(type);
        }
      }

      for (const auto &type : currentWayTypes) {
        maxLevel=std::max(maxLevel,level);

        progress.Info("Type "+type->GetName()+", "+NumberToString(wayTypeData[type->GetIndex()].indexCells)+" cells, "+NumberToString(wayTypeData[type->GetIndex()].indexEntries)+" objects");

        remainingWayTypes.Remove(type);
      }

      level++;
    }

    return !wayScanner.HasError() && wayScanner.Close();
  }