Ejemplo n.º 1
0
  bool AreaWayIndexGenerator::FitsIndexCriteria(const ImportParameter& parameter,
                                                Progress& progress,
                                                const TypeInfo& typeInfo,
                                                const TypeData& typeData,
                                                const CoordCountMap& cellFillCount)
  {
    if (typeData.indexCells==0) {
      return true;
    }

    size_t entryCount=0;
    size_t max=0;

    for (CoordCountMap::const_iterator cell=cellFillCount.begin();
         cell!=cellFillCount.end();
         ++cell) {
      entryCount+=cell->second;
      max=std::max(max,cell->second);
    }

    // Average number of entries per tile cell
    double average=entryCount*1.0/cellFillCount.size();

    // If the fill rate of the index is too low, we use this index level anyway
    if (typeData.indexCells/(1.0*typeData.cellXCount*typeData.cellYCount)<=
        parameter.GetAreaWayIndexMinFillRate()) {
      progress.Warning(typeInfo.GetName()+" ("+NumberToString(typeInfo.GetId())+") is not well distributed");
      return true;
    }

    // If average fill size and max fill size for tile cells
    // is within limits, store it now.
    if (max<=parameter.GetAreaWayIndexCellSizeMax() &&
        average<=parameter.GetAreaWayIndexCellSizeAverage()) {
      return true;
    }

    return false;
  }
Ejemplo n.º 2
0
  bool AreaNodeIndexGenerator::Import(const ImportParameter& parameter,
                                      Progress& progress,
                                      const TypeConfig& typeConfig)
  {
    FileScanner           nodeScanner;
    FileWriter            writer;
    std::set<TypeId>      remainingNodeTypes; //! Set of types we still must process
    std::vector<TypeData> nodeTypeData;
    size_t                level;
    size_t                maxLevel=0;

    nodeTypeData.resize(typeConfig.GetTypes().size());

    if (!nodeScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                         "nodes.dat"),
                          FileScanner::Sequential,
                          true)) {
      progress.Error("Cannot open 'nodes.dat'");
      return false;
    }

    //
    // Scanning distribution
    //

    progress.SetAction("Scanning level distribution of node types");

    // Initially we must process all types that represents nodes and that should
    // not be ignored
    for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
      if (typeConfig.GetTypeInfo(i).CanBeNode() &&
          !typeConfig.GetTypeInfo(i).GetIgnore()) {
        remainingNodeTypes.insert(i);
      }
    }

    level=parameter.GetAreaNodeMinMag();
    while (!remainingNodeTypes.empty()) {
      uint32_t         nodeCount=0;
      std::set<TypeId> currentNodeTypes(remainingNodeTypes);
      double           cellWidth=360.0/pow(2.0,(int)level);
      double           cellHeight=180.0/pow(2.0,(int)level);
      std::vector<std::map<Pixel,size_t> > cellFillCount;

      cellFillCount.resize(typeConfig.GetTypes().size());

      progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingNodeTypes.size())+" types still to process)");

      nodeScanner.GotoBegin();

      if (!nodeScanner.Read(nodeCount)) {
        progress.Error("Error while reading number of data entries in file");
        return false;
      }

      for (uint32_t n=1; n<=nodeCount; n++) {
        progress.SetProgress(n,nodeCount);

        FileOffset offset;
        Node       node;

        nodeScanner.GetPos(offset);

        if (!node.Read(nodeScanner)) {
          progress.Error(std::string("Error while reading data entry ")+
                         NumberToString(n)+" of "+
                         NumberToString(nodeCount)+
                         " in file '"+
                         nodeScanner.GetFilename()+"'");
          return false;
        }

        // If we still need to handle this type,
        // count number of entries per type and tile cell
        if (currentNodeTypes.find(node.GetType())!=currentNodeTypes.end()) {
          uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth);
          uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight);

          cellFillCount[node.GetType()][Pixel(xc,yc)]++;
        }
      }

      // Check statistics for each type
      // If statistics are within goal limits, use this level
      // for this type (else try again with the next higher level)
      for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
        if (currentNodeTypes.find(i)!=currentNodeTypes.end()) {
          size_t entryCount=0;
          size_t max=0;

          nodeTypeData[i].indexLevel=(uint32_t)level;
          nodeTypeData[i].indexCells=cellFillCount[i].size();
          nodeTypeData[i].indexEntries=0;

          if (!cellFillCount[i].empty()) {
            nodeTypeData[i].cellXStart=cellFillCount[i].begin()->first.x;
            nodeTypeData[i].cellYStart=cellFillCount[i].begin()->first.y;

            nodeTypeData[i].cellXEnd=nodeTypeData[i].cellXStart;
            nodeTypeData[i].cellYEnd=nodeTypeData[i].cellYStart;

            for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin();
                 cell!=cellFillCount[i].end();
                 ++cell) {
              nodeTypeData[i].indexEntries+=cell->second;

              nodeTypeData[i].cellXStart=std::min(nodeTypeData[i].cellXStart,cell->first.x);
              nodeTypeData[i].cellXEnd=std::max(nodeTypeData[i].cellXEnd,cell->first.x);

              nodeTypeData[i].cellYStart=std::min(nodeTypeData[i].cellYStart,cell->first.y);
              nodeTypeData[i].cellYEnd=std::max(nodeTypeData[i].cellYEnd,cell->first.y);
            }
          }

          nodeTypeData[i].cellXCount=nodeTypeData[i].cellXEnd-nodeTypeData[i].cellXStart+1;
          nodeTypeData[i].cellYCount=nodeTypeData[i].cellYEnd-nodeTypeData[i].cellYStart+1;

          // Count absolute number of entries
          for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin();
               cell!=cellFillCount[i].end();
               ++cell) {
            entryCount+=cell->second;
            max=std::max(max,cell->second);
          }

          // Average number of entries per tile cell
          double average=entryCount*1.0/cellFillCount[i].size();

          // If we do not have any entries, we store it now
          if (cellFillCount[i].empty()) {
            continue;
          }

          // If the fill rate of the index is too low, we use this index level anyway
          if (nodeTypeData[i].indexCells/(1.0*nodeTypeData[i].cellXCount*nodeTypeData[i].cellYCount)<=
              parameter.GetAreaNodeIndexMinFillRate()) {
            progress.Warning(typeConfig.GetTypeInfo(i).GetName()+" ("+NumberToString(i)+") is not well distributed");
            continue;
          }

          // If average fill size and max fill size for tile cells
          // is within limits, store it now.
          if (max<=parameter.GetAreaNodeIndexCellSizeMax() &&
              average<=parameter.GetAreaNodeIndexCellSizeAverage()) {
            continue;
          }

          // else, we remove it from the list and try again with an higher
          // level.
          currentNodeTypes.erase(i);
        }
      }

      // Now process all types for this limit, that are within the limits
      for (std::set<TypeId>::const_iterator cnt=currentNodeTypes.begin();
           cnt!=currentNodeTypes.end();
           cnt++) {
        maxLevel=std::max(maxLevel,level);

        progress.Info("Type "+typeConfig.GetTypeInfo(*cnt).GetName()+"(" + NumberToString(*cnt)+"), "+NumberToString(nodeTypeData[*cnt].indexCells)+" cells, "+NumberToString(nodeTypeData[*cnt].indexEntries)+" objects");

        remainingNodeTypes.erase(*cnt);
      }

      level++;
    }

    //
    // Writing index file
    //

    progress.SetAction("Generating 'areanode.idx'");

    if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                     "areanode.idx"))) {
      progress.Error("Cannot create 'areanode.idx'");
      return false;
    }

    uint32_t indexEntries=0;

    // Count number of types in index
    for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
    {
      if (typeConfig.GetTypeInfo(i).CanBeNode() &&
          nodeTypeData[i].HasEntries()) {
        indexEntries++;
      }
    }

    writer.Write(indexEntries);

    // Store index data for each type
    for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
    {
      if (typeConfig.GetTypeInfo(i).CanBeNode() &&
          nodeTypeData[i].HasEntries()) {
        FileOffset bitmapOffset=0;
        uint8_t    dataOffsetBytes=0;

        writer.WriteNumber(typeConfig.GetTypeInfo(i).GetId());

        writer.GetPos(nodeTypeData[i].indexOffset);

        writer.WriteFileOffset(bitmapOffset);
        writer.Write(dataOffsetBytes);

        writer.WriteNumber(nodeTypeData[i].indexLevel);
        writer.WriteNumber(nodeTypeData[i].cellXStart);
        writer.WriteNumber(nodeTypeData[i].cellXEnd);
        writer.WriteNumber(nodeTypeData[i].cellYStart);
        writer.WriteNumber(nodeTypeData[i].cellYEnd);
      }
    }

    // Now store index bitmap for each type in increasing level order (why?)
    for (size_t l=0; l<=maxLevel; l++) {
      std::set<TypeId> indexTypes;
      uint32_t         nodeCount;
      double           cellWidth=360.0/pow(2.0,(int)l);
      double           cellHeight=180.0/pow(2.0,(int)l);

      for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
        if (typeConfig.GetTypeInfo(i).CanBeNode() &&
            nodeTypeData[i].HasEntries() &&
            nodeTypeData[i].indexLevel==l) {
          indexTypes.insert(i);
        }
      }

      if (indexTypes.empty()) {
        continue;
      }

      progress.Info("Scanning nodes for index level "+NumberToString(l));

      std::vector<std::map<Pixel,std::list<FileOffset> > > typeCellOffsets;

      typeCellOffsets.resize(typeConfig.GetTypes().size());

      nodeScanner.GotoBegin();

      if (!nodeScanner.Read(nodeCount)) {
        progress.Error("Error while reading number of data entries in file");
        return false;
      }

      //
      // Collect all offsets
      //
      for (uint32_t n=1; n<=nodeCount; n++) {
        progress.SetProgress(n,nodeCount);

        FileOffset offset;
        Node       node;

        nodeScanner.GetPos(offset);

        if (!node.Read(nodeScanner)) {
          progress.Error(std::string("Error while reading data entry ")+
                         NumberToString(n)+" of "+
                         NumberToString(nodeCount)+
                         " in file '"+
                         nodeScanner.GetFilename()+"'");
          return false;
        }

        if (indexTypes.find(node.GetType())!=indexTypes.end()) {
          uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth);
          uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight);

          typeCellOffsets[node.GetType()][Pixel(xc,yc)].push_back(offset);
        }
      }

      //
      // Write bitmap
      //
      for (std::set<TypeId>::const_iterator type=indexTypes.begin();
           type!=indexTypes.end();
           ++type) {
        size_t indexEntries=0;
        size_t dataSize=0;
        char   buffer[10];

        for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin();
             cell!=typeCellOffsets[*type].end();
             ++cell) {
          indexEntries+=cell->second.size();

          dataSize+=EncodeNumber(cell->second.size(),buffer);

          FileOffset previousOffset=0;
          for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
               offset!=cell->second.end();
               ++offset) {
            FileOffset data=*offset-previousOffset;

            dataSize+=EncodeNumber(data,buffer);

            previousOffset=*offset;
          }
        }

        // "+1" because we add +1 to every offset, to generate offset > 0
        uint8_t dataOffsetBytes=BytesNeededToAddressFileData(dataSize);

        progress.Info("Writing map for "+
                      typeConfig.GetTypeInfo(*type).GetName()+", "+
                      NumberToString(typeCellOffsets[*type].size())+" cells, "+
                      NumberToString(indexEntries)+" entries, "+
                      ByteSizeToString(1.0*dataOffsetBytes*nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount));

        FileOffset bitmapOffset;

        if (!writer.GetPos(bitmapOffset)) {
          progress.Error("Cannot get type index start position in file");
          return false;
        }

        assert(nodeTypeData[*type].indexOffset!=0);

        if (!writer.SetPos(nodeTypeData[*type].indexOffset)) {
          progress.Error("Cannot go to type index offset in file");
          return false;
        }

        writer.WriteFileOffset(bitmapOffset);
        writer.Write(dataOffsetBytes);

        if (!writer.SetPos(bitmapOffset)) {
          progress.Error("Cannot go to type index start position in file");
          return false;
        }

        // Write the bitmap with offsets for each cell
        // We prefill with zero and only overwrite cells that have data
        // So zero means "no data for this cell"
        for (size_t i=0; i<nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount; i++) {
          FileOffset cellOffset=0;

          writer.WriteFileOffset(cellOffset,
                                 dataOffsetBytes);
        }

        FileOffset dataStartOffset;

        if (!writer.GetPos(dataStartOffset)) {
          progress.Error("Cannot get start of data section after bitmap");
          return false;
        }

        // Now write the list of offsets of objects for every cell with content
        for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin();
             cell!=typeCellOffsets[*type].end();
             ++cell) {
          FileOffset bitmapCellOffset=bitmapOffset+
                                      ((cell->first.y-nodeTypeData[*type].cellYStart)*nodeTypeData[*type].cellXCount+
                                       cell->first.x-nodeTypeData[*type].cellXStart)*dataOffsetBytes;
          FileOffset previousOffset=0;
          FileOffset cellOffset;

          if (!writer.GetPos(cellOffset)) {
            progress.Error("Cannot get cell start position in file");
            return false;
          }

          if (!writer.SetPos(bitmapCellOffset)) {
            progress.Error("Cannot go to cell start position in file");
            return false;
          }

          writer.WriteFileOffset(cellOffset-dataStartOffset+1,
                                 dataOffsetBytes);

          if (!writer.SetPos(cellOffset)) {
            progress.Error("Cannot go back to cell start position in file");
            return false;
          }

          writer.WriteNumber((uint32_t)cell->second.size());

          for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
               offset!=cell->second.end();
               ++offset) {
            writer.WriteNumber((FileOffset)(*offset-previousOffset));

            previousOffset=*offset;
          }
        }
      }
    }

    return !writer.HasError() && writer.Close();
  }
Ejemplo n.º 3
0
  bool AreaWayIndexGenerator::FitsIndexCriteria(const ImportParameter& /*parameter*/,
                                                Progress& progress,
                                                const TypeInfo& typeInfo,
                                                const TypeData& typeData,
                                                const CoordCountMap& cellFillCount) const
  {
    if (typeData.indexCells==0) {
      return true;
    }

    size_t overallCount=0;
    size_t maxCellCount=0;

    for (const auto& cell : cellFillCount) {
      overallCount+=cell.second;
      maxCellCount=std::max(maxCellCount,cell.second);
    }

    // Average number of entries per tile cell
    double average=overallCount*1.0/cellFillCount.size();

    size_t emptyCount=0;
    size_t toLowCount=0;
    size_t toHighCount=0;
    size_t inCount=0;
    size_t allCount=0;

    for (const auto& cell : cellFillCount) {
      if (cell.second==0) {
        emptyCount++;
      }
      else if (cell.second<0.4*average) {
        toLowCount++;
      }
      else if (cell.second>128){
        toHighCount++;
      }
      else {
        inCount++;
      }

      allCount++;
    }

    if (toHighCount*1.0/allCount>=0.05) {
      return false;
    }

    if (toLowCount*1.0/allCount>=0.2) {
      progress.Warning(typeInfo.GetName()+" has more than 20% cells with <40% of average filling ("+NumberToString(toLowCount)+"/"+NumberToString(allCount)+")");
    }

    /*
    // If the fill rate of the index is too low, we use this index level anyway
    if (fillRate<parameter.GetAreaWayIndexMinFillRate()) {
      progress.Warning(typeInfo.GetName()+" is not well distributed");
      return true;
    }

    // If average fill size and max fill size for tile cells
    // is within limits, store it now.
    if (maxCellCount<=parameter.GetAreaWayIndexCellSizeMax() &&
        average<=parameter.GetAreaWayIndexCellSizeAverage()) {
      return true;
    }*/

    return true;
  }