bool OptimizeAreasLowZoomGenerator::WriteAreaBitmap(Progress& progress,
FileWriter& writer,
const std::list<AreaRef>& areas,
const FileOffsetFileOffsetMap& offsets,
TypeData& data)
{
// We do not write a bitmap, if there is not data to map
if (areas.empty()) {
return true;
}
double cellWidth=cellDimension[data.indexLevel].width;
double cellHeight=cellDimension[data.indexLevel].height;
std::map<Pixel,std::list<FileOffset> > cellOffsets;
for (const auto& area : areas) {
GeoBox boundingBox;
FileOffsetFileOffsetMap::const_iterator offset=offsets.find(area->GetFileOffset());
if (offset==offsets.end()) {
continue;
}
area->GetBoundingBox(boundingBox);
//
// Calculate minimum and maximum tile ids that are covered
// by the way
// Renormated coordinate space (everything is >=0)
//
uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellWidth);
uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellWidth);
uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellHeight);
uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellHeight);
for (uint32_t y=minyc; y<=maxyc; y++) {
for (uint32_t x=minxc; x<=maxxc; x++) {
cellOffsets[Pixel(x,y)].push_back(offset->second);
}
}
}
size_t indexEntries=0;
size_t dataSize=0;
char buffer[10];
for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=cellOffsets.begin();
cell!=cellOffsets.end();
++cell) {
indexEntries+=cell->second.size();
dataSize+=EncodeNumber(cell->second.size(),buffer);
FileOffset previousOffset=0;
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
FileOffset data=*offset-previousOffset;
dataSize+=EncodeNumber(data,buffer);
previousOffset=*offset;
}
}
data.dataOffsetBytes=BytesNeededToEncodeNumber(dataSize);
progress.Info("Writing map for level "+
NumberToString(data.optLevel)+", index level "+
NumberToString(data.indexLevel)+", "+
NumberToString(cellOffsets.size())+" cells, "+
NumberToString(indexEntries)+" entries, "+
ByteSizeToString(1.0*data.cellXCount*data.cellYCount*data.dataOffsetBytes+dataSize));
data.bitmapOffset=writer.GetPos();
// Write the bitmap with offsets for each cell
// We prefill with zero and only overrite cells that have data
// So zero means "no data for this cell"
for (size_t i=0; i<data.cellXCount*data.cellYCount; i++) {
FileOffset cellOffset=0;
writer.WriteFileOffset(cellOffset,
data.dataOffsetBytes);
}
FileOffset dataStartOffset;
dataStartOffset=writer.GetPos();
// Now write the list of offsets of objects for every cell with content
for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=cellOffsets.begin();
cell!=cellOffsets.end();
++cell) {
FileOffset bitmapCellOffset=data.bitmapOffset+
((cell->first.y-data.cellYStart)*data.cellXCount+
cell->first.x-data.cellXStart)*data.dataOffsetBytes;
FileOffset previousOffset=0;
FileOffset cellOffset;
cellOffset=writer.GetPos();
writer.SetPos(bitmapCellOffset);
writer.WriteFileOffset(cellOffset-dataStartOffset,
data.dataOffsetBytes);
writer.SetPos(cellOffset);
writer.WriteNumber((uint32_t)cell->second.size());
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
writer.WriteNumber((FileOffset)(*offset-previousOffset));
previousOffset=*offset;
}
}
return true;
}
bool AreaNodeIndexGenerator::Import(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig)
{
FileScanner nodeScanner;
FileWriter writer;
std::set<TypeId> remainingNodeTypes; //! Set of types we still must process
std::vector<TypeData> nodeTypeData;
size_t level;
size_t maxLevel=0;
nodeTypeData.resize(typeConfig.GetTypes().size());
if (!nodeScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"nodes.dat"),
FileScanner::Sequential,
true)) {
progress.Error("Cannot open 'nodes.dat'");
return false;
}
//
// Scanning distribution
//
progress.SetAction("Scanning level distribution of node types");
// Initially we must process all types that represents nodes and that should
// not be ignored
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
!typeConfig.GetTypeInfo(i).GetIgnore()) {
remainingNodeTypes.insert(i);
}
}
level=parameter.GetAreaNodeMinMag();
while (!remainingNodeTypes.empty()) {
uint32_t nodeCount=0;
std::set<TypeId> currentNodeTypes(remainingNodeTypes);
double cellWidth=360.0/pow(2.0,(int)level);
double cellHeight=180.0/pow(2.0,(int)level);
std::vector<std::map<Pixel,size_t> > cellFillCount;
cellFillCount.resize(typeConfig.GetTypes().size());
progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingNodeTypes.size())+" types still to process)");
nodeScanner.GotoBegin();
if (!nodeScanner.Read(nodeCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
for (uint32_t n=1; n<=nodeCount; n++) {
progress.SetProgress(n,nodeCount);
FileOffset offset;
Node node;
nodeScanner.GetPos(offset);
if (!node.Read(nodeScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(n)+" of "+
NumberToString(nodeCount)+
" in file '"+
nodeScanner.GetFilename()+"'");
return false;
}
// If we still need to handle this type,
// count number of entries per type and tile cell
if (currentNodeTypes.find(node.GetType())!=currentNodeTypes.end()) {
uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth);
uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight);
cellFillCount[node.GetType()][Pixel(xc,yc)]++;
}
}
// Check statistics for each type
// If statistics are within goal limits, use this level
// for this type (else try again with the next higher level)
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (currentNodeTypes.find(i)!=currentNodeTypes.end()) {
size_t entryCount=0;
size_t max=0;
nodeTypeData[i].indexLevel=(uint32_t)level;
nodeTypeData[i].indexCells=cellFillCount[i].size();
nodeTypeData[i].indexEntries=0;
if (!cellFillCount[i].empty()) {
nodeTypeData[i].cellXStart=cellFillCount[i].begin()->first.x;
nodeTypeData[i].cellYStart=cellFillCount[i].begin()->first.y;
nodeTypeData[i].cellXEnd=nodeTypeData[i].cellXStart;
nodeTypeData[i].cellYEnd=nodeTypeData[i].cellYStart;
for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin();
cell!=cellFillCount[i].end();
++cell) {
nodeTypeData[i].indexEntries+=cell->second;
nodeTypeData[i].cellXStart=std::min(nodeTypeData[i].cellXStart,cell->first.x);
nodeTypeData[i].cellXEnd=std::max(nodeTypeData[i].cellXEnd,cell->first.x);
nodeTypeData[i].cellYStart=std::min(nodeTypeData[i].cellYStart,cell->first.y);
nodeTypeData[i].cellYEnd=std::max(nodeTypeData[i].cellYEnd,cell->first.y);
}
}
nodeTypeData[i].cellXCount=nodeTypeData[i].cellXEnd-nodeTypeData[i].cellXStart+1;
nodeTypeData[i].cellYCount=nodeTypeData[i].cellYEnd-nodeTypeData[i].cellYStart+1;
// Count absolute number of entries
for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin();
cell!=cellFillCount[i].end();
++cell) {
entryCount+=cell->second;
max=std::max(max,cell->second);
}
// Average number of entries per tile cell
double average=entryCount*1.0/cellFillCount[i].size();
// If we do not have any entries, we store it now
if (cellFillCount[i].empty()) {
continue;
}
// If the fill rate of the index is too low, we use this index level anyway
if (nodeTypeData[i].indexCells/(1.0*nodeTypeData[i].cellXCount*nodeTypeData[i].cellYCount)<=
parameter.GetAreaNodeIndexMinFillRate()) {
progress.Warning(typeConfig.GetTypeInfo(i).GetName()+" ("+NumberToString(i)+") is not well distributed");
continue;
}
// If average fill size and max fill size for tile cells
// is within limits, store it now.
if (max<=parameter.GetAreaNodeIndexCellSizeMax() &&
average<=parameter.GetAreaNodeIndexCellSizeAverage()) {
continue;
}
// else, we remove it from the list and try again with an higher
// level.
currentNodeTypes.erase(i);
}
}
// Now process all types for this limit, that are within the limits
for (std::set<TypeId>::const_iterator cnt=currentNodeTypes.begin();
cnt!=currentNodeTypes.end();
cnt++) {
maxLevel=std::max(maxLevel,level);
progress.Info("Type "+typeConfig.GetTypeInfo(*cnt).GetName()+"(" + NumberToString(*cnt)+"), "+NumberToString(nodeTypeData[*cnt].indexCells)+" cells, "+NumberToString(nodeTypeData[*cnt].indexEntries)+" objects");
remainingNodeTypes.erase(*cnt);
}
level++;
}
//
// Writing index file
//
progress.SetAction("Generating 'areanode.idx'");
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areanode.idx"))) {
progress.Error("Cannot create 'areanode.idx'");
return false;
}
uint32_t indexEntries=0;
// Count number of types in index
for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
{
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
nodeTypeData[i].HasEntries()) {
indexEntries++;
}
}
writer.Write(indexEntries);
// Store index data for each type
for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
{
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
nodeTypeData[i].HasEntries()) {
FileOffset bitmapOffset=0;
uint8_t dataOffsetBytes=0;
writer.WriteNumber(typeConfig.GetTypeInfo(i).GetId());
writer.GetPos(nodeTypeData[i].indexOffset);
writer.WriteFileOffset(bitmapOffset);
writer.Write(dataOffsetBytes);
writer.WriteNumber(nodeTypeData[i].indexLevel);
writer.WriteNumber(nodeTypeData[i].cellXStart);
writer.WriteNumber(nodeTypeData[i].cellXEnd);
writer.WriteNumber(nodeTypeData[i].cellYStart);
writer.WriteNumber(nodeTypeData[i].cellYEnd);
}
}
// Now store index bitmap for each type in increasing level order (why?)
for (size_t l=0; l<=maxLevel; l++) {
std::set<TypeId> indexTypes;
uint32_t nodeCount;
double cellWidth=360.0/pow(2.0,(int)l);
double cellHeight=180.0/pow(2.0,(int)l);
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
nodeTypeData[i].HasEntries() &&
nodeTypeData[i].indexLevel==l) {
indexTypes.insert(i);
}
}
if (indexTypes.empty()) {
continue;
}
progress.Info("Scanning nodes for index level "+NumberToString(l));
std::vector<std::map<Pixel,std::list<FileOffset> > > typeCellOffsets;
typeCellOffsets.resize(typeConfig.GetTypes().size());
nodeScanner.GotoBegin();
if (!nodeScanner.Read(nodeCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
//
// Collect all offsets
//
for (uint32_t n=1; n<=nodeCount; n++) {
progress.SetProgress(n,nodeCount);
FileOffset offset;
Node node;
nodeScanner.GetPos(offset);
if (!node.Read(nodeScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(n)+" of "+
NumberToString(nodeCount)+
" in file '"+
nodeScanner.GetFilename()+"'");
return false;
}
if (indexTypes.find(node.GetType())!=indexTypes.end()) {
uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth);
uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight);
typeCellOffsets[node.GetType()][Pixel(xc,yc)].push_back(offset);
}
}
//
// Write bitmap
//
for (std::set<TypeId>::const_iterator type=indexTypes.begin();
type!=indexTypes.end();
++type) {
size_t indexEntries=0;
size_t dataSize=0;
char buffer[10];
for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin();
cell!=typeCellOffsets[*type].end();
++cell) {
indexEntries+=cell->second.size();
dataSize+=EncodeNumber(cell->second.size(),buffer);
FileOffset previousOffset=0;
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
FileOffset data=*offset-previousOffset;
dataSize+=EncodeNumber(data,buffer);
previousOffset=*offset;
}
}
// "+1" because we add +1 to every offset, to generate offset > 0
uint8_t dataOffsetBytes=BytesNeededToAddressFileData(dataSize);
progress.Info("Writing map for "+
typeConfig.GetTypeInfo(*type).GetName()+", "+
NumberToString(typeCellOffsets[*type].size())+" cells, "+
NumberToString(indexEntries)+" entries, "+
ByteSizeToString(1.0*dataOffsetBytes*nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount));
FileOffset bitmapOffset;
if (!writer.GetPos(bitmapOffset)) {
progress.Error("Cannot get type index start position in file");
return false;
}
assert(nodeTypeData[*type].indexOffset!=0);
if (!writer.SetPos(nodeTypeData[*type].indexOffset)) {
progress.Error("Cannot go to type index offset in file");
return false;
}
writer.WriteFileOffset(bitmapOffset);
writer.Write(dataOffsetBytes);
if (!writer.SetPos(bitmapOffset)) {
progress.Error("Cannot go to type index start position in file");
return false;
}
// Write the bitmap with offsets for each cell
// We prefill with zero and only overwrite cells that have data
// So zero means "no data for this cell"
for (size_t i=0; i<nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount; i++) {
FileOffset cellOffset=0;
writer.WriteFileOffset(cellOffset,
dataOffsetBytes);
}
FileOffset dataStartOffset;
if (!writer.GetPos(dataStartOffset)) {
progress.Error("Cannot get start of data section after bitmap");
return false;
}
// Now write the list of offsets of objects for every cell with content
for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin();
cell!=typeCellOffsets[*type].end();
++cell) {
FileOffset bitmapCellOffset=bitmapOffset+
((cell->first.y-nodeTypeData[*type].cellYStart)*nodeTypeData[*type].cellXCount+
cell->first.x-nodeTypeData[*type].cellXStart)*dataOffsetBytes;
FileOffset previousOffset=0;
FileOffset cellOffset;
if (!writer.GetPos(cellOffset)) {
progress.Error("Cannot get cell start position in file");
return false;
}
if (!writer.SetPos(bitmapCellOffset)) {
progress.Error("Cannot go to cell start position in file");
return false;
}
writer.WriteFileOffset(cellOffset-dataStartOffset+1,
dataOffsetBytes);
if (!writer.SetPos(cellOffset)) {
progress.Error("Cannot go back to cell start position in file");
return false;
}
writer.WriteNumber((uint32_t)cell->second.size());
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
writer.WriteNumber((FileOffset)(*offset-previousOffset));
previousOffset=*offset;
}
}
}
}
return !writer.HasError() && writer.Close();
}
bool AreaWayIndexGenerator::WriteBitmap(Progress& progress,
FileWriter& writer,
const TypeInfo& typeInfo,
const TypeData& typeData,
const CoordOffsetsMap& typeCellOffsets)
{
size_t indexEntries=0;
size_t dataSize=0;
char buffer[10];
for (CoordOffsetsMap::const_iterator cell=typeCellOffsets.begin();
cell!=typeCellOffsets.end();
++cell) {
indexEntries+=cell->second.size();
dataSize+=EncodeNumber(cell->second.size(),buffer);
FileOffset previousOffset=0;
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
FileOffset data=*offset-previousOffset;
dataSize+=EncodeNumber(data,buffer);
previousOffset=*offset;
}
}
// "+1" because we add +1 to every offset, to generate offset > 0
uint8_t dataOffsetBytes=BytesNeededToAddressFileData(dataSize+1);
progress.Info("Writing map for "+
typeInfo.GetName()+" , "+
ByteSizeToString(1.0*dataOffsetBytes*typeData.cellXCount*typeData.cellYCount+dataSize));
FileOffset bitmapOffset;
if (!writer.GetPos(bitmapOffset)) {
progress.Error("Cannot get type index start position in file");
return false;
}
assert(typeData.indexOffset!=0);
if (!writer.SetPos(typeData.indexOffset)) {
progress.Error("Cannot go to type index offset in file");
return false;
}
writer.WriteFileOffset(bitmapOffset);
writer.Write(dataOffsetBytes);
if (!writer.SetPos(bitmapOffset)) {
progress.Error("Cannot go to type index start position in file");
return false;
}
// Write the bitmap with offsets for each cell
// We prefill with zero and only overwrite cells that have data
// So zero means "no data for this cell"
for (size_t i=0; i<typeData.cellXCount*typeData.cellYCount; i++) {
writer.WriteFileOffset(0,
dataOffsetBytes);
}
FileOffset dataStartOffset;
if (!writer.GetPos(dataStartOffset)) {
progress.Error("Cannot get start of data section after bitmap");
return false;
}
// Now write the list of offsets of objects for every cell with content
for (CoordOffsetsMap::const_iterator cell=typeCellOffsets.begin();
cell!=typeCellOffsets.end();
++cell) {
FileOffset bitmapCellOffset=bitmapOffset+
((cell->first.y-typeData.cellYStart)*typeData.cellXCount+
cell->first.x-typeData.cellXStart)*(FileOffset)dataOffsetBytes;
FileOffset previousOffset=0;
FileOffset cellOffset;
assert(bitmapCellOffset>=bitmapOffset);
if (!writer.GetPos(cellOffset)) {
progress.Error("Cannot get cell start position in file");
return false;
}
if (!writer.SetPos(bitmapCellOffset)) {
progress.Error("Cannot go to cell start position in file");
return false;
}
assert(cellOffset>bitmapCellOffset);
// We add +1 to make sure, that we can differentiate between "0" as "no entry" and "0" as first data entry.
writer.WriteFileOffset(cellOffset-dataStartOffset+1,dataOffsetBytes);
if (!writer.SetPos(cellOffset)) {
progress.Error("Cannot go back to cell start position in file");
return false;
}
writer.WriteNumber((uint32_t)cell->second.size());
// FileOffsets are already in increasing order, since
// File is scanned from start to end
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
writer.WriteNumber((FileOffset)(*offset-previousOffset));
previousOffset=*offset;
}
}
return true;
}
