void PerfTest::RunRecvBufferSizeTests(SecurityProvider::Enum secProvider) { console.WriteLine("========================================================="); console.WriteLine("security provider = {0}", secProvider); console.WriteLine(" test data total in bytes: {0}", testDataSize_); console.WriteLine("========================================================="); for (uint clientThreadCount = clientThreadMin_; clientThreadCount <= clientThreadMax_; ++clientThreadCount) { wstring outputFile = wformatString( "PerfTest-QueueReceived@{0}_ClientThread@{1}_Sec@{2}.csv", shouldQueueReceivedMessage_, clientThreadCount, secProvider); console.WriteLine("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++"); console.WriteLine("client thread count = {0}", clientThreadCount); console.WriteLine("ShouldQueueReceivedMessage = {0}", shouldQueueReceivedMessage_); console.WriteLine("output = {0}", outputFile); console.WriteLine("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++"); FileWriter csvFile; auto error = csvFile.TryOpen(outputFile); Invariant(error.IsSuccess()); KFinally([&] { csvFile.Close(); }); for (uint messageSize = messageSizeMin_; messageSize <= messageSizeMax_; messageSize *= 2) { csvFile.Write("{0}", messageSize); map<TimeSpan, uint> resultMap; for (uint tcpBufferSize = tcpBufferSizeMin_; tcpBufferSize <= tcpBufferSizeMax_; tcpBufferSize += 4*1024) { TransportConfig::GetConfig().TcpReceiveBufferSize = tcpBufferSize; console.WriteLine("---------------------------------------------------------"); console.WriteLine("tcp receive buffer = {0}", TransportConfig::GetConfig().TcpReceiveBufferSize); console.WriteLine("---------------------------------------------------------"); //csvFile.Write("{0}", tcpBufferSize); PerfTest perfTest(secProvider, messageSize, clientThreadCount); perfTest.Run(csvFile); resultMap.emplace(make_pair(perfTest.GetTestDuration(), tcpBufferSize)); } // record tcpBufferSize in KB, sorted by test duration console.WriteLine("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$"); console.WriteLine("TCP buffer sizes sorted by test duration:"); auto optimalBufferSize = resultMap.cbegin()->second / 1024; for (auto const & entry : resultMap) { auto bufferSize = entry.second/1024; csvFile.Write(",{0}", bufferSize); console.WriteLine("{0}", bufferSize); } csvFile.WriteLine(); csvFile.Flush(); console.WriteLine("\n@_@ optimal buffer size = {0} @_@", optimalBufferSize); console.WriteLine("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$"); } } }
bool RouteNode::Write(FileWriter& writer) const { writer.WriteNumber(id); writer.WriteCoord(coord); writer.WriteNumber((uint32_t)objects.size()); writer.WriteNumber((uint32_t)paths.size()); writer.WriteNumber((uint32_t)excludes.size()); Id lastFileOffset=0; for (const auto& object : objects) { FileOffset offset=object.object.GetFileOffset()-lastFileOffset; if (object.object.GetType()==refWay) { offset=offset*2; } else if (object.object.GetType()==refArea) { offset=offset*2+1; } else { assert(false); } writer.WriteNumber(offset); writer.WriteNumber(object.type); writer.Write(object.maxSpeed); writer.Write(object.grade); lastFileOffset=object.object.GetFileOffset(); } if (!paths.empty()) { for (const auto& path : paths) { writer.WriteFileOffset(path.offset); writer.WriteNumber(path.objectIndex); //writer.Write(paths[i].bearing); writer.Write(path.flags); writer.WriteNumber((uint32_t)floor(path.distance*(1000.0*100.0)+0.5)); } } for (const auto& exclude : excludes) { writer.Write(exclude.source); writer.WriteNumber(exclude.targetIndex); } return !writer.HasError(); }
bool Area::WriteOptimized(const TypeConfig& typeConfig, FileWriter& writer) const { std::vector<Ring>::const_iterator ring=rings.begin(); bool multipleRings=rings.size()>1; // Outer ring writer.WriteTypeId(ring->GetType()->GetAreaId(), typeConfig.GetAreaTypeIdBytes()); if (!ring->featureValueBuffer.Write(writer, multipleRings)) { return false; } if (multipleRings) { writer.WriteNumber((uint32_t)(rings.size()-1)); } if (!writer.Write(ring->nodes)) { return false; } ++ring; // Potential additional rings while (ring!=rings.end()) { writer.WriteTypeId(ring->GetType()->GetAreaId(), typeConfig.GetAreaTypeIdBytes()); if (ring->GetType()->GetAreaId()!=typeIgnore) { if (!ring->featureValueBuffer.Write(writer)) { return false; } } writer.Write(ring->ring); if (!writer.Write(ring->nodes)) { return false; } ++ring; } return !writer.HasError(); }
bool Way::Write(const TypeConfig& typeConfig, FileWriter& writer) const { assert(!nodes.empty()); writer.WriteTypeId(featureValueBuffer.GetType()->GetWayId(), typeConfig.GetWayTypeIdBytes()); if (!featureValueBuffer.Write(writer)) { return false; } if (!writer.Write(nodes)) { return false; } if (featureValueBuffer.GetType()->CanRoute() || featureValueBuffer.GetType()->GetOptimizeLowZoom()) { if (!WriteIds(writer)) { return false; } } return !writer.HasError(); }
bool OptimizeAreasLowZoomGenerator::WriteHeader(FileWriter& writer, const std::list<TypeData>& areaTypesData, uint32_t optimizeMaxMap) { writer.Write(optimizeMaxMap); writer.Write((uint32_t)areaTypesData.size()); for (const auto &typeData : areaTypesData) { if (!WriteTypeData(writer, typeData)) { return false; } } return true; }
Kumu::Result_t Kumu::WriteObjectIntoFile(const Kumu::IArchive& Object, const std::string& Filename) { ByteString Buffer; Result_t result = Buffer.Capacity(Object.ArchiveLength()); if ( KM_SUCCESS(result) ) { ui32_t write_count = 0; FileWriter Writer; MemIOWriter MemWriter(&Buffer); result = Object.Archive(&MemWriter) ? RESULT_OK : RESULT_WRITEFAIL; if ( KM_SUCCESS(result) ) { Buffer.Length(MemWriter.Length()); result = Writer.OpenWrite(Filename); } if ( KM_SUCCESS(result) ) result = Writer.Write(Buffer.RoData(), Buffer.Length(), &write_count); } return result; }
bool WayLocationProcessorFilter::AfterProcessingEnd(const ImportParameter& /*parameter*/, Progress& progress, const TypeConfig& /*typeConfig*/) { delete nameReader; nameReader=NULL; delete locationReader; locationReader=NULL; writer.SetPos(0); writer.Write(overallDataCount); try { writer.Close(); } catch (IOException& e) { progress.Error(e.GetDescription()); writer.CloseFailsafe(); return false; } return true; }
bool Relation::Write(FileWriter& writer) const { writer.Write(id); if (!attributes.Write(writer)) { return false; } writer.WriteNumber((uint32_t)roles.size()); for (size_t i=0; i<roles.size(); i++) { if (!roles[i].attributes.Write(writer)) { return false; } writer.Write(roles[i].role); writer.WriteNumber((uint32_t)roles[i].nodes.size()); if (!roles[i].nodes.empty()) { Id minId=std::numeric_limits<Id>::max(); uint32_t minLat=std::numeric_limits<uint32_t>::max(); uint32_t minLon=std::numeric_limits<uint32_t>::max(); for (size_t j=0; j<roles[i].nodes.size(); j++) { minId=std::min(minId,roles[i].nodes[j].GetId()); minLat=std::min(minLat,(uint32_t)floor((roles[i].nodes[j].lat+90.0)*conversionFactor+0.5)); minLon=std::min(minLon,(uint32_t)floor((roles[i].nodes[j].lon+180.0)*conversionFactor+0.5)); } writer.Write(minId); writer.Write(minLat); writer.Write(minLon); for (size_t j=0; j<roles[i].nodes.size(); j++) { uint32_t latValue=(uint32_t)floor((roles[i].nodes[j].lat+90.0)*conversionFactor+0.5); uint32_t lonValue=(uint32_t)floor((roles[i].nodes[j].lon+180.0)*conversionFactor+0.5); writer.WriteNumber(roles[i].nodes[j].id-minId); writer.WriteNumber(latValue-minLat); writer.WriteNumber(lonValue-minLon); } } } return !writer.HasError(); }
bool Preprocess::Callback::DumpDistribution() { FileWriter writer; progress.SetAction("Writing 'distribution.dat'"); if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "distribution.dat"))) { return false; } for (const auto &type : typeConfig->GetTypes()) { writer.Write(nodeStat[type->GetIndex()]); writer.Write(wayStat[type->GetIndex()]); writer.Write(areaStat[type->GetIndex()]); } return writer.Close(); }
bool WayLocationProcessorFilter::Process(Progress& progress, const FileOffset& offset, Way& way, bool& /*save*/) { try { if (!way.GetType()->GetIndexAsPOI()) { return true; } NameFeatureValue *nameValue=nameReader->GetValue(way.GetFeatureValueBuffer()); if (nameValue==NULL) { return true; } LocationFeatureValue *locationValue=locationReader->GetValue(way.GetFeatureValueBuffer()); std::string name; std::string location; std::string address; name=nameValue->GetName(); if (locationValue!=NULL) { location=locationValue->GetLocation(); } writer.WriteFileOffset(offset); writer.WriteNumber(way.GetType()->GetWayId()); writer.Write(name); writer.Write(location); writer.Write(way.nodes); overallDataCount++; } catch (IOException& e) { progress.Error(e.GetDescription()); return false; } return true; }
Kumu::Result_t Kumu::WriteStringIntoFile(const std::string& filename, const std::string& inString) { FileWriter File; ui32_t write_count = 0; Result_t result = File.OpenWrite(filename); if ( KM_SUCCESS(result) ) result = File.Write((byte_t*)inString.c_str(), inString.length(), &write_count); return result; }
bool Area::WriteIds(FileWriter& writer, const std::vector<Id>& ids) const { Id minId=0; for (size_t i=0; i<ids.size(); i++) { if (ids[i]!=0) { if (minId==0) { minId=ids[i]; } else { minId=std::min(minId,ids[i]); } } } writer.WriteNumber(minId); if (minId>0) { size_t idCurrent=0; while (idCurrent<ids.size()) { uint8_t bitset=0; uint8_t bitMask=1; size_t idEnd=std::min(idCurrent+8,ids.size()); for (size_t i=idCurrent; i<idEnd; i++) { if (ids[i]!=0) { bitset=bitset | bitMask; } bitMask*=2; } writer.Write(bitset); for (size_t i=idCurrent; i<idEnd; i++) { if (ids[i]!=0) { writer.WriteNumber(ids[i]-minId); } bitMask=bitMask*2; } idCurrent+=8; } } return !writer.HasError(); }
void PerfTest::WaitForResult(FileWriter & fw) { Invariant(allReceived_.WaitOne(testTimeout_)); auto elapsedMilliseconds = stopwatch_.ElapsedMilliseconds; console.WriteLine(">>> time elapsed: {0} ms", elapsedMilliseconds); auto totalReceivedBytes = recvBytes_.load(); auto recvRate = (totalReceivedBytes * 8.0) / elapsedMilliseconds / 1000.0; console.WriteLine(">>> received: {0} bytes", totalReceivedBytes); console.WriteLine(">>> receive rate: {0} mbps\n\n", recvRate); fw.Write(",{0},{1}", elapsedMilliseconds, recvRate); listener_->Stop(); listener_.reset(); }
Result_t Kumu::WriteBufferIntoFile(const Kumu::ByteString& Buffer, const std::string& Filename) { ui32_t write_count = 0; FileWriter Writer; Result_t result = Writer.OpenWrite(Filename); if ( KM_SUCCESS(result) ) result = Writer.Write(Buffer.RoData(), Buffer.Length(), &write_count); if ( KM_SUCCESS(result) && Buffer.Length() != write_count) return RESULT_WRITEFAIL; return result; }
bool OptimizeAreasLowZoomGenerator::WriteTypeData(FileWriter& writer, const TypeData& data) { assert(data.type); writer.Write(data.type->GetAreaId()); writer.Write(data.optLevel); writer.Write(data.indexLevel); writer.Write(data.cellXStart); writer.Write(data.cellXEnd); writer.Write(data.cellYStart); writer.Write(data.cellYEnd); writer.WriteFileOffset(data.bitmapOffset); writer.Write(data.dataOffsetBytes); return !writer.HasError(); }
bool WayLocationProcessorFilter::BeforeProcessingStart(const ImportParameter& parameter, Progress& progress, const TypeConfig& typeConfig) { overallDataCount=0; nameReader=new NameFeatureValueReader(typeConfig); locationReader=new LocationFeatureValueReader(typeConfig); try { writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(), SortWayDataGenerator::WAYADDRESS_DAT)); writer.Write(overallDataCount); } catch (IOException& e) { progress.Error(e.GetDescription()); return false; } return true; }
bool NodeDataGenerator::Import(const TypeConfigRef& typeConfig, const ImportParameter& parameter, Progress& progress) { uint32_t rawNodeCount=0; uint32_t nodesReadCount=0; uint32_t nodesWrittenCount=0; // // Iterator over all raw nodes, hcekc they type, and convert them from raw nodes // to nodes if the type is interesting (!=typeIgnore). // // Count the bounding box by the way... // progress.SetAction("Generating nodes.tmp"); FileScanner scanner; FileWriter writer; if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "rawnodes.dat"), FileScanner::Sequential, parameter.GetRawNodeDataMemoryMaped())) { progress.Error("Cannot open 'rawnodes.dat'"); return false; } if (!scanner.Read(rawNodeCount)) { progress.Error("Error while reading number of data entries in file"); return false; } if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "nodes.tmp"))) { progress.Error("Cannot create 'nodes.tmp'"); return false; } writer.Write(nodesWrittenCount); for (uint32_t n=1; n<=rawNodeCount; n++) { progress.SetProgress(n,rawNodeCount); RawNode rawNode; Node node; if (!rawNode.Read(typeConfig, scanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(n)+" of "+ NumberToString(rawNodeCount)+ " in file '"+ scanner.GetFilename()+"'"); return false; } nodesReadCount++; if (!rawNode.GetType()->GetIgnore()) { node.SetFeatures(rawNode.GetFeatureValueBuffer()); node.SetCoords(rawNode.GetCoords()); FileOffset fileOffset; if (!writer.GetPos(fileOffset)) { progress.Error(std::string("Error while reading current fileOffset in file '")+ writer.GetFilename()+"'"); return false; } writer.Write(rawNode.GetId()); node.Write(typeConfig, writer); nodesWrittenCount++; } } if (!scanner.Close()) { return false; } writer.SetPos(0); writer.Write(nodesWrittenCount); if (!writer.Close()) { return false; } progress.Info(std::string("Read "+NumberToString(nodesReadCount)+" nodes, wrote "+NumberToString(nodesWrittenCount)+" nodes")); return true; }
bool AreaWayIndexGenerator::WriteBitmap(Progress& progress, FileWriter& writer, const TypeInfo& typeInfo, const TypeData& typeData, const CoordOffsetsMap& typeCellOffsets) { size_t indexEntries=0; size_t dataSize=0; char buffer[10]; for (CoordOffsetsMap::const_iterator cell=typeCellOffsets.begin(); cell!=typeCellOffsets.end(); ++cell) { indexEntries+=cell->second.size(); dataSize+=EncodeNumber(cell->second.size(),buffer); FileOffset previousOffset=0; for (std::list<FileOffset>::const_iterator offset=cell->second.begin(); offset!=cell->second.end(); ++offset) { FileOffset data=*offset-previousOffset; dataSize+=EncodeNumber(data,buffer); previousOffset=*offset; } } // "+1" because we add +1 to every offset, to generate offset > 0 uint8_t dataOffsetBytes=BytesNeededToAddressFileData(dataSize+1); progress.Info("Writing map for "+ typeInfo.GetName()+" , "+ ByteSizeToString(1.0*dataOffsetBytes*typeData.cellXCount*typeData.cellYCount+dataSize)); FileOffset bitmapOffset; if (!writer.GetPos(bitmapOffset)) { progress.Error("Cannot get type index start position in file"); return false; } assert(typeData.indexOffset!=0); if (!writer.SetPos(typeData.indexOffset)) { progress.Error("Cannot go to type index offset in file"); return false; } writer.WriteFileOffset(bitmapOffset); writer.Write(dataOffsetBytes); if (!writer.SetPos(bitmapOffset)) { progress.Error("Cannot go to type index start position in file"); return false; } // Write the bitmap with offsets for each cell // We prefill with zero and only overwrite cells that have data // So zero means "no data for this cell" for (size_t i=0; i<typeData.cellXCount*typeData.cellYCount; i++) { writer.WriteFileOffset(0, dataOffsetBytes); } FileOffset dataStartOffset; if (!writer.GetPos(dataStartOffset)) { progress.Error("Cannot get start of data section after bitmap"); return false; } // Now write the list of offsets of objects for every cell with content for (CoordOffsetsMap::const_iterator cell=typeCellOffsets.begin(); cell!=typeCellOffsets.end(); ++cell) { FileOffset bitmapCellOffset=bitmapOffset+ ((cell->first.y-typeData.cellYStart)*typeData.cellXCount+ cell->first.x-typeData.cellXStart)*(FileOffset)dataOffsetBytes; FileOffset previousOffset=0; FileOffset cellOffset; assert(bitmapCellOffset>=bitmapOffset); if (!writer.GetPos(cellOffset)) { progress.Error("Cannot get cell start position in file"); return false; } if (!writer.SetPos(bitmapCellOffset)) { progress.Error("Cannot go to cell start position in file"); return false; } assert(cellOffset>bitmapCellOffset); // We add +1 to make sure, that we can differentiate between "0" as "no entry" and "0" as first data entry. writer.WriteFileOffset(cellOffset-dataStartOffset+1,dataOffsetBytes); if (!writer.SetPos(cellOffset)) { progress.Error("Cannot go back to cell start position in file"); return false; } writer.WriteNumber((uint32_t)cell->second.size()); // FileOffsets are already in increasing order, since // File is scanned from start to end for (std::list<FileOffset>::const_iterator offset=cell->second.begin(); offset!=cell->second.end(); ++offset) { writer.WriteNumber((FileOffset)(*offset-previousOffset)); previousOffset=*offset; } } return true; }
bool AreaWayIndexGenerator::Import(const ImportParameter& parameter, Progress& progress, const TypeConfig& typeConfig) { FileScanner wayScanner; FileWriter writer; std::set<TypeId> remainingWayTypes; std::vector<TypeData> wayTypeData; size_t level; size_t maxLevel=0; wayTypeData.resize(typeConfig.GetTypes().size()); if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "ways.dat"), FileScanner::Sequential, parameter.GetWayDataMemoryMaped())) { progress.Error("Cannot open 'ways.dat'"); return false; } // // Scanning distribution // progress.SetAction("Scanning level distribution of way types"); for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeWay() && !typeConfig.GetTypeInfo(i).GetIgnore()) { remainingWayTypes.insert(i); } } level=parameter.GetAreaWayMinMag(); while (!remainingWayTypes.empty()) { uint32_t wayCount=0; std::set<TypeId> currentWayTypes(remainingWayTypes); double cellWidth=360.0/pow(2.0,(int)level); double cellHeight=180.0/pow(2.0,(int)level); std::vector<CoordCountMap> cellFillCount(typeConfig.GetTypes().size()); progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingWayTypes.size())+" types remaining)"); wayScanner.GotoBegin(); if (!wayScanner.Read(wayCount)) { progress.Error("Error while reading number of data entries in file"); return false; } Way way; for (uint32_t w=1; w<=wayCount; w++) { progress.SetProgress(w,wayCount); if (!way.Read(wayScanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(w)+" of "+ NumberToString(wayCount)+ " in file '"+ wayScanner.GetFilename()+"'"); return false; } // Count number of entries per current type and coordinate if (currentWayTypes.find(way.GetType())==currentWayTypes.end()) { continue; } double minLon; double maxLon; double minLat; double maxLat; way.GetBoundingBox(minLon,maxLon,minLat,maxLat); // // Calculate minimum and maximum tile ids that are covered // by the way // Renormated coordinate space (everything is >=0) // uint32_t minxc=(uint32_t)floor((minLon+180.0)/cellWidth); uint32_t maxxc=(uint32_t)floor((maxLon+180.0)/cellWidth); uint32_t minyc=(uint32_t)floor((minLat+90.0)/cellHeight); uint32_t maxyc=(uint32_t)floor((maxLat+90.0)/cellHeight); for (uint32_t y=minyc; y<=maxyc; y++) { for (uint32_t x=minxc; x<=maxxc; x++) { cellFillCount[way.GetType()][Pixel(x,y)]++; } } } // Check if cell fill for current type is in defined limits for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (currentWayTypes.find(i)!=currentWayTypes.end()) { CalculateStatistics(level,wayTypeData[i],cellFillCount[i]); if (!FitsIndexCriteria(parameter, progress, typeConfig.GetTypeInfo(i), wayTypeData[i], cellFillCount[i])) { currentWayTypes.erase(i); } } } for (std::set<TypeId>::const_iterator cwt=currentWayTypes.begin(); cwt!=currentWayTypes.end(); cwt++) { maxLevel=std::max(maxLevel,level); progress.Info("Type "+typeConfig.GetTypeInfo(*cwt).GetName()+"(" + NumberToString(*cwt)+"), "+NumberToString(wayTypeData[*cwt].indexCells)+" cells, "+NumberToString(wayTypeData[*cwt].indexEntries)+" objects"); remainingWayTypes.erase(*cwt); } level++; } // // Writing index file // progress.SetAction("Generating 'areaway.idx'"); if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "areaway.idx"))) { progress.Error("Cannot create 'areaway.idx'"); return false; } uint32_t indexEntries=0; for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeWay() && wayTypeData[i].HasEntries()) { indexEntries++; } } writer.Write(indexEntries); for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeWay() && wayTypeData[i].HasEntries()) { uint8_t dataOffsetBytes=0; FileOffset bitmapOffset=0; writer.WriteNumber(typeConfig.GetTypeInfo(i).GetId()); writer.GetPos(wayTypeData[i].indexOffset); writer.WriteFileOffset(bitmapOffset); if (wayTypeData[i].HasEntries()) { writer.Write(dataOffsetBytes); writer.WriteNumber(wayTypeData[i].indexLevel); writer.WriteNumber(wayTypeData[i].cellXStart); writer.WriteNumber(wayTypeData[i].cellXEnd); writer.WriteNumber(wayTypeData[i].cellYStart); writer.WriteNumber(wayTypeData[i].cellYEnd); } } } for (size_t l=parameter.GetAreaWayMinMag(); l<=maxLevel; l++) { std::set<TypeId> indexTypes; uint32_t wayCount; double cellWidth=360.0/pow(2.0,(int)l); double cellHeight=180.0/pow(2.0,(int)l); for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeWay() && wayTypeData[i].HasEntries() && wayTypeData[i].indexLevel==l) { indexTypes.insert(i); } } if (indexTypes.empty()) { continue; } progress.Info("Scanning ways for index level "+NumberToString(l)); std::vector<CoordOffsetsMap> typeCellOffsets(typeConfig.GetTypes().size()); wayScanner.GotoBegin(); if (!wayScanner.Read(wayCount)) { progress.Error("Error while reading number of data entries in file"); return false; } Way way; for (uint32_t w=1; w<=wayCount; w++) { progress.SetProgress(w,wayCount); FileOffset offset; wayScanner.GetPos(offset); if (!way.Read(wayScanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(w)+" of "+ NumberToString(wayCount)+ " in file '"+ wayScanner.GetFilename()+"'"); return false; } if (indexTypes.find(way.GetType())==indexTypes.end()) { continue; } double minLon; double maxLon; double minLat; double maxLat; way.GetBoundingBox(minLon,maxLon,minLat,maxLat); // // Calculate minimum and maximum tile ids that are covered // by the way // Renormated coordinate space (everything is >=0) // uint32_t minxc=(uint32_t)floor((minLon+180.0)/cellWidth); uint32_t maxxc=(uint32_t)floor((maxLon+180.0)/cellWidth); uint32_t minyc=(uint32_t)floor((minLat+90.0)/cellHeight); uint32_t maxyc=(uint32_t)floor((maxLat+90.0)/cellHeight); for (uint32_t y=minyc; y<=maxyc; y++) { for (uint32_t x=minxc; x<=maxxc; x++) { typeCellOffsets[way.GetType()][Pixel(x,y)].push_back(offset); } } } for (std::set<TypeId>::const_iterator type=indexTypes.begin(); type!=indexTypes.end(); ++type) { if (!WriteBitmap(progress, writer, typeConfig.GetTypeInfo(*type), wayTypeData[*type], typeCellOffsets[*type])) { return false; } } } return !writer.HasError() && writer.Close(); }
bool MergeAreasGenerator::Import(const TypeConfigRef& typeConfig, const ImportParameter& parameter, Progress& progress) { TypeInfoSet mergeTypes; FileScanner scanner; FileWriter writer; uint32_t areasWritten=0; for (const auto& type : typeConfig->GetTypes()) { if (type->CanBeArea() && type->GetMergeAreas()) { mergeTypes.Set(type); } } std::unordered_set<Id> nodeUseMap; try { scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), MergeAreaDataGenerator::AREAS_TMP), FileScanner::Sequential, parameter.GetRawWayDataMemoryMaped()); if (!ScanAreaNodeIds(progress, *typeConfig, scanner, mergeTypes, nodeUseMap)) { return false; } uint32_t nodeCount=nodeUseMap.size(); progress.Info("Found "+NumberToString(nodeCount)+" nodes as possible connection points for areas"); /* ------ */ writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(), AREAS2_TMP)); writer.Write(areasWritten); while (true) { TypeInfoSet loadedTypes; std::vector<AreaMergeData> mergeJob(typeConfig->GetTypeCount()); // // Load type data // progress.SetAction("Collecting area data by type"); if (!GetAreas(parameter, progress, *typeConfig, mergeTypes, loadedTypes, nodeUseMap, scanner, writer, mergeJob, areasWritten)) { return false; } // Merge progress.SetAction("Merging areas"); for (const auto& type : loadedTypes) { if (!mergeJob[type->GetIndex()].areas.empty()) { progress.Info("Merging areas of type "+type->GetName()); MergeAreas(progress, nodeUseMap, mergeJob[type->GetIndex()]); progress.Info("Reduced areas of '"+type->GetName()+"' from "+NumberToString(mergeJob[type->GetIndex()].areaCount)+" to "+NumberToString(mergeJob[type->GetIndex()].areaCount-mergeJob[type->GetIndex()].mergedAway.size())); mergeJob[type->GetIndex()].areas.clear(); } } // Store back merge result if (!loadedTypes.Empty()) { if (!WriteMergeResult(progress, *typeConfig, scanner, writer, loadedTypes, mergeJob, areasWritten)) { return false; } mergeTypes.Remove(loadedTypes); } if (mergeTypes.Empty()) { break; } } scanner.Close(); writer.GotoBegin(); writer.Write(areasWritten); writer.Close(); } catch (IOException& e) { progress.Error(e.GetDescription()); scanner.CloseFailsafe(); writer.CloseFailsafe(); return false; } return true; }
void WriteRawData(FileWriter & writer, std::vector<uint8_t> const & rawData) { uint64_t const size = static_cast<uint64_t>(rawData.size()); WriteToSink(writer, size); writer.Write(rawData.data(), size); }
bool MergeAreasGenerator::WriteMergeResult(Progress& progress, const TypeConfig& typeConfig, FileScanner& scanner, FileWriter& writer, const TypeInfoSet& loadedTypes, std::vector<AreaMergeData>& mergeJob, uint32_t& areasWritten) { uint32_t areaCount=0; std::unordered_map<FileOffset,AreaRef> merges; std::unordered_set<FileOffset> ignores; for (const auto& type : loadedTypes) { for (const auto& area : mergeJob[type->GetIndex()].merges) { merges[area->GetFileOffset()]=area; } ignores.insert(mergeJob[type->GetIndex()].mergedAway.begin(), mergeJob[type->GetIndex()].mergedAway.end()); } scanner.GotoBegin(); scanner.Read(areaCount); for (uint32_t a=1; a<=areaCount; a++) { uint8_t type; Id id; AreaRef area=std::make_shared<Area>(); progress.SetProgress(a,areaCount); scanner.Read(type); scanner.Read(id); area->ReadImport(typeConfig, scanner); if (loadedTypes.IsSet(area->GetType())) { if (ignores.find(area->GetFileOffset())!=ignores.end()) { continue; } writer.Write(type); writer.Write(id); const auto& merge=merges.find(area->GetFileOffset()) ; if (merge!=merges.end()) { area=merge->second; } area->WriteImport(typeConfig, writer); areasWritten++; } } return true; }
/** * Load areas which has a one for the types given by types. If at leats one node * in one of the outer rings of the areas is marked in nodeUseMap as "used at least twice", * index it into the areas map. * * If the number of indexed areas is bigger than parameter.GetRawWayBlockSize() types are * dropped form areas until the number is again below the lmit. */ bool MergeAreasGenerator::GetAreas(const ImportParameter& parameter, Progress& progress, const TypeConfig& typeConfig, const TypeInfoSet& candidateTypes, TypeInfoSet& loadedTypes, const std::unordered_set<Id>& nodeUseMap, FileScanner& scanner, FileWriter& writer, std::vector<AreaMergeData>& mergeJob, uint32_t& areasWritten) { bool firstCall=areasWritten==0; // We are called for the first time uint32_t areaCount=0; size_t collectedAreasCount=0; size_t typesWithAreas=0; for (auto& data : mergeJob) { data.areaCount=0; } loadedTypes=candidateTypes; scanner.GotoBegin(); scanner.Read(areaCount); for (uint32_t a=1; a<=areaCount; a++) { uint8_t type; Id id; AreaRef area=std::make_shared<Area>(); progress.SetProgress(a,areaCount); scanner.Read(type); scanner.Read(id); area->ReadImport(typeConfig, scanner); mergeJob[area->GetType()->GetIndex()].areaCount++; // This is an area of a type that does not get merged, // we directly store it in the target file. if (!loadedTypes.IsSet(area->GetType())) { if (firstCall) { writer.Write(type); writer.Write(id); area->WriteImport(typeConfig, writer); areasWritten++; } continue; } bool isMergeCandidate=false; for (const auto& ring: area->rings) { if (!ring.IsOuterRing()) { continue; } for (const auto node : ring.nodes) { if (nodeUseMap.find(node.GetId())!=nodeUseMap.end()) { isMergeCandidate=true; break; } } if (isMergeCandidate) { break; } } if (!isMergeCandidate) { continue; } if (mergeJob[area->GetType()->GetIndex()].areas.empty()) { typesWithAreas++; } mergeJob[area->GetType()->GetIndex()].areas.push_back(area); collectedAreasCount++; while (collectedAreasCount>parameter.GetRawWayBlockSize() && typesWithAreas>1) { TypeInfoRef victimType; // Find the type with the smallest amount of ways loaded for (auto &type : loadedTypes) { if (!mergeJob[type->GetIndex()].areas.empty() && (!victimType || mergeJob[type->GetIndex()].areas.size()<mergeJob[victimType->GetIndex()].areas.size())) { victimType=type; } } // If there is more then one type of way, we always must find a "victim" type. assert(victimType); // Correct the statistics collectedAreasCount-=mergeJob[victimType->GetIndex()].areas.size(); // Clear already loaded data of th victim type mergeJob[victimType->GetIndex()].areas.clear(); typesWithAreas--; loadedTypes.Remove(victimType); } } progress.SetAction("Collected "+NumberToString(collectedAreasCount)+" areas for "+NumberToString(loadedTypes.Size())+" types"); return true; }
bool AreaWayIndexGenerator::Import(const TypeConfigRef& typeConfig, const ImportParameter& parameter, Progress& progress) { FileScanner wayScanner; FileWriter writer; std::vector<TypeData> wayTypeData; size_t maxLevel; progress.Info("Minimum magnification: "+NumberToString(parameter.GetAreaWayMinMag())); // // Scanning distribution // progress.SetAction("Scanning level distribution of way types"); if (!CalculateDistribution(typeConfig, parameter, progress, wayTypeData, maxLevel)) { return false; } // Calculate number of types which have data uint32_t indexEntries=0; for (const auto& type : typeConfig->GetWayTypes()) { if (wayTypeData[type->GetIndex()].HasEntries()) { indexEntries++; } } // // Writing index file // progress.SetAction("Generating 'areaway.idx'"); if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "areaway.idx"))) { progress.Error("Cannot create 'areaway.idx'"); return false; } writer.Write(indexEntries); for (const auto &type : typeConfig->GetWayTypes()) { size_t i=type->GetIndex(); if (wayTypeData[i].HasEntries()) { uint8_t dataOffsetBytes=0; FileOffset bitmapOffset=0; writer.WriteTypeId(type->GetWayId(), typeConfig->GetWayTypeIdBytes()); writer.GetPos(wayTypeData[i].indexOffset); writer.WriteFileOffset(bitmapOffset); writer.Write(dataOffsetBytes); writer.WriteNumber(wayTypeData[i].indexLevel); writer.WriteNumber(wayTypeData[i].cellXStart); writer.WriteNumber(wayTypeData[i].cellXEnd); writer.WriteNumber(wayTypeData[i].cellYStart); writer.WriteNumber(wayTypeData[i].cellYEnd); } } if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "ways.dat"), FileScanner::Sequential, parameter.GetWayDataMemoryMaped())) { progress.Error("Cannot open 'ways.dat'"); return false; } for (size_t l=parameter.GetAreaWayMinMag(); l<=maxLevel; l++) { TypeInfoSet indexTypes(*typeConfig); uint32_t wayCount; double cellWidth=360.0/pow(2.0,(int)l); double cellHeight=180.0/pow(2.0,(int)l); wayScanner.GotoBegin(); for (const auto &type : typeConfig->GetWayTypes()) { if (wayTypeData[type->GetIndex()].HasEntries() && wayTypeData[type->GetIndex()].indexLevel==l) { indexTypes.Set(type); } } if (indexTypes.Empty()) { continue; } progress.Info("Scanning ways for index level "+NumberToString(l)); std::vector<CoordOffsetsMap> typeCellOffsets(typeConfig->GetTypeCount()); if (!wayScanner.Read(wayCount)) { progress.Error("Error while reading number of data entries in file"); return false; } Way way; for (uint32_t w=1; w<=wayCount; w++) { progress.SetProgress(w,wayCount); FileOffset offset; wayScanner.GetPos(offset); if (!way.Read(*typeConfig, wayScanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(w)+" of "+ NumberToString(wayCount)+ " in file '"+ wayScanner.GetFilename()+"'"); return false; } if (!indexTypes.IsSet(way.GetType())) { continue; } GeoBox boundingBox; way.GetBoundingBox(boundingBox); // // Calculate minimum and maximum tile ids that are covered // by the way // Renormalized coordinate space (everything is >=0) // uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellWidth); uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellWidth); uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellHeight); uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellHeight); for (uint32_t y=minyc; y<=maxyc; y++) { for (uint32_t x=minxc; x<=maxxc; x++) { typeCellOffsets[way.GetType()->GetIndex()][Pixel(x,y)].push_back(offset); } } } for (const auto &type : indexTypes) { size_t index=type->GetIndex(); if (!WriteBitmap(progress, writer, *typeConfig->GetTypeInfo(index), wayTypeData[index], typeCellOffsets[index])) { return false; } } } return !writer.HasError() && writer.Close(); }
bool AreaNodeIndexGenerator::Import(const ImportParameter& parameter, Progress& progress, const TypeConfig& typeConfig) { FileScanner nodeScanner; FileWriter writer; std::set<TypeId> remainingNodeTypes; //! Set of types we still must process std::vector<TypeData> nodeTypeData; size_t level; size_t maxLevel=0; nodeTypeData.resize(typeConfig.GetTypes().size()); if (!nodeScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "nodes.dat"), FileScanner::Sequential, true)) { progress.Error("Cannot open 'nodes.dat'"); return false; } // // Scanning distribution // progress.SetAction("Scanning level distribution of node types"); // Initially we must process all types that represents nodes and that should // not be ignored for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeNode() && !typeConfig.GetTypeInfo(i).GetIgnore()) { remainingNodeTypes.insert(i); } } level=parameter.GetAreaNodeMinMag(); while (!remainingNodeTypes.empty()) { uint32_t nodeCount=0; std::set<TypeId> currentNodeTypes(remainingNodeTypes); double cellWidth=360.0/pow(2.0,(int)level); double cellHeight=180.0/pow(2.0,(int)level); std::vector<std::map<Pixel,size_t> > cellFillCount; cellFillCount.resize(typeConfig.GetTypes().size()); progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingNodeTypes.size())+" types still to process)"); nodeScanner.GotoBegin(); if (!nodeScanner.Read(nodeCount)) { progress.Error("Error while reading number of data entries in file"); return false; } for (uint32_t n=1; n<=nodeCount; n++) { progress.SetProgress(n,nodeCount); FileOffset offset; Node node; nodeScanner.GetPos(offset); if (!node.Read(nodeScanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(n)+" of "+ NumberToString(nodeCount)+ " in file '"+ nodeScanner.GetFilename()+"'"); return false; } // If we still need to handle this type, // count number of entries per type and tile cell if (currentNodeTypes.find(node.GetType())!=currentNodeTypes.end()) { uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth); uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight); cellFillCount[node.GetType()][Pixel(xc,yc)]++; } } // Check statistics for each type // If statistics are within goal limits, use this level // for this type (else try again with the next higher level) for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (currentNodeTypes.find(i)!=currentNodeTypes.end()) { size_t entryCount=0; size_t max=0; nodeTypeData[i].indexLevel=(uint32_t)level; nodeTypeData[i].indexCells=cellFillCount[i].size(); nodeTypeData[i].indexEntries=0; if (!cellFillCount[i].empty()) { nodeTypeData[i].cellXStart=cellFillCount[i].begin()->first.x; nodeTypeData[i].cellYStart=cellFillCount[i].begin()->first.y; nodeTypeData[i].cellXEnd=nodeTypeData[i].cellXStart; nodeTypeData[i].cellYEnd=nodeTypeData[i].cellYStart; for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin(); cell!=cellFillCount[i].end(); ++cell) { nodeTypeData[i].indexEntries+=cell->second; nodeTypeData[i].cellXStart=std::min(nodeTypeData[i].cellXStart,cell->first.x); nodeTypeData[i].cellXEnd=std::max(nodeTypeData[i].cellXEnd,cell->first.x); nodeTypeData[i].cellYStart=std::min(nodeTypeData[i].cellYStart,cell->first.y); nodeTypeData[i].cellYEnd=std::max(nodeTypeData[i].cellYEnd,cell->first.y); } } nodeTypeData[i].cellXCount=nodeTypeData[i].cellXEnd-nodeTypeData[i].cellXStart+1; nodeTypeData[i].cellYCount=nodeTypeData[i].cellYEnd-nodeTypeData[i].cellYStart+1; // Count absolute number of entries for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin(); cell!=cellFillCount[i].end(); ++cell) { entryCount+=cell->second; max=std::max(max,cell->second); } // Average number of entries per tile cell double average=entryCount*1.0/cellFillCount[i].size(); // If we do not have any entries, we store it now if (cellFillCount[i].empty()) { continue; } // If the fill rate of the index is too low, we use this index level anyway if (nodeTypeData[i].indexCells/(1.0*nodeTypeData[i].cellXCount*nodeTypeData[i].cellYCount)<= parameter.GetAreaNodeIndexMinFillRate()) { progress.Warning(typeConfig.GetTypeInfo(i).GetName()+" ("+NumberToString(i)+") is not well distributed"); continue; } // If average fill size and max fill size for tile cells // is within limits, store it now. if (max<=parameter.GetAreaNodeIndexCellSizeMax() && average<=parameter.GetAreaNodeIndexCellSizeAverage()) { continue; } // else, we remove it from the list and try again with an higher // level. currentNodeTypes.erase(i); } } // Now process all types for this limit, that are within the limits for (std::set<TypeId>::const_iterator cnt=currentNodeTypes.begin(); cnt!=currentNodeTypes.end(); cnt++) { maxLevel=std::max(maxLevel,level); progress.Info("Type "+typeConfig.GetTypeInfo(*cnt).GetName()+"(" + NumberToString(*cnt)+"), "+NumberToString(nodeTypeData[*cnt].indexCells)+" cells, "+NumberToString(nodeTypeData[*cnt].indexEntries)+" objects"); remainingNodeTypes.erase(*cnt); } level++; } // // Writing index file // progress.SetAction("Generating 'areanode.idx'"); if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "areanode.idx"))) { progress.Error("Cannot create 'areanode.idx'"); return false; } uint32_t indexEntries=0; // Count number of types in index for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeNode() && nodeTypeData[i].HasEntries()) { indexEntries++; } } writer.Write(indexEntries); // Store index data for each type for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeNode() && nodeTypeData[i].HasEntries()) { FileOffset bitmapOffset=0; uint8_t dataOffsetBytes=0; writer.WriteNumber(typeConfig.GetTypeInfo(i).GetId()); writer.GetPos(nodeTypeData[i].indexOffset); writer.WriteFileOffset(bitmapOffset); writer.Write(dataOffsetBytes); writer.WriteNumber(nodeTypeData[i].indexLevel); writer.WriteNumber(nodeTypeData[i].cellXStart); writer.WriteNumber(nodeTypeData[i].cellXEnd); writer.WriteNumber(nodeTypeData[i].cellYStart); writer.WriteNumber(nodeTypeData[i].cellYEnd); } } // Now store index bitmap for each type in increasing level order (why?) for (size_t l=0; l<=maxLevel; l++) { std::set<TypeId> indexTypes; uint32_t nodeCount; double cellWidth=360.0/pow(2.0,(int)l); double cellHeight=180.0/pow(2.0,(int)l); for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeNode() && nodeTypeData[i].HasEntries() && nodeTypeData[i].indexLevel==l) { indexTypes.insert(i); } } if (indexTypes.empty()) { continue; } progress.Info("Scanning nodes for index level "+NumberToString(l)); std::vector<std::map<Pixel,std::list<FileOffset> > > typeCellOffsets; typeCellOffsets.resize(typeConfig.GetTypes().size()); nodeScanner.GotoBegin(); if (!nodeScanner.Read(nodeCount)) { progress.Error("Error while reading number of data entries in file"); return false; } // // Collect all offsets // for (uint32_t n=1; n<=nodeCount; n++) { progress.SetProgress(n,nodeCount); FileOffset offset; Node node; nodeScanner.GetPos(offset); if (!node.Read(nodeScanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(n)+" of "+ NumberToString(nodeCount)+ " in file '"+ nodeScanner.GetFilename()+"'"); return false; } if (indexTypes.find(node.GetType())!=indexTypes.end()) { uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth); uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight); typeCellOffsets[node.GetType()][Pixel(xc,yc)].push_back(offset); } } // // Write bitmap // for (std::set<TypeId>::const_iterator type=indexTypes.begin(); type!=indexTypes.end(); ++type) { size_t indexEntries=0; size_t dataSize=0; char buffer[10]; for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin(); cell!=typeCellOffsets[*type].end(); ++cell) { indexEntries+=cell->second.size(); dataSize+=EncodeNumber(cell->second.size(),buffer); FileOffset previousOffset=0; for (std::list<FileOffset>::const_iterator offset=cell->second.begin(); offset!=cell->second.end(); ++offset) { FileOffset data=*offset-previousOffset; dataSize+=EncodeNumber(data,buffer); previousOffset=*offset; } } // "+1" because we add +1 to every offset, to generate offset > 0 uint8_t dataOffsetBytes=BytesNeededToAddressFileData(dataSize); progress.Info("Writing map for "+ typeConfig.GetTypeInfo(*type).GetName()+", "+ NumberToString(typeCellOffsets[*type].size())+" cells, "+ NumberToString(indexEntries)+" entries, "+ ByteSizeToString(1.0*dataOffsetBytes*nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount)); FileOffset bitmapOffset; if (!writer.GetPos(bitmapOffset)) { progress.Error("Cannot get type index start position in file"); return false; } assert(nodeTypeData[*type].indexOffset!=0); if (!writer.SetPos(nodeTypeData[*type].indexOffset)) { progress.Error("Cannot go to type index offset in file"); return false; } writer.WriteFileOffset(bitmapOffset); writer.Write(dataOffsetBytes); if (!writer.SetPos(bitmapOffset)) { progress.Error("Cannot go to type index start position in file"); return false; } // Write the bitmap with offsets for each cell // We prefill with zero and only overwrite cells that have data // So zero means "no data for this cell" for (size_t i=0; i<nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount; i++) { FileOffset cellOffset=0; writer.WriteFileOffset(cellOffset, dataOffsetBytes); } FileOffset dataStartOffset; if (!writer.GetPos(dataStartOffset)) { progress.Error("Cannot get start of data section after bitmap"); return false; } // Now write the list of offsets of objects for every cell with content for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin(); cell!=typeCellOffsets[*type].end(); ++cell) { FileOffset bitmapCellOffset=bitmapOffset+ ((cell->first.y-nodeTypeData[*type].cellYStart)*nodeTypeData[*type].cellXCount+ cell->first.x-nodeTypeData[*type].cellXStart)*dataOffsetBytes; FileOffset previousOffset=0; FileOffset cellOffset; if (!writer.GetPos(cellOffset)) { progress.Error("Cannot get cell start position in file"); return false; } if (!writer.SetPos(bitmapCellOffset)) { progress.Error("Cannot go to cell start position in file"); return false; } writer.WriteFileOffset(cellOffset-dataStartOffset+1, dataOffsetBytes); if (!writer.SetPos(cellOffset)) { progress.Error("Cannot go back to cell start position in file"); return false; } writer.WriteNumber((uint32_t)cell->second.size()); for (std::list<FileOffset>::const_iterator offset=cell->second.begin(); offset!=cell->second.end(); ++offset) { writer.WriteNumber((FileOffset)(*offset-previousOffset)); previousOffset=*offset; } } } } return !writer.HasError() && writer.Close(); }
bool OptimizeAreaWayIdsGenerator::CopyWays(const ImportParameter& parameter, Progress& progress, const TypeConfig& typeConfig, NodeUseMap& nodeUseMap) { FileScanner scanner; FileWriter writer; uint32_t dataCount=0; progress.SetAction("Copy data from 'wayway.tmp' to 'ways.tmp'"); if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "wayway.tmp"), FileScanner::Sequential, parameter.GetWayDataMemoryMaped())) { progress.Error(std::string("Cannot open '")+scanner.GetFilename()+"'"); return false; } if (!scanner.Read(dataCount)) { progress.Error("Error while reading number of data entries in file"); return false; } if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "ways.tmp"))) { progress.Error(std::string("Cannot create '")+writer.GetFilename()+"'"); return false; } writer.Write(dataCount); for (uint32_t current=1; current<=dataCount; current++) { uint8_t type; Id id; Way data; progress.SetProgress(current,dataCount); if (!scanner.Read(type) || !scanner.Read(id) || !data.Read(typeConfig, scanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(current)+" of "+ NumberToString(dataCount)+ " in file '"+ scanner.GetFilename()+"'"); return false; } for (auto& id : data.ids) { if (!nodeUseMap.IsNodeUsedAtLeastTwice(id)) { id=0; } } if (!writer.Write(type) || !writer.Write(id) || !data.Write(typeConfig, writer)) { progress.Error(std::string("Error while writing data entry to file '")+ writer.GetFilename()+"'"); return false; } } if (!scanner.Close()) { progress.Error(std::string("Error while closing file '")+ scanner.GetFilename()+"'"); return false; } if (!writer.Close()) { progress.Error(std::string("Error while closing file '")+ writer.GetFilename()+"'"); return false; } return true; }