void PerfTest::RunRecvBufferSizeTests(SecurityProvider::Enum secProvider)
{
console.WriteLine("=========================================================");
console.WriteLine("security provider = {0}", secProvider);
console.WriteLine(" test data total in bytes: {0}", testDataSize_);
console.WriteLine("=========================================================");
for (uint clientThreadCount = clientThreadMin_; clientThreadCount <= clientThreadMax_; ++clientThreadCount)
{
wstring outputFile = wformatString(
"PerfTest-QueueReceived@{0}_ClientThread@{1}_Sec@{2}.csv",
shouldQueueReceivedMessage_, clientThreadCount, secProvider);
console.WriteLine("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++");
console.WriteLine("client thread count = {0}", clientThreadCount);
console.WriteLine("ShouldQueueReceivedMessage = {0}", shouldQueueReceivedMessage_);
console.WriteLine("output = {0}", outputFile);
console.WriteLine("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++");
FileWriter csvFile;
auto error = csvFile.TryOpen(outputFile);
Invariant(error.IsSuccess());
KFinally([&] { csvFile.Close(); });
for (uint messageSize = messageSizeMin_; messageSize <= messageSizeMax_; messageSize *= 2)
{
csvFile.Write("{0}", messageSize);
map<TimeSpan, uint> resultMap;
for (uint tcpBufferSize = tcpBufferSizeMin_; tcpBufferSize <= tcpBufferSizeMax_; tcpBufferSize += 4*1024)
{
TransportConfig::GetConfig().TcpReceiveBufferSize = tcpBufferSize;
console.WriteLine("---------------------------------------------------------");
console.WriteLine("tcp receive buffer = {0}", TransportConfig::GetConfig().TcpReceiveBufferSize);
console.WriteLine("---------------------------------------------------------");
//csvFile.Write("{0}", tcpBufferSize);
PerfTest perfTest(secProvider, messageSize, clientThreadCount);
perfTest.Run(csvFile);
resultMap.emplace(make_pair(perfTest.GetTestDuration(), tcpBufferSize));
}
// record tcpBufferSize in KB, sorted by test duration
console.WriteLine("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$");
console.WriteLine("TCP buffer sizes sorted by test duration:");
auto optimalBufferSize = resultMap.cbegin()->second / 1024;
for (auto const & entry : resultMap)
{
auto bufferSize = entry.second/1024;
csvFile.Write(",{0}", bufferSize);
console.WriteLine("{0}", bufferSize);
}
csvFile.WriteLine();
csvFile.Flush();
console.WriteLine("\n@_@ optimal buffer size = {0} @_@", optimalBufferSize);
console.WriteLine("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$");
}
}
}
bool RouteNode::Write(FileWriter& writer) const
{
writer.WriteNumber(id);
writer.WriteCoord(coord);
writer.WriteNumber((uint32_t)objects.size());
writer.WriteNumber((uint32_t)paths.size());
writer.WriteNumber((uint32_t)excludes.size());
Id lastFileOffset=0;
for (const auto& object : objects) {
FileOffset offset=object.object.GetFileOffset()-lastFileOffset;
if (object.object.GetType()==refWay) {
offset=offset*2;
}
else if (object.object.GetType()==refArea) {
offset=offset*2+1;
}
else {
assert(false);
}
writer.WriteNumber(offset);
writer.WriteNumber(object.type);
writer.Write(object.maxSpeed);
writer.Write(object.grade);
lastFileOffset=object.object.GetFileOffset();
}
if (!paths.empty()) {
for (const auto& path : paths) {
writer.WriteFileOffset(path.offset);
writer.WriteNumber(path.objectIndex);
//writer.Write(paths[i].bearing);
writer.Write(path.flags);
writer.WriteNumber((uint32_t)floor(path.distance*(1000.0*100.0)+0.5));
}
}
for (const auto& exclude : excludes) {
writer.Write(exclude.source);
writer.WriteNumber(exclude.targetIndex);
}
return !writer.HasError();
}
bool Area::WriteOptimized(const TypeConfig& typeConfig,
FileWriter& writer) const
{
std::vector<Ring>::const_iterator ring=rings.begin();
bool multipleRings=rings.size()>1;
// Outer ring
writer.WriteTypeId(ring->GetType()->GetAreaId(),
typeConfig.GetAreaTypeIdBytes());
if (!ring->featureValueBuffer.Write(writer,
multipleRings)) {
return false;
}
if (multipleRings) {
writer.WriteNumber((uint32_t)(rings.size()-1));
}
if (!writer.Write(ring->nodes)) {
return false;
}
++ring;
// Potential additional rings
while (ring!=rings.end()) {
writer.WriteTypeId(ring->GetType()->GetAreaId(),
typeConfig.GetAreaTypeIdBytes());
if (ring->GetType()->GetAreaId()!=typeIgnore) {
if (!ring->featureValueBuffer.Write(writer)) {
return false;
}
}
writer.Write(ring->ring);
if (!writer.Write(ring->nodes)) {
return false;
}
++ring;
}
return !writer.HasError();
}
bool Way::Write(const TypeConfig& typeConfig,
FileWriter& writer) const
{
assert(!nodes.empty());
writer.WriteTypeId(featureValueBuffer.GetType()->GetWayId(),
typeConfig.GetWayTypeIdBytes());
if (!featureValueBuffer.Write(writer)) {
return false;
}
if (!writer.Write(nodes)) {
return false;
}
if (featureValueBuffer.GetType()->CanRoute() ||
featureValueBuffer.GetType()->GetOptimizeLowZoom()) {
if (!WriteIds(writer)) {
return false;
}
}
return !writer.HasError();
}
bool OptimizeAreasLowZoomGenerator::WriteHeader(FileWriter& writer,
const std::list<TypeData>& areaTypesData,
uint32_t optimizeMaxMap)
{
writer.Write(optimizeMaxMap);
writer.Write((uint32_t)areaTypesData.size());
for (const auto &typeData : areaTypesData) {
if (!WriteTypeData(writer,
typeData)) {
return false;
}
}
return true;
}
Kumu::Result_t
Kumu::WriteObjectIntoFile(const Kumu::IArchive& Object, const std::string& Filename)
{
ByteString Buffer;
Result_t result = Buffer.Capacity(Object.ArchiveLength());
if ( KM_SUCCESS(result) )
{
ui32_t write_count = 0;
FileWriter Writer;
MemIOWriter MemWriter(&Buffer);
result = Object.Archive(&MemWriter) ? RESULT_OK : RESULT_WRITEFAIL;
if ( KM_SUCCESS(result) )
{
Buffer.Length(MemWriter.Length());
result = Writer.OpenWrite(Filename);
}
if ( KM_SUCCESS(result) )
result = Writer.Write(Buffer.RoData(), Buffer.Length(), &write_count);
}
return result;
}
bool WayLocationProcessorFilter::AfterProcessingEnd(const ImportParameter& /*parameter*/,
Progress& progress,
const TypeConfig& /*typeConfig*/)
{
delete nameReader;
nameReader=NULL;
delete locationReader;
locationReader=NULL;
writer.SetPos(0);
writer.Write(overallDataCount);
try {
writer.Close();
}
catch (IOException& e) {
progress.Error(e.GetDescription());
writer.CloseFailsafe();
return false;
}
return true;
}
bool Relation::Write(FileWriter& writer) const
{
writer.Write(id);
if (!attributes.Write(writer)) {
return false;
}
writer.WriteNumber((uint32_t)roles.size());
for (size_t i=0; i<roles.size(); i++) {
if (!roles[i].attributes.Write(writer)) {
return false;
}
writer.Write(roles[i].role);
writer.WriteNumber((uint32_t)roles[i].nodes.size());
if (!roles[i].nodes.empty()) {
Id minId=std::numeric_limits<Id>::max();
uint32_t minLat=std::numeric_limits<uint32_t>::max();
uint32_t minLon=std::numeric_limits<uint32_t>::max();
for (size_t j=0; j<roles[i].nodes.size(); j++) {
minId=std::min(minId,roles[i].nodes[j].GetId());
minLat=std::min(minLat,(uint32_t)floor((roles[i].nodes[j].lat+90.0)*conversionFactor+0.5));
minLon=std::min(minLon,(uint32_t)floor((roles[i].nodes[j].lon+180.0)*conversionFactor+0.5));
}
writer.Write(minId);
writer.Write(minLat);
writer.Write(minLon);
for (size_t j=0; j<roles[i].nodes.size(); j++) {
uint32_t latValue=(uint32_t)floor((roles[i].nodes[j].lat+90.0)*conversionFactor+0.5);
uint32_t lonValue=(uint32_t)floor((roles[i].nodes[j].lon+180.0)*conversionFactor+0.5);
writer.WriteNumber(roles[i].nodes[j].id-minId);
writer.WriteNumber(latValue-minLat);
writer.WriteNumber(lonValue-minLon);
}
}
}
return !writer.HasError();
}
bool Preprocess::Callback::DumpDistribution()
{
FileWriter writer;
progress.SetAction("Writing 'distribution.dat'");
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"distribution.dat"))) {
return false;
}
for (const auto &type : typeConfig->GetTypes()) {
writer.Write(nodeStat[type->GetIndex()]);
writer.Write(wayStat[type->GetIndex()]);
writer.Write(areaStat[type->GetIndex()]);
}
return writer.Close();
}
bool WayLocationProcessorFilter::Process(Progress& progress,
const FileOffset& offset,
Way& way,
bool& /*save*/)
{
try {
if (!way.GetType()->GetIndexAsPOI()) {
return true;
}
NameFeatureValue *nameValue=nameReader->GetValue(way.GetFeatureValueBuffer());
if (nameValue==NULL) {
return true;
}
LocationFeatureValue *locationValue=locationReader->GetValue(way.GetFeatureValueBuffer());
std::string name;
std::string location;
std::string address;
name=nameValue->GetName();
if (locationValue!=NULL) {
location=locationValue->GetLocation();
}
writer.WriteFileOffset(offset);
writer.WriteNumber(way.GetType()->GetWayId());
writer.Write(name);
writer.Write(location);
writer.Write(way.nodes);
overallDataCount++;
}
catch (IOException& e) {
progress.Error(e.GetDescription());
return false;
}
return true;
}
Kumu::Result_t
Kumu::WriteStringIntoFile(const std::string& filename, const std::string& inString)
{
FileWriter File;
ui32_t write_count = 0;
Result_t result = File.OpenWrite(filename);
if ( KM_SUCCESS(result) )
result = File.Write((byte_t*)inString.c_str(), inString.length(), &write_count);
return result;
}
bool Area::WriteIds(FileWriter& writer,
const std::vector<Id>& ids) const
{
Id minId=0;
for (size_t i=0; i<ids.size(); i++) {
if (ids[i]!=0) {
if (minId==0) {
minId=ids[i];
}
else {
minId=std::min(minId,ids[i]);
}
}
}
writer.WriteNumber(minId);
if (minId>0) {
size_t idCurrent=0;
while (idCurrent<ids.size()) {
uint8_t bitset=0;
uint8_t bitMask=1;
size_t idEnd=std::min(idCurrent+8,ids.size());
for (size_t i=idCurrent; i<idEnd; i++) {
if (ids[i]!=0) {
bitset=bitset | bitMask;
}
bitMask*=2;
}
writer.Write(bitset);
for (size_t i=idCurrent; i<idEnd; i++) {
if (ids[i]!=0) {
writer.WriteNumber(ids[i]-minId);
}
bitMask=bitMask*2;
}
idCurrent+=8;
}
}
return !writer.HasError();
}
void PerfTest::WaitForResult(FileWriter & fw)
{
Invariant(allReceived_.WaitOne(testTimeout_));
auto elapsedMilliseconds = stopwatch_.ElapsedMilliseconds;
console.WriteLine(">>> time elapsed: {0} ms", elapsedMilliseconds);
auto totalReceivedBytes = recvBytes_.load();
auto recvRate = (totalReceivedBytes * 8.0) / elapsedMilliseconds / 1000.0;
console.WriteLine(">>> received: {0} bytes", totalReceivedBytes);
console.WriteLine(">>> receive rate: {0} mbps\n\n", recvRate);
fw.Write(",{0},{1}", elapsedMilliseconds, recvRate);
listener_->Stop();
listener_.reset();
}
Result_t
Kumu::WriteBufferIntoFile(const Kumu::ByteString& Buffer, const std::string& Filename)
{
ui32_t write_count = 0;
FileWriter Writer;
Result_t result = Writer.OpenWrite(Filename);
if ( KM_SUCCESS(result) )
result = Writer.Write(Buffer.RoData(), Buffer.Length(), &write_count);
if ( KM_SUCCESS(result) && Buffer.Length() != write_count)
return RESULT_WRITEFAIL;
return result;
}
bool OptimizeAreasLowZoomGenerator::WriteTypeData(FileWriter& writer,
const TypeData& data)
{
assert(data.type);
writer.Write(data.type->GetAreaId());
writer.Write(data.optLevel);
writer.Write(data.indexLevel);
writer.Write(data.cellXStart);
writer.Write(data.cellXEnd);
writer.Write(data.cellYStart);
writer.Write(data.cellYEnd);
writer.WriteFileOffset(data.bitmapOffset);
writer.Write(data.dataOffsetBytes);
return !writer.HasError();
}
bool WayLocationProcessorFilter::BeforeProcessingStart(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig)
{
overallDataCount=0;
nameReader=new NameFeatureValueReader(typeConfig);
locationReader=new LocationFeatureValueReader(typeConfig);
try {
writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
SortWayDataGenerator::WAYADDRESS_DAT));
writer.Write(overallDataCount);
}
catch (IOException& e) {
progress.Error(e.GetDescription());
return false;
}
return true;
}
bool NodeDataGenerator::Import(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress)
{
uint32_t rawNodeCount=0;
uint32_t nodesReadCount=0;
uint32_t nodesWrittenCount=0;
//
// Iterator over all raw nodes, hcekc they type, and convert them from raw nodes
// to nodes if the type is interesting (!=typeIgnore).
//
// Count the bounding box by the way...
//
progress.SetAction("Generating nodes.tmp");
FileScanner scanner;
FileWriter writer;
if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"rawnodes.dat"),
FileScanner::Sequential,
parameter.GetRawNodeDataMemoryMaped())) {
progress.Error("Cannot open 'rawnodes.dat'");
return false;
}
if (!scanner.Read(rawNodeCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"nodes.tmp"))) {
progress.Error("Cannot create 'nodes.tmp'");
return false;
}
writer.Write(nodesWrittenCount);
for (uint32_t n=1; n<=rawNodeCount; n++) {
progress.SetProgress(n,rawNodeCount);
RawNode rawNode;
Node node;
if (!rawNode.Read(typeConfig,
scanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(n)+" of "+
NumberToString(rawNodeCount)+
" in file '"+
scanner.GetFilename()+"'");
return false;
}
nodesReadCount++;
if (!rawNode.GetType()->GetIgnore()) {
node.SetFeatures(rawNode.GetFeatureValueBuffer());
node.SetCoords(rawNode.GetCoords());
FileOffset fileOffset;
if (!writer.GetPos(fileOffset)) {
progress.Error(std::string("Error while reading current fileOffset in file '")+
writer.GetFilename()+"'");
return false;
}
writer.Write(rawNode.GetId());
node.Write(typeConfig,
writer);
nodesWrittenCount++;
}
}
if (!scanner.Close()) {
return false;
}
writer.SetPos(0);
writer.Write(nodesWrittenCount);
if (!writer.Close()) {
return false;
}
progress.Info(std::string("Read "+NumberToString(nodesReadCount)+" nodes, wrote "+NumberToString(nodesWrittenCount)+" nodes"));
return true;
}
bool AreaWayIndexGenerator::WriteBitmap(Progress& progress,
FileWriter& writer,
const TypeInfo& typeInfo,
const TypeData& typeData,
const CoordOffsetsMap& typeCellOffsets)
{
size_t indexEntries=0;
size_t dataSize=0;
char buffer[10];
for (CoordOffsetsMap::const_iterator cell=typeCellOffsets.begin();
cell!=typeCellOffsets.end();
++cell) {
indexEntries+=cell->second.size();
dataSize+=EncodeNumber(cell->second.size(),buffer);
FileOffset previousOffset=0;
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
FileOffset data=*offset-previousOffset;
dataSize+=EncodeNumber(data,buffer);
previousOffset=*offset;
}
}
// "+1" because we add +1 to every offset, to generate offset > 0
uint8_t dataOffsetBytes=BytesNeededToAddressFileData(dataSize+1);
progress.Info("Writing map for "+
typeInfo.GetName()+" , "+
ByteSizeToString(1.0*dataOffsetBytes*typeData.cellXCount*typeData.cellYCount+dataSize));
FileOffset bitmapOffset;
if (!writer.GetPos(bitmapOffset)) {
progress.Error("Cannot get type index start position in file");
return false;
}
assert(typeData.indexOffset!=0);
if (!writer.SetPos(typeData.indexOffset)) {
progress.Error("Cannot go to type index offset in file");
return false;
}
writer.WriteFileOffset(bitmapOffset);
writer.Write(dataOffsetBytes);
if (!writer.SetPos(bitmapOffset)) {
progress.Error("Cannot go to type index start position in file");
return false;
}
// Write the bitmap with offsets for each cell
// We prefill with zero and only overwrite cells that have data
// So zero means "no data for this cell"
for (size_t i=0; i<typeData.cellXCount*typeData.cellYCount; i++) {
writer.WriteFileOffset(0,
dataOffsetBytes);
}
FileOffset dataStartOffset;
if (!writer.GetPos(dataStartOffset)) {
progress.Error("Cannot get start of data section after bitmap");
return false;
}
// Now write the list of offsets of objects for every cell with content
for (CoordOffsetsMap::const_iterator cell=typeCellOffsets.begin();
cell!=typeCellOffsets.end();
++cell) {
FileOffset bitmapCellOffset=bitmapOffset+
((cell->first.y-typeData.cellYStart)*typeData.cellXCount+
cell->first.x-typeData.cellXStart)*(FileOffset)dataOffsetBytes;
FileOffset previousOffset=0;
FileOffset cellOffset;
assert(bitmapCellOffset>=bitmapOffset);
if (!writer.GetPos(cellOffset)) {
progress.Error("Cannot get cell start position in file");
return false;
}
if (!writer.SetPos(bitmapCellOffset)) {
progress.Error("Cannot go to cell start position in file");
return false;
}
assert(cellOffset>bitmapCellOffset);
// We add +1 to make sure, that we can differentiate between "0" as "no entry" and "0" as first data entry.
writer.WriteFileOffset(cellOffset-dataStartOffset+1,dataOffsetBytes);
if (!writer.SetPos(cellOffset)) {
progress.Error("Cannot go back to cell start position in file");
return false;
}
writer.WriteNumber((uint32_t)cell->second.size());
// FileOffsets are already in increasing order, since
// File is scanned from start to end
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
writer.WriteNumber((FileOffset)(*offset-previousOffset));
previousOffset=*offset;
}
}
return true;
}
bool AreaWayIndexGenerator::Import(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig)
{
FileScanner wayScanner;
FileWriter writer;
std::set<TypeId> remainingWayTypes;
std::vector<TypeData> wayTypeData;
size_t level;
size_t maxLevel=0;
wayTypeData.resize(typeConfig.GetTypes().size());
if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"ways.dat"),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped())) {
progress.Error("Cannot open 'ways.dat'");
return false;
}
//
// Scanning distribution
//
progress.SetAction("Scanning level distribution of way types");
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (typeConfig.GetTypeInfo(i).CanBeWay() &&
!typeConfig.GetTypeInfo(i).GetIgnore()) {
remainingWayTypes.insert(i);
}
}
level=parameter.GetAreaWayMinMag();
while (!remainingWayTypes.empty()) {
uint32_t wayCount=0;
std::set<TypeId> currentWayTypes(remainingWayTypes);
double cellWidth=360.0/pow(2.0,(int)level);
double cellHeight=180.0/pow(2.0,(int)level);
std::vector<CoordCountMap> cellFillCount(typeConfig.GetTypes().size());
progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingWayTypes.size())+" types remaining)");
wayScanner.GotoBegin();
if (!wayScanner.Read(wayCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
Way way;
for (uint32_t w=1; w<=wayCount; w++) {
progress.SetProgress(w,wayCount);
if (!way.Read(wayScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(w)+" of "+
NumberToString(wayCount)+
" in file '"+
wayScanner.GetFilename()+"'");
return false;
}
// Count number of entries per current type and coordinate
if (currentWayTypes.find(way.GetType())==currentWayTypes.end()) {
continue;
}
double minLon;
double maxLon;
double minLat;
double maxLat;
way.GetBoundingBox(minLon,maxLon,minLat,maxLat);
//
// Calculate minimum and maximum tile ids that are covered
// by the way
// Renormated coordinate space (everything is >=0)
//
uint32_t minxc=(uint32_t)floor((minLon+180.0)/cellWidth);
uint32_t maxxc=(uint32_t)floor((maxLon+180.0)/cellWidth);
uint32_t minyc=(uint32_t)floor((minLat+90.0)/cellHeight);
uint32_t maxyc=(uint32_t)floor((maxLat+90.0)/cellHeight);
for (uint32_t y=minyc; y<=maxyc; y++) {
for (uint32_t x=minxc; x<=maxxc; x++) {
cellFillCount[way.GetType()][Pixel(x,y)]++;
}
}
}
// Check if cell fill for current type is in defined limits
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (currentWayTypes.find(i)!=currentWayTypes.end()) {
CalculateStatistics(level,wayTypeData[i],cellFillCount[i]);
if (!FitsIndexCriteria(parameter,
progress,
typeConfig.GetTypeInfo(i),
wayTypeData[i],
cellFillCount[i])) {
currentWayTypes.erase(i);
}
}
}
for (std::set<TypeId>::const_iterator cwt=currentWayTypes.begin();
cwt!=currentWayTypes.end();
cwt++) {
maxLevel=std::max(maxLevel,level);
progress.Info("Type "+typeConfig.GetTypeInfo(*cwt).GetName()+"(" + NumberToString(*cwt)+"), "+NumberToString(wayTypeData[*cwt].indexCells)+" cells, "+NumberToString(wayTypeData[*cwt].indexEntries)+" objects");
remainingWayTypes.erase(*cwt);
}
level++;
}
//
// Writing index file
//
progress.SetAction("Generating 'areaway.idx'");
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areaway.idx"))) {
progress.Error("Cannot create 'areaway.idx'");
return false;
}
uint32_t indexEntries=0;
for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
{
if (typeConfig.GetTypeInfo(i).CanBeWay() &&
wayTypeData[i].HasEntries()) {
indexEntries++;
}
}
writer.Write(indexEntries);
for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
{
if (typeConfig.GetTypeInfo(i).CanBeWay() &&
wayTypeData[i].HasEntries()) {
uint8_t dataOffsetBytes=0;
FileOffset bitmapOffset=0;
writer.WriteNumber(typeConfig.GetTypeInfo(i).GetId());
writer.GetPos(wayTypeData[i].indexOffset);
writer.WriteFileOffset(bitmapOffset);
if (wayTypeData[i].HasEntries()) {
writer.Write(dataOffsetBytes);
writer.WriteNumber(wayTypeData[i].indexLevel);
writer.WriteNumber(wayTypeData[i].cellXStart);
writer.WriteNumber(wayTypeData[i].cellXEnd);
writer.WriteNumber(wayTypeData[i].cellYStart);
writer.WriteNumber(wayTypeData[i].cellYEnd);
}
}
}
for (size_t l=parameter.GetAreaWayMinMag(); l<=maxLevel; l++) {
std::set<TypeId> indexTypes;
uint32_t wayCount;
double cellWidth=360.0/pow(2.0,(int)l);
double cellHeight=180.0/pow(2.0,(int)l);
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (typeConfig.GetTypeInfo(i).CanBeWay() &&
wayTypeData[i].HasEntries() &&
wayTypeData[i].indexLevel==l) {
indexTypes.insert(i);
}
}
if (indexTypes.empty()) {
continue;
}
progress.Info("Scanning ways for index level "+NumberToString(l));
std::vector<CoordOffsetsMap> typeCellOffsets(typeConfig.GetTypes().size());
wayScanner.GotoBegin();
if (!wayScanner.Read(wayCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
Way way;
for (uint32_t w=1; w<=wayCount; w++) {
progress.SetProgress(w,wayCount);
FileOffset offset;
wayScanner.GetPos(offset);
if (!way.Read(wayScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(w)+" of "+
NumberToString(wayCount)+
" in file '"+
wayScanner.GetFilename()+"'");
return false;
}
if (indexTypes.find(way.GetType())==indexTypes.end()) {
continue;
}
double minLon;
double maxLon;
double minLat;
double maxLat;
way.GetBoundingBox(minLon,maxLon,minLat,maxLat);
//
// Calculate minimum and maximum tile ids that are covered
// by the way
// Renormated coordinate space (everything is >=0)
//
uint32_t minxc=(uint32_t)floor((minLon+180.0)/cellWidth);
uint32_t maxxc=(uint32_t)floor((maxLon+180.0)/cellWidth);
uint32_t minyc=(uint32_t)floor((minLat+90.0)/cellHeight);
uint32_t maxyc=(uint32_t)floor((maxLat+90.0)/cellHeight);
for (uint32_t y=minyc; y<=maxyc; y++) {
for (uint32_t x=minxc; x<=maxxc; x++) {
typeCellOffsets[way.GetType()][Pixel(x,y)].push_back(offset);
}
}
}
for (std::set<TypeId>::const_iterator type=indexTypes.begin();
type!=indexTypes.end();
++type) {
if (!WriteBitmap(progress,
writer,
typeConfig.GetTypeInfo(*type),
wayTypeData[*type],
typeCellOffsets[*type])) {
return false;
}
}
}
return !writer.HasError() && writer.Close();
}
bool MergeAreasGenerator::Import(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress)
{
TypeInfoSet mergeTypes;
FileScanner scanner;
FileWriter writer;
uint32_t areasWritten=0;
for (const auto& type : typeConfig->GetTypes()) {
if (type->CanBeArea() &&
type->GetMergeAreas()) {
mergeTypes.Set(type);
}
}
std::unordered_set<Id> nodeUseMap;
try {
scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
MergeAreaDataGenerator::AREAS_TMP),
FileScanner::Sequential,
parameter.GetRawWayDataMemoryMaped());
if (!ScanAreaNodeIds(progress,
*typeConfig,
scanner,
mergeTypes,
nodeUseMap)) {
return false;
}
uint32_t nodeCount=nodeUseMap.size();
progress.Info("Found "+NumberToString(nodeCount)+" nodes as possible connection points for areas");
/* ------ */
writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
AREAS2_TMP));
writer.Write(areasWritten);
while (true) {
TypeInfoSet loadedTypes;
std::vector<AreaMergeData> mergeJob(typeConfig->GetTypeCount());
//
// Load type data
//
progress.SetAction("Collecting area data by type");
if (!GetAreas(parameter,
progress,
*typeConfig,
mergeTypes,
loadedTypes,
nodeUseMap,
scanner,
writer,
mergeJob,
areasWritten)) {
return false;
}
// Merge
progress.SetAction("Merging areas");
for (const auto& type : loadedTypes) {
if (!mergeJob[type->GetIndex()].areas.empty()) {
progress.Info("Merging areas of type "+type->GetName());
MergeAreas(progress,
nodeUseMap,
mergeJob[type->GetIndex()]);
progress.Info("Reduced areas of '"+type->GetName()+"' from "+NumberToString(mergeJob[type->GetIndex()].areaCount)+" to "+NumberToString(mergeJob[type->GetIndex()].areaCount-mergeJob[type->GetIndex()].mergedAway.size()));
mergeJob[type->GetIndex()].areas.clear();
}
}
// Store back merge result
if (!loadedTypes.Empty()) {
if (!WriteMergeResult(progress,
*typeConfig,
scanner,
writer,
loadedTypes,
mergeJob,
areasWritten)) {
return false;
}
mergeTypes.Remove(loadedTypes);
}
if (mergeTypes.Empty()) {
break;
}
}
scanner.Close();
writer.GotoBegin();
writer.Write(areasWritten);
writer.Close();
}
catch (IOException& e) {
progress.Error(e.GetDescription());
scanner.CloseFailsafe();
writer.CloseFailsafe();
return false;
}
return true;
}
void WriteRawData(FileWriter & writer, std::vector<uint8_t> const & rawData)
{
uint64_t const size = static_cast<uint64_t>(rawData.size());
WriteToSink(writer, size);
writer.Write(rawData.data(), size);
}
bool MergeAreasGenerator::WriteMergeResult(Progress& progress,
const TypeConfig& typeConfig,
FileScanner& scanner,
FileWriter& writer,
const TypeInfoSet& loadedTypes,
std::vector<AreaMergeData>& mergeJob,
uint32_t& areasWritten)
{
uint32_t areaCount=0;
std::unordered_map<FileOffset,AreaRef> merges;
std::unordered_set<FileOffset> ignores;
for (const auto& type : loadedTypes) {
for (const auto& area : mergeJob[type->GetIndex()].merges) {
merges[area->GetFileOffset()]=area;
}
ignores.insert(mergeJob[type->GetIndex()].mergedAway.begin(),
mergeJob[type->GetIndex()].mergedAway.end());
}
scanner.GotoBegin();
scanner.Read(areaCount);
for (uint32_t a=1; a<=areaCount; a++) {
uint8_t type;
Id id;
AreaRef area=std::make_shared<Area>();
progress.SetProgress(a,areaCount);
scanner.Read(type);
scanner.Read(id);
area->ReadImport(typeConfig,
scanner);
if (loadedTypes.IsSet(area->GetType())) {
if (ignores.find(area->GetFileOffset())!=ignores.end()) {
continue;
}
writer.Write(type);
writer.Write(id);
const auto& merge=merges.find(area->GetFileOffset()) ;
if (merge!=merges.end()) {
area=merge->second;
}
area->WriteImport(typeConfig,
writer);
areasWritten++;
}
}
return true;
}
/**
* Load areas which has a one for the types given by types. If at leats one node
* in one of the outer rings of the areas is marked in nodeUseMap as "used at least twice",
* index it into the areas map.
*
* If the number of indexed areas is bigger than parameter.GetRawWayBlockSize() types are
* dropped form areas until the number is again below the lmit.
*/
bool MergeAreasGenerator::GetAreas(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig,
const TypeInfoSet& candidateTypes,
TypeInfoSet& loadedTypes,
const std::unordered_set<Id>& nodeUseMap,
FileScanner& scanner,
FileWriter& writer,
std::vector<AreaMergeData>& mergeJob,
uint32_t& areasWritten)
{
bool firstCall=areasWritten==0; // We are called for the first time
uint32_t areaCount=0;
size_t collectedAreasCount=0;
size_t typesWithAreas=0;
for (auto& data : mergeJob) {
data.areaCount=0;
}
loadedTypes=candidateTypes;
scanner.GotoBegin();
scanner.Read(areaCount);
for (uint32_t a=1; a<=areaCount; a++) {
uint8_t type;
Id id;
AreaRef area=std::make_shared<Area>();
progress.SetProgress(a,areaCount);
scanner.Read(type);
scanner.Read(id);
area->ReadImport(typeConfig,
scanner);
mergeJob[area->GetType()->GetIndex()].areaCount++;
// This is an area of a type that does not get merged,
// we directly store it in the target file.
if (!loadedTypes.IsSet(area->GetType())) {
if (firstCall) {
writer.Write(type);
writer.Write(id);
area->WriteImport(typeConfig,
writer);
areasWritten++;
}
continue;
}
bool isMergeCandidate=false;
for (const auto& ring: area->rings) {
if (!ring.IsOuterRing()) {
continue;
}
for (const auto node : ring.nodes) {
if (nodeUseMap.find(node.GetId())!=nodeUseMap.end()) {
isMergeCandidate=true;
break;
}
}
if (isMergeCandidate) {
break;
}
}
if (!isMergeCandidate) {
continue;
}
if (mergeJob[area->GetType()->GetIndex()].areas.empty()) {
typesWithAreas++;
}
mergeJob[area->GetType()->GetIndex()].areas.push_back(area);
collectedAreasCount++;
while (collectedAreasCount>parameter.GetRawWayBlockSize() &&
typesWithAreas>1) {
TypeInfoRef victimType;
// Find the type with the smallest amount of ways loaded
for (auto &type : loadedTypes) {
if (!mergeJob[type->GetIndex()].areas.empty() &&
(!victimType ||
mergeJob[type->GetIndex()].areas.size()<mergeJob[victimType->GetIndex()].areas.size())) {
victimType=type;
}
}
// If there is more then one type of way, we always must find a "victim" type.
assert(victimType);
// Correct the statistics
collectedAreasCount-=mergeJob[victimType->GetIndex()].areas.size();
// Clear already loaded data of th victim type
mergeJob[victimType->GetIndex()].areas.clear();
typesWithAreas--;
loadedTypes.Remove(victimType);
}
}
progress.SetAction("Collected "+NumberToString(collectedAreasCount)+" areas for "+NumberToString(loadedTypes.Size())+" types");
return true;
}
bool AreaWayIndexGenerator::Import(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress)
{
FileScanner wayScanner;
FileWriter writer;
std::vector<TypeData> wayTypeData;
size_t maxLevel;
progress.Info("Minimum magnification: "+NumberToString(parameter.GetAreaWayMinMag()));
//
// Scanning distribution
//
progress.SetAction("Scanning level distribution of way types");
if (!CalculateDistribution(typeConfig,
parameter,
progress,
wayTypeData,
maxLevel)) {
return false;
}
// Calculate number of types which have data
uint32_t indexEntries=0;
for (const auto& type : typeConfig->GetWayTypes())
{
if (wayTypeData[type->GetIndex()].HasEntries()) {
indexEntries++;
}
}
//
// Writing index file
//
progress.SetAction("Generating 'areaway.idx'");
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areaway.idx"))) {
progress.Error("Cannot create 'areaway.idx'");
return false;
}
writer.Write(indexEntries);
for (const auto &type : typeConfig->GetWayTypes()) {
size_t i=type->GetIndex();
if (wayTypeData[i].HasEntries()) {
uint8_t dataOffsetBytes=0;
FileOffset bitmapOffset=0;
writer.WriteTypeId(type->GetWayId(),
typeConfig->GetWayTypeIdBytes());
writer.GetPos(wayTypeData[i].indexOffset);
writer.WriteFileOffset(bitmapOffset);
writer.Write(dataOffsetBytes);
writer.WriteNumber(wayTypeData[i].indexLevel);
writer.WriteNumber(wayTypeData[i].cellXStart);
writer.WriteNumber(wayTypeData[i].cellXEnd);
writer.WriteNumber(wayTypeData[i].cellYStart);
writer.WriteNumber(wayTypeData[i].cellYEnd);
}
}
if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"ways.dat"),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped())) {
progress.Error("Cannot open 'ways.dat'");
return false;
}
for (size_t l=parameter.GetAreaWayMinMag(); l<=maxLevel; l++) {
TypeInfoSet indexTypes(*typeConfig);
uint32_t wayCount;
double cellWidth=360.0/pow(2.0,(int)l);
double cellHeight=180.0/pow(2.0,(int)l);
wayScanner.GotoBegin();
for (const auto &type : typeConfig->GetWayTypes()) {
if (wayTypeData[type->GetIndex()].HasEntries() &&
wayTypeData[type->GetIndex()].indexLevel==l) {
indexTypes.Set(type);
}
}
if (indexTypes.Empty()) {
continue;
}
progress.Info("Scanning ways for index level "+NumberToString(l));
std::vector<CoordOffsetsMap> typeCellOffsets(typeConfig->GetTypeCount());
if (!wayScanner.Read(wayCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
Way way;
for (uint32_t w=1; w<=wayCount; w++) {
progress.SetProgress(w,wayCount);
FileOffset offset;
wayScanner.GetPos(offset);
if (!way.Read(*typeConfig,
wayScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(w)+" of "+
NumberToString(wayCount)+
" in file '"+
wayScanner.GetFilename()+"'");
return false;
}
if (!indexTypes.IsSet(way.GetType())) {
continue;
}
GeoBox boundingBox;
way.GetBoundingBox(boundingBox);
//
// Calculate minimum and maximum tile ids that are covered
// by the way
// Renormalized coordinate space (everything is >=0)
//
uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellWidth);
uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellWidth);
uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellHeight);
uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellHeight);
for (uint32_t y=minyc; y<=maxyc; y++) {
for (uint32_t x=minxc; x<=maxxc; x++) {
typeCellOffsets[way.GetType()->GetIndex()][Pixel(x,y)].push_back(offset);
}
}
}
for (const auto &type : indexTypes) {
size_t index=type->GetIndex();
if (!WriteBitmap(progress,
writer,
*typeConfig->GetTypeInfo(index),
wayTypeData[index],
typeCellOffsets[index])) {
return false;
}
}
}
return !writer.HasError() && writer.Close();
}
bool AreaNodeIndexGenerator::Import(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig)
{
FileScanner nodeScanner;
FileWriter writer;
std::set<TypeId> remainingNodeTypes; //! Set of types we still must process
std::vector<TypeData> nodeTypeData;
size_t level;
size_t maxLevel=0;
nodeTypeData.resize(typeConfig.GetTypes().size());
if (!nodeScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"nodes.dat"),
FileScanner::Sequential,
true)) {
progress.Error("Cannot open 'nodes.dat'");
return false;
}
//
// Scanning distribution
//
progress.SetAction("Scanning level distribution of node types");
// Initially we must process all types that represents nodes and that should
// not be ignored
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
!typeConfig.GetTypeInfo(i).GetIgnore()) {
remainingNodeTypes.insert(i);
}
}
level=parameter.GetAreaNodeMinMag();
while (!remainingNodeTypes.empty()) {
uint32_t nodeCount=0;
std::set<TypeId> currentNodeTypes(remainingNodeTypes);
double cellWidth=360.0/pow(2.0,(int)level);
double cellHeight=180.0/pow(2.0,(int)level);
std::vector<std::map<Pixel,size_t> > cellFillCount;
cellFillCount.resize(typeConfig.GetTypes().size());
progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingNodeTypes.size())+" types still to process)");
nodeScanner.GotoBegin();
if (!nodeScanner.Read(nodeCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
for (uint32_t n=1; n<=nodeCount; n++) {
progress.SetProgress(n,nodeCount);
FileOffset offset;
Node node;
nodeScanner.GetPos(offset);
if (!node.Read(nodeScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(n)+" of "+
NumberToString(nodeCount)+
" in file '"+
nodeScanner.GetFilename()+"'");
return false;
}
// If we still need to handle this type,
// count number of entries per type and tile cell
if (currentNodeTypes.find(node.GetType())!=currentNodeTypes.end()) {
uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth);
uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight);
cellFillCount[node.GetType()][Pixel(xc,yc)]++;
}
}
// Check statistics for each type
// If statistics are within goal limits, use this level
// for this type (else try again with the next higher level)
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (currentNodeTypes.find(i)!=currentNodeTypes.end()) {
size_t entryCount=0;
size_t max=0;
nodeTypeData[i].indexLevel=(uint32_t)level;
nodeTypeData[i].indexCells=cellFillCount[i].size();
nodeTypeData[i].indexEntries=0;
if (!cellFillCount[i].empty()) {
nodeTypeData[i].cellXStart=cellFillCount[i].begin()->first.x;
nodeTypeData[i].cellYStart=cellFillCount[i].begin()->first.y;
nodeTypeData[i].cellXEnd=nodeTypeData[i].cellXStart;
nodeTypeData[i].cellYEnd=nodeTypeData[i].cellYStart;
for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin();
cell!=cellFillCount[i].end();
++cell) {
nodeTypeData[i].indexEntries+=cell->second;
nodeTypeData[i].cellXStart=std::min(nodeTypeData[i].cellXStart,cell->first.x);
nodeTypeData[i].cellXEnd=std::max(nodeTypeData[i].cellXEnd,cell->first.x);
nodeTypeData[i].cellYStart=std::min(nodeTypeData[i].cellYStart,cell->first.y);
nodeTypeData[i].cellYEnd=std::max(nodeTypeData[i].cellYEnd,cell->first.y);
}
}
nodeTypeData[i].cellXCount=nodeTypeData[i].cellXEnd-nodeTypeData[i].cellXStart+1;
nodeTypeData[i].cellYCount=nodeTypeData[i].cellYEnd-nodeTypeData[i].cellYStart+1;
// Count absolute number of entries
for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin();
cell!=cellFillCount[i].end();
++cell) {
entryCount+=cell->second;
max=std::max(max,cell->second);
}
// Average number of entries per tile cell
double average=entryCount*1.0/cellFillCount[i].size();
// If we do not have any entries, we store it now
if (cellFillCount[i].empty()) {
continue;
}
// If the fill rate of the index is too low, we use this index level anyway
if (nodeTypeData[i].indexCells/(1.0*nodeTypeData[i].cellXCount*nodeTypeData[i].cellYCount)<=
parameter.GetAreaNodeIndexMinFillRate()) {
progress.Warning(typeConfig.GetTypeInfo(i).GetName()+" ("+NumberToString(i)+") is not well distributed");
continue;
}
// If average fill size and max fill size for tile cells
// is within limits, store it now.
if (max<=parameter.GetAreaNodeIndexCellSizeMax() &&
average<=parameter.GetAreaNodeIndexCellSizeAverage()) {
continue;
}
// else, we remove it from the list and try again with an higher
// level.
currentNodeTypes.erase(i);
}
}
// Now process all types for this limit, that are within the limits
for (std::set<TypeId>::const_iterator cnt=currentNodeTypes.begin();
cnt!=currentNodeTypes.end();
cnt++) {
maxLevel=std::max(maxLevel,level);
progress.Info("Type "+typeConfig.GetTypeInfo(*cnt).GetName()+"(" + NumberToString(*cnt)+"), "+NumberToString(nodeTypeData[*cnt].indexCells)+" cells, "+NumberToString(nodeTypeData[*cnt].indexEntries)+" objects");
remainingNodeTypes.erase(*cnt);
}
level++;
}
//
// Writing index file
//
progress.SetAction("Generating 'areanode.idx'");
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areanode.idx"))) {
progress.Error("Cannot create 'areanode.idx'");
return false;
}
uint32_t indexEntries=0;
// Count number of types in index
for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
{
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
nodeTypeData[i].HasEntries()) {
indexEntries++;
}
}
writer.Write(indexEntries);
// Store index data for each type
for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
{
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
nodeTypeData[i].HasEntries()) {
FileOffset bitmapOffset=0;
uint8_t dataOffsetBytes=0;
writer.WriteNumber(typeConfig.GetTypeInfo(i).GetId());
writer.GetPos(nodeTypeData[i].indexOffset);
writer.WriteFileOffset(bitmapOffset);
writer.Write(dataOffsetBytes);
writer.WriteNumber(nodeTypeData[i].indexLevel);
writer.WriteNumber(nodeTypeData[i].cellXStart);
writer.WriteNumber(nodeTypeData[i].cellXEnd);
writer.WriteNumber(nodeTypeData[i].cellYStart);
writer.WriteNumber(nodeTypeData[i].cellYEnd);
}
}
// Now store index bitmap for each type in increasing level order (why?)
for (size_t l=0; l<=maxLevel; l++) {
std::set<TypeId> indexTypes;
uint32_t nodeCount;
double cellWidth=360.0/pow(2.0,(int)l);
double cellHeight=180.0/pow(2.0,(int)l);
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
nodeTypeData[i].HasEntries() &&
nodeTypeData[i].indexLevel==l) {
indexTypes.insert(i);
}
}
if (indexTypes.empty()) {
continue;
}
progress.Info("Scanning nodes for index level "+NumberToString(l));
std::vector<std::map<Pixel,std::list<FileOffset> > > typeCellOffsets;
typeCellOffsets.resize(typeConfig.GetTypes().size());
nodeScanner.GotoBegin();
if (!nodeScanner.Read(nodeCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
//
// Collect all offsets
//
for (uint32_t n=1; n<=nodeCount; n++) {
progress.SetProgress(n,nodeCount);
FileOffset offset;
Node node;
nodeScanner.GetPos(offset);
if (!node.Read(nodeScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(n)+" of "+
NumberToString(nodeCount)+
" in file '"+
nodeScanner.GetFilename()+"'");
return false;
}
if (indexTypes.find(node.GetType())!=indexTypes.end()) {
uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth);
uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight);
typeCellOffsets[node.GetType()][Pixel(xc,yc)].push_back(offset);
}
}
//
// Write bitmap
//
for (std::set<TypeId>::const_iterator type=indexTypes.begin();
type!=indexTypes.end();
++type) {
size_t indexEntries=0;
size_t dataSize=0;
char buffer[10];
for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin();
cell!=typeCellOffsets[*type].end();
++cell) {
indexEntries+=cell->second.size();
dataSize+=EncodeNumber(cell->second.size(),buffer);
FileOffset previousOffset=0;
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
FileOffset data=*offset-previousOffset;
dataSize+=EncodeNumber(data,buffer);
previousOffset=*offset;
}
}
// "+1" because we add +1 to every offset, to generate offset > 0
uint8_t dataOffsetBytes=BytesNeededToAddressFileData(dataSize);
progress.Info("Writing map for "+
typeConfig.GetTypeInfo(*type).GetName()+", "+
NumberToString(typeCellOffsets[*type].size())+" cells, "+
NumberToString(indexEntries)+" entries, "+
ByteSizeToString(1.0*dataOffsetBytes*nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount));
FileOffset bitmapOffset;
if (!writer.GetPos(bitmapOffset)) {
progress.Error("Cannot get type index start position in file");
return false;
}
assert(nodeTypeData[*type].indexOffset!=0);
if (!writer.SetPos(nodeTypeData[*type].indexOffset)) {
progress.Error("Cannot go to type index offset in file");
return false;
}
writer.WriteFileOffset(bitmapOffset);
writer.Write(dataOffsetBytes);
if (!writer.SetPos(bitmapOffset)) {
progress.Error("Cannot go to type index start position in file");
return false;
}
// Write the bitmap with offsets for each cell
// We prefill with zero and only overwrite cells that have data
// So zero means "no data for this cell"
for (size_t i=0; i<nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount; i++) {
FileOffset cellOffset=0;
writer.WriteFileOffset(cellOffset,
dataOffsetBytes);
}
FileOffset dataStartOffset;
if (!writer.GetPos(dataStartOffset)) {
progress.Error("Cannot get start of data section after bitmap");
return false;
}
// Now write the list of offsets of objects for every cell with content
for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin();
cell!=typeCellOffsets[*type].end();
++cell) {
FileOffset bitmapCellOffset=bitmapOffset+
((cell->first.y-nodeTypeData[*type].cellYStart)*nodeTypeData[*type].cellXCount+
cell->first.x-nodeTypeData[*type].cellXStart)*dataOffsetBytes;
FileOffset previousOffset=0;
FileOffset cellOffset;
if (!writer.GetPos(cellOffset)) {
progress.Error("Cannot get cell start position in file");
return false;
}
if (!writer.SetPos(bitmapCellOffset)) {
progress.Error("Cannot go to cell start position in file");
return false;
}
writer.WriteFileOffset(cellOffset-dataStartOffset+1,
dataOffsetBytes);
if (!writer.SetPos(cellOffset)) {
progress.Error("Cannot go back to cell start position in file");
return false;
}
writer.WriteNumber((uint32_t)cell->second.size());
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
writer.WriteNumber((FileOffset)(*offset-previousOffset));
previousOffset=*offset;
}
}
}
}
return !writer.HasError() && writer.Close();
}
bool OptimizeAreaWayIdsGenerator::CopyWays(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig,
NodeUseMap& nodeUseMap)
{
FileScanner scanner;
FileWriter writer;
uint32_t dataCount=0;
progress.SetAction("Copy data from 'wayway.tmp' to 'ways.tmp'");
if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"wayway.tmp"),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped())) {
progress.Error(std::string("Cannot open '")+scanner.GetFilename()+"'");
return false;
}
if (!scanner.Read(dataCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"ways.tmp"))) {
progress.Error(std::string("Cannot create '")+writer.GetFilename()+"'");
return false;
}
writer.Write(dataCount);
for (uint32_t current=1; current<=dataCount; current++) {
uint8_t type;
Id id;
Way data;
progress.SetProgress(current,dataCount);
if (!scanner.Read(type) ||
!scanner.Read(id) ||
!data.Read(typeConfig,
scanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(current)+" of "+
NumberToString(dataCount)+
" in file '"+
scanner.GetFilename()+"'");
return false;
}
for (auto& id : data.ids) {
if (!nodeUseMap.IsNodeUsedAtLeastTwice(id)) {
id=0;
}
}
if (!writer.Write(type) ||
!writer.Write(id) ||
!data.Write(typeConfig,
writer)) {
progress.Error(std::string("Error while writing data entry to file '")+
writer.GetFilename()+"'");
return false;
}
}
if (!scanner.Close()) {
progress.Error(std::string("Error while closing file '")+
scanner.GetFilename()+"'");
return false;
}
if (!writer.Close()) {
progress.Error(std::string("Error while closing file '")+
writer.GetFilename()+"'");
return false;
}
return true;
}
