/*-------------------------------------------------------------------*
| dxfReadBlockHeader |
| Inputs: |
| HDXF hDxf = handle to the openning DXF file structure |
| PDXFBLOCKHEADER pBlockHeader = |
| pointer to blockheader structure |
| Output: TRUE if everything is ok |
*-------------------------------------------------------------------*/
BOOL dxfReadBlockHeader( HDXF hDxf, PDXFBLOCKHEADER pBlockHeader )
{
PDXF pDxf;
int GCode;
char strValue[2048];
// Initialize pDxf ------------------
if((pDxf = InitilizePDXF(hDxf))==NULL)
return FALSE;
// Check if current section is BLOCKS
if(pDxf->Read.CurrentSection!=SEC_BLOCKS)
{
// Current section is not BLOCKS
GlobalUnlock(hDxf);
return FALSE;
}
dxfStorePos(pDxf);
ReadParamFromDxfFile(pDxf, GCode, strValue);
if((GCode==0) && (strcmp(strValue, "BLOCK")==0))
ReadBlockHeader(pDxf, pBlockHeader);
else
{
// Can not read block header
dxfRestorePos(pDxf);
GlobalUnlock(hDxf);
return FALSE;
}
// UnInitilize hDxf -----------------
return UnInitilizePDXF(hDxf);
}
/*-------------------------------------------------------------------*
| dxfFindBlockHeader |
| Inputs: |
| HDXF hDxf = handle to the openning DXF file structure |
| PDXFBLOCKHEADER pBlockHeader = |
| pointer to blockheader structure |
| Output: TRUE if everything is ok |
*-------------------------------------------------------------------*/
BOOL dxfFindBlockHeader( HDXF hDxf, PDXFBLOCKHEADER pBlockHeader )
{
PDXF pDxf;
// Initialize pDxf ------------------
if((pDxf = InitilizePDXF(hDxf))==NULL)
return FALSE;
// Check if current section is BLOCKS
if(pDxf->Read.CurrentSection!=SEC_BLOCKS)
{
// Current section is not BLOCKS
GlobalUnlock(hDxf);
return FALSE;
}
dxfStorePos(pDxf);
if(FindParamFromDxfFile(pDxf, 0, "BLOCK"))
ReadBlockHeader(pDxf, pBlockHeader);
else
{
// Can not read block header
dxfRestorePos(pDxf);
GlobalUnlock(hDxf);
return FALSE;
}
// UnInitilize hDxf -----------------
return UnInitilizePDXF(hDxf);
}
void Unpack::Unpack5(bool Solid)
{
FileExtracted=true;
if (!Suspended)
{
UnpInitData(Solid);
if (!UnpReadBuf())
return;
if (!ReadBlockHeader(Inp,BlockHeader) || !ReadTables(Inp,BlockHeader,BlockTables))
return;
}
while (true)
{
UnpPtr&=MaxWinMask;
if (Inp.InAddr>=ReadBorder)
{
bool FileDone=false;
// We use 'while', because for empty block containing only Huffman table,
// we'll be on the block border once again just after reading the table.
while (Inp.InAddr>BlockHeader.BlockStart+BlockHeader.BlockSize-1 ||
Inp.InAddr==BlockHeader.BlockStart+BlockHeader.BlockSize-1 &&
Inp.InBit>=BlockHeader.BlockBitSize)
{
if (BlockHeader.LastBlockInFile)
{
FileDone=true;
break;
}
if (!ReadBlockHeader(Inp,BlockHeader) || !ReadTables(Inp,BlockHeader,BlockTables))
return;
}
if (FileDone || !UnpReadBuf())
break;
}
if (((WriteBorder-UnpPtr) & MaxWinMask)<MAX_LZ_MATCH+3 && WriteBorder!=UnpPtr)
{
UnpWriteBuf();
if (WrittenFileSize>DestUnpSize)
return;
if (Suspended)
{
FileExtracted=false;
return;
}
}
uint MainSlot=DecodeNumber(Inp,&BlockTables.LD);
if (MainSlot<256)
{
if (Fragmented)
FragWindow[UnpPtr++]=(byte)MainSlot;
else
Window[UnpPtr++]=(byte)MainSlot;
continue;
}
if (MainSlot>=262)
{
uint Length=SlotToLength(Inp,MainSlot-262);
uint DBits,Distance=1,DistSlot=DecodeNumber(Inp,&BlockTables.DD);
if (DistSlot<4)
{
DBits=0;
Distance+=DistSlot;
}
else
{
DBits=DistSlot/2 - 1;
Distance+=(2 | (DistSlot & 1)) << DBits;
}
if (DBits>0)
{
if (DBits>=4)
{
if (DBits>4)
{
Distance+=((Inp.getbits32()>>(36-DBits))<<4);
Inp.addbits(DBits-4);
}
uint LowDist=DecodeNumber(Inp,&BlockTables.LDD);
Distance+=LowDist;
}
else
{
Distance+=Inp.getbits32()>>(32-DBits);
Inp.addbits(DBits);
}
}
void Unpack::Unpack5MT(bool Solid)
{
InitMT();
UnpInitData(Solid);
for (uint I=0;I<MaxUserThreads*UNP_BLOCKS_PER_THREAD;I++)
{
UnpackThreadData *CurData=UnpThreadData+I;
CurData->LargeBlock=false;
CurData->Incomplete=false;
}
UnpThreadData[0].BlockHeader=BlockHeader;
UnpThreadData[0].BlockTables=BlockTables;
uint LastBlockNum=0;
int DataSize=0;
int BlockStart=0;
// 'true' if we found a block too large for multithreaded extraction,
// so we switched to single threaded mode until the end of file.
// Large blocks could cause too high memory use in multithreaded mode.
bool LargeBlock=false;
bool Done=false;
while (!Done)
{
int ReadSize=UnpIO->UnpRead(ReadBufMT+DataSize,(UNP_READ_SIZE_MT-DataSize)&~0xf);
if (ReadSize<0)
break;
DataSize+=ReadSize;
if (DataSize==0)
break;
bool BufferProcessed=false;
while (BlockStart<DataSize && !Done)
{
// Data amount, which is guaranteed to fit block header and tables,
// so we can safely read them without additional checks.
const int TooSmallToProcess=1024;
uint BlockNumber=0,BlockNumberMT=0;
while (BlockNumber<MaxUserThreads*UNP_BLOCKS_PER_THREAD)
{
UnpackThreadData *CurData=UnpThreadData+BlockNumber;
LastBlockNum=BlockNumber;
CurData->UnpackPtr=this;
// 'Incomplete' thread is present. This is a thread processing block
// in the end of buffer, split between two read operations.
if (CurData->Incomplete)
CurData->DataSize=DataSize;
else
{
CurData->Inp.SetExternalBuffer(ReadBufMT+BlockStart);
CurData->Inp.InitBitInput();
CurData->DataSize=DataSize-BlockStart;
if (CurData->DataSize==0)
break;
CurData->DamagedData=false;
CurData->HeaderRead=false;
CurData->TableRead=false;
}
// We should not use 'last block in file' block flag here unless
// we'll check the block size, because even if block is last in file,
// it can exceed the current buffer and require more reading.
CurData->NoDataLeft=(ReadSize==0);
CurData->Incomplete=false;
CurData->ThreadNumber=BlockNumber;
if (!CurData->HeaderRead)
{
CurData->HeaderRead=true;
if (!ReadBlockHeader(CurData->Inp,CurData->BlockHeader))
{
Done=true;
break;
}
}
// To prevent too high memory use we switch to single threaded mode
// if block exceeds this size. Typically RAR blocks do not exceed
// 64 KB, so this protection should not affect most of valid archives.
const int LargeBlockSize=0x20000;
if (LargeBlock || CurData->BlockHeader.BlockSize>LargeBlockSize)
LargeBlock=CurData->LargeBlock=true;
else
BlockNumberMT++; // Number of normal blocks processed in MT mode.
BlockStart+=CurData->BlockHeader.HeaderSize+CurData->BlockHeader.BlockSize;
BlockNumber++;
int DataLeft=DataSize-BlockStart;
if (DataLeft>=0 && CurData->BlockHeader.LastBlockInFile)
break;
// For second and following threads we move smaller blocks to buffer
// start to ensure that we have enough data to fit block header
// and tables.
if (DataLeft<TooSmallToProcess)
break;
}
//#undef USE_THREADS
UnpackThreadDataList UTDArray[MaxPoolThreads];
uint UTDArrayPos=0;
uint MaxBlockPerThread=BlockNumberMT/MaxUserThreads;
if (BlockNumberMT%MaxUserThreads!=0)
MaxBlockPerThread++;
// Decode all normal blocks until the first 'large' if any.
for (uint CurBlock=0;CurBlock<BlockNumberMT;CurBlock+=MaxBlockPerThread)
{
UnpackThreadDataList *UTD=UTDArray+UTDArrayPos++;
UTD->D=UnpThreadData+CurBlock;
UTD->BlockCount=Min(MaxBlockPerThread,BlockNumberMT-CurBlock);
#ifdef USE_THREADS
if (BlockNumber==1)
UnpackDecode(*UTD->D);
else
UnpThreadPool->AddTask(UnpackDecodeThread,(void*)UTD);
#else
for (uint I=0;I<UTD->BlockCount;I++)
UnpackDecode(UTD->D[I]);
#endif
}
if (BlockNumber==0)
break;
#ifdef USE_THREADS
UnpThreadPool->WaitDone();
#endif
bool IncompleteThread=false;
for (uint Block=0;Block<BlockNumber;Block++)
{
UnpackThreadData *CurData=UnpThreadData+Block;
if (!CurData->LargeBlock && !ProcessDecoded(*CurData) ||
CurData->LargeBlock && !UnpackLargeBlock(*CurData) ||
CurData->DamagedData)
{
Done=true;
break;
}
if (CurData->Incomplete)
{
int BufPos=int(CurData->Inp.InBuf+CurData->Inp.InAddr-ReadBufMT);
if (DataSize<=BufPos) // Thread exceeded input buffer boundary.
{
Done=true;
break;
}
IncompleteThread=true;
memmove(ReadBufMT,ReadBufMT+BufPos,DataSize-BufPos);
CurData->BlockHeader.BlockSize-=CurData->Inp.InAddr-CurData->BlockHeader.BlockStart;
CurData->BlockHeader.HeaderSize=0;
CurData->BlockHeader.BlockStart=0;
CurData->Inp.InBuf=ReadBufMT;
CurData->Inp.InAddr=0;
if (Block!=0)
{
// Move the incomplete thread entry to the first position,
// so we'll start processing from it. Preserve the original
// buffer for decoded data.
UnpackDecodedItem *Decoded=UnpThreadData[0].Decoded;
uint DecodedAllocated=UnpThreadData[0].DecodedAllocated;
UnpThreadData[0]=*CurData;
UnpThreadData[0].Decoded=Decoded;
UnpThreadData[0].DecodedAllocated=DecodedAllocated;
CurData->Incomplete=false;
}
BlockStart=0;
DataSize-=BufPos;
break;
}
else
if (CurData->BlockHeader.LastBlockInFile)
{
Done=true;
break;
}
}
if (IncompleteThread || Done)
break; // Current buffer is done, read more data or quit.
else
{
int DataLeft=DataSize-BlockStart;
if (DataLeft<TooSmallToProcess)
{
if (DataLeft<0) // Invalid data, must not happen in valid archive.
{
Done=true;
break;
}
// If we do not have incomplete thread and have some data
// in the end of buffer, too small for single thread,
// let's move it to beginning of next buffer.
if (DataLeft>0)
memmove(ReadBufMT,ReadBufMT+BlockStart,DataLeft);
DataSize=DataLeft;
BlockStart=0;
break; // Current buffer is done, try to read more data.
}
}
}
}
UnpWriteBuf();
BlockHeader=UnpThreadData[LastBlockNum].BlockHeader;
BlockTables=UnpThreadData[LastBlockNum].BlockTables;
}
bool PreprocessPBF::Import(const TypeConfigRef& typeConfig,
const ImportParameter& /*parameter*/,
Progress& progress,
const std::string& filename)
{
FileOffset fileSize;
FileOffset currentPosition;
progress.SetAction(std::string("Parsing *.osm.pbf file '")+filename+"'");
try {
fileSize=GetFileSize(filename);
FILE* file;
file=fopen(filename.c_str(),"rb");
if (file==NULL) {
progress.Error("Cannot open file!");
return false;
}
// BlockHeader
PBF::BlockHeader blockHeader;
if (!ReadBlockHeader(progress,file,blockHeader,false)) {
fclose(file);
return false;
}
if (blockHeader.type()!="OSMHeader") {
progress.Error("File '"+filename+"' is not an OSM PBF file!");
fclose(file);
return false;
}
PBF::HeaderBlock headerBlock;
if (!ReadHeaderBlock(progress,
file,
blockHeader,
headerBlock)) {
fclose(file);
return false;
}
for (int i=0; i<headerBlock.required_features_size(); i++) {
std::string feature=headerBlock.required_features(i);
if (feature!="OsmSchema-V0.6" &&
feature!="DenseNodes") {
progress.Error(std::string("Unsupported feature '")+feature+"'");
fclose(file);
return false;
}
}
nodes.reserve(20000);
members.reserve(2000);
std::future<void> currentBlockTask;
while (true) {
PBF::BlockHeader blockHeader;
if (!GetPos(file,
currentPosition)) {
progress.Error("Cannot read current position in '"+filename+"'!");
fclose(file);
return false;
}
progress.SetProgress(currentPosition,
fileSize);
if (!ReadBlockHeader(progress,
file,
blockHeader,
true)) {
fclose(file);
break;
}
if (blockHeader.type()!="OSMData") {
progress.Error("File '"+filename+"' is not an OSM PBF file!");
fclose(file);
return false;
}
std::unique_ptr<PBF::PrimitiveBlock> block(new PBF::PrimitiveBlock());
if (!ReadPrimitiveBlock(progress,
file,
blockHeader,
*block)) {
fclose(file);
return false;
}
if (currentBlockTask.valid()) {
currentBlockTask.get();
}
currentBlockTask=std::async(std::launch::async,
&PreprocessPBF::ProcessBlock,this,
typeConfig,
std::move(block));
}
if (currentBlockTask.valid()) {
currentBlockTask.get();
}
}
catch (IOException& e) {
progress.Error(e.GetDescription());
return false;
}
return true;
}
