void OsmPbfWriterPrivate::writeFileblock(const QString &type, const QByteArray &unpackedData, const bool useCompression, const QByteArray &indexData)
{
if (type.isNull() || unpackedData.isNull())
qFatal("%s", qPrintable(QObject::tr("Type and/or unpacked data is null")));
if (!fileOpen)
throw OsmPbfWriter::FileNotOpenException();
if (unpackedData.size() > maxBlockSize)
maxBlockSize = unpackedData.size();
OSMPBF::Blob blobParsed;
if (useCompression) {
QByteArray payloadPacked = qCompress(unpackedData).mid(4);
blobParsed.set_raw_size(unpackedData.size());
blobParsed.set_zlib_data(payloadPacked.constData(), payloadPacked.size());
fileZlibDelta += (unpackedData.size() - payloadPacked.size()); // FIXME: this doesn't account for raw_size which is not set without compression
} else {
blobParsed.set_raw(unpackedData.constData(), unpackedData.length());
}
QByteArray blobPacked = packPbfMessageToArray(blobParsed);
OSMPBF::BlobHeader blobHeaderParsed;
blobHeaderParsed.set_type(type.toUtf8().constData(), type.toUtf8().size());
blobHeaderParsed.set_datasize(blobPacked.length());
if (!indexData.isNull())
blobHeaderParsed.set_indexdata(indexData.constData(), indexData.length());
QByteArray blobHeaderPacked = packPbfMessageToArray(blobHeaderParsed);
fileOut.write(arrayFromBEU(blobHeaderPacked.length(), 4));
fileOut.write(blobHeaderPacked);
fileOut.write(blobPacked);
blocksCount++;
fileRealSize += (4 + blobHeaderPacked.size() + blobPacked.size());
}
void CThreadLoader::Start(CThreadUnit** pTasks, int countTasks)
{
m_pprimblock=new OSMPBF::PrimitiveBlock;
if(m_nCount >0)
{
assert(false);
}
boost::timer::nanosecond_type const secunda(1000000000LL);
boost::timer::cpu_timer uTimer;
//unsigned uTime=0;
if(m_nThredNumber==0)
{
uTimer.start();
}
std::vector<char> buffer_blob_header;
std::vector<char> buffer(OSMPBF::max_uncompressed_blob_size);
std::vector<unsigned char> unpack_buffer(OSMPBF::max_uncompressed_blob_size);
const char* pbWork;
//for(int l=0;l<20;++l)
for(;;)
{
// storage of size, used multiple times
__int32 sz;
OSMPBF::BlobHeader blobheader;
{
//Начало паралельной работы
boost::lock_guard<boost::mutex> l(*m_pcsFile);
if(feof(m_fp))
break;
// read the first 4 bytes of the file, this is the size of the blob-header
if(fread(&sz, sizeof(sz), 1, m_fp) != 1)
break; // end of file reached
// convert the size from network byte-order to host byte-order
sz = ntohl(sz);
// ensure the blob-header is smaller then MAX_BLOB_HEADER_SIZE
if(sz > OSMPBF::max_blob_header_size)
err("blob-header-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_blob_header_size);
buffer_blob_header.resize(sz);
// read the blob-header from the file
if(fread(&buffer_blob_header[0], sz, 1, m_fp) != 1)
err("unable to read blob-header from file");
// parse the blob-header from the read-buffer
if(!blobheader.ParseFromArray(&buffer_blob_header[0], sz))
err("unable to parse blob header");
// size of the following blob
sz = blobheader.m_datasize.m_val;
// ensure the blob is smaller then MAX_BLOB_SIZE
if(sz > OSMPBF::max_uncompressed_blob_size)
err("blob-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_uncompressed_blob_size);
// read the blob from the file
if(fread(&buffer[0], sz, 1, m_fp) != 1)
err("unable to read blob from file");
//Отсюда можно работать паралельно
}
// ++m_nCount; continue;
// parse the blob from the read-buffer
OSMPBF::Blob blob;
if(!blob.ParseFromArray(&buffer[0], sz))
err("unable to parse blob");
// set when we find at least one data stream
bool found_data = false;
// if the blob has uncompressed data
if(!blob.m_raw.empty()) {
// we have at least one datastream
found_data = true;
// size of the blob-data
sz = blob.m_raw.size();
// check that raw_size is set correctly
if(sz != blob.m_raw_size.m_val)
warn(" reports wrong raw_size: %u bytes", blob.m_raw_size.m_val);
// copy the uncompressed data over to the unpack_buffer
//memcpy(&unpack_buffer[0], &buffer[0], sz);
pbWork=&buffer[0];
}
// if the blob has zlib-compressed data
if(!blob.m_zlib_data.empty())
{
// issue a warning if there is more than one data steam, a blob may only contain one data stream
if(found_data)
warn(" contains several data streams");
// we have at least one datastream
found_data = true;
// the size of the compressesd data
sz = blob.m_zlib_data.size();
// zlib information
z_stream z;
// next byte to decompress
z.next_in = (z_const Bytef *) blob.m_zlib_data.m_pBegin;
// number of bytes to decompress
z.avail_in = sz;
// place of next decompressed byte
z.next_out = &unpack_buffer[0];
pbWork= (const char*) &unpack_buffer[0];
// space for decompressed data
z.avail_out = blob.m_raw_size.m_val;
// misc
z.zalloc = Z_NULL;
z.zfree = Z_NULL;
z.opaque = Z_NULL;
if(inflateInit(&z) != Z_OK) {
err(" failed to init zlib stream");
}
if(inflate(&z, Z_FINISH) != Z_STREAM_END) {
err(" failed to inflate zlib stream");
}
if(inflateEnd(&z) != Z_OK) {
err(" failed to deinit zlib stream");
}
// unpacked size
sz = z.total_out;
}
// if the blob has lzma-compressed data
if(!blob.m_lzma_data.empty()) {
// issue a warning if there is more than one data steam, a blob may only contain one data stream
if(found_data)
warn(" contains several data streams");
// we have at least one datastream
found_data = true;
// issue a warning, lzma compression is not yet supported
err(" lzma-decompression is not supported");
}
// check we have at least one data-stream
if(!found_data)
err(" does not contain any known data stream");
// switch between different blob-types
if(blobheader.m_type.compareString("OSMHeader")==0)
{
OSMPBF::Headerblock headerblock ;
// parse the HeaderBlock from the blob
if(!headerblock.ParseFromArray(pbWork, sz))
err("unable to parse header block");
}
else if(blobheader.m_type.compareString("OSMData")==0)
{
m_pprimblock->Clear();
// parse the PrimitiveBlock from the blob
if(!m_pprimblock->ParseFromArray(pbWork, sz))
err("unable to parse primitive block");
// iterate over all PrimitiveGroups
for(int i = 0, l = m_pprimblock->m_primitivegroup.size(); i < l; i++)
{
// one PrimitiveGroup from the the Block
OSMPBF::PrimitiveGroup& pg = m_pprimblock->m_primitivegroup[i];
bool found_items=false;
// tell about nodes
if(pg.m_nodes.size() > 0)
{
found_items = true;
for(size_t i=0;i<pg.m_nodes.size();++i)
AddNode(pg.m_nodes[i]);
}
// tell about dense nodes
if(!pg.m_dense.empty()) {
found_items = true;
AddDense(pg.m_dense);
}
// tell about ways
if(pg.m_ways.size() > 0) {
found_items = true;
for(size_t i=0;i<pg.m_ways.size();++i)
AddWay(pg.m_ways[i]);
}
// tell about relations
if(pg.m_relations.size() > 0) {
found_items = true;
for(size_t i=0;i<pg.m_relations.size();++i)
AddRelations(pg.m_relations[i]);
}
if(!found_items)
warn(" contains no items");
}
}
else {
// unknown blob type
warn(" unknown blob type: %s", blobheader.m_type.toString().c_str());
}
//////////////////////////////////////////////////////////////////////////
LONG l =INTERLOCKED_INCREMENT(m_nCount);
if(m_nThredNumber==0)
{
boost::timer::cpu_times const elapsed_times(uTimer.elapsed());
boost::timer::nanosecond_type elapsed(elapsed_times.wall);
if(elapsed >= secunda)
{
info("Bloks=%d",l);
uTimer.start();
}
}
}
m_tabNode_Cash.Save();
m_tabNode_NotVisible_Cash.Save();
m_tabNI_Cash.Save();
m_tabNI_NotVisible_Cash.Save();
m_tabWI_Cash.Save();
m_tabWI_NotVisible_Cash.Save();
m_tabRI_Cash.Save();
m_tabRI_NotVisible_Cash.Save();
m_tabkvNode_cash.Save();
m_tabkvWay_cash.Save();
m_tabkvRelation_cash.Save();
delete m_pprimblock;
m_pprimblock=NULL;
}
// application main method
int main(int argc, char *argv[]) {
// check if the output is a tty so we can use colors
#ifdef WIN32
usecolor = 0;
#else
usecolor = isatty(1);
#endif
static struct option long_options[] = {
{"color", no_argument, 0, 'c'},
{0, 0, 0, 0}
};
while (1) {
int c = getopt_long(argc, argv, "c", long_options, 0);
if (c == -1) {
break;
}
switch (c) {
case 'c':
usecolor = true;
break;
default:
exit(1);
}
}
// check for proper command line args
if (optind != argc-1) {
err("usage: %s [--color] file.osm.pbf", argv[0]);
}
// open specified file
FILE *fp = fopen(argv[optind], "rb");
if (!fp) {
err("can't open file '%s'", argv[optind]);
}
// read while the file has not reached its end
while (!feof(fp)) {
// storage of size, used multiple times
int32_t sz;
// read the first 4 bytes of the file, this is the size of the blob-header
if (fread(&sz, sizeof(sz), 1, fp) != 1) {
break; // end of file reached
}
// convert the size from network byte-order to host byte-order
sz = ntohl(sz);
// ensure the blob-header is smaller then MAX_BLOB_HEADER_SIZE
if (sz > OSMPBF::max_blob_header_size) {
err("blob-header-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_blob_header_size);
}
// read the blob-header from the file
if (fread(buffer, sz, 1, fp) != 1) {
err("unable to read blob-header from file");
}
// parse the blob-header from the read-buffer
if (!blobheader.ParseFromArray(buffer, sz)) {
err("unable to parse blob header");
}
// tell about the blob-header
info("\nBlobHeader (%d bytes)", sz);
debug(" type = %s", blobheader.type().c_str());
// size of the following blob
sz = blobheader.datasize();
debug(" datasize = %u", sz);
// optional indexdata
if (blobheader.has_indexdata()) {
debug(" indexdata = %u bytes", blobheader.indexdata().size());
}
// ensure the blob is smaller then MAX_BLOB_SIZE
if (sz > OSMPBF::max_uncompressed_blob_size) {
err("blob-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_uncompressed_blob_size);
}
// read the blob from the file
if (fread(buffer, sz, 1, fp) != 1) {
err("unable to read blob from file");
}
// parse the blob from the read-buffer
if (!blob.ParseFromArray(buffer, sz)) {
err("unable to parse blob");
}
// tell about the blob-header
info("Blob (%d bytes)", sz);
// set when we find at least one data stream
bool found_data = false;
// if the blob has uncompressed data
if (blob.has_raw()) {
// we have at least one datastream
found_data = true;
// size of the blob-data
sz = blob.raw().size();
// check that raw_size is set correctly
if (sz != blob.raw_size()) {
warn(" reports wrong raw_size: %u bytes", blob.raw_size());
}
// tell about the blob-data
debug(" contains uncompressed data: %u bytes", sz);
// copy the uncompressed data over to the unpack_buffer
memcpy(unpack_buffer, buffer, sz);
}
// if the blob has zlib-compressed data
if (blob.has_zlib_data()) {
// issue a warning if there is more than one data steam, a blob may only contain one data stream
if (found_data) {
warn(" contains several data streams");
}
// we have at least one datastream
found_data = true;
// the size of the compressesd data
sz = blob.zlib_data().size();
// tell about the compressed data
debug(" contains zlib-compressed data: %u bytes", sz);
debug(" uncompressed size: %u bytes", blob.raw_size());
// zlib information
z_stream z;
// next byte to decompress
z.next_in = (unsigned char*) blob.zlib_data().c_str();
// number of bytes to decompress
z.avail_in = sz;
// place of next decompressed byte
z.next_out = (unsigned char*) unpack_buffer;
// space for decompressed data
z.avail_out = blob.raw_size();
// misc
z.zalloc = Z_NULL;
z.zfree = Z_NULL;
z.opaque = Z_NULL;
if (inflateInit(&z) != Z_OK) {
err(" failed to init zlib stream");
}
if (inflate(&z, Z_FINISH) != Z_STREAM_END) {
err(" failed to inflate zlib stream");
}
if (inflateEnd(&z) != Z_OK) {
err(" failed to deinit zlib stream");
}
// unpacked size
sz = z.total_out;
}
// if the blob has lzma-compressed data
if (blob.has_lzma_data()) {
// issue a warning if there is more than one data steam, a blob may only contain one data stream
if (found_data) {
warn(" contains several data streams");
}
// we have at least one datastream
found_data = true;
// tell about the compressed data
debug(" contains lzma-compressed data: %u bytes", blob.lzma_data().size());
debug(" uncompressed size: %u bytes", blob.raw_size());
// issue a warning, lzma compression is not yet supported
err(" lzma-decompression is not supported");
}
// check we have at least one data-stream
if (!found_data) {
err(" does not contain any known data stream");
}
// switch between different blob-types
if (blobheader.type() == "OSMHeader") {
// tell about the OSMHeader blob
info(" OSMHeader");
// parse the HeaderBlock from the blob
if (!headerblock.ParseFromArray(unpack_buffer, sz)) {
err("unable to parse header block");
}
// tell about the bbox
if (headerblock.has_bbox()) {
OSMPBF::HeaderBBox bbox = headerblock.bbox();
debug(" bbox: %.7f,%.7f,%.7f,%.7f",
(double)bbox.left() / OSMPBF::lonlat_resolution,
(double)bbox.bottom() / OSMPBF::lonlat_resolution,
(double)bbox.right() / OSMPBF::lonlat_resolution,
(double)bbox.top() / OSMPBF::lonlat_resolution);
}
// tell about the required features
for (int i = 0, l = headerblock.required_features_size(); i < l; i++) {
debug(" required_feature: %s", headerblock.required_features(i).c_str());
}
// tell about the optional features
for (int i = 0, l = headerblock.optional_features_size(); i < l; i++) {
debug(" optional_feature: %s", headerblock.optional_features(i).c_str());
}
// tell about the writing program
if (headerblock.has_writingprogram()) {
debug(" writingprogram: %s", headerblock.writingprogram().c_str());
}
// tell about the source
if (headerblock.has_source()) {
debug(" source: %s", headerblock.source().c_str());
}
} else if (blobheader.type() == "OSMData") {
// tell about the OSMData blob
info(" OSMData");
// parse the PrimitiveBlock from the blob
if (!primblock.ParseFromArray(unpack_buffer, sz)) {
err("unable to parse primitive block");
}
// tell about the block's meta info
debug(" granularity: %u", primblock.granularity());
debug(" lat_offset: %u", primblock.lat_offset());
debug(" lon_offset: %u", primblock.lon_offset());
debug(" date_granularity: %u", primblock.date_granularity());
// tell about the stringtable
debug(" stringtable: %u items", primblock.stringtable().s_size());
// number of PrimitiveGroups
debug(" primitivegroups: %u groups", primblock.primitivegroup_size());
// iterate over all PrimitiveGroups
for (int i = 0, l = primblock.primitivegroup_size(); i < l; i++) {
// one PrimitiveGroup from the the Block
OSMPBF::PrimitiveGroup pg = primblock.primitivegroup(i);
bool found_items=false;
// tell about nodes
if (pg.nodes_size() > 0) {
found_items = true;
debug(" nodes: %d", pg.nodes_size());
if (pg.nodes(0).has_info()) {
debug(" with meta-info");
}
}
// tell about dense nodes
if (pg.has_dense()) {
found_items = true;
debug(" dense nodes: %d", pg.dense().id_size());
if (pg.dense().has_denseinfo()) {
debug(" with meta-info");
}
}
// tell about ways
if (pg.ways_size() > 0) {
found_items = true;
debug(" ways: %d", pg.ways_size());
if (pg.ways(0).has_info()) {
debug(" with meta-info");
}
}
// tell about relations
if (pg.relations_size() > 0) {
found_items = true;
debug(" relations: %d", pg.relations_size());
if (pg.relations(0).has_info()) {
debug(" with meta-info");
}
}
if (!found_items) {
warn(" contains no items");
}
}
}
else {
// unknown blob type
warn(" unknown blob type: %s", blobheader.type().c_str());
}
}
// close the file pointer
fclose(fp);
// clean up the protobuf lib
google::protobuf::ShutdownProtobufLibrary();
}