예제 #1
0
void OsmPbfWriterPrivate::writeFileblock(const QString &type, const QByteArray &unpackedData, const bool useCompression, const QByteArray &indexData)
{
    if (type.isNull() || unpackedData.isNull())
        qFatal("%s", qPrintable(QObject::tr("Type and/or unpacked data is null")));

    if (!fileOpen)
        throw OsmPbfWriter::FileNotOpenException();

    if (unpackedData.size() > maxBlockSize)
        maxBlockSize = unpackedData.size();

    OSMPBF::Blob blobParsed;
    if (useCompression) {
        QByteArray payloadPacked = qCompress(unpackedData).mid(4);
        blobParsed.set_raw_size(unpackedData.size());
        blobParsed.set_zlib_data(payloadPacked.constData(), payloadPacked.size());
        fileZlibDelta += (unpackedData.size() - payloadPacked.size());  // FIXME: this doesn't account for raw_size which is not set without compression
    } else {
        blobParsed.set_raw(unpackedData.constData(), unpackedData.length());
    }
    QByteArray blobPacked = packPbfMessageToArray(blobParsed);

    OSMPBF::BlobHeader blobHeaderParsed;
    blobHeaderParsed.set_type(type.toUtf8().constData(), type.toUtf8().size());
    blobHeaderParsed.set_datasize(blobPacked.length());
    if (!indexData.isNull())
        blobHeaderParsed.set_indexdata(indexData.constData(), indexData.length());
    QByteArray blobHeaderPacked = packPbfMessageToArray(blobHeaderParsed);

    fileOut.write(arrayFromBEU(blobHeaderPacked.length(), 4));
    fileOut.write(blobHeaderPacked);
    fileOut.write(blobPacked);

    blocksCount++;
    fileRealSize += (4 + blobHeaderPacked.size() + blobPacked.size());
}
예제 #2
0
void CThreadLoader::Start(CThreadUnit** pTasks, int countTasks)
{
    m_pprimblock=new OSMPBF::PrimitiveBlock;

	 if(m_nCount >0)
	 {
		 assert(false);
	 }

	 boost::timer::nanosecond_type const secunda(1000000000LL);
	 boost::timer::cpu_timer uTimer;
	 //unsigned uTime=0;
	 if(m_nThredNumber==0)
	 {
		 uTimer.start();

	 }




	 std::vector<char> buffer_blob_header;
	 std::vector<char>  buffer(OSMPBF::max_uncompressed_blob_size);
	 std::vector<unsigned char> unpack_buffer(OSMPBF::max_uncompressed_blob_size);

	 const char* pbWork;

	 //for(int l=0;l<20;++l)
	 for(;;)
	 {
		 // storage of size, used multiple times
		 __int32 sz;
		 OSMPBF::BlobHeader blobheader;

		 {
			 //Начало паралельной работы
			boost::lock_guard<boost::mutex> l(*m_pcsFile);
			if(feof(m_fp))
				break;


			 // read the first 4 bytes of the file, this is the size of the blob-header
			 if(fread(&sz, sizeof(sz), 1, m_fp) != 1)
				 break; // end of file reached

			 // convert the size from network byte-order to host byte-order
			 sz = ntohl(sz);

			 // ensure the blob-header is smaller then MAX_BLOB_HEADER_SIZE
			 if(sz > OSMPBF::max_blob_header_size)
				 err("blob-header-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_blob_header_size);

			 buffer_blob_header.resize(sz);
			 // read the blob-header from the file
			 if(fread(&buffer_blob_header[0], sz, 1, m_fp) != 1)
				 err("unable to read blob-header from file");


			 // parse the blob-header from the read-buffer
			 if(!blobheader.ParseFromArray(&buffer_blob_header[0], sz))
				 err("unable to parse blob header");

			 // size of the following blob
			 sz = blobheader.m_datasize.m_val;

			 // ensure the blob is smaller then MAX_BLOB_SIZE
			 if(sz > OSMPBF::max_uncompressed_blob_size)
				 err("blob-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_uncompressed_blob_size);

			 // read the blob from the file
			 if(fread(&buffer[0], sz, 1, m_fp) != 1)
				 err("unable to read blob from file");

			 //Отсюда можно работать паралельно

		 }
		// ++m_nCount; continue;


		 // parse the blob from the read-buffer
		 OSMPBF::Blob blob;
		 if(!blob.ParseFromArray(&buffer[0], sz))
			 err("unable to parse blob");

		 // set when we find at least one data stream
		 bool found_data = false;

		 // if the blob has uncompressed data
		 if(!blob.m_raw.empty()) {
			 // we have at least one datastream
			 found_data = true;

			 // size of the blob-data
			 sz = blob.m_raw.size();

			 // check that raw_size is set correctly
			 if(sz != blob.m_raw_size.m_val)
				 warn("  reports wrong raw_size: %u bytes", blob.m_raw_size.m_val);

			 // copy the uncompressed data over to the unpack_buffer
			 //memcpy(&unpack_buffer[0], &buffer[0], sz);

			 pbWork=&buffer[0];
		 }

		 // if the blob has zlib-compressed data
		 if(!blob.m_zlib_data.empty())
		 {
			 // issue a warning if there is more than one data steam, a blob may only contain one data stream
			 if(found_data)
				 warn("  contains several data streams");

			 // we have at least one datastream
			 found_data = true;

			 // the size of the compressesd data
			 sz = blob.m_zlib_data.size();

			 // zlib information
			 z_stream z;

			 // next byte to decompress
			 z.next_in   = (z_const Bytef *) blob.m_zlib_data.m_pBegin;

			 // number of bytes to decompress
			 z.avail_in  = sz;

			 // place of next decompressed byte
			  z.next_out  =  &unpack_buffer[0];
			  pbWork= (const char*) &unpack_buffer[0];


			 // space for decompressed data
			 z.avail_out = blob.m_raw_size.m_val;

			 // misc
			 z.zalloc    = Z_NULL;
			 z.zfree     = Z_NULL;
			 z.opaque    = Z_NULL;

			 if(inflateInit(&z) != Z_OK) {
				 err("  failed to init zlib stream");
			 }
			 if(inflate(&z, Z_FINISH) != Z_STREAM_END) {
				 err("  failed to inflate zlib stream");
			 }
			 if(inflateEnd(&z) != Z_OK) {
				 err("  failed to deinit zlib stream");
			 }

			 // unpacked size
			 sz = z.total_out;

		 }

		 // if the blob has lzma-compressed data
		 if(!blob.m_lzma_data.empty()) {
			 // issue a warning if there is more than one data steam, a blob may only contain one data stream
			 if(found_data)
				 warn("  contains several data streams");

			 // we have at least one datastream
			 found_data = true;

			 // issue a warning, lzma compression is not yet supported
			 err("  lzma-decompression is not supported");
		 }
		 // check we have at least one data-stream
		 if(!found_data)
			 err("  does not contain any known data stream");


		 // switch between different blob-types
		 if(blobheader.m_type.compareString("OSMHeader")==0)
		 {

			 OSMPBF::Headerblock headerblock ;
			 // parse the HeaderBlock from the blob
			 if(!headerblock.ParseFromArray(pbWork, sz))
				 err("unable to parse header block");


		 }
		 else if(blobheader.m_type.compareString("OSMData")==0)
		 {

			 m_pprimblock->Clear();



			 // parse the PrimitiveBlock from the blob
			 if(!m_pprimblock->ParseFromArray(pbWork, sz))
				 err("unable to parse primitive block");

			 // iterate over all PrimitiveGroups
			 for(int i = 0, l = m_pprimblock->m_primitivegroup.size(); i < l; i++)
			 {
				 // one PrimitiveGroup from the the Block
				 OSMPBF::PrimitiveGroup& pg = m_pprimblock->m_primitivegroup[i];

				 bool found_items=false;

				 // tell about nodes
				 if(pg.m_nodes.size() > 0)
				 {
					 found_items = true;
					 for(size_t i=0;i<pg.m_nodes.size();++i)
						AddNode(pg.m_nodes[i]);


				 }

				 // tell about dense nodes
				 if(!pg.m_dense.empty()) {
					 found_items = true;
					 AddDense(pg.m_dense);

				 }

				 // tell about ways
				 if(pg.m_ways.size() > 0) {
					 found_items = true;
					 for(size_t i=0;i<pg.m_ways.size();++i)
						AddWay(pg.m_ways[i]);

				 }

				 // tell about relations
				 if(pg.m_relations.size() > 0) {
					found_items = true;

					for(size_t i=0;i<pg.m_relations.size();++i)
						AddRelations(pg.m_relations[i]);

				 }

				 if(!found_items)
					 warn("      contains no items");

			 }
		 }

		 else {
			 // unknown blob type
			 warn("  unknown blob type: %s", blobheader.m_type.toString().c_str());
		 }

		 //////////////////////////////////////////////////////////////////////////
		 LONG l =INTERLOCKED_INCREMENT(m_nCount);
		 if(m_nThredNumber==0)
		 {
			 boost::timer::cpu_times const elapsed_times(uTimer.elapsed());
		     boost::timer::nanosecond_type elapsed(elapsed_times.wall);
			 if(elapsed >= secunda)
			 {
				 info("Bloks=%d",l);
				 uTimer.start();
			 }
		 }
	 }

    m_tabNode_Cash.Save();
    m_tabNode_NotVisible_Cash.Save();

    m_tabNI_Cash.Save();
    m_tabNI_NotVisible_Cash.Save();

    m_tabWI_Cash.Save();
    m_tabWI_NotVisible_Cash.Save();

    m_tabRI_Cash.Save();
    m_tabRI_NotVisible_Cash.Save();

    m_tabkvNode_cash.Save();
	m_tabkvWay_cash.Save();
	m_tabkvRelation_cash.Save();

	 delete m_pprimblock;
	 m_pprimblock=NULL;




}
예제 #3
0
// application main method
int main(int argc, char *argv[]) {
    // check if the output is a tty so we can use colors

#ifdef WIN32
    usecolor = 0;
#else
    usecolor = isatty(1);
#endif

    static struct option long_options[] = {
        {"color", no_argument, 0, 'c'},
        {0, 0, 0, 0}
    };

    while (1) {
        int c = getopt_long(argc, argv, "c", long_options, 0);

        if (c == -1) {
            break;
        }

        switch (c) {
            case 'c':
                usecolor = true;
                break;
            default:
                exit(1);
        }
    }

    // check for proper command line args
    if (optind != argc-1) {
        err("usage: %s [--color] file.osm.pbf", argv[0]);
    }

    // open specified file
    FILE *fp = fopen(argv[optind], "rb");

    if (!fp) {
        err("can't open file '%s'", argv[optind]);
    }

    // read while the file has not reached its end
    while (!feof(fp)) {
        // storage of size, used multiple times
        int32_t sz;

        // read the first 4 bytes of the file, this is the size of the blob-header
        if (fread(&sz, sizeof(sz), 1, fp) != 1) {
            break; // end of file reached
        }

        // convert the size from network byte-order to host byte-order
        sz = ntohl(sz);

        // ensure the blob-header is smaller then MAX_BLOB_HEADER_SIZE
        if (sz > OSMPBF::max_blob_header_size) {
            err("blob-header-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_blob_header_size);
        }

        // read the blob-header from the file
        if (fread(buffer, sz, 1, fp) != 1) {
            err("unable to read blob-header from file");
        }

        // parse the blob-header from the read-buffer
        if (!blobheader.ParseFromArray(buffer, sz)) {
            err("unable to parse blob header");
        }

        // tell about the blob-header
        info("\nBlobHeader (%d bytes)", sz);
        debug("  type = %s", blobheader.type().c_str());

        // size of the following blob
        sz = blobheader.datasize();
        debug("  datasize = %u", sz);

        // optional indexdata
        if (blobheader.has_indexdata()) {
            debug("  indexdata = %u bytes", blobheader.indexdata().size());
        }

        // ensure the blob is smaller then MAX_BLOB_SIZE
        if (sz > OSMPBF::max_uncompressed_blob_size) {
            err("blob-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_uncompressed_blob_size);
        }

        // read the blob from the file
        if (fread(buffer, sz, 1, fp) != 1) {
            err("unable to read blob from file");
        }

        // parse the blob from the read-buffer
        if (!blob.ParseFromArray(buffer, sz)) {
            err("unable to parse blob");
        }

        // tell about the blob-header
        info("Blob (%d bytes)", sz);

        // set when we find at least one data stream
        bool found_data = false;

        // if the blob has uncompressed data
        if (blob.has_raw()) {
            // we have at least one datastream
            found_data = true;

            // size of the blob-data
            sz = blob.raw().size();

            // check that raw_size is set correctly
            if (sz != blob.raw_size()) {
                warn("  reports wrong raw_size: %u bytes", blob.raw_size());
            }

            // tell about the blob-data
            debug("  contains uncompressed data: %u bytes", sz);

            // copy the uncompressed data over to the unpack_buffer
            memcpy(unpack_buffer, buffer, sz);
        }

        // if the blob has zlib-compressed data
        if (blob.has_zlib_data()) {
            // issue a warning if there is more than one data steam, a blob may only contain one data stream
            if (found_data) {
                warn("  contains several data streams");
            }

            // we have at least one datastream
            found_data = true;

            // the size of the compressesd data
            sz = blob.zlib_data().size();

            // tell about the compressed data
            debug("  contains zlib-compressed data: %u bytes", sz);
            debug("  uncompressed size: %u bytes", blob.raw_size());

            // zlib information
            z_stream z;

            // next byte to decompress
            z.next_in   = (unsigned char*) blob.zlib_data().c_str();

            // number of bytes to decompress
            z.avail_in  = sz;

            // place of next decompressed byte
            z.next_out  = (unsigned char*) unpack_buffer;

            // space for decompressed data
            z.avail_out = blob.raw_size();

            // misc
            z.zalloc    = Z_NULL;
            z.zfree     = Z_NULL;
            z.opaque    = Z_NULL;

            if (inflateInit(&z) != Z_OK) {
                err("  failed to init zlib stream");
            }
            if (inflate(&z, Z_FINISH) != Z_STREAM_END) {
                err("  failed to inflate zlib stream");
            }
            if (inflateEnd(&z) != Z_OK) {
                err("  failed to deinit zlib stream");
            }

            // unpacked size
            sz = z.total_out;
        }

        // if the blob has lzma-compressed data
        if (blob.has_lzma_data()) {
            // issue a warning if there is more than one data steam, a blob may only contain one data stream
            if (found_data) {
                warn("  contains several data streams");
            }

            // we have at least one datastream
            found_data = true;

            // tell about the compressed data
            debug("  contains lzma-compressed data: %u bytes", blob.lzma_data().size());
            debug("  uncompressed size: %u bytes", blob.raw_size());

            // issue a warning, lzma compression is not yet supported
            err("  lzma-decompression is not supported");
        }

        // check we have at least one data-stream
        if (!found_data) {
            err("  does not contain any known data stream");
        }

        // switch between different blob-types
        if (blobheader.type() == "OSMHeader") {
            // tell about the OSMHeader blob
            info("  OSMHeader");

            // parse the HeaderBlock from the blob
            if (!headerblock.ParseFromArray(unpack_buffer, sz)) {
                err("unable to parse header block");
            }

            // tell about the bbox
            if (headerblock.has_bbox()) {
                OSMPBF::HeaderBBox bbox = headerblock.bbox();
                debug("    bbox: %.7f,%.7f,%.7f,%.7f",
                    (double)bbox.left() / OSMPBF::lonlat_resolution,
                    (double)bbox.bottom() / OSMPBF::lonlat_resolution,
                    (double)bbox.right() / OSMPBF::lonlat_resolution,
                    (double)bbox.top() / OSMPBF::lonlat_resolution);
            }

            // tell about the required features
            for (int i = 0, l = headerblock.required_features_size(); i < l; i++) {
                debug("    required_feature: %s", headerblock.required_features(i).c_str());
            }

            // tell about the optional features
            for (int i = 0, l = headerblock.optional_features_size(); i < l; i++) {
                debug("    optional_feature: %s", headerblock.optional_features(i).c_str());
            }

            // tell about the writing program
            if (headerblock.has_writingprogram()) {
                debug("    writingprogram: %s", headerblock.writingprogram().c_str());
            }

            // tell about the source
            if (headerblock.has_source()) {
                debug("    source: %s", headerblock.source().c_str());
            }
        } else if (blobheader.type() == "OSMData") {
            // tell about the OSMData blob
            info("  OSMData");

            // parse the PrimitiveBlock from the blob
            if (!primblock.ParseFromArray(unpack_buffer, sz)) {
                err("unable to parse primitive block");
            }

            // tell about the block's meta info
            debug("    granularity: %u", primblock.granularity());
            debug("    lat_offset: %u", primblock.lat_offset());
            debug("    lon_offset: %u", primblock.lon_offset());
            debug("    date_granularity: %u", primblock.date_granularity());

            // tell about the stringtable
            debug("    stringtable: %u items", primblock.stringtable().s_size());

            // number of PrimitiveGroups
            debug("    primitivegroups: %u groups", primblock.primitivegroup_size());

            // iterate over all PrimitiveGroups
            for (int i = 0, l = primblock.primitivegroup_size(); i < l; i++) {
                // one PrimitiveGroup from the the Block
                OSMPBF::PrimitiveGroup pg = primblock.primitivegroup(i);

                bool found_items=false;

                // tell about nodes
                if (pg.nodes_size() > 0) {
                    found_items = true;

                    debug("      nodes: %d", pg.nodes_size());
                    if (pg.nodes(0).has_info()) {
                        debug("        with meta-info");
                    }
                }

                // tell about dense nodes
                if (pg.has_dense()) {
                    found_items = true;

                    debug("      dense nodes: %d", pg.dense().id_size());
                    if (pg.dense().has_denseinfo()) {
                        debug("        with meta-info");
                    }
                }

                // tell about ways
                if (pg.ways_size() > 0) {
                    found_items = true;

                    debug("      ways: %d", pg.ways_size());
                    if (pg.ways(0).has_info()) {
                        debug("        with meta-info");
                    }
                }

                // tell about relations
                if (pg.relations_size() > 0) {
                    found_items = true;

                    debug("      relations: %d", pg.relations_size());
                    if (pg.relations(0).has_info()) {
                        debug("        with meta-info");
                    }
                }

                if (!found_items) {
                    warn("      contains no items");
                }
            }
        }

        else {
            // unknown blob type
            warn("  unknown blob type: %s", blobheader.type().c_str());
        }
    }

    // close the file pointer
    fclose(fp);

    // clean up the protobuf lib
    google::protobuf::ShutdownProtobufLibrary();
}