boost::uint32_t Crop::readBufferImpl(PointBuffer& dstData) { // The client has asked us for dstData.getCapacity() points. // We will read from our previous stage until we get that amount (or // until the previous stage runs out of points). boost::uint32_t numPointsNeeded = dstData.getCapacity(); assert(dstData.getNumPoints() == 0); while (numPointsNeeded > 0) { if (getPrevIterator().atEnd()) break; // set up buffer to be filled by prev stage PointBuffer srcData(dstData.getSchema(), numPointsNeeded); // read from prev stage const boost::uint32_t numSrcPointsRead = getPrevIterator().read(srcData); assert(numSrcPointsRead == srcData.getNumPoints()); assert(numSrcPointsRead <= numPointsNeeded); // we got no data, and there is no more to get -- exit the loop if (numSrcPointsRead == 0) break; // copy points from src (prev stage) into dst (our stage), // based on the CropFilter's rules (i.e. its bounds) const boost::uint32_t numPointsProcessed = m_cropFilter.processBuffer(dstData, srcData); numPointsNeeded -= numPointsProcessed; } const boost::uint32_t numPointsAchieved = dstData.getNumPoints(); return numPointsAchieved; }
boost::uint32_t Reader::processBuffer(PointBuffer& data, boost::uint64_t index) const { const Schema& schema = data.getSchema(); // how many are they asking for? boost::uint64_t numPointsWanted = data.getCapacity(); // we can only give them as many as we have left boost::uint64_t numPointsAvailable = getNumPoints() - index; if (numPointsAvailable < numPointsWanted) numPointsWanted = numPointsAvailable; schema::DimensionMap* d = m_buffer.getSchema().mapDimensions(data.getSchema()); data.setNumPoints(0); PointBuffer::copyLikeDimensions(m_buffer, data, *d, index, 0, numPointsWanted); data.setNumPoints(numPointsWanted); delete d; return numPointsWanted; }
boost::uint32_t DecimationFilterSequentialIterator::readBufferImpl(PointBuffer& dstData) { // The client has asked us for dstData.getCapacity() points. // We will read from our previous stage until we get that amount (or // until the previous stage runs out of points). boost::uint32_t numPointsNeeded = dstData.getCapacity(); assert(dstData.getNumPoints() == 0); // set up buffer to be filled by prev stage PointBuffer srcData(dstData.getSchemaLayout(), numPointsNeeded); while (numPointsNeeded > 0) { // read from prev stage const boost::uint64_t srcStartIndex = getPrevIterator().getIndex(); const boost::uint32_t numSrcPointsRead = getPrevIterator().read(srcData); assert(numSrcPointsRead <= srcData.getNumPoints()); //assert(numSrcPointsRead <= numPointsNeeded); // we got no data, and there is no more to get -- exit the loop if (numSrcPointsRead == 0) break; // copy points from src (prev stage) into dst (our stage), // based on the CropFilter's rules (i.e. its bounds) const boost::uint32_t numPointsAdded = m_filter.processBuffer(dstData, srcData, srcStartIndex); numPointsNeeded -= numPointsAdded; //printf(".");fflush(stdout); } const boost::uint32_t numPointsAchieved = dstData.getNumPoints(); return numPointsAchieved; }
boost::uint32_t Mosaic::readBufferImpl(PointBuffer& destData) { boost::uint32_t totalNumPointsToRead = destData.getCapacity(); boost::uint32_t totalNumPointsRead = 0; boost::uint32_t destPointIndex = 0; // for each stage, we read as many points as we can while (totalNumPointsRead < totalNumPointsToRead) { assert(m_iteratorIndex < m_prevIterators.size()); m_prevIterator = m_prevIterators[m_iteratorIndex]; // read as much as we can into temp buffer PointBuffer tmp(destData.getSchema(), totalNumPointsToRead-totalNumPointsRead); boost::uint32_t numRead = m_prevIterator->read(tmp); totalNumPointsRead += numRead; // concat the temp buffer on to end of real dest buffer destData.copyPointsFast(destPointIndex, 0, tmp, numRead); destPointIndex += numRead; destData.setNumPoints(destData.getNumPoints() + numRead); if (m_prevIterator->atEnd()) { ++m_iteratorIndex; } if (m_iteratorIndex == m_prevIterators.size()) { break; } } return totalNumPointsRead; }
// BUG: this duplicates the code above boost::uint32_t Cache::readBufferImpl(PointBuffer& data) { const boost::uint32_t cacheBlockSize = m_filter.getCacheBlockSize(); const boost::uint64_t currentPointIndex = getIndex(); // for now, we only read from the cache if they are asking for one point // (this avoids the problem of an N-point request needing more than one // cached block to satisfy it) if (data.getCapacity() != 1) { const boost::uint32_t numRead = getPrevIterator().read(data); // if they asked for a full block and we got a full block, // and the block we got is properly aligned and not already cached, // then let's cache it! const bool isCacheable = (data.getCapacity() == cacheBlockSize) && (numRead == cacheBlockSize) && (currentPointIndex % cacheBlockSize == 0); if (isCacheable && (m_filter.lookupInCache(currentPointIndex) == NULL)) { m_filter.addToCache(currentPointIndex, data); } m_filter.updateStats(numRead, data.getCapacity()); return numRead; } // they asked for just one point -- first, check Mister Cache const PointBuffer* block = m_filter.lookupInCache(currentPointIndex); if (block != NULL) { // A hit! A palpable hit! data.copyPointFast(0, currentPointIndex % cacheBlockSize, *block); m_filter.updateStats(0, 1); return 1; } // Not in the cache, so do a normal read :-( const boost::uint32_t numRead = getPrevIterator().read(data); m_filter.updateStats(numRead, numRead); return numRead; }
void Selector::alterSchema(PointBuffer& buffer) { Schema const& original_schema = buffer.getSchema(); Schema new_schema = buffer.getSchema(); std::map<std::string, bool> const& ignoredMap = m_selectorFilter.getIgnoredMap(); // for (std::map<std::string, bool>::const_iterator i = ignoredMap.begin(); // i != ignoredMap.end(); ++i) // { // boost::optional<Dimension const&> d = original_schema.getDimensionOptional(i->first); // if (d) // { // Dimension d2(*d); // boost::uint32_t flags = d2.getFlags(); // if (i->second) // d2.setFlags(flags | dimension::IsIgnored); // new_schema.setDimension(d2); // } // } // schema::Map dimensions = original_schema.getDimensions(); schema::index_by_index const& dims = dimensions.get<schema::index>(); for (schema::index_by_index::const_iterator t = dims.begin(); t != dims.end(); ++t) { std::map<std::string, bool>::const_iterator ignored = ignoredMap.find(t->getName()); if (ignored != ignoredMap.end()) { if (ignored->second) // marked to be dropped { // set to ignored Dimension d2(*t); boost::uint32_t flags = d2.getFlags(); d2.setFlags(flags | dimension::IsIgnored); new_schema.setDimension(d2); } } else { // didn't find it in our map of specified dimensions if (m_selectorFilter.doIgnoreUnspecifiedDimensions()) { // set to ignored Dimension d2(*t); boost::uint32_t flags = d2.getFlags(); d2.setFlags(flags | dimension::IsIgnored); new_schema.setDimension(d2); } } } buffer = PointBuffer(new_schema, buffer.getCapacity()); }
PointBuffer* makeTestBuffer() { Dimension d1("Classification", dimension::UnsignedInteger, 1); Dimension d2("X", dimension::SignedInteger, 4); Dimension d3("Y", dimension::Float, 8); Schema schema; schema.appendDimension(d1); schema.appendDimension(d2); schema.appendDimension(d3); std::size_t offX = schema.getDimension(0).getByteOffset(); BOOST_CHECK(offX==0); std::size_t offY = schema.getDimension(1).getByteOffset(); BOOST_CHECK(offY==1); std::size_t offZ = schema.getDimension(2).getByteOffset(); BOOST_CHECK(offZ==5); boost::uint32_t capacity = 17; PointBuffer* data = new PointBuffer(schema, capacity); BOOST_CHECK(data->getCapacity() == capacity); Dimension const& dimC = data->getSchema().getDimension("Classification"); Dimension const& dimX = data->getSchema().getDimension("X"); Dimension const& dimY = data->getSchema().getDimension("Y"); // write the data into the buffer for (boost::uint32_t i=0; i<data->getCapacity(); i++) { const boost::uint8_t x = static_cast<boost::uint8_t>(i)+1; const boost::int32_t y = i*10; const double z = i * 100; data->setField(dimC, i, x); data->setField(dimX, i, y); data->setField(dimY, i, z); data->setNumPoints(i+1); } BOOST_CHECK(data->getCapacity() ==17); BOOST_CHECK(data->getNumPoints() ==17); return data; }
boost::uint32_t Predicate::processBuffer(PointBuffer& data, pdal::plang::BufferedInvocation& python) const { python.resetArguments(); python.beginChunk(data); python.execute(); if (!python.hasOutputVariable("Mask")) { throw python_error("Mask variable not set in predicate filter function"); } boost::uint8_t* mask = new boost::uint8_t[data.getNumPoints()]; PointBuffer dstData(data.getSchema(), data.getCapacity()); python.extractResult("Mask", (boost::uint8_t*)mask, data.getNumPoints(), 1, pdal::dimension::RawByte, 1); boost::uint8_t* dst = dstData.getData(0); boost::uint8_t* src = data.getData(0); const Schema& schema = dstData.getSchema(); boost::uint32_t numBytes = schema.getByteSize(); assert(numBytes == data.getSchema().getByteSize()); boost::uint32_t numSrcPoints = data.getNumPoints(); boost::uint32_t count = 0; for (boost::uint32_t srcIndex=0; srcIndex<numSrcPoints; srcIndex++) { if (mask[srcIndex]) { memcpy(dst, src, numBytes); dst += numBytes; ++count; dstData.setNumPoints(count); } src += numBytes; } data.copyPointsFast(0, 0, dstData, count); data.setNumPoints(count); delete[] mask; return count; }
void IteratorBase::fillUserBuffer(PointBuffer& user_buffer) { Schema const& user_schema = user_buffer.getSchema(); schema::index_by_index const& idx = user_schema.getDimensions().get<schema::index>(); boost::int32_t numUserSpace = user_buffer.getCapacity() - user_buffer.getNumPoints(); if (numUserSpace < 0) throw pdal_error("We ran out of space!"); boost::int32_t numOraclePoints = m_active_buffer->getNumPoints() - m_buffer_position; schema::index_by_index::size_type i(0); for (i = 0; i < idx.size(); ++i) { copyDatabaseData(*m_active_buffer, user_buffer, idx[i], m_buffer_position, user_buffer.getNumPoints(), (std::min)(numOraclePoints,numUserSpace)); } bool bSetPointSourceId = getReader().getOptions().getValueOrDefault<bool>("populate_pointsourceid", false); if (bSetPointSourceId) { Dimension const* point_source_field = &(user_buffer.getSchema().getDimensionOptional("PointSourceId").get()); if (point_source_field) { for (boost::int32_t i = 0; i < numUserSpace; ++i) { if (i < 0) throw sqlite_driver_error("point_source_field point index is less than 0!"); user_buffer.setField(*point_source_field, i, m_active_cloud_id); } } } if (numOraclePoints > numUserSpace) m_buffer_position = m_buffer_position + numUserSpace; else if (numOraclePoints < numUserSpace) m_buffer_position = 0; boost::uint32_t howManyThisRead = (std::min)(numUserSpace, numOraclePoints); user_buffer.setNumPoints(howManyThisRead + user_buffer.getNumPoints()); }
boost::uint32_t Reader::processBuffer(PointBuffer& data, boost::uint64_t index) const { const Schema& schema = data.getSchema(); // make up some data and put it into the buffer // how many are they asking for? boost::uint64_t numPointsWanted = data.getCapacity(); // we can only give them as many as we have left boost::uint64_t numPointsAvailable = getNumPoints() - index; if (numPointsAvailable < numPointsWanted) numPointsWanted = numPointsAvailable; const Bounds<double>& bounds = getBounds(); const std::vector< Range<double> >& dims = bounds.dimensions(); const double minX = dims[0].getMinimum(); const double maxX = dims[0].getMaximum(); const double minY = dims[1].getMinimum(); const double maxY = dims[1].getMaximum(); const double minZ = dims[2].getMinimum(); const double maxZ = dims[2].getMaximum(); const double numDeltas = (double)getNumPoints() - 1.0; const double delX = (maxX - minX) / numDeltas; const double delY = (maxY - minY) / numDeltas; const double delZ = (maxZ - minZ) / numDeltas; const Dimension& dimX = schema.getDimension("X", getName()); const Dimension& dimY = schema.getDimension("Y", getName()); const Dimension& dimZ = schema.getDimension("Z", getName()); const Dimension& dimTime = schema.getDimension("Time", getName()); boost::uint64_t time = index; const Reader::Mode mode = getMode(); boost::uint32_t cnt = 0; data.setNumPoints(0); for (boost::uint32_t pointIndex=0; pointIndex<numPointsWanted; pointIndex++) { double x; double y; double z; switch (mode) { case Reader::Random: x = Utils::random(minX, maxX); y = Utils::random(minY, maxY); z = Utils::random(minZ, maxZ); break; case Reader::Constant: x = minX; y = minY; z = minZ; break; case Reader::Ramp: x = minX + delX * pointIndex; y = minY + delY * pointIndex; z = minZ + delZ * pointIndex; break; default: throw pdal_error("invalid mode in FauxReader"); break; } data.setField<double>(dimX, pointIndex, x); data.setField<double>(dimY, pointIndex, y); data.setField<double>(dimZ, pointIndex, z); data.setField<boost::uint64_t>(dimTime, pointIndex, time); ++time; ++cnt; data.setNumPoints(cnt); assert(cnt <= data.getCapacity()); } return cnt; }
uint32_t MrsidReader::processBuffer(PointBuffer& data, uint64_t index) const { const Schema& schema = data.getSchema(); // how many are they asking for? uint64_t numPointsWanted = data.getCapacity(); // we can only give them as many as we have left uint64_t numPointsAvailable = getNumPoints() - index; if (numPointsAvailable < numPointsWanted) numPointsWanted = numPointsAvailable; LizardTech::PointData points; // to do: specify a PointInfo structure that reads only the channels we will output. points.init(m_PS->getPointInfo(), (size_t)numPointsWanted); size_t count = m_iter->getNextPoints(points); uint32_t cnt = 0; data.setNumPoints(0); schema::index_by_index const& dims = schema.getDimensions().get<schema::index>(); for (uint32_t pointIndex=0; pointIndex<count; pointIndex++) { ++cnt; for (unsigned int i=0; i < dims.size(); i++) { Dimension const& d = dims[i]; if (d.getName() == "X" && d.getInterpretation() == dimension::Float && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_X)) { double *pData = static_cast<double*>(points.getChannel(CHANNEL_NAME_X)->getData()); double value = static_cast<double>(pData[pointIndex]); data.setField<double>(d, pointIndex, value); } else if (d.getName() == "X" && d.getInterpretation() == dimension::SignedInteger && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_X)) { double *pData = static_cast<double*>(points.getChannel(CHANNEL_NAME_X)->getData()); int32_t value = static_cast<int32_t>(pData[pointIndex]); data.setField<int32_t>(d, pointIndex, value); } else if (d.getName() == "Y" && d.getInterpretation() == dimension::Float && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_Y)) { double *pData = static_cast<double*>(points.getChannel(CHANNEL_NAME_Y)->getData()); double value = static_cast<double>(pData[pointIndex]); data.setField<double>(d, pointIndex, value); } else if (d.getName() == "Y" && d.getInterpretation() == dimension::SignedInteger && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_Y)) { double *pData = static_cast<double*>(points.getChannel(CHANNEL_NAME_Y)->getData()); int32_t value = static_cast<int32_t>(pData[pointIndex]); data.setField<int32_t>(d, pointIndex, value); } else if (d.getName() == "Z" && d.getInterpretation() == dimension::Float && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_Z)) { double *pData = static_cast<double*>(points.getChannel(CHANNEL_NAME_Z)->getData()); double value = static_cast<double>(pData[pointIndex]); data.setField<double>(d, pointIndex, value); } else if (d.getName() == "Z" && d.getInterpretation() == dimension::SignedInteger && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_Z)) { double *pData = static_cast<double*>(points.getChannel(CHANNEL_NAME_Z)->getData()); int32_t value = static_cast<int32_t>(pData[pointIndex]); data.setField<int32_t>(d, pointIndex, value); } else if (d.getName() == "Time" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_GPSTime)) { double *pData = static_cast<double*>(points.getChannel(CHANNEL_NAME_GPSTime)->getData()); uint64_t value = static_cast<uint64_t>(pData[pointIndex]); data.setField<uint64_t>(d, pointIndex, value); } else if (d.getName() == "Intensity" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_Intensity)) { uint16_t *pData = static_cast<uint16_t*>(points.getChannel(CHANNEL_NAME_Intensity)->getData()); uint16_t value = static_cast<uint16_t>(pData[pointIndex]); data.setField<uint16_t>(d, pointIndex, value); } else if (d.getName() == "ReturnNumber" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_ReturnNum)) { uint8_t *pData = static_cast<uint8_t*>(points.getChannel(CHANNEL_NAME_ReturnNum)->getData()); uint8_t value = static_cast<uint8_t>(pData[pointIndex]); data.setField<uint8_t>(d, pointIndex, value); } else if (d.getName() == "NumberOfReturns" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_NumReturns)) { uint8_t *pData = static_cast<uint8_t*>(points.getChannel(CHANNEL_NAME_NumReturns)->getData()); uint8_t value = static_cast<uint8_t>(pData[pointIndex]); data.setField<uint8_t>(d, pointIndex, value); } else if (d.getName() == "ScanDirectionFlag" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_ScanDir)) { uint8_t *pData = static_cast<uint8_t*>(points.getChannel(CHANNEL_NAME_NumReturns)->getData()); uint8_t value = static_cast<uint8_t>(pData[pointIndex]); data.setField<uint8_t>(d, pointIndex, value); } else if (d.getName() == "ScanAngleRank" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_ScanAngle)) { int8_t *pData = static_cast<int8_t*>(points.getChannel(CHANNEL_NAME_NumReturns)->getData()); int8_t value = static_cast<int8_t>(pData[pointIndex]); data.setField<int8_t>(d, pointIndex, value); } else if (d.getName() == "EdgeOfFlightLine" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_EdgeFlightLine)) { uint8_t *pData = static_cast<uint8_t*>(points.getChannel(CHANNEL_NAME_NumReturns)->getData()); uint8_t value = static_cast<uint8_t>(pData[pointIndex]); data.setField<uint8_t>(d, pointIndex, value); } else if (d.getName() == "Classification" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_ClassId)) { uint8_t *pData = static_cast<uint8_t*>(points.getChannel(CHANNEL_NAME_NumReturns)->getData()); uint8_t value = static_cast<uint8_t>(pData[pointIndex]); data.setField<uint8_t>(d, pointIndex, value); } else if (d.getName() == "UserData" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_UserData)) { uint8_t *pData = static_cast<uint8_t*>(points.getChannel(CHANNEL_NAME_NumReturns)->getData()); uint8_t value = static_cast<uint8_t>(pData[pointIndex]); data.setField<uint8_t>(d, pointIndex, value); } else if (d.getName() == "PointSourceId" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_SourceId)) { uint16_t *pData = static_cast<uint16_t*>(points.getChannel(CHANNEL_NAME_NumReturns)->getData()); uint16_t value = static_cast<uint16_t>(pData[pointIndex]); data.setField<uint16_t>(d, pointIndex, value); } } } data.setNumPoints(cnt); assert(cnt <= data.getCapacity()); return cnt; }
boost::uint32_t IteratorBase::myReadBlocks(PointBuffer& user_buffer) { boost::uint32_t numPointsRead = 0; std::string const& query = getReader().getOptions().getValueOrThrow<std::string>("query"); ::soci::row block; ::soci::indicator ind = ::soci::i_null; ::soci::statement blocks = (m_session->prepare << query, ::soci::into(block, ind)); blocks.execute(); bool bDidRead = blocks.fetch(); // if (ind == ::soci::i_null) // { // // We have no points to return // getReader().log()->get(logDEBUG) << "Query returned no points" << std::endl; // return 0; // } // // size_t size = block.size(); // for (size_t i = 0; i < size; ++i) // { // getReader().log()->get(logDEBUG3) << "column: " << block.get_properties(i).get_name() << std::endl; // } if (!m_active_buffer) { m_active_buffer = fetchPointBuffer(block.get<int>("cloud_id"), block.get<std::string>("schema"), user_buffer.getCapacity()); m_active_cloud_id = block.get<int>("cloud_id"); } // // This shouldn't ever happen int num_points = block.get<int>("num_points"); if (num_points > static_cast<boost::int32_t>(m_active_buffer->getCapacity())) { std::ostringstream oss; oss << "Block size, " << num_points <<", is too large to fit in " << "buffer of size " << user_buffer.getCapacity() <<". Increase buffer capacity with writer's \"chunk_size\" option " << "or increase the read buffer size"; throw buffer_too_small(oss.str()); } while (bDidRead) { boost::uint32_t numReadThisBlock = static_cast<boost::uint32_t>(block.get<int>("num_points")); boost::uint32_t numSpaceLeftThisBuffer = user_buffer.getCapacity() - user_buffer.getNumPoints(); getReader().log()->get(logDEBUG4) << "IteratorBase::myReadBlocks:" "numReadThisBlock: " << numReadThisBlock << " numSpaceLeftThisBlock: " << numSpaceLeftThisBuffer << " total numPointsRead: " << numPointsRead << std::endl; numPointsRead = numPointsRead + numReadThisBlock; readBlob(block, (std::min)(numReadThisBlock, numSpaceLeftThisBuffer)); fillUserBuffer(user_buffer); bDidRead = blocks.fetch(); // if (!bDidRead) // return user_buffer.getNumPoints(); boost::int32_t const& current_cloud_id = block.get<int>("cloud_id"); if (current_cloud_id != m_active_cloud_id) { getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: current_cloud_id: " << current_cloud_id << " m_active_cloud_id: " << m_active_cloud_id << std::endl; m_active_buffer = fetchPointBuffer(current_cloud_id, block.get<std::string>("schema"), user_buffer.getCapacity()); m_active_cloud_id = current_cloud_id; return user_buffer.getNumPoints(); } } return numPointsRead; }
// append all points from src buffer to end of dst buffer, based on the our bounds boost::uint32_t Crop::processBuffer(PointBuffer& dstData, const PointBuffer& srcData) const { const Schema& schema = dstData.getSchema(); const Bounds<double>& bounds = this->getBounds(); boost::uint32_t numSrcPoints = srcData.getNumPoints(); boost::uint32_t dstIndex = dstData.getNumPoints(); boost::uint32_t numPointsAdded = 0; boost::optional<Dimension const&> dimX = schema.getDimension("X"); boost::optional<Dimension const&> dimY = schema.getDimension("Y"); boost::optional<Dimension const&> dimZ = schema.getDimension("Z"); for (boost::uint32_t srcIndex=0; srcIndex<numSrcPoints; srcIndex++) { // need to scale the values double x(0.0); double y(0.0); double z(0.0); if (dimX->getInterpretation() == dimension::SignedInteger ) { boost::int32_t xi = srcData.getField<boost::int32_t>(*dimX, srcIndex); boost::int32_t yi = srcData.getField<boost::int32_t>(*dimY, srcIndex); boost::int32_t zi = srcData.getField<boost::int32_t>(*dimZ, srcIndex); x = dimX->applyScaling(xi); y = dimY->applyScaling(yi); z = dimZ->applyScaling(zi); } else if (dimX->getInterpretation() == dimension::UnsignedInteger) { boost::uint32_t xi = srcData.getField<boost::uint32_t>(*dimX, srcIndex); boost::uint32_t yi = srcData.getField<boost::uint32_t>(*dimY, srcIndex); boost::uint32_t zi = srcData.getField<boost::uint32_t>(*dimZ, srcIndex); x = dimX->applyScaling(xi); y = dimY->applyScaling(yi); z = dimZ->applyScaling(zi); } else { x = srcData.getField<double>(*dimX, srcIndex); y = srcData.getField<double>(*dimY, srcIndex); z = srcData.getField<double>(*dimZ, srcIndex); } Vector<double> point(x,y,z); if (bounds.contains(point)) { dstData.copyPointFast(dstIndex, srcIndex, srcData); dstData.setNumPoints(dstIndex+1); ++dstIndex; ++numPointsAdded; } } assert(dstIndex <= dstData.getCapacity()); return numPointsAdded; }
void IteratorBase::fillUserBuffer(PointBuffer& user_buffer) { Schema const& user_schema = user_buffer.getSchema(); schema::index_by_index const& idx = user_schema.getDimensions().get<schema::index>(); boost::int32_t numOraclePoints = m_oracle_buffer->getNumPoints() - m_buffer_position; boost::int32_t numUserSpace = user_buffer.getCapacity() - user_buffer.getNumPoints(); boost::int32_t howManyThisRead = (std::min)(numUserSpace, numOraclePoints); if (numUserSpace < 0) { std::ostringstream oss; oss << "numUserSpace < 0! : " << numUserSpace; throw pdal_error(oss.str()); } if (numOraclePoints < 0) { std::ostringstream oss; oss << "numOraclePoints < 0! : " << numOraclePoints; throw pdal_error(oss.str()); } if (user_buffer.getNumPoints() + howManyThisRead > user_buffer.getCapacity()) { std::ostringstream oss; oss << "user_buffer.getNumPoints() + howManyThisRead == " << user_buffer.getNumPoints() + howManyThisRead << " but user_buffer.getCapacity() == " << user_buffer.getCapacity(); throw pdal_error(oss.str()); } PointBuffer::copyLikeDimensions(*m_oracle_buffer, user_buffer, *m_dimension_map, m_buffer_position, user_buffer.getNumPoints(), howManyThisRead); Schema const& src = m_oracle_buffer->getSchema(); Dimension const& src_x = src.getDimension("drivers.oci.reader.X"); Dimension const& src_y = src.getDimension("drivers.oci.reader.Y"); Dimension const& src_z = src.getDimension("drivers.oci.reader.Z"); Schema const& dst = user_buffer.getSchema(); Dimension const& dst_x = dst.getDimension("drivers.oci.reader.X"); Dimension const& dst_y = dst.getDimension("drivers.oci.reader.Y"); Dimension const& dst_z = dst.getDimension("drivers.oci.reader.Z"); bool bDifferentScales = (!pdal::Utils::compare_distance(dst_x.getNumericScale(), src_x.getNumericScale()) || !pdal::Utils::compare_distance(dst_y.getNumericScale(), src_y.getNumericScale()) || !pdal::Utils::compare_distance(dst_z.getNumericScale(), src_z.getNumericScale())); bool bDifferentOffsets = (!pdal::Utils::compare_distance(dst_x.getNumericOffset(), src_x.getNumericOffset()) || !pdal::Utils::compare_distance(dst_y.getNumericOffset(), src_y.getNumericOffset()) || !pdal::Utils::compare_distance(dst_z.getNumericOffset(), src_z.getNumericOffset())); bool bNormalizeXYZ = getReader().getOptions().getValueOrDefault<bool>("do_normalize_xyz", true); if ((bDifferentScales || bDifferentOffsets) && bNormalizeXYZ) { for (unsigned i = m_buffer_position; i < howManyThisRead; ++i) { double x = m_oracle_buffer->applyScaling(src_x, i); boost::int32_t xi = dst_x.removeScaling<boost::int32_t>(x); user_buffer.setField<boost::int32_t>(dst_x, user_buffer.getNumPoints()+i, xi); double y = m_oracle_buffer->applyScaling(src_y, i); boost::int32_t yi = dst_y.removeScaling<boost::int32_t>(y); user_buffer.setField<boost::int32_t>(dst_y, user_buffer.getNumPoints()+i, yi); double z = m_oracle_buffer->applyScaling(src_z, i); boost::int32_t zi = dst_z.removeScaling<boost::int32_t>(z); user_buffer.setField<boost::int32_t>(dst_z, user_buffer.getNumPoints()+i, zi); } } getReader().log()->get(logDEBUG2) << "IteratorBase::fillUserBuffer m_buffer_position: " << m_buffer_position << std::endl; if (numOraclePoints > numUserSpace) m_buffer_position = m_buffer_position + numUserSpace; else if (numOraclePoints < numUserSpace) m_buffer_position = 0; getReader().log()->get(logDEBUG2) << "IteratorBase::fillUserBuffer m_buffer_position: " << m_buffer_position << std::endl; bool bSetPointSourceId = getReader().getOptions().getValueOrDefault<bool>("populate_pointsourceid", false); if (bSetPointSourceId) { Dimension const* point_source_field = user_buffer.getSchema().getDimensionPtr("PointSourceId"); if (point_source_field) { for (boost::int32_t i = m_buffer_position; i < howManyThisRead; ++i) { assert(user_buffer.getNumPoints() + i < user_buffer.getCapacity()); if (point_source_field->getByteSize() == 2) user_buffer.setField(*point_source_field, user_buffer.getNumPoints() + i, static_cast<boost::uint16_t>(m_active_cloud_id)); else if (point_source_field->getByteSize() == 4) { user_buffer.setField(*point_source_field, user_buffer.getNumPoints() + i, m_active_cloud_id); } } } } user_buffer.setNumPoints(howManyThisRead + user_buffer.getNumPoints()); }
boost::uint32_t IteratorBase::myReadBlocks(PointBuffer& user_buffer) { boost::uint32_t numPointsRead = 0; user_buffer.setNumPoints(0); bool bDidRead = false; if (!m_oracle_buffer) { m_oracle_buffer = fetchPointBuffer(m_initialQueryStatement, m_block->pc); if (!m_oracle_buffer) throw pdal_error("m_oracle_buffer was NULL!"); m_dimension_map = fetchDimensionMap(m_initialQueryStatement, m_block->pc, *m_oracle_buffer, user_buffer); boost::int32_t current_cloud_id(0); current_cloud_id = m_initialQueryStatement->GetInteger(&m_block->pc->pc_id); m_active_cloud_id = current_cloud_id; } // This shouldn't ever happen if (m_block->num_points > static_cast<boost::int32_t>(m_oracle_buffer->getCapacity())) { m_oracle_buffer->resize(m_block->num_points); } if (!m_block->num_points) { // We still have a block of data from the last readBuffer call // that was partially read. getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: fetching first block" << std::endl; bDidRead = m_initialQueryStatement->Fetch(); if (!bDidRead) { m_at_end = true; return 0; } user_buffer.setSpatialBounds(getBounds(m_initialQueryStatement, m_block)); } else { // Our read was already "done" last readBuffer call, but if we're done, // we're done if (m_at_end) getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: we are at end of the blocks;" << std::endl; else getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: we have points left to read on this block" << std::endl; if (m_at_end) return 0; bDidRead = true; } while (bDidRead) { boost::uint32_t numReadThisBlock = m_block->num_points; boost::uint32_t numSpaceLeftThisBuffer = user_buffer.getCapacity() - user_buffer.getNumPoints(); getReader().log()->get(logDEBUG4) << "IteratorBase::myReadBlocks:" "numReadThisBlock: " << numReadThisBlock << " numSpaceLeftThisBlock: " << numSpaceLeftThisBuffer << " total numPointsRead: " << numPointsRead << std::endl; numPointsRead = numPointsRead + numReadThisBlock; readBlob(m_initialQueryStatement, m_block, m_block->num_points); fillUserBuffer(user_buffer); if (m_buffer_position != 0) { return user_buffer.getNumPoints(); } else { bDidRead = m_initialQueryStatement->Fetch(); if (!bDidRead) { getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: done reading block. Read " << numPointsRead << " points" << std::endl; m_at_end = true; return user_buffer.getNumPoints(); } } boost::int32_t current_cloud_id(0); current_cloud_id = m_initialQueryStatement->GetInteger(&m_block->pc->pc_id); getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: current_cloud_id: " << current_cloud_id << " m_active_cloud_id: " << m_active_cloud_id << std::endl; if (current_cloud_id != m_active_cloud_id) { m_oracle_buffer = fetchPointBuffer(m_initialQueryStatement, m_block->pc); if (!m_oracle_buffer) throw pdal_error("m_oracle_buffer was NULL!"); m_dimension_map = fetchDimensionMap(m_initialQueryStatement, m_block->pc, *m_oracle_buffer, user_buffer); m_active_cloud_id = current_cloud_id; return user_buffer.getNumPoints(); } } return numPointsRead; }
boost::uint32_t Reader::readBufferImpl(PointBuffer& data) { m_log->get(logDEBUG5) << "Reading a point buffer of " << data.getCapacity() << " points." << std::endl; return m_reader.processBuffer(data, getIndex()); }
boost::uint32_t Reader::processBuffer(PointBuffer& data, std::istream& stream, boost::uint64_t numPointsLeft) const { // we must not read more points than are left in the file const boost::uint64_t numPoints64 = std::min<boost::uint64_t>(data.getCapacity(), numPointsLeft); const boost::uint32_t numPoints = (boost::uint32_t)std::min<boost::uint64_t>(numPoints64, std::numeric_limits<boost::uint32_t>::max()); const Schema& schema = data.getSchema(); const int pointByteCount = getPointDataSize(); const PointDimensions dimensions(schema, getName()); boost::uint8_t* buf = new boost::uint8_t[pointByteCount * numPoints]; Utils::read_n(buf, stream, pointByteCount * numPoints); for (boost::uint32_t pointIndex=0; pointIndex<numPoints; pointIndex++) { boost::uint8_t* p = buf + pointByteCount * pointIndex; if (m_format == TERRASOLID_Format_1) { boost::uint8_t classification = Utils::read_field<boost::uint8_t>(p); if (dimensions.Classification) data.setField<boost::uint8_t>(*dimensions.Classification, pointIndex, classification); boost::uint8_t flight_line = Utils::read_field<boost::uint8_t>(p); if (dimensions.PointSourceId) data.setField<boost::uint8_t>(*dimensions.PointSourceId, pointIndex, flight_line); boost::uint16_t echo_int = Utils::read_field<boost::uint16_t>(p); if (dimensions.ReturnNumber) data.setField<boost::uint16_t>(*dimensions.ReturnNumber, pointIndex, echo_int); boost::int32_t x = Utils::read_field<boost::int32_t>(p); if (dimensions.X) data.setField<boost::int32_t>(*dimensions.X, pointIndex, x); boost::int32_t y = Utils::read_field<boost::int32_t>(p); if (dimensions.Y) data.setField<boost::int32_t>(*dimensions.Y, pointIndex, y); boost::int32_t z = Utils::read_field<boost::int32_t>(p); if (dimensions.Z) data.setField<boost::int32_t>(*dimensions.Z, pointIndex, z); boost::uint32_t time = Utils::read_field<boost::uint32_t>(p); if (dimensions.Time) data.setField<boost::uint32_t>(*dimensions.Time, pointIndex, time); boost::uint8_t red = Utils::read_field<boost::uint8_t>(p); if (dimensions.Red) data.setField<boost::uint8_t>(*dimensions.Red, pointIndex, red); boost::uint8_t green = Utils::read_field<boost::uint8_t>(p); if (dimensions.Green) data.setField<boost::uint8_t>(*dimensions.Green, pointIndex, green); boost::uint8_t blue = Utils::read_field<boost::uint8_t>(p); if (dimensions.Blue) data.setField<boost::uint8_t>(*dimensions.Blue, pointIndex, blue); boost::uint8_t alpha = Utils::read_field<boost::uint8_t>(p); if (dimensions.Alpha) data.setField<boost::uint8_t>(*dimensions.Alpha, pointIndex, alpha); } if (m_format == TERRASOLID_Format_2) { boost::int32_t x = Utils::read_field<boost::int32_t>(p); if (dimensions.X) data.setField<boost::int32_t>(*dimensions.X, pointIndex, x); boost::int32_t y = Utils::read_field<boost::int32_t>(p); if (dimensions.Y) data.setField<boost::int32_t>(*dimensions.Y, pointIndex, y); boost::int32_t z = Utils::read_field<boost::int32_t>(p); if (dimensions.Z) data.setField<boost::int32_t>(*dimensions.Z, pointIndex, z); boost::uint8_t classification = Utils::read_field<boost::uint8_t>(p); if (dimensions.Classification) data.setField<boost::uint8_t>(*dimensions.Classification, pointIndex, classification); boost::uint8_t return_number = Utils::read_field<boost::uint8_t>(p); if (dimensions.ReturnNumber) data.setField<boost::uint8_t>(*dimensions.ReturnNumber, pointIndex, return_number); boost::uint8_t flag = Utils::read_field<boost::uint8_t>(p); if (dimensions.Flag) data.setField<boost::uint8_t>(*dimensions.Flag, pointIndex, flag); boost::uint8_t mark = Utils::read_field<boost::uint8_t>(p); if (dimensions.Mark) data.setField<boost::uint8_t>(*dimensions.Mark, pointIndex, mark); boost::uint16_t flight_line = Utils::read_field<boost::uint16_t>(p); if (dimensions.PointSourceId) data.setField<boost::uint16_t>(*dimensions.PointSourceId, pointIndex, flight_line); boost::uint16_t intensity = Utils::read_field<boost::uint16_t>(p); if (dimensions.Intensity) data.setField<boost::uint16_t>(*dimensions.Intensity, pointIndex, intensity); boost::uint32_t time = Utils::read_field<boost::uint32_t>(p); if (dimensions.Time) data.setField<boost::uint32_t>(*dimensions.Time, pointIndex, time); boost::uint8_t red = Utils::read_field<boost::uint8_t>(p); if (dimensions.Red) data.setField<boost::uint8_t>(*dimensions.Red, pointIndex, red); boost::uint8_t green = Utils::read_field<boost::uint8_t>(p); if (dimensions.Green) data.setField<boost::uint8_t>(*dimensions.Green, pointIndex, green); boost::uint8_t blue = Utils::read_field<boost::uint8_t>(p); if (dimensions.Blue) data.setField<boost::uint8_t>(*dimensions.Blue, pointIndex, blue); boost::uint8_t alpha = Utils::read_field<boost::uint8_t>(p); if (dimensions.Alpha) data.setField<boost::uint8_t>(*dimensions.Alpha, pointIndex, alpha); } data.setNumPoints(pointIndex+1); } delete[] buf; return numPoints; }
boost::uint32_t Index::readBufferImpl(PointBuffer& data) { const boost::uint32_t numRead = getPrevIterator().read(data); bool logOutput = m_stage.log()->getLevel() > logDEBUG4; m_stage.log()->floatPrecision(8); if (logOutput) { m_stage.log()->get(logDEBUG2) << "inserting data into index data array of capacity: " << data.getCapacity() << std::endl; } #ifdef PDAL_HAVE_FLANN for (boost::uint32_t pointIndex=0; pointIndex<numRead; pointIndex++) { float x = static_cast<float>(getScaledValue(data, *m_xDim, pointIndex)); float y = static_cast<float>(getScaledValue(data, *m_yDim, pointIndex)); float z = static_cast<float>(getScaledValue(data, *m_zDim, pointIndex)); m_data.push_back(x); m_data.push_back(y); if (m_stage.getDimensions() > 2) m_data.push_back(z); if (logOutput) { m_stage.log()->get(logDEBUG4) << "index: " << pointIndex << " x: " << x << " y: " << y << " z: " << z << std::endl; } } #endif return numRead; }