boost::uint32_t Crop::readBufferImpl(PointBuffer& dstData)
{
// The client has asked us for dstData.getCapacity() points.
// We will read from our previous stage until we get that amount (or
// until the previous stage runs out of points).
boost::uint32_t numPointsNeeded = dstData.getCapacity();
assert(dstData.getNumPoints() == 0);
while (numPointsNeeded > 0)
{
if (getPrevIterator().atEnd()) break;
// set up buffer to be filled by prev stage
PointBuffer srcData(dstData.getSchema(), numPointsNeeded);
// read from prev stage
const boost::uint32_t numSrcPointsRead = getPrevIterator().read(srcData);
assert(numSrcPointsRead == srcData.getNumPoints());
assert(numSrcPointsRead <= numPointsNeeded);
// we got no data, and there is no more to get -- exit the loop
if (numSrcPointsRead == 0) break;
// copy points from src (prev stage) into dst (our stage),
// based on the CropFilter's rules (i.e. its bounds)
const boost::uint32_t numPointsProcessed = m_cropFilter.processBuffer(dstData, srcData);
numPointsNeeded -= numPointsProcessed;
}
const boost::uint32_t numPointsAchieved = dstData.getNumPoints();
return numPointsAchieved;
}
boost::uint32_t Reader::processBuffer(PointBuffer& data, boost::uint64_t index) const
{
const Schema& schema = data.getSchema();
// how many are they asking for?
boost::uint64_t numPointsWanted = data.getCapacity();
// we can only give them as many as we have left
boost::uint64_t numPointsAvailable = getNumPoints() - index;
if (numPointsAvailable < numPointsWanted)
numPointsWanted = numPointsAvailable;
schema::DimensionMap* d = m_buffer.getSchema().mapDimensions(data.getSchema());
data.setNumPoints(0);
PointBuffer::copyLikeDimensions(m_buffer, data,
*d,
index,
0,
numPointsWanted);
data.setNumPoints(numPointsWanted);
delete d;
return numPointsWanted;
}
boost::uint32_t DecimationFilterSequentialIterator::readBufferImpl(PointBuffer& dstData)
{
// The client has asked us for dstData.getCapacity() points.
// We will read from our previous stage until we get that amount (or
// until the previous stage runs out of points).
boost::uint32_t numPointsNeeded = dstData.getCapacity();
assert(dstData.getNumPoints() == 0);
// set up buffer to be filled by prev stage
PointBuffer srcData(dstData.getSchemaLayout(), numPointsNeeded);
while (numPointsNeeded > 0)
{
// read from prev stage
const boost::uint64_t srcStartIndex = getPrevIterator().getIndex();
const boost::uint32_t numSrcPointsRead = getPrevIterator().read(srcData);
assert(numSrcPointsRead <= srcData.getNumPoints());
//assert(numSrcPointsRead <= numPointsNeeded);
// we got no data, and there is no more to get -- exit the loop
if (numSrcPointsRead == 0) break;
// copy points from src (prev stage) into dst (our stage),
// based on the CropFilter's rules (i.e. its bounds)
const boost::uint32_t numPointsAdded = m_filter.processBuffer(dstData, srcData, srcStartIndex);
numPointsNeeded -= numPointsAdded;
//printf(".");fflush(stdout);
}
const boost::uint32_t numPointsAchieved = dstData.getNumPoints();
return numPointsAchieved;
}
boost::uint32_t Mosaic::readBufferImpl(PointBuffer& destData)
{
boost::uint32_t totalNumPointsToRead = destData.getCapacity();
boost::uint32_t totalNumPointsRead = 0;
boost::uint32_t destPointIndex = 0;
// for each stage, we read as many points as we can
while (totalNumPointsRead < totalNumPointsToRead)
{
assert(m_iteratorIndex < m_prevIterators.size());
m_prevIterator = m_prevIterators[m_iteratorIndex];
// read as much as we can into temp buffer
PointBuffer tmp(destData.getSchema(), totalNumPointsToRead-totalNumPointsRead);
boost::uint32_t numRead = m_prevIterator->read(tmp);
totalNumPointsRead += numRead;
// concat the temp buffer on to end of real dest buffer
destData.copyPointsFast(destPointIndex, 0, tmp, numRead);
destPointIndex += numRead;
destData.setNumPoints(destData.getNumPoints() + numRead);
if (m_prevIterator->atEnd())
{
++m_iteratorIndex;
}
if (m_iteratorIndex == m_prevIterators.size())
{
break;
}
}
return totalNumPointsRead;
}
// BUG: this duplicates the code above
boost::uint32_t Cache::readBufferImpl(PointBuffer& data)
{
const boost::uint32_t cacheBlockSize = m_filter.getCacheBlockSize();
const boost::uint64_t currentPointIndex = getIndex();
// for now, we only read from the cache if they are asking for one point
// (this avoids the problem of an N-point request needing more than one
// cached block to satisfy it)
if (data.getCapacity() != 1)
{
const boost::uint32_t numRead = getPrevIterator().read(data);
// if they asked for a full block and we got a full block,
// and the block we got is properly aligned and not already cached,
// then let's cache it!
const bool isCacheable = (data.getCapacity() == cacheBlockSize) &&
(numRead == cacheBlockSize) &&
(currentPointIndex % cacheBlockSize == 0);
if (isCacheable && (m_filter.lookupInCache(currentPointIndex) == NULL))
{
m_filter.addToCache(currentPointIndex, data);
}
m_filter.updateStats(numRead, data.getCapacity());
return numRead;
}
// they asked for just one point -- first, check Mister Cache
const PointBuffer* block = m_filter.lookupInCache(currentPointIndex);
if (block != NULL)
{
// A hit! A palpable hit!
data.copyPointFast(0, currentPointIndex % cacheBlockSize, *block);
m_filter.updateStats(0, 1);
return 1;
}
// Not in the cache, so do a normal read :-(
const boost::uint32_t numRead = getPrevIterator().read(data);
m_filter.updateStats(numRead, numRead);
return numRead;
}
void Selector::alterSchema(PointBuffer& buffer)
{
Schema const& original_schema = buffer.getSchema();
Schema new_schema = buffer.getSchema();
std::map<std::string, bool> const& ignoredMap = m_selectorFilter.getIgnoredMap();
// for (std::map<std::string, bool>::const_iterator i = ignoredMap.begin();
// i != ignoredMap.end(); ++i)
// {
// boost::optional<Dimension const&> d = original_schema.getDimensionOptional(i->first);
// if (d)
// {
// Dimension d2(*d);
// boost::uint32_t flags = d2.getFlags();
// if (i->second)
// d2.setFlags(flags | dimension::IsIgnored);
// new_schema.setDimension(d2);
// }
// }
//
schema::Map dimensions = original_schema.getDimensions();
schema::index_by_index const& dims = dimensions.get<schema::index>();
for (schema::index_by_index::const_iterator t = dims.begin();
t != dims.end();
++t)
{
std::map<std::string, bool>::const_iterator ignored = ignoredMap.find(t->getName());
if (ignored != ignoredMap.end())
{
if (ignored->second) // marked to be dropped
{
// set to ignored
Dimension d2(*t);
boost::uint32_t flags = d2.getFlags();
d2.setFlags(flags | dimension::IsIgnored);
new_schema.setDimension(d2);
}
}
else { // didn't find it in our map of specified dimensions
if (m_selectorFilter.doIgnoreUnspecifiedDimensions())
{
// set to ignored
Dimension d2(*t);
boost::uint32_t flags = d2.getFlags();
d2.setFlags(flags | dimension::IsIgnored);
new_schema.setDimension(d2);
}
}
}
buffer = PointBuffer(new_schema, buffer.getCapacity());
}
PointBuffer* makeTestBuffer()
{
Dimension d1("Classification", dimension::UnsignedInteger, 1);
Dimension d2("X", dimension::SignedInteger, 4);
Dimension d3("Y", dimension::Float, 8);
Schema schema;
schema.appendDimension(d1);
schema.appendDimension(d2);
schema.appendDimension(d3);
std::size_t offX = schema.getDimension(0).getByteOffset();
BOOST_CHECK(offX==0);
std::size_t offY = schema.getDimension(1).getByteOffset();
BOOST_CHECK(offY==1);
std::size_t offZ = schema.getDimension(2).getByteOffset();
BOOST_CHECK(offZ==5);
boost::uint32_t capacity = 17;
PointBuffer* data = new PointBuffer(schema, capacity);
BOOST_CHECK(data->getCapacity() == capacity);
Dimension const& dimC = data->getSchema().getDimension("Classification");
Dimension const& dimX = data->getSchema().getDimension("X");
Dimension const& dimY = data->getSchema().getDimension("Y");
// write the data into the buffer
for (boost::uint32_t i=0; i<data->getCapacity(); i++)
{
const boost::uint8_t x = static_cast<boost::uint8_t>(i)+1;
const boost::int32_t y = i*10;
const double z = i * 100;
data->setField(dimC, i, x);
data->setField(dimX, i, y);
data->setField(dimY, i, z);
data->setNumPoints(i+1);
}
BOOST_CHECK(data->getCapacity() ==17);
BOOST_CHECK(data->getNumPoints() ==17);
return data;
}
boost::uint32_t Predicate::processBuffer(PointBuffer& data, pdal::plang::BufferedInvocation& python) const
{
python.resetArguments();
python.beginChunk(data);
python.execute();
if (!python.hasOutputVariable("Mask"))
{
throw python_error("Mask variable not set in predicate filter function");
}
boost::uint8_t* mask = new boost::uint8_t[data.getNumPoints()];
PointBuffer dstData(data.getSchema(), data.getCapacity());
python.extractResult("Mask", (boost::uint8_t*)mask, data.getNumPoints(), 1, pdal::dimension::RawByte, 1);
boost::uint8_t* dst = dstData.getData(0);
boost::uint8_t* src = data.getData(0);
const Schema& schema = dstData.getSchema();
boost::uint32_t numBytes = schema.getByteSize();
assert(numBytes == data.getSchema().getByteSize());
boost::uint32_t numSrcPoints = data.getNumPoints();
boost::uint32_t count = 0;
for (boost::uint32_t srcIndex=0; srcIndex<numSrcPoints; srcIndex++)
{
if (mask[srcIndex])
{
memcpy(dst, src, numBytes);
dst += numBytes;
++count;
dstData.setNumPoints(count);
}
src += numBytes;
}
data.copyPointsFast(0, 0, dstData, count);
data.setNumPoints(count);
delete[] mask;
return count;
}
void IteratorBase::fillUserBuffer(PointBuffer& user_buffer)
{
Schema const& user_schema = user_buffer.getSchema();
schema::index_by_index const& idx = user_schema.getDimensions().get<schema::index>();
boost::int32_t numUserSpace = user_buffer.getCapacity() - user_buffer.getNumPoints();
if (numUserSpace < 0)
throw pdal_error("We ran out of space!");
boost::int32_t numOraclePoints = m_active_buffer->getNumPoints() - m_buffer_position;
schema::index_by_index::size_type i(0);
for (i = 0; i < idx.size(); ++i)
{
copyDatabaseData(*m_active_buffer,
user_buffer,
idx[i],
m_buffer_position,
user_buffer.getNumPoints(),
(std::min)(numOraclePoints,numUserSpace));
}
bool bSetPointSourceId = getReader().getOptions().getValueOrDefault<bool>("populate_pointsourceid", false);
if (bSetPointSourceId)
{
Dimension const* point_source_field = &(user_buffer.getSchema().getDimensionOptional("PointSourceId").get());
if (point_source_field)
{
for (boost::int32_t i = 0; i < numUserSpace; ++i)
{
if (i < 0)
throw sqlite_driver_error("point_source_field point index is less than 0!");
user_buffer.setField(*point_source_field, i, m_active_cloud_id);
}
}
}
if (numOraclePoints > numUserSpace)
m_buffer_position = m_buffer_position + numUserSpace;
else if (numOraclePoints < numUserSpace)
m_buffer_position = 0;
boost::uint32_t howManyThisRead = (std::min)(numUserSpace, numOraclePoints);
user_buffer.setNumPoints(howManyThisRead + user_buffer.getNumPoints());
}
boost::uint32_t Reader::processBuffer(PointBuffer& data, boost::uint64_t index) const
{
const Schema& schema = data.getSchema();
// make up some data and put it into the buffer
// how many are they asking for?
boost::uint64_t numPointsWanted = data.getCapacity();
// we can only give them as many as we have left
boost::uint64_t numPointsAvailable = getNumPoints() - index;
if (numPointsAvailable < numPointsWanted)
numPointsWanted = numPointsAvailable;
const Bounds<double>& bounds = getBounds();
const std::vector< Range<double> >& dims = bounds.dimensions();
const double minX = dims[0].getMinimum();
const double maxX = dims[0].getMaximum();
const double minY = dims[1].getMinimum();
const double maxY = dims[1].getMaximum();
const double minZ = dims[2].getMinimum();
const double maxZ = dims[2].getMaximum();
const double numDeltas = (double)getNumPoints() - 1.0;
const double delX = (maxX - minX) / numDeltas;
const double delY = (maxY - minY) / numDeltas;
const double delZ = (maxZ - minZ) / numDeltas;
const Dimension& dimX = schema.getDimension("X", getName());
const Dimension& dimY = schema.getDimension("Y", getName());
const Dimension& dimZ = schema.getDimension("Z", getName());
const Dimension& dimTime = schema.getDimension("Time", getName());
boost::uint64_t time = index;
const Reader::Mode mode = getMode();
boost::uint32_t cnt = 0;
data.setNumPoints(0);
for (boost::uint32_t pointIndex=0; pointIndex<numPointsWanted; pointIndex++)
{
double x;
double y;
double z;
switch (mode)
{
case Reader::Random:
x = Utils::random(minX, maxX);
y = Utils::random(minY, maxY);
z = Utils::random(minZ, maxZ);
break;
case Reader::Constant:
x = minX;
y = minY;
z = minZ;
break;
case Reader::Ramp:
x = minX + delX * pointIndex;
y = minY + delY * pointIndex;
z = minZ + delZ * pointIndex;
break;
default:
throw pdal_error("invalid mode in FauxReader");
break;
}
data.setField<double>(dimX, pointIndex, x);
data.setField<double>(dimY, pointIndex, y);
data.setField<double>(dimZ, pointIndex, z);
data.setField<boost::uint64_t>(dimTime, pointIndex, time);
++time;
++cnt;
data.setNumPoints(cnt);
assert(cnt <= data.getCapacity());
}
return cnt;
}
uint32_t MrsidReader::processBuffer(PointBuffer& data, uint64_t index) const
{
const Schema& schema = data.getSchema();
// how many are they asking for?
uint64_t numPointsWanted = data.getCapacity();
// we can only give them as many as we have left
uint64_t numPointsAvailable = getNumPoints() - index;
if (numPointsAvailable < numPointsWanted)
numPointsWanted = numPointsAvailable;
LizardTech::PointData points;
// to do: specify a PointInfo structure that reads only the channels we will output.
points.init(m_PS->getPointInfo(), (size_t)numPointsWanted);
size_t count = m_iter->getNextPoints(points);
uint32_t cnt = 0;
data.setNumPoints(0);
schema::index_by_index const& dims = schema.getDimensions().get<schema::index>();
for (uint32_t pointIndex=0; pointIndex<count; pointIndex++)
{
++cnt;
for (unsigned int i=0; i < dims.size(); i++)
{
Dimension const& d = dims[i];
if (d.getName() == "X" && d.getInterpretation() == dimension::Float && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_X))
{
double *pData = static_cast<double*>(points.getChannel(CHANNEL_NAME_X)->getData());
double value = static_cast<double>(pData[pointIndex]);
data.setField<double>(d, pointIndex, value);
}
else if (d.getName() == "X" && d.getInterpretation() == dimension::SignedInteger && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_X))
{
double *pData = static_cast<double*>(points.getChannel(CHANNEL_NAME_X)->getData());
int32_t value = static_cast<int32_t>(pData[pointIndex]);
data.setField<int32_t>(d, pointIndex, value);
}
else if (d.getName() == "Y" && d.getInterpretation() == dimension::Float && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_Y))
{
double *pData = static_cast<double*>(points.getChannel(CHANNEL_NAME_Y)->getData());
double value = static_cast<double>(pData[pointIndex]);
data.setField<double>(d, pointIndex, value);
}
else if (d.getName() == "Y" && d.getInterpretation() == dimension::SignedInteger && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_Y))
{
double *pData = static_cast<double*>(points.getChannel(CHANNEL_NAME_Y)->getData());
int32_t value = static_cast<int32_t>(pData[pointIndex]);
data.setField<int32_t>(d, pointIndex, value);
}
else if (d.getName() == "Z" && d.getInterpretation() == dimension::Float && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_Z))
{
double *pData = static_cast<double*>(points.getChannel(CHANNEL_NAME_Z)->getData());
double value = static_cast<double>(pData[pointIndex]);
data.setField<double>(d, pointIndex, value);
}
else if (d.getName() == "Z" && d.getInterpretation() == dimension::SignedInteger && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_Z))
{
double *pData = static_cast<double*>(points.getChannel(CHANNEL_NAME_Z)->getData());
int32_t value = static_cast<int32_t>(pData[pointIndex]);
data.setField<int32_t>(d, pointIndex, value);
}
else if (d.getName() == "Time" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_GPSTime))
{
double *pData = static_cast<double*>(points.getChannel(CHANNEL_NAME_GPSTime)->getData());
uint64_t value = static_cast<uint64_t>(pData[pointIndex]);
data.setField<uint64_t>(d, pointIndex, value);
}
else if (d.getName() == "Intensity" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_Intensity))
{
uint16_t *pData = static_cast<uint16_t*>(points.getChannel(CHANNEL_NAME_Intensity)->getData());
uint16_t value = static_cast<uint16_t>(pData[pointIndex]);
data.setField<uint16_t>(d, pointIndex, value);
}
else if (d.getName() == "ReturnNumber" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_ReturnNum))
{
uint8_t *pData = static_cast<uint8_t*>(points.getChannel(CHANNEL_NAME_ReturnNum)->getData());
uint8_t value = static_cast<uint8_t>(pData[pointIndex]);
data.setField<uint8_t>(d, pointIndex, value);
}
else if (d.getName() == "NumberOfReturns" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_NumReturns))
{
uint8_t *pData = static_cast<uint8_t*>(points.getChannel(CHANNEL_NAME_NumReturns)->getData());
uint8_t value = static_cast<uint8_t>(pData[pointIndex]);
data.setField<uint8_t>(d, pointIndex, value);
}
else if (d.getName() == "ScanDirectionFlag" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_ScanDir))
{
uint8_t *pData = static_cast<uint8_t*>(points.getChannel(CHANNEL_NAME_NumReturns)->getData());
uint8_t value = static_cast<uint8_t>(pData[pointIndex]);
data.setField<uint8_t>(d, pointIndex, value);
}
else if (d.getName() == "ScanAngleRank" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_ScanAngle))
{
int8_t *pData = static_cast<int8_t*>(points.getChannel(CHANNEL_NAME_NumReturns)->getData());
int8_t value = static_cast<int8_t>(pData[pointIndex]);
data.setField<int8_t>(d, pointIndex, value);
}
else if (d.getName() == "EdgeOfFlightLine" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_EdgeFlightLine))
{
uint8_t *pData = static_cast<uint8_t*>(points.getChannel(CHANNEL_NAME_NumReturns)->getData());
uint8_t value = static_cast<uint8_t>(pData[pointIndex]);
data.setField<uint8_t>(d, pointIndex, value);
}
else if (d.getName() == "Classification" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_ClassId))
{
uint8_t *pData = static_cast<uint8_t*>(points.getChannel(CHANNEL_NAME_NumReturns)->getData());
uint8_t value = static_cast<uint8_t>(pData[pointIndex]);
data.setField<uint8_t>(d, pointIndex, value);
}
else if (d.getName() == "UserData" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_UserData))
{
uint8_t *pData = static_cast<uint8_t*>(points.getChannel(CHANNEL_NAME_NumReturns)->getData());
uint8_t value = static_cast<uint8_t>(pData[pointIndex]);
data.setField<uint8_t>(d, pointIndex, value);
}
else if (d.getName() == "PointSourceId" && m_PS->getPointInfo().hasChannel(CHANNEL_NAME_SourceId))
{
uint16_t *pData = static_cast<uint16_t*>(points.getChannel(CHANNEL_NAME_NumReturns)->getData());
uint16_t value = static_cast<uint16_t>(pData[pointIndex]);
data.setField<uint16_t>(d, pointIndex, value);
}
}
}
data.setNumPoints(cnt);
assert(cnt <= data.getCapacity());
return cnt;
}
boost::uint32_t IteratorBase::myReadBlocks(PointBuffer& user_buffer)
{
boost::uint32_t numPointsRead = 0;
std::string const& query = getReader().getOptions().getValueOrThrow<std::string>("query");
::soci::row block;
::soci::indicator ind = ::soci::i_null;
::soci::statement blocks = (m_session->prepare << query, ::soci::into(block, ind));
blocks.execute();
bool bDidRead = blocks.fetch();
// if (ind == ::soci::i_null)
// {
// // We have no points to return
// getReader().log()->get(logDEBUG) << "Query returned no points" << std::endl;
// return 0;
// }
//
// size_t size = block.size();
// for (size_t i = 0; i < size; ++i)
// {
// getReader().log()->get(logDEBUG3) << "column: " << block.get_properties(i).get_name() << std::endl;
// }
if (!m_active_buffer)
{
m_active_buffer = fetchPointBuffer(block.get<int>("cloud_id"),
block.get<std::string>("schema"),
user_buffer.getCapacity());
m_active_cloud_id = block.get<int>("cloud_id");
}
//
// This shouldn't ever happen
int num_points = block.get<int>("num_points");
if (num_points > static_cast<boost::int32_t>(m_active_buffer->getCapacity()))
{
std::ostringstream oss;
oss << "Block size, " << num_points <<", is too large to fit in "
<< "buffer of size " << user_buffer.getCapacity() <<". Increase buffer capacity with writer's \"chunk_size\" option "
<< "or increase the read buffer size";
throw buffer_too_small(oss.str());
}
while (bDidRead)
{
boost::uint32_t numReadThisBlock = static_cast<boost::uint32_t>(block.get<int>("num_points"));
boost::uint32_t numSpaceLeftThisBuffer = user_buffer.getCapacity() - user_buffer.getNumPoints();
getReader().log()->get(logDEBUG4) << "IteratorBase::myReadBlocks:" "numReadThisBlock: "
<< numReadThisBlock << " numSpaceLeftThisBlock: "
<< numSpaceLeftThisBuffer << " total numPointsRead: "
<< numPointsRead << std::endl;
numPointsRead = numPointsRead + numReadThisBlock;
readBlob(block, (std::min)(numReadThisBlock, numSpaceLeftThisBuffer));
fillUserBuffer(user_buffer);
bDidRead = blocks.fetch();
// if (!bDidRead)
// return user_buffer.getNumPoints();
boost::int32_t const& current_cloud_id = block.get<int>("cloud_id");
if (current_cloud_id != m_active_cloud_id)
{
getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: current_cloud_id: "
<< current_cloud_id << " m_active_cloud_id: "
<< m_active_cloud_id << std::endl;
m_active_buffer = fetchPointBuffer(current_cloud_id, block.get<std::string>("schema"), user_buffer.getCapacity());
m_active_cloud_id = current_cloud_id;
return user_buffer.getNumPoints();
}
}
return numPointsRead;
}
// append all points from src buffer to end of dst buffer, based on the our bounds
boost::uint32_t Crop::processBuffer(PointBuffer& dstData, const PointBuffer& srcData) const
{
const Schema& schema = dstData.getSchema();
const Bounds<double>& bounds = this->getBounds();
boost::uint32_t numSrcPoints = srcData.getNumPoints();
boost::uint32_t dstIndex = dstData.getNumPoints();
boost::uint32_t numPointsAdded = 0;
boost::optional<Dimension const&> dimX = schema.getDimension("X");
boost::optional<Dimension const&> dimY = schema.getDimension("Y");
boost::optional<Dimension const&> dimZ = schema.getDimension("Z");
for (boost::uint32_t srcIndex=0; srcIndex<numSrcPoints; srcIndex++)
{
// need to scale the values
double x(0.0);
double y(0.0);
double z(0.0);
if (dimX->getInterpretation() == dimension::SignedInteger )
{
boost::int32_t xi = srcData.getField<boost::int32_t>(*dimX, srcIndex);
boost::int32_t yi = srcData.getField<boost::int32_t>(*dimY, srcIndex);
boost::int32_t zi = srcData.getField<boost::int32_t>(*dimZ, srcIndex);
x = dimX->applyScaling(xi);
y = dimY->applyScaling(yi);
z = dimZ->applyScaling(zi);
}
else if (dimX->getInterpretation() == dimension::UnsignedInteger)
{
boost::uint32_t xi = srcData.getField<boost::uint32_t>(*dimX, srcIndex);
boost::uint32_t yi = srcData.getField<boost::uint32_t>(*dimY, srcIndex);
boost::uint32_t zi = srcData.getField<boost::uint32_t>(*dimZ, srcIndex);
x = dimX->applyScaling(xi);
y = dimY->applyScaling(yi);
z = dimZ->applyScaling(zi);
} else
{
x = srcData.getField<double>(*dimX, srcIndex);
y = srcData.getField<double>(*dimY, srcIndex);
z = srcData.getField<double>(*dimZ, srcIndex);
}
Vector<double> point(x,y,z);
if (bounds.contains(point))
{
dstData.copyPointFast(dstIndex, srcIndex, srcData);
dstData.setNumPoints(dstIndex+1);
++dstIndex;
++numPointsAdded;
}
}
assert(dstIndex <= dstData.getCapacity());
return numPointsAdded;
}
void IteratorBase::fillUserBuffer(PointBuffer& user_buffer)
{
Schema const& user_schema = user_buffer.getSchema();
schema::index_by_index const& idx = user_schema.getDimensions().get<schema::index>();
boost::int32_t numOraclePoints = m_oracle_buffer->getNumPoints() - m_buffer_position;
boost::int32_t numUserSpace = user_buffer.getCapacity() - user_buffer.getNumPoints();
boost::int32_t howManyThisRead = (std::min)(numUserSpace, numOraclePoints);
if (numUserSpace < 0)
{
std::ostringstream oss;
oss << "numUserSpace < 0! : " << numUserSpace;
throw pdal_error(oss.str());
}
if (numOraclePoints < 0)
{
std::ostringstream oss;
oss << "numOraclePoints < 0! : " << numOraclePoints;
throw pdal_error(oss.str());
}
if (user_buffer.getNumPoints() + howManyThisRead > user_buffer.getCapacity())
{
std::ostringstream oss;
oss << "user_buffer.getNumPoints() + howManyThisRead == " << user_buffer.getNumPoints() + howManyThisRead
<< " but user_buffer.getCapacity() == " << user_buffer.getCapacity();
throw pdal_error(oss.str());
}
PointBuffer::copyLikeDimensions(*m_oracle_buffer, user_buffer,
*m_dimension_map,
m_buffer_position, user_buffer.getNumPoints(),
howManyThisRead);
Schema const& src = m_oracle_buffer->getSchema();
Dimension const& src_x = src.getDimension("drivers.oci.reader.X");
Dimension const& src_y = src.getDimension("drivers.oci.reader.Y");
Dimension const& src_z = src.getDimension("drivers.oci.reader.Z");
Schema const& dst = user_buffer.getSchema();
Dimension const& dst_x = dst.getDimension("drivers.oci.reader.X");
Dimension const& dst_y = dst.getDimension("drivers.oci.reader.Y");
Dimension const& dst_z = dst.getDimension("drivers.oci.reader.Z");
bool bDifferentScales = (!pdal::Utils::compare_distance(dst_x.getNumericScale(), src_x.getNumericScale()) ||
!pdal::Utils::compare_distance(dst_y.getNumericScale(), src_y.getNumericScale()) ||
!pdal::Utils::compare_distance(dst_z.getNumericScale(), src_z.getNumericScale()));
bool bDifferentOffsets = (!pdal::Utils::compare_distance(dst_x.getNumericOffset(), src_x.getNumericOffset()) ||
!pdal::Utils::compare_distance(dst_y.getNumericOffset(), src_y.getNumericOffset()) ||
!pdal::Utils::compare_distance(dst_z.getNumericOffset(), src_z.getNumericOffset()));
bool bNormalizeXYZ = getReader().getOptions().getValueOrDefault<bool>("do_normalize_xyz", true);
if ((bDifferentScales || bDifferentOffsets) && bNormalizeXYZ)
{
for (unsigned i = m_buffer_position; i < howManyThisRead; ++i)
{
double x = m_oracle_buffer->applyScaling(src_x, i);
boost::int32_t xi = dst_x.removeScaling<boost::int32_t>(x);
user_buffer.setField<boost::int32_t>(dst_x, user_buffer.getNumPoints()+i, xi);
double y = m_oracle_buffer->applyScaling(src_y, i);
boost::int32_t yi = dst_y.removeScaling<boost::int32_t>(y);
user_buffer.setField<boost::int32_t>(dst_y, user_buffer.getNumPoints()+i, yi);
double z = m_oracle_buffer->applyScaling(src_z, i);
boost::int32_t zi = dst_z.removeScaling<boost::int32_t>(z);
user_buffer.setField<boost::int32_t>(dst_z, user_buffer.getNumPoints()+i, zi);
}
}
getReader().log()->get(logDEBUG2) << "IteratorBase::fillUserBuffer m_buffer_position: "
<< m_buffer_position << std::endl;
if (numOraclePoints > numUserSpace)
m_buffer_position = m_buffer_position + numUserSpace;
else if (numOraclePoints < numUserSpace)
m_buffer_position = 0;
getReader().log()->get(logDEBUG2) << "IteratorBase::fillUserBuffer m_buffer_position: "
<< m_buffer_position << std::endl;
bool bSetPointSourceId = getReader().getOptions().getValueOrDefault<bool>("populate_pointsourceid", false);
if (bSetPointSourceId)
{
Dimension const* point_source_field = user_buffer.getSchema().getDimensionPtr("PointSourceId");
if (point_source_field)
{
for (boost::int32_t i = m_buffer_position; i < howManyThisRead; ++i)
{
assert(user_buffer.getNumPoints() + i < user_buffer.getCapacity());
if (point_source_field->getByteSize() == 2)
user_buffer.setField(*point_source_field,
user_buffer.getNumPoints() + i,
static_cast<boost::uint16_t>(m_active_cloud_id));
else if (point_source_field->getByteSize() == 4)
{
user_buffer.setField(*point_source_field,
user_buffer.getNumPoints() + i,
m_active_cloud_id);
}
}
}
}
user_buffer.setNumPoints(howManyThisRead + user_buffer.getNumPoints());
}
boost::uint32_t IteratorBase::myReadBlocks(PointBuffer& user_buffer)
{
boost::uint32_t numPointsRead = 0;
user_buffer.setNumPoints(0);
bool bDidRead = false;
if (!m_oracle_buffer)
{
m_oracle_buffer = fetchPointBuffer(m_initialQueryStatement, m_block->pc);
if (!m_oracle_buffer) throw pdal_error("m_oracle_buffer was NULL!");
m_dimension_map = fetchDimensionMap(m_initialQueryStatement, m_block->pc, *m_oracle_buffer, user_buffer);
boost::int32_t current_cloud_id(0);
current_cloud_id = m_initialQueryStatement->GetInteger(&m_block->pc->pc_id);
m_active_cloud_id = current_cloud_id;
}
// This shouldn't ever happen
if (m_block->num_points > static_cast<boost::int32_t>(m_oracle_buffer->getCapacity()))
{
m_oracle_buffer->resize(m_block->num_points);
}
if (!m_block->num_points)
{
// We still have a block of data from the last readBuffer call
// that was partially read.
getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: fetching first block" << std::endl;
bDidRead = m_initialQueryStatement->Fetch();
if (!bDidRead)
{
m_at_end = true;
return 0;
}
user_buffer.setSpatialBounds(getBounds(m_initialQueryStatement, m_block));
}
else
{
// Our read was already "done" last readBuffer call, but if we're done,
// we're done
if (m_at_end)
getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: we are at end of the blocks;" << std::endl;
else
getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: we have points left to read on this block" << std::endl;
if (m_at_end) return 0;
bDidRead = true;
}
while (bDidRead)
{
boost::uint32_t numReadThisBlock = m_block->num_points;
boost::uint32_t numSpaceLeftThisBuffer = user_buffer.getCapacity() - user_buffer.getNumPoints();
getReader().log()->get(logDEBUG4) << "IteratorBase::myReadBlocks:" "numReadThisBlock: "
<< numReadThisBlock << " numSpaceLeftThisBlock: "
<< numSpaceLeftThisBuffer << " total numPointsRead: "
<< numPointsRead << std::endl;
numPointsRead = numPointsRead + numReadThisBlock;
readBlob(m_initialQueryStatement, m_block, m_block->num_points);
fillUserBuffer(user_buffer);
if (m_buffer_position != 0)
{
return user_buffer.getNumPoints();
}
else
{
bDidRead = m_initialQueryStatement->Fetch();
if (!bDidRead)
{
getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: done reading block. Read " << numPointsRead << " points" << std::endl;
m_at_end = true;
return user_buffer.getNumPoints();
}
}
boost::int32_t current_cloud_id(0);
current_cloud_id = m_initialQueryStatement->GetInteger(&m_block->pc->pc_id);
getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: current_cloud_id: "
<< current_cloud_id << " m_active_cloud_id: "
<< m_active_cloud_id << std::endl;
if (current_cloud_id != m_active_cloud_id)
{
m_oracle_buffer = fetchPointBuffer(m_initialQueryStatement, m_block->pc);
if (!m_oracle_buffer) throw pdal_error("m_oracle_buffer was NULL!");
m_dimension_map = fetchDimensionMap(m_initialQueryStatement, m_block->pc, *m_oracle_buffer, user_buffer);
m_active_cloud_id = current_cloud_id;
return user_buffer.getNumPoints();
}
}
return numPointsRead;
}
boost::uint32_t Reader::readBufferImpl(PointBuffer& data)
{
m_log->get(logDEBUG5) << "Reading a point buffer of " <<
data.getCapacity() << " points." << std::endl;
return m_reader.processBuffer(data, getIndex());
}
boost::uint32_t Reader::processBuffer(PointBuffer& data, std::istream& stream, boost::uint64_t numPointsLeft) const
{
// we must not read more points than are left in the file
const boost::uint64_t numPoints64 = std::min<boost::uint64_t>(data.getCapacity(), numPointsLeft);
const boost::uint32_t numPoints = (boost::uint32_t)std::min<boost::uint64_t>(numPoints64, std::numeric_limits<boost::uint32_t>::max());
const Schema& schema = data.getSchema();
const int pointByteCount = getPointDataSize();
const PointDimensions dimensions(schema, getName());
boost::uint8_t* buf = new boost::uint8_t[pointByteCount * numPoints];
Utils::read_n(buf, stream, pointByteCount * numPoints);
for (boost::uint32_t pointIndex=0; pointIndex<numPoints; pointIndex++)
{
boost::uint8_t* p = buf + pointByteCount * pointIndex;
if (m_format == TERRASOLID_Format_1)
{
boost::uint8_t classification = Utils::read_field<boost::uint8_t>(p);
if (dimensions.Classification)
data.setField<boost::uint8_t>(*dimensions.Classification, pointIndex, classification);
boost::uint8_t flight_line = Utils::read_field<boost::uint8_t>(p);
if (dimensions.PointSourceId)
data.setField<boost::uint8_t>(*dimensions.PointSourceId, pointIndex, flight_line);
boost::uint16_t echo_int = Utils::read_field<boost::uint16_t>(p);
if (dimensions.ReturnNumber)
data.setField<boost::uint16_t>(*dimensions.ReturnNumber, pointIndex, echo_int);
boost::int32_t x = Utils::read_field<boost::int32_t>(p);
if (dimensions.X)
data.setField<boost::int32_t>(*dimensions.X, pointIndex, x);
boost::int32_t y = Utils::read_field<boost::int32_t>(p);
if (dimensions.Y)
data.setField<boost::int32_t>(*dimensions.Y, pointIndex, y);
boost::int32_t z = Utils::read_field<boost::int32_t>(p);
if (dimensions.Z)
data.setField<boost::int32_t>(*dimensions.Z, pointIndex, z);
boost::uint32_t time = Utils::read_field<boost::uint32_t>(p);
if (dimensions.Time)
data.setField<boost::uint32_t>(*dimensions.Time, pointIndex, time);
boost::uint8_t red = Utils::read_field<boost::uint8_t>(p);
if (dimensions.Red)
data.setField<boost::uint8_t>(*dimensions.Red, pointIndex, red);
boost::uint8_t green = Utils::read_field<boost::uint8_t>(p);
if (dimensions.Green)
data.setField<boost::uint8_t>(*dimensions.Green, pointIndex, green);
boost::uint8_t blue = Utils::read_field<boost::uint8_t>(p);
if (dimensions.Blue)
data.setField<boost::uint8_t>(*dimensions.Blue, pointIndex, blue);
boost::uint8_t alpha = Utils::read_field<boost::uint8_t>(p);
if (dimensions.Alpha)
data.setField<boost::uint8_t>(*dimensions.Alpha, pointIndex, alpha);
}
if (m_format == TERRASOLID_Format_2)
{
boost::int32_t x = Utils::read_field<boost::int32_t>(p);
if (dimensions.X)
data.setField<boost::int32_t>(*dimensions.X, pointIndex, x);
boost::int32_t y = Utils::read_field<boost::int32_t>(p);
if (dimensions.Y)
data.setField<boost::int32_t>(*dimensions.Y, pointIndex, y);
boost::int32_t z = Utils::read_field<boost::int32_t>(p);
if (dimensions.Z)
data.setField<boost::int32_t>(*dimensions.Z, pointIndex, z);
boost::uint8_t classification = Utils::read_field<boost::uint8_t>(p);
if (dimensions.Classification)
data.setField<boost::uint8_t>(*dimensions.Classification, pointIndex, classification);
boost::uint8_t return_number = Utils::read_field<boost::uint8_t>(p);
if (dimensions.ReturnNumber)
data.setField<boost::uint8_t>(*dimensions.ReturnNumber, pointIndex, return_number);
boost::uint8_t flag = Utils::read_field<boost::uint8_t>(p);
if (dimensions.Flag)
data.setField<boost::uint8_t>(*dimensions.Flag, pointIndex, flag);
boost::uint8_t mark = Utils::read_field<boost::uint8_t>(p);
if (dimensions.Mark)
data.setField<boost::uint8_t>(*dimensions.Mark, pointIndex, mark);
boost::uint16_t flight_line = Utils::read_field<boost::uint16_t>(p);
if (dimensions.PointSourceId)
data.setField<boost::uint16_t>(*dimensions.PointSourceId, pointIndex, flight_line);
boost::uint16_t intensity = Utils::read_field<boost::uint16_t>(p);
if (dimensions.Intensity)
data.setField<boost::uint16_t>(*dimensions.Intensity, pointIndex, intensity);
boost::uint32_t time = Utils::read_field<boost::uint32_t>(p);
if (dimensions.Time)
data.setField<boost::uint32_t>(*dimensions.Time, pointIndex, time);
boost::uint8_t red = Utils::read_field<boost::uint8_t>(p);
if (dimensions.Red)
data.setField<boost::uint8_t>(*dimensions.Red, pointIndex, red);
boost::uint8_t green = Utils::read_field<boost::uint8_t>(p);
if (dimensions.Green)
data.setField<boost::uint8_t>(*dimensions.Green, pointIndex, green);
boost::uint8_t blue = Utils::read_field<boost::uint8_t>(p);
if (dimensions.Blue)
data.setField<boost::uint8_t>(*dimensions.Blue, pointIndex, blue);
boost::uint8_t alpha = Utils::read_field<boost::uint8_t>(p);
if (dimensions.Alpha)
data.setField<boost::uint8_t>(*dimensions.Alpha, pointIndex, alpha);
}
data.setNumPoints(pointIndex+1);
}
delete[] buf;
return numPoints;
}
boost::uint32_t Index::readBufferImpl(PointBuffer& data)
{
const boost::uint32_t numRead = getPrevIterator().read(data);
bool logOutput = m_stage.log()->getLevel() > logDEBUG4;
m_stage.log()->floatPrecision(8);
if (logOutput)
{
m_stage.log()->get(logDEBUG2) << "inserting data into index data array of capacity: " << data.getCapacity() << std::endl;
}
#ifdef PDAL_HAVE_FLANN
for (boost::uint32_t pointIndex=0; pointIndex<numRead; pointIndex++)
{
float x = static_cast<float>(getScaledValue(data, *m_xDim, pointIndex));
float y = static_cast<float>(getScaledValue(data, *m_yDim, pointIndex));
float z = static_cast<float>(getScaledValue(data, *m_zDim, pointIndex));
m_data.push_back(x);
m_data.push_back(y);
if (m_stage.getDimensions() > 2)
m_data.push_back(z);
if (logOutput)
{
m_stage.log()->get(logDEBUG4) << "index: " << pointIndex << " x: " << x << " y: " << y << " z: " << z << std::endl;
}
}
#endif
return numRead;
}
