boost::uint32_t Crop::readBufferImpl(PointBuffer& dstData)
{
// The client has asked us for dstData.getCapacity() points.
// We will read from our previous stage until we get that amount (or
// until the previous stage runs out of points).
boost::uint32_t numPointsNeeded = dstData.getCapacity();
assert(dstData.getNumPoints() == 0);
while (numPointsNeeded > 0)
{
if (getPrevIterator().atEnd()) break;
// set up buffer to be filled by prev stage
PointBuffer srcData(dstData.getSchema(), numPointsNeeded);
// read from prev stage
const boost::uint32_t numSrcPointsRead = getPrevIterator().read(srcData);
assert(numSrcPointsRead == srcData.getNumPoints());
assert(numSrcPointsRead <= numPointsNeeded);
// we got no data, and there is no more to get -- exit the loop
if (numSrcPointsRead == 0) break;
// copy points from src (prev stage) into dst (our stage),
// based on the CropFilter's rules (i.e. its bounds)
const boost::uint32_t numPointsProcessed = m_cropFilter.processBuffer(dstData, srcData);
numPointsNeeded -= numPointsProcessed;
}
const boost::uint32_t numPointsAchieved = dstData.getNumPoints();
return numPointsAchieved;
}
boost::uint32_t DecimationFilterSequentialIterator::readBufferImpl(PointBuffer& dstData)
{
// The client has asked us for dstData.getCapacity() points.
// We will read from our previous stage until we get that amount (or
// until the previous stage runs out of points).
boost::uint32_t numPointsNeeded = dstData.getCapacity();
assert(dstData.getNumPoints() == 0);
// set up buffer to be filled by prev stage
PointBuffer srcData(dstData.getSchemaLayout(), numPointsNeeded);
while (numPointsNeeded > 0)
{
// read from prev stage
const boost::uint64_t srcStartIndex = getPrevIterator().getIndex();
const boost::uint32_t numSrcPointsRead = getPrevIterator().read(srcData);
assert(numSrcPointsRead <= srcData.getNumPoints());
//assert(numSrcPointsRead <= numPointsNeeded);
// we got no data, and there is no more to get -- exit the loop
if (numSrcPointsRead == 0) break;
// copy points from src (prev stage) into dst (our stage),
// based on the CropFilter's rules (i.e. its bounds)
const boost::uint32_t numPointsAdded = m_filter.processBuffer(dstData, srcData, srcStartIndex);
numPointsNeeded -= numPointsAdded;
//printf(".");fflush(stdout);
}
const boost::uint32_t numPointsAchieved = dstData.getNumPoints();
return numPointsAchieved;
}
boost::uint32_t ByteSwap::processBuffer(PointBuffer& dstData, const PointBuffer& srcData) const
{
const Schema& dstSchema = dstData.getSchema();
schema::index_by_index const& dstDims = dstSchema.getDimensions().get<schema::index>();
dstData.setSpatialBounds(srcData.getSpatialBounds());
dstData.copyPointsFast(0, 0, srcData, srcData.getNumPoints());
dstData.setNumPoints(srcData.getNumPoints());
for (boost::uint32_t i = 0; i != dstData.getNumPoints(); ++i)
{
boost::uint8_t* data = dstData.getData(i);
std::size_t position = 0;
for (boost::uint32_t n = 0; n < dstDims.size(); ++n)
{
const Dimension& d = dstSchema.getDimension(n);
std::size_t size = d.getByteSize();
boost::uint8_t* pos = data + position;
SWAP_ENDIANNESS_N(*pos, size);
position = position + size;
}
}
return dstData.getNumPoints();
}
boost::uint32_t Predicate::processBuffer(PointBuffer& data, pdal::plang::BufferedInvocation& python) const
{
python.resetArguments();
python.beginChunk(data);
python.execute();
if (!python.hasOutputVariable("Mask"))
{
throw python_error("Mask variable not set in predicate filter function");
}
boost::uint8_t* mask = new boost::uint8_t[data.getNumPoints()];
PointBuffer dstData(data.getSchema(), data.getCapacity());
python.extractResult("Mask", (boost::uint8_t*)mask, data.getNumPoints(), 1, pdal::dimension::RawByte, 1);
boost::uint8_t* dst = dstData.getData(0);
boost::uint8_t* src = data.getData(0);
const Schema& schema = dstData.getSchema();
boost::uint32_t numBytes = schema.getByteSize();
assert(numBytes == data.getSchema().getByteSize());
boost::uint32_t numSrcPoints = data.getNumPoints();
boost::uint32_t count = 0;
for (boost::uint32_t srcIndex=0; srcIndex<numSrcPoints; srcIndex++)
{
if (mask[srcIndex])
{
memcpy(dst, src, numBytes);
dst += numBytes;
++count;
dstData.setNumPoints(count);
}
src += numBytes;
}
data.copyPointsFast(0, 0, dstData, count);
data.setNumPoints(count);
delete[] mask;
return count;
}
void InPlaceReprojection::processBuffer(PointBuffer& data) const
{
const boost::uint32_t numPoints = data.getNumPoints();
const Schema& schema = this->getSchema();
Dimension const& d_x = schema.getDimension(getOptions().getValueOrDefault<std::string>("x_dim", "X"));
Dimension const& d_y = schema.getDimension(getOptions().getValueOrDefault<std::string>("y_dim", "Y"));
Dimension const& d_z = schema.getDimension(getOptions().getValueOrDefault<std::string>("z_dim", "Z"));
for (boost::uint32_t pointIndex=0; pointIndex<numPoints; pointIndex++)
{
double x = getScaledValue(data, m_x, pointIndex);
double y = getScaledValue(data, m_y, pointIndex);
double z = getScaledValue(data, m_z, pointIndex);
// std::cout << "input: " << x << " y: " << y << " z: " << z << std::endl;
this->transform(x,y,z);
// std::cout << "output: " << x << " y: " << y << " z: " << z << std::endl;
setScaledValue(data, x, d_x, pointIndex);
setScaledValue(data, y, d_y, pointIndex);
setScaledValue(data, z, d_z, pointIndex);
// std::cout << "set: " << getScaledValue(data, d_x, pointIndex, indexX)
// << " y: " << getScaledValue(data, d_y, pointIndex, indexY)
// << " z: " << getScaledValue(data, d_z, pointIndex, indexZ) << std::endl;
data.setNumPoints(pointIndex+1);
}
return;
}
boost::uint32_t Mosaic::readBufferImpl(PointBuffer& destData)
{
boost::uint32_t totalNumPointsToRead = destData.getCapacity();
boost::uint32_t totalNumPointsRead = 0;
boost::uint32_t destPointIndex = 0;
// for each stage, we read as many points as we can
while (totalNumPointsRead < totalNumPointsToRead)
{
assert(m_iteratorIndex < m_prevIterators.size());
m_prevIterator = m_prevIterators[m_iteratorIndex];
// read as much as we can into temp buffer
PointBuffer tmp(destData.getSchema(), totalNumPointsToRead-totalNumPointsRead);
boost::uint32_t numRead = m_prevIterator->read(tmp);
totalNumPointsRead += numRead;
// concat the temp buffer on to end of real dest buffer
destData.copyPointsFast(destPointIndex, 0, tmp, numRead);
destPointIndex += numRead;
destData.setNumPoints(destData.getNumPoints() + numRead);
if (m_prevIterator->atEnd())
{
++m_iteratorIndex;
}
if (m_iteratorIndex == m_prevIterators.size())
{
break;
}
}
return totalNumPointsRead;
}
void Reprojection::processBuffer(PointBuffer& data) const
{
const boost::uint32_t numPoints = data.getNumPoints();
const Schema& schema = data.getSchema();
Dimension const& dimX = schema.getDimension("X");
Dimension const& dimY = schema.getDimension("Y");
Dimension const& dimZ = schema.getDimension("Z");
for (boost::uint32_t pointIndex=0; pointIndex<numPoints; pointIndex++)
{
double x = data.getField<double>(dimX, pointIndex);
double y = data.getField<double>(dimY, pointIndex);
double z = data.getField<double>(dimZ, pointIndex);
this->transform(x,y,z);
data.setField<double>(dimX, pointIndex, x);
data.setField<double>(dimY, pointIndex, y);
data.setField<double>(dimZ, pointIndex, z);
data.setNumPoints(pointIndex+1);
}
return;
}
void IteratorBase::fillUserBuffer(PointBuffer& user_buffer)
{
Schema const& user_schema = user_buffer.getSchema();
schema::index_by_index const& idx = user_schema.getDimensions().get<schema::index>();
boost::int32_t numUserSpace = user_buffer.getCapacity() - user_buffer.getNumPoints();
if (numUserSpace < 0)
throw pdal_error("We ran out of space!");
boost::int32_t numOraclePoints = m_active_buffer->getNumPoints() - m_buffer_position;
schema::index_by_index::size_type i(0);
for (i = 0; i < idx.size(); ++i)
{
copyDatabaseData(*m_active_buffer,
user_buffer,
idx[i],
m_buffer_position,
user_buffer.getNumPoints(),
(std::min)(numOraclePoints,numUserSpace));
}
bool bSetPointSourceId = getReader().getOptions().getValueOrDefault<bool>("populate_pointsourceid", false);
if (bSetPointSourceId)
{
Dimension const* point_source_field = &(user_buffer.getSchema().getDimensionOptional("PointSourceId").get());
if (point_source_field)
{
for (boost::int32_t i = 0; i < numUserSpace; ++i)
{
if (i < 0)
throw sqlite_driver_error("point_source_field point index is less than 0!");
user_buffer.setField(*point_source_field, i, m_active_cloud_id);
}
}
}
if (numOraclePoints > numUserSpace)
m_buffer_position = m_buffer_position + numUserSpace;
else if (numOraclePoints < numUserSpace)
m_buffer_position = 0;
boost::uint32_t howManyThisRead = (std::min)(numUserSpace, numOraclePoints);
user_buffer.setNumPoints(howManyThisRead + user_buffer.getNumPoints());
}
boost::uint32_t Predicate::readBufferImpl(PointBuffer& dstData)
{
// read in a full block of points
const boost::uint32_t numRead = getPrevIterator().read(dstData);
if (numRead > 0)
{
m_numPointsProcessed = dstData.getNumPoints();
// copies the points as they pass in-place
m_predicateFilter.processBuffer(dstData, *m_pythonMethod);
}
m_numPointsPassed = dstData.getNumPoints();
return dstData.getNumPoints();
}
boost::uint32_t Writer::writeBuffer(const PointBuffer& buffer)
{
boost::uint32_t numPoints = buffer.getNumPoints();
WriteBlock(buffer);
return numPoints;
}
void Writer::PackPointData(PointBuffer const& buffer,
boost::uint8_t** point_data,
boost::uint32_t& point_data_len,
boost::uint32_t& schema_byte_size)
{
// Creates a new buffer that has the ignored dimensions removed from
// it.
schema::index_by_index const& idx = buffer.getSchema().getDimensions().get<schema::index>();
schema_byte_size = 0;
schema::index_by_index::size_type i(0);
for (i = 0; i < idx.size(); ++i)
{
if (! idx[i].isIgnored())
schema_byte_size = schema_byte_size+idx[i].getByteSize();
}
log()->get(logDEBUG) << "Packed schema byte size " << schema_byte_size << std::endl;;
point_data_len = buffer.getNumPoints() * schema_byte_size;
*point_data = new boost::uint8_t[point_data_len];
boost::uint8_t* current_position = *point_data;
for (boost::uint32_t i = 0; i < buffer.getNumPoints(); ++i)
{
boost::uint8_t* data = buffer.getData(i);
for (boost::uint32_t d = 0; d < idx.size(); ++d)
{
if (! idx[d].isIgnored())
{
memcpy(current_position, data, idx[d].getByteSize());
current_position = current_position+idx[d].getByteSize();
}
data = data + idx[d].getByteSize();
}
}
}
Reader::Reader(const Options& options, const PointBuffer& buffer)
: pdal::Reader(options)
, m_buffer(buffer)
{
setNumPoints(buffer.getNumPoints());
setBounds(buffer.getSpatialBounds());
setSchema(buffer.getSchema());
setPointCountType(PointCount_Fixed);
return;
}
void PDALtoPCD(PointBuffer& data, CloudT &cloud)
{
#ifdef PDAL_HAVE_PCL
typedef typename pcl::traits::fieldList<typename CloudT::PointType>::type FieldList;
const pdal::Schema &buffer_schema = data.getSchema();
cloud.width = data.getNumPoints();
cloud.height = 1; // unorganized point cloud
cloud.is_dense = false;
cloud.points.resize(cloud.width);
const pdal::Dimension &dX = buffer_schema.getDimension("X");
const pdal::Dimension &dY = buffer_schema.getDimension("Y");
const pdal::Dimension &dZ = buffer_schema.getDimension("Z");
if (pcl::traits::has_xyz<typename CloudT::PointType>::value)
{
for (size_t i = 0; i < cloud.points.size(); ++i)
{
float xd = data.getField<int32_t>(dX, i) * dX.getNumericScale();
float yd = data.getField<int32_t>(dY, i) * dY.getNumericScale();
float zd = data.getField<int32_t>(dZ, i) * dZ.getNumericScale();
typename CloudT::PointType p = cloud.points[i];
p.x = xd; p.y = yd; p.z = zd;
cloud.points[i] = p;
}
}
boost::optional<pdal::Dimension const &> dI = buffer_schema.getDimensionOptional("Intensity");
if (pcl::traits::has_intensity<typename CloudT::PointType>::value && dI)
{
for (size_t i = 0; i < cloud.points.size(); ++i)
{
boost::int32_t vi = data.getField<boost::int32_t>(*dI, i);
float vd = dI->applyScaling(vi);
typename CloudT::PointType p = cloud.points[i];
pcl::for_each_type<FieldList> (pcl::SetIfFieldExists<typename CloudT::PointType, float> (p, "intensity", vd));
cloud.points[i] = p;
}
}
#endif
}
PointBuffer* makeTestBuffer()
{
Dimension d1("Classification", dimension::UnsignedInteger, 1);
Dimension d2("X", dimension::SignedInteger, 4);
Dimension d3("Y", dimension::Float, 8);
Schema schema;
schema.appendDimension(d1);
schema.appendDimension(d2);
schema.appendDimension(d3);
std::size_t offX = schema.getDimension(0).getByteOffset();
BOOST_CHECK(offX==0);
std::size_t offY = schema.getDimension(1).getByteOffset();
BOOST_CHECK(offY==1);
std::size_t offZ = schema.getDimension(2).getByteOffset();
BOOST_CHECK(offZ==5);
boost::uint32_t capacity = 17;
PointBuffer* data = new PointBuffer(schema, capacity);
BOOST_CHECK(data->getCapacity() == capacity);
Dimension const& dimC = data->getSchema().getDimension("Classification");
Dimension const& dimX = data->getSchema().getDimension("X");
Dimension const& dimY = data->getSchema().getDimension("Y");
// write the data into the buffer
for (boost::uint32_t i=0; i<data->getCapacity(); i++)
{
const boost::uint8_t x = static_cast<boost::uint8_t>(i)+1;
const boost::int32_t y = i*10;
const double z = i * 100;
data->setField(dimC, i, x);
data->setField(dimX, i, y);
data->setField(dimY, i, z);
data->setNumPoints(i+1);
}
BOOST_CHECK(data->getCapacity() ==17);
BOOST_CHECK(data->getNumPoints() ==17);
return data;
}
boost::uint32_t Writer::writeBuffer(const PointBuffer& data)
{
const Schema& schema = data.getSchema();
std::string z_name = getOptions().getValueOrDefault<std::string>("Z", "Z");
pdal::Dimension const& dimX = schema.getDimension("X");
pdal::Dimension const& dimY = schema.getDimension("Y");
pdal::Dimension const& dimZ = schema.getDimension(z_name);
boost::uint32_t numPoints = 0;
double xd(0.0);
double yd(0.0);
double zd(0.0);
for (boost::uint32_t pointIndex=0; pointIndex < data.getNumPoints(); pointIndex++)
{
boost::int32_t x = data.getField<boost::int32_t>(dimX, pointIndex);
boost::int32_t y = data.getField<boost::int32_t>(dimY, pointIndex);
boost::int32_t z = data.getField<boost::int32_t>(dimZ, pointIndex);
xd = dimX.applyScaling<boost::int32_t>(x);
yd = dimY.applyScaling<boost::int32_t>(y);
zd = dimZ.applyScaling<boost::int32_t>(z);
m_bounds.setMinimum(0, (std::min)(xd, m_bounds.getMinimum(0)));
m_bounds.setMinimum(1, (std::min)(yd, m_bounds.getMinimum(1)));
m_bounds.setMaximum(0, (std::max)(xd, m_bounds.getMaximum(0)));
m_bounds.setMaximum(1, (std::max)(yd, m_bounds.getMaximum(1)));
m_coordinates.push_back(boost::tuple<double, double, double>(xd, yd, zd));
numPoints++;
}
return numPoints;
}
pdal::Bounds<double> Writer::CalculateBounds(PointBuffer const& buffer)
{
pdal::Schema const& schema = buffer.getSchema();
pdal::Bounds<double> output;
boost::optional<Dimension const&> dimX = schema.getDimension("X");
boost::optional<Dimension const&> dimY = schema.getDimension("Y");
boost::optional<Dimension const&> dimZ = schema.getDimension("Z");
bool first = true;
for (boost::uint32_t pointIndex=0; pointIndex<buffer.getNumPoints(); pointIndex++)
{
const boost::int32_t xi = buffer.getField<boost::int32_t>(*dimX, pointIndex);
const boost::int32_t yi = buffer.getField<boost::int32_t>(*dimY, pointIndex);
const boost::int32_t zi = buffer.getField<boost::int32_t>(*dimZ, pointIndex);
const double xd = dimX->applyScaling(xi);
const double yd = dimY->applyScaling(yi);
const double zd = dimZ->applyScaling(zi);
Vector<double> v(xd, yd, zd);
if (first)
{
output = pdal::Bounds<double>(xd, yd, zd, xd, yd, zd);
if (m_pcExtent.empty())
m_pcExtent = output;
first = false;
}
output.grow(v);
}
m_pcExtent.grow(output);
return output;
}
bool Writer::WriteBlock(PointBuffer const& buffer)
{
bool bUsePartition = m_block_table_partition_column.size() != 0;
// Pluck the block id out of the first point in the buffer
pdal::Schema const& schema = buffer.getSchema();
Dimension const& blockDim = schema.getDimension("BlockID");
boost::int32_t block_id = buffer.getField<boost::int32_t>(blockDim, 0);
std::ostringstream oss;
std::ostringstream partition;
if (bUsePartition)
{
partition << "," << m_block_table_partition_column;
}
oss << "INSERT INTO "<< m_block_table_name <<
"(OBJ_ID, BLK_ID, NUM_POINTS, POINTS, "
"PCBLK_MIN_RES, BLK_EXTENT, PCBLK_MAX_RES, NUM_UNSORTED_POINTS, PT_SORT_DIM";
if (bUsePartition)
oss << partition.str();
oss << ") "
"VALUES ( :1, :2, :3, :4, 1, mdsys.sdo_geometry(:5, :6, null,:7, :8)"
", 1, 0, 1";
if (bUsePartition)
oss << ", :9";
oss <<")";
// TODO: If gotdata == false below, this memory probably leaks --mloskot
OCILobLocator** locator =(OCILobLocator**) VSIMalloc(sizeof(OCILobLocator*) * 1);
Statement statement = Statement(m_connection->CreateStatement(oss.str().c_str()));
long* p_pc_id = (long*) malloc(1 * sizeof(long));
p_pc_id[0] = m_pc_id;
long* p_result_id = (long*) malloc(1 * sizeof(long));
p_result_id[0] = (long)block_id;
long* p_num_points = (long*) malloc(1 * sizeof(long));
p_num_points[0] = (long)buffer.getNumPoints();
// std::cout << "point count on write: " << buffer.getNumPoints() << std::endl;
// :1
statement->Bind(&m_pc_id);
// :2
statement->Bind(p_result_id);
// :3
statement->Bind(p_num_points);
// :4
statement->Define(locator, 1);
// std::vector<liblas::uint8_t> data;
// bool gotdata = GetResultData(result, reader, data, 3);
// if (! gotdata) throw std::runtime_error("unable to fetch point data byte array");
// boost::uint8_t* point_data = buffer.getData(0);
boost::uint8_t* point_data;
boost::uint32_t point_data_length;
boost::uint32_t schema_byte_size;
bool pack = getOptions().getValueOrDefault<bool>("pack_ignored_fields", true);
if (pack)
PackPointData(buffer, &point_data, point_data_length, schema_byte_size);
else
{
point_data = buffer.getData(0);
point_data_length = buffer.getSchema().getByteSize() * buffer.getNumPoints();
}
// statement->Bind((char*)point_data,(long)(buffer.getSchema().getByteSize()*buffer.getNumPoints()));
statement->Bind((char*)point_data,(long)(point_data_length));
// :5
long* p_gtype = (long*) malloc(1 * sizeof(long));
p_gtype[0] = m_gtype;
statement->Bind(p_gtype);
// :6
long* p_srid = 0;
if (m_srid != 0)
{
p_srid = (long*) malloc(1 * sizeof(long));
p_srid[0] = m_srid;
}
statement->Bind(p_srid);
// :7
OCIArray* sdo_elem_info=0;
m_connection->CreateType(&sdo_elem_info, m_connection->GetElemInfoType());
SetElements(statement, sdo_elem_info);
statement->Bind(&sdo_elem_info, m_connection->GetElemInfoType());
// :8
OCIArray* sdo_ordinates=0;
m_connection->CreateType(&sdo_ordinates, m_connection->GetOrdinateType());
// x0, x1, y0, y1, z0, z1, bUse3d
pdal::Bounds<double> bounds = CalculateBounds(buffer);
SetOrdinates(statement, sdo_ordinates, bounds);
statement->Bind(&sdo_ordinates, m_connection->GetOrdinateType());
// :9
long* p_partition_d = 0;
if (bUsePartition)
{
p_partition_d = (long*) malloc(1 * sizeof(long));
p_partition_d[0] = m_block_table_partition_value;
statement->Bind(p_partition_d);
}
try
{
statement->Execute();
}
catch (std::runtime_error const& e)
{
std::ostringstream oss;
oss << "Failed to insert block # into '" << m_block_table_name << "' table. Does the table exist? " << std::endl << e.what() << std::endl;
throw std::runtime_error(oss.str());
}
oss.str("");
OWStatement::Free(locator, 1);
if (p_pc_id != 0) free(p_pc_id);
if (p_result_id != 0) free(p_result_id);
if (p_num_points != 0) free(p_num_points);
if (p_gtype != 0) free(p_gtype);
if (p_srid != 0) free(p_srid);
if (p_partition_d != 0) free(p_partition_d);
m_connection->DestroyType(&sdo_elem_info);
m_connection->DestroyType(&sdo_ordinates);
return true;
}
bool Writer::WriteBlock(PointBuffer const& buffer)
{
boost::uint8_t* point_data;
boost::uint32_t point_data_length;
boost::uint32_t schema_byte_size;
bool pack = getOptions().getValueOrDefault<bool>("pack_ignored_fields", true);
if (pack)
PackPointData(buffer, &point_data, point_data_length, schema_byte_size);
else
{
point_data = buffer.getData(0);
point_data_length = buffer.getSchema().getByteSize() * buffer.getNumPoints();
}
bool doKDTree = getOptions().getValueOrDefault<bool>("kdtreeify", true);
if (doKDTree)
{
// kdtreeify(buffer, &point_data, point_data_length, schema_byte_size);
}
std::string block_table = getOptions().getValueOrThrow<std::string>("block_table");
// // Pluck the block id out of the first point in the buffer
pdal::Schema const& schema = buffer.getSchema();
Dimension const& blockDim = schema.getDimension("BlockID");
m_block_id = buffer.getField<boost::int32_t>(blockDim, 0);
m_obj_id = getOptions().getValueOrThrow<boost::int32_t>("pc_id");
m_num_points = static_cast<boost::int64_t>(buffer.getNumPoints());
// if (m_type == DATABASE_POSTGRESQL)
// {
// std::string cloud_column = getOptions().getValueOrDefault<std::string>("cloud_column", "id");
// bool is3d = getOptions().getValueOrDefault<bool>("is3d", false);
//
//
// std::vector<boost::uint8_t> block_data;
// for (boost::uint32_t i = 0; i < point_data_length; ++i)
// {
// block_data.push_back(point_data[i]);
// }
//
// if (pack)
// delete point_data;
// m_block_bytes.str("");
// Utils::binary_to_hex_stream(point_data, m_block_bytes, 0, point_data_length);
// m_block_data = m_block_bytes.str();
// //std::cout << "hex: " << hex.substr(0, 30) << std::endl;
// m_srid = getOptions().getValueOrDefault<boost::uint32_t>("srid", 4326);
//
// boost::uint32_t precision(9);
// pdal::Bounds<double> bounds = buffer.calculateBounds(3);
// // m_extent.str("");
// m_extent = bounds.toWKT(precision); // polygons are only 2d, not cubes
// // m_bbox.str("");
// m_bbox = bounds.toBox(precision, 3);
// log()->get(logDEBUG) << "extent: " << m_extent << std::endl;
// log()->get(logDEBUG) << "bbox: " << m_bbox << std::endl;
//
// if (!m_block_statement)
// {
// // m_block_statement = (m_session->prepare << m_block_insert_query.str(), \
// // ::soci::use(m_obj_id, "obj_id"), \
// // ::soci::use(m_block_id, "block_id"), \
// // ::soci::use(m_num_points, "num_points"), \
// // ::soci::use(m_block_bytes.str(),"hex"), \
// // ::soci::use(m_extent.str(), "extent"), \
// // ::soci::use(m_srid, "srid"), \
// // ::soci::use(m_bbox.str(), "bbox"));
// m_block_statement = new ::soci::statement(*m_session);
//
// m_block_statement->exchange(::soci::use(m_obj_id, "obj_id"));
// m_block_statement->exchange(::soci::use(m_block_id, "block_id"));
// m_block_statement->exchange(::soci::use(m_num_points, "num_points"));
// m_block_statement->exchange(::soci::use(m_block_data,"hex"));
// m_block_statement->exchange(::soci::use(m_extent, "extent"));
// m_block_statement->exchange(::soci::use(m_srid, "srid"));
// m_block_statement->exchange(::soci::use(m_bbox, "bbox"));
// m_block_statement->alloc();
// m_block_statement->prepare(m_block_insert_query.str());
// m_block_statement->define_and_bind();
//
// }
// // ::soci::statement st = (m_session->prepare << m_block_insert_query.str(), \
// // ::soci::use(m_obj_id, "obj_id"), \
// // ::soci::use(m_block_id, "block_id"), \
// // ::soci::use(m_num_points, "num_points"), \
// // ::soci::use(m_block_bytes.str(),"hex"), \
// // ::soci::use(m_extent.str(), "extent"), \
// // ::soci::use(m_srid, "srid"), \
// // ::soci::use(m_bbox.str(), "bbox"));
// try
// {
// m_block_statement->execute(true);
// }
// catch (std::exception const& e)
// {
// std::ostringstream oss;
// oss << "Insert query failed with error '" << e.what() << "'";
// m_session->rollback();
// throw pdal_error(oss.str());
// }
//
// }
return true;
}
void Writer::PackPointData(PointBuffer const& buffer,
boost::uint8_t** point_data,
boost::uint32_t& point_data_len,
boost::uint32_t& schema_byte_size)
{
// Creates a new buffer that has the ignored dimensions removed from
// it.
schema::index_by_index const& idx = buffer.getSchema().getDimensions().get<schema::index>();
schema_byte_size = 0;
schema::index_by_index::size_type i(0);
for (i = 0; i < idx.size(); ++i)
{
if (! idx[i].isIgnored())
schema_byte_size = schema_byte_size+idx[i].getByteSize();
}
log()->get(logDEBUG) << "Packed schema byte size " << schema_byte_size;
point_data_len = buffer.getNumPoints() * schema_byte_size;
*point_data = new boost::uint8_t[point_data_len];
boost::uint8_t* current_position = *point_data;
for (boost::uint32_t i = 0; i < buffer.getNumPoints(); ++i)
{
boost::uint8_t* data = buffer.getData(i);
for (boost::uint32_t d = 0; d < idx.size(); ++d)
{
if (! idx[d].isIgnored())
{
memcpy(current_position, data, idx[d].getByteSize());
current_position = current_position+idx[d].getByteSize();
}
data = data + idx[d].getByteSize();
}
}
// Create a vector of two element vectors that states how long of a byte
// run to copy for the point based on whether or not the ignored/used
// flag of the dimension switches. This is to a) eliminate panning through
// the dimension multi_index repeatedly per point, and to b) eliminate
// tons of small memcpy in exchange for larger ones.
// std::vector< std::vector< boost::uint32_t> > runs;
// // bool switched = idx[0].isIgnored(); // start at with the first dimension
// for (boost::uint32_t d = 0; d < idx.size(); ++d)
// {
// std::vector<boost::uint32_t> t;
// if ( idx[d].isIgnored())
// {
// t.push_back(0);
// t.push_back(idx[d].getByteSize());
// }
// else
// {
// t.push_back(1);
// t.push_back(idx[d].getByteSize());
// }
//
// runs.push_back(t);
// }
//
// for (boost::uint32_t i = 0; i < buffer.getNumPoints(); ++i)
// {
// boost::uint8_t* data = buffer.getData(i);
// for (std::vector< std::vector<boost::uint32_t> >::size_type d=0; d < runs.size(); d++)
// // for (boost::uint32_t d = 0; d < idx.size(); ++d)
// {
// boost::uint32_t isUsed = runs[d][0];
// boost::uint32_t byteSize = runs[d][1];
// if (isUsed != 0)
// {
// memcpy(current_position, data, byteSize);
// current_position = current_position+byteSize;
// }
// data = data + byteSize;
//
// }
// }
}
boost::uint32_t IteratorBase::myReadBlocks(PointBuffer& user_buffer)
{
boost::uint32_t numPointsRead = 0;
std::string const& query = getReader().getOptions().getValueOrThrow<std::string>("query");
::soci::row block;
::soci::indicator ind = ::soci::i_null;
::soci::statement blocks = (m_session->prepare << query, ::soci::into(block, ind));
blocks.execute();
bool bDidRead = blocks.fetch();
// if (ind == ::soci::i_null)
// {
// // We have no points to return
// getReader().log()->get(logDEBUG) << "Query returned no points" << std::endl;
// return 0;
// }
//
// size_t size = block.size();
// for (size_t i = 0; i < size; ++i)
// {
// getReader().log()->get(logDEBUG3) << "column: " << block.get_properties(i).get_name() << std::endl;
// }
if (!m_active_buffer)
{
m_active_buffer = fetchPointBuffer(block.get<int>("cloud_id"),
block.get<std::string>("schema"),
user_buffer.getCapacity());
m_active_cloud_id = block.get<int>("cloud_id");
}
//
// This shouldn't ever happen
int num_points = block.get<int>("num_points");
if (num_points > static_cast<boost::int32_t>(m_active_buffer->getCapacity()))
{
std::ostringstream oss;
oss << "Block size, " << num_points <<", is too large to fit in "
<< "buffer of size " << user_buffer.getCapacity() <<". Increase buffer capacity with writer's \"chunk_size\" option "
<< "or increase the read buffer size";
throw buffer_too_small(oss.str());
}
while (bDidRead)
{
boost::uint32_t numReadThisBlock = static_cast<boost::uint32_t>(block.get<int>("num_points"));
boost::uint32_t numSpaceLeftThisBuffer = user_buffer.getCapacity() - user_buffer.getNumPoints();
getReader().log()->get(logDEBUG4) << "IteratorBase::myReadBlocks:" "numReadThisBlock: "
<< numReadThisBlock << " numSpaceLeftThisBlock: "
<< numSpaceLeftThisBuffer << " total numPointsRead: "
<< numPointsRead << std::endl;
numPointsRead = numPointsRead + numReadThisBlock;
readBlob(block, (std::min)(numReadThisBlock, numSpaceLeftThisBuffer));
fillUserBuffer(user_buffer);
bDidRead = blocks.fetch();
// if (!bDidRead)
// return user_buffer.getNumPoints();
boost::int32_t const& current_cloud_id = block.get<int>("cloud_id");
if (current_cloud_id != m_active_cloud_id)
{
getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: current_cloud_id: "
<< current_cloud_id << " m_active_cloud_id: "
<< m_active_cloud_id << std::endl;
m_active_buffer = fetchPointBuffer(current_cloud_id, block.get<std::string>("schema"), user_buffer.getCapacity());
m_active_cloud_id = current_cloud_id;
return user_buffer.getNumPoints();
}
}
return numPointsRead;
}
void Diff::checkPoints( StageSequentialIterator* source_iter,
PointBuffer& source_data,
StageSequentialIterator* candidate_iter,
PointBuffer& candidate_data,
ptree& errors)
{
boost::uint32_t i(0);
boost::uint32_t chunk(0);
boost::uint32_t MAX_BADBYTES(20);
boost::uint32_t badbytes(0);
while (!source_iter->atEnd())
{
const boost::uint32_t numSrcRead = source_iter->read(source_data);
const boost::uint32_t numCandidateRead = candidate_iter->read(candidate_data);
if (numSrcRead != numCandidateRead)
{
std::ostringstream oss;
oss << "Unable to read same number of points for chunk number";
errors.put<std::string>("points.error", oss.str());
errors.put<boost::uint32_t>("points.candidate" , numCandidateRead);
errors.put<boost::uint32_t>("points.source" , numSrcRead);
}
chunk++;
pdal::pointbuffer::PointBufferByteSize source_byte_length(0);
pdal::pointbuffer::PointBufferByteSize candidate_byte_length(0);
source_byte_length = static_cast<pdal::pointbuffer::PointBufferByteSize>(source_data.getSchema().getByteSize()) *
static_cast<pdal::pointbuffer::PointBufferByteSize>(source_data.getNumPoints());
candidate_byte_length = static_cast<pdal::pointbuffer::PointBufferByteSize>(candidate_data.getSchema().getByteSize()) *
static_cast<pdal::pointbuffer::PointBufferByteSize>(candidate_data.getNumPoints());
if (source_byte_length != candidate_byte_length)
{
std::ostringstream oss;
oss << "Source byte length != candidate byte length";
errors.put<std::string>("buffer.error", oss.str());
errors.put<boost::uint32_t>("buffer.candidate" , candidate_byte_length);
errors.put<boost::uint32_t>("buffer.source" , source_byte_length);
}
boost::uint8_t* s = source_data.getData(0);
boost::uint8_t* c = candidate_data.getData(0);
for (boost::uint32_t p = 0; p < std::min(source_byte_length, candidate_byte_length); ++p)
{
if (s[p] != c[p])
{
std::ostringstream oss;
oss << "Byte number " << p << " is not equal for source and candidate";
errors.put<std::string>("data.error", oss.str());
badbytes++;
}
}
if (badbytes > MAX_BADBYTES )
break;
}
}
// append all points from src buffer to end of dst buffer, based on the our bounds
boost::uint32_t Crop::processBuffer(PointBuffer& dstData, const PointBuffer& srcData) const
{
const Schema& schema = dstData.getSchema();
const Bounds<double>& bounds = this->getBounds();
boost::uint32_t numSrcPoints = srcData.getNumPoints();
boost::uint32_t dstIndex = dstData.getNumPoints();
boost::uint32_t numPointsAdded = 0;
boost::optional<Dimension const&> dimX = schema.getDimension("X");
boost::optional<Dimension const&> dimY = schema.getDimension("Y");
boost::optional<Dimension const&> dimZ = schema.getDimension("Z");
for (boost::uint32_t srcIndex=0; srcIndex<numSrcPoints; srcIndex++)
{
// need to scale the values
double x(0.0);
double y(0.0);
double z(0.0);
if (dimX->getInterpretation() == dimension::SignedInteger )
{
boost::int32_t xi = srcData.getField<boost::int32_t>(*dimX, srcIndex);
boost::int32_t yi = srcData.getField<boost::int32_t>(*dimY, srcIndex);
boost::int32_t zi = srcData.getField<boost::int32_t>(*dimZ, srcIndex);
x = dimX->applyScaling(xi);
y = dimY->applyScaling(yi);
z = dimZ->applyScaling(zi);
}
else if (dimX->getInterpretation() == dimension::UnsignedInteger)
{
boost::uint32_t xi = srcData.getField<boost::uint32_t>(*dimX, srcIndex);
boost::uint32_t yi = srcData.getField<boost::uint32_t>(*dimY, srcIndex);
boost::uint32_t zi = srcData.getField<boost::uint32_t>(*dimZ, srcIndex);
x = dimX->applyScaling(xi);
y = dimY->applyScaling(yi);
z = dimZ->applyScaling(zi);
} else
{
x = srcData.getField<double>(*dimX, srcIndex);
y = srcData.getField<double>(*dimY, srcIndex);
z = srcData.getField<double>(*dimZ, srcIndex);
}
Vector<double> point(x,y,z);
if (bounds.contains(point))
{
dstData.copyPointFast(dstIndex, srcIndex, srcData);
dstData.setNumPoints(dstIndex+1);
++dstIndex;
++numPointsAdded;
}
}
assert(dstIndex <= dstData.getCapacity());
return numPointsAdded;
}
void IteratorBase::fillUserBuffer(PointBuffer& user_buffer)
{
Schema const& user_schema = user_buffer.getSchema();
schema::index_by_index const& idx = user_schema.getDimensions().get<schema::index>();
boost::int32_t numOraclePoints = m_oracle_buffer->getNumPoints() - m_buffer_position;
boost::int32_t numUserSpace = user_buffer.getCapacity() - user_buffer.getNumPoints();
boost::int32_t howManyThisRead = (std::min)(numUserSpace, numOraclePoints);
if (numUserSpace < 0)
{
std::ostringstream oss;
oss << "numUserSpace < 0! : " << numUserSpace;
throw pdal_error(oss.str());
}
if (numOraclePoints < 0)
{
std::ostringstream oss;
oss << "numOraclePoints < 0! : " << numOraclePoints;
throw pdal_error(oss.str());
}
if (user_buffer.getNumPoints() + howManyThisRead > user_buffer.getCapacity())
{
std::ostringstream oss;
oss << "user_buffer.getNumPoints() + howManyThisRead == " << user_buffer.getNumPoints() + howManyThisRead
<< " but user_buffer.getCapacity() == " << user_buffer.getCapacity();
throw pdal_error(oss.str());
}
PointBuffer::copyLikeDimensions(*m_oracle_buffer, user_buffer,
*m_dimension_map,
m_buffer_position, user_buffer.getNumPoints(),
howManyThisRead);
Schema const& src = m_oracle_buffer->getSchema();
Dimension const& src_x = src.getDimension("drivers.oci.reader.X");
Dimension const& src_y = src.getDimension("drivers.oci.reader.Y");
Dimension const& src_z = src.getDimension("drivers.oci.reader.Z");
Schema const& dst = user_buffer.getSchema();
Dimension const& dst_x = dst.getDimension("drivers.oci.reader.X");
Dimension const& dst_y = dst.getDimension("drivers.oci.reader.Y");
Dimension const& dst_z = dst.getDimension("drivers.oci.reader.Z");
bool bDifferentScales = (!pdal::Utils::compare_distance(dst_x.getNumericScale(), src_x.getNumericScale()) ||
!pdal::Utils::compare_distance(dst_y.getNumericScale(), src_y.getNumericScale()) ||
!pdal::Utils::compare_distance(dst_z.getNumericScale(), src_z.getNumericScale()));
bool bDifferentOffsets = (!pdal::Utils::compare_distance(dst_x.getNumericOffset(), src_x.getNumericOffset()) ||
!pdal::Utils::compare_distance(dst_y.getNumericOffset(), src_y.getNumericOffset()) ||
!pdal::Utils::compare_distance(dst_z.getNumericOffset(), src_z.getNumericOffset()));
bool bNormalizeXYZ = getReader().getOptions().getValueOrDefault<bool>("do_normalize_xyz", true);
if ((bDifferentScales || bDifferentOffsets) && bNormalizeXYZ)
{
for (unsigned i = m_buffer_position; i < howManyThisRead; ++i)
{
double x = m_oracle_buffer->applyScaling(src_x, i);
boost::int32_t xi = dst_x.removeScaling<boost::int32_t>(x);
user_buffer.setField<boost::int32_t>(dst_x, user_buffer.getNumPoints()+i, xi);
double y = m_oracle_buffer->applyScaling(src_y, i);
boost::int32_t yi = dst_y.removeScaling<boost::int32_t>(y);
user_buffer.setField<boost::int32_t>(dst_y, user_buffer.getNumPoints()+i, yi);
double z = m_oracle_buffer->applyScaling(src_z, i);
boost::int32_t zi = dst_z.removeScaling<boost::int32_t>(z);
user_buffer.setField<boost::int32_t>(dst_z, user_buffer.getNumPoints()+i, zi);
}
}
getReader().log()->get(logDEBUG2) << "IteratorBase::fillUserBuffer m_buffer_position: "
<< m_buffer_position << std::endl;
if (numOraclePoints > numUserSpace)
m_buffer_position = m_buffer_position + numUserSpace;
else if (numOraclePoints < numUserSpace)
m_buffer_position = 0;
getReader().log()->get(logDEBUG2) << "IteratorBase::fillUserBuffer m_buffer_position: "
<< m_buffer_position << std::endl;
bool bSetPointSourceId = getReader().getOptions().getValueOrDefault<bool>("populate_pointsourceid", false);
if (bSetPointSourceId)
{
Dimension const* point_source_field = user_buffer.getSchema().getDimensionPtr("PointSourceId");
if (point_source_field)
{
for (boost::int32_t i = m_buffer_position; i < howManyThisRead; ++i)
{
assert(user_buffer.getNumPoints() + i < user_buffer.getCapacity());
if (point_source_field->getByteSize() == 2)
user_buffer.setField(*point_source_field,
user_buffer.getNumPoints() + i,
static_cast<boost::uint16_t>(m_active_cloud_id));
else if (point_source_field->getByteSize() == 4)
{
user_buffer.setField(*point_source_field,
user_buffer.getNumPoints() + i,
m_active_cloud_id);
}
}
}
}
user_buffer.setNumPoints(howManyThisRead + user_buffer.getNumPoints());
}
void Delta::outputDetail(PointBuffer& source_data,
IndexedPointBuffer& candidate_data,
std::map<Point, Point> *points) const
{
Schema const& candidate_schema = candidate_data.getSchema();
Dimension const& cDimX = candidate_schema.getDimension("X");
Dimension const& cDimY = candidate_schema.getDimension("Y");
Dimension const& cDimZ = candidate_schema.getDimension("Z");
Schema const& source_schema = source_data.getSchema();
Dimension const& sDimX = source_schema.getDimension("X");
Dimension const& sDimY = source_schema.getDimension("Y");
Dimension const& sDimZ = source_schema.getDimension("Z");
bool bWroteHeader(false);
std::ostream& ostr = m_outputStream ? *m_outputStream : std::cout;
candidate_data.build(m_3d);
boost::uint32_t count(std::min(source_data.getNumPoints(), candidate_data.getNumPoints()));
for (boost::uint32_t i = 0; i < count; ++i)
{
double sx = source_data.applyScaling(sDimX, i);
double sy = source_data.applyScaling(sDimY, i);
double sz = source_data.applyScaling(sDimZ, i);
std::vector<std::size_t> ids = candidate_data.neighbors(sx, sy, sz, 1);
if (!ids.size())
{
std::ostringstream oss;
oss << "unable to find point for id '" << i <<"'";
throw app_runtime_error(oss.str() );
}
std::size_t id = ids[0];
double cx = candidate_data.applyScaling(cDimX, id);
double cy = candidate_data.applyScaling(cDimY, id);
double cz = candidate_data.applyScaling(cDimZ, id);
Point s(sx, sy, sz, id);
Point c(cx, cy, cz, id);
double xd = sx - cx;
double yd = sy - cy;
double zd = sz - cz;
if (!bWroteHeader)
{
writeHeader(ostr, m_3d);
bWroteHeader = true;
}
ostr << i << ",";
boost::uint32_t precision = Utils::getStreamPrecision(cDimX.getNumericScale());
ostr.setf(std::ios_base::fixed, std::ios_base::floatfield);
ostr.precision(precision);
ostr << xd << ",";
precision = Utils::getStreamPrecision(cDimY.getNumericScale());
ostr.precision(precision);
ostr << yd;
if (m_3d)
{
ostr << ",";
precision = Utils::getStreamPrecision(cDimZ.getNumericScale());
ostr.precision(precision);
ostr << zd;
}
ostr << std::endl;
}
if (m_outputStream)
{
FileUtils::closeFile(m_outputStream);
}
}
boost::uint32_t IteratorBase::myReadBlocks(PointBuffer& user_buffer)
{
boost::uint32_t numPointsRead = 0;
user_buffer.setNumPoints(0);
bool bDidRead = false;
if (!m_oracle_buffer)
{
m_oracle_buffer = fetchPointBuffer(m_initialQueryStatement, m_block->pc);
if (!m_oracle_buffer) throw pdal_error("m_oracle_buffer was NULL!");
m_dimension_map = fetchDimensionMap(m_initialQueryStatement, m_block->pc, *m_oracle_buffer, user_buffer);
boost::int32_t current_cloud_id(0);
current_cloud_id = m_initialQueryStatement->GetInteger(&m_block->pc->pc_id);
m_active_cloud_id = current_cloud_id;
}
// This shouldn't ever happen
if (m_block->num_points > static_cast<boost::int32_t>(m_oracle_buffer->getCapacity()))
{
m_oracle_buffer->resize(m_block->num_points);
}
if (!m_block->num_points)
{
// We still have a block of data from the last readBuffer call
// that was partially read.
getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: fetching first block" << std::endl;
bDidRead = m_initialQueryStatement->Fetch();
if (!bDidRead)
{
m_at_end = true;
return 0;
}
user_buffer.setSpatialBounds(getBounds(m_initialQueryStatement, m_block));
}
else
{
// Our read was already "done" last readBuffer call, but if we're done,
// we're done
if (m_at_end)
getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: we are at end of the blocks;" << std::endl;
else
getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: we have points left to read on this block" << std::endl;
if (m_at_end) return 0;
bDidRead = true;
}
while (bDidRead)
{
boost::uint32_t numReadThisBlock = m_block->num_points;
boost::uint32_t numSpaceLeftThisBuffer = user_buffer.getCapacity() - user_buffer.getNumPoints();
getReader().log()->get(logDEBUG4) << "IteratorBase::myReadBlocks:" "numReadThisBlock: "
<< numReadThisBlock << " numSpaceLeftThisBlock: "
<< numSpaceLeftThisBuffer << " total numPointsRead: "
<< numPointsRead << std::endl;
numPointsRead = numPointsRead + numReadThisBlock;
readBlob(m_initialQueryStatement, m_block, m_block->num_points);
fillUserBuffer(user_buffer);
if (m_buffer_position != 0)
{
return user_buffer.getNumPoints();
}
else
{
bDidRead = m_initialQueryStatement->Fetch();
if (!bDidRead)
{
getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: done reading block. Read " << numPointsRead << " points" << std::endl;
m_at_end = true;
return user_buffer.getNumPoints();
}
}
boost::int32_t current_cloud_id(0);
current_cloud_id = m_initialQueryStatement->GetInteger(&m_block->pc->pc_id);
getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: current_cloud_id: "
<< current_cloud_id << " m_active_cloud_id: "
<< m_active_cloud_id << std::endl;
if (current_cloud_id != m_active_cloud_id)
{
m_oracle_buffer = fetchPointBuffer(m_initialQueryStatement, m_block->pc);
if (!m_oracle_buffer) throw pdal_error("m_oracle_buffer was NULL!");
m_dimension_map = fetchDimensionMap(m_initialQueryStatement, m_block->pc, *m_oracle_buffer, user_buffer);
m_active_cloud_id = current_cloud_id;
return user_buffer.getNumPoints();
}
}
return numPointsRead;
}
