void PooledPointTable::reset()
{
BinaryPointTable table(m_schema);
pdal::PointRef pointRef(table, 0);
assert(m_cellNodes.size() >= outstanding());
Cell::PooledStack cells(m_cellNodes.pop(outstanding()));
for (auto& cell : cells)
{
auto data(m_dataNodes.popOne());
table.setPoint(*data);
if (m_origin != invalidOrigin)
{
pointRef.setField(pdal::Dimension::Id::PointId, m_index);
pointRef.setField(pdal::Dimension::Id::OriginId, m_origin);
++m_index;
}
cell.set(pointRef, std::move(data));
}
cells = m_process(std::move(cells));
for (auto& cell : cells) m_dataNodes.push(cell.acquire());
m_cellNodes.push(std::move(cells));
allocate();
}
virtual Cell::PooledStack read(
const arbiter::Endpoint& endpoint,
PointPool& pool,
const Id& id) const override
{
auto data(io::ensureGet(endpoint, m_metadata.basename(id)));
const Tail tail(*data, m_tailFields);
const char* pos(data->data());
const Schema& schema(pool.schema());
const std::size_t pointSize(schema.pointSize());
const std::size_t numPoints(data->size() / pointSize);
const std::size_t numBytes(data->size() + tail.size());
BinaryPointTable table(schema);
pdal::PointRef pointRef(table, 0);
if (pointSize * numPoints != data->size())
{
throw std::runtime_error("Invalid binary chunk size");
}
if (tail.numPoints() && tail.numPoints() != numPoints)
{
throw std::runtime_error("Invalid binary chunk numPoints");
}
if (tail.numBytes() && tail.numBytes() != numBytes)
{
throw std::runtime_error("Invalid binary chunk numBytes");
}
Data::PooledStack dataStack(pool.dataPool().acquire(numPoints));
Cell::PooledStack cellStack(pool.cellPool().acquire(numPoints));
for (Cell& cell : cellStack)
{
table.setPoint(pos);
Data::PooledNode dataNode(dataStack.popOne());
std::copy(pos, pos + pointSize, *dataNode);
cell.set(pointRef, std::move(dataNode));
pos += pointSize;
}
assert(dataStack.empty());
return cellStack;
}
Cell::PooledStack LazPerfStorage::read(
const arbiter::Endpoint& endpoint,
PointPool& pool,
const Id& id) const
{
auto compressed(io::ensureGet(endpoint, m_metadata.basename(id)));
const Tail tail(*compressed, m_tailFields);
const Schema& schema(pool.schema());
const std::size_t pointSize(schema.pointSize());
const std::size_t numPoints(tail.numPoints());
const std::size_t numBytes(compressed->size() + tail.size());
BinaryPointTable table(schema);
pdal::PointRef pointRef(table, 0);
if (id >= m_metadata.structure().coldIndexBegin() && !numPoints)
{
throw std::runtime_error("Invalid lazperf chunk - no numPoints");
}
if (tail.numBytes() && tail.numBytes() != numBytes)
{
std::cout << tail.numBytes() << " != " << numBytes << std::endl;
throw std::runtime_error("Invalid lazperf chunk numBytes");
}
Data::PooledStack dataStack(pool.dataPool().acquire(numPoints));
Cell::PooledStack cellStack(pool.cellPool().acquire(numPoints));
DecompressionStream stream(*compressed);
pdal::LazPerfDecompressor<DecompressionStream> decompressor(
stream,
schema.pdalLayout().dimTypes());
for (Cell& cell : cellStack)
{
Data::PooledNode dataNode(dataStack.popOne());
table.setPoint(*dataNode);
decompressor.decompress(*dataNode, pointSize);
cell.set(pointRef, std::move(dataNode));
}
assert(dataStack.empty());
return cellStack;
}
CelledBaseChunkReader::CelledBaseChunkReader(
const Metadata& m,
PointPool& pool,
const arbiter::Endpoint& endpoint)
: BaseChunkReader(m, pool)
{
DimList dims;
dims.push_back(DimInfo("TubeId", "unsigned", 8));
dims.insert(dims.end(), m.schema().dims().begin(), m.schema().dims().end());
const Schema celledSchema(dims);
PointPool celledPool(celledSchema, m.delta());
auto tubedCells(m.storage().deserialize(endpoint, celledPool, m_id));
Data::PooledStack tubedData(celledPool.dataPool());
auto dataNodes(m_pool.dataPool().acquire(tubedCells.size()));
m_cells = m_pool.cellPool().acquire(tubedCells.size());
const std::size_t celledPointSize(celledSchema.pointSize());
const std::size_t tubeIdSize(sizeof(uint64_t));
uint64_t tube(0);
char* tPos(reinterpret_cast<char*>(&tube));
BinaryPointTable table(m.schema());
pdal::PointRef pointRef(table, 0);
for (auto& cell : m_cells)
{
auto tubedCell(tubedCells.popOne());
const char* src(tubedCell->uniqueData());
Data::PooledNode data(dataNodes.popOne());
std::copy(src, src + tubeIdSize, tPos);
std::copy(src + tubeIdSize, src + celledPointSize, *data);
table.setPoint(*data);
cell.set(pointRef, std::move(data));
m_points.at(tube).emplace_back(cell.point(), cell.uniqueData());
tubedData.push(tubedCell->acquire());
}
}
std::unique_ptr<std::vector<char>> Compression::decompress(
const std::vector<char>& data,
const Schema& nativeSchema,
const Schema* const wantedSchema,
const std::size_t numPoints)
{
if (!wantedSchema || *wantedSchema == nativeSchema)
{
return decompress(data, nativeSchema, numPoints);
}
// Get decompressor in the native schema.
DecompressionStream decompressionStream(data);
pdal::LazPerfDecompressor<DecompressionStream> decompressor(
decompressionStream,
nativeSchema.pdalLayout().dimTypes());
// Allocate room for a single point in the native schema.
std::vector<char> nativePoint(nativeSchema.pointSize());
BinaryPointTable table(nativeSchema, nativePoint.data());
pdal::PointRef pointRef(table, 0);
// Get our result space, in the desired schema, ready.
std::unique_ptr<std::vector<char>> decompressed(
new std::vector<char>(numPoints * wantedSchema->pointSize(), 0));
char* pos(decompressed->data());
const char* end(pos + decompressed->size());
while (pos < end)
{
decompressor.decompress(nativePoint.data(), nativePoint.size());
for (const auto& d : wantedSchema->dims())
{
pointRef.getField(pos, d.id(), d.type());
pos += d.size();
}
}
return decompressed;
}
PooledInfoStack Compression::decompress(
const std::vector<char>& data,
const std::size_t numPoints,
PointPool& pointPool)
{
PooledDataStack dataStack(pointPool.dataPool().acquire(numPoints));
PooledInfoStack infoStack(pointPool.infoPool().acquire(numPoints));
BinaryPointTable table(pointPool.schema());
pdal::PointRef pointRef(table, 0);
const std::size_t pointSize(pointPool.schema().pointSize());
DecompressionStream decompressionStream(data);
pdal::LazPerfDecompressor<DecompressionStream> decompressor(
decompressionStream,
pointPool.schema().pdalLayout().dimTypes());
RawInfoNode* info(infoStack.head());
char* pos(nullptr);
while (info)
{
info->construct(dataStack.popOne());
pos = info->val().data();
decompressor.decompress(pos, pointSize);
table.setPoint(pos);
info->val().point(pointRef);
info = info->next();
}
return infoStack;
}
