point_count_t BpfReader::readDimMajor(PointViewPtr data, point_count_t count)
{
PointId idx(0);
PointId startId = data->size();
point_count_t numRead = 0;
for (size_t d = 0; d < m_dims.size(); ++d)
{
idx = m_index;
PointId nextId = startId;
numRead = 0;
seekDimMajor(d, idx);
for (; numRead < count && idx < numPoints(); idx++, numRead++, nextId++)
{
float f;
m_stream >> f;
data->setField(m_dims[d].m_id, nextId, f + m_dims[d].m_offset);
}
}
m_index = idx;
// Transformation only applies to X, Y and Z
for (PointId idx = startId; idx < data->size(); idx++)
{
double x = data->getFieldAs<double>(Dimension::Id::X, idx);
double y = data->getFieldAs<double>(Dimension::Id::Y, idx);
double z = data->getFieldAs<double>(Dimension::Id::Z, idx);
m_header.m_xform.apply(x, y, z);
data->setField(Dimension::Id::X, idx, x);
data->setField(Dimension::Id::Y, idx, y);
data->setField(Dimension::Id::Z, idx, z);
if (m_cb)
m_cb(*data, idx);
}
return numRead;
}
TEST(CropFilterTest, test_crop_polygon)
{
#ifdef PDAL_HAVE_GEOS
Options ops1;
ops1.add("filename", Support::datapath("las/1.2-with-color.las"));
LasReader reader;
reader.setOptions(ops1);
Options options;
Option debug("debug", true, "");
Option verbose("verbose", 9, "");
std::istream* wkt_stream =
FileUtils::openFile(Support::datapath("autzen/autzen-selection.wkt"));
std::stringstream strbuf;
strbuf << wkt_stream->rdbuf();
std::string wkt(strbuf.str());
Option polygon("polygon", wkt, "");
options.add(polygon);
CropFilter crop;
crop.setInput(reader);
crop.setOptions(options);
PointTable table;
crop.prepare(table);
PointViewSet viewSet = crop.execute(table);
EXPECT_EQ(viewSet.size(), 1u);
PointViewPtr view = *viewSet.begin();
EXPECT_EQ(view->size(), 47u);
FileUtils::closeFile(wkt_stream);
#endif
}
TEST(RangeFilterTest, nan)
{
LasReader reader;
Options options;
options.add("filename", Support::datapath("las/gps-time-nan.las"));
reader.setOptions(options);
Options rangeOptions;
rangeOptions.add("limits", "GpsTime[-1:1]");
RangeFilter filter;
filter.setOptions(rangeOptions);
filter.setInput(reader);
PointTable table;
filter.prepare(table);
PointViewSet viewSet = filter.execute(table);
PointViewPtr view = *viewSet.begin();
EXPECT_EQ(1u, viewSet.size());
EXPECT_EQ(0u, view->size());
}
void PcdWriter::write(const PointViewPtr view)
{
pcl::PointCloud<XYZIRGBA>::Ptr cloud(new pcl::PointCloud<XYZIRGBA>);
BOX3D buffer_bounds;
view->calculateBounds(buffer_bounds);
pclsupport::PDALtoPCD(view, *cloud, buffer_bounds);
pcl::PCDWriter w;
if (m_compressed)
w.writeBinaryCompressed<XYZIRGBA>(m_filename, *cloud);
else
w.writeASCII<XYZIRGBA>(m_filename, *cloud);
}
// Make sure that dimension names containing digits works
TEST(RangeFilterTest, case_1659)
{
TextReader reader;
Options ops;
ops.add("filename", Support::datapath("text/numeric_dim.txt"));
reader.setOptions(ops);
Options rangeOps;
rangeOps.add("limits", "Eigenvalue0[:35]");
RangeFilter filter;
filter.setOptions(rangeOps);
filter.setInput(reader);
PointTable table;
filter.prepare(table);
PointViewSet viewSet = filter.execute(table);
PointViewPtr view = *viewSet.begin();
EXPECT_EQ(1u, viewSet.size());
EXPECT_EQ(3u, view->size());
}
TEST(MergeTest, test3)
{
using namespace pdal;
PipelineManager mgr;
PipelineReader specReader(mgr);
specReader.readPipeline(Support::configuredpath("filters/merge3.xml"));
std::ostringstream oss;
std::ostream& o = std::clog;
auto ctx = Utils::redirect(o, oss);
mgr.execute();
std::string s = oss.str();
EXPECT_TRUE(s.find("inconsistent spatial references") != s.npos);
Utils::restore(o, ctx);
PointViewSet viewSet = mgr.views();
EXPECT_EQ(viewSet.size(), 1u);
PointViewPtr view = *viewSet.begin();
EXPECT_EQ(view->size(), 2130u);
}
TEST(OldPCLBlockTests, StatisticalOutliers2)
{
StageFactory f;
Options ro;
ro.add("filename", Support::datapath("autzen/autzen-point-format-3.las"));
Stage* r(f.createStage("readers.las"));
EXPECT_TRUE(r);
r->setOptions(ro);
Options fo;
fo.add("method", "statistical");
fo.add("multiplier", 0.0);
fo.add("mean_k", 5);
Stage* outlier(f.createStage("filters.outlier"));
EXPECT_TRUE(outlier);
outlier->setOptions(fo);
outlier->setInput(*r);
Options rangeOpts;
rangeOpts.add("limits", "Classification![7:7]");
Stage* range(f.createStage("filters.range"));
EXPECT_TRUE(range);
range->setOptions(rangeOpts);
range->setInput(*outlier);
PointTable table;
range->prepare(table);
PointViewSet viewSet = range->execute(table);
EXPECT_EQ(1u, viewSet.size());
PointViewPtr view = *viewSet.begin();
EXPECT_EQ(63u, view->size());
}
TEST(SbetReaderTest, testRead)
{
Option filename("filename", Support::datapath("sbet/2-points.sbet"), "");
Options options(filename);
std::shared_ptr<SbetReader> reader(new SbetReader);
reader->setOptions(options);
PointTable table;
reader->prepare(table);
PointViewSet viewSet = reader->execute(table);
EXPECT_EQ(viewSet.size(), 1u);
PointViewPtr view = *viewSet.begin();
EXPECT_EQ(view->size(), 2u);
checkPoint(*view.get(), 0,
1.516310028360710e+05, 5.680211852972264e-01,
-2.041654392303940e+00, 1.077152953296560e+02,
-2.332420866600025e+00, -3.335067504871401e-01,
-3.093961631767838e-02, -2.813407149321339e-02,
-2.429905393889139e-02, 3.046773230278662e+00,
-2.198414736922658e-02, 7.859639737752390e-01,
7.849084719295495e-01, -2.978807916450262e-01,
6.226807982589819e-05, 9.312162756440178e-03,
7.217812320996525e-02);
checkPoint(*view.get(), 1,
1.516310078318641e+05, 5.680211834722869e-01,
-2.041654392034053e+00, 1.077151424357507e+02,
-2.336228229691271e+00, -3.324663118952635e-01,
-3.022948961008987e-02, -2.813856631423094e-02,
-2.425215669392169e-02, 3.047131105236811e+00,
-2.198416007932108e-02, 8.397590491636475e-01,
3.252165276637165e-01, -1.558883225990844e-01,
8.379685112283802e-04, 7.372886784718076e-03,
7.179027672314571e-02);
}
void compareTextLas(const std::string& textFilename,
Options& textOptions, const std::string& lasFilename)
{
TextReader t;
textOptions.add("filename", textFilename);
t.setOptions(textOptions);
LasReader l;
Options lo;
lo.add("filename", lasFilename);
l.setOptions(lo);
PointTable tt;
t.prepare(tt);
PointViewSet ts = t.execute(tt);
EXPECT_EQ(ts.size(), 1U);
PointViewPtr tv = *ts.begin();
PointTable lt;
l.prepare(lt);
PointViewSet ls = l.execute(lt);
EXPECT_EQ(ls.size(), 1U);
PointViewPtr lv = *ls.begin();
EXPECT_EQ(tv->size(), lv->size());
// Validate some point data.
for (PointId i = 0; i < lv->size(); ++i)
{
EXPECT_DOUBLE_EQ(tv->getFieldAs<double>(Dimension::Id::X, i),
lv->getFieldAs<double>(Dimension::Id::X, i));
EXPECT_DOUBLE_EQ(tv->getFieldAs<double>(Dimension::Id::Y, i),
lv->getFieldAs<double>(Dimension::Id::Y, i));
EXPECT_DOUBLE_EQ(tv->getFieldAs<double>(Dimension::Id::Z, i),
lv->getFieldAs<double>(Dimension::Id::Z, i));
}
}
point_count_t NumpyReader::read(PointViewPtr view, point_count_t numToRead)
{
PointId idx = view->size();
point_count_t numRead(0);
while (numRead < numToRead)
{
PointRef point(*view, idx);
if (!processOne(point))
break;
numRead++;
idx++;
}
return numRead;
}
point_count_t TextReader::read(PointViewPtr view, point_count_t numPts)
{
PointId idx = view->size();
point_count_t cnt = 0;
PointRef point(*view, idx);
while (cnt < numPts)
{
point.setPointId(idx);
if (!processOne(point))
break;
cnt++;
idx++;
}
return cnt;
}
TEST(QFITReaderTest, test_14_word)
{
Options options;
options.add("filename", Support::datapath("qfit/14-word.qi"),
"Input filename for reader to use");
options.add("flip_coordinates", false,
"Flip coordinates from 0-360 to -180-180");
options.add("scale_z", 0.001f, "Z scale from mm to m");
options.add("count", 3);
PointTable table;
std::shared_ptr<QfitReader> reader(new QfitReader);
reader->setOptions(options);
reader->prepare(table);
PointViewSet viewSet = reader->execute(table);
EXPECT_EQ(viewSet.size(), 1u);
PointViewPtr view = *viewSet.begin();
EXPECT_EQ(view->size(), 3u);
Check_Point(*view, 0, 244.306337, 35.623317, 1056.830000000, 903);
Check_Point(*view, 1, 244.306260, 35.623280, 1056.409000000, 903);
Check_Point(*view, 2, 244.306204, 35.623257, 1056.483000000, 903);
}
void RialtoReader::doQuery(const TileMath& tmm,
const GpkgTile& tile,
PointViewPtr view,
double qMinX, double qMinY, double qMaxX, double qMaxY)
{
const uint32_t level = tile.getLevel();
const uint32_t column = tile.getColumn();
const uint32_t row = tile.getRow();
const uint32_t numPoints = tile.getNumPoints();
// if this tile is entirely inside the query box, then
// we won't need to check each point
double tileMinX, tileMinY, tileMaxX, tileMaxY;
tmm.getTileBounds(column, row, level,
tileMinX, tileMinY, tileMaxX, tileMaxY);
const bool tileEntirelyInsideQueryBox =
tmm.rectContainsRect(qMinX, qMinY, qMaxX, qMaxY,
tileMinX, tileMinY, tileMaxX, tileMaxY);
log()->get(LogLevel::Debug) << " intersecting tile "
<< "(" << level << "," << column << "," << row << ")"
<< " contains " << numPoints << " points" << std::endl;
PointViewPtr tempView = view->makeNew();
GpkgTile::exportToPV(numPoints, tempView, tile.getBlob());
for (uint32_t i=0; i<tempView->size(); i++) {
const double x = tempView->getFieldAs<double>(Dimension::Id::X, i);
const double y = tempView->getFieldAs<double>(Dimension::Id::Y, i);
if (tileEntirelyInsideQueryBox || (x >= qMinX && x <= qMaxX && y >= qMinY && y <= qMaxY))
{
view->appendPoint(*tempView, i);
}
}
}
PointViewSet SampleFilter::run(PointViewPtr inView)
{
point_count_t np = inView->size();
// Return empty PointViewSet if the input PointView has no points.
// Otherwise, make a new output PointView.
PointViewSet viewSet;
if (!np)
return viewSet;
PointViewPtr outView = inView->makeNew();
// Build the 3D KD-tree.
KD3Index index(*inView);
index.build();
// All points are marked as kept (1) by default. As they are masked by
// neighbors within the user-specified radius, their value is changed to 0.
std::vector<int> keep(np, 1);
// We are able to subsample in a single pass over the shuffled indices.
for (PointId i = 0; i < np; ++i)
{
// If a point is masked, it is forever masked, and cannot be part of the
// sampled cloud. Otherwise, the current index is appended to the output
// PointView.
if (keep[i] == 0)
continue;
outView->appendPoint(*inView, i);
// We now proceed to mask all neighbors within m_radius of the kept
// point.
auto ids = index.radius(i, m_radius);
for (auto const& id : ids)
keep[id] = 0;
}
// Simply calculate the percentage of retained points.
double frac = (double)outView->size() / (double)inView->size();
log()->get(LogLevel::Debug2)
<< "Retaining " << outView->size() << " of " << inView->size()
<< " points (" << 100 * frac << "%)\n";
viewSet.insert(outView);
return viewSet;
}
void HAGFilter::filter(PointView& view)
{
PointViewPtr gView = view.makeNew();
PointViewPtr ngView = view.makeNew();
std::vector<PointId> gIdx, ngIdx;
// First pass: Separate into ground and non-ground views.
for (PointId i = 0; i < view.size(); ++i)
{
double c = view.getFieldAs<double>(Dimension::Id::Classification, i);
if (c == 2)
{
gView->appendPoint(view, i);
gIdx.push_back(i);
}
else
{
ngView->appendPoint(view, i);
ngIdx.push_back(i);
}
}
// Bail if there weren't any points classified as ground.
if (gView->size() == 0)
throwError("Input PointView does not have any points classified "
"as ground");
// Build the 2D KD-tree.
KD2Index& kdi = gView->build2dIndex();
// Second pass: Find Z difference between non-ground points and the nearest
// neighbor (2D) in the ground view.
for (PointId i = 0; i < ngView->size(); ++i)
{
PointRef point = ngView->point(i);
double z0 = point.getFieldAs<double>(Dimension::Id::Z);
auto ids = kdi.neighbors(point, 1);
double z1 = gView->getFieldAs<double>(Dimension::Id::Z, ids[0]);
view.setField(Dimension::Id::HeightAboveGround, ngIdx[i], z0 - z1);
}
// Final pass: Ensure that all ground points have height value pegged at 0.
for (auto const& i : gIdx)
view.setField(Dimension::Id::HeightAboveGround, i, 0.0);
}
PointViewSet run(PointViewPtr view)
{
if (m_count > view->size())
log()->get(LogLevel::Warning)
<< "Requested number of points (count=" << m_count
<< ") exceeds number of available points.\n";
PointViewSet viewSet;
PointViewPtr outView = view->makeNew();
for (PointId i = view->size() - std::min(m_count, view->size());
i < view->size(); ++i)
outView->appendPoint(*view, i);
viewSet.insert(outView);
return viewSet;
}
void MyWriter::write(PointViewPtr view)
{
for (PointId idx = 0; idx < view->size(); ++idx)
{
double x = view->getFieldAs<double>(Dimension::Id::X, idx);
double y = view->getFieldAs<double>(Dimension::Id::Y, idx);
double z = view->getFieldAs<double>(Dimension::Id::Z, idx);
unsigned int myData = 0;
if (!m_datafield.empty()) {
myData = (int)(view->getFieldAs<double>(m_dataDim, idx) + 0.5);
}
*m_stream << x << ":" << y << ":" << z << ":"
<< myData << m_newline;
}
}
std::vector<double> PMFFilter::morphOpen(PointViewPtr view, float radius)
{
point_count_t np(view->size());
QuadIndex idx(*view);
std::vector<double> minZ(np), maxZ(np);
// erode
for (PointId i = 0; i < np; ++i)
{
double x = view->getFieldAs<double>(Dimension::Id::X, i);
double y = view->getFieldAs<double>(Dimension::Id::Y, i);
std::vector<PointId> ids = idx.getPoints(x-radius, y-radius, x+radius, y+radius);
double localMin(std::numeric_limits<double>::max());
for (auto const& j : ids)
{
double z = view->getFieldAs<double>(Dimension::Id::Z, j);
if (z < localMin)
localMin = z;
}
minZ[i] = localMin;
}
// dilate
for (PointId i = 0; i < np; ++i)
{
double x = view->getFieldAs<double>(Dimension::Id::X, i);
double y = view->getFieldAs<double>(Dimension::Id::Y, i);
std::vector<PointId> ids = idx.getPoints(x-radius, y-radius, x+radius, y+radius);
double localMax(std::numeric_limits<double>::lowest());
for (auto const& j : ids)
{
double z = minZ[j];
if (z > localMax)
localMax = z;
}
maxZ[i] = localMax;
}
return maxZ;
}
std::vector<double> PMFFilter::morphOpen(PointViewPtr view, float radius)
{
point_count_t np(view->size());
KD2Index index(*view);
index.build();
std::vector<double> minZ(np), maxZ(np);
typedef std::vector<PointId> PointIdVec;
std::map<PointId, PointIdVec> neighborMap;
// erode
for (PointId i = 0; i < np; ++i)
{
double x = view->getFieldAs<double>(Dimension::Id::X, i);
double y = view->getFieldAs<double>(Dimension::Id::Y, i);
auto ids = index.radius(x, y, radius);
// neighborMap.insert(std::pair<PointId, std::vector<PointId>(i, ids));
neighborMap[i] = ids;
double localMin(std::numeric_limits<double>::max());
for (auto const& j : ids)
{
double z = view->getFieldAs<double>(Dimension::Id::Z, j);
if (z < localMin)
localMin = z;
}
minZ[i] = localMin;
}
// dilate
for (PointId i = 0; i < np; ++i)
{
auto ids = neighborMap[i];
double localMax(std::numeric_limits<double>::lowest());
for (auto const& j : ids)
{
double z = minZ[j];
if (z > localMax)
localMax = z;
}
maxZ[i] = localMax;
}
return maxZ;
}
int DiffKernel::execute()
{
PointTable sourceTable;
Stage& source = makeReader(m_sourceFile, m_driverOverride);
source.prepare(sourceTable);
PointViewSet sourceSet = source.execute(sourceTable);
MetadataNode errors;
PointTable candidateTable;
Stage& candidate = makeReader(m_candidateFile, m_driverOverride);
candidate.prepare(candidateTable);
PointViewSet candidateSet = candidate.execute(candidateTable);
assert(sourceSet.size() == 1);
assert(candidateSet.size() == 1);
PointViewPtr sourceView = *sourceSet.begin();
PointViewPtr candidateView = *candidateSet.begin();
if (candidateView->size() != sourceView->size())
{
std::ostringstream oss;
oss << "Source and candidate files do not have the same point count";
errors.add("count.error", oss.str());
errors.add("count.candidate", candidateView->size());
errors.add("count.source", sourceView->size());
}
MetadataNode source_metadata = sourceTable.metadata();
MetadataNode candidate_metadata = candidateTable.metadata();
if (source_metadata != candidate_metadata)
{
std::ostringstream oss;
oss << "Source and candidate files do not have the same metadata count";
errors.add("metadata.error", oss.str());
errors.add(source_metadata);
errors.add(candidate_metadata);
}
if (candidateTable.layout()->dims().size() !=
sourceTable.layout()->dims().size())
{
std::ostringstream oss;
oss << "Source and candidate files do not have the same "
"number of dimensions";
}
return 0;
}
point_count_t Ilvis2Reader::read(PointViewPtr view, point_count_t count)
{
PointId idx = view->size();
point_count_t numRead = 0;
PointRef point = PointRef(*view, 0);
while (numRead < count)
{
point.setPointId(idx++);
if (!processOne(point))
break;
if (m_cb)
m_cb(*view, idx);
numRead++;
}
return numRead;
}
std::vector<char> getBytes(PointViewPtr view)
{
std::vector<char> bytes(view->pointSize() * view->size());
DimTypeList dimTypes = view->dimTypes();
char *p = bytes.data();
for (PointId idx = 0; idx < view->size(); ++idx)
{
view->getPackedPoint(dimTypes, idx, p);
p += view->pointSize();
}
return bytes;
}
void RialtoTest::verifyPointsInBounds(PointViewPtr view,
double minx, double miny, double maxx, double maxy)
{
for (uint32_t i=0; i<view->size(); i++)
{
const double x = view->getFieldAs<double>(Dimension::Id::X, i);
const double y = view->getFieldAs<double>(Dimension::Id::Y, i);
if (!(x >= minx && x <= maxx))
{
printf("query x: min=%lf max=%lf x=%lf\n", minx, maxx, x);
printf("query y: min=%lf max=%lf y=%lf\n", miny, maxy, y);
}
if (!(y >= miny && y <= maxy))
{
printf("query y: min=%lf max=%lf y=%lf\n", miny, maxy, y);
}
EXPECT_TRUE(x >= minx && x <= maxx);
EXPECT_TRUE(y >= miny && y <= maxy);
}
}
MetadataNode DeltaKernel::dump(PointViewPtr& srcView, PointViewPtr& candView,
KD3Index& index, DimIndexMap& dims)
{
MetadataNode root;
for (PointId id = 0; id < srcView->size(); ++id)
{
double x = srcView->getFieldAs<double>(Dimension::Id::X, id);
double y = srcView->getFieldAs<double>(Dimension::Id::Y, id);
double z = srcView->getFieldAs<double>(Dimension::Id::Z, id);
PointId candId = index.neighbor(x, y, z);
// It may be faster to put in a special case to avoid having to
// fetch X, Y and Z, more than once but this is simpler and
// I'm thinking in most cases it will make no practical difference.
for (auto di = dims.begin(); di != dims.end(); ++di)
{
DimIndex& d = di->second;
double sv = srcView->getFieldAs<double>(d.m_srcId, id);
double cv = candView->getFieldAs<double>(d.m_candId, candId);
accumulate(d, sv - cv);
}
}
root.add("source", m_sourceFile);
root.add("candidate", m_candidateFile);
for (auto dpair : dims)
{
DimIndex& d = dpair.second;
MetadataNode dimNode = root.add(d.m_name);
dimNode.add("min", d.m_min);
dimNode.add("max", d.m_max);
dimNode.add("mean", d.m_avg);
}
return root;
}
point_count_t PlyReader::read(PointViewPtr view, point_count_t num)
{
CallbackContext context;
context.view = view;
context.dimensionMap = m_vertexDimensions;
// It's possible that point_count_t holds a value that's larger than the
// long that is the maximum rply (don't know about ply) point count.
long cnt;
cnt = Utils::inRange<long>(num) ? num : (std::numeric_limits<long>::max)();
for (auto it : m_vertexDimensions)
{
ply_set_read_cb(m_ply, "vertex", it.first.c_str(), readPlyCallback,
&context, cnt);
}
if (!ply_read(m_ply))
{
std::stringstream ss;
ss << "Error reading " << m_filename << ".";
throw pdal_error(ss.str());
}
return view->size();
}
point_count_t OciReader::read(PointViewPtr view, point_count_t count)
{
if (eof())
return 0;
point_count_t totalNumRead = 0;
while (totalNumRead < count)
{
if (m_block->numRemaining() == 0)
if (!readOci(m_stmt, m_block))
return totalNumRead;
PointId bufBegin = view->size();
point_count_t numRead = 0;
if (orientation() == Orientation::DimensionMajor)
numRead = readDimMajor(*view, m_block, count - totalNumRead);
else if (orientation() == Orientation::PointMajor)
numRead = readPointMajor(*view, m_block, count - totalNumRead);
PointId bufEnd = bufBegin + numRead;
totalNumRead += numRead;
}
return totalNumRead;
}
PointViewSet RangeFilter::run(PointViewPtr inView)
{
PointViewSet viewSet;
if (!inView->size())
return viewSet;
PointViewPtr outView = inView->makeNew();
for (PointId i = 0; i < inView->size(); ++i)
{
PointRef point = inView->point(i);
if (processOne(point))
outView->appendPoint(*inView, i);
}
viewSet.insert(outView);
return viewSet;
}
void LasWriter::writeView(const PointViewPtr view)
{
Utils::writeProgress(m_progressFd, "READYVIEW",
std::to_string(view->size()));
m_scaling.setAutoXForm(view);
point_count_t pointLen = m_lasHeader.pointLen();
// Since we use the LASzip API, we can't benefit from building
// a buffer of multiple points, so loop.
if (m_compression == LasCompression::LasZip)
{
PointRef point(*view, 0);
for (PointId idx = 0; idx < view->size(); ++idx)
{
point.setPointId(idx);
processPoint(point);
}
}
else
{
// Make a buffer of at most a meg.
m_pointBuf.resize((std::min)((point_count_t)1000000,
pointLen * view->size()));
const PointView& viewRef(*view.get());
point_count_t remaining = view->size();
PointId idx = 0;
while (remaining)
{
point_count_t filled = fillWriteBuf(viewRef, idx, m_pointBuf);
idx += filled;
remaining -= filled;
if (m_compression == LasCompression::LazPerf)
writeLazPerfBuf(m_pointBuf.data(), pointLen, filled);
else
m_ostream->write(m_pointBuf.data(), filled * pointLen);
}
}
Utils::writeProgress(m_progressFd, "DONEVIEW",
std::to_string(view->size()));
}
PointViewSet MADFilter::run(PointViewPtr view)
{
using namespace Dimension;
PointViewSet viewSet;
PointViewPtr output = view->makeNew();
auto estimate_median = [](std::vector<double> vals)
{
std::nth_element(vals.begin(), vals.begin()+vals.size()/2, vals.end());
return *(vals.begin()+vals.size()/2);
};
std::vector<double> z(view->size());
for (PointId j = 0; j < view->size(); ++j)
z[j] = view->getFieldAs<double>(m_dimId, j);
double median = estimate_median(z);
log()->get(LogLevel::Debug) << getName() << " estimated median value: " << median << std::endl;
std::transform(z.begin(), z.end(), z.begin(),
[median](double v) { return std::fabs(v - median); });
double mad = estimate_median(z)*m_madMultiplier;
log()->get(LogLevel::Debug) << getName() << " mad " << mad << std::endl;
for (PointId j = 0; j < view->size(); ++j)
{
if (z[j]/mad < m_multiplier)
output->appendPoint(*view, j);
}
double low_fence = median - m_multiplier * mad;
double hi_fence = median + m_multiplier * mad;
log()->get(LogLevel::Debug) << getName() << " cropping " << m_dimName
<< " in the range (" << low_fence
<< "," << hi_fence << ")" << std::endl;
viewSet.erase(view);
viewSet.insert(output);
return viewSet;
}
MetadataNode InfoKernel::dumpPoints(PointViewPtr inView) const
{
MetadataNode root;
PointViewPtr outView = inView->makeNew();
// Stick points in a inViewfer.
std::vector<PointId> points = getListOfPoints(m_pointIndexes);
for (size_t i = 0; i < points.size(); ++i)
{
PointId id = (PointId)points[i];
if (id < inView->size())
outView->appendPoint(*inView.get(), id);
}
MetadataNode tree = outView->toMetadata();
std::string prefix("point ");
for (size_t i = 0; i < outView->size(); ++i)
{
MetadataNode n = tree.findChild(std::to_string(i));
n.add("PointId", points[i]);
root.add(n.clone("point"));
}
return root;
}
