INLINE void RTCameraPacketGen::generateRayMorton(RayPacket &pckt, int x, int y) const
{
const sse3f org(aOrg.xxxx(), aOrg.yyyy(), aOrg.zzzz());
const ssef left = ssef(ssei(x));
const ssef top = ssef(ssei(y));
uint32 id = 0;
// avoid issues with unused w
pckt.iaMinOrg = pckt.iaMaxOrg = aOrg.xyzz();
for(uint32 j = 0; j < pckt.height; ++j) {
for(uint32 i = 0; i < pckt.width; i += ssef::laneNum(), ++id) {
const ssei xcoord = ssei(mortonX + 4 * id);
const ssei ycoord = ssei(mortonY + 4 * id);
const ssef column = left + ssef(xcoord);
const ssef row = top + ssef(ycoord);
pckt.org[id] = org;
pckt.dir[id].x = fixup(imagePlaneOrg.x + column*xAxis.x + row*zAxis.x);
pckt.dir[id].y = fixup(imagePlaneOrg.y + column*xAxis.y + row*zAxis.y);
pckt.dir[id].z = fixup(imagePlaneOrg.z + column*xAxis.z + row*zAxis.z);
pckt.dir[id] = normalize(pckt.dir[id]);
pckt.rdir[id].x = rcp(pckt.dir[id].x);
pckt.rdir[id].y = rcp(pckt.dir[id].y);
pckt.rdir[id].z = rcp(pckt.dir[id].z);
}
}
}
void BVH4Intersector4Single<types,robust,PrimitiveIntersector4>::occluded(sseb* valid_i, BVH4* bvh, Ray4& ray)
{
/* load ray */
const sseb valid = *valid_i;
sseb terminated = !valid;
sse3f ray_org = ray.org, ray_dir = ray.dir;
ssef ray_tnear = ray.tnear, ray_tfar = ray.tfar;
const sse3f rdir = rcp_safe(ray_dir);
const sse3f org(ray_org), org_rdir = org * rdir;
ray_tnear = select(valid,ray_tnear,ssef(pos_inf));
ray_tfar = select(valid,ray_tfar ,ssef(neg_inf));
const ssef inf = ssef(pos_inf);
Precalculations pre(valid,ray);
/* compute near/far per ray */
sse3i nearXYZ;
nearXYZ.x = select(rdir.x >= 0.0f,ssei(0*(int)sizeof(ssef)),ssei(1*(int)sizeof(ssef)));
nearXYZ.y = select(rdir.y >= 0.0f,ssei(2*(int)sizeof(ssef)),ssei(3*(int)sizeof(ssef)));
nearXYZ.z = select(rdir.z >= 0.0f,ssei(4*(int)sizeof(ssef)),ssei(5*(int)sizeof(ssef)));
/* we have no packet implementation for OBB nodes yet */
size_t bits = movemask(valid);
for (size_t i=__bsf(bits); bits!=0; bits=__btc(bits,i), i=__bsf(bits)) {
if (occluded1(bvh,bvh->root,i,pre,ray,ray_org,ray_dir,rdir,ray_tnear,ray_tfar,nearXYZ))
terminated[i] = -1;
}
store4i(valid & terminated,&ray.geomID,0);
AVX_ZERO_UPPER();
}
INLINE void RTCameraPacketGen::generateRay(RayPacket &pckt, int x, int y) const
{
const sse3f org(aOrg.xxxx(), aOrg.yyyy(), aOrg.zzzz());
const ssef left = ssef(ssei(x))+ ssef::identity();
ssef row = ssef(ssei(y));
uint32 id = 0;
// avoid issues with unused w
pckt.iaMinOrg = pckt.iaMaxOrg = aOrg.xyzz();
for(uint32 j = 0; j < pckt.height; ++j) {
ssef column = left;
for(uint32 i = 0; i < pckt.width; i += ssef::laneNum(), ++id) {
pckt.org[id] = org;
pckt.dir[id].x = fixup(imagePlaneOrg.x + column*xAxis.x + row*zAxis.x);
pckt.dir[id].y = fixup(imagePlaneOrg.y + column*xAxis.y + row*zAxis.y);
pckt.dir[id].z = fixup(imagePlaneOrg.z + column*xAxis.z + row*zAxis.z);
pckt.dir[id].x = (imagePlaneOrg.x + column*xAxis.x + row*zAxis.x);
pckt.dir[id].y = (imagePlaneOrg.y + column*xAxis.y + row*zAxis.y);
pckt.dir[id].z = (imagePlaneOrg.z + column*xAxis.z + row*zAxis.z);
pckt.dir[id] = normalize(pckt.dir[id]);
pckt.rdir[id].x = rcp(pckt.dir[id].x);
pckt.rdir[id].y = rcp(pckt.dir[id].y);
pckt.rdir[id].z = rcp(pckt.dir[id].z);
column += ssef::laneNumv();
}
row += ssef::one();
}
}
__forceinline SpatialSplit::Split SpatialSplit::BinInfo::best(const PrimInfo& pinfo, const Mapping& mapping, const size_t blocks_shift)
{
/* sweep from right to left and compute parallel prefix of merged bounds */
ssef rAreas[BINS];
ssei rCounts[BINS];
ssei count = 0; BBox3fa bx = empty; BBox3fa by = empty; BBox3fa bz = empty;
for (size_t i=BINS-1; i>0; i--)
{
count += numEnd[i];
rCounts[i] = count;
bx.extend(bounds[i][0]); rAreas[i][0] = halfArea(bx);
by.extend(bounds[i][1]); rAreas[i][1] = halfArea(by);
bz.extend(bounds[i][2]); rAreas[i][2] = halfArea(bz);
}
/* sweep from left to right and compute SAH */
ssei blocks_add = (1 << blocks_shift)-1;
ssei ii = 1; ssef vbestSAH = pos_inf; ssei vbestPos = 0;
count = 0; bx = empty; by = empty; bz = empty;
for (size_t i=1; i<BINS; i++, ii+=1)
{
count += numBegin[i-1];
bx.extend(bounds[i-1][0]); float Ax = halfArea(bx);
by.extend(bounds[i-1][1]); float Ay = halfArea(by);
bz.extend(bounds[i-1][2]); float Az = halfArea(bz);
const ssef lArea = ssef(Ax,Ay,Az,Az);
const ssef rArea = rAreas[i];
const ssei lCount = (count +blocks_add) >> blocks_shift;
const ssei rCount = (rCounts[i]+blocks_add) >> blocks_shift;
const ssef sah = lArea*ssef(lCount) + rArea*ssef(rCount);
vbestPos = select(sah < vbestSAH,ii ,vbestPos);
vbestSAH = select(sah < vbestSAH,sah,vbestSAH);
}
/* find best dimension */
float bestSAH = inf;
int bestDim = -1;
int bestPos = 0;
for (size_t dim=0; dim<3; dim++)
{
/* ignore zero sized dimensions */
if (unlikely(mapping.invalid(dim)))
continue;
/* test if this is a better dimension */
if (vbestSAH[dim] < bestSAH && vbestPos[dim] != 0) {
bestDim = dim;
bestPos = vbestPos[dim];
bestSAH = vbestSAH[dim];
}
}
/* return invalid split if no split found */
if (bestDim == -1)
return Split(inf,-1,0,mapping);
/* return best found split */
return Split(bestSAH,bestDim,bestPos,mapping);
}
// Unused since we do not use back face culling right now
INLINE void RTCameraPacketGen::generateCR(RayPacket &pckt, int x, int y) const
{
const ssef left = ssef(ssei(x)) + pckt.crx;
const ssef top = ssef(ssei(y)) + pckt.cry;
pckt.crdir.x = imagePlaneOrg.x + left*xAxis.x + top*zAxis.x;
pckt.crdir.y = imagePlaneOrg.y + left*xAxis.y + top*zAxis.y;
pckt.crdir.z = imagePlaneOrg.z + left*xAxis.z + top*zAxis.z;
}
void test(void)
{
f = essef(f);
d = essed(d);
f = ssef(f);
d = ssed(d);
}
void test(void)
{
f = essef(f); /* { dg-error "SSE" } */
d = essed(d); /* { dg-error "SSE" } */
f = ssef(f); /* { dg-error "SSE" } */
d = ssed(d); /* { dg-error "SSE" } */
}
PerspectiveFrustum::PerspectiveFrustum(const RTCamera &cam, Ref<HiZ> hiz)
: hiz(hiz)
{
this->org_aos = ssef(cam.org.x, cam.org.y, cam.org.z, 0.f);
this->view_aos = ssef(cam.view.x, cam.view.y, cam.view.z, 0.f);
this->org = sse3f(cam.org.x, cam.org.y, cam.org.z);
this->view = sse3f(cam.view.x, cam.view.y, cam.view.z);
this->xAxis = sse3f(cam.xAxis.x, cam.xAxis.y, cam.xAxis.z);
this->zAxis = sse3f(cam.zAxis.x, cam.zAxis.y, cam.zAxis.z);
this->yMaxSinAngle = sin(cam.fov * float(pi) / 360.f);
this->xMaxSinAngle = cam.ratio * this->yMaxSinAngle;
this->yMaxInvTanAngle = 1.f / tan(cam.fov * float(pi) / 360.f);
this->xMaxInvTanAngle = this->yMaxInvTanAngle / cam.ratio;
this->xAxis = normalize(this->xAxis);
this->zAxis = normalize(this->zAxis);
this->windowing = ssef(float(hiz->tileXNum) * 0.5f,
float(hiz->tileXNum) * 0.5f,
float(hiz->tileYNum) * 0.5f,
float(hiz->tileYNum) * 0.5f);
this->hizExtent = ssef(float(hiz->tileXNum-1),
float(hiz->tileXNum-1),
float(hiz->tileYNum-1),
float(hiz->tileYNum-1));
}
void BVH4MBIntersector1<TriangleIntersector>::occluded(const BVH4MB* bvh, Ray& ray)
{
AVX_ZERO_UPPER();
STAT3(shadow.travs,1,1,1);
/*! stack state */
Base* stack[1+3*BVH4MB::maxDepth]; //!< stack of nodes that still need to get traversed
Base** stackPtr = stack+1; //!< current stack pointer
stack[0] = bvh->root; //!< push first node onto stack
/*! offsets to select the side that becomes the lower or upper bound */
const size_t nearX = (ray.dir.x >= 0) ? 0*2*sizeof(ssef) : 1*2*sizeof(ssef);
const size_t nearY = (ray.dir.y >= 0) ? 2*2*sizeof(ssef) : 3*2*sizeof(ssef);
const size_t nearZ = (ray.dir.z >= 0) ? 4*2*sizeof(ssef) : 5*2*sizeof(ssef);
const size_t farX = nearX ^ 32;
const size_t farY = nearY ^ 32;
const size_t farZ = nearZ ^ 32;
/*! load the ray into SIMD registers */
const sse3f norg(-ray.org.x,-ray.org.y,-ray.org.z);
const Vec3fa ray_rdir = rcp_safe(ray.dir);
const sse3f rdir(ray_rdir.x,ray_rdir.y,ray_rdir.z);
const ssef rayNear(ray.tnear);
const ssef rayFar (ray.tfar);
/*! pop node from stack */
while (true)
{
/* finish when the stack is empty */
if (unlikely(stackPtr == stack)) break;
Base* cur = *(--stackPtr);
/*! this is an inner node */
if (likely(cur->isNode()))
{
STAT3(shadow.trav_nodes,1,1,1);
/*! single ray intersection with 4 boxes */
const Node* node = cur->node();
const ssef* pNearX = (const ssef*)((const char*)node+nearX);
const ssef* pNearY = (const ssef*)((const char*)node+nearY);
const ssef* pNearZ = (const ssef*)((const char*)node+nearZ);
const ssef tNearX = (norg.x + ssef(pNearX[0]) + ray.time*pNearX[1]) * rdir.x;
const ssef tNearY = (norg.y + ssef(pNearY[0]) + ray.time*pNearY[1]) * rdir.y;
const ssef tNearZ = (norg.z + ssef(pNearZ[0]) + ray.time*pNearZ[1]) * rdir.z;
const ssef tNear = max(tNearX,tNearY,tNearZ,rayNear);
const ssef* pFarX = (const ssef*)((const char*)node+farX);
const ssef* pFarY = (const ssef*)((const char*)node+farY);
const ssef* pFarZ = (const ssef*)((const char*)node+farZ);
const ssef tFarX = (norg.x + ssef(pFarX[0]) + ray.time*pFarX[1]) * rdir.x;
const ssef tFarY = (norg.y + ssef(pFarY[0]) + ray.time*pFarY[1]) * rdir.y;
const ssef tFarZ = (norg.z + ssef(pFarZ[0]) + ray.time*pFarZ[1]) * rdir.z;
const ssef tFar = min(tFarX,tFarY,tFarZ,rayFar);
size_t _hit = movemask(tNear <= tFar);
/*! push hit nodes onto stack */
if (likely(_hit == 0)) continue;
size_t r = __bsf(_hit); _hit = __btc(_hit,r);
*stackPtr = node->child[r]; stackPtr++;
if (likely(_hit == 0)) continue;
r = __bsf(_hit); _hit = __btc(_hit,r);
*stackPtr = node->child[r]; stackPtr++;
if (likely(_hit == 0)) continue;
r = __bsf(_hit); _hit = __btc(_hit,r);
*stackPtr = node->child[r]; stackPtr++;
if (likely(_hit == 0)) continue;
r = __bsf(_hit); _hit = __btc(_hit,r);
*stackPtr = node->child[r]; stackPtr++;
}
/*! this is a leaf node */
else
{
STAT3(shadow.trav_leaves,1,1,1);
size_t num; Triangle* tri = (Triangle*) cur->leaf(num);
for (size_t i=0; i<num; i++)
if (TriangleIntersector::occluded(ray,tri[i],bvh->geometry)) {
ray.geomID = 0;
break;
}
}
}
AVX_ZERO_UPPER();
}
void BVH2Intersector<TriangleIntersector>::intersect(const Ray& ray, Hit& hit) const
{
AVX_ZERO_UPPER();
STAT3(normal.travs,1,1,1);
struct StackItem {
Base* ptr; //!< node pointer
float dist; //!< distance of node
};
/*! stack state */
StackItem stack[1+BVH2::maxDepth]; //!< stack of nodes that still need to get traversed
StackItem* stackPtr = stack; //!< current stack pointer
Base* cur = bvh->root; //!< in cur we track the ID of the current node
/*! precomputed shuffles, to switch lower and upper bounds depending on ray direction */
const ssei identity = _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
const ssei swap = _mm_set_epi8( 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8);
const ssei shuffleX = ray.dir.x >= 0 ? identity : swap;
const ssei shuffleY = ray.dir.y >= 0 ? identity : swap;
const ssei shuffleZ = ray.dir.z >= 0 ? identity : swap;
/*! load the ray into SIMD registers */
const ssei pn = ssei(0x00000000,0x00000000,0x80000000,0x80000000);
const sse3f norg(-ray.org.x,-ray.org.y,-ray.org.z);
const sse3f rdir = sse3f(ssef(ray.rdir.x) ^ pn, ssef(ray.rdir.y) ^ pn, ssef(ray.rdir.z) ^ pn);
ssef nearFar(ray.near, ray.near, -ray.far, -ray.far);
hit.t = min(hit.t,ray.far);
while (true)
{
/*! downtraversal loop */
while (likely(cur->isNode()))
{
/*! single ray intersection with box of both children. */
const Node* node = cur->node();
const ssef tNearFarX = (shuffle8(node->lower_upper_x,shuffleX) + norg.x) * rdir.x;
const ssef tNearFarY = (shuffle8(node->lower_upper_y,shuffleY) + norg.y) * rdir.y;
const ssef tNearFarZ = (shuffle8(node->lower_upper_z,shuffleZ) + norg.z) * rdir.z;
const ssef tNearFar = max(tNearFarX,tNearFarY,tNearFarZ,nearFar) ^ pn;
const sseb lrhit = tNearFar <= shuffle8(tNearFar,swap);
/*! if two children hit, push far node onto stack and continue with closer node */
if (likely(lrhit[0] != 0 && lrhit[1] != 0)) {
if (likely(tNearFar[0] < tNearFar[1])) {
stackPtr->ptr = node->child[1];
stackPtr->dist = tNearFar[1];
cur = node->child[0];
stackPtr++;
}
else {
stackPtr->ptr = node->child[0];
stackPtr->dist = tNearFar[0];
cur = node->child[1];
stackPtr++;
}
}
/*! if one child hit, continue with that child */
else {
if (likely(lrhit[0] != 0)) cur = node->child[0];
else if (likely(lrhit[1] != 0)) cur = node->child[1];
else goto pop_node;
}
}
/*! leaf node, intersect all triangles */
{
STAT3(shadow.trav_leaves,1,1,1);
size_t num; Triangle* tri = (Triangle*) cur->leaf(num);
for (size_t i=0; i<num; i++)
TriangleIntersector::intersect(ray,hit,tri[i],bvh->vertices);
nearFar = shuffle<0,1,2,3>(nearFar,-hit.t);
}
/*! pop next node from stack */
pop_node:
if (unlikely(stackPtr == stack)) break;
--stackPtr;
cur = stackPtr->ptr;
if (unlikely(stackPtr->dist > hit.t)) goto pop_node;
}
AVX_ZERO_UPPER();
}
void BVH4Intersector4Chunk<PrimitiveIntersector4>::intersect(sseb* valid_i, BVH4* bvh, Ray4& ray)
{
/* load ray */
const sseb valid0 = *valid_i;
const sse3f rdir = rcp_safe(ray.dir);
const sse3f org(ray.org), org_rdir = org * rdir;
ssef ray_tnear = select(valid0,ray.tnear,ssef(pos_inf));
ssef ray_tfar = select(valid0,ray.tfar ,ssef(neg_inf));
const ssef inf = ssef(pos_inf);
Precalculations pre(valid0,ray);
/* allocate stack and push root node */
ssef stack_near[stackSize];
NodeRef stack_node[stackSize];
stack_node[0] = BVH4::invalidNode;
stack_near[0] = inf;
stack_node[1] = bvh->root;
stack_near[1] = ray_tnear;
NodeRef* stackEnd = stack_node+stackSize;
NodeRef* __restrict__ sptr_node = stack_node + 2;
ssef* __restrict__ sptr_near = stack_near + 2;
while (1)
{
/* pop next node from stack */
assert(sptr_node > stack_node);
sptr_node--;
sptr_near--;
NodeRef curNode = *sptr_node;
if (unlikely(curNode == BVH4::invalidNode)) {
assert(sptr_node == stack_node);
break;
}
/* cull node if behind closest hit point */
ssef curDist = *sptr_near;
if (unlikely(none(ray_tfar > curDist)))
continue;
while (1)
{
/* test if this is a leaf node */
if (unlikely(curNode.isLeaf()))
break;
const sseb valid_node = ray_tfar > curDist;
STAT3(normal.trav_nodes,1,popcnt(valid_node),4);
const Node* __restrict__ const node = curNode.node();
/* pop of next node */
assert(sptr_node > stack_node);
sptr_node--;
sptr_near--;
curNode = *sptr_node;
curDist = *sptr_near;
#pragma unroll(4)
for (unsigned i=0; i<BVH4::N; i++)
{
const NodeRef child = node->children[i];
if (unlikely(child == BVH4::emptyNode)) break;
#if defined(__AVX2__)
const ssef lclipMinX = msub(node->lower_x[i],rdir.x,org_rdir.x);
const ssef lclipMinY = msub(node->lower_y[i],rdir.y,org_rdir.y);
const ssef lclipMinZ = msub(node->lower_z[i],rdir.z,org_rdir.z);
const ssef lclipMaxX = msub(node->upper_x[i],rdir.x,org_rdir.x);
const ssef lclipMaxY = msub(node->upper_y[i],rdir.y,org_rdir.y);
const ssef lclipMaxZ = msub(node->upper_z[i],rdir.z,org_rdir.z);
#else
const ssef lclipMinX = (node->lower_x[i] - org.x) * rdir.x;
const ssef lclipMinY = (node->lower_y[i] - org.y) * rdir.y;
const ssef lclipMinZ = (node->lower_z[i] - org.z) * rdir.z;
const ssef lclipMaxX = (node->upper_x[i] - org.x) * rdir.x;
const ssef lclipMaxY = (node->upper_y[i] - org.y) * rdir.y;
const ssef lclipMaxZ = (node->upper_z[i] - org.z) * rdir.z;
#endif
#if defined(__SSE4_1__)
const ssef lnearP = maxi(maxi(mini(lclipMinX, lclipMaxX), mini(lclipMinY, lclipMaxY)), mini(lclipMinZ, lclipMaxZ));
const ssef lfarP = mini(mini(maxi(lclipMinX, lclipMaxX), maxi(lclipMinY, lclipMaxY)), maxi(lclipMinZ, lclipMaxZ));
const sseb lhit = maxi(lnearP,ray_tnear) <= mini(lfarP,ray_tfar);
#else
const ssef lnearP = max(max(min(lclipMinX, lclipMaxX), min(lclipMinY, lclipMaxY)), min(lclipMinZ, lclipMaxZ));
const ssef lfarP = min(min(max(lclipMinX, lclipMaxX), max(lclipMinY, lclipMaxY)), max(lclipMinZ, lclipMaxZ));
const sseb lhit = max(lnearP,ray_tnear) <= min(lfarP,ray_tfar);
#endif
/* if we hit the child we choose to continue with that child if it
is closer than the current next child, or we push it onto the stack */
if (likely(any(lhit)))
{
assert(sptr_node < stackEnd);
const ssef childDist = select(lhit,lnearP,inf);
const NodeRef child = node->children[i];
assert(child != BVH4::emptyNode);
sptr_node++;
sptr_near++;
/* push cur node onto stack and continue with hit child */
if (any(childDist < curDist))
{
*(sptr_node-1) = curNode;
*(sptr_near-1) = curDist;
curDist = childDist;
curNode = child;
}
/* push hit child onto stack */
else {
*(sptr_node-1) = child;
*(sptr_near-1) = childDist;
}
}
}
}
/* return if stack is empty */
if (unlikely(curNode == BVH4::invalidNode)) {
assert(sptr_node == stack_node);
break;
}
/* intersect leaf */
const sseb valid_leaf = ray_tfar > curDist;
STAT3(normal.trav_leaves,1,popcnt(valid_leaf),4);
size_t items; const Primitive* prim = (Primitive*) curNode.leaf(items);
PrimitiveIntersector4::intersect(valid_leaf,pre,ray,prim,items,bvh->geometry);
ray_tfar = select(valid_leaf,ray.tfar,ray_tfar);
}
AVX_ZERO_UPPER();
}
void BVH4Intersector4Hybrid<types,robust,PrimitiveIntersector4>::intersect(sseb* valid_i, BVH4* bvh, Ray4& ray)
{
/* load ray */
const sseb valid0 = *valid_i;
sse3f ray_org = ray.org;
sse3f ray_dir = ray.dir;
ssef ray_tnear = ray.tnear, ray_tfar = ray.tfar;
const sse3f rdir = rcp_safe(ray_dir);
const sse3f org(ray_org), org_rdir = org * rdir;
ray_tnear = select(valid0,ray_tnear,ssef(pos_inf));
ray_tfar = select(valid0,ray_tfar ,ssef(neg_inf));
const ssef inf = ssef(pos_inf);
Precalculations pre(valid0,ray);
/* compute near/far per ray */
sse3i nearXYZ;
nearXYZ.x = select(rdir.x >= 0.0f,ssei(0*(int)sizeof(ssef)),ssei(1*(int)sizeof(ssef)));
nearXYZ.y = select(rdir.y >= 0.0f,ssei(2*(int)sizeof(ssef)),ssei(3*(int)sizeof(ssef)));
nearXYZ.z = select(rdir.z >= 0.0f,ssei(4*(int)sizeof(ssef)),ssei(5*(int)sizeof(ssef)));
/* allocate stack and push root node */
ssef stack_near[stackSizeChunk];
NodeRef stack_node[stackSizeChunk];
stack_node[0] = BVH4::invalidNode;
stack_near[0] = inf;
stack_node[1] = bvh->root;
stack_near[1] = ray_tnear;
NodeRef* stackEnd = stack_node+stackSizeChunk;
NodeRef* __restrict__ sptr_node = stack_node + 2;
ssef* __restrict__ sptr_near = stack_near + 2;
while (1) pop:
{
/* pop next node from stack */
assert(sptr_node > stack_node);
sptr_node--;
sptr_near--;
NodeRef cur = *sptr_node;
if (unlikely(cur == BVH4::invalidNode)) {
assert(sptr_node == stack_node);
break;
}
/* cull node if behind closest hit point */
ssef curDist = *sptr_near;
const sseb active = curDist < ray_tfar;
if (unlikely(none(active)))
continue;
/* switch to single ray traversal */
#if !defined(__WIN32__) || defined(__X86_64__)
size_t bits = movemask(active);
if (unlikely(__popcnt(bits) <= SWITCH_THRESHOLD)) {
for (size_t i=__bsf(bits); bits!=0; bits=__btc(bits,i), i=__bsf(bits)) {
BVH4Intersector4Single<types,robust,PrimitiveIntersector4>::intersect1(bvh, cur, i, pre, ray, ray_org, ray_dir, rdir, ray_tnear, ray_tfar, nearXYZ);
}
ray_tfar = min(ray_tfar,ray.tfar);
continue;
}
#endif
while (1)
{
/* process normal nodes */
if (likely((types & 0x1) && cur.isNode()))
{
const sseb valid_node = ray_tfar > curDist;
STAT3(normal.trav_nodes,1,popcnt(valid_node),4);
const Node* __restrict__ const node = cur.node();
/* pop of next node */
assert(sptr_node > stack_node);
sptr_node--;
sptr_near--;
cur = *sptr_node;
curDist = *sptr_near;
#pragma unroll(4)
for (unsigned i=0; i<BVH4::N; i++)
{
const NodeRef child = node->children[i];
if (unlikely(child == BVH4::emptyNode)) break;
ssef lnearP; const sseb lhit = node->intersect<robust>(i,org,rdir,org_rdir,ray_tnear,ray_tfar,lnearP);
/* if we hit the child we choose to continue with that child if it
is closer than the current next child, or we push it onto the stack */
if (likely(any(lhit)))
{
assert(sptr_node < stackEnd);
assert(child != BVH4::emptyNode);
const ssef childDist = select(lhit,lnearP,inf);
sptr_node++;
sptr_near++;
/* push cur node onto stack and continue with hit child */
if (any(childDist < curDist))
{
*(sptr_node-1) = cur;
*(sptr_near-1) = curDist;
curDist = childDist;
cur = child;
}
/* push hit child onto stack */
else {
*(sptr_node-1) = child;
*(sptr_near-1) = childDist;
}
}
}
#if SWITCH_DURING_DOWN_TRAVERSAL == 1
// seems to be the best place for testing utilization
if (unlikely(popcnt(ray_tfar > curDist) <= SWITCH_THRESHOLD))
{
*sptr_node++ = cur;
*sptr_near++ = curDist;
goto pop;
}
#endif
}
/* process motion blur nodes */
else if (likely((types & 0x10) && cur.isNodeMB()))
{
const sseb valid_node = ray_tfar > curDist;
STAT3(normal.trav_nodes,1,popcnt(valid_node),4);
const BVH4::NodeMB* __restrict__ const node = cur.nodeMB();
/* pop of next node */
assert(sptr_node > stack_node);
sptr_node--;
sptr_near--;
cur = *sptr_node;
curDist = *sptr_near;
#pragma unroll(4)
for (unsigned i=0; i<BVH4::N; i++)
{
const NodeRef child = node->child(i);
if (unlikely(child == BVH4::emptyNode)) break;
ssef lnearP; const sseb lhit = node->intersect(i,org,rdir,org_rdir,ray_tnear,ray_tfar,ray.time,lnearP);
/* if we hit the child we choose to continue with that child if it
is closer than the current next child, or we push it onto the stack */
if (likely(any(lhit)))
{
assert(sptr_node < stackEnd);
assert(child != BVH4::emptyNode);
const ssef childDist = select(lhit,lnearP,inf);
sptr_node++;
sptr_near++;
/* push cur node onto stack and continue with hit child */
if (any(childDist < curDist))
{
*(sptr_node-1) = cur;
*(sptr_near-1) = curDist;
curDist = childDist;
cur = child;
}
/* push hit child onto stack */
else {
*(sptr_node-1) = child;
*(sptr_near-1) = childDist;
}
}
}
#if SWITCH_DURING_DOWN_TRAVERSAL == 1
// seems to be the best place for testing utilization
if (unlikely(popcnt(ray_tfar > curDist) <= SWITCH_THRESHOLD))
{
*sptr_node++ = cur;
*sptr_near++ = curDist;
goto pop;
}
#endif
}
else
break;
}
void kdtreebenthin::draw<1>(scene& scene, ray4* r, hit4* hit4)
{
unsigned int signx = movemask(r->D().x());
unsigned int signy = movemask(r->D().y());
unsigned int signz = movemask(r->D().z());
//If the traversal direction is not same for all rays we
// do a single ray traversal
if (((signx - 1) < 14) // sign of x is 0xF or 0
|| ((signy - 1) < 14) // sign of y is 0xF or 0
|| ((signz - 1) < 14)) { // sign of z is 0xF or 0
hit hit[4];
for (int i = 0; i < 4; ++i) {
vec3f d(r->D().x()[i], r->D().y()[i], r->D().z()[i]);
ray ray(r->O(), d);
hit[i].prim = -1;
draw(scene, ray, hit[i]);
}
hit4->prim = ssei(hit[0].prim, hit[1].prim, hit[2].prim, hit[3].prim);
hit4->u = ssef(hit[0].u, hit[1].u, hit[2].u, hit[3].u);
hit4->v = ssef(hit[0].v, hit[1].v, hit[2].v, hit[3].v);
return;
}
ssef tnear, tfar;
_boundingBox.clip(*r, tnear, tfar);
if (movemask(tnear >= tfar) == 0xF)
return;
const unsigned int dir[3][2] = {
{ signx & 1 , 1 - (signx & 1) },
{ signy & 1 , 1 - (signy & 1) },
{ signz & 1 , 1 - (signz & 1) } };
ssef far[MAX_STACK_SIZE];
ssef near[MAX_STACK_SIZE];
int nodes[MAX_STACK_SIZE];
//push dummyNode onto stack which will cause us to exit
nodes[0] = 0;
far[0] = BPRAY_INF;
uint32_t stackptr = 1;
kdnode* currNode = _nodes + 1;
int activemask = 0xF;
#if MAILBOX
static uint64_t rayid = 0;
__sync_add_and_fetch(&rayid, 1);
#endif
while (true) {
if (!currNode->isLeaf()) {
const int axis = currNode->getAxis();
const int front = currNode->getLeft() + dir[axis][0];
const int back = currNode->getLeft() + dir[axis][1];
const ssef dist = currNode->getSplit() - r->O()[axis];
const ssef t = dist * r->rcpD()[axis];
currNode = _nodes + back;
if (!(movemask(tnear <= t) & activemask)) continue;
currNode = _nodes + front;
if (!(movemask(tfar >= t) & activemask)) continue;
nodes[stackptr] = back;
near[stackptr] = max(tnear, t);
far[stackptr] = tfar;
tfar = min(tfar, t);
activemask &= movemask(tnear <= tfar);
++stackptr;
} else {
int primidx = currNode->getPrimitiveOffset();
int primcount = currNode->getNumPrims();
for (int i = 0; i != primcount; ++i) {
int t = _prims[primidx + i];
//prefetch
int t2 = _prims[primidx + i + 1];
_mm_prefetch((char*)&scene._accels[t2], _MM_HINT_T0);
#if MAILBOX
//mailboxing
if (mbox.find(scene, rayid, t)) continue;
#endif
scene.intersect(t, *r, *hit4);
#if MAILBOX
mbox.add(scene, rayid, t);
#endif
}
if (movemask(tfar < r->tfar) == 0) return;
--stackptr;
currNode = nodes[stackptr] + _nodes;
tfar = far[stackptr];
tnear = near[stackptr];
activemask = movemask(tnear <= tfar);
}
}
}
bool BVH2Intersector<TriangleIntersector>::occluded(const Ray& ray) const
{
AVX_ZERO_UPPER();
/*! stack state */
Base* stack[1+BVH2::maxDepth]; //!< stack of nodes that still need to get traversed
Base** stackPtr = stack; //!< current stack pointer
Base* cur = bvh->root; //!< in cur we track the ID of the current node
/*! precomputed shuffles, to switch lower and upper bounds depending on ray direction */
const ssei identity = _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
const ssei swap = _mm_set_epi8( 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8);
const ssei shuffleX = ray.dir.x >= 0 ? identity : swap;
const ssei shuffleY = ray.dir.y >= 0 ? identity : swap;
const ssei shuffleZ = ray.dir.z >= 0 ? identity : swap;
/*! load the ray into SIMD registers */
const ssei pn = ssei(0x00000000,0x00000000,0x80000000,0x80000000);
const sse3f norg(-ray.org.x,-ray.org.y,-ray.org.z);
const sse3f rdir = sse3f(ssef(ray.rdir.x) ^ pn, ssef(ray.rdir.y) ^ pn, ssef(ray.rdir.z) ^ pn);
ssef nearFar(ray.near, ray.near, -ray.far, -ray.far);
while (true)
{
/*! this is an inner node */
while (likely(cur->isNode()))
{
/*! Single ray intersection with box of both children. See bvh2i.h for node layout. */
const Node* node = cur->node();
const ssef tNearFarX = (shuffle8(node->lower_upper_x,shuffleX) + norg.x) * rdir.x;
const ssef tNearFarY = (shuffle8(node->lower_upper_y,shuffleY) + norg.y) * rdir.y;
const ssef tNearFarZ = (shuffle8(node->lower_upper_z,shuffleZ) + norg.z) * rdir.z;
const ssef tNearFar = max(tNearFarX,tNearFarY,tNearFarZ,nearFar) ^ pn;
const sseb lrhit = tNearFar <= shuffle8(tNearFar,swap);
/*! if two children hit, push far node onto stack and continue with closer node */
if (likely(lrhit[0] != 0 && lrhit[1] != 0)) {
*stackPtr++ = node->child[0]; cur = node->child[1];
}
/*! if one child hit, continue with that child */
else {
if (lrhit[0] != 0) cur = node->child[0];
else if (lrhit[1] != 0) cur = node->child[1];
else goto pop_node;
}
}
/*! leaf node, intersect all triangles */
{
STAT3(shadow.trav_leaves,1,1,1);
size_t num; Triangle* tri = (Triangle*) cur->leaf(num);
for (size_t i=0; i<num; i++)
if (TriangleIntersector::occluded(ray,tri[i],bvh->vertices)) {
AVX_ZERO_UPPER();
return true;
}
}
/*! pop next node from stack */
pop_node:
if (unlikely(stackPtr == stack)) break;
cur = *(--stackPtr);
}
AVX_ZERO_UPPER();
return false;
}
__m128 BVH4Intersector4Hybrid<TriangleIntersector4>::occluded(const BVH4Intersector4Hybrid* This, Ray4& ray, const __m128 valid_i)
{
sseb valid = valid_i;
NodeRef invalid = (NodeRef)1;
sseb terminated = !valid;
const BVH4* bvh = This->bvh;
/* load ray into registers */
ssef ray_near = select(valid,ray.tnear,pos_inf);
ssef ray_far = select(valid,ray.tfar ,neg_inf);
sse3f rdir = rcp_safe(ray.dir);
/* allocate stack and push root node */
NodeRef stack_node[3*BVH4::maxDepth+1];
ssef stack_near[3*BVH4::maxDepth+1];
stack_node[0] = invalid;
stack_near[0] = ssef(inf);
stack_node[1] = bvh->root;
stack_near[1] = ray_near;
NodeRef* sptr_node = stack_node+2;
ssef * sptr_near = stack_near+2;
while (1)
{
/* pop next node from stack */
sptr_node--;
sptr_near--;
ssef curDist = *sptr_near;
NodeRef curNode = *sptr_node;
if (unlikely(curNode == invalid))
break;
/* cull node if behind closest hit point */
const sseb active = curDist < ray_far;
if (unlikely(none(active)))
continue;
/* switch to single ray traversal */
size_t bits = movemask(active);
if (unlikely(__popcnt(bits) <= SWITCH_THRESHOLD)) {
for (size_t i=__bsf(bits); bits!=0; bits=__btc(bits,i), i=__bsf(bits)) {
if (BVH4Intersector1<TriangleIntersector1>::occluded1(bvh,curNode,i,ray,rdir))
terminated[i] = -1;
}
if (all(terminated)) return terminated;
/* let terminated rays miss all boxes */
ray_far = select(terminated,neg_inf,ray_far);
continue;
}
while (1)
{
/* test if this is a leaf node */
if (unlikely(curNode.isLeaf()))
break;
const Node* const node = curNode.node(bvh->nodePtr()); //NodeRef(curNode).node(nodes);
//prefetch<PFHINT_L1>((ssef*)node + 1); // depth first order prefetch
/* pop of next node */
sptr_node--;
sptr_near--;
curNode = *sptr_node;
curDist = *sptr_near;
for (unsigned i=0;i<4;i++)
{
const ssef dminx = (ssef(node->lower_x[i]) - ray.org.x) * rdir.x;
const ssef dmaxx = (ssef(node->upper_x[i]) - ray.org.x) * rdir.x;
const ssef dminy = (ssef(node->lower_y[i]) - ray.org.y) * rdir.y;
const ssef dmaxy = (ssef(node->upper_y[i]) - ray.org.y) * rdir.y;
const ssef dminz = (ssef(node->lower_z[i]) - ray.org.z) * rdir.z;
const ssef dmaxz = (ssef(node->upper_z[i]) - ray.org.z) * rdir.z;
const ssef dlowerx = min(dminx,dmaxx);
const ssef dupperx = max(dminx,dmaxx);
const ssef dlowery = min(dminy,dmaxy);
const ssef duppery = max(dminy,dmaxy);
const ssef dlowerz = min(dminz,dmaxz);
const ssef dupperz = max(dminz,dmaxz);
const ssef near = max(dlowerx,dlowery,dlowerz,ray_near);
const ssef far = min(dupperx,duppery,dupperz,ray_far );
const sseb mhit = near <= far;
const ssef childDist = select(mhit,near,inf);
const sseb closer = childDist < curDist;
/* if we hit the child we choose to continue with that child if it
is closer than the current next child, or we push it onto the stack */
if (likely(any(mhit)))
{
const NodeRef child = node->child(i);
//if (child != invalid)
{
sptr_node++;
sptr_near++;
/* push cur node onto stack and continue with hit child */
if (any(closer)) {
*(sptr_node-1) = curNode;
*(sptr_near-1) = curDist;
curDist = childDist;
curNode = child;
}
/* push hit child onto stack*/
else {
*(sptr_node-1) = child;
*(sptr_near-1) = childDist;
}
}
}
}
}
/* return if stack is empty */
if (unlikely(curNode == invalid))
break;
/* decode leaf node */
size_t num;
Triangle* tri = (Triangle*) curNode.leaf(bvh->triPtr(),num);
/* intersect triangles */
for (size_t i=0; i<num; i++) {
terminated |= TriangleIntersector4::occluded(valid,ray,tri[i],bvh->vertices);
if (all(terminated)) return terminated;
}
ray_far = select(terminated,neg_inf,ray_far);
}
return valid & terminated;
}
__forceinline bool occludedT(const BVH4* bvh, Ray& ray)
{
typedef typename TriangleIntersector::Triangle Triangle;
typedef StackItemT<size_t> StackItem;
typedef typename BVH4::NodeRef NodeRef;
typedef typename BVH4::Node Node;
/*! stack state */
NodeRef stack[1+3*BVH4::maxDepth]; //!< stack of nodes that still need to get traversed
NodeRef* stackPtr = stack+1; //!< current stack pointer
stack[0] = bvh->root;
/*! load the ray into SIMD registers */
const avxf pos_neg = avxf(ssef(+0.0f),ssef(-0.0f));
const avxf neg_pos = avxf(ssef(-0.0f),ssef(+0.0f));
const avxf flipSignX = swapX ? neg_pos : pos_neg;
const avxf flipSignY = swapY ? neg_pos : pos_neg;
const avxf flipSignZ = swapZ ? neg_pos : pos_neg;
const avx3f norg(-ray.org.x,-ray.org.y,-ray.org.z);
const Vector3f ray_rdir = rcp_safe(ray.dir);
const avx3f rdir(ray_rdir.x^flipSignX,ray_rdir.y^flipSignY,ray_rdir.z^flipSignZ);
const avx3f org_rdir(avx3f(ray.org.x,ray.org.y,ray.org.z)*rdir);
const avxf rayNearFar(ssef(ray.tnear),-ssef(ray.tfar));
const void* nodePtr = bvh->nodePtr();
const void* triPtr = bvh->triPtr();
/* pop loop */
while (true) pop:
{
/*! pop next node */
if (unlikely(stackPtr == stack)) break;
stackPtr--;
NodeRef cur = (NodeRef) *stackPtr;
/* downtraversal loop */
while (true)
{
/*! stop if we found a leaf */
if (unlikely(cur.isLeaf())) break;
STAT3(shadow.trav_nodes,1,1,1);
/*! single ray intersection with 4 boxes */
const Node* node = cur.node(nodePtr);
#if defined (__AVX2__) || defined(__MIC__)
const avxf tLowerUpperX = msub(avxf::load(&node->lower_x), rdir.x, org_rdir.x);
const avxf tLowerUpperY = msub(avxf::load(&node->lower_y), rdir.y, org_rdir.y);
const avxf tLowerUpperZ = msub(avxf::load(&node->lower_z), rdir.z, org_rdir.z);
#else
const avxf tLowerUpperX = (norg.x + avxf::load(&node->lower_x)) * rdir.x;
const avxf tLowerUpperY = (norg.y + avxf::load(&node->lower_y)) * rdir.y;
const avxf tLowerUpperZ = (norg.z + avxf::load(&node->lower_z)) * rdir.z;
#endif
const avxf tNearFarX = swapX ? shuffle<1,0>(tLowerUpperX) : tLowerUpperX;
const avxf tNearFarY = swapY ? shuffle<1,0>(tLowerUpperY) : tLowerUpperY;
const avxf tNearFarZ = swapZ ? shuffle<1,0>(tLowerUpperZ) : tLowerUpperZ;
const avxf tNearFar = max(tNearFarX,tNearFarY,tNearFarZ,rayNearFar);
const ssef tNear = extract<0>(tNearFar);
const ssef tFar = extract<1>(tNearFar);
size_t mask = movemask(-tNear >= tFar);
/*! if no child is hit, pop next node */
if (unlikely(mask == 0))
goto pop;
/*! one child is hit, continue with that child */
size_t r = __bsf(mask); mask = __btc(mask,r);
if (likely(mask == 0)) {
cur = node->child(r);
continue;
}
/*! two children are hit, push far child, and continue with closer child */
NodeRef c0 = node->child(r); const float d0 = tNear[r];
r = __bsf(mask); mask = __btc(mask,r);
NodeRef c1 = node->child(r); const float d1 = tNear[r];
if (likely(mask == 0)) {
if (d0 < d1) { *stackPtr = c1; stackPtr++; cur = c0; continue; }
else { *stackPtr = c0; stackPtr++; cur = c1; continue; }
}
*stackPtr = c0; stackPtr++;
*stackPtr = c1; stackPtr++;
/*! three children are hit */
r = __bsf(mask); mask = __btc(mask,r);
cur = node->child(r); *stackPtr = cur; stackPtr++;
if (likely(mask == 0)) {
stackPtr--;
continue;
}
/*! four children are hit */
cur = node->child(3);
}
/*! this is a leaf node */
STAT3(shadow.trav_leaves,1,1,1);
size_t num; Triangle* tri = (Triangle*) cur.leaf(triPtr,num);
for (size_t i=0; i<num; i++) {
if (TriangleIntersector::occluded(ray,tri[i],bvh->vertices)) {
AVX_ZERO_UPPER();
return true;
}
}
}
AVX_ZERO_UPPER();
return false;
}
bool BVH2Traverser::occluded(const Ray& ray) const
{
/*! stack state */
int stackPtr = 0; //!< current stack pointer
int stack[1+BVH2<Triangle4>::maxDepth]; //!< stack of nodes that still need to get traversed
int cur = bvh->root; //!< in cur we track the ID of the current node
/*! precomputed shuffles, to switch lower and upper bounds depending on ray direction */
const ssei identity = _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
const ssei swap = _mm_set_epi8( 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8);
const ssei shuffleX = ray.dir.x >= 0 ? identity : swap;
const ssei shuffleY = ray.dir.y >= 0 ? identity : swap;
const ssei shuffleZ = ray.dir.z >= 0 ? identity : swap;
/*! load the ray into SIMD registers */
const ssei pn = ssei(0x00000000,0x00000000,0x80000000,0x80000000);
const sse3f norg(-ray.org.x,-ray.org.y,-ray.org.z);
const sse3f rdir = sse3f(ssef(ray.rdir.x) ^ pn, ssef(ray.rdir.y) ^ pn, ssef(ray.rdir.z) ^ pn);
ssef nearFar(ray.near, ray.near, -ray.far, -ray.far);
BVH2<Triangle4>::Node* nodes = bvh->nodes;
while (true)
{
/*! this is an inner node */
while (__builtin_expect(cur >= 0, true))
{
/*! Single ray intersection with box of both children. See bvh2.h for node layout. */
const BVH2<Triangle4>::Node& node = bvh->node(nodes,cur);
const ssef tNearFarX = (shuffle8(node.lower_upper_x,shuffleX) + norg.x) * rdir.x;
const ssef tNearFarY = (shuffle8(node.lower_upper_y,shuffleY) + norg.y) * rdir.y;
const ssef tNearFarZ = (shuffle8(node.lower_upper_z,shuffleZ) + norg.z) * rdir.z;
const ssef tNearFar = max(tNearFarX,tNearFarY,tNearFarZ,nearFar) ^ pn;
const sseb lrhit = tNearFar <= shuffle8(tNearFar,swap);
/*! if two children hit, push far node onto stack and continue with closer node */
if (__builtin_expect(lrhit[0] != 0 && lrhit[1] != 0, true)) {
if (tNearFar[0] < tNearFar[1]) { stack[stackPtr++] = node.child[1]; cur = node.child[0]; }
else { stack[stackPtr++] = node.child[0]; cur = node.child[1]; }
}
/*! if one child hit, continue with that child */
else {
if (lrhit[0] != 0) cur = node.child[0];
else if (lrhit[1] != 0) cur = node.child[1];
else goto pop_node;
}
}
/*! leaf node, intersect all triangles */
{
cur ^= 0x80000000;
const size_t ofs = size_t(cur) >> 5;
const size_t num = size_t(cur) & 0x1F;
for (size_t i=ofs; i<ofs+num; i++)
if (bvh->triangles[i].occluded(ray))
return true;
}
/*! pop next node from stack */
pop_node:
if (__builtin_expect(stackPtr == 0, false)) break;
cur = stack[--stackPtr];
}
return false;
}
void BVH4iIntersector1<TriangleIntersector>::intersect(const BVH4iIntersector1* This, Ray& ray)
{
AVX_ZERO_UPPER();
STAT3(normal.travs,1,1,1);
/*! stack state */
const BVH4i* bvh = This->bvh;
StackItem stack[1+3*BVH4i::maxDepth]; //!< stack of nodes
StackItem* stackPtr = stack+1; //!< current stack pointer
stack[0].ptr = bvh->root;
stack[0].dist = neg_inf;
/*! offsets to select the side that becomes the lower or upper bound */
const size_t nearX = ray.dir.x >= 0.0f ? 0*sizeof(ssef_m) : 1*sizeof(ssef_m);
const size_t nearY = ray.dir.y >= 0.0f ? 2*sizeof(ssef_m) : 3*sizeof(ssef_m);
const size_t nearZ = ray.dir.z >= 0.0f ? 4*sizeof(ssef_m) : 5*sizeof(ssef_m);
/*! load the ray into SIMD registers */
const sse3f norg(-ray.org.x,-ray.org.y,-ray.org.z);
const Vector3f ray_rdir = rcp_safe(ray.dir);
const sse3f rdir(ray_rdir.x,ray_rdir.y,ray_rdir.z);
const Vector3f ray_org_rdir = ray.org*ray_rdir;
const sse3f org_rdir(ray_org_rdir.x,ray_org_rdir.y,ray_org_rdir.z);
const ssef rayNear(ray.tnear);
ssef rayFar(ray.tfar);
const void* nodePtr = bvh->nodePtr();
const void* triPtr = bvh->triPtr();
/* pop loop */
while (true) pop:
{
/*! pop next node */
if (unlikely(stackPtr == stack)) break;
stackPtr--;
NodeRef cur = NodeRef(stackPtr->ptr);
/*! if popped node is too far, pop next one */
if (unlikely(stackPtr->dist > ray.tfar))
continue;
/* downtraversal loop */
while (true)
{
/*! stop if we found a leaf */
if (unlikely(cur.isLeaf())) break;
STAT3(normal.trav_nodes,1,1,1);
/*! single ray intersection with 4 boxes */
const Node* node = cur.node(nodePtr);
const size_t farX = nearX ^ 16, farY = nearY ^ 16, farZ = nearZ ^ 16;
#if defined (__AVX2__)
const ssef tNearX = msub(ssef((const char*)nodePtr+(size_t)cur+nearX), rdir.x, org_rdir.x);
const ssef tNearY = msub(ssef((const char*)nodePtr+(size_t)cur+nearY), rdir.y, org_rdir.y);
const ssef tNearZ = msub(ssef((const char*)nodePtr+(size_t)cur+nearZ), rdir.z, org_rdir.z);
const ssef tFarX = msub(ssef((const char*)nodePtr+(size_t)cur+farX ), rdir.x, org_rdir.x);
const ssef tFarY = msub(ssef((const char*)nodePtr+(size_t)cur+farY ), rdir.y, org_rdir.y);
const ssef tFarZ = msub(ssef((const char*)nodePtr+(size_t)cur+farZ ), rdir.z, org_rdir.z);
#else
const ssef tNearX = (norg.x + ssef((const char*)nodePtr+(size_t)cur+nearX)) * rdir.x;
const ssef tNearY = (norg.y + ssef((const char*)nodePtr+(size_t)cur+nearY)) * rdir.y;
const ssef tNearZ = (norg.z + ssef((const char*)nodePtr+(size_t)cur+nearZ)) * rdir.z;
const ssef tFarX = (norg.x + ssef((const char*)nodePtr+(size_t)cur+farX )) * rdir.x;
const ssef tFarY = (norg.y + ssef((const char*)nodePtr+(size_t)cur+farY )) * rdir.y;
const ssef tFarZ = (norg.z + ssef((const char*)nodePtr+(size_t)cur+farZ )) * rdir.z;
#endif
const ssef tNear = max(tNearX,tNearY,tNearZ,rayNear);
const ssef tFar = min(tFarX ,tFarY ,tFarZ ,rayFar);
size_t mask = movemask(tNear <= tFar);
/*! if no child is hit, pop next node */
if (unlikely(mask == 0))
goto pop;
/*! one child is hit, continue with that child */
size_t r = __bsf(mask); mask = __btc(mask,r);
if (likely(mask == 0)) {
cur = node->child(r);
continue;
}
/*! two children are hit, push far child, and continue with closer child */
NodeRef c0 = node->child(r); const float d0 = tNear[r];
r = __bsf(mask); mask = __btc(mask,r);
NodeRef c1 = node->child(r); const float d1 = tNear[r];
if (likely(mask == 0)) {
if (d0 < d1) { stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; cur = c0; continue; }
else { stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; cur = c1; continue; }
}
/*! Here starts the slow path for 3 or 4 hit children. We push
* all nodes onto the stack to sort them there. */
stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++;
stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++;
/*! three children are hit, push all onto stack and sort 3 stack items, continue with closest child */
r = __bsf(mask); mask = __btc(mask,r);
NodeRef c = node->child(r); float d = tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++;
if (likely(mask == 0)) {
sort(stackPtr[-1],stackPtr[-2],stackPtr[-3]);
cur = (NodeRef) stackPtr[-1].ptr; stackPtr--;
continue;
}
/*! four children are hit, push all onto stack and sort 4 stack items, continue with closest child */
r = __bsf(mask); mask = __btc(mask,r);
c = node->child(r); d = tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++;
sort(stackPtr[-1],stackPtr[-2],stackPtr[-3],stackPtr[-4]);
cur = (NodeRef) stackPtr[-1].ptr; stackPtr--;
}
/*! this is a leaf node */
STAT3(normal.trav_leaves,1,1,1);
size_t num; Triangle* tri = (Triangle*) cur.leaf(triPtr,num);
for (size_t i=0; i<num; i++)
TriangleIntersector::intersect(ray,tri[i],bvh->vertices);
rayFar = ray.tfar;
}
AVX_ZERO_UPPER();
}
bool BVH4iIntersector1<TriangleIntersector>::occluded(const BVH4iIntersector1* This, Ray& ray)
{
AVX_ZERO_UPPER();
STAT3(shadow.travs,1,1,1);
/*! stack state */
const BVH4i* bvh = This->bvh;
NodeRef stack[1+3*BVH4i::maxDepth]; //!< stack of nodes that still need to get traversed
NodeRef* stackPtr = stack+1; //!< current stack pointer
stack[0] = bvh->root;
/*! offsets to select the side that becomes the lower or upper bound */
const size_t nearX = ray.dir.x >= 0 ? 0*sizeof(ssef_m) : 1*sizeof(ssef_m);
const size_t nearY = ray.dir.y >= 0 ? 2*sizeof(ssef_m) : 3*sizeof(ssef_m);
const size_t nearZ = ray.dir.z >= 0 ? 4*sizeof(ssef_m) : 5*sizeof(ssef_m);
/*! load the ray into SIMD registers */
const sse3f norg(-ray.org.x,-ray.org.y,-ray.org.z);
const Vector3f ray_rdir = rcp_safe(ray.dir);
const sse3f rdir(ray_rdir.x,ray_rdir.y,ray_rdir.z);
const Vector3f ray_org_rdir = ray.org*ray_rdir;
const sse3f org_rdir(ray_org_rdir.x,ray_org_rdir.y,ray_org_rdir.z);
const ssef rayNear(ray.tnear);
const ssef rayFar(ray.tfar);
const void* nodePtr = bvh->nodePtr();
const void* triPtr = bvh->triPtr();
/* pop loop */
while (true) pop:
{
/*! pop next node */
if (unlikely(stackPtr == stack)) break;
stackPtr--;
NodeRef cur = (NodeRef) *stackPtr;
/* downtraversal loop */
while (true)
{
/*! stop if we found a leaf */
if (unlikely(cur.isLeaf())) break;
STAT3(shadow.trav_nodes,1,1,1);
/*! single ray intersection with 4 boxes */
const Node* node = cur.node(nodePtr);
const size_t farX = nearX ^ 16, farY = nearY ^ 16, farZ = nearZ ^ 16;
#if defined (__AVX2__)
const ssef tNearX = msub(ssef((const char*)nodePtr+(size_t)cur+nearX), rdir.x, org_rdir.x);
const ssef tNearY = msub(ssef((const char*)nodePtr+(size_t)cur+nearY), rdir.y, org_rdir.y);
const ssef tNearZ = msub(ssef((const char*)nodePtr+(size_t)cur+nearZ), rdir.z, org_rdir.z);
const ssef tFarX = msub(ssef((const char*)nodePtr+(size_t)cur+farX ), rdir.x, org_rdir.x);
const ssef tFarY = msub(ssef((const char*)nodePtr+(size_t)cur+farY ), rdir.y, org_rdir.y);
const ssef tFarZ = msub(ssef((const char*)nodePtr+(size_t)cur+farZ ), rdir.z, org_rdir.z);
#else
const ssef tNearX = (norg.x + ssef((const char*)nodePtr+(size_t)cur+nearX)) * rdir.x;
const ssef tNearY = (norg.y + ssef((const char*)nodePtr+(size_t)cur+nearY)) * rdir.y;
const ssef tNearZ = (norg.z + ssef((const char*)nodePtr+(size_t)cur+nearZ)) * rdir.z;
const ssef tFarX = (norg.x + ssef((const char*)nodePtr+(size_t)cur+farX )) * rdir.x;
const ssef tFarY = (norg.y + ssef((const char*)nodePtr+(size_t)cur+farY )) * rdir.y;
const ssef tFarZ = (norg.z + ssef((const char*)nodePtr+(size_t)cur+farZ )) * rdir.z;
#endif
const ssef tNear = max(tNearX,tNearY,tNearZ,rayNear);
const ssef tFar = min(tFarX ,tFarY ,tFarZ ,rayFar);
size_t mask = movemask(tNear <= tFar);
/*! if no child is hit, pop next node */
if (unlikely(mask == 0))
goto pop;
/*! one child is hit, continue with that child */
size_t r = __bsf(mask); mask = __btc(mask,r);
if (likely(mask == 0)) {
cur = node->child(r);
continue;
}
/*! two children are hit, push far child, and continue with closer child */
NodeRef c0 = node->child(r); const float d0 = tNear[r];
r = __bsf(mask); mask = __btc(mask,r);
NodeRef c1 = node->child(r); const float d1 = tNear[r];
if (likely(mask == 0)) {
if (d0 < d1) { *stackPtr = c1; stackPtr++; cur = c0; continue; }
else { *stackPtr = c0; stackPtr++; cur = c1; continue; }
}
*stackPtr = c0; stackPtr++;
*stackPtr = c1; stackPtr++;
/*! three children are hit */
r = __bsf(mask); mask = __btc(mask,r);
cur = node->child(r); *stackPtr = cur; stackPtr++;
if (likely(mask == 0)) {
stackPtr--;
continue;
}
/*! four children are hit */
cur = node->child(3);
}
/*! this is a leaf node */
STAT3(shadow.trav_leaves,1,1,1);
size_t num; Triangle* tri = (Triangle*) cur.leaf(triPtr,num);
for (size_t i=0; i<num; i++) {
if (TriangleIntersector::occluded(ray,tri[i],bvh->vertices)) {
AVX_ZERO_UPPER();
return true;
}
}
}
AVX_ZERO_UPPER();
return false;
}
void BVH4MBIntersector1<TriangleIntersector>::intersect(const BVH4MB* bvh, Ray& ray)
{
AVX_ZERO_UPPER();
STAT3(normal.travs,1,1,1);
/*! stack state */
Base* popCur = bvh->root; //!< pre-popped top node from the stack
float popDist = neg_inf; //!< pre-popped distance of top node from the stack
StackItem stack[1+3*BVH4MB::maxDepth]; //!< stack of nodes that still need to get traversed
StackItem* stackPtr = stack+1; //!< current stack pointer
/*! offsets to select the side that becomes the lower or upper bound */
const size_t nearX = ray.dir.x >= 0 ? 0*2*sizeof(ssef) : 1*2*sizeof(ssef);
const size_t nearY = ray.dir.y >= 0 ? 2*2*sizeof(ssef) : 3*2*sizeof(ssef);
const size_t nearZ = ray.dir.z >= 0 ? 4*2*sizeof(ssef) : 5*2*sizeof(ssef);
const size_t farX = nearX ^ 32;
const size_t farY = nearY ^ 32;
const size_t farZ = nearZ ^ 32;
/*! load the ray into SIMD registers */
const sse3f norg(-ray.org.x,-ray.org.y,-ray.org.z);
const Vec3fa ray_rdir = rcp_safe(ray.dir);
const sse3f rdir(ray_rdir.x,ray_rdir.y,ray_rdir.z);
const ssef rayNear(ray.tnear);
ssef rayFar(ray.tfar);
while (true)
{
/*! pop next node */
if (unlikely(stackPtr == stack)) break;
stackPtr--;
Base* cur = popCur;
/*! if popped node is too far, pop next one */
if (unlikely(popDist > ray.tfar)) {
popCur = (Base*)stackPtr[-1].ptr;
popDist = stackPtr[-1].dist;
continue;
}
next:
/*! we mostly go into the inner node case */
if (likely(cur->isNode()))
{
STAT3(normal.trav_nodes,1,1,1);
/*! single ray intersection with 4 boxes */
const Node* node = cur->node();
const ssef* pNearX = (const ssef*)((const char*)node+nearX);
const ssef* pNearY = (const ssef*)((const char*)node+nearY);
const ssef* pNearZ = (const ssef*)((const char*)node+nearZ);
const ssef tNearX = (norg.x + ssef(pNearX[0]) + ray.time*pNearX[1]) * rdir.x;
const ssef tNearY = (norg.y + ssef(pNearY[0]) + ray.time*pNearY[1]) * rdir.y;
const ssef tNearZ = (norg.z + ssef(pNearZ[0]) + ray.time*pNearZ[1]) * rdir.z;
const ssef tNear = max(tNearX,tNearY,tNearZ,rayNear);
const ssef* pFarX = (const ssef*)((const char*)node+farX);
const ssef* pFarY = (const ssef*)((const char*)node+farY);
const ssef* pFarZ = (const ssef*)((const char*)node+farZ);
const ssef tFarX = (norg.x + ssef(pFarX[0]) + ray.time*pFarX[1]) * rdir.x;
const ssef tFarY = (norg.y + ssef(pFarY[0]) + ray.time*pFarY[1]) * rdir.y;
const ssef tFarZ = (norg.z + ssef(pFarZ[0]) + ray.time*pFarZ[1]) * rdir.z;
popCur = (Base*) stackPtr[-1].ptr; //!< pre-pop of topmost stack item
popDist = stackPtr[-1].dist; //!< pre-pop of distance of topmost stack item
const ssef tFar = min(tFarX,tFarY,tFarZ,rayFar);
size_t _hit = movemask(tNear <= tFar);
/*! if no child is hit, pop next node */
if (unlikely(_hit == 0))
continue;
/*! one child is hit, continue with that child */
size_t r = __bsf(_hit); _hit = __btc(_hit,r);
if (likely(_hit == 0)) {
cur = node->child[r];
goto next;
}
/*! two children are hit, push far child, and continue with closer child */
Base* c0 = node->child[r]; const float d0 = tNear[r];
r = __bsf(_hit); _hit = __btc(_hit,r);
Base* c1 = node->child[r]; const float d1 = tNear[r];
if (likely(_hit == 0)) {
if (d0 < d1) { stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; cur = c0; goto next; }
else { stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; cur = c1; goto next; }
}
/*! Here starts the slow path for 3 or 4 hit children. We push
* all nodes onto the stack to sort them there. */
stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++;
stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++;
/*! three children are hit, push all onto stack and sort 3 stack items, continue with closest child */
r = __bsf(_hit); _hit = __btc(_hit,r);
Base* c = node->child[r]; float d = tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++;
if (likely(_hit == 0)) {
sort(stackPtr[-1],stackPtr[-2],stackPtr[-3]);
cur = (Base*) stackPtr[-1].ptr; stackPtr--;
goto next;
}
/*! four children are hit, push all onto stack and sort 4 stack items, continue with closest child */
r = __bsf(_hit); _hit = __btc(_hit,r);
c = node->child[r]; d = tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++;
sort(stackPtr[-1],stackPtr[-2],stackPtr[-3],stackPtr[-4]);
cur = (Base*) stackPtr[-1].ptr; stackPtr--;
goto next;
}
/*! this is a leaf node */
else
{
STAT3(normal.trav_leaves,1,1,1);
size_t num; Triangle* tri = (Triangle*) cur->leaf(num);
for (size_t i=0; i<num; i++)
TriangleIntersector::intersect(ray,tri[i],bvh->geometry);
popCur = (Base*) stackPtr[-1].ptr; //!< pre-pop of topmost stack item
popDist = stackPtr[-1].dist; //!< pre-pop of distance of topmost stack item
rayFar = ray.tfar;
}
}
AVX_ZERO_UPPER();
}
void BVH4Intersector4Chunk<TriangleIntersector4>::intersect(const BVH4Intersector4Chunk* This, Ray4& ray, const __m128 valid_i)
{
sseb valid = valid_i;
NodeRef invalid = (NodeRef)1;
const BVH4* bvh = This->bvh;
STAT3(normal.travs,1,popcnt(valid),4);
/* load ray into registers */
ssef ray_near = select(valid,ray.tnear,pos_inf);
ssef ray_far = select(valid,ray.tfar ,neg_inf);
sse3f rdir = rcp_safe(ray.dir);
ray.tfar = ray_far;
/* allocate stack and push root node */
NodeRef stack_node[3*BVH4::maxDepth+1];
ssef stack_near[3*BVH4::maxDepth+1];
stack_node[0] = invalid;
stack_near[0] = ssef(inf);
stack_node[1] = bvh->root;
stack_near[1] = ray_near;
NodeRef* sptr_node = stack_node+2;
ssef * sptr_near = stack_near+2;
while (1)
{
/* pop next node from stack */
sptr_node--;
sptr_near--;
ssef curDist = *sptr_near;
NodeRef curNode = *sptr_node;
if (unlikely(curNode == invalid))
break;
/* cull node if behind closest hit point */
const sseb m_dist = curDist < ray_far;
if (unlikely(none(m_dist)))
continue;
while (1)
{
/* test if this is a leaf node */
if (unlikely(curNode.isLeaf()))
break;
STAT3(normal.trav_nodes,1,popcnt(valid),4);
const Node* const node = curNode.node(bvh->nodePtr()); //NodeRef(curNode).node(nodes);
//prefetch<PFHINT_L1>((ssef*)node + 1); // depth first order prefetch
/* pop of next node */
sptr_node--;
sptr_near--;
curNode = *sptr_node;
curDist = *sptr_near;
for (unsigned i=0;i<4;i++)
{
const ssef dminx = (ssef(node->lower_x[i]) - ray.org.x) * rdir.x;
const ssef dmaxx = (ssef(node->upper_x[i]) - ray.org.x) * rdir.x;
const ssef dminy = (ssef(node->lower_y[i]) - ray.org.y) * rdir.y;
const ssef dmaxy = (ssef(node->upper_y[i]) - ray.org.y) * rdir.y;
const ssef dminz = (ssef(node->lower_z[i]) - ray.org.z) * rdir.z;
const ssef dmaxz = (ssef(node->upper_z[i]) - ray.org.z) * rdir.z;
const ssef dlowerx = min(dminx,dmaxx);
const ssef dupperx = max(dminx,dmaxx);
const ssef dlowery = min(dminy,dmaxy);
const ssef duppery = max(dminy,dmaxy);
const ssef dlowerz = min(dminz,dmaxz);
const ssef dupperz = max(dminz,dmaxz);
const ssef near = max(dlowerx,dlowery,dlowerz,ray_near);
const ssef far = min(dupperx,duppery,dupperz,ray_far );
const sseb mhit = near <= far;
const ssef childDist = select(mhit,near,inf);
const sseb closer = childDist < curDist;
/* if we hit the child we choose to continue with that child if it
is closer than the current next child, or we push it onto the stack */
if (likely(any(mhit)))
{
const NodeRef child = node->child(i);
//if (child != invalid)
{
sptr_node++;
sptr_near++;
/* push cur node onto stack and continue with hit child */
if (any(closer)) {
*(sptr_node-1) = curNode;
*(sptr_near-1) = curDist;
curDist = childDist;
curNode = child;
}
/* push hit child onto stack*/
else {
*(sptr_node-1) = child;
*(sptr_near-1) = childDist;
}
}
}
}
}
/* return if stack is empty */
if (unlikely(curNode == invalid))
break;
/* decode leaf node */
size_t num;
STAT3(normal.trav_leaves,1,popcnt(valid),4);
Triangle* tri = (Triangle*) curNode.leaf(bvh->triPtr(),num);
/* intersect triangles */
for (size_t i=0; i<num; i++)
TriangleIntersector4::intersect(valid,ray,tri[i],bvh->vertices);
ray_far = ray.tfar;
}
}
void BVH4iIntersector4Chunk<TriangleIntersector4>::intersect(sseb* valid_i, BVH4i* bvh, Ray4& ray)
{
/* load node and primitive array */
const Node * __restrict__ nodes = (Node *)bvh->nodePtr();
const Triangle * __restrict__ accel = (Triangle*)bvh->triPtr();
/* load ray */
const sseb valid0 = *valid_i;
const sse3f rdir = rcp_safe(ray.dir);
const sse3f org_rdir = ray.org * rdir;
ssef ray_tnear = select(valid0,ray.tnear,pos_inf);
ssef ray_tfar = select(valid0,ray.tfar ,neg_inf);
const ssef inf = ssef(pos_inf);
/* allocate stack and push root node */
ssef stack_near[3*BVH4i::maxDepth+1];
NodeRef stack_node[3*BVH4i::maxDepth+1];
stack_node[0] = BVH4i::invalidNode;
stack_near[0] = inf;
stack_node[1] = bvh->root;
stack_near[1] = ray_tnear;
NodeRef* __restrict__ sptr_node = stack_node + 2;
ssef* __restrict__ sptr_near = stack_near + 2;
while (1)
{
/* pop next node from stack */
sptr_node--;
sptr_near--;
NodeRef curNode = *sptr_node;
if (unlikely(curNode == BVH4i::invalidNode))
break;
/* cull node if behind closest hit point */
ssef curDist = *sptr_near;
if (unlikely(none(ray_tfar > curDist)))
continue;
while (1)
{
/* test if this is a leaf node */
if (unlikely(curNode.isLeaf()))
break;
const sseb valid_node = ray_tfar > curDist;
STAT3(normal.trav_nodes,1,popcnt(valid_node),4);
const Node* __restrict__ const node = curNode.node(nodes);
/* pop of next node */
sptr_node--;
sptr_near--;
curNode = *sptr_node; // FIXME: this trick creates issues with stack depth
curDist = *sptr_near;
#pragma unroll(4)
for (unsigned i=0; i<4; i++)
{
const NodeRef child = node->children[i];
if (unlikely(child == BVH4i::emptyNode)) break;
#if defined(__AVX2__)
const ssef lclipMinX = msub(node->lower_x[i],rdir.x,org_rdir.x);
const ssef lclipMinY = msub(node->lower_y[i],rdir.y,org_rdir.y);
const ssef lclipMinZ = msub(node->lower_z[i],rdir.z,org_rdir.z);
const ssef lclipMaxX = msub(node->upper_x[i],rdir.x,org_rdir.x);
const ssef lclipMaxY = msub(node->upper_y[i],rdir.y,org_rdir.y);
const ssef lclipMaxZ = msub(node->upper_z[i],rdir.z,org_rdir.z);
const ssef lnearP = maxi(maxi(mini(lclipMinX, lclipMaxX), mini(lclipMinY, lclipMaxY)), mini(lclipMinZ, lclipMaxZ));
const ssef lfarP = mini(mini(maxi(lclipMinX, lclipMaxX), maxi(lclipMinY, lclipMaxY)), maxi(lclipMinZ, lclipMaxZ));
const sseb lhit = maxi(lnearP,ray_tnear) <= mini(lfarP,ray_tfar);
#else
const ssef lclipMinX = node->lower_x[i] * rdir.x - org_rdir.x;
const ssef lclipMinY = node->lower_y[i] * rdir.y - org_rdir.y;
const ssef lclipMinZ = node->lower_z[i] * rdir.z - org_rdir.z;
const ssef lclipMaxX = node->upper_x[i] * rdir.x - org_rdir.x;
const ssef lclipMaxY = node->upper_y[i] * rdir.y - org_rdir.y;
const ssef lclipMaxZ = node->upper_z[i] * rdir.z - org_rdir.z;
const ssef lnearP = max(max(min(lclipMinX, lclipMaxX), min(lclipMinY, lclipMaxY)), min(lclipMinZ, lclipMaxZ));
const ssef lfarP = min(min(max(lclipMinX, lclipMaxX), max(lclipMinY, lclipMaxY)), max(lclipMinZ, lclipMaxZ));
const sseb lhit = max(lnearP,ray_tnear) <= min(lfarP,ray_tfar);
#endif
/* if we hit the child we choose to continue with that child if it
is closer than the current next child, or we push it onto the stack */
if (likely(any(lhit)))
{
const ssef childDist = select(lhit,lnearP,inf);
sptr_node++;
sptr_near++;
/* push cur node onto stack and continue with hit child */
if (any(childDist < curDist))
{
*(sptr_node-1) = curNode;
*(sptr_near-1) = curDist;
curDist = childDist;
curNode = child;
}
/* push hit child onto stack */
else {
*(sptr_node-1) = child;
*(sptr_near-1) = childDist;
}
assert(sptr_node - stack_node < BVH4i::maxDepth);
}
}
}
/* return if stack is empty */
if (unlikely(curNode == BVH4i::invalidNode))
break;
/* intersect leaf */
const sseb valid_leaf = ray_tfar > curDist;
STAT3(normal.trav_leaves,1,popcnt(valid_leaf),4);
size_t items; const Triangle* tri = (Triangle*) curNode.leaf(accel, items);
TriangleIntersector4::intersect(valid_leaf,ray,tri,items,bvh->geometry);
ray_tfar = select(valid_leaf,ray.tfar,ray_tfar);
}
AVX_ZERO_UPPER();
}
void BVH4Intersector4Chunk<types,robust,PrimitiveIntersector4>::intersect(sseb* valid_i, BVH4* bvh, Ray4& ray)
{
/* load ray */
const sseb valid0 = *valid_i;
const sse3f rdir = rcp_safe(ray.dir);
const sse3f org(ray.org), org_rdir = org * rdir;
ssef ray_tnear = select(valid0,ray.tnear,ssef(pos_inf));
ssef ray_tfar = select(valid0,ray.tfar ,ssef(neg_inf));
const ssef inf = ssef(pos_inf);
Precalculations pre(valid0,ray);
/* allocate stack and push root node */
ssef stack_near[stackSize];
NodeRef stack_node[stackSize];
stack_node[0] = BVH4::invalidNode;
stack_near[0] = inf;
stack_node[1] = bvh->root;
stack_near[1] = ray_tnear;
NodeRef* stackEnd = stack_node+stackSize;
NodeRef* __restrict__ sptr_node = stack_node + 2;
ssef* __restrict__ sptr_near = stack_near + 2;
while (1)
{
/* pop next node from stack */
assert(sptr_node > stack_node);
sptr_node--;
sptr_near--;
NodeRef cur = *sptr_node;
if (unlikely(cur == BVH4::invalidNode)) {
assert(sptr_node == stack_node);
break;
}
/* cull node if behind closest hit point */
ssef curDist = *sptr_near;
if (unlikely(none(ray_tfar > curDist)))
continue;
while (1)
{
/* process normal nodes */
if (likely((types & 0x1) && cur.isNode()))
{
const sseb valid_node = ray_tfar > curDist;
STAT3(normal.trav_nodes,1,popcnt(valid_node),8);
const Node* __restrict__ const node = cur.node();
/* pop of next node */
assert(sptr_node > stack_node);
sptr_node--;
sptr_near--;
cur = *sptr_node;
curDist = *sptr_near;
#pragma unroll(4)
for (unsigned i=0; i<BVH4::N; i++)
{
const NodeRef child = node->children[i];
if (unlikely(child == BVH4::emptyNode)) break;
ssef lnearP; const sseb lhit = node->intersect<robust>(i,org,rdir,org_rdir,ray_tnear,ray_tfar,lnearP);
/* if we hit the child we choose to continue with that child if it
is closer than the current next child, or we push it onto the stack */
if (likely(any(lhit)))
{
assert(sptr_node < stackEnd);
const ssef childDist = select(lhit,lnearP,inf);
const NodeRef child = node->children[i];
assert(child != BVH4::emptyNode);
sptr_node++;
sptr_near++;
/* push cur node onto stack and continue with hit child */
if (any(childDist < curDist))
{
*(sptr_node-1) = cur;
*(sptr_near-1) = curDist;
curDist = childDist;
cur = child;
}
/* push hit child onto stack */
else {
*(sptr_node-1) = child;
*(sptr_near-1) = childDist;
}
}
}
}
/* process motion blur nodes */
else if (likely((types & 0x10) && cur.isNodeMB()))
{
const sseb valid_node = ray_tfar > curDist;
STAT3(normal.trav_nodes,1,popcnt(valid_node),8);
const BVH4::NodeMB* __restrict__ const node = cur.nodeMB();
/* pop of next node */
assert(sptr_node > stack_node);
sptr_node--;
sptr_near--;
cur = *sptr_node;
curDist = *sptr_near;
#pragma unroll(4)
for (unsigned i=0; i<BVH4::N; i++)
{
const NodeRef child = node->child(i);
if (unlikely(child == BVH4::emptyNode)) break;
ssef lnearP; const sseb lhit = node->intersect(i,org,rdir,org_rdir,ray_tnear,ray_tfar,ray.time,lnearP);
/* if we hit the child we choose to continue with that child if it
is closer than the current next child, or we push it onto the stack */
if (likely(any(lhit)))
{
assert(sptr_node < stackEnd);
assert(child != BVH4::emptyNode);
const ssef childDist = select(lhit,lnearP,inf);
sptr_node++;
sptr_near++;
/* push cur node onto stack and continue with hit child */
if (any(childDist < curDist))
{
*(sptr_node-1) = cur;
*(sptr_near-1) = curDist;
curDist = childDist;
cur = child;
}
/* push hit child onto stack */
else {
*(sptr_node-1) = child;
*(sptr_near-1) = childDist;
}
}
}
}
else
break;
}
__forceinline void intersectT(const BVH4* bvh, Ray& ray)
{
typedef typename TriangleIntersector::Triangle Triangle;
typedef StackItemT<size_t> StackItem;
typedef typename BVH4::NodeRef NodeRef;
typedef typename BVH4::Node Node;
/*! stack state */
StackItem stack[1+3*BVH4::maxDepth]; //!< stack of nodes
StackItem* stackPtr = stack+1; //!< current stack pointer
stack[0].ptr = bvh->root;
stack[0].dist = neg_inf;
/*! load the ray into SIMD registers */
const avxf pos_neg = avxf(ssef(+0.0f),ssef(-0.0f));
const avxf neg_pos = avxf(ssef(-0.0f),ssef(+0.0f));
const avxf flipSignX = swapX ? neg_pos : pos_neg;
const avxf flipSignY = swapY ? neg_pos : pos_neg;
const avxf flipSignZ = swapZ ? neg_pos : pos_neg;
const Vector3f ray_rdir = rcp_safe(ray.dir);
const avx3f norg(-ray.org.x,-ray.org.y,-ray.org.z);
const avx3f rdir(ray_rdir.x^flipSignX,ray_rdir.y^flipSignY,ray_rdir.z^flipSignZ);
const avx3f org_rdir(avx3f(ray.org.x,ray.org.y,ray.org.z)*rdir);
avxf rayNearFar(ssef(ray.tnear),-ssef(ray.tfar));
const void* nodePtr = bvh->nodePtr();
const void* triPtr = bvh->triPtr();
/* pop loop */
while (true) pop:
{
/*! pop next node */
if (unlikely(stackPtr == stack)) break;
stackPtr--;
NodeRef cur = NodeRef(stackPtr->ptr);
/*! if popped node is too far, pop next one */
if (unlikely(stackPtr->dist > ray.tfar))
continue;
/* downtraversal loop */
while (true)
{
/*! stop if we found a leaf */
if (unlikely(cur.isLeaf())) break;
STAT3(normal.trav_nodes,1,1,1);
/*! single ray intersection with 4 boxes */
const Node* node = cur.node(nodePtr);
#if defined (__AVX2__) || defined(__MIC__)
const avxf tLowerUpperX = msub(avxf::load(&node->lower_x), rdir.x, org_rdir.x);
const avxf tLowerUpperY = msub(avxf::load(&node->lower_y), rdir.y, org_rdir.y);
const avxf tLowerUpperZ = msub(avxf::load(&node->lower_z), rdir.z, org_rdir.z);
#else
const avxf tLowerUpperX = (norg.x + avxf::load(&node->lower_x)) * rdir.x;
const avxf tLowerUpperY = (norg.y + avxf::load(&node->lower_y)) * rdir.y;
const avxf tLowerUpperZ = (norg.z + avxf::load(&node->lower_z)) * rdir.z;
#endif
const avxf tNearFarX = swapX ? shuffle<1,0>(tLowerUpperX) : tLowerUpperX;
const avxf tNearFarY = swapY ? shuffle<1,0>(tLowerUpperY) : tLowerUpperY;
const avxf tNearFarZ = swapZ ? shuffle<1,0>(tLowerUpperZ) : tLowerUpperZ;
const avxf tNearFar = max(tNearFarX,tNearFarY,tNearFarZ,rayNearFar);
const ssef tNear = extract<0>(tNearFar);
const ssef tFar = extract<1>(tNearFar);
size_t mask = movemask(-tNear >= tFar);
/*! if no child is hit, pop next node */
if (unlikely(mask == 0))
goto pop;
/*! one child is hit, continue with that child */
size_t r = __bsf(mask); mask = __btc(mask,r);
if (likely(mask == 0)) {
cur = node->child(r);
continue;
}
/*! two children are hit, push far child, and continue with closer child */
NodeRef c0 = node->child(r); const float d0 = tNear[r];
r = __bsf(mask); mask = __btc(mask,r);
NodeRef c1 = node->child(r); const float d1 = tNear[r];
if (likely(mask == 0)) {
if (d0 < d1) { stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; cur = c0; continue; }
else { stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; cur = c1; continue; }
}
/*! Here starts the slow path for 3 or 4 hit children. We push
* all nodes onto the stack to sort them there. */
stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++;
stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++;
/*! three children are hit, push all onto stack and sort 3 stack items, continue with closest child */
r = __bsf(mask); mask = __btc(mask,r);
NodeRef c = node->child(r); float d = tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++;
if (likely(mask == 0)) {
sort(stackPtr[-1],stackPtr[-2],stackPtr[-3]);
cur = (NodeRef) stackPtr[-1].ptr; stackPtr--;
continue;
}
/*! four children are hit, push all onto stack and sort 4 stack items, continue with closest child */
r = __bsf(mask); mask = __btc(mask,r);
c = node->child(r); d = tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++;
sort(stackPtr[-1],stackPtr[-2],stackPtr[-3],stackPtr[-4]);
cur = (NodeRef) stackPtr[-1].ptr; stackPtr--;
}
/*! this is a leaf node */
STAT3(normal.trav_leaves,1,1,1);
size_t num; Triangle* tri = (Triangle*) cur.leaf(triPtr,num);
for (size_t i=0; i<num; i++)
TriangleIntersector::intersect(ray,tri[i],bvh->vertices);
rayNearFar = insert<1>(rayNearFar,-ssef(ray.tfar));
}
}
size_t BVH4MB::rotate(Base* nodeID, size_t depth)
{
/*! nothing to rotate if we reached a leaf node. */
if (nodeID->isLeaf()) return 0;
Node* parent = nodeID->node();
/*! rotate all children first */
ssei cdepth;
for (size_t c=0; c<4; c++)
cdepth[c] = (int)rotate(parent->child[c],depth+1);
/* compute current area of all children */
ssef sizeX = parent->upper_x-parent->lower_x;
ssef sizeY = parent->upper_y-parent->lower_y;
ssef sizeZ = parent->upper_z-parent->lower_z;
ssef childArea = sizeX*(sizeY + sizeZ) + sizeY*sizeZ;
/*! transpose node bounds */
ssef plower0,plower1,plower2,plower3; transpose(parent->lower_x,parent->lower_y,parent->lower_z,ssef(zero),plower0,plower1,plower2,plower3);
ssef pupper0,pupper1,pupper2,pupper3; transpose(parent->upper_x,parent->upper_y,parent->upper_z,ssef(zero),pupper0,pupper1,pupper2,pupper3);
BBox<ssef> other0(plower0,pupper0), other1(plower1,pupper1), other2(plower2,pupper2), other3(plower3,pupper3);
/*! Find best rotation. We pick a target child of a first child,
and swap this with an other child. We perform the best such
swap. */
float bestCost = pos_inf;
int bestChild = -1, bestTarget = -1, bestOther = -1;
for (size_t c=0; c<4; c++)
{
/*! ignore leaf nodes as we cannot descent into */
if (parent->child[c]->isLeaf()) continue;
Node* child = parent->child[c]->node();
/*! transpose child bounds */
ssef clower0,clower1,clower2,clower3; transpose(child->lower_x,child->lower_y,child->lower_z,ssef(zero),clower0,clower1,clower2,clower3);
ssef cupper0,cupper1,cupper2,cupper3; transpose(child->upper_x,child->upper_y,child->upper_z,ssef(zero),cupper0,cupper1,cupper2,cupper3);
BBox<ssef> target0(clower0,cupper0), target1(clower1,cupper1), target2(clower2,cupper2), target3(clower3,cupper3);
/*! put other0 at each target position */
float cost00 = halfArea3f(merge(other0 ,target1,target2,target3));
float cost01 = halfArea3f(merge(target0,other0 ,target2,target3));
float cost02 = halfArea3f(merge(target0,target1,other0 ,target3));
float cost03 = halfArea3f(merge(target0,target1,target2,other0 ));
ssef cost0 = ssef(cost00,cost01,cost02,cost03);
ssef min0 = vreduce_min(cost0);
int pos0 = (int)__bsf(movemask(min0 == cost0));
/*! put other1 at each target position */
float cost10 = halfArea3f(merge(other1 ,target1,target2,target3));
float cost11 = halfArea3f(merge(target0,other1 ,target2,target3));
float cost12 = halfArea3f(merge(target0,target1,other1 ,target3));
float cost13 = halfArea3f(merge(target0,target1,target2,other1 ));
ssef cost1 = ssef(cost10,cost11,cost12,cost13);
ssef min1 = vreduce_min(cost1);
int pos1 = (int)__bsf(movemask(min1 == cost1));
/*! put other2 at each target position */
float cost20 = halfArea3f(merge(other2 ,target1,target2,target3));
float cost21 = halfArea3f(merge(target0,other2 ,target2,target3));
float cost22 = halfArea3f(merge(target0,target1,other2 ,target3));
float cost23 = halfArea3f(merge(target0,target1,target2,other2 ));
ssef cost2 = ssef(cost20,cost21,cost22,cost23);
ssef min2 = vreduce_min(cost2);
int pos2 = (int)__bsf(movemask(min2 == cost2));
/*! put other3 at each target position */
float cost30 = halfArea3f(merge(other3 ,target1,target2,target3));
float cost31 = halfArea3f(merge(target0,other3 ,target2,target3));
float cost32 = halfArea3f(merge(target0,target1,other3 ,target3));
float cost33 = halfArea3f(merge(target0,target1,target2,other3 ));
ssef cost3 = ssef(cost30,cost31,cost32,cost33);
ssef min3 = vreduce_min(cost3);
int pos3 = (int)__bsf(movemask(min3 == cost3));
/*! find best other child */
ssef otherCost = ssef(extract<0>(min0),extract<0>(min1),extract<0>(min2),extract<0>(min3));
int pos[4] = { pos0,pos1,pos2,pos3 };
sseb valid = ssei(int(depth+1))+cdepth <= ssei(maxDepth); // only select swaps that fulfill depth constraints
if (none(valid)) continue;
size_t n = select_min(valid,otherCost);
float cost = otherCost[n]-childArea[c]; //< increasing the original child bound is bad, decreasing good
/*! accept a swap when it reduces cost and is not swapping a node with itself */
if (cost < bestCost && n != c) {
bestCost = cost;
bestChild = (int)c;
bestOther = (int)n;
bestTarget = pos[n];
}
}
/*! if we did not find a swap that improves the SAH then do nothing */
if (bestCost >= 0) return 1+reduce_max(cdepth);
/*! perform the best found tree rotation */
Node* child = parent->child[bestChild]->node();
swap(parent,bestOther,child,bestTarget);
parent->lower_x[bestChild] = reduce_min(child->lower_x);
parent->lower_y[bestChild] = reduce_min(child->lower_y);
parent->lower_z[bestChild] = reduce_min(child->lower_z);
parent->upper_x[bestChild] = reduce_max(child->upper_x);
parent->upper_y[bestChild] = reduce_max(child->upper_y);
parent->upper_z[bestChild] = reduce_max(child->upper_z);
parent->lower_dx[bestChild] = reduce_min(child->lower_dx);
parent->lower_dy[bestChild] = reduce_min(child->lower_dy);
parent->lower_dz[bestChild] = reduce_min(child->lower_dz);
parent->upper_dx[bestChild] = reduce_max(child->upper_dx);
parent->upper_dy[bestChild] = reduce_max(child->upper_dy);
parent->upper_dz[bestChild] = reduce_max(child->upper_dz);
/*! This returned depth is conservative as the child that was
* pulled up in the tree could have been on the critical path. */
cdepth[bestOther]++; // bestOther was pushed down one level
return 1+reduce_max(cdepth);
}
void BVH4Intersector4Hybrid<PrimitiveIntersector4>::occluded(sseb* valid_i, BVH4* bvh, Ray4& ray)
{
/* load ray */
const sseb valid = *valid_i;
sseb terminated = !valid;
sse3f ray_org = ray.org, ray_dir = ray.dir;
ssef ray_tnear = ray.tnear, ray_tfar = ray.tfar;
#if defined(__FIX_RAYS__)
const ssef float_range = 0.1f*FLT_MAX;
ray_org = clamp(ray_org,sse3f(-float_range),sse3f(+float_range));
ray_dir = clamp(ray_dir,sse3f(-float_range),sse3f(+float_range));
ray_tnear = max(ray_tnear,FLT_MIN);
ray_tfar = min(ray_tfar,float(inf));
#endif
const sse3f rdir = rcp_safe(ray_dir);
const sse3f org(ray_org), org_rdir = org * rdir;
ray_tnear = select(valid,ray_tnear,ssef(pos_inf));
ray_tfar = select(valid,ray_tfar ,ssef(neg_inf));
const ssef inf = ssef(pos_inf);
/* allocate stack and push root node */
ssef stack_near[stackSizeChunk];
NodeRef stack_node[stackSizeChunk];
stack_node[0] = BVH4::invalidNode;
stack_near[0] = inf;
stack_node[1] = bvh->root;
stack_near[1] = ray_tnear;
NodeRef* stackEnd = stack_node+stackSizeChunk;
NodeRef* __restrict__ sptr_node = stack_node + 2;
ssef* __restrict__ sptr_near = stack_near + 2;
while (1)
{
/* pop next node from stack */
assert(sptr_node > stack_node);
sptr_node--;
sptr_near--;
NodeRef curNode = *sptr_node;
if (unlikely(curNode == BVH4::invalidNode)) {
assert(sptr_node == stack_node);
break;
}
/* cull node if behind closest hit point */
ssef curDist = *sptr_near;
const sseb active = curDist < ray_tfar;
if (unlikely(none(active)))
continue;
/* switch to single ray traversal */
#if !defined(__WIN32__) || defined(__X86_64__)
size_t bits = movemask(active);
if (unlikely(__popcnt(bits) <= SWITCH_THRESHOLD)) {
for (size_t i=__bsf(bits); bits!=0; bits=__btc(bits,i), i=__bsf(bits)) {
if (occluded1(bvh,curNode,i,ray,ray_org,ray_dir,rdir,ray_tnear,ray_tfar))
terminated[i] = -1;
}
if (all(terminated)) break;
ray_tfar = select(terminated,ssef(neg_inf),ray_tfar);
continue;
}
#endif
while (1)
{
/* test if this is a leaf node */
if (unlikely(curNode.isLeaf()))
break;
const sseb valid_node = ray_tfar > curDist;
STAT3(shadow.trav_nodes,1,popcnt(valid_node),4);
const Node* __restrict__ const node = curNode.node();
/* pop of next node */
assert(sptr_node > stack_node);
sptr_node--;
sptr_near--;
curNode = *sptr_node;
curDist = *sptr_near;
#pragma unroll(4)
for (unsigned i=0; i<4; i++)
{
const NodeRef child = node->children[i];
if (unlikely(child == BVH4::emptyNode)) break;
#if defined(__AVX2__)
const ssef lclipMinX = msub(node->lower_x[i],rdir.x,org_rdir.x);
const ssef lclipMinY = msub(node->lower_y[i],rdir.y,org_rdir.y);
const ssef lclipMinZ = msub(node->lower_z[i],rdir.z,org_rdir.z);
const ssef lclipMaxX = msub(node->upper_x[i],rdir.x,org_rdir.x);
const ssef lclipMaxY = msub(node->upper_y[i],rdir.y,org_rdir.y);
const ssef lclipMaxZ = msub(node->upper_z[i],rdir.z,org_rdir.z);
#else
const ssef lclipMinX = (node->lower_x[i] - org.x) * rdir.x;
const ssef lclipMinY = (node->lower_y[i] - org.y) * rdir.y;
const ssef lclipMinZ = (node->lower_z[i] - org.z) * rdir.z;
const ssef lclipMaxX = (node->upper_x[i] - org.x) * rdir.x;
const ssef lclipMaxY = (node->upper_y[i] - org.y) * rdir.y;
const ssef lclipMaxZ = (node->upper_z[i] - org.z) * rdir.z;
#endif
#if defined(__SSE4_1__)
const ssef lnearP = maxi(maxi(mini(lclipMinX, lclipMaxX), mini(lclipMinY, lclipMaxY)), mini(lclipMinZ, lclipMaxZ));
const ssef lfarP = mini(mini(maxi(lclipMinX, lclipMaxX), maxi(lclipMinY, lclipMaxY)), maxi(lclipMinZ, lclipMaxZ));
const sseb lhit = maxi(lnearP,ray_tnear) <= mini(lfarP,ray_tfar);
#else
const ssef lnearP = max(max(min(lclipMinX, lclipMaxX), min(lclipMinY, lclipMaxY)), min(lclipMinZ, lclipMaxZ));
const ssef lfarP = min(min(max(lclipMinX, lclipMaxX), max(lclipMinY, lclipMaxY)), max(lclipMinZ, lclipMaxZ));
const sseb lhit = max(lnearP,ray_tnear) <= min(lfarP,ray_tfar);
#endif
/* if we hit the child we choose to continue with that child if it
is closer than the current next child, or we push it onto the stack */
if (likely(any(lhit)))
{
assert(sptr_node < stackEnd);
assert(child != BVH4::emptyNode);
const ssef childDist = select(lhit,lnearP,inf);
sptr_node++;
sptr_near++;
/* push cur node onto stack and continue with hit child */
if (any(childDist < curDist))
{
*(sptr_node-1) = curNode;
*(sptr_near-1) = curDist;
curDist = childDist;
curNode = child;
}
/* push hit child onto stack */
else {
*(sptr_node-1) = child;
*(sptr_near-1) = childDist;
}
}
}
}
/* return if stack is empty */
if (unlikely(curNode == BVH4::invalidNode)) {
assert(sptr_node == stack_node);
break;
}
/* intersect leaf */
const sseb valid_leaf = ray_tfar > curDist;
STAT3(shadow.trav_leaves,1,popcnt(valid_leaf),4);
size_t items; const Primitive* prim = (Primitive*) curNode.leaf(items);
terminated |= PrimitiveIntersector4::occluded(!terminated,ray,prim,items,bvh->geometry);
if (all(terminated)) break;
ray_tfar = select(terminated,ssef(neg_inf),ray_tfar);
}
store4i(valid & terminated,&ray.geomID,0);
AVX_ZERO_UPPER();
}
void BVH4Intersector1Bezier<PrimitiveIntersector>::intersect(const BVH4* bvh, Ray& ray)
{
/*! perform per ray precalculations required by the primitive intersector */
Precalculations pre(ray);
/*! stack state */
StackItemInt32<NodeRef> stack[stackSize]; //!< stack of nodes
StackItemInt32<NodeRef>* stackPtr = stack+1; //!< current stack pointer
StackItemInt32<NodeRef>* stackEnd = stack+stackSize;
stack[0].ptr = bvh->root;
stack[0].dist = neg_inf;
/*! offsets to select the side that becomes the lower or upper bound */
const size_t nearX = ray.dir.x >= 0.0f ? 0*sizeof(ssef) : 1*sizeof(ssef);
const size_t nearY = ray.dir.y >= 0.0f ? 2*sizeof(ssef) : 3*sizeof(ssef);
const size_t nearZ = ray.dir.z >= 0.0f ? 4*sizeof(ssef) : 5*sizeof(ssef);
/*! load the ray into SIMD registers */
const sse3f norg(-ray.org.x,-ray.org.y,-ray.org.z);
const Vec3fa ray_rdir = rcp_safe(ray.dir);
const sse3f rdir(ray_rdir.x,ray_rdir.y,ray_rdir.z);
const Vec3fa ray_org_rdir = ray.org*ray_rdir;
const sse3f org_rdir(ray_org_rdir.x,ray_org_rdir.y,ray_org_rdir.z);
const ssef ray_near(ray.tnear);
ssef ray_far(ray.tfar);
/* pop loop */
while (true) pop:
{
/*! pop next node */
if (unlikely(stackPtr == stack)) break;
stackPtr--;
NodeRef cur = NodeRef(stackPtr->ptr);
/*! if popped node is too far, pop next one */
if (unlikely(*(float*)&stackPtr->dist > ray.tfar))
continue;
/* downtraversal loop */
while (true)
{
/*! stop if we found a leaf */
if (unlikely(cur.isLeaf())) break;
STAT3(normal.trav_nodes,1,1,1);
/*! single ray intersection with 4 boxes */
const Node* node = cur.node();
const size_t farX = nearX ^ 16, farY = nearY ^ 16, farZ = nearZ ^ 16;
const ssef tFarX0 = abs((norg.x + load4f((const char*)node+farX )) * rdir.x);
const ssef tFarY0 = abs((norg.y + load4f((const char*)node+farY )) * rdir.y);
const ssef tFarZ0 = abs((norg.z + load4f((const char*)node+farZ )) * rdir.z);
const ssef tFar0 = min(tFarX0 ,tFarY0 ,tFarZ0);
const ssef radius = abs(ssef(ray.org.w) + tFar0 * ssef(ray.dir.w));
//const ssef radius = zero;
//PRINT2(tFar0,radius);
const ssef tLowerX = (norg.x + node->lower_x - radius) * rdir.x;
const ssef tLowerY = (norg.y + node->lower_y - radius) * rdir.y;
const ssef tLowerZ = (norg.z + node->lower_z - radius) * rdir.z;
const ssef tUpperX = (norg.x + node->upper_x + radius) * rdir.x;
const ssef tUpperY = (norg.y + node->upper_y + radius) * rdir.y;
const ssef tUpperZ = (norg.z + node->upper_z + radius) * rdir.z;
const ssef tNearX = min(tLowerX,tUpperX);
const ssef tNearY = min(tLowerY,tUpperY);
const ssef tNearZ = min(tLowerZ,tUpperZ);
const ssef tFarX = max(tLowerX,tUpperX);
const ssef tFarY = max(tLowerY,tUpperY);
const ssef tFarZ = max(tLowerZ,tUpperZ);
const ssef tNear = max(tNearX,tNearY,tNearZ,ray_near);
const ssef tFar = min(tFarX ,tFarY ,tFarZ ,ray_far);
const sseb vmask = tNear <= tFar;
size_t mask = movemask(vmask);
/*! if no child is hit, pop next node */
if (unlikely(mask == 0))
goto pop;
/*! one child is hit, continue with that child */
size_t r = __bscf(mask);
if (likely(mask == 0)) {
cur = node->child(r);
//assert(cur != BVH4::emptyNode); // FIXME: enable these assertions again, currently traversing empty children
continue;
}
/*! two children are hit, push far child, and continue with closer child */
NodeRef c0 = node->child(r); const unsigned int d0 = ((unsigned int*)&tNear)[r];
r = __bscf(mask);
NodeRef c1 = node->child(r); const unsigned int d1 = ((unsigned int*)&tNear)[r];
//assert(c0 != BVH4::emptyNode);
//assert(c1 != BVH4::emptyNode);
if (likely(mask == 0)) {
assert(stackPtr < stackEnd);
if (d0 < d1) { stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; cur = c0; continue; }
else { stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; cur = c1; continue; }
}
/*! Here starts the slow path for 3 or 4 hit children. We push
* all nodes onto the stack to sort them there. */
assert(stackPtr < stackEnd);
stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++;
assert(stackPtr < stackEnd);
stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++;
/*! three children are hit, push all onto stack and sort 3 stack items, continue with closest child */
assert(stackPtr < stackEnd);
r = __bscf(mask);
NodeRef c = node->child(r); unsigned int d = ((unsigned int*)&tNear)[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++;
//assert(c != BVH4::emptyNode);
if (likely(mask == 0)) {
sort(stackPtr[-1],stackPtr[-2],stackPtr[-3]);
cur = (NodeRef) stackPtr[-1].ptr; stackPtr--;
continue;
}
/*! four children are hit, push all onto stack and sort 4 stack items, continue with closest child */
assert(stackPtr < stackEnd);
r = __bscf(mask);
c = node->child(r); d = *(unsigned int*)&tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++;
//assert(c != BVH4::emptyNode);
sort(stackPtr[-1],stackPtr[-2],stackPtr[-3],stackPtr[-4]);
cur = (NodeRef) stackPtr[-1].ptr; stackPtr--;
}
/*! this is a leaf node */
STAT3(normal.trav_leaves,1,1,1);
size_t num; Primitive* prim = (Primitive*) cur.leaf(num);
PrimitiveIntersector::intersect(pre,ray,prim,num,bvh->geometry);
ray_far = ray.tfar;
}
}
