void BVHN<N>::clearBarrier(NodeRef& node) { if (node.isBarrier()) node.clearBarrier(); else if (!node.isLeaf()) { BaseNode* n = node.baseNode(BVH_FLAG_ALIGNED_NODE); // FIXME: flags should be stored in BVH for (size_t c=0; c<N; c++) clearBarrier(n->child(c)); } }
void BVHN<N>::clearBarrier(NodeRef& node) { if (node.isBarrier()) node.clearBarrier(); else if (!node.isLeaf()) { Node* n = node.node(); for (size_t c=0; c<N; c++) clearBarrier(n->child(c)); } }
BBox3fa BVHNRefitter<N>::recurse_bottom(NodeRef& ref) { /* this is a leaf node */ if (unlikely(ref.isLeaf())) return leafBounds.leafBounds(ref); /* recurse if this is an internal node */ AlignedNode* node = ref.alignedNode(); /* enable exclusive prefetch for >= AVX platforms */ #if defined(__AVX__) ref.prefetchW(); #endif BBox3fa bounds[N]; for (size_t i=0; i<N; i++) if (unlikely(node->child(i) == BVH::emptyNode)) { bounds[i] = BBox3fa(empty); } else bounds[i] = recurse_bottom(node->child(i)); /* AOS to SOA transform */ BBox3vf<N> boundsT = transpose<N>(bounds); /* set new bounds */ node->lower_x = boundsT.lower.x; node->lower_y = boundsT.lower.y; node->lower_z = boundsT.lower.z; node->upper_x = boundsT.upper.x; node->upper_y = boundsT.upper.y; node->upper_z = boundsT.upper.z; return merge<N>(bounds); }
void BVH8Intersector8Hybrid<PrimitiveIntersector8>::occluded(bool8* valid_i, BVH8* bvh, Ray8& ray) { /* load ray */ const bool8 valid = *valid_i; bool8 terminated = !valid; Vec3f8 ray_org = ray.org, ray_dir = ray.dir; float8 ray_tnear = ray.tnear, ray_tfar = ray.tfar; const Vec3f8 rdir = rcp_safe(ray_dir); const Vec3f8 org(ray_org), org_rdir = org * rdir; ray_tnear = select(valid,ray_tnear,float8(pos_inf)); ray_tfar = select(valid,ray_tfar ,float8(neg_inf)); const float8 inf = float8(pos_inf); Precalculations pre(valid,ray); /* compute near/far per ray */ Vec3i8 nearXYZ; nearXYZ.x = select(rdir.x >= 0.0f,int8(0*(int)sizeof(float8)),int8(1*(int)sizeof(float8))); nearXYZ.y = select(rdir.y >= 0.0f,int8(2*(int)sizeof(float8)),int8(3*(int)sizeof(float8))); nearXYZ.z = select(rdir.z >= 0.0f,int8(4*(int)sizeof(float8)),int8(5*(int)sizeof(float8))); /* allocate stack and push root node */ float8 stack_near[stackSizeChunk]; NodeRef stack_node[stackSizeChunk]; stack_node[0] = BVH8::invalidNode; stack_near[0] = inf; stack_node[1] = bvh->root; stack_near[1] = ray_tnear; NodeRef* stackEnd = stack_node+stackSizeChunk; NodeRef* __restrict__ sptr_node = stack_node + 2; float8* __restrict__ sptr_near = stack_near + 2; while (1) { /* pop next node from stack */ assert(sptr_node > stack_node); sptr_node--; sptr_near--; NodeRef cur = *sptr_node; if (unlikely(cur == BVH8::invalidNode)) { assert(sptr_node == stack_node); break; } /* cull node if behind closest hit point */ float8 curDist = *sptr_near; const bool8 active = curDist < ray_tfar; if (unlikely(none(active))) continue; /* switch to single ray traversal */ #if !defined(__WIN32__) || defined(__X86_64__) size_t bits = movemask(active); if (unlikely(__popcnt(bits) <= SWITCH_THRESHOLD)) { for (size_t i=__bsf(bits); bits!=0; bits=__btc(bits,i), i=__bsf(bits)) { if (occluded1(bvh,cur,i,pre,ray,ray_org,ray_dir,rdir,ray_tnear,ray_tfar,nearXYZ)) terminated[i] = -1; } if (all(terminated)) break; ray_tfar = select(terminated,float8(neg_inf),ray_tfar); continue; } #endif while (1) { /* test if this is a leaf node */ if (unlikely(cur.isLeaf())) break; const bool8 valid_node = ray_tfar > curDist; STAT3(shadow.trav_nodes,1,popcnt(valid_node),8); const Node* __restrict__ const node = (Node*)cur.node(); /* pop of next node */ assert(sptr_node > stack_node); sptr_node--; sptr_near--; cur = *sptr_node; curDist = *sptr_near; for (unsigned i=0; i<BVH8::N; i++) { const NodeRef child = node->children[i]; if (unlikely(child == BVH8::emptyNode)) break; #if defined(__AVX2__) const float8 lclipMinX = msub(node->lower_x[i],rdir.x,org_rdir.x); const float8 lclipMinY = msub(node->lower_y[i],rdir.y,org_rdir.y); const float8 lclipMinZ = msub(node->lower_z[i],rdir.z,org_rdir.z); const float8 lclipMaxX = msub(node->upper_x[i],rdir.x,org_rdir.x); const float8 lclipMaxY = msub(node->upper_y[i],rdir.y,org_rdir.y); const float8 lclipMaxZ = msub(node->upper_z[i],rdir.z,org_rdir.z); const float8 lnearP = maxi(maxi(mini(lclipMinX, lclipMaxX), mini(lclipMinY, lclipMaxY)), mini(lclipMinZ, lclipMaxZ)); const float8 lfarP = mini(mini(maxi(lclipMinX, lclipMaxX), maxi(lclipMinY, lclipMaxY)), maxi(lclipMinZ, lclipMaxZ)); const bool8 lhit = maxi(lnearP,ray_tnear) <= mini(lfarP,ray_tfar); #else const float8 lclipMinX = (node->lower_x[i] - org.x) * rdir.x; const float8 lclipMinY = (node->lower_y[i] - org.y) * rdir.y; const float8 lclipMinZ = (node->lower_z[i] - org.z) * rdir.z; const float8 lclipMaxX = (node->upper_x[i] - org.x) * rdir.x; const float8 lclipMaxY = (node->upper_y[i] - org.y) * rdir.y; const float8 lclipMaxZ = (node->upper_z[i] - org.z) * rdir.z; const float8 lnearP = max(max(min(lclipMinX, lclipMaxX), min(lclipMinY, lclipMaxY)), min(lclipMinZ, lclipMaxZ)); const float8 lfarP = min(min(max(lclipMinX, lclipMaxX), max(lclipMinY, lclipMaxY)), max(lclipMinZ, lclipMaxZ)); const bool8 lhit = max(lnearP,ray_tnear) <= min(lfarP,ray_tfar); #endif /* if we hit the child we choose to continue with that child if it is closer than the current next child, or we push it onto the stack */ if (likely(any(lhit))) { assert(sptr_node < stackEnd); assert(child != BVH8::emptyNode); const float8 childDist = select(lhit,lnearP,inf); sptr_node++; sptr_near++; /* push cur node onto stack and continue with hit child */ if (any(childDist < curDist)) { *(sptr_node-1) = cur; *(sptr_near-1) = curDist; curDist = childDist; cur = child; } /* push hit child onto stack */ else { *(sptr_node-1) = child; *(sptr_near-1) = childDist; } } } } /* return if stack is empty */ if (unlikely(cur == BVH8::invalidNode)) { assert(sptr_node == stack_node); break; } /* intersect leaf */ assert(cur != BVH8::emptyNode); const bool8 valid_leaf = ray_tfar > curDist; STAT3(shadow.trav_leaves,1,popcnt(valid_leaf),8); size_t items; const Triangle* prim = (Triangle*) cur.leaf(items); terminated |= PrimitiveIntersector8::occluded(!terminated,pre,ray,prim,items,bvh->scene); if (all(terminated)) break; ray_tfar = select(terminated,float8(neg_inf),ray_tfar); } store8i(valid & terminated,&ray.geomID,0); AVX_ZERO_UPPER(); }
void BVH4Intersector1<PrimitiveIntersector>::intersect(const BVH4* bvh, Ray& ray) { /*! stack state */ StackItemInt32<NodeRef> stack[stackSize]; //!< stack of nodes StackItemInt32<NodeRef>* stackPtr = stack+1; //!< current stack pointer StackItemInt32<NodeRef>* stackEnd = stack+stackSize; stack[0].ptr = bvh->root; stack[0].dist = neg_inf; /*! offsets to select the side that becomes the lower or upper bound */ const size_t nearX = ray.dir.x >= 0.0f ? 0*sizeof(ssef) : 1*sizeof(ssef); const size_t nearY = ray.dir.y >= 0.0f ? 2*sizeof(ssef) : 3*sizeof(ssef); const size_t nearZ = ray.dir.z >= 0.0f ? 4*sizeof(ssef) : 5*sizeof(ssef); #if 0 // FIXME: why is this slower /*! load the ray */ Vec3fa ray_org = ray.org; Vec3fa ray_dir = ray.dir; ssef ray_near = max(ray.tnear,FLT_MIN); // we do not support negative tnear values in this kernel due to integer optimizations ssef ray_far = ray.tfar; #if defined(__FIX_RAYS__) const float float_range = 0.1f*FLT_MAX; ray_org = clamp(ray_org,Vec3fa(-float_range),Vec3fa(+float_range)); ray_dir = clamp(ray_dir,Vec3fa(-float_range),Vec3fa(+float_range)); ray_far = min(ray_far,float(inf)); #endif const Vec3fa ray_rdir = rcp_safe(ray_dir); const sse3f org(ray_org), dir(ray_dir); const sse3f norg(-ray_org), rdir(ray_rdir), org_rdir(ray_org*ray_rdir); #else /*! load the ray into SIMD registers */ const sse3f norg(-ray.org.x,-ray.org.y,-ray.org.z); const Vec3fa ray_rdir = rcp_safe(ray.dir); const sse3f rdir(ray_rdir.x,ray_rdir.y,ray_rdir.z); const Vec3fa ray_org_rdir = ray.org*ray_rdir; const sse3f org_rdir(ray_org_rdir.x,ray_org_rdir.y,ray_org_rdir.z); const ssef ray_near(ray.tnear); ssef ray_far(ray.tfar); #endif /* pop loop */ while (true) pop: { /*! pop next node */ if (unlikely(stackPtr == stack)) break; stackPtr--; NodeRef cur = NodeRef(stackPtr->ptr); /*! if popped node is too far, pop next one */ if (unlikely(*(float*)&stackPtr->dist > ray.tfar)) continue; /* downtraversal loop */ while (true) { /*! stop if we found a leaf */ if (unlikely(cur.isLeaf())) break; STAT3(normal.trav_nodes,1,1,1); /*! single ray intersection with 4 boxes */ const Node* node = cur.node(); const size_t farX = nearX ^ 16, farY = nearY ^ 16, farZ = nearZ ^ 16; #if defined (__AVX2__) const ssef tNearX = msub(load4f((const char*)node+nearX), rdir.x, org_rdir.x); const ssef tNearY = msub(load4f((const char*)node+nearY), rdir.y, org_rdir.y); const ssef tNearZ = msub(load4f((const char*)node+nearZ), rdir.z, org_rdir.z); const ssef tFarX = msub(load4f((const char*)node+farX ), rdir.x, org_rdir.x); const ssef tFarY = msub(load4f((const char*)node+farY ), rdir.y, org_rdir.y); const ssef tFarZ = msub(load4f((const char*)node+farZ ), rdir.z, org_rdir.z); #else const ssef tNearX = (norg.x + load4f((const char*)node+nearX)) * rdir.x; const ssef tNearY = (norg.y + load4f((const char*)node+nearY)) * rdir.y; const ssef tNearZ = (norg.z + load4f((const char*)node+nearZ)) * rdir.z; const ssef tFarX = (norg.x + load4f((const char*)node+farX )) * rdir.x; const ssef tFarY = (norg.y + load4f((const char*)node+farY )) * rdir.y; const ssef tFarZ = (norg.z + load4f((const char*)node+farZ )) * rdir.z; #endif #if defined(__SSE4_1__) const ssef tNear = maxi(maxi(tNearX,tNearY),maxi(tNearZ,ray_near)); const ssef tFar = mini(mini(tFarX ,tFarY ),mini(tFarZ ,ray_far )); const sseb vmask = cast(tNear) > cast(tFar); size_t mask = movemask(vmask)^0xf; #else const ssef tNear = max(tNearX,tNearY,tNearZ,ray_near); const ssef tFar = min(tFarX ,tFarY ,tFarZ ,ray_far); const sseb vmask = tNear <= tFar; size_t mask = movemask(vmask); #endif /*! if no child is hit, pop next node */ if (unlikely(mask == 0)) goto pop; /*! one child is hit, continue with that child */ size_t r = __bscf(mask); if (likely(mask == 0)) { cur = node->child(r); assert(cur != BVH4::emptyNode); continue; } /*! two children are hit, push far child, and continue with closer child */ NodeRef c0 = node->child(r); const unsigned int d0 = ((unsigned int*)&tNear)[r]; r = __bscf(mask); NodeRef c1 = node->child(r); const unsigned int d1 = ((unsigned int*)&tNear)[r]; assert(c0 != BVH4::emptyNode); assert(c1 != BVH4::emptyNode); if (likely(mask == 0)) { assert(stackPtr < stackEnd); if (d0 < d1) { stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; cur = c0; continue; } else { stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; cur = c1; continue; } } /*! Here starts the slow path for 3 or 4 hit children. We push * all nodes onto the stack to sort them there. */ assert(stackPtr < stackEnd); stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; assert(stackPtr < stackEnd); stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; /*! three children are hit, push all onto stack and sort 3 stack items, continue with closest child */ assert(stackPtr < stackEnd); r = __bscf(mask); NodeRef c = node->child(r); unsigned int d = ((unsigned int*)&tNear)[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; assert(c != BVH4::emptyNode); if (likely(mask == 0)) { sort(stackPtr[-1],stackPtr[-2],stackPtr[-3]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; continue; } /*! four children are hit, push all onto stack and sort 4 stack items, continue with closest child */ assert(stackPtr < stackEnd); r = __bscf(mask); c = node->child(r); d = *(unsigned int*)&tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; assert(c != BVH4::emptyNode); sort(stackPtr[-1],stackPtr[-2],stackPtr[-3],stackPtr[-4]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; } /*! this is a leaf node */ STAT3(normal.trav_leaves,1,1,1); size_t num; Primitive* prim = (Primitive*) cur.leaf(num); PrimitiveIntersector::intersect(ray,prim,num,bvh->geometry); ray_far = ray.tfar; } }
void BVH4Intersector1<types,robust,PrimitiveIntersector>::intersect(const BVH4* bvh, Ray& ray) { /*! perform per ray precalculations required by the primitive intersector */ Precalculations pre(ray); BVH4::UnalignedNodeMB::Precalculations pre1(ray); /*! stack state */ StackItemInt32<NodeRef> stack[stackSize]; //!< stack of nodes StackItemInt32<NodeRef>* stackPtr = stack+1; //!< current stack pointer StackItemInt32<NodeRef>* stackEnd = stack+stackSize; stack[0].ptr = bvh->root; stack[0].dist = neg_inf; /*! load the ray into SIMD registers */ const Vec3fa ray_rdir = rcp_safe(ray.dir); const Vec3fa ray_org_rdir = ray.org*ray_rdir; const sse3f org(ray.org.x,ray.org.y,ray.org.z); const sse3f dir(ray.dir.x,ray.dir.y,ray.dir.z); const sse3f rdir(ray_rdir.x,ray_rdir.y,ray_rdir.z); const sse3f org_rdir(ray_org_rdir.x,ray_org_rdir.y,ray_org_rdir.z); const ssef ray_near(ray.tnear); ssef ray_far(ray.tfar); /*! offsets to select the side that becomes the lower or upper bound */ const size_t nearX = ray_rdir.x >= 0.0f ? 0*sizeof(ssef) : 1*sizeof(ssef); const size_t nearY = ray_rdir.y >= 0.0f ? 2*sizeof(ssef) : 3*sizeof(ssef); const size_t nearZ = ray_rdir.z >= 0.0f ? 4*sizeof(ssef) : 5*sizeof(ssef); /* pop loop */ while (true) pop: { /*! pop next node */ if (unlikely(stackPtr == stack)) break; stackPtr--; NodeRef cur = NodeRef(stackPtr->ptr); /*! if popped node is too far, pop next one */ if (unlikely(*(float*)&stackPtr->dist > ray.tfar)) continue; /* downtraversal loop */ while (true) { size_t mask; ssef tNear; /*! stop if we found a leaf node */ if (unlikely(cur.isLeaf(types))) break; STAT3(normal.trav_nodes,1,1,1); /* process standard nodes */ if (likely(cur.isNode(types))) mask = cur.node()->intersect<robust>(nearX,nearY,nearZ,org,rdir,org_rdir,ray_near,ray_far,tNear); /* process motion blur nodes */ else if (likely(cur.isNodeMB(types))) mask = cur.nodeMB()->intersect(nearX,nearY,nearZ,org,rdir,org_rdir,ray_near,ray_far,ray.time,tNear); /*! process nodes with unaligned bounds */ else if (unlikely(cur.isUnalignedNode(types))) mask = cur.unalignedNode()->intersect(org,dir,ray_near,ray_far,tNear); /*! process nodes with unaligned bounds and motion blur */ else if (unlikely(cur.isUnalignedNodeMB(types))) mask = cur.unalignedNodeMB()->intersect(pre1,org,dir,ray_near,ray_far,ray.time,tNear); /*! if no child is hit, pop next node */ const BVH4::BaseNode* node = cur.baseNode(types); if (unlikely(mask == 0)) goto pop; /*! one child is hit, continue with that child */ size_t r = __bscf(mask); if (likely(mask == 0)) { cur = node->child(r); cur.prefetch(types); assert(cur != BVH4::emptyNode); continue; } /*! two children are hit, push far child, and continue with closer child */ NodeRef c0 = node->child(r); c0.prefetch(types); const unsigned int d0 = ((unsigned int*)&tNear)[r]; r = __bscf(mask); NodeRef c1 = node->child(r); c1.prefetch(types); const unsigned int d1 = ((unsigned int*)&tNear)[r]; assert(c0 != BVH4::emptyNode); assert(c1 != BVH4::emptyNode); if (likely(mask == 0)) { assert(stackPtr < stackEnd); if (d0 < d1) { stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; cur = c0; continue; } else { stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; cur = c1; continue; } } /*! Here starts the slow path for 3 or 4 hit children. We push * all nodes onto the stack to sort them there. */ assert(stackPtr < stackEnd); stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; assert(stackPtr < stackEnd); stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; /*! three children are hit, push all onto stack and sort 3 stack items, continue with closest child */ assert(stackPtr < stackEnd); r = __bscf(mask); NodeRef c = node->child(r); c.prefetch(types); unsigned int d = ((unsigned int*)&tNear)[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; assert(c != BVH4::emptyNode); if (likely(mask == 0)) { sort(stackPtr[-1],stackPtr[-2],stackPtr[-3]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; continue; } /*! four children are hit, push all onto stack and sort 4 stack items, continue with closest child */ assert(stackPtr < stackEnd); r = __bscf(mask); c = node->child(r); c.prefetch(types); d = *(unsigned int*)&tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; assert(c != BVH4::emptyNode); sort(stackPtr[-1],stackPtr[-2],stackPtr[-3],stackPtr[-4]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; } /*! this is a leaf node */ assert(cur != BVH4::emptyNode); STAT3(normal.trav_leaves,1,1,1); size_t num; Primitive* prim = (Primitive*) cur.leaf(num); PrimitiveIntersector::intersect(pre,ray,prim,num,bvh->geometry); ray_far = ray.tfar; } AVX_ZERO_UPPER(); }
void BVH4mbIntersector16Single::occluded(mic_i* valid_i, BVH4mb* bvh, Ray16& ray16) { /* near and node stack */ __align(64) NodeRef stack_node[3*BVH4i::maxDepth+1]; /* setup */ const mic_m m_valid = *(mic_i*)valid_i != mic_i(0); const mic3f rdir16 = rcp_safe(ray16.dir); unsigned int terminated = toInt(!m_valid); const mic_f inf = mic_f(pos_inf); const mic_f zero = mic_f::zero(); const Node * __restrict__ nodes = (Node *)bvh->nodePtr(); const BVH4mb::Triangle01 * __restrict__ accel = (BVH4mb::Triangle01 *)bvh->triPtr(); stack_node[0] = BVH4i::invalidNode; long rayIndex = -1; while((rayIndex = bitscan64(rayIndex,toInt(m_valid))) != BITSCAN_NO_BIT_SET_64) { stack_node[1] = bvh->root; size_t sindex = 2; const mic_f org_xyz = loadAOS4to16f(rayIndex,ray16.org.x,ray16.org.y,ray16.org.z); const mic_f dir_xyz = loadAOS4to16f(rayIndex,ray16.dir.x,ray16.dir.y,ray16.dir.z); const mic_f rdir_xyz = loadAOS4to16f(rayIndex,rdir16.x,rdir16.y,rdir16.z); const mic_f org_rdir_xyz = org_xyz * rdir_xyz; const mic_f min_dist_xyz = broadcast1to16f(&ray16.tnear[rayIndex]); const mic_f max_dist_xyz = broadcast1to16f(&ray16.tfar[rayIndex]); const mic_f time = broadcast1to16f(&ray16.time[rayIndex]); const unsigned int leaf_mask = BVH4I_LEAF_MASK; while (1) { NodeRef curNode = stack_node[sindex-1]; sindex--; const mic_f one_time = (mic_f::one() - time); while (1) { /* test if this is a leaf node */ if (unlikely(curNode.isLeaf(leaf_mask))) break; const Node* __restrict__ const node = curNode.node(nodes); const float* __restrict const plower = (float*)node->lower; const float* __restrict const pupper = (float*)node->upper; prefetch<PFHINT_L1>((char*)node + 0*64); prefetch<PFHINT_L1>((char*)node + 1*64); prefetch<PFHINT_L1>((char*)node + 2*64); prefetch<PFHINT_L1>((char*)node + 3*64); const BVH4mb::Node* __restrict__ const nodeMB = (BVH4mb::Node*)node; const mic_f lower = one_time * load16f((float*)nodeMB->lower) + time * load16f((float*)nodeMB->lower_t1); const mic_f upper = one_time * load16f((float*)nodeMB->upper) + time * load16f((float*)nodeMB->upper_t1); /* intersect single ray with 4 bounding boxes */ const mic_f tLowerXYZ = lower * rdir_xyz - org_rdir_xyz; const mic_f tUpperXYZ = upper * rdir_xyz - org_rdir_xyz; const mic_f tLower = mask_min(0x7777,min_dist_xyz,tLowerXYZ,tUpperXYZ); const mic_f tUpper = mask_max(0x7777,max_dist_xyz,tLowerXYZ,tUpperXYZ); const Node* __restrict__ const next = curNode.node(nodes); prefetch<PFHINT_L2>((char*)next + 0); prefetch<PFHINT_L2>((char*)next + 64); sindex--; const mic_f tNear = vreduce_max4(tLower); const mic_f tFar = vreduce_min4(tUpper); const mic_m hitm = le(0x8888,tNear,tFar); const mic_f tNear_pos = select(hitm,tNear,inf); curNode = stack_node[sindex]; // early pop of next node /* if no child is hit, continue with early popped child */ if (unlikely(none(hitm))) continue; sindex++; const unsigned long hiti = toInt(hitm); const unsigned long pos_first = bitscan64(hiti); const unsigned long num_hitm = countbits(hiti); /* if a single child is hit, continue with that child */ curNode = ((unsigned int *)plower)[pos_first]; if (likely(num_hitm == 1)) continue; /* if two children are hit, push in correct order */ const unsigned long pos_second = bitscan64(pos_first,hiti); if (likely(num_hitm == 2)) { const unsigned int dist_first = ((unsigned int*)&tNear)[pos_first]; const unsigned int dist_second = ((unsigned int*)&tNear)[pos_second]; const unsigned int node_first = curNode; const unsigned int node_second = ((unsigned int*)plower)[pos_second]; if (dist_first <= dist_second) { stack_node[sindex] = node_second; sindex++; assert(sindex < 3*BVH4i::maxDepth+1); continue; } else { stack_node[sindex] = curNode; curNode = node_second; sindex++; assert(sindex < 3*BVH4i::maxDepth+1); continue; } } /* continue with closest child and push all others */ const mic_f min_dist = set_min_lanes(tNear_pos); const unsigned int old_sindex = sindex; sindex += countbits(hiti) - 1; assert(sindex < 3*BVH4i::maxDepth+1); const mic_m closest_child = eq(hitm,min_dist,tNear); const unsigned long closest_child_pos = bitscan64(closest_child); const mic_m m_pos = andn(hitm,andn(closest_child,(mic_m)((unsigned int)closest_child - 1))); const mic_i plower_node = load16i((int*)plower); curNode = ((unsigned int*)plower)[closest_child_pos]; compactustore16i(m_pos,&stack_node[old_sindex],plower_node); } /* return if stack is empty */ if (unlikely(curNode == BVH4i::invalidNode)) break; /* intersect one ray against four triangles */ ////////////////////////////////////////////////////////////////////////////////////////////////// const BVH4mb::Triangle01* tptr = (BVH4mb::Triangle01*) curNode.leaf(accel); prefetch<PFHINT_L1>((mic_f*)tptr + 0); prefetch<PFHINT_L1>((mic_f*)tptr + 1); prefetch<PFHINT_L1>((mic_f*)tptr + 2); prefetch<PFHINT_L1>((mic_f*)tptr + 3); const mic_i and_mask = broadcast4to16i(zlc4); const mic_f v0_t0 = gather_4f_zlc(and_mask, (float*)&tptr[0].t0.v0, (float*)&tptr[1].t0.v0, (float*)&tptr[2].t0.v0, (float*)&tptr[3].t0.v0); const mic_f v1_t0 = gather_4f_zlc(and_mask, (float*)&tptr[0].t0.v1, (float*)&tptr[1].t0.v1, (float*)&tptr[2].t0.v1, (float*)&tptr[3].t0.v1); const mic_f v2_t0 = gather_4f_zlc(and_mask, (float*)&tptr[0].t0.v2, (float*)&tptr[1].t0.v2, (float*)&tptr[2].t0.v2, (float*)&tptr[3].t0.v2); prefetch<PFHINT_L2>((mic_f*)tptr + 4); prefetch<PFHINT_L2>((mic_f*)tptr + 5); prefetch<PFHINT_L2>((mic_f*)tptr + 6); prefetch<PFHINT_L2>((mic_f*)tptr + 7); const mic_f v0_t1 = gather_4f_zlc(and_mask, (float*)&tptr[0].t1.v0, (float*)&tptr[1].t1.v0, (float*)&tptr[2].t1.v0, (float*)&tptr[3].t1.v0); const mic_f v1_t1 = gather_4f_zlc(and_mask, (float*)&tptr[0].t1.v1, (float*)&tptr[1].t1.v1, (float*)&tptr[2].t1.v1, (float*)&tptr[3].t1.v1); const mic_f v2_t1 = gather_4f_zlc(and_mask, (float*)&tptr[0].t1.v2, (float*)&tptr[1].t1.v2, (float*)&tptr[2].t1.v2, (float*)&tptr[3].t1.v2); const mic_f v0 = v0_t0 * one_time + time * v0_t1; const mic_f v1 = v1_t0 * one_time + time * v1_t1; const mic_f v2 = v2_t0 * one_time + time * v2_t1; const mic_f e1 = v1 - v0; const mic_f e2 = v0 - v2; const mic_f normal = lcross_zxy(e1,e2); const mic_f org = v0 - org_xyz; const mic_f odzxy = msubr231(org * swizzle(dir_xyz,_MM_SWIZ_REG_DACB), dir_xyz, swizzle(org,_MM_SWIZ_REG_DACB)); const mic_f den = ldot3_zxy(dir_xyz,normal); const mic_f rcp_den = rcp(den); const mic_f uu = ldot3_zxy(e2,odzxy); const mic_f vv = ldot3_zxy(e1,odzxy); const mic_f u = uu * rcp_den; const mic_f v = vv * rcp_den; #if defined(__BACKFACE_CULLING__) const mic_m m_init = (mic_m)0x1111 & (den > zero); #else const mic_m m_init = 0x1111; #endif const mic_m valid_u = ge((mic_m)m_init,u,zero); const mic_m valid_v = ge(valid_u,v,zero); const mic_m m_aperture = le(valid_v,u+v,mic_f::one()); const mic_f nom = ldot3_zxy(org,normal); const mic_f t = rcp_den*nom; if (unlikely(none(m_aperture))) continue; mic_m m_final = lt(lt(m_aperture,min_dist_xyz,t),t,max_dist_xyz); #if defined(__USE_RAY_MASK__) const mic_i rayMask(ray16.mask[rayIndex]); const mic_i triMask = swDDDD(gather16i_4i_align(&tptr[0].t0.v2,&tptr[1].t0.v2,&tptr[2].t0.v2,&tptr[3].t0.v2)); const mic_m m_ray_mask = (rayMask & triMask) != mic_i::zero(); m_final &= m_ray_mask; #endif if (unlikely(any(m_final))) { terminated |= mic_m::shift1[rayIndex]; break; } ////////////////////////////////////////////////////////////////////////////////////////////////// } if (unlikely(all(toMask(terminated)))) break; } store16i(m_valid & toMask(terminated),&ray16.geomID,0); }
void BVH8Intersector8Chunk<PrimitiveIntersector8>::intersect(avxb* valid_i, BVH8* bvh, Ray8& ray) { #if defined(__AVX__) /* load ray */ const avxb valid0 = *valid_i; const avx3f rdir = rcp_safe(ray.dir); const avx3f org_rdir = ray.org * rdir; avxf ray_tnear = select(valid0,ray.tnear,pos_inf); avxf ray_tfar = select(valid0,ray.tfar ,neg_inf); const avxf inf = avxf(pos_inf); Precalculations pre(valid0,ray); /* allocate stack and push root node */ avxf stack_near[3*BVH8::maxDepth+1]; NodeRef stack_node[3*BVH8::maxDepth+1]; stack_node[0] = BVH8::invalidNode; stack_near[0] = inf; stack_node[1] = bvh->root; stack_near[1] = ray_tnear; NodeRef* __restrict__ sptr_node = stack_node + 2; avxf* __restrict__ sptr_near = stack_near + 2; while (1) { /* pop next node from stack */ sptr_node--; sptr_near--; NodeRef cur = *sptr_node; if (unlikely(cur == BVH8::invalidNode)) break; /* cull node if behind closest hit point */ avxf curDist = *sptr_near; if (unlikely(none(ray_tfar > curDist))) continue; while (1) { /* test if this is a leaf node */ if (unlikely(cur.isLeaf())) break; const avxb valid_node = ray_tfar > curDist; STAT3(normal.trav_nodes,1,popcnt(valid_node),8); const Node* __restrict__ const node = (BVH8::Node*)cur.node(); /* pop of next node */ sptr_node--; sptr_near--; cur = *sptr_node; // FIXME: this trick creates issues with stack depth curDist = *sptr_near; for (unsigned i=0; i<BVH8::N; i++) { const NodeRef child = node->children[i]; if (unlikely(child == BVH8::emptyNode)) break; #if defined(__AVX2__) const avxf lclipMinX = msub(node->lower_x[i],rdir.x,org_rdir.x); const avxf lclipMinY = msub(node->lower_y[i],rdir.y,org_rdir.y); const avxf lclipMinZ = msub(node->lower_z[i],rdir.z,org_rdir.z); const avxf lclipMaxX = msub(node->upper_x[i],rdir.x,org_rdir.x); const avxf lclipMaxY = msub(node->upper_y[i],rdir.y,org_rdir.y); const avxf lclipMaxZ = msub(node->upper_z[i],rdir.z,org_rdir.z); const avxf lnearP = maxi(maxi(mini(lclipMinX, lclipMaxX), mini(lclipMinY, lclipMaxY)), mini(lclipMinZ, lclipMaxZ)); const avxf lfarP = mini(mini(maxi(lclipMinX, lclipMaxX), maxi(lclipMinY, lclipMaxY)), maxi(lclipMinZ, lclipMaxZ)); const avxb lhit = maxi(lnearP,ray_tnear) <= mini(lfarP,ray_tfar); #else const avxf lclipMinX = node->lower_x[i] * rdir.x - org_rdir.x; const avxf lclipMinY = node->lower_y[i] * rdir.y - org_rdir.y; const avxf lclipMinZ = node->lower_z[i] * rdir.z - org_rdir.z; const avxf lclipMaxX = node->upper_x[i] * rdir.x - org_rdir.x; const avxf lclipMaxY = node->upper_y[i] * rdir.y - org_rdir.y; const avxf lclipMaxZ = node->upper_z[i] * rdir.z - org_rdir.z; const avxf lnearP = max(max(min(lclipMinX, lclipMaxX), min(lclipMinY, lclipMaxY)), min(lclipMinZ, lclipMaxZ)); const avxf lfarP = min(min(max(lclipMinX, lclipMaxX), max(lclipMinY, lclipMaxY)), max(lclipMinZ, lclipMaxZ)); const avxb lhit = max(lnearP,ray_tnear) <= min(lfarP,ray_tfar); #endif /* if we hit the child we choose to continue with that child if it is closer than the current next child, or we push it onto the stack */ if (likely(any(lhit))) { const avxf childDist = select(lhit,lnearP,inf); const NodeRef child = node->children[i]; /* push cur node onto stack and continue with hit child */ if (any(childDist < curDist)) { *sptr_node = cur; *sptr_near = curDist; sptr_node++; sptr_near++; curDist = childDist; cur = child; } /* push hit child onto stack*/ else { *sptr_node = child; *sptr_near = childDist; sptr_node++; sptr_near++; } assert(sptr_node - stack_node < BVH8::maxDepth); } } } /* return if stack is empty */ if (unlikely(cur == BVH8::invalidNode)) break; /* intersect leaf */ assert(cur != BVH8::emptyNode); const avxb valid_leaf = ray_tfar > curDist; STAT3(normal.trav_leaves,1,popcnt(valid_leaf),8); size_t items; const Triangle* tri = (Triangle*) cur.leaf(items); PrimitiveIntersector8::intersect(valid_leaf,pre,ray,tri,items,bvh->geometry); ray_tfar = select(valid_leaf,ray.tfar,ray_tfar); } AVX_ZERO_UPPER(); #endif }
void BVH4iIntersector4Chunk<TriangleIntersector4>::occluded(sseb* valid_i, BVH4i* bvh, Ray4& ray) { /* load node and primitive array */ const Node * __restrict__ nodes = (Node *)bvh->nodePtr(); const Triangle * __restrict__ accel = (Triangle*)bvh->triPtr(); /* load ray */ const sseb valid = *valid_i; sseb terminated = !valid; const sse3f rdir = rcp_safe(ray.dir); const sse3f org_rdir = ray.org * rdir; ssef ray_tnear = select(valid,ray.tnear,pos_inf); ssef ray_tfar = select(valid,ray.tfar ,neg_inf); const ssef inf = ssef(pos_inf); /* allocate stack and push root node */ ssef stack_near[3*BVH4i::maxDepth+1]; NodeRef stack_node[3*BVH4i::maxDepth+1]; stack_node[0] = BVH4i::invalidNode; stack_near[0] = inf; stack_node[1] = bvh->root; stack_near[1] = ray_tnear; NodeRef* __restrict__ sptr_node = stack_node + 2; ssef* __restrict__ sptr_near = stack_near + 2; while (1) { /* pop next node from stack */ sptr_node--; sptr_near--; NodeRef curNode = *sptr_node; if (unlikely(curNode == BVH4i::invalidNode)) break; /* cull node if behind closest hit point */ ssef curDist = *sptr_near; if (unlikely(none(ray_tfar > curDist))) continue; while (1) { /* test if this is a leaf node */ if (unlikely(curNode.isLeaf())) break; const sseb valid_node = ray_tfar > curDist; STAT3(shadow.trav_nodes,1,popcnt(valid_node),4); const Node* __restrict__ const node = curNode.node(nodes); /* pop of next node */ sptr_node--; sptr_near--; curNode = *sptr_node; // FIXME: this trick creates issues with stack depth curDist = *sptr_near; #pragma unroll(4) for (unsigned i=0; i<4; i++) { const NodeRef child = node->children[i]; if (unlikely(child == BVH4i::emptyNode)) break; #if defined(__AVX2__) const ssef lclipMinX = msub(node->lower_x[i],rdir.x,org_rdir.x); const ssef lclipMinY = msub(node->lower_y[i],rdir.y,org_rdir.y); const ssef lclipMinZ = msub(node->lower_z[i],rdir.z,org_rdir.z); const ssef lclipMaxX = msub(node->upper_x[i],rdir.x,org_rdir.x); const ssef lclipMaxY = msub(node->upper_y[i],rdir.y,org_rdir.y); const ssef lclipMaxZ = msub(node->upper_z[i],rdir.z,org_rdir.z); const ssef lnearP = maxi(maxi(mini(lclipMinX, lclipMaxX), mini(lclipMinY, lclipMaxY)), mini(lclipMinZ, lclipMaxZ)); const ssef lfarP = mini(mini(maxi(lclipMinX, lclipMaxX), maxi(lclipMinY, lclipMaxY)), maxi(lclipMinZ, lclipMaxZ)); const sseb lhit = maxi(lnearP,ray_tnear) <= mini(lfarP,ray_tfar); #else const ssef lclipMinX = node->lower_x[i] * rdir.x - org_rdir.x; const ssef lclipMinY = node->lower_y[i] * rdir.y - org_rdir.y; const ssef lclipMinZ = node->lower_z[i] * rdir.z - org_rdir.z; const ssef lclipMaxX = node->upper_x[i] * rdir.x - org_rdir.x; const ssef lclipMaxY = node->upper_y[i] * rdir.y - org_rdir.y; const ssef lclipMaxZ = node->upper_z[i] * rdir.z - org_rdir.z; const ssef lnearP = max(max(min(lclipMinX, lclipMaxX), min(lclipMinY, lclipMaxY)), min(lclipMinZ, lclipMaxZ)); const ssef lfarP = min(min(max(lclipMinX, lclipMaxX), max(lclipMinY, lclipMaxY)), max(lclipMinZ, lclipMaxZ)); const sseb lhit = max(lnearP,ray_tnear) <= min(lfarP,ray_tfar); #endif /* if we hit the child we choose to continue with that child if it is closer than the current next child, or we push it onto the stack */ if (likely(any(lhit))) { const ssef childDist = select(lhit,lnearP,inf); sptr_node++; sptr_near++; /* push cur node onto stack and continue with hit child */ if (any(childDist < curDist)) { *(sptr_node-1) = curNode; *(sptr_near-1) = curDist; curDist = childDist; curNode = child; } /* push hit child onto stack*/ else { *(sptr_node-1) = child; *(sptr_near-1) = childDist; } assert(sptr_node - stack_node < BVH4i::maxDepth); } } } /* return if stack is empty */ if (unlikely(curNode == BVH4i::invalidNode)) break; /* intersect leaf */ const sseb valid_leaf = ray_tfar > curDist; STAT3(shadow.trav_leaves,1,popcnt(valid_leaf),4); size_t items; const Triangle* tri = (Triangle*) curNode.leaf(accel, items); terminated |= TriangleIntersector4::occluded(!terminated,ray,tri,items,bvh->geometry); if (all(terminated)) break; ray_tfar = select(terminated,neg_inf,ray_tfar); } store4i(valid & terminated,&ray.geomID,0); AVX_ZERO_UPPER(); }
__forceinline bool BVH8iIntersector8Hybrid<TriangleIntersector8>::occluded1(const BVH8i* bvh, NodeRef root, const size_t k, Ray8& ray,const avx3f &ray_org, const avx3f &ray_dir, const avx3f &ray_rdir, const avxf &ray_tnear, const avxf &ray_tfar, const avx3i& nearXYZ) { /*! stack state */ NodeRef stack[stackSizeSingle]; //!< stack of nodes that still need to get traversed NodeRef* stackPtr = stack+1; //!< current stack pointer NodeRef* stackEnd = stack+stackSizeSingle; stack[0] = root; /*! offsets to select the side that becomes the lower or upper bound */ const size_t nearX = nearXYZ.x[k]; const size_t nearY = nearXYZ.y[k]; const size_t nearZ = nearXYZ.z[k]; /*! load the ray into SIMD registers */ const avx3f org (ray_org .x[k],ray_org .y[k],ray_org .z[k]); const avx3f rdir(ray_rdir.x[k],ray_rdir.y[k],ray_rdir.z[k]); const avx3f norg = -org, org_rdir(org*rdir); const avxf rayNear(ray_tnear[k]), rayFar(ray_tfar[k]); const Node * __restrict__ nodes = (Node *)bvh->nodePtr(); const Triangle * __restrict__ accel = (Triangle*)bvh->triPtr(); /* pop loop */ while (true) pop: { /*! pop next node */ if (unlikely(stackPtr == stack)) break; stackPtr--; NodeRef cur = (NodeRef) *stackPtr; /* downtraversal loop */ while (true) { /*! stop if we found a leaf */ if (unlikely(cur.isLeaf())) break; STAT3(shadow.trav_nodes,1,1,1); /*! single ray intersection with 4 boxes */ const Node* node = (Node*)cur.node(nodes); const size_t farX = nearX ^ sizeof(avxf), farY = nearY ^ sizeof(avxf), farZ = nearZ ^ sizeof(avxf); #if defined (__AVX2__) const avxf tNearX = msub(load8f((const char*)node+nearX), rdir.x, org_rdir.x); const avxf tNearY = msub(load8f((const char*)node+nearY), rdir.y, org_rdir.y); const avxf tNearZ = msub(load8f((const char*)node+nearZ), rdir.z, org_rdir.z); const avxf tFarX = msub(load8f((const char*)node+farX ), rdir.x, org_rdir.x); const avxf tFarY = msub(load8f((const char*)node+farY ), rdir.y, org_rdir.y); const avxf tFarZ = msub(load8f((const char*)node+farZ ), rdir.z, org_rdir.z); #else const avxf tNearX = (norg.x + load8f((const char*)node+nearX)) * rdir.x; const avxf tNearY = (norg.y + load8f((const char*)node+nearY)) * rdir.y; const avxf tNearZ = (norg.z + load8f((const char*)node+nearZ)) * rdir.z; const avxf tFarX = (norg.x + load8f((const char*)node+farX )) * rdir.x; const avxf tFarY = (norg.y + load8f((const char*)node+farY )) * rdir.y; const avxf tFarZ = (norg.z + load8f((const char*)node+farZ )) * rdir.z; #endif #if defined(__AVX2__) const avxf tNear = maxi(maxi(tNearX,tNearY),maxi(tNearZ,rayNear)); const avxf tFar = mini(mini(tFarX ,tFarY ),mini(tFarZ ,rayFar )); const avxb vmask = cast(tNear) > cast(tFar); unsigned int mask = movemask(vmask)^0xff; #else const avxf tNear = max(tNearX,tNearY,tNearZ,rayNear); const avxf tFar = min(tFarX ,tFarY ,tFarZ ,rayFar); const avxb vmask = tNear <= tFar; unsigned int mask = movemask(vmask); #endif /*! if no child is hit, pop next node */ if (unlikely(mask == 0)) goto pop; /*! one child is hit, continue with that child */ size_t r = __bscf(mask); if (likely(mask == 0)) { cur = node->child(r); assert(cur != BVH4i::emptyNode); continue; } /*! two children are hit, push far child, and continue with closer child */ NodeRef c0 = node->child(r); const unsigned int d0 = ((unsigned int*)&tNear)[r]; r = __bscf(mask); NodeRef c1 = node->child(r); const unsigned int d1 = ((unsigned int*)&tNear)[r]; assert(c0 != BVH4i::emptyNode); assert(c1 != BVH4i::emptyNode); if (likely(mask == 0)) { assert(stackPtr < stackEnd); if (d0 < d1) { *stackPtr = c1; stackPtr++; cur = c0; continue; } else { *stackPtr = c0; stackPtr++; cur = c1; continue; } } assert(stackPtr < stackEnd); *stackPtr = c0; stackPtr++; assert(stackPtr < stackEnd); *stackPtr = c1; stackPtr++; /*! three children are hit */ r = __bscf(mask); cur = node->child(r); assert(cur != BVH4i::emptyNode); if (likely(mask == 0)) continue; while(1) { r = __bscf(mask); NodeRef c = node->child(r); *stackPtr = c; stackPtr++; if (unlikely(mask == 0)) break; } cur = (NodeRef) stackPtr[-1]; stackPtr--; // assert(stackPtr < stackEnd); // *stackPtr = cur; stackPtr++; // /*! four children are hit */ // cur = node->child(3); // assert(cur != BVH4i::emptyNode); } /*! this is a leaf node */ STAT3(shadow.trav_leaves,1,1,1); size_t num; Triangle* prim = (Triangle*) cur.leaf(accel,num); if (TriangleIntersector8::occluded(ray,k,prim,num,bvh->geometry)) { ray.geomID[k] = 0; return true; } } return false; }
__forceinline void intersectT(const BVH4* bvh, Ray& ray) { typedef typename TriangleIntersector::Triangle Triangle; typedef StackItemT<size_t> StackItem; typedef typename BVH4::NodeRef NodeRef; typedef typename BVH4::Node Node; /*! stack state */ StackItem stack[1+3*BVH4::maxDepth]; //!< stack of nodes StackItem* stackPtr = stack+1; //!< current stack pointer stack[0].ptr = bvh->root; stack[0].dist = neg_inf; /*! load the ray into SIMD registers */ const avxf pos_neg = avxf(ssef(+0.0f),ssef(-0.0f)); const avxf neg_pos = avxf(ssef(-0.0f),ssef(+0.0f)); const avxf flipSignX = swapX ? neg_pos : pos_neg; const avxf flipSignY = swapY ? neg_pos : pos_neg; const avxf flipSignZ = swapZ ? neg_pos : pos_neg; const Vector3f ray_rdir = rcp_safe(ray.dir); const avx3f norg(-ray.org.x,-ray.org.y,-ray.org.z); const avx3f rdir(ray_rdir.x^flipSignX,ray_rdir.y^flipSignY,ray_rdir.z^flipSignZ); const avx3f org_rdir(avx3f(ray.org.x,ray.org.y,ray.org.z)*rdir); avxf rayNearFar(ssef(ray.tnear),-ssef(ray.tfar)); const void* nodePtr = bvh->nodePtr(); const void* triPtr = bvh->triPtr(); /* pop loop */ while (true) pop: { /*! pop next node */ if (unlikely(stackPtr == stack)) break; stackPtr--; NodeRef cur = NodeRef(stackPtr->ptr); /*! if popped node is too far, pop next one */ if (unlikely(stackPtr->dist > ray.tfar)) continue; /* downtraversal loop */ while (true) { /*! stop if we found a leaf */ if (unlikely(cur.isLeaf())) break; STAT3(normal.trav_nodes,1,1,1); /*! single ray intersection with 4 boxes */ const Node* node = cur.node(nodePtr); #if defined (__AVX2__) || defined(__MIC__) const avxf tLowerUpperX = msub(avxf::load(&node->lower_x), rdir.x, org_rdir.x); const avxf tLowerUpperY = msub(avxf::load(&node->lower_y), rdir.y, org_rdir.y); const avxf tLowerUpperZ = msub(avxf::load(&node->lower_z), rdir.z, org_rdir.z); #else const avxf tLowerUpperX = (norg.x + avxf::load(&node->lower_x)) * rdir.x; const avxf tLowerUpperY = (norg.y + avxf::load(&node->lower_y)) * rdir.y; const avxf tLowerUpperZ = (norg.z + avxf::load(&node->lower_z)) * rdir.z; #endif const avxf tNearFarX = swapX ? shuffle<1,0>(tLowerUpperX) : tLowerUpperX; const avxf tNearFarY = swapY ? shuffle<1,0>(tLowerUpperY) : tLowerUpperY; const avxf tNearFarZ = swapZ ? shuffle<1,0>(tLowerUpperZ) : tLowerUpperZ; const avxf tNearFar = max(tNearFarX,tNearFarY,tNearFarZ,rayNearFar); const ssef tNear = extract<0>(tNearFar); const ssef tFar = extract<1>(tNearFar); size_t mask = movemask(-tNear >= tFar); /*! if no child is hit, pop next node */ if (unlikely(mask == 0)) goto pop; /*! one child is hit, continue with that child */ size_t r = __bsf(mask); mask = __btc(mask,r); if (likely(mask == 0)) { cur = node->child(r); continue; } /*! two children are hit, push far child, and continue with closer child */ NodeRef c0 = node->child(r); const float d0 = tNear[r]; r = __bsf(mask); mask = __btc(mask,r); NodeRef c1 = node->child(r); const float d1 = tNear[r]; if (likely(mask == 0)) { if (d0 < d1) { stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; cur = c0; continue; } else { stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; cur = c1; continue; } } /*! Here starts the slow path for 3 or 4 hit children. We push * all nodes onto the stack to sort them there. */ stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; /*! three children are hit, push all onto stack and sort 3 stack items, continue with closest child */ r = __bsf(mask); mask = __btc(mask,r); NodeRef c = node->child(r); float d = tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; if (likely(mask == 0)) { sort(stackPtr[-1],stackPtr[-2],stackPtr[-3]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; continue; } /*! four children are hit, push all onto stack and sort 4 stack items, continue with closest child */ r = __bsf(mask); mask = __btc(mask,r); c = node->child(r); d = tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; sort(stackPtr[-1],stackPtr[-2],stackPtr[-3],stackPtr[-4]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; } /*! this is a leaf node */ STAT3(normal.trav_leaves,1,1,1); size_t num; Triangle* tri = (Triangle*) cur.leaf(triPtr,num); for (size_t i=0; i<num; i++) TriangleIntersector::intersect(ray,tri[i],bvh->vertices); rayNearFar = insert<1>(rayNearFar,-ssef(ray.tfar)); } }
__forceinline bool occludedT(const BVH4* bvh, Ray& ray) { typedef typename TriangleIntersector::Triangle Triangle; typedef StackItemT<size_t> StackItem; typedef typename BVH4::NodeRef NodeRef; typedef typename BVH4::Node Node; /*! stack state */ NodeRef stack[1+3*BVH4::maxDepth]; //!< stack of nodes that still need to get traversed NodeRef* stackPtr = stack+1; //!< current stack pointer stack[0] = bvh->root; /*! load the ray into SIMD registers */ const avxf pos_neg = avxf(ssef(+0.0f),ssef(-0.0f)); const avxf neg_pos = avxf(ssef(-0.0f),ssef(+0.0f)); const avxf flipSignX = swapX ? neg_pos : pos_neg; const avxf flipSignY = swapY ? neg_pos : pos_neg; const avxf flipSignZ = swapZ ? neg_pos : pos_neg; const avx3f norg(-ray.org.x,-ray.org.y,-ray.org.z); const Vector3f ray_rdir = rcp_safe(ray.dir); const avx3f rdir(ray_rdir.x^flipSignX,ray_rdir.y^flipSignY,ray_rdir.z^flipSignZ); const avx3f org_rdir(avx3f(ray.org.x,ray.org.y,ray.org.z)*rdir); const avxf rayNearFar(ssef(ray.tnear),-ssef(ray.tfar)); const void* nodePtr = bvh->nodePtr(); const void* triPtr = bvh->triPtr(); /* pop loop */ while (true) pop: { /*! pop next node */ if (unlikely(stackPtr == stack)) break; stackPtr--; NodeRef cur = (NodeRef) *stackPtr; /* downtraversal loop */ while (true) { /*! stop if we found a leaf */ if (unlikely(cur.isLeaf())) break; STAT3(shadow.trav_nodes,1,1,1); /*! single ray intersection with 4 boxes */ const Node* node = cur.node(nodePtr); #if defined (__AVX2__) || defined(__MIC__) const avxf tLowerUpperX = msub(avxf::load(&node->lower_x), rdir.x, org_rdir.x); const avxf tLowerUpperY = msub(avxf::load(&node->lower_y), rdir.y, org_rdir.y); const avxf tLowerUpperZ = msub(avxf::load(&node->lower_z), rdir.z, org_rdir.z); #else const avxf tLowerUpperX = (norg.x + avxf::load(&node->lower_x)) * rdir.x; const avxf tLowerUpperY = (norg.y + avxf::load(&node->lower_y)) * rdir.y; const avxf tLowerUpperZ = (norg.z + avxf::load(&node->lower_z)) * rdir.z; #endif const avxf tNearFarX = swapX ? shuffle<1,0>(tLowerUpperX) : tLowerUpperX; const avxf tNearFarY = swapY ? shuffle<1,0>(tLowerUpperY) : tLowerUpperY; const avxf tNearFarZ = swapZ ? shuffle<1,0>(tLowerUpperZ) : tLowerUpperZ; const avxf tNearFar = max(tNearFarX,tNearFarY,tNearFarZ,rayNearFar); const ssef tNear = extract<0>(tNearFar); const ssef tFar = extract<1>(tNearFar); size_t mask = movemask(-tNear >= tFar); /*! if no child is hit, pop next node */ if (unlikely(mask == 0)) goto pop; /*! one child is hit, continue with that child */ size_t r = __bsf(mask); mask = __btc(mask,r); if (likely(mask == 0)) { cur = node->child(r); continue; } /*! two children are hit, push far child, and continue with closer child */ NodeRef c0 = node->child(r); const float d0 = tNear[r]; r = __bsf(mask); mask = __btc(mask,r); NodeRef c1 = node->child(r); const float d1 = tNear[r]; if (likely(mask == 0)) { if (d0 < d1) { *stackPtr = c1; stackPtr++; cur = c0; continue; } else { *stackPtr = c0; stackPtr++; cur = c1; continue; } } *stackPtr = c0; stackPtr++; *stackPtr = c1; stackPtr++; /*! three children are hit */ r = __bsf(mask); mask = __btc(mask,r); cur = node->child(r); *stackPtr = cur; stackPtr++; if (likely(mask == 0)) { stackPtr--; continue; } /*! four children are hit */ cur = node->child(3); } /*! this is a leaf node */ STAT3(shadow.trav_leaves,1,1,1); size_t num; Triangle* tri = (Triangle*) cur.leaf(triPtr,num); for (size_t i=0; i<num; i++) { if (TriangleIntersector::occluded(ray,tri[i],bvh->vertices)) { AVX_ZERO_UPPER(); return true; } } } AVX_ZERO_UPPER(); return false; }
void BVH4iIntersector1Scalar<TriangleIntersector>::occluded(const BVH4i* bvh, Ray& ray) { /*! stack state */ StackItem stack[1+3*BVH4i::maxDepth]; //!< stack of nodes StackItem *stackPtr = stack+1; //!< current stack pointer stack[0].ptr = bvh->root; stack[0].dist = neg_inf; /*! load the ray into SIMD registers */ const Vec3f rdir = rcp_safe(ray.dir); const Vec3f org_rdir = ray.org*rdir; const void* nodePtr = bvh->nodePtr(); const void* triPtr = bvh->triPtr(); /* pop loop */ while (true) pop: { if (unlikely(stackPtr == stack)) break; stackPtr--; NodeRef cur = NodeRef(stackPtr->ptr); /* downtraversal loop */ while (true) { /*! stop if we found a leaf */ if (unlikely(cur.isLeaf())) break; STAT3(shadow.trav_nodes,1,1,1); /*! single ray intersection with 4 boxes */ const Node* node = cur.node(nodePtr); size_t pushed = 0; for (size_t i=0;i<4;i++) { const float nearX = node->lower_x[i] * rdir.x - org_rdir.x; const float farX = node->upper_x[i] * rdir.x - org_rdir.x; const float nearY = node->lower_y[i] * rdir.y - org_rdir.y; const float farY = node->upper_y[i] * rdir.y - org_rdir.y; const float nearZ = node->lower_z[i] * rdir.z - org_rdir.z; const float farZ = node->upper_z[i] * rdir.z - org_rdir.z; const float tNearX = min(nearX,farX); const float tFarX = max(nearX,farX); const float tNearY = min(nearY,farY); const float tFarY = max(nearY,farY); const float tNearZ = min(nearZ,farZ); const float tFarZ = max(nearZ,farZ); const float tNear = max(tNearX,tNearY,tNearZ,ray.tnear); const float tFar = min(tFarX ,tFarY ,tFarZ ,ray.tfar); if (tNear <= tFar) { stackPtr->ptr = node->child(i); stackPtr->dist = tNear; stackPtr++; pushed++; } } if (pushed == 0) { goto pop; } else if (pushed == 1) { cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; continue; } else if (pushed == 2) { sort(stackPtr[-1],stackPtr[-2]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; continue; } else if (pushed == 3) { sort(stackPtr[-1],stackPtr[-2],stackPtr[-3]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; continue; } else { sort(stackPtr[-1],stackPtr[-2],stackPtr[-3],stackPtr[-4]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; } } /*! this is a leaf node */ STAT3(shadow.trav_leaves,1,1,1); size_t num; Triangle1* tri = (Triangle1*) cur.leaf(triPtr,num); for (size_t i=0;i<num;i++) if (occluded_vec3f(ray,tri[i],bvh->geometry)) { ray.geomID = 0; break; } if (ray.geomID == 0) break; } AVX_ZERO_UPPER(); }
void BVH4iIntersector1<TriangleIntersector>::intersect(const BVH4iIntersector1* This, Ray& ray) { AVX_ZERO_UPPER(); STAT3(normal.travs,1,1,1); /*! stack state */ const BVH4i* bvh = This->bvh; StackItem stack[1+3*BVH4i::maxDepth]; //!< stack of nodes StackItem* stackPtr = stack+1; //!< current stack pointer stack[0].ptr = bvh->root; stack[0].dist = neg_inf; /*! offsets to select the side that becomes the lower or upper bound */ const size_t nearX = ray.dir.x >= 0.0f ? 0*sizeof(ssef_m) : 1*sizeof(ssef_m); const size_t nearY = ray.dir.y >= 0.0f ? 2*sizeof(ssef_m) : 3*sizeof(ssef_m); const size_t nearZ = ray.dir.z >= 0.0f ? 4*sizeof(ssef_m) : 5*sizeof(ssef_m); /*! load the ray into SIMD registers */ const sse3f norg(-ray.org.x,-ray.org.y,-ray.org.z); const Vector3f ray_rdir = rcp_safe(ray.dir); const sse3f rdir(ray_rdir.x,ray_rdir.y,ray_rdir.z); const Vector3f ray_org_rdir = ray.org*ray_rdir; const sse3f org_rdir(ray_org_rdir.x,ray_org_rdir.y,ray_org_rdir.z); const ssef rayNear(ray.tnear); ssef rayFar(ray.tfar); const void* nodePtr = bvh->nodePtr(); const void* triPtr = bvh->triPtr(); /* pop loop */ while (true) pop: { /*! pop next node */ if (unlikely(stackPtr == stack)) break; stackPtr--; NodeRef cur = NodeRef(stackPtr->ptr); /*! if popped node is too far, pop next one */ if (unlikely(stackPtr->dist > ray.tfar)) continue; /* downtraversal loop */ while (true) { /*! stop if we found a leaf */ if (unlikely(cur.isLeaf())) break; STAT3(normal.trav_nodes,1,1,1); /*! single ray intersection with 4 boxes */ const Node* node = cur.node(nodePtr); const size_t farX = nearX ^ 16, farY = nearY ^ 16, farZ = nearZ ^ 16; #if defined (__AVX2__) const ssef tNearX = msub(ssef((const char*)nodePtr+(size_t)cur+nearX), rdir.x, org_rdir.x); const ssef tNearY = msub(ssef((const char*)nodePtr+(size_t)cur+nearY), rdir.y, org_rdir.y); const ssef tNearZ = msub(ssef((const char*)nodePtr+(size_t)cur+nearZ), rdir.z, org_rdir.z); const ssef tFarX = msub(ssef((const char*)nodePtr+(size_t)cur+farX ), rdir.x, org_rdir.x); const ssef tFarY = msub(ssef((const char*)nodePtr+(size_t)cur+farY ), rdir.y, org_rdir.y); const ssef tFarZ = msub(ssef((const char*)nodePtr+(size_t)cur+farZ ), rdir.z, org_rdir.z); #else const ssef tNearX = (norg.x + ssef((const char*)nodePtr+(size_t)cur+nearX)) * rdir.x; const ssef tNearY = (norg.y + ssef((const char*)nodePtr+(size_t)cur+nearY)) * rdir.y; const ssef tNearZ = (norg.z + ssef((const char*)nodePtr+(size_t)cur+nearZ)) * rdir.z; const ssef tFarX = (norg.x + ssef((const char*)nodePtr+(size_t)cur+farX )) * rdir.x; const ssef tFarY = (norg.y + ssef((const char*)nodePtr+(size_t)cur+farY )) * rdir.y; const ssef tFarZ = (norg.z + ssef((const char*)nodePtr+(size_t)cur+farZ )) * rdir.z; #endif const ssef tNear = max(tNearX,tNearY,tNearZ,rayNear); const ssef tFar = min(tFarX ,tFarY ,tFarZ ,rayFar); size_t mask = movemask(tNear <= tFar); /*! if no child is hit, pop next node */ if (unlikely(mask == 0)) goto pop; /*! one child is hit, continue with that child */ size_t r = __bsf(mask); mask = __btc(mask,r); if (likely(mask == 0)) { cur = node->child(r); continue; } /*! two children are hit, push far child, and continue with closer child */ NodeRef c0 = node->child(r); const float d0 = tNear[r]; r = __bsf(mask); mask = __btc(mask,r); NodeRef c1 = node->child(r); const float d1 = tNear[r]; if (likely(mask == 0)) { if (d0 < d1) { stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; cur = c0; continue; } else { stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; cur = c1; continue; } } /*! Here starts the slow path for 3 or 4 hit children. We push * all nodes onto the stack to sort them there. */ stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; /*! three children are hit, push all onto stack and sort 3 stack items, continue with closest child */ r = __bsf(mask); mask = __btc(mask,r); NodeRef c = node->child(r); float d = tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; if (likely(mask == 0)) { sort(stackPtr[-1],stackPtr[-2],stackPtr[-3]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; continue; } /*! four children are hit, push all onto stack and sort 4 stack items, continue with closest child */ r = __bsf(mask); mask = __btc(mask,r); c = node->child(r); d = tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; sort(stackPtr[-1],stackPtr[-2],stackPtr[-3],stackPtr[-4]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; } /*! this is a leaf node */ STAT3(normal.trav_leaves,1,1,1); size_t num; Triangle* tri = (Triangle*) cur.leaf(triPtr,num); for (size_t i=0; i<num; i++) TriangleIntersector::intersect(ray,tri[i],bvh->vertices); rayFar = ray.tfar; } AVX_ZERO_UPPER(); }
bool BVH4iIntersector1<TriangleIntersector>::occluded(const BVH4iIntersector1* This, Ray& ray) { AVX_ZERO_UPPER(); STAT3(shadow.travs,1,1,1); /*! stack state */ const BVH4i* bvh = This->bvh; NodeRef stack[1+3*BVH4i::maxDepth]; //!< stack of nodes that still need to get traversed NodeRef* stackPtr = stack+1; //!< current stack pointer stack[0] = bvh->root; /*! offsets to select the side that becomes the lower or upper bound */ const size_t nearX = ray.dir.x >= 0 ? 0*sizeof(ssef_m) : 1*sizeof(ssef_m); const size_t nearY = ray.dir.y >= 0 ? 2*sizeof(ssef_m) : 3*sizeof(ssef_m); const size_t nearZ = ray.dir.z >= 0 ? 4*sizeof(ssef_m) : 5*sizeof(ssef_m); /*! load the ray into SIMD registers */ const sse3f norg(-ray.org.x,-ray.org.y,-ray.org.z); const Vector3f ray_rdir = rcp_safe(ray.dir); const sse3f rdir(ray_rdir.x,ray_rdir.y,ray_rdir.z); const Vector3f ray_org_rdir = ray.org*ray_rdir; const sse3f org_rdir(ray_org_rdir.x,ray_org_rdir.y,ray_org_rdir.z); const ssef rayNear(ray.tnear); const ssef rayFar(ray.tfar); const void* nodePtr = bvh->nodePtr(); const void* triPtr = bvh->triPtr(); /* pop loop */ while (true) pop: { /*! pop next node */ if (unlikely(stackPtr == stack)) break; stackPtr--; NodeRef cur = (NodeRef) *stackPtr; /* downtraversal loop */ while (true) { /*! stop if we found a leaf */ if (unlikely(cur.isLeaf())) break; STAT3(shadow.trav_nodes,1,1,1); /*! single ray intersection with 4 boxes */ const Node* node = cur.node(nodePtr); const size_t farX = nearX ^ 16, farY = nearY ^ 16, farZ = nearZ ^ 16; #if defined (__AVX2__) const ssef tNearX = msub(ssef((const char*)nodePtr+(size_t)cur+nearX), rdir.x, org_rdir.x); const ssef tNearY = msub(ssef((const char*)nodePtr+(size_t)cur+nearY), rdir.y, org_rdir.y); const ssef tNearZ = msub(ssef((const char*)nodePtr+(size_t)cur+nearZ), rdir.z, org_rdir.z); const ssef tFarX = msub(ssef((const char*)nodePtr+(size_t)cur+farX ), rdir.x, org_rdir.x); const ssef tFarY = msub(ssef((const char*)nodePtr+(size_t)cur+farY ), rdir.y, org_rdir.y); const ssef tFarZ = msub(ssef((const char*)nodePtr+(size_t)cur+farZ ), rdir.z, org_rdir.z); #else const ssef tNearX = (norg.x + ssef((const char*)nodePtr+(size_t)cur+nearX)) * rdir.x; const ssef tNearY = (norg.y + ssef((const char*)nodePtr+(size_t)cur+nearY)) * rdir.y; const ssef tNearZ = (norg.z + ssef((const char*)nodePtr+(size_t)cur+nearZ)) * rdir.z; const ssef tFarX = (norg.x + ssef((const char*)nodePtr+(size_t)cur+farX )) * rdir.x; const ssef tFarY = (norg.y + ssef((const char*)nodePtr+(size_t)cur+farY )) * rdir.y; const ssef tFarZ = (norg.z + ssef((const char*)nodePtr+(size_t)cur+farZ )) * rdir.z; #endif const ssef tNear = max(tNearX,tNearY,tNearZ,rayNear); const ssef tFar = min(tFarX ,tFarY ,tFarZ ,rayFar); size_t mask = movemask(tNear <= tFar); /*! if no child is hit, pop next node */ if (unlikely(mask == 0)) goto pop; /*! one child is hit, continue with that child */ size_t r = __bsf(mask); mask = __btc(mask,r); if (likely(mask == 0)) { cur = node->child(r); continue; } /*! two children are hit, push far child, and continue with closer child */ NodeRef c0 = node->child(r); const float d0 = tNear[r]; r = __bsf(mask); mask = __btc(mask,r); NodeRef c1 = node->child(r); const float d1 = tNear[r]; if (likely(mask == 0)) { if (d0 < d1) { *stackPtr = c1; stackPtr++; cur = c0; continue; } else { *stackPtr = c0; stackPtr++; cur = c1; continue; } } *stackPtr = c0; stackPtr++; *stackPtr = c1; stackPtr++; /*! three children are hit */ r = __bsf(mask); mask = __btc(mask,r); cur = node->child(r); *stackPtr = cur; stackPtr++; if (likely(mask == 0)) { stackPtr--; continue; } /*! four children are hit */ cur = node->child(3); } /*! this is a leaf node */ STAT3(shadow.trav_leaves,1,1,1); size_t num; Triangle* tri = (Triangle*) cur.leaf(triPtr,num); for (size_t i=0; i<num; i++) { if (TriangleIntersector::occluded(ray,tri[i],bvh->vertices)) { AVX_ZERO_UPPER(); return true; } } } AVX_ZERO_UPPER(); return false; }
void BVH8Intersector1<robust,PrimitiveIntersector>::intersect(const BVH8* bvh, Ray& ray) { /*! perform per ray precalculations required by the primitive intersector */ Precalculations pre(ray,bvh); /*! stack state */ StackItemT<NodeRef> stack[stackSize]; //!< stack of nodes StackItemT<NodeRef>* stackPtr = stack+1; //!< current stack pointer StackItemT<NodeRef>* stackEnd = stack+stackSize; stack[0].ptr = bvh->root; stack[0].dist = neg_inf; /* filter out invalid rays */ #if defined(RTCORE_IGNORE_INVALID_RAYS) if (!ray.valid()) return; #endif /* verify correct input */ assert(ray.tnear > -FLT_MIN); //assert(!(types & BVH4::FLAG_NODE_MB) || (ray.time >= 0.0f && ray.time <= 1.0f)); /*! load the ray into SIMD registers */ const Vec3f8 norg(-ray.org.x,-ray.org.y,-ray.org.z); const Vec3fa ray_rdir = rcp_safe(ray.dir); const Vec3f8 rdir(ray_rdir.x,ray_rdir.y,ray_rdir.z); const Vec3fa ray_org_rdir = ray.org*ray_rdir; const Vec3f8 org_rdir(ray_org_rdir.x,ray_org_rdir.y,ray_org_rdir.z); const float8 ray_near(ray.tnear); float8 ray_far(ray.tfar); /*! offsets to select the side that becomes the lower or upper bound */ const size_t nearX = ray_rdir.x >= 0.0f ? 0*sizeof(float8) : 1*sizeof(float8); const size_t nearY = ray_rdir.y >= 0.0f ? 2*sizeof(float8) : 3*sizeof(float8); const size_t nearZ = ray_rdir.z >= 0.0f ? 4*sizeof(float8) : 5*sizeof(float8); /* pop loop */ while (true) pop: { /*! pop next node */ if (unlikely(stackPtr == stack)) break; stackPtr--; NodeRef cur = NodeRef(stackPtr->ptr); /*! if popped node is too far, pop next one */ if (unlikely(*(float*)&stackPtr->dist > ray.tfar)) continue; /* downtraversal loop */ while (true) { /*! stop if we found a leaf */ if (unlikely(cur.isLeaf())) break; STAT3(normal.trav_nodes,1,1,1); /*! single ray intersection with 4 boxes */ const Node* node = cur.node(); const size_t farX = nearX ^ sizeof(float8), farY = nearY ^ sizeof(float8), farZ = nearZ ^ sizeof(float8); #if defined (__AVX2__) const float8 tNearX = msub(load8f((const char*)node+nearX), rdir.x, org_rdir.x); const float8 tNearY = msub(load8f((const char*)node+nearY), rdir.y, org_rdir.y); const float8 tNearZ = msub(load8f((const char*)node+nearZ), rdir.z, org_rdir.z); const float8 tFarX = msub(load8f((const char*)node+farX ), rdir.x, org_rdir.x); const float8 tFarY = msub(load8f((const char*)node+farY ), rdir.y, org_rdir.y); const float8 tFarZ = msub(load8f((const char*)node+farZ ), rdir.z, org_rdir.z); #else const float8 tNearX = (norg.x + load8f((const char*)node+nearX)) * rdir.x; const float8 tNearY = (norg.y + load8f((const char*)node+nearY)) * rdir.y; const float8 tNearZ = (norg.z + load8f((const char*)node+nearZ)) * rdir.z; const float8 tFarX = (norg.x + load8f((const char*)node+farX )) * rdir.x; const float8 tFarY = (norg.y + load8f((const char*)node+farY )) * rdir.y; const float8 tFarZ = (norg.z + load8f((const char*)node+farZ )) * rdir.z; #endif const float round_down = 1.0f-2.0f*float(ulp); const float round_up = 1.0f+2.0f*float(ulp); #if defined(__AVX2__) const float8 tNear = maxi(maxi(tNearX,tNearY),maxi(tNearZ,ray_near)); const float8 tFar = mini(mini(tFarX ,tFarY ),mini(tFarZ ,ray_far )); const bool8 vmask = robust ? (round_down*tNear > round_up*tFar) : cast(tNear) > cast(tFar); size_t mask = movemask(vmask)^0xff; #else const float8 tNear = max(tNearX,tNearY,tNearZ,ray_near); const float8 tFar = min(tFarX ,tFarY ,tFarZ ,ray_far); const bool8 vmask = robust ? (round_down*tNear > round_up*tFar) : tNear <= tFar; size_t mask = movemask(vmask); #endif /*! if no child is hit, pop next node */ if (unlikely(mask == 0)) goto pop; /*! one child is hit, continue with that child */ size_t r = __bscf(mask); if (likely(mask == 0)) { cur = node->child(r); cur.prefetch(); assert(cur != BVH8::emptyNode); continue; } /*! two children are hit, push far child, and continue with closer child */ NodeRef c0 = node->child(r); c0.prefetch(); const unsigned int d0 = ((unsigned int*)&tNear)[r]; r = __bscf(mask); NodeRef c1 = node->child(r); c1.prefetch(); const unsigned int d1 = ((unsigned int*)&tNear)[r]; assert(c0 != BVH8::emptyNode); assert(c1 != BVH8::emptyNode); if (likely(mask == 0)) { assert(stackPtr < stackEnd); if (d0 < d1) { stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; cur = c0; continue; } else { stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; cur = c1; continue; } } /*! Here starts the slow path for 3 or 4 hit children. We push * all nodes onto the stack to sort them there. */ assert(stackPtr < stackEnd); stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; assert(stackPtr < stackEnd); stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; /*! three children are hit, push all onto stack and sort 3 stack items, continue with closest child */ assert(stackPtr < stackEnd); r = __bscf(mask); NodeRef c = node->child(r); c.prefetch(); unsigned int d = ((unsigned int*)&tNear)[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; assert(c != BVH8::emptyNode); if (likely(mask == 0)) { sort(stackPtr[-1],stackPtr[-2],stackPtr[-3]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; continue; } /*! four children are hit, push all onto stack and sort 4 stack items, continue with closest child */ r = __bscf(mask); c = node->child(r); c.prefetch(); d = *(unsigned int*)&tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; if (likely(mask == 0)) { sort(stackPtr[-1],stackPtr[-2],stackPtr[-3],stackPtr[-4]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; continue; } /*! fallback case if more than 4 children are hit */ while (1) { r = __bscf(mask); assert(stackPtr < stackEnd); c = node->child(r); c.prefetch(); d = *(unsigned int*)&tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; if (unlikely(mask == 0)) break; } cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; } /*! this is a leaf node */ assert(cur != BVH8::emptyNode); STAT3(normal.trav_leaves,1,1,1); size_t num; Primitive* prim = (Primitive*) cur.leaf(num); size_t lazy_node = 0; PrimitiveIntersector::intersect(pre,ray,prim,num,bvh->scene,lazy_node); ray_far = ray.tfar; if (unlikely(lazy_node)) { stackPtr->ptr = lazy_node; stackPtr->dist = inf; stackPtr++; } } AVX_ZERO_UPPER(); }
void BVH8Intersector1<robust,PrimitiveIntersector>::occluded(const BVH8* bvh, Ray& ray) { /*! perform per ray precalculations required by the primitive intersector */ Precalculations pre(ray,bvh); /*! stack state */ NodeRef stack[stackSize]; //!< stack of nodes that still need to get traversed NodeRef* stackPtr = stack+1; //!< current stack pointer NodeRef* stackEnd = stack+stackSize; stack[0] = bvh->root; /* filter out invalid rays */ #if defined(RTCORE_IGNORE_INVALID_RAYS) if (!ray.valid()) return; #endif /* verify correct input */ assert(ray.tnear > -FLT_MIN); //assert(!(types & BVH4::FLAG_NODE_MB) || (ray.time >= 0.0f && ray.time <= 1.0f)); /*! load the ray into SIMD registers */ const Vec3f8 norg(-ray.org.x,-ray.org.y,-ray.org.z); const Vec3fa ray_rdir = rcp_safe(ray.dir); const Vec3f8 rdir(ray_rdir.x,ray_rdir.y,ray_rdir.z); const Vec3fa ray_org_rdir = ray.org*ray_rdir; const Vec3f8 org_rdir(ray_org_rdir.x,ray_org_rdir.y,ray_org_rdir.z); const float8 ray_near(ray.tnear); float8 ray_far(ray.tfar); /*! offsets to select the side that becomes the lower or upper bound */ const size_t nearX = ray_rdir.x >= 0 ? 0*sizeof(float8) : 1*sizeof(float8); const size_t nearY = ray_rdir.y >= 0 ? 2*sizeof(float8) : 3*sizeof(float8); const size_t nearZ = ray_rdir.z >= 0 ? 4*sizeof(float8) : 5*sizeof(float8); /* pop loop */ while (true) pop: { /*! pop next node */ if (unlikely(stackPtr == stack)) break; stackPtr--; NodeRef cur = (NodeRef) *stackPtr; /* downtraversal loop */ while (true) { /*! stop if we found a leaf */ if (unlikely(cur.isLeaf())) break; STAT3(shadow.trav_nodes,1,1,1); /*! single ray intersection with 4 boxes */ const Node* node = cur.node(); const size_t farX = nearX ^ sizeof(float8), farY = nearY ^ sizeof(float8), farZ = nearZ ^ sizeof(float8); #if defined (__AVX2__) const float8 tNearX = msub(load8f((const char*)node+nearX), rdir.x, org_rdir.x); const float8 tNearY = msub(load8f((const char*)node+nearY), rdir.y, org_rdir.y); const float8 tNearZ = msub(load8f((const char*)node+nearZ), rdir.z, org_rdir.z); const float8 tFarX = msub(load8f((const char*)node+farX ), rdir.x, org_rdir.x); const float8 tFarY = msub(load8f((const char*)node+farY ), rdir.y, org_rdir.y); const float8 tFarZ = msub(load8f((const char*)node+farZ ), rdir.z, org_rdir.z); #else const float8 tNearX = (norg.x + load8f((const char*)node+nearX)) * rdir.x; const float8 tNearY = (norg.y + load8f((const char*)node+nearY)) * rdir.y; const float8 tNearZ = (norg.z + load8f((const char*)node+nearZ)) * rdir.z; const float8 tFarX = (norg.x + load8f((const char*)node+farX )) * rdir.x; const float8 tFarY = (norg.y + load8f((const char*)node+farY )) * rdir.y; const float8 tFarZ = (norg.z + load8f((const char*)node+farZ )) * rdir.z; #endif #if defined(__AVX2__) const float8 tNear = maxi(maxi(tNearX,tNearY),maxi(tNearZ,ray_near)); const float8 tFar = mini(mini(tFarX ,tFarY ),mini(tFarZ ,ray_far )); const bool8 vmask = cast(tNear) > cast(tFar); size_t mask = movemask(vmask)^0xff; #else const float8 tNear = max(tNearX,tNearY,tNearZ,ray_near); const float8 tFar = min(tFarX ,tFarY ,tFarZ ,ray_far); const bool8 vmask = tNear <= tFar; size_t mask = movemask(vmask); #endif /*! if no child is hit, pop next node */ if (unlikely(mask == 0)) goto pop; /*! one child is hit, continue with that child */ size_t r = __bscf(mask); if (likely(mask == 0)) { cur = node->child(r); cur.prefetch(); assert(cur != BVH8::emptyNode); continue; } /*! two children are hit, push far child, and continue with closer child */ NodeRef c0 = node->child(r); c0.prefetch(); const unsigned int d0 = ((unsigned int*)&tNear)[r]; r = __bscf(mask); NodeRef c1 = node->child(r); c1.prefetch(); const unsigned int d1 = ((unsigned int*)&tNear)[r]; assert(c0 != BVH8::emptyNode); assert(c1 != BVH8::emptyNode); if (likely(mask == 0)) { assert(stackPtr < stackEnd); if (d0 < d1) { *stackPtr = c1; stackPtr++; cur = c0; continue; } else { *stackPtr = c0; stackPtr++; cur = c1; continue; } } assert(stackPtr < stackEnd); *stackPtr = c0; stackPtr++; assert(stackPtr < stackEnd); *stackPtr = c1; stackPtr++; /*! three children are hit */ r = __bscf(mask); cur = node->child(r); cur.prefetch(); *stackPtr = cur; stackPtr++; if (likely(mask == 0)) { stackPtr--; continue; } /*! process more than three children */ while(1) { r = __bscf(mask); NodeRef c = node->child(r); c.prefetch(); *stackPtr = c; stackPtr++; if (unlikely(mask == 0)) break; } cur = (NodeRef) stackPtr[-1]; stackPtr--; } /*! this is a leaf node */ assert(cur != BVH8::emptyNode); STAT3(shadow.trav_leaves,1,1,1); size_t num; Primitive* prim = (Primitive*) cur.leaf(num); size_t lazy_node = 0; if (PrimitiveIntersector::occluded(pre,ray,prim,num,bvh->scene,lazy_node)) { ray.geomID = 0; break; } if (unlikely(lazy_node)) { *stackPtr = (NodeRef)lazy_node; stackPtr++; } } AVX_ZERO_UPPER(); }
void BVH4Intersector4Chunk<PrimitiveIntersector4>::intersect(sseb* valid_i, BVH4* bvh, Ray4& ray) { /* load ray */ const sseb valid0 = *valid_i; const sse3f rdir = rcp_safe(ray.dir); const sse3f org(ray.org), org_rdir = org * rdir; ssef ray_tnear = select(valid0,ray.tnear,ssef(pos_inf)); ssef ray_tfar = select(valid0,ray.tfar ,ssef(neg_inf)); const ssef inf = ssef(pos_inf); Precalculations pre(valid0,ray); /* allocate stack and push root node */ ssef stack_near[stackSize]; NodeRef stack_node[stackSize]; stack_node[0] = BVH4::invalidNode; stack_near[0] = inf; stack_node[1] = bvh->root; stack_near[1] = ray_tnear; NodeRef* stackEnd = stack_node+stackSize; NodeRef* __restrict__ sptr_node = stack_node + 2; ssef* __restrict__ sptr_near = stack_near + 2; while (1) { /* pop next node from stack */ assert(sptr_node > stack_node); sptr_node--; sptr_near--; NodeRef curNode = *sptr_node; if (unlikely(curNode == BVH4::invalidNode)) { assert(sptr_node == stack_node); break; } /* cull node if behind closest hit point */ ssef curDist = *sptr_near; if (unlikely(none(ray_tfar > curDist))) continue; while (1) { /* test if this is a leaf node */ if (unlikely(curNode.isLeaf())) break; const sseb valid_node = ray_tfar > curDist; STAT3(normal.trav_nodes,1,popcnt(valid_node),4); const Node* __restrict__ const node = curNode.node(); /* pop of next node */ assert(sptr_node > stack_node); sptr_node--; sptr_near--; curNode = *sptr_node; curDist = *sptr_near; #pragma unroll(4) for (unsigned i=0; i<BVH4::N; i++) { const NodeRef child = node->children[i]; if (unlikely(child == BVH4::emptyNode)) break; #if defined(__AVX2__) const ssef lclipMinX = msub(node->lower_x[i],rdir.x,org_rdir.x); const ssef lclipMinY = msub(node->lower_y[i],rdir.y,org_rdir.y); const ssef lclipMinZ = msub(node->lower_z[i],rdir.z,org_rdir.z); const ssef lclipMaxX = msub(node->upper_x[i],rdir.x,org_rdir.x); const ssef lclipMaxY = msub(node->upper_y[i],rdir.y,org_rdir.y); const ssef lclipMaxZ = msub(node->upper_z[i],rdir.z,org_rdir.z); #else const ssef lclipMinX = (node->lower_x[i] - org.x) * rdir.x; const ssef lclipMinY = (node->lower_y[i] - org.y) * rdir.y; const ssef lclipMinZ = (node->lower_z[i] - org.z) * rdir.z; const ssef lclipMaxX = (node->upper_x[i] - org.x) * rdir.x; const ssef lclipMaxY = (node->upper_y[i] - org.y) * rdir.y; const ssef lclipMaxZ = (node->upper_z[i] - org.z) * rdir.z; #endif #if defined(__SSE4_1__) const ssef lnearP = maxi(maxi(mini(lclipMinX, lclipMaxX), mini(lclipMinY, lclipMaxY)), mini(lclipMinZ, lclipMaxZ)); const ssef lfarP = mini(mini(maxi(lclipMinX, lclipMaxX), maxi(lclipMinY, lclipMaxY)), maxi(lclipMinZ, lclipMaxZ)); const sseb lhit = maxi(lnearP,ray_tnear) <= mini(lfarP,ray_tfar); #else const ssef lnearP = max(max(min(lclipMinX, lclipMaxX), min(lclipMinY, lclipMaxY)), min(lclipMinZ, lclipMaxZ)); const ssef lfarP = min(min(max(lclipMinX, lclipMaxX), max(lclipMinY, lclipMaxY)), max(lclipMinZ, lclipMaxZ)); const sseb lhit = max(lnearP,ray_tnear) <= min(lfarP,ray_tfar); #endif /* if we hit the child we choose to continue with that child if it is closer than the current next child, or we push it onto the stack */ if (likely(any(lhit))) { assert(sptr_node < stackEnd); const ssef childDist = select(lhit,lnearP,inf); const NodeRef child = node->children[i]; assert(child != BVH4::emptyNode); sptr_node++; sptr_near++; /* push cur node onto stack and continue with hit child */ if (any(childDist < curDist)) { *(sptr_node-1) = curNode; *(sptr_near-1) = curDist; curDist = childDist; curNode = child; } /* push hit child onto stack */ else { *(sptr_node-1) = child; *(sptr_near-1) = childDist; } } } } /* return if stack is empty */ if (unlikely(curNode == BVH4::invalidNode)) { assert(sptr_node == stack_node); break; } /* intersect leaf */ const sseb valid_leaf = ray_tfar > curDist; STAT3(normal.trav_leaves,1,popcnt(valid_leaf),4); size_t items; const Primitive* prim = (Primitive*) curNode.leaf(items); PrimitiveIntersector4::intersect(valid_leaf,pre,ray,prim,items,bvh->geometry); ray_tfar = select(valid_leaf,ray.tfar,ray_tfar); } AVX_ZERO_UPPER(); }
__forceinline void BVH8iIntersector8Hybrid<TriangleIntersector8>::intersect1(const BVH8i* bvh, NodeRef root, const size_t k, Ray8& ray,const avx3f &ray_org, const avx3f &ray_dir, const avx3f &ray_rdir, const avxf &ray_tnear, const avxf &ray_tfar, const avx3i& nearXYZ) { /*! stack state */ StackItemInt64 stack[stackSizeSingle]; //!< stack of nodes StackItemInt64* stackPtr = stack+1; //!< current stack pointer StackItemInt64* stackEnd = stack+stackSizeSingle; stack[0].ptr = root; stack[0].dist = neg_inf; /*! offsets to select the side that becomes the lower or upper bound */ const size_t nearX = nearXYZ.x[k]; const size_t nearY = nearXYZ.y[k]; const size_t nearZ = nearXYZ.z[k]; /*! load the ray into SIMD registers */ const avx3f org (ray_org .x[k],ray_org .y[k],ray_org .z[k]); const avx3f rdir(ray_rdir.x[k],ray_rdir.y[k],ray_rdir.z[k]); const avx3f org_rdir(org*rdir); avxf rayNear(ray_tnear[k]), rayFar(ray_tfar[k]); const Node * __restrict__ nodes = (Node *)bvh->nodePtr(); const Triangle * __restrict__ accel = (Triangle*)bvh->triPtr(); /* pop loop */ while (true) pop: { /*! pop next node */ if (unlikely(stackPtr == stack)) break; stackPtr--; NodeRef cur = NodeRef(stackPtr->ptr); /*! if popped node is too far, pop next one */ if (unlikely(*(float*)&stackPtr->dist > ray.tfar[k])) continue; /* downtraversal loop */ while (true) { /*! stop if we found a leaf */ if (unlikely(cur.isLeaf())) break; STAT3(normal.trav_nodes,1,1,1); /*! single ray intersection with 4 boxes */ const Node* node = (Node*)cur.node(nodes); const size_t farX = nearX ^ sizeof(avxf), farY = nearY ^ sizeof(avxf), farZ = nearZ ^ sizeof(avxf); #if defined (__AVX2__) const avxf tNearX = msub(load8f((const char*)node+nearX), rdir.x, org_rdir.x); const avxf tNearY = msub(load8f((const char*)node+nearY), rdir.y, org_rdir.y); const avxf tNearZ = msub(load8f((const char*)node+nearZ), rdir.z, org_rdir.z); const avxf tFarX = msub(load8f((const char*)node+farX ), rdir.x, org_rdir.x); const avxf tFarY = msub(load8f((const char*)node+farY ), rdir.y, org_rdir.y); const avxf tFarZ = msub(load8f((const char*)node+farZ ), rdir.z, org_rdir.z); #else const avxf tNearX = (load8f((const char*)node+nearX) - org.x) * rdir.x; const avxf tNearY = (load8f((const char*)node+nearY) - org.y) * rdir.y; const avxf tNearZ = (load8f((const char*)node+nearZ) - org.z) * rdir.z; const avxf tFarX = (load8f((const char*)node+farX ) - org.x) * rdir.x; const avxf tFarY = (load8f((const char*)node+farY ) - org.y) * rdir.y; const avxf tFarZ = (load8f((const char*)node+farZ ) - org.z) * rdir.z; #endif #if defined(__AVX2__) const avxf tNear = maxi(maxi(tNearX,tNearY),maxi(tNearZ,rayNear)); const avxf tFar = mini(mini(tFarX ,tFarY ),mini(tFarZ ,rayFar )); const avxb vmask = cast(tNear) > cast(tFar); unsigned int mask = movemask(vmask)^0xff; #else const avxf tNear = max(tNearX,tNearY,tNearZ,rayNear); const avxf tFar = min(tFarX ,tFarY ,tFarZ ,rayFar); const avxb vmask = tNear <= tFar; unsigned int mask = movemask(vmask); #endif /*! if no child is hit, pop next node */ if (unlikely(mask == 0)) goto pop; /*! one child is hit, continue with that child */ size_t r = __bscf(mask); if (likely(mask == 0)) { cur = node->child(r); assert(cur != BVH4i::emptyNode); continue; } /*! two children are hit, push far child, and continue with closer child */ NodeRef c0 = node->child(r); const unsigned int d0 = ((unsigned int*)&tNear)[r]; r = __bscf(mask); NodeRef c1 = node->child(r); const unsigned int d1 = ((unsigned int*)&tNear)[r]; assert(c0 != BVH4i::emptyNode); assert(c1 != BVH4i::emptyNode); if (likely(mask == 0)) { assert(stackPtr < stackEnd); if (d0 < d1) { stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; cur = c0; continue; } else { stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; cur = c1; continue; } } /*! Here starts the slow path for 3 or 4 hit children. We push * all nodes onto the stack to sort them there. */ assert(stackPtr < stackEnd); stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; assert(stackPtr < stackEnd); stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; /*! three children are hit, push all onto stack and sort 3 stack items, continue with closest child */ assert(stackPtr < stackEnd); r = __bscf(mask); NodeRef c = node->child(r); unsigned int d = ((unsigned int*)&tNear)[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; assert(c0 != BVH4i::emptyNode); if (likely(mask == 0)) { sort(stackPtr[-1],stackPtr[-2],stackPtr[-3]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; continue; } /*! four children are hit, push all onto stack and sort 4 stack items, continue with closest child */ assert(stackPtr < stackEnd); r = __bscf(mask); c = node->child(r); d = ((unsigned int*)&tNear)[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; assert(c != BVH4i::emptyNode); if (likely(mask == 0)) { sort(stackPtr[-1],stackPtr[-2],stackPtr[-3],stackPtr[-4]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; continue; } while(1) { r = __bscf(mask); c = node->child(r); d = ((unsigned int*)&tNear)[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; if (unlikely(mask == 0)) break; } cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; } /*! this is a leaf node */ STAT3(normal.trav_leaves,1,1,1); size_t num; Triangle* prim = (Triangle*) cur.leaf(accel,num); TriangleIntersector8::intersect(ray,k,prim,num,bvh->geometry); rayFar = ray.tfar[k]; } }
void BVH4Intersector4Hybrid<PrimitiveIntersector4>::intersect(sseb* valid_i, BVH4* bvh, Ray4& ray) { /* load ray */ const sseb valid0 = *valid_i; sse3f ray_org = ray.org, ray_dir = ray.dir; ssef ray_tnear = ray.tnear, ray_tfar = ray.tfar; #if defined(__FIX_RAYS__) const ssef float_range = 0.1f*FLT_MAX; ray_org = clamp(ray_org,sse3f(-float_range),sse3f(+float_range)); ray_dir = clamp(ray_dir,sse3f(-float_range),sse3f(+float_range)); ray_tnear = max(ray_tnear,FLT_MIN); ray_tfar = min(ray_tfar,float(inf)); #endif const sse3f rdir = rcp_safe(ray_dir); const sse3f org(ray_org), org_rdir = org * rdir; ray_tnear = select(valid0,ray_tnear,ssef(pos_inf)); ray_tfar = select(valid0,ray_tfar ,ssef(neg_inf)); const ssef inf = ssef(pos_inf); /* allocate stack and push root node */ ssef stack_near[stackSizeChunk]; NodeRef stack_node[stackSizeChunk]; stack_node[0] = BVH4::invalidNode; stack_near[0] = inf; stack_node[1] = bvh->root; stack_near[1] = ray_tnear; NodeRef* stackEnd = stack_node+stackSizeChunk; NodeRef* __restrict__ sptr_node = stack_node + 2; ssef* __restrict__ sptr_near = stack_near + 2; while (1) { /* pop next node from stack */ assert(sptr_node > stack_node); sptr_node--; sptr_near--; NodeRef curNode = *sptr_node; if (unlikely(curNode == BVH4::invalidNode)) { assert(sptr_node == stack_node); break; } /* cull node if behind closest hit point */ ssef curDist = *sptr_near; const sseb active = curDist < ray_tfar; if (unlikely(none(active))) continue; /* switch to single ray traversal */ #if !defined(__WIN32__) || defined(__X86_64__) size_t bits = movemask(active); if (unlikely(__popcnt(bits) <= SWITCH_THRESHOLD)) { for (size_t i=__bsf(bits); bits!=0; bits=__btc(bits,i), i=__bsf(bits)) { intersect1(bvh,curNode,i,ray,ray_org,ray_dir,rdir,ray_tnear,ray_tfar); } ray_tfar = ray.tfar; continue; } #endif while (1) { /* test if this is a leaf node */ if (unlikely(curNode.isLeaf())) break; const sseb valid_node = ray_tfar > curDist; STAT3(normal.trav_nodes,1,popcnt(valid_node),4); const Node* __restrict__ const node = curNode.node(); /* pop of next node */ assert(sptr_node > stack_node); sptr_node--; sptr_near--; curNode = *sptr_node; curDist = *sptr_near; #pragma unroll(4) for (unsigned i=0; i<4; i++) { const NodeRef child = node->children[i]; if (unlikely(child == BVH4::emptyNode)) break; #if defined(__AVX2__) const ssef lclipMinX = msub(node->lower_x[i],rdir.x,org_rdir.x); const ssef lclipMinY = msub(node->lower_y[i],rdir.y,org_rdir.y); const ssef lclipMinZ = msub(node->lower_z[i],rdir.z,org_rdir.z); const ssef lclipMaxX = msub(node->upper_x[i],rdir.x,org_rdir.x); const ssef lclipMaxY = msub(node->upper_y[i],rdir.y,org_rdir.y); const ssef lclipMaxZ = msub(node->upper_z[i],rdir.z,org_rdir.z); #else const ssef lclipMinX = (node->lower_x[i] - org.x) * rdir.x; const ssef lclipMinY = (node->lower_y[i] - org.y) * rdir.y; const ssef lclipMinZ = (node->lower_z[i] - org.z) * rdir.z; const ssef lclipMaxX = (node->upper_x[i] - org.x) * rdir.x; const ssef lclipMaxY = (node->upper_y[i] - org.y) * rdir.y; const ssef lclipMaxZ = (node->upper_z[i] - org.z) * rdir.z; #endif #if defined(__SSE4_1__) const ssef lnearP = maxi(maxi(mini(lclipMinX, lclipMaxX), mini(lclipMinY, lclipMaxY)), mini(lclipMinZ, lclipMaxZ)); const ssef lfarP = mini(mini(maxi(lclipMinX, lclipMaxX), maxi(lclipMinY, lclipMaxY)), maxi(lclipMinZ, lclipMaxZ)); const sseb lhit = maxi(lnearP,ray_tnear) <= mini(lfarP,ray_tfar); #else const ssef lnearP = max(max(min(lclipMinX, lclipMaxX), min(lclipMinY, lclipMaxY)), min(lclipMinZ, lclipMaxZ)); const ssef lfarP = min(min(max(lclipMinX, lclipMaxX), max(lclipMinY, lclipMaxY)), max(lclipMinZ, lclipMaxZ)); const sseb lhit = max(lnearP,ray_tnear) <= min(lfarP,ray_tfar); #endif /* if we hit the child we choose to continue with that child if it is closer than the current next child, or we push it onto the stack */ if (likely(any(lhit))) { assert(sptr_node < stackEnd); const ssef childDist = select(lhit,lnearP,inf); const NodeRef child = node->children[i]; assert(child != BVH4::emptyNode); sptr_node++; sptr_near++; /* push cur node onto stack and continue with hit child */ if (any(childDist < curDist)) { *(sptr_node-1) = curNode; *(sptr_near-1) = curDist; curDist = childDist; curNode = child; } /* push hit child onto stack */ else { *(sptr_node-1) = child; *(sptr_near-1) = childDist; } } } } /* return if stack is empty */ if (unlikely(curNode == BVH4::invalidNode)) { assert(sptr_node == stack_node); break; } /* intersect leaf */ const sseb valid_leaf = ray_tfar > curDist; STAT3(normal.trav_leaves,1,popcnt(valid_leaf),4); size_t items; const Primitive* prim = (Primitive*) curNode.leaf(items); PrimitiveIntersector4::intersect(valid_leaf,ray,prim,items,bvh->geometry); ray_tfar = select(valid_leaf,ray.tfar,ray_tfar); } AVX_ZERO_UPPER(); }
void BVH8iIntersector8Hybrid<TriangleIntersector8>::occluded(avxb* valid_i, BVH8i* bvh, Ray8& ray) { /* load ray */ const avxb valid = *valid_i; avxb terminated = !valid; avx3f ray_org = ray.org, ray_dir = ray.dir; avxf ray_tnear = ray.tnear, ray_tfar = ray.tfar; #if defined(__FIX_RAYS__) const avxf float_range = 0.1f*FLT_MAX; ray_org = clamp(ray_org,avx3f(-float_range),avx3f(+float_range)); ray_dir = clamp(ray_dir,avx3f(-float_range),avx3f(+float_range)); ray_tnear = max(ray_tnear,FLT_MIN); ray_tfar = min(ray_tfar,float(inf)); #endif const avx3f rdir = rcp_safe(ray_dir); const avx3f org(ray_org), org_rdir = org * rdir; ray_tnear = select(valid,ray_tnear,avxf(pos_inf)); ray_tfar = select(valid,ray_tfar ,avxf(neg_inf)); const avxf inf = avxf(pos_inf); /* compute near/far per ray */ avx3i nearXYZ; nearXYZ.x = select(rdir.x >= 0.0f,avxi(0*(int)sizeof(avxf)),avxi(1*(int)sizeof(avxf))); nearXYZ.y = select(rdir.y >= 0.0f,avxi(2*(int)sizeof(avxf)),avxi(3*(int)sizeof(avxf))); nearXYZ.z = select(rdir.z >= 0.0f,avxi(4*(int)sizeof(avxf)),avxi(5*(int)sizeof(avxf))); /* allocate stack and push root node */ avxf stack_near[stackSizeChunk]; NodeRef stack_node[stackSizeChunk]; stack_node[0] = BVH4i::invalidNode; stack_near[0] = inf; stack_node[1] = bvh->root; stack_near[1] = ray_tnear; NodeRef* stackEnd = stack_node+stackSizeChunk; NodeRef* __restrict__ sptr_node = stack_node + 2; avxf* __restrict__ sptr_near = stack_near + 2; const Node * __restrict__ nodes = (Node *)bvh->nodePtr(); const Triangle * __restrict__ accel = (Triangle*)bvh->triPtr(); while (1) { /* pop next node from stack */ assert(sptr_node > stack_node); sptr_node--; sptr_near--; NodeRef curNode = *sptr_node; if (unlikely(curNode == BVH4i::invalidNode)) { assert(sptr_node == stack_node); break; } /* cull node if behind closest hit point */ avxf curDist = *sptr_near; const avxb active = curDist < ray_tfar; if (unlikely(none(active))) continue; /* switch to single ray traversal */ #if !defined(__WIN32__) || defined(__X86_64__) size_t bits = movemask(active); if (unlikely(__popcnt(bits) <= SWITCH_THRESHOLD)) { for (size_t i=__bsf(bits); bits!=0; bits=__btc(bits,i), i=__bsf(bits)) { if (occluded1(bvh,curNode,i,ray,ray_org,ray_dir,rdir,ray_tnear,ray_tfar,nearXYZ)) terminated[i] = -1; } if (all(terminated)) break; ray_tfar = select(terminated,avxf(neg_inf),ray_tfar); continue; } #endif while (1) { /* test if this is a leaf node */ if (unlikely(curNode.isLeaf())) break; const avxb valid_node = ray_tfar > curDist; STAT3(shadow.trav_nodes,1,popcnt(valid_node),8); const Node* __restrict__ const node = (Node*)curNode.node(nodes); /* pop of next node */ assert(sptr_node > stack_node); sptr_node--; sptr_near--; curNode = *sptr_node; curDist = *sptr_near; for (unsigned i=0; i<8; i++) { const NodeRef child = node->children[i]; if (unlikely(child == BVH4i::emptyNode)) break; #if defined(__AVX2__) const avxf lclipMinX = msub(node->lower_x[i],rdir.x,org_rdir.x); const avxf lclipMinY = msub(node->lower_y[i],rdir.y,org_rdir.y); const avxf lclipMinZ = msub(node->lower_z[i],rdir.z,org_rdir.z); const avxf lclipMaxX = msub(node->upper_x[i],rdir.x,org_rdir.x); const avxf lclipMaxY = msub(node->upper_y[i],rdir.y,org_rdir.y); const avxf lclipMaxZ = msub(node->upper_z[i],rdir.z,org_rdir.z); const avxf lnearP = maxi(maxi(mini(lclipMinX, lclipMaxX), mini(lclipMinY, lclipMaxY)), mini(lclipMinZ, lclipMaxZ)); const avxf lfarP = mini(mini(maxi(lclipMinX, lclipMaxX), maxi(lclipMinY, lclipMaxY)), maxi(lclipMinZ, lclipMaxZ)); const avxb lhit = maxi(lnearP,ray_tnear) <= mini(lfarP,ray_tfar); #else const avxf lclipMinX = (node->lower_x[i] - org.x) * rdir.x; const avxf lclipMinY = (node->lower_y[i] - org.y) * rdir.y; const avxf lclipMinZ = (node->lower_z[i] - org.z) * rdir.z; const avxf lclipMaxX = (node->upper_x[i] - org.x) * rdir.x; const avxf lclipMaxY = (node->upper_y[i] - org.y) * rdir.y; const avxf lclipMaxZ = (node->upper_z[i] - org.z) * rdir.z; const avxf lnearP = max(max(min(lclipMinX, lclipMaxX), min(lclipMinY, lclipMaxY)), min(lclipMinZ, lclipMaxZ)); const avxf lfarP = min(min(max(lclipMinX, lclipMaxX), max(lclipMinY, lclipMaxY)), max(lclipMinZ, lclipMaxZ)); const avxb lhit = max(lnearP,ray_tnear) <= min(lfarP,ray_tfar); #endif /* if we hit the child we choose to continue with that child if it is closer than the current next child, or we push it onto the stack */ if (likely(any(lhit))) { assert(sptr_node < stackEnd); assert(child != BVH4i::emptyNode); const avxf childDist = select(lhit,lnearP,inf); sptr_node++; sptr_near++; /* push cur node onto stack and continue with hit child */ if (any(childDist < curDist)) { *(sptr_node-1) = curNode; *(sptr_near-1) = curDist; curDist = childDist; curNode = child; } /* push hit child onto stack */ else { *(sptr_node-1) = child; *(sptr_near-1) = childDist; } } } } /* return if stack is empty */ if (unlikely(curNode == BVH4i::invalidNode)) { assert(sptr_node == stack_node); break; } /* intersect leaf */ const avxb valid_leaf = ray_tfar > curDist; STAT3(shadow.trav_leaves,1,popcnt(valid_leaf),8); size_t items; const Triangle* prim = (Triangle*) curNode.leaf(accel,items); terminated |= TriangleIntersector8::occluded(!terminated,ray,prim,items,bvh->geometry); if (all(terminated)) break; ray_tfar = select(terminated,avxf(neg_inf),ray_tfar); } store8i(valid & terminated,&ray.geomID,0); AVX_ZERO_UPPER(); }
__forceinline void BVH4Intersector4Hybrid<PrimitiveIntersector4>::intersect1(const BVH4* bvh, NodeRef root, size_t k, Ray4& ray, const sse3f& ray_org, const sse3f& ray_dir, const sse3f& ray_rdir, const ssef& ray_tnear, const ssef& ray_tfar) { /*! stack state */ StackItem stack[stackSizeSingle]; //!< stack of nodes StackItem* stackPtr = stack+1; //!< current stack pointer StackItem* stackEnd = stack+stackSizeSingle; stack[0].ptr = root; stack[0].dist = neg_inf; /*! offsets to select the side that becomes the lower or upper bound */ const size_t nearX = ray_dir.x[k] >= 0.0f ? 0*sizeof(ssef) : 1*sizeof(ssef); const size_t nearY = ray_dir.y[k] >= 0.0f ? 2*sizeof(ssef) : 3*sizeof(ssef); const size_t nearZ = ray_dir.z[k] >= 0.0f ? 4*sizeof(ssef) : 5*sizeof(ssef); /*! load the ray into SIMD registers */ const sse3f org (ray_org .x[k],ray_org .y[k],ray_org .z[k]); const sse3f rdir(ray_rdir.x[k],ray_rdir.y[k],ray_rdir.z[k]); const sse3f norg = -org, org_rdir(org*rdir); ssef rayNear(ray_tnear[k]), rayFar(ray_tfar[k]); /* pop loop */ while (true) pop: { /*! pop next node */ if (unlikely(stackPtr == stack)) break; stackPtr--; NodeRef cur = NodeRef(stackPtr->ptr); /*! if popped node is too far, pop next one */ if (unlikely(stackPtr->dist > ray.tfar[k])) continue; /* downtraversal loop */ while (true) { /*! stop if we found a leaf */ if (unlikely(cur.isLeaf())) break; STAT3(normal.trav_nodes,1,1,1); /*! single ray intersection with 4 boxes */ const Node* node = cur.node(); const size_t farX = nearX ^ 16, farY = nearY ^ 16, farZ = nearZ ^ 16; #if defined (__AVX2__) const ssef tNearX = msub(load4f((const char*)node+nearX), rdir.x, org_rdir.x); const ssef tNearY = msub(load4f((const char*)node+nearY), rdir.y, org_rdir.y); const ssef tNearZ = msub(load4f((const char*)node+nearZ), rdir.z, org_rdir.z); const ssef tFarX = msub(load4f((const char*)node+farX ), rdir.x, org_rdir.x); const ssef tFarY = msub(load4f((const char*)node+farY ), rdir.y, org_rdir.y); const ssef tFarZ = msub(load4f((const char*)node+farZ ), rdir.z, org_rdir.z); #else const ssef tNearX = (norg.x + load4f((const char*)node+nearX)) * rdir.x; const ssef tNearY = (norg.y + load4f((const char*)node+nearY)) * rdir.y; const ssef tNearZ = (norg.z + load4f((const char*)node+nearZ)) * rdir.z; const ssef tFarX = (norg.x + load4f((const char*)node+farX )) * rdir.x; const ssef tFarY = (norg.y + load4f((const char*)node+farY )) * rdir.y; const ssef tFarZ = (norg.z + load4f((const char*)node+farZ )) * rdir.z; #endif #if defined(__SSE4_1__) const ssef tNear = maxi(maxi(tNearX,tNearY),maxi(tNearZ,rayNear)); const ssef tFar = mini(mini(tFarX ,tFarY ),mini(tFarZ ,rayFar )); const sseb vmask = cast(tNear) > cast(tFar); size_t mask = movemask(vmask)^0xf; #else const ssef tNear = max(tNearX,tNearY,tNearZ,rayNear); const ssef tFar = min(tFarX ,tFarY ,tFarZ ,rayFar); const sseb vmask = tNear <= tFar; size_t mask = movemask(vmask); #endif /*! if no child is hit, pop next node */ if (unlikely(mask == 0)) goto pop; /*! one child is hit, continue with that child */ size_t r = __bscf(mask); if (likely(mask == 0)) { cur = node->child(r); assert(cur != BVH4::emptyNode); continue; } /*! two children are hit, push far child, and continue with closer child */ NodeRef c0 = node->child(r); const float d0 = tNear[r]; r = __bscf(mask); NodeRef c1 = node->child(r); const float d1 = tNear[r]; assert(c0 != BVH4::emptyNode); assert(c1 != BVH4::emptyNode); if (likely(mask == 0)) { assert(stackPtr < stackEnd); if (d0 < d1) { stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; cur = c0; continue; } else { stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; cur = c1; continue; } } /*! Here starts the slow path for 3 or 4 hit children. We push * all nodes onto the stack to sort them there. */ assert(stackPtr < stackEnd); stackPtr->ptr = c0; stackPtr->dist = d0; stackPtr++; assert(stackPtr < stackEnd); stackPtr->ptr = c1; stackPtr->dist = d1; stackPtr++; /*! three children are hit, push all onto stack and sort 3 stack items, continue with closest child */ assert(stackPtr < stackEnd); r = __bscf(mask); NodeRef c = node->child(r); float d = tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; assert(c != BVH4::emptyNode); if (likely(mask == 0)) { sort(stackPtr[-1],stackPtr[-2],stackPtr[-3]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; continue; } /*! four children are hit, push all onto stack and sort 4 stack items, continue with closest child */ assert(stackPtr < stackEnd); r = __bscf(mask); c = node->child(r); d = tNear[r]; stackPtr->ptr = c; stackPtr->dist = d; stackPtr++; assert(c != BVH4::emptyNode); sort(stackPtr[-1],stackPtr[-2],stackPtr[-3],stackPtr[-4]); cur = (NodeRef) stackPtr[-1].ptr; stackPtr--; } /*! this is a leaf node */ STAT3(normal.trav_leaves,1,1,1); size_t num; Primitive* prim = (Primitive*) cur.leaf(num); PrimitiveIntersector4::intersect(ray,k,prim,num,bvh->geometry); rayFar = ray.tfar[k]; } }
typename BVHNStatistics<N>::Statistics BVHNStatistics<N>::statistics(NodeRef node, const double A, const BBox1f t0t1) { Statistics s; double dt = max(0.0f,t0t1.size()); if (node.isAlignedNode()) { AlignedNode* n = node.alignedNode(); for (size_t i=0; i<N; i++) { if (n->child(i) == BVH::emptyNode) continue; s.statAlignedNodes.numChildren++; const double Ai = max(0.0f,halfArea(n->extend(i))); s = s + statistics(n->child(i),Ai,t0t1); } s.statAlignedNodes.numNodes++; s.statAlignedNodes.nodeSAH += dt*A; s.depth++; } else if (node.isUnalignedNode()) { UnalignedNode* n = node.unalignedNode(); for (size_t i=0; i<N; i++) { if (n->child(i) == BVH::emptyNode) continue; s.statUnalignedNodes.numChildren++; const double Ai = max(0.0f,halfArea(n->extend(i))); s = s + statistics(n->child(i),Ai,t0t1); } s.statUnalignedNodes.numNodes++; s.statUnalignedNodes.nodeSAH += dt*A; s.depth++; } else if (node.isAlignedNodeMB()) { AlignedNodeMB* n = node.alignedNodeMB(); for (size_t i=0; i<N; i++) { if (n->child(i) == BVH::emptyNode) continue; s.statAlignedNodesMB.numChildren++; const double Ai = max(0.0f,halfArea(n->extend0(i))); s = s + statistics(n->child(i),Ai,t0t1); } s.statAlignedNodesMB.numNodes++; s.statAlignedNodesMB.nodeSAH += dt*A; s.depth++; } else if (node.isUnalignedNodeMB()) { UnalignedNodeMB* n = node.unalignedNodeMB(); for (size_t i=0; i<N; i++) { if (n->child(i) == BVH::emptyNode) continue; s.statUnalignedNodesMB.numChildren++; const double Ai = max(0.0f,halfArea(n->extend0(i))); s = s + statistics(n->child(i),Ai,t0t1); } s.statUnalignedNodesMB.numNodes++; s.statUnalignedNodesMB.nodeSAH += dt*A; s.depth++; } else if (node.isTransformNode()) { s.statTransformNodes.numNodes++; s.statTransformNodes.nodeSAH += dt*A; s.depth++; } else if (node.isQuantizedNode()) { QuantizedNode* n = node.quantizedNode(); for (size_t i=0; i<N; i++) { if (n->child(i) == BVH::emptyNode) continue; s.statQuantizedNodes.numChildren++; const double Ai = max(0.0f,halfArea(n->extend(i))); s = s + statistics(n->child(i),Ai,t0t1); } s.statQuantizedNodes.numNodes++; s.statQuantizedNodes.nodeSAH += dt*A; s.depth++; } else if (node.isLeaf()) { size_t num; const char* tri = node.leaf(num); if (num) { for (size_t i=0; i<num; i++) { s.statLeaf.numPrims += bvh->primTy.size(tri+i*bvh->primTy.bytes); } s.statLeaf.numLeaves++; s.statLeaf.numPrimBlocks += num; s.statLeaf.leafSAH += dt*A*num; if (num-1 < Statistics::LeafStat::NHIST) { s.statLeaf.numPrimBlocksHistogram[num-1]++; } } } else { throw std::runtime_error("not supported node type in bvh_statistics"); } return s; }
__forceinline NodeArea(NodeRef& node, const BBox3fa& bounds) : node(&node), A(node.isLeaf() ? float(neg_inf) : area(bounds)) {}
__forceinline bool BVH4Intersector4Hybrid<PrimitiveIntersector4>::occluded1(const BVH4* bvh, NodeRef root, size_t k, Ray4& ray, const sse3f& ray_org, const sse3f& ray_dir, const sse3f& ray_rdir, const ssef& ray_tnear, const ssef& ray_tfar) { /*! stack state */ NodeRef stack[stackSizeSingle]; //!< stack of nodes that still need to get traversed NodeRef* stackPtr = stack+1; //!< current stack pointer NodeRef* stackEnd = stack+stackSizeSingle; stack[0] = root; /*! offsets to select the side that becomes the lower or upper bound */ const size_t nearX = ray_dir.x[k] >= 0.0f ? 0*sizeof(ssef) : 1*sizeof(ssef); const size_t nearY = ray_dir.y[k] >= 0.0f ? 2*sizeof(ssef) : 3*sizeof(ssef); const size_t nearZ = ray_dir.z[k] >= 0.0f ? 4*sizeof(ssef) : 5*sizeof(ssef); /*! load the ray into SIMD registers */ const sse3f org (ray_org .x[k],ray_org .y[k],ray_org .z[k]); const sse3f rdir(ray_rdir.x[k],ray_rdir.y[k],ray_rdir.z[k]); const sse3f norg = -org, org_rdir(org*rdir); const ssef rayNear(ray_tnear[k]), rayFar(ray_tfar[k]); /* pop loop */ while (true) pop: { /*! pop next node */ if (unlikely(stackPtr == stack)) break; stackPtr--; NodeRef cur = (NodeRef) *stackPtr; /* downtraversal loop */ while (true) { /*! stop if we found a leaf */ if (unlikely(cur.isLeaf())) break; STAT3(shadow.trav_nodes,1,1,1); /*! single ray intersection with 4 boxes */ const Node* node = cur.node(); const size_t farX = nearX ^ 16, farY = nearY ^ 16, farZ = nearZ ^ 16; #if defined (__AVX2__) const ssef tNearX = msub(load4f((const char*)node+nearX), rdir.x, org_rdir.x); const ssef tNearY = msub(load4f((const char*)node+nearY), rdir.y, org_rdir.y); const ssef tNearZ = msub(load4f((const char*)node+nearZ), rdir.z, org_rdir.z); const ssef tFarX = msub(load4f((const char*)node+farX ), rdir.x, org_rdir.x); const ssef tFarY = msub(load4f((const char*)node+farY ), rdir.y, org_rdir.y); const ssef tFarZ = msub(load4f((const char*)node+farZ ), rdir.z, org_rdir.z); #else const ssef tNearX = (norg.x + load4f((const char*)node+nearX)) * rdir.x; const ssef tNearY = (norg.y + load4f((const char*)node+nearY)) * rdir.y; const ssef tNearZ = (norg.z + load4f((const char*)node+nearZ)) * rdir.z; const ssef tFarX = (norg.x + load4f((const char*)node+farX )) * rdir.x; const ssef tFarY = (norg.y + load4f((const char*)node+farY )) * rdir.y; const ssef tFarZ = (norg.z + load4f((const char*)node+farZ )) * rdir.z; #endif #if defined(__SSE4_1__) const ssef tNear = maxi(maxi(tNearX,tNearY),maxi(tNearZ,rayNear)); const ssef tFar = mini(mini(tFarX ,tFarY ),mini(tFarZ ,rayFar )); const sseb vmask = cast(tNear) > cast(tFar); size_t mask = movemask(vmask)^0xf; #else const ssef tNear = max(tNearX,tNearY,tNearZ,rayNear); const ssef tFar = min(tFarX ,tFarY ,tFarZ ,rayFar); const sseb vmask = tNear <= tFar; size_t mask = movemask(vmask); #endif /*! if no child is hit, pop next node */ if (unlikely(mask == 0)) goto pop; /*! one child is hit, continue with that child */ size_t r = __bscf(mask); if (likely(mask == 0)) { cur = node->child(r); assert(cur != BVH4::emptyNode); continue; } /*! two children are hit, push far child, and continue with closer child */ NodeRef c0 = node->child(r); const float d0 = tNear[r]; r = __bscf(mask); NodeRef c1 = node->child(r); const float d1 = tNear[r]; assert(c0 != BVH4::emptyNode); assert(c1 != BVH4::emptyNode); if (likely(mask == 0)) { assert(stackPtr < stackEnd); if (d0 < d1) { *stackPtr = c1; stackPtr++; cur = c0; continue; } else { *stackPtr = c0; stackPtr++; cur = c1; continue; } } assert(stackPtr < stackEnd); *stackPtr = c0; stackPtr++; assert(stackPtr < stackEnd); *stackPtr = c1; stackPtr++; /*! three children are hit */ r = __bscf(mask); cur = node->child(r); assert(cur != BVH4::emptyNode); if (likely(mask == 0)) continue; assert(stackPtr < stackEnd); *stackPtr = cur; stackPtr++; /*! four children are hit */ cur = node->child(3); assert(cur != BVH4::emptyNode); } /*! this is a leaf node */ STAT3(shadow.trav_leaves,1,1,1); size_t num; Primitive* prim = (Primitive*) cur.leaf(num); if (PrimitiveIntersector4::occluded(ray,k,prim,num,bvh->geometry)) { ray.geomID[k] = 0; return true; } } return false; }
void BVH8Intersector16Chunk<PrimitiveIntersector16>::occluded(bool16* valid_i, BVH8* bvh, Ray16& ray) { #if defined(__AVX512__) /* load ray */ const bool16 valid = *valid_i; bool16 terminated = !valid; const Vec3f16 rdir = rcp_safe(ray.dir); const Vec3f16 org_rdir = ray.org * rdir; float16 ray_tnear = select(valid,ray.tnear,pos_inf); float16 ray_tfar = select(valid,ray.tfar ,neg_inf); const float16 inf = float16(pos_inf); Precalculations pre(valid,ray); /* allocate stack and push root node */ float16 stack_near[3*BVH8::maxDepth+1]; NodeRef stack_node[3*BVH8::maxDepth+1]; stack_node[0] = BVH8::invalidNode; stack_near[0] = inf; stack_node[1] = bvh->root; stack_near[1] = ray_tnear; NodeRef* __restrict__ sptr_node = stack_node + 2; float16* __restrict__ sptr_near = stack_near + 2; while (1) { /* pop next node from stack */ sptr_node--; sptr_near--; NodeRef cur = *sptr_node; if (unlikely(cur == BVH8::invalidNode)) break; /* cull node if behind closest hit point */ float16 curDist = *sptr_near; if (unlikely(none(ray_tfar > curDist))) continue; while (1) { /* test if this is a leaf node */ if (unlikely(cur.isLeaf())) break; const bool16 valid_node = ray_tfar > curDist; STAT3(shadow.trav_nodes,1,popcnt(valid_node),8); const Node* __restrict__ const node = (Node*)cur.node(); /* pop of next node */ sptr_node--; sptr_near--; cur = *sptr_node; // FIXME: this trick creates issues with stack depth curDist = *sptr_near; for (unsigned i=0; i<BVH8::N; i++) { const NodeRef child = node->children[i]; if (unlikely(child == BVH8::emptyNode)) break; const float16 lclipMinX = msub(node->lower_x[i],rdir.x,org_rdir.x); const float16 lclipMinY = msub(node->lower_y[i],rdir.y,org_rdir.y); const float16 lclipMinZ = msub(node->lower_z[i],rdir.z,org_rdir.z); const float16 lclipMaxX = msub(node->upper_x[i],rdir.x,org_rdir.x); const float16 lclipMaxY = msub(node->upper_y[i],rdir.y,org_rdir.y); const float16 lclipMaxZ = msub(node->upper_z[i],rdir.z,org_rdir.z); const float16 lnearP = max(max(min(lclipMinX, lclipMaxX), min(lclipMinY, lclipMaxY)), min(lclipMinZ, lclipMaxZ)); const float16 lfarP = min(min(max(lclipMinX, lclipMaxX), max(lclipMinY, lclipMaxY)), max(lclipMinZ, lclipMaxZ)); const bool16 lhit = max(lnearP,ray_tnear) <= min(lfarP,ray_tfar); /* if we hit the child we choose to continue with that child if it is closer than the current next child, or we push it onto the stack */ if (likely(any(lhit))) { const float16 childDist = select(lhit,lnearP,inf); sptr_node++; sptr_near++; /* push cur node onto stack and continue with hit child */ if (any(childDist < curDist)) { *(sptr_node-1) = cur; *(sptr_near-1) = curDist; curDist = childDist; cur = child; } /* push hit child onto stack*/ else { *(sptr_node-1) = child; *(sptr_near-1) = childDist; } assert(sptr_node - stack_node < BVH8::maxDepth); } } } /* return if stack is empty */ if (unlikely(cur == BVH8::invalidNode)) break; /* intersect leaf */ assert(cur != BVH8::emptyNode); const bool16 valid_leaf = ray_tfar > curDist; STAT3(shadow.trav_leaves,1,popcnt(valid_leaf),8); size_t items; const Triangle* tri = (Triangle*) cur.leaf(items); terminated |= PrimitiveIntersector16::occluded(!terminated,pre,ray,tri,items,bvh->scene); if (all(terminated)) break; ray_tfar = select(terminated,neg_inf,ray_tfar); } store16i(valid & terminated,&ray.geomID,0); AVX_ZERO_UPPER(); #endif }
void BVH4Intersector4Chunk<TriangleIntersector4>::intersect(const BVH4Intersector4Chunk* This, Ray4& ray, const __m128 valid_i) { sseb valid = valid_i; NodeRef invalid = (NodeRef)1; const BVH4* bvh = This->bvh; STAT3(normal.travs,1,popcnt(valid),4); /* load ray into registers */ ssef ray_near = select(valid,ray.tnear,pos_inf); ssef ray_far = select(valid,ray.tfar ,neg_inf); sse3f rdir = rcp_safe(ray.dir); ray.tfar = ray_far; /* allocate stack and push root node */ NodeRef stack_node[3*BVH4::maxDepth+1]; ssef stack_near[3*BVH4::maxDepth+1]; stack_node[0] = invalid; stack_near[0] = ssef(inf); stack_node[1] = bvh->root; stack_near[1] = ray_near; NodeRef* sptr_node = stack_node+2; ssef * sptr_near = stack_near+2; while (1) { /* pop next node from stack */ sptr_node--; sptr_near--; ssef curDist = *sptr_near; NodeRef curNode = *sptr_node; if (unlikely(curNode == invalid)) break; /* cull node if behind closest hit point */ const sseb m_dist = curDist < ray_far; if (unlikely(none(m_dist))) continue; while (1) { /* test if this is a leaf node */ if (unlikely(curNode.isLeaf())) break; STAT3(normal.trav_nodes,1,popcnt(valid),4); const Node* const node = curNode.node(bvh->nodePtr()); //NodeRef(curNode).node(nodes); //prefetch<PFHINT_L1>((ssef*)node + 1); // depth first order prefetch /* pop of next node */ sptr_node--; sptr_near--; curNode = *sptr_node; curDist = *sptr_near; for (unsigned i=0;i<4;i++) { const ssef dminx = (ssef(node->lower_x[i]) - ray.org.x) * rdir.x; const ssef dmaxx = (ssef(node->upper_x[i]) - ray.org.x) * rdir.x; const ssef dminy = (ssef(node->lower_y[i]) - ray.org.y) * rdir.y; const ssef dmaxy = (ssef(node->upper_y[i]) - ray.org.y) * rdir.y; const ssef dminz = (ssef(node->lower_z[i]) - ray.org.z) * rdir.z; const ssef dmaxz = (ssef(node->upper_z[i]) - ray.org.z) * rdir.z; const ssef dlowerx = min(dminx,dmaxx); const ssef dupperx = max(dminx,dmaxx); const ssef dlowery = min(dminy,dmaxy); const ssef duppery = max(dminy,dmaxy); const ssef dlowerz = min(dminz,dmaxz); const ssef dupperz = max(dminz,dmaxz); const ssef near = max(dlowerx,dlowery,dlowerz,ray_near); const ssef far = min(dupperx,duppery,dupperz,ray_far ); const sseb mhit = near <= far; const ssef childDist = select(mhit,near,inf); const sseb closer = childDist < curDist; /* if we hit the child we choose to continue with that child if it is closer than the current next child, or we push it onto the stack */ if (likely(any(mhit))) { const NodeRef child = node->child(i); //if (child != invalid) { sptr_node++; sptr_near++; /* push cur node onto stack and continue with hit child */ if (any(closer)) { *(sptr_node-1) = curNode; *(sptr_near-1) = curDist; curDist = childDist; curNode = child; } /* push hit child onto stack*/ else { *(sptr_node-1) = child; *(sptr_near-1) = childDist; } } } } } /* return if stack is empty */ if (unlikely(curNode == invalid)) break; /* decode leaf node */ size_t num; STAT3(normal.trav_leaves,1,popcnt(valid),4); Triangle* tri = (Triangle*) curNode.leaf(bvh->triPtr(),num); /* intersect triangles */ for (size_t i=0; i<num; i++) TriangleIntersector4::intersect(valid,ray,tri[i],bvh->vertices); ray_far = ray.tfar; } }
void BVH4Intersector1<PrimitiveIntersector>::occluded(const BVH4* bvh, Ray& ray) { /*! stack state */ NodeRef stack[stackSize]; //!< stack of nodes that still need to get traversed NodeRef* stackPtr = stack+1; //!< current stack pointer NodeRef* stackEnd = stack+stackSize; stack[0] = bvh->root; /*! offsets to select the side that becomes the lower or upper bound */ const size_t nearX = ray.dir.x >= 0 ? 0*sizeof(ssef) : 1*sizeof(ssef); const size_t nearY = ray.dir.y >= 0 ? 2*sizeof(ssef) : 3*sizeof(ssef); const size_t nearZ = ray.dir.z >= 0 ? 4*sizeof(ssef) : 5*sizeof(ssef); #if 0 // FIXME: why is this slower /*! load the ray */ Vec3fa ray_org = ray.org; Vec3fa ray_dir = ray.dir; ssef ray_near = max(ray.tnear,FLT_MIN); // we do not support negative tnear values in this kernel due to integer optimizations ssef ray_far = ray.tfar; #if defined(__FIX_RAYS__) const float float_range = 0.1f*FLT_MAX; ray_org = clamp(ray_org,Vec3fa(-float_range),Vec3fa(+float_range)); ray_dir = clamp(ray_dir,Vec3fa(-float_range),Vec3fa(+float_range)); ray_far = min(ray_far,float(inf)); #endif const Vec3fa ray_rdir = rcp_safe(ray_dir); const sse3f org(ray_org), dir(ray_dir); const sse3f norg(-ray_org), rdir(ray_rdir), org_rdir(ray_org*ray_rdir); #else /*! load the ray into SIMD registers */ const sse3f norg(-ray.org.x,-ray.org.y,-ray.org.z); const Vec3fa ray_rdir = rcp_safe(ray.dir); const sse3f rdir(ray_rdir.x,ray_rdir.y,ray_rdir.z); const Vec3fa ray_org_rdir = ray.org*ray_rdir; const sse3f org_rdir(ray_org_rdir.x,ray_org_rdir.y,ray_org_rdir.z); const ssef ray_near(ray.tnear); ssef ray_far(ray.tfar); #endif /* pop loop */ while (true) pop: { /*! pop next node */ if (unlikely(stackPtr == stack)) break; stackPtr--; NodeRef cur = (NodeRef) *stackPtr; /* downtraversal loop */ while (true) { /*! stop if we found a leaf */ if (unlikely(cur.isLeaf())) break; STAT3(shadow.trav_nodes,1,1,1); /*! single ray intersection with 4 boxes */ const Node* node = cur.node(); const size_t farX = nearX ^ 16, farY = nearY ^ 16, farZ = nearZ ^ 16; #if defined (__AVX2__) const ssef tNearX = msub(load4f((const char*)node+nearX), rdir.x, org_rdir.x); const ssef tNearY = msub(load4f((const char*)node+nearY), rdir.y, org_rdir.y); const ssef tNearZ = msub(load4f((const char*)node+nearZ), rdir.z, org_rdir.z); const ssef tFarX = msub(load4f((const char*)node+farX ), rdir.x, org_rdir.x); const ssef tFarY = msub(load4f((const char*)node+farY ), rdir.y, org_rdir.y); const ssef tFarZ = msub(load4f((const char*)node+farZ ), rdir.z, org_rdir.z); #else const ssef tNearX = (norg.x + load4f((const char*)node+nearX)) * rdir.x; const ssef tNearY = (norg.y + load4f((const char*)node+nearY)) * rdir.y; const ssef tNearZ = (norg.z + load4f((const char*)node+nearZ)) * rdir.z; const ssef tFarX = (norg.x + load4f((const char*)node+farX )) * rdir.x; const ssef tFarY = (norg.y + load4f((const char*)node+farY )) * rdir.y; const ssef tFarZ = (norg.z + load4f((const char*)node+farZ )) * rdir.z; #endif #if defined(__SSE4_1__) const ssef tNear = maxi(maxi(tNearX,tNearY),maxi(tNearZ,ray_near)); const ssef tFar = mini(mini(tFarX ,tFarY ),mini(tFarZ ,ray_far )); const sseb vmask = cast(tNear) > cast(tFar); size_t mask = movemask(vmask)^0xf; #else const ssef tNear = max(tNearX,tNearY,tNearZ,ray_near); const ssef tFar = min(tFarX ,tFarY ,tFarZ ,ray_far); const sseb vmask = tNear <= tFar; size_t mask = movemask(vmask); #endif /*! if no child is hit, pop next node */ if (unlikely(mask == 0)) goto pop; /*! one child is hit, continue with that child */ size_t r = __bscf(mask); if (likely(mask == 0)) { cur = node->child(r); assert(cur != BVH4::emptyNode); continue; } /*! two children are hit, push far child, and continue with closer child */ NodeRef c0 = node->child(r); const unsigned int d0 = ((unsigned int*)&tNear)[r]; r = __bscf(mask); NodeRef c1 = node->child(r); const unsigned int d1 = ((unsigned int*)&tNear)[r]; assert(c0 != BVH4::emptyNode); assert(c1 != BVH4::emptyNode); if (likely(mask == 0)) { assert(stackPtr < stackEnd); if (d0 < d1) { *stackPtr = c1; stackPtr++; cur = c0; continue; } else { *stackPtr = c0; stackPtr++; cur = c1; continue; } } assert(stackPtr < stackEnd); *stackPtr = c0; stackPtr++; assert(stackPtr < stackEnd); *stackPtr = c1; stackPtr++; /*! three children are hit */ r = __bscf(mask); cur = node->child(r); assert(cur != BVH4::emptyNode); if (likely(mask == 0)) continue; assert(stackPtr < stackEnd); *stackPtr = cur; stackPtr++; /*! four children are hit */ cur = node->child(3); assert(cur != BVH4::emptyNode); } /*! this is a leaf node */ STAT3(shadow.trav_leaves,1,1,1); size_t num; Primitive* prim = (Primitive*) cur.leaf(num); if (PrimitiveIntersector::occluded(ray,prim,num,bvh->geometry)) { ray.geomID = 0; break; } } AVX_ZERO_UPPER(); }
void BVH4iIntersector1::occluded(BVH4i* bvh, Ray& ray) { /* near and node stack */ __aligned(64) NodeRef stack_node[3*BVH4i::maxDepth+1]; /* setup */ const mic3f rdir16 = rcp_safe(mic3f(ray.dir.x,ray.dir.y,ray.dir.z)); const mic_f inf = mic_f(pos_inf); const mic_f zero = mic_f::zero(); const Node * __restrict__ nodes = (Node *)bvh->nodePtr(); const Triangle1 * __restrict__ accel = (Triangle1*)bvh->triPtr(); stack_node[0] = BVH4i::invalidNode; stack_node[1] = bvh->root; size_t sindex = 2; const mic_f org_xyz = loadAOS4to16f(ray.org.x,ray.org.y,ray.org.z); const mic_f dir_xyz = loadAOS4to16f(ray.dir.x,ray.dir.y,ray.dir.z); const mic_f rdir_xyz = loadAOS4to16f(rdir16.x[0],rdir16.y[0],rdir16.z[0]); const mic_f org_rdir_xyz = org_xyz * rdir_xyz; const mic_f min_dist_xyz = broadcast1to16f(&ray.tnear); const mic_f max_dist_xyz = broadcast1to16f(&ray.tfar); const unsigned int leaf_mask = BVH4I_LEAF_MASK; while (1) { NodeRef curNode = stack_node[sindex-1]; sindex--; while (1) { /* test if this is a leaf node */ if (unlikely(curNode.isLeaf(leaf_mask))) break; const Node* __restrict__ const node = curNode.node(nodes); const float* __restrict const plower = (float*)node->lower; const float* __restrict const pupper = (float*)node->upper; prefetch<PFHINT_L1>((char*)node + 0); prefetch<PFHINT_L1>((char*)node + 64); /* intersect single ray with 4 bounding boxes */ const mic_f tLowerXYZ = load16f(plower) * rdir_xyz - org_rdir_xyz; const mic_f tUpperXYZ = load16f(pupper) * rdir_xyz - org_rdir_xyz; const mic_f tLower = mask_min(0x7777,min_dist_xyz,tLowerXYZ,tUpperXYZ); const mic_f tUpper = mask_max(0x7777,max_dist_xyz,tLowerXYZ,tUpperXYZ); sindex--; curNode = stack_node[sindex]; const Node* __restrict__ const next = curNode.node(nodes); prefetch<PFHINT_L2>((char*)next + 0); prefetch<PFHINT_L2>((char*)next + 64); const mic_f tNear = vreduce_max4(tLower); const mic_f tFar = vreduce_min4(tUpper); const mic_m hitm = le(0x8888,tNear,tFar); const mic_f tNear_pos = select(hitm,tNear,inf); /* if no child is hit, continue with early popped child */ if (unlikely(none(hitm))) continue; sindex++; const unsigned long hiti = toInt(hitm); const unsigned long pos_first = bitscan64(hiti); const unsigned long num_hitm = countbits(hiti); /* if a single child is hit, continue with that child */ curNode = ((unsigned int *)plower)[pos_first]; if (likely(num_hitm == 1)) continue; /* if two children are hit, push in correct order */ const unsigned long pos_second = bitscan64(pos_first,hiti); if (likely(num_hitm == 2)) { const unsigned int dist_first = ((unsigned int*)&tNear)[pos_first]; const unsigned int dist_second = ((unsigned int*)&tNear)[pos_second]; const unsigned int node_first = curNode; const unsigned int node_second = ((unsigned int*)plower)[pos_second]; if (dist_first <= dist_second) { stack_node[sindex] = node_second; sindex++; assert(sindex < 3*BVH4i::maxDepth+1); continue; } else { stack_node[sindex] = curNode; curNode = node_second; sindex++; assert(sindex < 3*BVH4i::maxDepth+1); continue; } } /* continue with closest child and push all others */ const mic_f min_dist = set_min_lanes(tNear_pos); const unsigned old_sindex = sindex; sindex += countbits(hiti) - 1; assert(sindex < 3*BVH4i::maxDepth+1); const mic_m closest_child = eq(hitm,min_dist,tNear); const unsigned long closest_child_pos = bitscan64(closest_child); const mic_m m_pos = andn(hitm,andn(closest_child,(mic_m)((unsigned int)closest_child - 1))); const mic_i plower_node = load16i((int*)plower); curNode = ((unsigned int*)plower)[closest_child_pos]; compactustore16i(m_pos,&stack_node[old_sindex],plower_node); } /* return if stack is empty */ if (unlikely(curNode == BVH4i::invalidNode)) break; /* intersect one ray against four triangles */ ////////////////////////////////////////////////////////////////////////////////////////////////// const Triangle1* tptr = (Triangle1*) curNode.leaf(accel); prefetch<PFHINT_L1>(tptr + 3); prefetch<PFHINT_L1>(tptr + 2); prefetch<PFHINT_L1>(tptr + 1); prefetch<PFHINT_L1>(tptr + 0); const mic_i and_mask = broadcast4to16i(zlc4); const mic_f v0 = gather_4f_zlc(and_mask, (float*)&tptr[0].v0, (float*)&tptr[1].v0, (float*)&tptr[2].v0, (float*)&tptr[3].v0); const mic_f v1 = gather_4f_zlc(and_mask, (float*)&tptr[0].v1, (float*)&tptr[1].v1, (float*)&tptr[2].v1, (float*)&tptr[3].v1); const mic_f v2 = gather_4f_zlc(and_mask, (float*)&tptr[0].v2, (float*)&tptr[1].v2, (float*)&tptr[2].v2, (float*)&tptr[3].v2); const mic_f e1 = v1 - v0; const mic_f e2 = v0 - v2; const mic_f normal = lcross_zxy(e1,e2); const mic_f org = v0 - org_xyz; const mic_f odzxy = msubr231(org * swizzle(dir_xyz,_MM_SWIZ_REG_DACB), dir_xyz, swizzle(org,_MM_SWIZ_REG_DACB)); const mic_f den = ldot3_zxy(dir_xyz,normal); const mic_f rcp_den = rcp(den); const mic_f uu = ldot3_zxy(e2,odzxy); const mic_f vv = ldot3_zxy(e1,odzxy); const mic_f u = uu * rcp_den; const mic_f v = vv * rcp_den; #if defined(__BACKFACE_CULLING__) const mic_m m_init = (mic_m)0x1111 & (den > zero); #else const mic_m m_init = 0x1111; #endif const mic_m valid_u = ge(m_init,u,zero); const mic_m valid_v = ge(valid_u,v,zero); const mic_m m_aperture = le(valid_v,u+v,mic_f::one()); const mic_f nom = ldot3_zxy(org,normal); const mic_f t = rcp_den*nom; if (unlikely(none(m_aperture))) continue; mic_m m_final = lt(lt(m_aperture,min_dist_xyz,t),t,max_dist_xyz); #if defined(__USE_RAY_MASK__) const mic_i rayMask(ray.mask); const mic_i triMask = swDDDD(gather16i_4i_align(&tptr[0].v2,&tptr[1].v2,&tptr[2].v2,&tptr[3].v2)); const mic_m m_ray_mask = (rayMask & triMask) != mic_i::zero(); m_final &= m_ray_mask; #endif #if defined(__INTERSECTION_FILTER__) /* did the ray hit one of the four triangles? */ while (any(m_final)) { const mic_f temp_t = select(m_final,t,max_dist_xyz); const mic_f min_dist = vreduce_min(temp_t); const mic_m m_dist = eq(min_dist,temp_t); const size_t vecIndex = bitscan(toInt(m_dist)); const size_t triIndex = vecIndex >> 2; const Triangle1 *__restrict__ tri_ptr = tptr + triIndex; const mic_m m_tri = m_dist^(m_dist & (mic_m)((unsigned int)m_dist - 1)); const mic_f gnormalx = mic_f(tri_ptr->Ng.x); const mic_f gnormaly = mic_f(tri_ptr->Ng.y); const mic_f gnormalz = mic_f(tri_ptr->Ng.z); const int geomID = tri_ptr->geomID(); const int primID = tri_ptr->primID(); Geometry* geom = ((Scene*)bvh->geometry)->get(geomID); if (likely(!geom->hasOcclusionFilter1())) break; if (runOcclusionFilter1(geom,ray,u,v,min_dist,gnormalx,gnormaly,gnormalz,m_tri,geomID,primID)) break; m_final ^= m_tri; /* clear bit */ } #endif if (unlikely(any(m_final))) { ray.geomID = 0; return; } ////////////////////////////////////////////////////////////////////////////////////////////////// } }
void BVH4Intersector1<types,robust,PrimitiveIntersector>::occluded(const BVH4* bvh, Ray& ray) { /*! perform per ray precalculations required by the primitive intersector */ Precalculations pre(ray); BVH4::UnalignedNodeMB::Precalculations pre1(ray); /*! stack state */ NodeRef stack[stackSize]; //!< stack of nodes that still need to get traversed NodeRef* stackPtr = stack+1; //!< current stack pointer NodeRef* stackEnd = stack+stackSize; stack[0] = bvh->root; /*! load the ray into SIMD registers */ const Vec3fa ray_rdir = rcp_safe(ray.dir); const Vec3fa ray_org_rdir = ray.org*ray_rdir; const sse3f org(ray.org.x,ray.org.y,ray.org.z); const sse3f dir(ray.dir.x,ray.dir.y,ray.dir.z); const sse3f rdir(ray_rdir.x,ray_rdir.y,ray_rdir.z); const sse3f org_rdir(ray_org_rdir.x,ray_org_rdir.y,ray_org_rdir.z); const ssef ray_near(ray.tnear); ssef ray_far(ray.tfar); /*! offsets to select the side that becomes the lower or upper bound */ const size_t nearX = ray_rdir.x >= 0 ? 0*sizeof(ssef) : 1*sizeof(ssef); const size_t nearY = ray_rdir.y >= 0 ? 2*sizeof(ssef) : 3*sizeof(ssef); const size_t nearZ = ray_rdir.z >= 0 ? 4*sizeof(ssef) : 5*sizeof(ssef); /* pop loop */ while (true) pop: { /*! pop next node */ if (unlikely(stackPtr == stack)) break; stackPtr--; NodeRef cur = (NodeRef) *stackPtr; /* downtraversal loop */ while (true) { size_t mask; ssef tNear; /*! stop if we found a leaf node */ if (unlikely(cur.isLeaf(types))) break; STAT3(shadow.trav_nodes,1,1,1); /* process standard nodes */ if (likely(cur.isNode(types))) mask = cur.node()->intersect<robust>(nearX,nearY,nearZ,org,rdir,org_rdir,ray_near,ray_far,tNear); /* process motion blur nodes */ else if (likely(cur.isNodeMB(types))) mask = cur.nodeMB()->intersect(nearX,nearY,nearZ,org,rdir,org_rdir,ray_near,ray_far,ray.time,tNear); /*! process nodes with unaligned bounds */ else if (unlikely(cur.isUnalignedNode(types))) mask = cur.unalignedNode()->intersect(org,dir,ray_near,ray_far,tNear); /*! process nodes with unaligned bounds and motion blur */ else if (unlikely(cur.isUnalignedNodeMB(types))) mask = cur.unalignedNodeMB()->intersect(pre1,org,dir,ray_near,ray_far,ray.time,tNear); /*! if no child is hit, pop next node */ const BVH4::BaseNode* node = cur.baseNode(types); if (unlikely(mask == 0)) goto pop; /*! one child is hit, continue with that child */ size_t r = __bscf(mask); if (likely(mask == 0)) { cur = node->child(r); cur.prefetch(types); assert(cur != BVH4::emptyNode); continue; } /*! two children are hit, push far child, and continue with closer child */ NodeRef c0 = node->child(r); c0.prefetch(types); const unsigned int d0 = ((unsigned int*)&tNear)[r]; r = __bscf(mask); NodeRef c1 = node->child(r); c1.prefetch(types); const unsigned int d1 = ((unsigned int*)&tNear)[r]; assert(c0 != BVH4::emptyNode); assert(c1 != BVH4::emptyNode); if (likely(mask == 0)) { assert(stackPtr < stackEnd); if (d0 < d1) { *stackPtr = c1; stackPtr++; cur = c0; continue; } else { *stackPtr = c0; stackPtr++; cur = c1; continue; } } assert(stackPtr < stackEnd); *stackPtr = c0; stackPtr++; assert(stackPtr < stackEnd); *stackPtr = c1; stackPtr++; /*! three children are hit */ r = __bscf(mask); cur = node->child(r); cur.prefetch(types); assert(cur != BVH4::emptyNode); if (likely(mask == 0)) continue; assert(stackPtr < stackEnd); *stackPtr = cur; stackPtr++; /*! four children are hit */ cur = node->child(3); cur.prefetch(types); assert(cur != BVH4::emptyNode); } /*! this is a leaf node */ assert(cur != BVH4::emptyNode); STAT3(shadow.trav_leaves,1,1,1); size_t num; Primitive* prim = (Primitive*) cur.leaf(num); if (PrimitiveIntersector::occluded(pre,ray,prim,num,bvh->geometry)) { ray.geomID = 0; break; } } AVX_ZERO_UPPER(); }