void ED_view3d_clipping_calc(BoundBox *bb, float planes[4][4], bglMats *mats, const rcti *rect)
{
float modelview[4][4];
double xs, ys, p[3];
int val, flip_sign, a;
/* near zero floating point values can give issues with gluUnProject
* in side view on some implementations */
if (fabs(mats->modelview[0]) < 1e-6) mats->modelview[0] = 0.0;
if (fabs(mats->modelview[5]) < 1e-6) mats->modelview[5] = 0.0;
/* Set up viewport so that gluUnProject will give correct values */
mats->viewport[0] = 0;
mats->viewport[1] = 0;
/* four clipping planes and bounding volume */
/* first do the bounding volume */
for (val = 0; val < 4; val++) {
xs = (val == 0 || val == 3) ? rect->xmin : rect->xmax;
ys = (val == 0 || val == 1) ? rect->ymin : rect->ymax;
gluUnProject(xs, ys, 0.0, mats->modelview, mats->projection, mats->viewport, &p[0], &p[1], &p[2]);
copy_v3fl_v3db(bb->vec[val], p);
gluUnProject(xs, ys, 1.0, mats->modelview, mats->projection, mats->viewport, &p[0], &p[1], &p[2]);
copy_v3fl_v3db(bb->vec[4 + val], p);
}
/* verify if we have negative scale. doing the transform before cross
* product flips the sign of the vector compared to doing cross product
* before transform then, so we correct for that. */
for (a = 0; a < 16; a++)
((float *)modelview)[a] = mats->modelview[a];
flip_sign = is_negative_m4(modelview);
/* then plane equations */
for (val = 0; val < 4; val++) {
normal_tri_v3(planes[val], bb->vec[val], bb->vec[val == 3 ? 0 : val + 1], bb->vec[val + 4]);
if (flip_sign)
negate_v3(planes[val]);
planes[val][3] = -dot_v3v3(planes[val], bb->vec[val]);
}
}
static bool depth_unproject(
const ARegion *ar, const bglMats *mats,
const int mval[2], const double depth,
float r_location_world[3])
{
double p[3];
if (gluUnProject(
(double)ar->winrct.xmin + mval[0] + 0.5,
(double)ar->winrct.ymin + mval[1] + 0.5,
depth, mats->modelview, mats->projection, (const GLint *)mats->viewport,
&p[0], &p[1], &p[2]))
{
copy_v3fl_v3db(r_location_world, p);
return true;
}
return false;
}
