void ED_view3d_clipping_calc(BoundBox *bb, float planes[4][4], bglMats *mats, const rcti *rect) { float modelview[4][4]; double xs, ys, p[3]; int val, flip_sign, a; /* near zero floating point values can give issues with gluUnProject * in side view on some implementations */ if (fabs(mats->modelview[0]) < 1e-6) mats->modelview[0] = 0.0; if (fabs(mats->modelview[5]) < 1e-6) mats->modelview[5] = 0.0; /* Set up viewport so that gluUnProject will give correct values */ mats->viewport[0] = 0; mats->viewport[1] = 0; /* four clipping planes and bounding volume */ /* first do the bounding volume */ for (val = 0; val < 4; val++) { xs = (val == 0 || val == 3) ? rect->xmin : rect->xmax; ys = (val == 0 || val == 1) ? rect->ymin : rect->ymax; gluUnProject(xs, ys, 0.0, mats->modelview, mats->projection, mats->viewport, &p[0], &p[1], &p[2]); copy_v3fl_v3db(bb->vec[val], p); gluUnProject(xs, ys, 1.0, mats->modelview, mats->projection, mats->viewport, &p[0], &p[1], &p[2]); copy_v3fl_v3db(bb->vec[4 + val], p); } /* verify if we have negative scale. doing the transform before cross * product flips the sign of the vector compared to doing cross product * before transform then, so we correct for that. */ for (a = 0; a < 16; a++) ((float *)modelview)[a] = mats->modelview[a]; flip_sign = is_negative_m4(modelview); /* then plane equations */ for (val = 0; val < 4; val++) { normal_tri_v3(planes[val], bb->vec[val], bb->vec[val == 3 ? 0 : val + 1], bb->vec[val + 4]); if (flip_sign) negate_v3(planes[val]); planes[val][3] = -dot_v3v3(planes[val], bb->vec[val]); } }
static bool depth_unproject( const ARegion *ar, const bglMats *mats, const int mval[2], const double depth, float r_location_world[3]) { double p[3]; if (gluUnProject( (double)ar->winrct.xmin + mval[0] + 0.5, (double)ar->winrct.ymin + mval[1] + 0.5, depth, mats->modelview, mats->projection, (const GLint *)mats->viewport, &p[0], &p[1], &p[2])) { copy_v3fl_v3db(r_location_world, p); return true; } return false; }