/* Mark non-sinkable allocations using single-pass backward propagation.
**
** Roots for the marking process are:
** - Some PHIs or snapshots (see below).
** - Non-PHI, non-constant values stored to PHI allocations.
** - All guards.
** - Any remaining loads not eliminated by store-to-load forwarding.
** - Stores with non-constant keys.
** - All stored values.
*/
static void sink_mark_ins(jit_State *J)
{
IRIns *ir, *irlast = IR(J->cur.nins-1);
for (ir = irlast ; ; ir--) {
switch (ir->o) {
case IR_BASE:
return; /* Finished. */
case IR_CALLL: /* IRCALL_lj_tab_len */
case IR_ALOAD: case IR_HLOAD: case IR_XLOAD: case IR_TBAR:
irt_setmark(IR(ir->op1)->t); /* Mark ref for remaining loads. */
break;
case IR_FLOAD:
if (irt_ismarked(ir->t) || ir->op2 == IRFL_TAB_META)
irt_setmark(IR(ir->op1)->t); /* Mark table for remaining loads. */
break;
case IR_ASTORE: case IR_HSTORE: case IR_FSTORE: case IR_XSTORE: {
IRIns *ira = sink_checkalloc(J, ir);
if (!ira || (irt_isphi(ira->t) && !sink_checkphi(J, ira, ir->op2)))
irt_setmark(IR(ir->op1)->t); /* Mark ineligible ref. */
irt_setmark(IR(ir->op2)->t); /* Mark stored value. */
break;
}
#if LJ_HASFFI
case IR_CNEWI:
if (irt_isphi(ir->t) &&
(!sink_checkphi(J, ir, ir->op2) ||
(LJ_32 && ir+1 < irlast && (ir+1)->o == IR_HIOP &&
!sink_checkphi(J, ir, (ir+1)->op2))))
irt_setmark(ir->t); /* Mark ineligible allocation. */
/* fallthrough */
#endif
case IR_USTORE:
irt_setmark(IR(ir->op2)->t); /* Mark stored value. */
break;
#if LJ_HASFFI
case IR_CALLXS:
#endif
case IR_CALLS:
irt_setmark(IR(ir->op1)->t); /* Mark (potentially) stored values. */
break;
case IR_PHI: {
IRIns *irl = IR(ir->op1), *irr = IR(ir->op2);
irl->prev = irr->prev = 0; /* Clear PHI value counts. */
if (irl->o == irr->o &&
(irl->o == IR_TNEW || irl->o == IR_TDUP ||
(LJ_HASFFI && (irl->o == IR_CNEW || irl->o == IR_CNEWI))))
break;
irt_setmark(irl->t);
irt_setmark(irr->t);
break;
}
default:
if (irt_ismarked(ir->t) || irt_isguard(ir->t)) { /* Propagate mark. */
if (ir->op1 >= REF_FIRST) irt_setmark(IR(ir->op1)->t);
if (ir->op2 >= REF_FIRST) irt_setmark(IR(ir->op2)->t);
}
break;
}
}
}
/* Sweep instructions and tag sunken allocations and stores. */
static void sink_sweep_ins(jit_State *J)
{
IRIns *ir, *irbase = IR(REF_BASE);
for (ir = IR(J->cur.nins-1) ; ir >= irbase; ir--) {
switch (ir->o) {
case IR_ASTORE: case IR_HSTORE: case IR_FSTORE: case IR_XSTORE: {
IRIns *ira = sink_checkalloc(J, ir);
if (ira && !irt_ismarked(ira->t)) {
int delta = (int)(ir - ira);
ir->prev = REGSP(RID_SINK, delta > 255 ? 255 : delta);
} else {
ir->prev = REGSP_INIT;
}
break;
}
case IR_NEWREF:
if (!irt_ismarked(IR(ir->op1)->t)) {
ir->prev = REGSP(RID_SINK, 0);
} else {
irt_clearmark(ir->t);
ir->prev = REGSP_INIT;
}
break;
#if LJ_HASFFI
case IR_CNEW: case IR_CNEWI:
#endif
case IR_TNEW: case IR_TDUP:
if (!irt_ismarked(ir->t)) {
ir->t.irt &= ~IRT_GUARD;
ir->prev = REGSP(RID_SINK, 0);
J->cur.sinktags = 1; /* Signal present SINK tags to assembler. */
} else {
irt_clearmark(ir->t);
ir->prev = REGSP_INIT;
}
break;
case IR_PHI: {
IRIns *ira = IR(ir->op2);
if (!irt_ismarked(ira->t) &&
(ira->o == IR_TNEW || ira->o == IR_TDUP ||
(LJ_HASFFI && (ira->o == IR_CNEW || ira->o == IR_CNEWI)))) {
ir->prev = REGSP(RID_SINK, 0);
} else {
ir->prev = REGSP_INIT;
}
break;
}
default:
irt_clearmark(ir->t);
ir->prev = REGSP_INIT;
break;
}
}
for (ir = IR(J->cur.nk); ir < irbase; ir++) {
irt_clearmark(ir->t);
ir->prev = REGSP_INIT;
if (irt_is64(ir->t) && ir->o != IR_KNULL)
ir++;
}
}