/* Backwards propagate marks. Replace unused instructions with NOPs. */ static void dce_propagate(jit_State *J) { IRRef1 *pchain[IR__MAX]; IRRef ins; uint32_t i; for (i = 0; i < IR__MAX; i++) pchain[i] = &J->chain[i]; for (ins = J->cur.nins-1; ins >= REF_FIRST; ins--) { IRIns *ir = IR(ins); if (irt_ismarked(ir->t)) { irt_clearmark(ir->t); pchain[ir->o] = &ir->prev; } else if (!ir_sideeff(ir)) { *pchain[ir->o] = ir->prev; /* Reroute original instruction chain. */ *pchain[IR_NOP] = (IRRef1)ins; ir->t.irt = IRT_NIL; ir->o = IR_NOP; /* Replace instruction with NOP. */ ir->op1 = ir->op2 = 0; pchain[IR_NOP] = &ir->prev; continue; } if (ir->op1 >= REF_FIRST) irt_setmark(IR(ir->op1)->t); if (ir->op2 >= REF_FIRST) irt_setmark(IR(ir->op2)->t); } *pchain[IR_NOP] = 0; /* Terminate NOP chain. */ }
/* Emit or eliminate collected PHIs. */ static void loop_emit_phi(jit_State *J, IRRef1 *subst, IRRef1 *phi, IRRef nphi) { int pass2 = 0; IRRef i, nslots; IRRef invar = J->chain[IR_LOOP]; /* Pass #1: mark redundant and potentially redundant PHIs. */ for (i = 0; i < nphi; i++) { IRRef lref = phi[i]; IRRef rref = subst[lref]; if (lref == rref || rref == REF_DROP) { /* Invariants are redundant. */ irt_setmark(IR(lref)->t); } else if (!(IR(rref)->op1 == lref || IR(rref)->op2 == lref)) { /* Quick check for simple recurrences failed, need pass2. */ irt_setmark(IR(lref)->t); pass2 = 1; } } /* Pass #2: traverse variant part and clear marks of non-redundant PHIs. */ if (pass2) { for (i = J->cur.nins-1; i > invar; i--) { IRIns *ir = IR(i); if (!irref_isk(ir->op1)) irt_clearmark(IR(ir->op1)->t); if (!irref_isk(ir->op2)) irt_clearmark(IR(ir->op2)->t); } } /* Pass #3: add PHIs for variant slots without a corresponding SLOAD. */ nslots = J->baseslot+J->maxslot; for (i = 1; i < nslots; i++) { IRRef ref = tref_ref(J->slot[i]); while (!irref_isk(ref) && ref != subst[ref]) { IRIns *ir = IR(ref); irt_clearmark(ir->t); /* Unmark potential uses, too. */ if (irt_isphi(ir->t) || irt_ispri(ir->t)) break; irt_setphi(ir->t); if (nphi >= LJ_MAX_PHI) lj_trace_err(J, LJ_TRERR_PHIOV); phi[nphi++] = (IRRef1)ref; ref = subst[ref]; if (ref > invar) break; } } /* Pass #4: emit PHI instructions or eliminate PHIs. */ for (i = 0; i < nphi; i++) { IRRef lref = phi[i]; IRIns *ir = IR(lref); if (!irt_ismarked(ir->t)) { /* Emit PHI if not marked. */ IRRef rref = subst[lref]; if (rref > invar) irt_setphi(IR(rref)->t); emitir_raw(IRT(IR_PHI, irt_type(ir->t)), lref, rref); } else { /* Otherwise eliminate PHI. */ irt_clearmark(ir->t); irt_clearphi(ir->t); } } }
/* Iteratively remark PHI refs with differing marks or PHI value counts. */ static void sink_remark_phi(jit_State *J) { IRIns *ir; int remark; do { remark = 0; for (ir = IR(J->cur.nins-1); ir->o == IR_PHI; ir--) { IRIns *irl = IR(ir->op1), *irr = IR(ir->op2); if (!((irl->t.irt ^ irr->t.irt) & IRT_MARK) && irl->prev == irr->prev) continue; remark |= (~(irl->t.irt & irr->t.irt) & IRT_MARK); irt_setmark(IR(ir->op1)->t); irt_setmark(IR(ir->op2)->t); } } while (remark); }
/* Mark all instructions referenced by a snapshot. */ static void sink_mark_snap(jit_State *J, SnapShot *snap) { SnapEntry *map = &J->cur.snapmap[snap->mapofs]; MSize n, nent = snap->nent; for (n = 0; n < nent; n++) { IRRef ref = snap_ref(map[n]); if (!irref_isk(ref)) irt_setmark(IR(ref)->t); } }
/* Scan through all snapshots and mark all referenced instructions. */ static void dce_marksnap(jit_State *J) { SnapNo i, nsnap = J->cur.nsnap; for (i = 0; i < nsnap; i++) { SnapShot *snap = &J->cur.snap[i]; SnapEntry *map = &J->cur.snapmap[snap->mapofs]; MSize n, nent = snap->nent; for (n = 0; n < nent; n++) { IRRef ref = snap_ref(map[n]); if (ref >= REF_FIRST) irt_setmark(IR(ref)->t); } } }
/* Scan through all snapshots and mark all referenced instructions. */ static void dce_marksnap(jit_State *J) { SnapNo i, nsnap = J->cur.nsnap; for (i = 0; i < nsnap; i++) { SnapShot *snap = &J->cur.snap[i]; IRRef2 *map = &J->cur.snapmap[snap->mapofs]; BCReg s, nslots = snap->nslots; for (s = 0; s < nslots; s++) { IRRef ref = snap_ref(map[s]); if (!irref_isk(ref)) irt_setmark(IR(ref)->t); } } }
/* Mark non-sinkable allocations using single-pass backward propagation. ** ** Roots for the marking process are: ** - Some PHIs or snapshots (see below). ** - Non-PHI, non-constant values stored to PHI allocations. ** - All guards. ** - Any remaining loads not eliminated by store-to-load forwarding. ** - Stores with non-constant keys. ** - All stored values. */ static void sink_mark_ins(jit_State *J) { IRIns *ir, *irlast = IR(J->cur.nins-1); for (ir = irlast ; ; ir--) { switch (ir->o) { case IR_BASE: return; /* Finished. */ case IR_CALLL: /* IRCALL_lj_tab_len */ case IR_ALOAD: case IR_HLOAD: case IR_XLOAD: case IR_TBAR: irt_setmark(IR(ir->op1)->t); /* Mark ref for remaining loads. */ break; case IR_FLOAD: if (irt_ismarked(ir->t) || ir->op2 == IRFL_TAB_META) irt_setmark(IR(ir->op1)->t); /* Mark table for remaining loads. */ break; case IR_ASTORE: case IR_HSTORE: case IR_FSTORE: case IR_XSTORE: { IRIns *ira = sink_checkalloc(J, ir); if (!ira || (irt_isphi(ira->t) && !sink_checkphi(J, ira, ir->op2))) irt_setmark(IR(ir->op1)->t); /* Mark ineligible ref. */ irt_setmark(IR(ir->op2)->t); /* Mark stored value. */ break; } #if LJ_HASFFI case IR_CNEWI: if (irt_isphi(ir->t) && (!sink_checkphi(J, ir, ir->op2) || (LJ_32 && ir+1 < irlast && (ir+1)->o == IR_HIOP && !sink_checkphi(J, ir, (ir+1)->op2)))) irt_setmark(ir->t); /* Mark ineligible allocation. */ /* fallthrough */ #endif case IR_USTORE: irt_setmark(IR(ir->op2)->t); /* Mark stored value. */ break; #if LJ_HASFFI case IR_CALLXS: #endif case IR_CALLS: irt_setmark(IR(ir->op1)->t); /* Mark (potentially) stored values. */ break; case IR_PHI: { IRIns *irl = IR(ir->op1), *irr = IR(ir->op2); irl->prev = irr->prev = 0; /* Clear PHI value counts. */ if (irl->o == irr->o && (irl->o == IR_TNEW || irl->o == IR_TDUP || (LJ_HASFFI && (irl->o == IR_CNEW || irl->o == IR_CNEWI)))) break; irt_setmark(irl->t); irt_setmark(irr->t); break; } default: if (irt_ismarked(ir->t) || irt_isguard(ir->t)) { /* Propagate mark. */ if (ir->op1 >= REF_FIRST) irt_setmark(IR(ir->op1)->t); if (ir->op2 >= REF_FIRST) irt_setmark(IR(ir->op2)->t); } break; } } }
/* Emit or eliminate collected PHIs. */ static void loop_emit_phi(jit_State *J, IRRef1 *subst, IRRef1 *phi, IRRef nphi, SnapNo onsnap) { int passx = 0; IRRef i, j, nslots; IRRef invar = J->chain[IR_LOOP]; /* Pass #1: mark redundant and potentially redundant PHIs. */ for (i = 0, j = 0; i < nphi; i++) { IRRef lref = phi[i]; IRRef rref = subst[lref]; if (lref == rref || rref == REF_DROP) { /* Invariants are redundant. */ irt_clearphi(IR(lref)->t); } else { phi[j++] = (IRRef1)lref; if (!(IR(rref)->op1 == lref || IR(rref)->op2 == lref)) { /* Quick check for simple recurrences failed, need pass2. */ irt_setmark(IR(lref)->t); passx = 1; } } } nphi = j; /* Pass #2: traverse variant part and clear marks of non-redundant PHIs. */ if (passx) { SnapNo s; for (i = J->cur.nins-1; i > invar; i--) { IRIns *ir = IR(i); if (!irref_isk(ir->op2)) irt_clearmark(IR(ir->op2)->t); if (!irref_isk(ir->op1)) { irt_clearmark(IR(ir->op1)->t); if (ir->op1 < invar && ir->o >= IR_CALLN && ir->o <= IR_CARG) { /* ORDER IR */ ir = IR(ir->op1); while (ir->o == IR_CARG) { if (!irref_isk(ir->op2)) irt_clearmark(IR(ir->op2)->t); if (irref_isk(ir->op1)) break; ir = IR(ir->op1); irt_clearmark(ir->t); } } } } for (s = J->cur.nsnap-1; s >= onsnap; s--) { SnapShot *snap = &J->cur.snap[s]; SnapEntry *map = &J->cur.snapmap[snap->mapofs]; MSize n, nent = snap->nent; for (n = 0; n < nent; n++) { IRRef ref = snap_ref(map[n]); if (!irref_isk(ref)) irt_clearmark(IR(ref)->t); } } } /* Pass #3: add PHIs for variant slots without a corresponding SLOAD. */ nslots = J->baseslot+J->maxslot; for (i = 1; i < nslots; i++) { IRRef ref = tref_ref(J->slot[i]); while (!irref_isk(ref) && ref != subst[ref]) { IRIns *ir = IR(ref); irt_clearmark(ir->t); /* Unmark potential uses, too. */ if (irt_isphi(ir->t) || irt_ispri(ir->t)) break; irt_setphi(ir->t); if (nphi >= LJ_MAX_PHI) lj_trace_err(J, LJ_TRERR_PHIOV); phi[nphi++] = (IRRef1)ref; ref = subst[ref]; if (ref > invar) break; } } /* Pass #4: propagate non-redundant PHIs. */ while (passx) { passx = 0; for (i = 0; i < nphi; i++) { IRRef lref = phi[i]; IRIns *ir = IR(lref); if (!irt_ismarked(ir->t)) { /* Propagate only from unmarked PHIs. */ IRIns *irr = IR(subst[lref]); if (irt_ismarked(irr->t)) { /* Right ref points to other PHI? */ irt_clearmark(irr->t); /* Mark that PHI as non-redundant. */ passx = 1; /* Retry. */ } } } } /* Pass #5: emit PHI instructions or eliminate PHIs. */ for (i = 0; i < nphi; i++) { IRRef lref = phi[i]; IRIns *ir = IR(lref); if (!irt_ismarked(ir->t)) { /* Emit PHI if not marked. */ IRRef rref = subst[lref]; if (rref > invar) irt_setphi(IR(rref)->t); emitir_raw(IRT(IR_PHI, irt_type(ir->t)), lref, rref); } else { /* Otherwise eliminate PHI. */ irt_clearmark(ir->t); irt_clearphi(ir->t); } } }