void visit(const For *op) {
do_indent();
out << op->for_type << ' ' << simplify_var_name(op->name);
// If the min or extent are constants, print them. At this
// stage they're all variables.
Expr min_val = op->min, extent_val = op->extent;
const Variable *min_var = min_val.as<Variable>();
const Variable *extent_var = extent_val.as<Variable>();
if (min_var && constants.contains(min_var->name)) {
min_val = constants.get(min_var->name);
}
if (extent_var && constants.contains(extent_var->name)) {
extent_val = constants.get(extent_var->name);
}
if (extent_val.defined() && is_const(extent_val) &&
min_val.defined() && is_const(min_val)) {
Expr max_val = simplify(min_val + extent_val - 1);
out << " in [" << min_val << ", " << max_val << "]";
}
out << ":\n";
indent += 2;
op->body.accept(this);
indent -= 2;
}
void Closure::pack_struct(llvm::Type *
#if LLVM_VERSION >= 37
type
#endif
,
Value *dst,
const Scope<Value *> &src,
IRBuilder<> *builder) {
// type, type of dst should be a pointer to a struct of the type returned by build_type
int idx = 0;
LLVMContext &context = builder->getContext();
vector<string> nm = names();
vector<llvm::Type*> ty = llvm_types(&context);
for (size_t i = 0; i < nm.size(); i++) {
#if LLVM_VERSION >= 37
Value *ptr = builder->CreateConstInBoundsGEP2_32(type, dst, 0, idx);
#else
Value *ptr = builder->CreateConstInBoundsGEP2_32(dst, 0, idx);
#endif
Value *val;
if (!ends_with(nm[i], ".buffer") || src.contains(nm[i])) {
val = src.get(nm[i]);
if (val->getType() != ty[i]) {
val = builder->CreateBitCast(val, ty[i]);
}
} else {
// Skip over buffers not in the symbol table. They must not be needed.
val = ConstantPointerNull::get(buffer_t->getPointerTo());
}
builder->CreateStore(val, ptr);
idx++;
}
}
void visit(const Free *op) {
int idx = get_func_id(op->name);
AllocSize alloc = func_alloc_sizes.get(op->name);
func_alloc_sizes.pop(op->name);
IRMutator::visit(op);
if (!is_zero(alloc.size)) {
Expr profiler_pipeline_state = Variable::make(Handle(), "profiler_pipeline_state");
if (!alloc.on_stack) {
debug(3) << " Free on heap: " << op->name << "(" << alloc.size << ") in pipeline " << pipeline_name << "\n";
Expr set_task = Call::make(Int(32), "halide_profiler_memory_free",
{profiler_pipeline_state, idx, alloc.size}, Call::Extern);
stmt = Block::make(Evaluate::make(set_task), stmt);
} else {
const int64_t *int_size = as_const_int(alloc.size);
internal_assert(int_size != nullptr);
func_stack_current[idx] -= *int_size;
debug(3) << " Free on stack: " << op->name << "(" << alloc.size << ") in pipeline " << pipeline_name
<< "; current: " << func_stack_current[idx] << "; peak: " << func_stack_peak[idx] << "\n";
}
}
}
void visit(const Variable *op) {
if (scope.contains(op->name)) {
expr = scope.get(op->name);
} else {
expr = op;
}
}
void visit(const Variable *op) {
if (op->name == var) {
result = Monotonic::Increasing;
} else if (scope.contains(op->name)) {
result = scope.get(op->name);
} else {
result = Monotonic::Constant;
}
}
void visit(const Variable *op) {
if (op->name == var) {
expr = make_one(op->type);
} else if (scope.contains(op->name)) {
expr = scope.get(op->name);
} else {
expr = make_zero(op->type);
}
}
void ModulusRemainder::visit(const Variable *op) {
if (scope.contains(op->name)) {
pair<int, int> mod_rem = scope.get(op->name);
modulus = mod_rem.first;
remainder = mod_rem.second;
} else {
modulus = 1;
remainder = 0;
}
}
void ComputeModulusRemainder::visit(const Variable *op) {
if (scope.contains(op->name)) {
ModulusRemainder mod_rem = scope.get(op->name);
modulus = mod_rem.modulus;
remainder = mod_rem.remainder;
} else {
modulus = 1;
remainder = 0;
}
}
virtual void visit(const Variable *op) {
if (op->name == var) {
expr = replacement;
} else if (scope.contains(op->name)) {
// The type of a var may have changed. E.g. if
// we're vectorizing across x we need to know the
// type of y has changed in the following example:
// let y = x + 1 in y*3
expr = Variable::make(scope.get(op->name), op->name);
} else {
expr = op;
}
}
llvm::Value * get(std::string name)
{
auto it = table.find(name);
if (it != table.end()) {
return it->second;
}
if (parent != nullptr) {
return parent->get(name);
} else {
return nullptr;
}
}
string var(const string &x) {
int id;
if (scope.contains(x)) {
id = scope.get(x);
} else {
id = unique_id();
scope.push(x, id);
}
std::stringstream s;
s << "<b class='Variable Matched' id='" << id << "-" << unique_id() << "'>";
s << x;
s << "</b>";
return s.str();
}
void Closure::pack_struct(Value *dst, const Scope<Value *> &src, IRBuilder<> *builder) {
// dst should be a pointer to a struct of the type returned by build_type
int idx = 0;
LLVMContext &context = builder->getContext();
vector<string> nm = names();
vector<llvm::Type*> ty = llvm_types(&context);
for (size_t i = 0; i < nm.size(); i++) {
if (!ends_with(nm[i], ".buffer") || src.contains(nm[i])) {
Value *val = src.get(nm[i]);
Value *ptr = builder->CreateConstInBoundsGEP2_32(dst, 0, idx);
if (val->getType() != ty[i]) {
val = builder->CreateBitCast(val, ty[i]);
}
builder->CreateStore(val, ptr);
}
idx++;
}
}
void visit(const Pipeline *op) {
if (op->name != func.name()) {
IRMutator::visit(op);
} else {
// We're interested in the case where exactly one of the
// mins of the buffer depends on the loop_var, and none of
// the extents do.
string dim = "";
Expr min, extent;
for (size_t i = 0; i < func.args().size(); i++) {
string min_name = func.name() + "." + func.args()[i] + ".min";
string extent_name = func.name() + "." + func.args()[i] + ".extent";
assert(scope.contains(min_name) && scope.contains(extent_name));
Expr this_min = scope.get(min_name);
Expr this_extent = scope.get(extent_name);
if (expr_depends_on_var(this_extent, loop_var)) {
min = Expr();
extent = Expr();
break;
}
if (expr_depends_on_var(this_min, loop_var)) {
if (min.defined()) {
min = Expr();
extent = Expr();
break;
} else {
dim = func.args()[i];
min = this_min;
extent = this_extent;
}
}
}
if (min.defined()) {
// Ok, we've isolated a function, a dimension to slide along, and loop variable to slide over
debug(2) << "Sliding " << func.name() << " over dimension " << dim << " along loop variable " << loop_var << "\n";
Expr loop_var_expr = Variable::make(Int(32), loop_var);
Expr steady_state = loop_var_expr > loop_min;
// The new min is one beyond the max we reached on the last loop iteration
Expr new_min = substitute(loop_var, loop_var_expr - 1, min + extent);
// The new extent is the old extent shrunk by how much we trimmed off the min
Expr new_extent = extent + min - new_min;
new_min = Select::make(steady_state, new_min, min);
new_extent = Select::make(steady_state, new_extent, extent);
stmt = LetStmt::make(func.name() + "." + dim + ".extent", new_extent, op);
stmt = LetStmt::make(func.name() + "." + dim + ".min", new_min, stmt);
} else {
debug(2) << "Could not perform sliding window optimization of " << func.name() << " over " << loop_var << "\n";
stmt = op;
}
}
}
void visit(const ProducerConsumer *op) {
if (op->name != func.name()) {
IRMutator::visit(op);
} else {
stmt = op;
// We're interested in the case where exactly one of the
// dimensions of the buffer has a min/extent that depends
// on the loop_var.
string dim = "";
int dim_idx = 0;
Expr min_required, max_required;
debug(3) << "Considering sliding " << func.name()
<< " along loop variable " << loop_var << "\n"
<< "Region provided:\n";
string prefix = func.name() + ".s" + std::to_string(func.updates().size()) + ".";
const std::vector<string> func_args = func.args();
for (int i = 0; i < func.dimensions(); i++) {
// Look up the region required of this function's last stage
string var = prefix + func_args[i];
internal_assert(scope.contains(var + ".min") && scope.contains(var + ".max"));
Expr min_req = scope.get(var + ".min");
Expr max_req = scope.get(var + ".max");
min_req = expand_expr(min_req, scope);
max_req = expand_expr(max_req, scope);
debug(3) << func_args[i] << ":" << min_req << ", " << max_req << "\n";
if (expr_depends_on_var(min_req, loop_var) ||
expr_depends_on_var(max_req, loop_var)) {
if (!dim.empty()) {
dim = "";
min_required = Expr();
max_required = Expr();
break;
} else {
dim = func_args[i];
dim_idx = i;
min_required = min_req;
max_required = max_req;
}
}
}
if (!min_required.defined()) {
debug(3) << "Could not perform sliding window optimization of "
<< func.name() << " over " << loop_var << " because either zero "
<< "or many dimensions of the function dependended on the loop var\n";
return;
}
// If the function is not pure in the given dimension, give up. We also
// need to make sure that it is pure in all the specializations
bool pure = true;
for (const Definition &def : func.updates()) {
pure = is_dim_always_pure(def, dim, dim_idx);
if (!pure) {
break;
}
}
if (!pure) {
debug(3) << "Could not performance sliding window optimization of "
<< func.name() << " over " << loop_var << " because the function "
<< "scatters along the related axis.\n";
return;
}
bool can_slide_up = false;
bool can_slide_down = false;
Monotonic monotonic_min = is_monotonic(min_required, loop_var);
Monotonic monotonic_max = is_monotonic(max_required, loop_var);
if (monotonic_min == Monotonic::Increasing ||
monotonic_min == Monotonic::Constant) {
can_slide_up = true;
}
if (monotonic_max == Monotonic::Decreasing ||
monotonic_max == Monotonic::Constant) {
can_slide_down = true;
}
if (!can_slide_up && !can_slide_down) {
debug(3) << "Not sliding " << func.name()
<< " over dimension " << dim
<< " along loop variable " << loop_var
<< " because I couldn't prove it moved monotonically along that dimension\n"
<< "Min is " << min_required << "\n"
<< "Max is " << max_required << "\n";
return;
}
// Ok, we've isolated a function, a dimension to slide
// along, and loop variable to slide over.
debug(3) << "Sliding " << func.name()
<< " over dimension " << dim
<< " along loop variable " << loop_var << "\n";
Expr loop_var_expr = Variable::make(Int(32), loop_var);
Expr prev_max_plus_one = substitute(loop_var, loop_var_expr - 1, max_required) + 1;
Expr prev_min_minus_one = substitute(loop_var, loop_var_expr - 1, min_required) - 1;
// If there's no overlap between adjacent iterations, we shouldn't slide.
if (can_prove(min_required >= prev_max_plus_one) ||
can_prove(max_required <= prev_min_minus_one)) {
debug(3) << "Not sliding " << func.name()
<< " over dimension " << dim
<< " along loop variable " << loop_var
<< " there's no overlap in the region computed across iterations\n"
<< "Min is " << min_required << "\n"
<< "Max is " << max_required << "\n";
return;
}
Expr new_min, new_max;
if (can_slide_up) {
new_min = select(loop_var_expr <= loop_min, min_required, likely(prev_max_plus_one));
new_max = max_required;
} else {
new_min = min_required;
new_max = select(loop_var_expr <= loop_min, max_required, likely(prev_min_minus_one));
}
Expr early_stages_min_required = new_min;
Expr early_stages_max_required = new_max;
debug(3) << "Sliding " << func.name() << ", " << dim << "\n"
<< "Pushing min up from " << min_required << " to " << new_min << "\n"
<< "Shrinking max from " << max_required << " to " << new_max << "\n";
// Now redefine the appropriate regions required
if (can_slide_up) {
replacements[prefix + dim + ".min"] = new_min;
} else {
replacements[prefix + dim + ".max"] = new_max;
}
for (size_t i = 0; i < func.updates().size(); i++) {
string n = func.name() + ".s" + std::to_string(i) + "." + dim;
replacements[n + ".min"] = Variable::make(Int(32), prefix + dim + ".min");
replacements[n + ".max"] = Variable::make(Int(32), prefix + dim + ".max");
}
// Ok, we have a new min/max required and we're going to
// rewrite all the lets that define bounds required. Now
// we need to additionally expand the bounds required of
// the last stage to cover values produced by stages
// before the last one. Because, e.g., an intermediate
// stage may be unrolled, expanding its bounds provided.
if (op->update.defined()) {
Box b = box_provided(op->produce, func.name());
merge_boxes(b, box_provided(op->update, func.name()));
if (can_slide_up) {
string n = prefix + dim + ".min";
Expr var = Variable::make(Int(32), n);
stmt = LetStmt::make(n, min(var, b[dim_idx].min), stmt);
} else {
string n = prefix + dim + ".max";
Expr var = Variable::make(Int(32), n);
stmt = LetStmt::make(n, max(var, b[dim_idx].max), stmt);
}
}
}
}