void visit(const For *op) {
do_indent();
out << op->for_type << ' ' << simplify_var_name(op->name);
// If the min or extent are constants, print them. At this
// stage they're all variables.
Expr min_val = op->min, extent_val = op->extent;
const Variable *min_var = min_val.as<Variable>();
const Variable *extent_var = extent_val.as<Variable>();
if (min_var && constants.contains(min_var->name)) {
min_val = constants.get(min_var->name);
}
if (extent_var && constants.contains(extent_var->name)) {
extent_val = constants.get(extent_var->name);
}
if (extent_val.defined() && is_const(extent_val) &&
min_val.defined() && is_const(min_val)) {
Expr max_val = simplify(min_val + extent_val - 1);
out << " in [" << min_val << ", " << max_val << "]";
}
out << ":\n";
indent += 2;
op->body.accept(this);
indent -= 2;
}
void visit(const Call *call) {
if (!inside_kernel_loop ||
call->call_type == Call::Intrinsic || call->call_type == Call::Extern) {
IRMutator::visit(call);
return;
}
string name = call->name;
if (call->call_type == Call::Halide && call->func.outputs() > 1) {
name = name + '.' + int_to_string(call->value_index);
}
user_assert(call->args.size() == 3) << "GLSL loads require three coordinates.\n";
// Create glsl_texture_load(name, name.buffer, x, y, c) intrinsic.
vector<Expr> args(5);
args[0] = call->name;
args[1] = Variable::make(Handle(), call->name + ".buffer");
for (size_t i = 0; i < call->args.size(); i++) {
string d = int_to_string(i);
string min_name = name + ".min." + d;
string min_name_constrained = min_name + ".constrained";
if (scope.contains(min_name_constrained)) {
min_name = min_name_constrained;
}
string extent_name = name + ".extent." + d;
string extent_name_constrained = extent_name + ".constrained";
if (scope.contains(extent_name_constrained)) {
extent_name = extent_name_constrained;
}
Expr min = Variable::make(Int(32), min_name);
Expr extent = Variable::make(Int(32), extent_name);
// Remind users to explicitly specify the 'min' values of
// ImageParams accessed by GLSL filters.
if (i == 2 && call->param.defined()) {
bool const_min_constraint = call->param.min_constraint(i).defined() &&
is_const(call->param.min_constraint(i));
if (!const_min_constraint) {
user_warning
<< "GLSL: Assuming min[2]==0 for ImageParam '" << name << "'. "
<< "Call set_min(2, min) or set_bounds(2, min, extent) to override.\n";
min = Expr(0);
}
}
if (i < 2) {
// Convert spatial coordinates x,y into texture coordinates by normalization.
args[i + 2] = (Cast::make(Float(32), call->args[i] - min) + 0.5f) / extent;
} else {
args[i + 2] = call->args[i] - min;
}
}
expr = Call::make(call->type, Call::glsl_texture_load,
args, Call::Intrinsic,
Function(), 0, call->image, call->param);
}
void visit(const Variable *var) {
// Is it a parameter?
if (var->param.defined()) return;
// Was it defined internally by a let expression?
if (defined_internally.contains(var->name)) return;
// Is it a pure argument?
for (size_t i = 0; i < pure_args.size(); i++) {
if (var->name == pure_args[i]) return;
}
// Is it in a reduction domain?
if (var->reduction_domain.defined()) {
if (!reduction_domain.defined()) {
reduction_domain = var->reduction_domain;
return;
} else if (var->reduction_domain.same_as(reduction_domain)) {
// It's in a reduction domain we already know about
return;
} else {
user_error << "Multiple reduction domains found in definition of Func \"" << name << "\"\n";
}
}
user_error << "Undefined variable \"" << var->name << "\" in definition of Func \"" << name << "\"\n";
}
void visit(const Variable *op) {
if (scope.contains(op->name)) {
expr = scope.get(op->name);
} else {
expr = op;
}
}
void visit(const Variable *v) {
if (internal.contains(v->name)) {
// Don't capture internally defined vars
return;
}
if (starts_with(v->name, "iv.")) {
// Don't capture implicit vars
return;
}
if (v->reduction_domain.defined()) {
rdom = RDom(v->reduction_domain);
return;
}
if (v->param.defined()) {
// Skip parameters
return;
}
for (size_t i = 0; i < free_vars.size(); i++) {
if (v->name == free_vars[i].name()) return;
}
free_vars.push_back(Var(v->name));
}
virtual void visit(const For *for_loop) {
// Compute the region required of each function within this loop body
map<string, Region> regions = regions_required(for_loop->body);
Stmt body = mutate(for_loop->body);
log(3) << "Bounds inference considering loop over " << for_loop->name << '\n';
// Inject let statements defining those bounds
for (size_t i = 0; i < funcs.size(); i++) {
if (in_update.contains(funcs[i])) continue;
const Region ®ion = regions[funcs[i]];
const Function &f = env.find(funcs[i])->second;
if (region.empty()) continue;
log(3) << "Injecting bounds for " << funcs[i] << '\n';
assert(region.size() == f.args().size() && "Dimensionality mismatch between function and region required");
for (size_t j = 0; j < region.size(); j++) {
const string &arg_name = f.args()[j];
body = new LetStmt(f.name() + "." + arg_name + ".min", region[j].min, body);
body = new LetStmt(f.name() + "." + arg_name + ".extent", region[j].extent, body);
}
}
if (body.same_as(for_loop->body)) {
stmt = for_loop;
} else {
stmt = new For(for_loop->name, for_loop->min, for_loop->extent, for_loop->for_type, body);
}
}
void visit(const Store *op) {
if (allocs.contains(op->name)) {
allocs.pop(op->name);
}
IRMutator::visit(op);
}
void visit(const Variable *var) {
// Is it a parameter?
if (var->param.defined()) return;
// Was it defined internally by a let expression?
if (defined_internally.contains(var->name)) return;
// Is it a pure argument?
for (size_t i = 0; i < pure_args.size(); i++) {
if (var->name == pure_args[i]) return;
}
// Is it in a reduction domain?
if (var->reduction_domain.defined()) {
if (!reduction_domain.defined()) {
reduction_domain = var->reduction_domain;
return;
} else if (var->reduction_domain.same_as(reduction_domain)) {
// It's in a reduction domain we already know about
return;
} else {
assertf(false, "Multiple reduction domains found in function definition", name);
}
}
std::cerr << "Undefined variable in function definition: " << var->name
<< " (Func: " << name << ")"
<< std::endl;
assert(false);
}
void Closure::pack_struct(llvm::Type *
#if LLVM_VERSION >= 37
type
#endif
,
Value *dst,
const Scope<Value *> &src,
IRBuilder<> *builder) {
// type, type of dst should be a pointer to a struct of the type returned by build_type
int idx = 0;
LLVMContext &context = builder->getContext();
vector<string> nm = names();
vector<llvm::Type*> ty = llvm_types(&context);
for (size_t i = 0; i < nm.size(); i++) {
#if LLVM_VERSION >= 37
Value *ptr = builder->CreateConstInBoundsGEP2_32(type, dst, 0, idx);
#else
Value *ptr = builder->CreateConstInBoundsGEP2_32(dst, 0, idx);
#endif
Value *val;
if (!ends_with(nm[i], ".buffer") || src.contains(nm[i])) {
val = src.get(nm[i]);
if (val->getType() != ty[i]) {
val = builder->CreateBitCast(val, ty[i]);
}
} else {
// Skip over buffers not in the symbol table. They must not be needed.
val = ConstantPointerNull::get(buffer_t->getPointerTo());
}
builder->CreateStore(val, ptr);
idx++;
}
}
void visit(const Call *call) {
if (!inside_kernel_loop || call->call_type == Call::Intrinsic) {
IRMutator::visit(call);
return;
}
string name = call->name;
if (call->call_type == Call::Halide && call->func.outputs() > 1) {
name = name + '.' + int_to_string(call->value_index);
}
user_assert(call->args.size() == 3) << "GLSL loads requires three coordinates.\n";
// Record that this buffer is accessed from a GPU kernel
need_buffer_t.push(call->name, 0);
// Create glsl_texture_load(name, name.buffer, x, y, c) intrinsic.
vector<Expr> args(5);
args[0] = call->name;
args[1] = Variable::make(Handle(), call->name + ".buffer");
for (size_t i = 0; i < call->args.size(); i++) {
string d = int_to_string(i);
string min_name = name + ".min." + d;
string min_name_constrained = min_name + ".constrained";
if (scope.contains(min_name_constrained)) {
min_name = min_name_constrained;
}
string extent_name = name + ".extent." + d;
string extent_name_constrained = extent_name + ".constrained";
if (scope.contains(extent_name_constrained)) {
extent_name = extent_name_constrained;
}
Expr min = Variable::make(Int(32), min_name);
Expr extent = Variable::make(Int(32), extent_name);
// Normalize the two spatial coordinates x,y
args[i + 2] = (i < 2)
? (Cast::make(Float(32), call->args[i] - min) + 0.5f) / extent
: call->args[i] - min;
}
expr = Call::make(call->type, Call::glsl_texture_load,
args, Call::Intrinsic,
Function(), 0, call->image, call->param);
}
void visit(const Free *op) {
if (allocs.contains(op->name)) {
// We have reached a Free Stmt without ever using this buffer, do nothing.
stmt = Evaluate::make(0);
} else {
stmt = op;
}
}
Expr find_replacement(const string &s) {
map<string, Expr>::const_iterator iter = replace.find(s);
if (iter != replace.end() && !hidden.contains(s)) {
return iter->second;
} else {
return Expr();
}
}
void visit(const Variable *op) {
if (op->name == var) {
result = Monotonic::Increasing;
} else if (scope.contains(op->name)) {
result = scope.get(op->name);
} else {
result = Monotonic::Constant;
}
}
void visit(const Variable *op) {
if (op->name == var) {
expr = make_one(op->type);
} else if (scope.contains(op->name)) {
expr = scope.get(op->name);
} else {
expr = make_zero(op->type);
}
}
void ModulusRemainder::visit(const Variable *op) {
if (scope.contains(op->name)) {
pair<int, int> mod_rem = scope.get(op->name);
modulus = mod_rem.first;
remainder = mod_rem.second;
} else {
modulus = 1;
remainder = 0;
}
}
void ComputeModulusRemainder::visit(const Variable *op) {
if (scope.contains(op->name)) {
ModulusRemainder mod_rem = scope.get(op->name);
modulus = mod_rem.modulus;
remainder = mod_rem.remainder;
} else {
modulus = 1;
remainder = 0;
}
}
void visit(const Variable *v) {
string var_name = v->name;
expr = v;
if (internal.contains(var_name)) {
// Don't capture internally defined vars
return;
}
if (v->reduction_domain.defined()) {
if (explicit_rdom) {
if (v->reduction_domain.same_as(rdom.domain())) {
// This variable belongs to the explicit reduction domain, so
// skip it.
return;
} else {
// This should be converted to a pure variable and
// added to the free vars list.
var_name = unique_name('v');
expr = Variable::make(v->type, var_name);
}
} else {
if (!rdom.defined()) {
// We're looking for a reduction domain, and this variable
// has one. Capture it.
rdom = RDom(v->reduction_domain);
return;
} else if (!rdom.domain().same_as(v->reduction_domain)) {
// We were looking for a reduction domain, and already
// found one. This one is different!
user_error << "Inline reduction \"" << name
<< "\" refers to reduction variables from multiple reduction domains: "
<< v->name << ", " << rdom.x.name() << "\n";
} else {
// Recapturing an already-known reduction domain
return;
}
}
}
if (v->param.defined()) {
// Skip parameters
return;
}
for (size_t i = 0; i < free_vars.size(); i++) {
if (var_name == free_vars[i].name()) return;
}
free_vars.push_back(Var(var_name));
call_args.push_back(v);
}
void visit(const Allocate *op) {
allocs.push(op->name, 1);
Stmt body = mutate(op->body);
if (allocs.contains(op->name)) {
stmt = body;
allocs.pop(op->name);
} else if (body.same_as(op->body)) {
stmt = op;
} else {
stmt = Allocate::make(op->name, op->type, op->extents, op->condition, body, op->new_expr, op->free_function);
}
}
virtual void visit(const Variable *op) {
if (op->name == var) {
expr = replacement;
} else if (scope.contains(op->name)) {
// The type of a var may have changed. E.g. if
// we're vectorizing across x we need to know the
// type of y has changed in the following example:
// let y = x + 1 in y*3
expr = Variable::make(scope.get(op->name), op->name);
} else {
expr = op;
}
}
void visit(const Call *op) {
if (op->call_type == Call::Extern) {
for (size_t i = 0; i < op->args.size(); i++) {
const Variable *var = op->args[i].as<Variable>();
if (var && ends_with(var->name, ".buffer")) {
std::string func = var->name.substr(0, var->name.find_first_of('.'));
if (allocs.contains(func)) {
allocs.pop(func);
}
}
}
}
IRMutator::visit(op);
}
string var(const string &x) {
int id;
if (scope.contains(x)) {
id = scope.get(x);
} else {
id = unique_id();
scope.push(x, id);
}
std::stringstream s;
s << "<b class='Variable Matched' id='" << id << "-" << unique_id() << "'>";
s << x;
s << "</b>";
return s.str();
}
void visit(const Variable *op) {
Type t = op->type;
t.width = new_width;
if (internal.contains(op->name)) {
expr = Variable::make(t, op->name, op->param, op->reduction_domain);
} else {
// Uh-oh, we don't know how to deinterleave this vector expression
// Make llvm do it
std::vector<Expr> args;
args.push_back(op);
for (int i = 0; i < new_width; i++) {
args.push_back(starting_lane + lane_stride * i);
}
expr = Call::make(t, Call::shuffle_vector, args, Call::Intrinsic);
}
}
void Closure::pack_struct(Value *dst, const Scope<Value *> &src, IRBuilder<> *builder) {
// dst should be a pointer to a struct of the type returned by build_type
int idx = 0;
LLVMContext &context = builder->getContext();
vector<string> nm = names();
vector<llvm::Type*> ty = llvm_types(&context);
for (size_t i = 0; i < nm.size(); i++) {
if (!ends_with(nm[i], ".buffer") || src.contains(nm[i])) {
Value *val = src.get(nm[i]);
Value *ptr = builder->CreateConstInBoundsGEP2_32(dst, 0, idx);
if (val->getType() != ty[i]) {
val = builder->CreateBitCast(val, ty[i]);
}
builder->CreateStore(val, ptr);
}
idx++;
}
}
CFA::CFA(const Scope& scope)
: scope_(scope)
, entry_(node(scope.entry()))
, exit_ (node(scope.exit() ))
{
std::queue<Continuation*> cfg_queue;
ContinuationSet cfg_done;
auto cfg_enqueue = [&] (Continuation* continuation) {
if (cfg_done.emplace(continuation).second)
cfg_queue.push(continuation);
};
cfg_queue.push(scope.entry());
while (!cfg_queue.empty()) {
auto src = pop(cfg_queue);
std::queue<const Def*> queue;
DefSet done;
auto enqueue = [&] (const Def* def) {
if (def->order() > 0 && scope.contains(def) && done.emplace(def).second) {
if (auto dst = def->isa_continuation()) {
cfg_enqueue(dst);
node(src)->link(node(dst));
} else
queue.push(def);
}
};
queue.push(src);
while (!queue.empty()) {
auto def = pop(queue);
for (auto op : def->ops())
enqueue(op);
}
}
link_to_exit();
verify();
}
virtual void visit(const For *for_loop) {
Scope<pair<Expr, Expr> > scope;
// Compute the region required of each function within this loop body
map<string, vector<pair<Expr, Expr> > > regions = regions_required(for_loop->body, scope);
// TODO: For reductions we also need to consider the region
// provided within any update statements over this function
// (but not within the produce statement)
Stmt body = mutate(for_loop->body);
log(3) << "Bounds inference considering loop over " << for_loop->name << '\n';
// Inject let statements defining those bounds
for (size_t i = 0; i < funcs.size(); i++) {
if (in_update.contains(funcs[i])) continue;
const vector<pair<Expr, Expr> > ®ion = regions[funcs[i]];
const Function &f = env.find(funcs[i])->second;
if (region.empty()) continue;
log(3) << "Injecting bounds for " << funcs[i] << '\n';
assert(region.size() == f.args().size() && "Dimensionality mismatch between function and region required");
for (size_t j = 0; j < region.size(); j++) {
const string &arg_name = f.args()[j];
body = new LetStmt(f.name() + "." + arg_name + ".min", region[j].first, body);
body = new LetStmt(f.name() + "." + arg_name + ".extent", region[j].second, body);
}
}
if (body.same_as(for_loop->body)) {
stmt = for_loop;
} else {
stmt = new For(for_loop->name, for_loop->min, for_loop->extent, for_loop->for_type, body);
}
}
void visit(const Call *call) {
if (!inside_kernel_loop ||
call->call_type == Call::Intrinsic || call->call_type == Call::Extern) {
IRMutator::visit(call);
return;
}
string name = call->name;
if (call->call_type == Call::Halide && call->func.outputs() > 1) {
name = name + '.' + int_to_string(call->value_index);
}
// Check to see if we are reading from a one or two dimension function
// and pad to three dimensions.
vector<Expr> call_args = call->args;
while (call_args.size() < 3) {
call_args.push_back(IntImm::make(0));
}
// Create glsl_texture_load(name, name.buffer, x, y, c) intrinsic.
vector<Expr> args(5);
args[0] = call->name;
args[1] = Variable::make(Handle(), call->name + ".buffer");
for (size_t i = 0; i < call_args.size(); i++) {
string d = int_to_string(i);
string min_name = name + ".min." + d;
string min_name_constrained = min_name + ".constrained";
if (scope.contains(min_name_constrained)) {
min_name = min_name_constrained;
}
string extent_name = name + ".extent." + d;
string extent_name_constrained = extent_name + ".constrained";
if (scope.contains(extent_name_constrained)) {
extent_name = extent_name_constrained;
}
Expr min = Variable::make(Int(32), min_name);
Expr extent = Variable::make(Int(32), extent_name);
// Remind users to explicitly specify the 'min' values of
// ImageParams accessed by GLSL filters.
if (i == 2 && call->param.defined()) {
bool const_min_constraint = call->param.min_constraint(i).defined() &&
is_const(call->param.min_constraint(i));
if (!const_min_constraint) {
user_warning
<< "GLSL: Assuming min[2]==0 for ImageParam '" << name << "'. "
<< "Call set_min(2, min) or set_bounds(2, min, extent) to override.\n";
min = Expr(0);
}
}
// Inject intrinsics into the call argument
Expr arg = mutate(call_args[i]);
if (i < 2) {
// Convert spatial coordinates x,y into texture coordinates by normalization.
args[i + 2] = (Cast::make(Float(32), arg - min) + 0.5f) / extent;
} else {
args[i + 2] = arg - min;
}
}
// This intrinsic represents the GLSL texture2D function, and that
// function returns a vec4 result.
Type load_type = call->type;
load_type.width = 4;
Expr load_call = Call::make(load_type, Call::glsl_texture_load,
vector<Expr>(&args[0],&args[4]),
Call::Intrinsic,
Function(), 0, call->image, call->param);
// Add a shuffle_vector intrinsic to swizzle a single channel scalar out
// of the vec4 loaded by glsl_texture_load. This may be widened to the
// size of the Halide function color dimension during vectorization.
expr = Call::make(call->type, Call::shuffle_vector,
vec(load_call, args[4]), Call::Intrinsic);
}
void visit(const Call *call) {
if (!inside_kernel_loop || call->call_type == Call::Intrinsic ||
call->call_type == Call::Extern) {
IRMutator::visit(call);
return;
}
string name = call->name;
if (call->call_type == Call::Halide && call->func.outputs() > 1) {
name = name + '.' + std::to_string(call->value_index);
}
vector<Expr> padded_call_args = call->args;
// Check to see if we are reading from a one or two dimension function
// and pad to three dimensions.
while (padded_call_args.size() < 3) {
padded_call_args.push_back(0);
}
// Create image_load("name", name.buffer, x, x_extent, y, y_extent, ...).
// Extents can be used by successive passes. OpenGL, for example, uses them
// for coordinates normalization.
vector<Expr> args(2);
args[0] = call->name;
args[1] = Variable::make(Handle(), call->name + ".buffer");
for (size_t i = 0; i < padded_call_args.size(); i++) {
// If this is an ordinary dimension, insert a variable that will be
// subsequently defined by StorageFlattening to with the min and
// extent. Otherwise, add a default value for the padded dimension.
// If 'i' is greater or equal to the number of args in the original
// node, it must be a padded dimension we added above.
if (i < call->args.size()) {
string d = std::to_string(i);
string min_name = name + ".min." + d;
string min_name_constrained = min_name + ".constrained";
if (scope.contains(min_name_constrained)) {
min_name = min_name_constrained;
}
string extent_name = name + ".extent." + d;
string extent_name_constrained = extent_name + ".constrained";
if (scope.contains(extent_name_constrained)) {
extent_name = extent_name_constrained;
}
Expr min = Variable::make(Int(32), min_name);
args.push_back(mutate(padded_call_args[i]) - min);
args.push_back(Variable::make(Int(32), extent_name));
} else {
args.push_back(0);
args.push_back(1);
}
}
Type load_type = call->type;
// load_type = load_type.with_lanes(4);
Expr load_call = Call::make(load_type,
Call::image_load,
args,
Call::Intrinsic,
Function(),
0,
call->image,
call->param);
expr = load_call;
// expr = Call::make(call->type, Call::shuffle_vector,
// vec(load_call, args[4]), Call::Intrinsic);
}
void visit(const ProducerConsumer *op) {
if (op->name != func.name()) {
IRMutator::visit(op);
} else {
stmt = op;
// We're interested in the case where exactly one of the
// dimensions of the buffer has a min/extent that depends
// on the loop_var.
string dim = "";
int dim_idx = 0;
Expr min_required, max_required;
debug(3) << "Considering sliding " << func.name()
<< " along loop variable " << loop_var << "\n"
<< "Region provided:\n";
string prefix = func.name() + ".s" + std::to_string(func.updates().size()) + ".";
const std::vector<string> func_args = func.args();
for (int i = 0; i < func.dimensions(); i++) {
// Look up the region required of this function's last stage
string var = prefix + func_args[i];
internal_assert(scope.contains(var + ".min") && scope.contains(var + ".max"));
Expr min_req = scope.get(var + ".min");
Expr max_req = scope.get(var + ".max");
min_req = expand_expr(min_req, scope);
max_req = expand_expr(max_req, scope);
debug(3) << func_args[i] << ":" << min_req << ", " << max_req << "\n";
if (expr_depends_on_var(min_req, loop_var) ||
expr_depends_on_var(max_req, loop_var)) {
if (!dim.empty()) {
dim = "";
min_required = Expr();
max_required = Expr();
break;
} else {
dim = func_args[i];
dim_idx = i;
min_required = min_req;
max_required = max_req;
}
}
}
if (!min_required.defined()) {
debug(3) << "Could not perform sliding window optimization of "
<< func.name() << " over " << loop_var << " because either zero "
<< "or many dimensions of the function dependended on the loop var\n";
return;
}
// If the function is not pure in the given dimension, give up. We also
// need to make sure that it is pure in all the specializations
bool pure = true;
for (const Definition &def : func.updates()) {
pure = is_dim_always_pure(def, dim, dim_idx);
if (!pure) {
break;
}
}
if (!pure) {
debug(3) << "Could not performance sliding window optimization of "
<< func.name() << " over " << loop_var << " because the function "
<< "scatters along the related axis.\n";
return;
}
bool can_slide_up = false;
bool can_slide_down = false;
Monotonic monotonic_min = is_monotonic(min_required, loop_var);
Monotonic monotonic_max = is_monotonic(max_required, loop_var);
if (monotonic_min == Monotonic::Increasing ||
monotonic_min == Monotonic::Constant) {
can_slide_up = true;
}
if (monotonic_max == Monotonic::Decreasing ||
monotonic_max == Monotonic::Constant) {
can_slide_down = true;
}
if (!can_slide_up && !can_slide_down) {
debug(3) << "Not sliding " << func.name()
<< " over dimension " << dim
<< " along loop variable " << loop_var
<< " because I couldn't prove it moved monotonically along that dimension\n"
<< "Min is " << min_required << "\n"
<< "Max is " << max_required << "\n";
return;
}
// Ok, we've isolated a function, a dimension to slide
// along, and loop variable to slide over.
debug(3) << "Sliding " << func.name()
<< " over dimension " << dim
<< " along loop variable " << loop_var << "\n";
Expr loop_var_expr = Variable::make(Int(32), loop_var);
Expr prev_max_plus_one = substitute(loop_var, loop_var_expr - 1, max_required) + 1;
Expr prev_min_minus_one = substitute(loop_var, loop_var_expr - 1, min_required) - 1;
// If there's no overlap between adjacent iterations, we shouldn't slide.
if (can_prove(min_required >= prev_max_plus_one) ||
can_prove(max_required <= prev_min_minus_one)) {
debug(3) << "Not sliding " << func.name()
<< " over dimension " << dim
<< " along loop variable " << loop_var
<< " there's no overlap in the region computed across iterations\n"
<< "Min is " << min_required << "\n"
<< "Max is " << max_required << "\n";
return;
}
Expr new_min, new_max;
if (can_slide_up) {
new_min = select(loop_var_expr <= loop_min, min_required, likely(prev_max_plus_one));
new_max = max_required;
} else {
new_min = min_required;
new_max = select(loop_var_expr <= loop_min, max_required, likely(prev_min_minus_one));
}
Expr early_stages_min_required = new_min;
Expr early_stages_max_required = new_max;
debug(3) << "Sliding " << func.name() << ", " << dim << "\n"
<< "Pushing min up from " << min_required << " to " << new_min << "\n"
<< "Shrinking max from " << max_required << " to " << new_max << "\n";
// Now redefine the appropriate regions required
if (can_slide_up) {
replacements[prefix + dim + ".min"] = new_min;
} else {
replacements[prefix + dim + ".max"] = new_max;
}
for (size_t i = 0; i < func.updates().size(); i++) {
string n = func.name() + ".s" + std::to_string(i) + "." + dim;
replacements[n + ".min"] = Variable::make(Int(32), prefix + dim + ".min");
replacements[n + ".max"] = Variable::make(Int(32), prefix + dim + ".max");
}
// Ok, we have a new min/max required and we're going to
// rewrite all the lets that define bounds required. Now
// we need to additionally expand the bounds required of
// the last stage to cover values produced by stages
// before the last one. Because, e.g., an intermediate
// stage may be unrolled, expanding its bounds provided.
if (op->update.defined()) {
Box b = box_provided(op->produce, func.name());
merge_boxes(b, box_provided(op->update, func.name()));
if (can_slide_up) {
string n = prefix + dim + ".min";
Expr var = Variable::make(Int(32), n);
stmt = LetStmt::make(n, min(var, b[dim_idx].min), stmt);
} else {
string n = prefix + dim + ".max";
Expr var = Variable::make(Int(32), n);
stmt = LetStmt::make(n, max(var, b[dim_idx].max), stmt);
}
}
}
}
void visit(const Pipeline *op) {
if (op->name != func.name()) {
IRMutator::visit(op);
} else {
// We're interested in the case where exactly one of the
// mins of the buffer depends on the loop_var, and none of
// the extents do.
string dim = "";
Expr min, extent;
for (size_t i = 0; i < func.args().size(); i++) {
string min_name = func.name() + "." + func.args()[i] + ".min";
string extent_name = func.name() + "." + func.args()[i] + ".extent";
assert(scope.contains(min_name) && scope.contains(extent_name));
Expr this_min = scope.get(min_name);
Expr this_extent = scope.get(extent_name);
if (expr_depends_on_var(this_extent, loop_var)) {
min = Expr();
extent = Expr();
break;
}
if (expr_depends_on_var(this_min, loop_var)) {
if (min.defined()) {
min = Expr();
extent = Expr();
break;
} else {
dim = func.args()[i];
min = this_min;
extent = this_extent;
}
}
}
if (min.defined()) {
// Ok, we've isolated a function, a dimension to slide along, and loop variable to slide over
debug(2) << "Sliding " << func.name() << " over dimension " << dim << " along loop variable " << loop_var << "\n";
Expr loop_var_expr = Variable::make(Int(32), loop_var);
Expr steady_state = loop_var_expr > loop_min;
// The new min is one beyond the max we reached on the last loop iteration
Expr new_min = substitute(loop_var, loop_var_expr - 1, min + extent);
// The new extent is the old extent shrunk by how much we trimmed off the min
Expr new_extent = extent + min - new_min;
new_min = Select::make(steady_state, new_min, min);
new_extent = Select::make(steady_state, new_extent, extent);
stmt = LetStmt::make(func.name() + "." + dim + ".extent", new_extent, op);
stmt = LetStmt::make(func.name() + "." + dim + ".min", new_min, stmt);
} else {
debug(2) << "Could not perform sliding window optimization of " << func.name() << " over " << loop_var << "\n";
stmt = op;
}
}
}
void visit(const Variable *op) {
bool this_varies = varying.contains(op->name);
varies |= this_varies;
}