static void
get_inline_stack (location_t locus, inline_stack *stack)
{
if (LOCATION_LOCUS (locus) == UNKNOWN_LOCATION)
return;
tree block = LOCATION_BLOCK (locus);
if (block && TREE_CODE (block) == BLOCK)
{
int level = 0;
for (block = BLOCK_SUPERCONTEXT (block);
block && (TREE_CODE (block) == BLOCK);
block = BLOCK_SUPERCONTEXT (block))
{
location_t tmp_locus = BLOCK_SOURCE_LOCATION (block);
if (LOCATION_LOCUS (tmp_locus) == UNKNOWN_LOCATION)
continue;
tree decl = get_function_decl_from_block (block);
stack->safe_push (
std::make_pair (decl, get_combined_location (locus, decl)));
locus = tmp_locus;
level++;
}
}
stack->safe_push (
std::make_pair (current_function_decl,
get_combined_location (locus, current_function_decl)));
}
static unsigned
get_relative_location_for_stmt (gimple *stmt)
{
location_t locus = gimple_location (stmt);
if (LOCATION_LOCUS (locus) == UNKNOWN_LOCATION)
return UNKNOWN_LOCATION;
for (tree block = gimple_block (stmt); block && (TREE_CODE (block) == BLOCK);
block = BLOCK_SUPERCONTEXT (block))
if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (block)) != UNKNOWN_LOCATION)
return get_combined_location (locus,
get_function_decl_from_block (block));
return get_combined_location (locus, current_function_decl);
}
static void
write_ts_decl_minimal_tree_pointers (struct output_block *ob, tree expr,
bool ref_p)
{
stream_write_tree (ob, DECL_NAME (expr), ref_p);
stream_write_tree (ob, DECL_CONTEXT (expr), ref_p);
lto_output_location (ob, LOCATION_LOCUS (DECL_SOURCE_LOCATION (expr)));
}
static void
write_ts_exp_tree_pointers (struct output_block *ob, tree expr, bool ref_p)
{
int i;
streamer_write_hwi (ob, TREE_OPERAND_LENGTH (expr));
for (i = 0; i < TREE_OPERAND_LENGTH (expr); i++)
stream_write_tree (ob, TREE_OPERAND (expr, i), ref_p);
lto_output_location (ob, LOCATION_LOCUS (EXPR_LOCATION (expr)));
stream_write_tree (ob, TREE_BLOCK (expr), ref_p);
}
static expanded_location
expand_location_1 (source_location loc,
bool expansion_point_p)
{
expanded_location xloc;
const struct line_map *map;
enum location_resolution_kind lrk = LRK_MACRO_EXPANSION_POINT;
tree block = NULL;
if (IS_ADHOC_LOC (loc))
{
block = LOCATION_BLOCK (loc);
loc = LOCATION_LOCUS (loc);
}
/* If LOC describes a location with a discriminator, extract the
discriminator and map it to the real location. */
if (min_discriminator_location != UNKNOWN_LOCATION
&& loc >= min_discriminator_location
&& loc < next_discriminator_location)
loc = map_discriminator_location (loc);
memset (&xloc, 0, sizeof (xloc));
if (loc >= RESERVED_LOCATION_COUNT)
{
if (!expansion_point_p)
{
/* We want to resolve LOC to its spelling location.
But if that spelling location is a reserved location that
appears in the context of a macro expansion (like for a
location for a built-in token), let's consider the first
location (toward the expansion point) that is not reserved;
that is, the first location that is in real source code. */
loc = linemap_unwind_to_first_non_reserved_loc (line_table,
loc, &map);
lrk = LRK_SPELLING_LOCATION;
}
loc = linemap_resolve_location (line_table, loc,
lrk, &map);
xloc = linemap_expand_location (line_table, map, loc);
}
xloc.data = block;
if (loc <= BUILTINS_LOCATION)
xloc.file = loc == UNKNOWN_LOCATION ? NULL : _("<built-in>");
return xloc;
}
bool
autofdo_source_profile::get_count_info (gimple *stmt, count_info *info) const
{
if (LOCATION_LOCUS (gimple_location (stmt)) == cfun->function_end_locus)
return false;
inline_stack stack;
get_inline_stack (gimple_location (stmt), &stack);
if (stack.length () == 0)
return false;
function_instance *s = get_function_instance_by_inline_stack (stack);
if (s == NULL)
return false;
return s->get_count_info (stack[0].second, info);
}
static tree
get_function_decl_from_block (tree block)
{
tree decl;
if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (block) == UNKNOWN_LOCATION))
return NULL_TREE;
for (decl = BLOCK_ABSTRACT_ORIGIN (block);
decl && (TREE_CODE (decl) == BLOCK);
decl = BLOCK_ABSTRACT_ORIGIN (decl))
if (TREE_CODE (decl) == FUNCTION_DECL)
break;
return decl;
}
bool
autofdo_source_profile::update_inlined_ind_target (gcall *stmt,
count_info *info)
{
if (LOCATION_LOCUS (gimple_location (stmt)) == cfun->function_end_locus)
return false;
count_info old_info;
get_count_info (stmt, &old_info);
gcov_type total = 0;
for (icall_target_map::const_iterator iter = old_info.targets.begin ();
iter != old_info.targets.end (); ++iter)
total += iter->second;
/* Program behavior changed, original promoted (and inlined) target is not
hot any more. Will avoid promote the original target.
To check if original promoted target is still hot, we check the total
count of the unpromoted targets (stored in old_info). If it is no less
than half of the callsite count (stored in INFO), the original promoted
target is considered not hot any more. */
if (total >= info->count / 2)
return false;
inline_stack stack;
get_inline_stack (gimple_location (stmt), &stack);
if (stack.length () == 0)
return false;
function_instance *s = get_function_instance_by_inline_stack (stack);
if (s == NULL)
return false;
icall_target_map map;
if (s->find_icall_target_map (stmt, &map) == 0)
return false;
for (icall_target_map::const_iterator iter = map.begin ();
iter != map.end (); ++iter)
info->targets[iter->first] = iter->second;
return true;
}
bool
autofdo_source_profile::update_inlined_ind_target (gcall *stmt,
count_info *info)
{
if (dump_file)
{
fprintf (dump_file, "Checking indirect call -> direct call ");
print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
}
if (LOCATION_LOCUS (gimple_location (stmt)) == cfun->function_end_locus)
{
if (dump_file)
fprintf (dump_file, " good locus\n");
return false;
}
count_info old_info;
get_count_info (stmt, &old_info);
gcov_type total = 0;
for (icall_target_map::const_iterator iter = old_info.targets.begin ();
iter != old_info.targets.end (); ++iter)
total += iter->second;
/* Program behavior changed, original promoted (and inlined) target is not
hot any more. Will avoid promote the original target.
To check if original promoted target is still hot, we check the total
count of the unpromoted targets (stored in TOTAL). If a callsite count
(stored in INFO) is smaller than half of the total count, the original
promoted target is considered not hot any more. */
if (info->count < total / 2)
{
if (dump_file)
fprintf (dump_file, " not hot anymore %ld < %ld",
(long)info->count,
(long)total /2);
return false;
}
inline_stack stack;
get_inline_stack (gimple_location (stmt), &stack);
if (stack.length () == 0)
{
if (dump_file)
fprintf (dump_file, " no inline stack\n");
return false;
}
function_instance *s = get_function_instance_by_inline_stack (stack);
if (s == NULL)
{
if (dump_file)
fprintf (dump_file, " function not found in inline stack\n");
return false;
}
icall_target_map map;
if (s->find_icall_target_map (stmt, &map) == 0)
{
if (dump_file)
fprintf (dump_file, " no target map\n");
return false;
}
for (icall_target_map::const_iterator iter = map.begin ();
iter != map.end (); ++iter)
info->targets[iter->first] = iter->second;
if (dump_file)
fprintf (dump_file, " looks good\n");
return true;
}
static void
output_gimple_stmt (struct output_block *ob, gimple stmt)
{
unsigned i;
enum gimple_code code;
enum LTO_tags tag;
struct bitpack_d bp;
histogram_value hist;
/* Emit identifying tag. */
code = gimple_code (stmt);
tag = lto_gimple_code_to_tag (code);
streamer_write_record_start (ob, tag);
/* Emit the tuple header. */
bp = bitpack_create (ob->main_stream);
bp_pack_var_len_unsigned (&bp, gimple_num_ops (stmt));
bp_pack_value (&bp, gimple_no_warning_p (stmt), 1);
if (is_gimple_assign (stmt))
bp_pack_value (&bp, gimple_assign_nontemporal_move_p (stmt), 1);
bp_pack_value (&bp, gimple_has_volatile_ops (stmt), 1);
hist = gimple_histogram_value (cfun, stmt);
bp_pack_value (&bp, hist != NULL, 1);
bp_pack_var_len_unsigned (&bp, stmt->gsbase.subcode);
/* Emit location information for the statement. */
stream_output_location (ob, &bp, LOCATION_LOCUS (gimple_location (stmt)));
streamer_write_bitpack (&bp);
/* Emit the lexical block holding STMT. */
stream_write_tree (ob, gimple_block (stmt), true);
/* Emit the operands. */
switch (gimple_code (stmt))
{
case GIMPLE_RESX:
streamer_write_hwi (ob, gimple_resx_region (stmt));
break;
case GIMPLE_EH_MUST_NOT_THROW:
stream_write_tree (ob, gimple_eh_must_not_throw_fndecl (stmt), true);
break;
case GIMPLE_EH_DISPATCH:
streamer_write_hwi (ob, gimple_eh_dispatch_region (stmt));
break;
case GIMPLE_ASM:
streamer_write_uhwi (ob, gimple_asm_ninputs (stmt));
streamer_write_uhwi (ob, gimple_asm_noutputs (stmt));
streamer_write_uhwi (ob, gimple_asm_nclobbers (stmt));
streamer_write_uhwi (ob, gimple_asm_nlabels (stmt));
streamer_write_string (ob, ob->main_stream, gimple_asm_string (stmt),
true);
/* Fallthru */
case GIMPLE_ASSIGN:
case GIMPLE_CALL:
case GIMPLE_RETURN:
case GIMPLE_SWITCH:
case GIMPLE_LABEL:
case GIMPLE_COND:
case GIMPLE_GOTO:
case GIMPLE_DEBUG:
for (i = 0; i < gimple_num_ops (stmt); i++)
{
tree op = gimple_op (stmt, i);
tree *basep = NULL;
/* Wrap all uses of non-automatic variables inside MEM_REFs
so that we do not have to deal with type mismatches on
merged symbols during IL read in. The first operand
of GIMPLE_DEBUG must be a decl, not MEM_REF, though. */
if (op && (i || !is_gimple_debug (stmt)))
{
basep = &op;
while (handled_component_p (*basep))
basep = &TREE_OPERAND (*basep, 0);
if (TREE_CODE (*basep) == VAR_DECL
&& !auto_var_in_fn_p (*basep, current_function_decl)
&& !DECL_REGISTER (*basep))
{
bool volatilep = TREE_THIS_VOLATILE (*basep);
*basep = build2 (MEM_REF, TREE_TYPE (*basep),
build_fold_addr_expr (*basep),
build_int_cst (build_pointer_type
(TREE_TYPE (*basep)), 0));
TREE_THIS_VOLATILE (*basep) = volatilep;
}
else
basep = NULL;
}
stream_write_tree (ob, op, true);
/* Restore the original base if we wrapped it inside a MEM_REF. */
if (basep)
*basep = TREE_OPERAND (TREE_OPERAND (*basep, 0), 0);
}
if (is_gimple_call (stmt))
{
if (gimple_call_internal_p (stmt))
streamer_write_enum (ob->main_stream, internal_fn,
IFN_LAST, gimple_call_internal_fn (stmt));
else
stream_write_tree (ob, gimple_call_fntype (stmt), true);
}
break;
case GIMPLE_NOP:
case GIMPLE_PREDICT:
break;
case GIMPLE_TRANSACTION:
gcc_assert (gimple_transaction_body (stmt) == NULL);
stream_write_tree (ob, gimple_transaction_label (stmt), true);
break;
default:
gcc_unreachable ();
}
if (hist)
stream_out_histogram_value (ob, hist);
}
tree
ubsan_create_data (const char *name, const location_t *ploc,
const struct ubsan_mismatch_data *mismatch, ...)
{
va_list args;
tree ret, t;
tree fields[5];
vec<tree, va_gc> *saved_args = NULL;
size_t i = 0;
location_t loc = UNKNOWN_LOCATION;
/* Firstly, create a pointer to type descriptor type. */
tree td_type = ubsan_type_descriptor_type ();
TYPE_READONLY (td_type) = 1;
td_type = build_pointer_type (td_type);
/* Create the structure type. */
ret = make_node (RECORD_TYPE);
if (ploc != NULL)
{
loc = LOCATION_LOCUS (*ploc);
fields[i] = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
ubsan_source_location_type ());
DECL_CONTEXT (fields[i]) = ret;
i++;
}
va_start (args, mismatch);
for (t = va_arg (args, tree); t != NULL_TREE;
i++, t = va_arg (args, tree))
{
gcc_checking_assert (i < 3);
/* Save the tree arguments for later use. */
vec_safe_push (saved_args, t);
fields[i] = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
td_type);
DECL_CONTEXT (fields[i]) = ret;
if (i)
DECL_CHAIN (fields[i - 1]) = fields[i];
}
va_end (args);
if (mismatch != NULL)
{
/* We have to add two more decls. */
fields[i] = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
pointer_sized_int_node);
DECL_CONTEXT (fields[i]) = ret;
DECL_CHAIN (fields[i - 1]) = fields[i];
i++;
fields[i] = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
unsigned_char_type_node);
DECL_CONTEXT (fields[i]) = ret;
DECL_CHAIN (fields[i - 1]) = fields[i];
i++;
}
TYPE_FIELDS (ret) = fields[0];
TYPE_NAME (ret) = get_identifier (name);
layout_type (ret);
/* Now, fill in the type. */
char tmp_name[32];
static unsigned int ubsan_var_id_num;
ASM_GENERATE_INTERNAL_LABEL (tmp_name, "Lubsan_data", ubsan_var_id_num++);
tree var = build_decl (UNKNOWN_LOCATION, VAR_DECL, get_identifier (tmp_name),
ret);
TREE_STATIC (var) = 1;
TREE_PUBLIC (var) = 0;
DECL_ARTIFICIAL (var) = 1;
DECL_IGNORED_P (var) = 1;
DECL_EXTERNAL (var) = 0;
vec<constructor_elt, va_gc> *v;
vec_alloc (v, i);
tree ctor = build_constructor (ret, v);
/* If desirable, set the __ubsan_source_location element. */
if (ploc != NULL)
CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, ubsan_source_location (loc));
size_t nelts = vec_safe_length (saved_args);
for (i = 0; i < nelts; i++)
{
t = (*saved_args)[i];
CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, t);
}
if (mismatch != NULL)
{
/* Append the pointer data. */
CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, mismatch->align);
CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, mismatch->ckind);
}
TREE_CONSTANT (ctor) = 1;
TREE_STATIC (ctor) = 1;
DECL_INITIAL (var) = ctor;
varpool_finalize_decl (var);
return var;
}
unsigned
vectorize_loops (void)
{
unsigned int i;
unsigned int num_vectorized_loops = 0;
unsigned int vect_loops_num;
loop_iterator li;
struct loop *loop;
vect_loops_num = number_of_loops ();
/* Bail out if there are no loops. */
if (vect_loops_num <= 1)
return 0;
init_stmt_vec_info_vec ();
/* ----------- Analyze loops. ----------- */
/* If some loop was duplicated, it gets bigger number
than all previously defined loops. This fact allows us to run
only over initial loops skipping newly generated ones. */
FOR_EACH_LOOP (li, loop, 0)
if (optimize_loop_nest_for_speed_p (loop))
{
loop_vec_info loop_vinfo;
vect_location = find_loop_location (loop);
if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOC
&& dump_enabled_p ())
dump_printf (MSG_ALL, "\nAnalyzing loop at %s:%d\n",
LOC_FILE (vect_location), LOC_LINE (vect_location));
loop_vinfo = vect_analyze_loop (loop);
loop->aux = loop_vinfo;
if (!loop_vinfo || !LOOP_VINFO_VECTORIZABLE_P (loop_vinfo))
continue;
if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOC
&& dump_enabled_p ())
dump_printf (MSG_ALL, "\n\nVectorizing loop at %s:%d\n",
LOC_FILE (vect_location), LOC_LINE (vect_location));
vect_transform_loop (loop_vinfo);
num_vectorized_loops++;
}
vect_location = UNKNOWN_LOC;
statistics_counter_event (cfun, "Vectorized loops", num_vectorized_loops);
if (dump_enabled_p ()
|| (num_vectorized_loops > 0 && dump_enabled_p ()))
dump_printf_loc (MSG_ALL, vect_location,
"vectorized %u loops in function.\n",
num_vectorized_loops);
/* ----------- Finalize. ----------- */
for (i = 1; i < vect_loops_num; i++)
{
loop_vec_info loop_vinfo;
loop = get_loop (i);
if (!loop)
continue;
loop_vinfo = (loop_vec_info) loop->aux;
destroy_loop_vec_info (loop_vinfo, true);
loop->aux = NULL;
}
free_stmt_vec_info_vec ();
return num_vectorized_loops > 0 ? TODO_cleanup_cfg : 0;
}
unsigned
vectorize_loops (void)
{
unsigned int i;
unsigned int num_vectorized_loops = 0;
unsigned int vect_loops_num;
loop_iterator li;
struct loop *loop;
vect_loops_num = number_of_loops (cfun);
/* Bail out if there are no loops. */
if (vect_loops_num <= 1)
return 0;
init_stmt_vec_info_vec ();
/* ----------- Analyze loops. ----------- */
/* If some loop was duplicated, it gets bigger number
than all previously defined loops. This fact allows us to run
only over initial loops skipping newly generated ones. */
FOR_EACH_LOOP (li, loop, 0)
if (optimize_loop_nest_for_speed_p (loop))
{
loop_vec_info loop_vinfo;
vect_location = find_loop_location (loop);
if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOC
&& dump_enabled_p ())
dump_printf (MSG_NOTE, "\nAnalyzing loop at %s:%d\n",
LOC_FILE (vect_location), LOC_LINE (vect_location));
loop_vinfo = vect_analyze_loop (loop);
loop->aux = loop_vinfo;
if (!loop_vinfo || !LOOP_VINFO_VECTORIZABLE_P (loop_vinfo))
continue;
if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOC
&& dump_enabled_p ())
dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
"Vectorized loop\n");
vect_transform_loop (loop_vinfo);
num_vectorized_loops++;
}
vect_location = UNKNOWN_LOC;
statistics_counter_event (cfun, "Vectorized loops", num_vectorized_loops);
if (dump_enabled_p ()
|| (num_vectorized_loops > 0 && dump_enabled_p ()))
dump_printf_loc (MSG_NOTE, vect_location,
"vectorized %u loops in function.\n",
num_vectorized_loops);
/* ----------- Finalize. ----------- */
for (i = 1; i < vect_loops_num; i++)
{
loop_vec_info loop_vinfo;
loop = get_loop (cfun, i);
if (!loop)
continue;
loop_vinfo = (loop_vec_info) loop->aux;
destroy_loop_vec_info (loop_vinfo, true);
loop->aux = NULL;
}
free_stmt_vec_info_vec ();
if (num_vectorized_loops > 0)
{
/* If we vectorized any loop only virtual SSA form needs to be updated.
??? Also while we try hard to update loop-closed SSA form we fail
to properly do this in some corner-cases (see PR56286). */
rewrite_into_loop_closed_ssa (NULL, TODO_update_ssa_only_virtuals);
return TODO_cleanup_cfg;
}
return 0;
}
