static void mark_bb_visited (basic_block bb, int trace) { bb->rbi->visited = trace; if (bbd[bb->index].heap) { fibheap_delete_node (bbd[bb->index].heap, bbd[bb->index].node); bbd[bb->index].heap = NULL; bbd[bb->index].node = NULL; } }
/* Propagate the constant parameters found by ipcp_iterate_stage() to the function's code. */ static void ipcp_insert_stage (void) { struct cgraph_node *node, *node1 = NULL; int i; VEC (cgraph_edge_p, heap) * redirect_callers; VEC (ipa_replace_map_p,gc)* replace_trees; int node_callers, count; tree parm_tree; struct ipa_replace_map *replace_param; fibheap_t heap; long overall_size = 0, new_size = 0; long max_new_size; ipa_check_create_node_params (); ipa_check_create_edge_args (); if (dump_file) fprintf (dump_file, "\nIPA insert stage:\n\n"); dead_nodes = BITMAP_ALLOC (NULL); for (node = cgraph_nodes; node; node = node->next) if (node->analyzed) { if (node->count > max_count) max_count = node->count; overall_size += node->local.inline_summary.self_size; } max_new_size = overall_size; if (max_new_size < PARAM_VALUE (PARAM_LARGE_UNIT_INSNS)) max_new_size = PARAM_VALUE (PARAM_LARGE_UNIT_INSNS); max_new_size = max_new_size * PARAM_VALUE (PARAM_IPCP_UNIT_GROWTH) / 100 + 1; /* First collect all functions we proved to have constant arguments to heap. */ heap = fibheap_new (); for (node = cgraph_nodes; node; node = node->next) { struct ipa_node_params *info; /* Propagation of the constant is forbidden in certain conditions. */ if (!node->analyzed || !ipcp_node_modifiable_p (node)) continue; info = IPA_NODE_REF (node); if (ipa_is_called_with_var_arguments (info)) continue; if (ipcp_const_param_count (node)) node->aux = fibheap_insert (heap, ipcp_estimate_cloning_cost (node), node); } /* Now clone in priority order until code size growth limits are met or heap is emptied. */ while (!fibheap_empty (heap)) { struct ipa_node_params *info; int growth = 0; bitmap args_to_skip; struct cgraph_edge *cs; node = (struct cgraph_node *)fibheap_extract_min (heap); node->aux = NULL; if (dump_file) fprintf (dump_file, "considering function %s\n", cgraph_node_name (node)); growth = ipcp_estimate_growth (node); if (new_size + growth > max_new_size) break; if (growth && optimize_function_for_size_p (DECL_STRUCT_FUNCTION (node->decl))) { if (dump_file) fprintf (dump_file, "Not versioning, cold code would grow"); continue; } info = IPA_NODE_REF (node); count = ipa_get_param_count (info); replace_trees = VEC_alloc (ipa_replace_map_p, gc, 1); if (node->local.can_change_signature) args_to_skip = BITMAP_GGC_ALLOC (); else args_to_skip = NULL; for (i = 0; i < count; i++) { struct ipcp_lattice *lat = ipcp_get_lattice (info, i); parm_tree = ipa_get_param (info, i); /* We can proactively remove obviously unused arguments. */ if (!ipa_is_param_used (info, i)) { if (args_to_skip) bitmap_set_bit (args_to_skip, i); continue; } if (lat->type == IPA_CONST_VALUE) { replace_param = ipcp_create_replace_map (parm_tree, lat); if (replace_param == NULL) break; VEC_safe_push (ipa_replace_map_p, gc, replace_trees, replace_param); if (args_to_skip) bitmap_set_bit (args_to_skip, i); } } if (i < count) { if (dump_file) fprintf (dump_file, "Not versioning, some parameters couldn't be replaced"); continue; } new_size += growth; /* Look if original function becomes dead after cloning. */ for (cs = node->callers; cs != NULL; cs = cs->next_caller) if (cs->caller == node || ipcp_need_redirect_p (cs)) break; if (!cs && cgraph_will_be_removed_from_program_if_no_direct_calls (node)) bitmap_set_bit (dead_nodes, node->uid); /* Compute how many callers node has. */ node_callers = 0; for (cs = node->callers; cs != NULL; cs = cs->next_caller) node_callers++; redirect_callers = VEC_alloc (cgraph_edge_p, heap, node_callers); for (cs = node->callers; cs != NULL; cs = cs->next_caller) if (!cs->indirect_inlining_edge) VEC_quick_push (cgraph_edge_p, redirect_callers, cs); /* Redirecting all the callers of the node to the new versioned node. */ node1 = cgraph_create_virtual_clone (node, redirect_callers, replace_trees, args_to_skip, "constprop"); args_to_skip = NULL; VEC_free (cgraph_edge_p, heap, redirect_callers); replace_trees = NULL; if (node1 == NULL) continue; ipcp_process_devirtualization_opportunities (node1); if (dump_file) fprintf (dump_file, "versioned function %s with growth %i, overall %i\n", cgraph_node_name (node), (int)growth, (int)new_size); ipcp_init_cloned_node (node, node1); info = IPA_NODE_REF (node); for (i = 0; i < count; i++) { struct ipcp_lattice *lat = ipcp_get_lattice (info, i); if (lat->type == IPA_CONST_VALUE) ipcp_discover_new_direct_edges (node1, i, lat->constant); } if (dump_file) dump_function_to_file (node1->decl, dump_file, dump_flags); for (cs = node->callees; cs; cs = cs->next_callee) if (cs->callee->aux) { fibheap_delete_node (heap, (fibnode_t) cs->callee->aux); cs->callee->aux = fibheap_insert (heap, ipcp_estimate_cloning_cost (cs->callee), cs->callee); } } while (!fibheap_empty (heap)) { if (dump_file) fprintf (dump_file, "skipping function %s\n", cgraph_node_name (node)); node = (struct cgraph_node *) fibheap_extract_min (heap); node->aux = NULL; } fibheap_delete (heap); BITMAP_FREE (dead_nodes); ipcp_update_callgraph (); ipcp_update_profiling (); }
static bool tail_duplicate (void) { fibnode_t *blocks = XCNEWVEC (fibnode_t, last_basic_block); basic_block *trace = XNEWVEC (basic_block, n_basic_blocks); int *counts = XNEWVEC (int, last_basic_block); int ninsns = 0, nduplicated = 0; gcov_type weighted_insns = 0, traced_insns = 0; fibheap_t heap = fibheap_new (); gcov_type cover_insns; int max_dup_insns; basic_block bb; bool changed = false; /* Create an oversized sbitmap to reduce the chance that we need to resize it. */ bb_seen = sbitmap_alloc (last_basic_block * 2); bitmap_clear (bb_seen); initialize_original_copy_tables (); if (profile_info && flag_branch_probabilities) probability_cutoff = PARAM_VALUE (TRACER_MIN_BRANCH_PROBABILITY_FEEDBACK); else probability_cutoff = PARAM_VALUE (TRACER_MIN_BRANCH_PROBABILITY); probability_cutoff = REG_BR_PROB_BASE / 100 * probability_cutoff; branch_ratio_cutoff = (REG_BR_PROB_BASE / 100 * PARAM_VALUE (TRACER_MIN_BRANCH_RATIO)); FOR_EACH_BB (bb) { int n = count_insns (bb); if (!ignore_bb_p (bb)) blocks[bb->index] = fibheap_insert (heap, -bb->frequency, bb); counts [bb->index] = n; ninsns += n; weighted_insns += n * bb->frequency; } if (profile_info && flag_branch_probabilities) cover_insns = PARAM_VALUE (TRACER_DYNAMIC_COVERAGE_FEEDBACK); else cover_insns = PARAM_VALUE (TRACER_DYNAMIC_COVERAGE); cover_insns = (weighted_insns * cover_insns + 50) / 100; max_dup_insns = (ninsns * PARAM_VALUE (TRACER_MAX_CODE_GROWTH) + 50) / 100; while (traced_insns < cover_insns && nduplicated < max_dup_insns && !fibheap_empty (heap)) { basic_block bb = (basic_block) fibheap_extract_min (heap); int n, pos; if (!bb) break; blocks[bb->index] = NULL; if (ignore_bb_p (bb)) continue; gcc_assert (!bb_seen_p (bb)); n = find_trace (bb, trace); bb = trace[0]; traced_insns += bb->frequency * counts [bb->index]; if (blocks[bb->index]) { fibheap_delete_node (heap, blocks[bb->index]); blocks[bb->index] = NULL; } for (pos = 1; pos < n; pos++) { basic_block bb2 = trace[pos]; if (blocks[bb2->index]) { fibheap_delete_node (heap, blocks[bb2->index]); blocks[bb2->index] = NULL; } traced_insns += bb2->frequency * counts [bb2->index]; if (EDGE_COUNT (bb2->preds) > 1 && can_duplicate_block_p (bb2) /* We have the tendency to duplicate the loop header of all do { } while loops. Do not do that - it is not profitable and it might create a loop with multiple entries or at least rotate the loop. */ && (!current_loops || bb2->loop_father->header != bb2)) { edge e; basic_block copy; nduplicated += counts [bb2->index]; e = find_edge (bb, bb2); copy = duplicate_block (bb2, e, bb); flush_pending_stmts (e); add_phi_args_after_copy (©, 1, NULL); /* Reconsider the original copy of block we've duplicated. Removing the most common predecessor may make it to be head. */ blocks[bb2->index] = fibheap_insert (heap, -bb2->frequency, bb2); if (dump_file) fprintf (dump_file, "Duplicated %i as %i [%i]\n", bb2->index, copy->index, copy->frequency); bb2 = copy; changed = true; } mark_bb_seen (bb2); bb = bb2; /* In case the trace became infrequent, stop duplicating. */ if (ignore_bb_p (bb)) break; } if (dump_file) fprintf (dump_file, " covered now %.1f\n\n", traced_insns * 100.0 / weighted_insns); } if (dump_file) fprintf (dump_file, "Duplicated %i insns (%i%%)\n", nduplicated, nduplicated * 100 / ninsns); free_original_copy_tables (); sbitmap_free (bb_seen); free (blocks); free (trace); free (counts); fibheap_delete (heap); return changed; }
static void tail_duplicate (void) { fibnode_t *blocks = XCNEWVEC (fibnode_t, last_basic_block); basic_block *trace = XNEWVEC (basic_block, n_basic_blocks); int *counts = XNEWVEC (int, last_basic_block); int ninsns = 0, nduplicated = 0; gcov_type weighted_insns = 0, traced_insns = 0; fibheap_t heap = fibheap_new (); gcov_type cover_insns; int max_dup_insns; basic_block bb; if (profile_info && flag_branch_probabilities) probability_cutoff = PARAM_VALUE (TRACER_MIN_BRANCH_PROBABILITY_FEEDBACK); else probability_cutoff = PARAM_VALUE (TRACER_MIN_BRANCH_PROBABILITY); probability_cutoff = REG_BR_PROB_BASE / 100 * probability_cutoff; branch_ratio_cutoff = (REG_BR_PROB_BASE / 100 * PARAM_VALUE (TRACER_MIN_BRANCH_RATIO)); FOR_EACH_BB (bb) { int n = count_insns (bb); if (!ignore_bb_p (bb)) blocks[bb->index] = fibheap_insert (heap, -bb->frequency, bb); counts [bb->index] = n; ninsns += n; weighted_insns += n * bb->frequency; } if (profile_info && flag_branch_probabilities) cover_insns = PARAM_VALUE (TRACER_DYNAMIC_COVERAGE_FEEDBACK); else cover_insns = PARAM_VALUE (TRACER_DYNAMIC_COVERAGE); cover_insns = (weighted_insns * cover_insns + 50) / 100; max_dup_insns = (ninsns * PARAM_VALUE (TRACER_MAX_CODE_GROWTH) + 50) / 100; while (traced_insns < cover_insns && nduplicated < max_dup_insns && !fibheap_empty (heap)) { basic_block bb = fibheap_extract_min (heap); int n, pos; if (!bb) break; blocks[bb->index] = NULL; if (ignore_bb_p (bb)) continue; gcc_assert (!seen (bb)); n = find_trace (bb, trace); bb = trace[0]; traced_insns += bb->frequency * counts [bb->index]; if (blocks[bb->index]) { fibheap_delete_node (heap, blocks[bb->index]); blocks[bb->index] = NULL; } for (pos = 1; pos < n; pos++) { basic_block bb2 = trace[pos]; if (blocks[bb2->index]) { fibheap_delete_node (heap, blocks[bb2->index]); blocks[bb2->index] = NULL; } traced_insns += bb2->frequency * counts [bb2->index]; if (EDGE_COUNT (bb2->preds) > 1 && can_duplicate_block_p (bb2)) { edge e; basic_block old = bb2; e = find_edge (bb, bb2); nduplicated += counts [bb2->index]; bb2 = duplicate_block (bb2, e, bb); /* Reconsider the original copy of block we've duplicated. Removing the most common predecessor may make it to be head. */ blocks[old->index] = fibheap_insert (heap, -old->frequency, old); if (dump_file) fprintf (dump_file, "Duplicated %i as %i [%i]\n", old->index, bb2->index, bb2->frequency); } bb->aux = bb2; bb2->il.rtl->visited = 1; bb = bb2; /* In case the trace became infrequent, stop duplicating. */ if (ignore_bb_p (bb)) break; } if (dump_file) fprintf (dump_file, " covered now %.1f\n\n", traced_insns * 100.0 / weighted_insns); } if (dump_file) fprintf (dump_file, "Duplicated %i insns (%i%%)\n", nduplicated, nduplicated * 100 / ninsns); free (blocks); free (trace); free (counts); fibheap_delete (heap); }