void SingleImplPass::run_pass(DexStoresVector& stores, ConfigFiles& cfg, PassManager& mgr) {
auto scope = build_class_scope(stores);
ClassHierarchy ch = build_type_hierarchy(scope);
int max_steps = 0;
size_t previous_invoke_intf_count = s_invoke_intf_count;
removed_count = 0;
while (true) {
DEBUG_ONLY size_t scope_size = scope.size();
TypeToTypes intfs_to_classes;
TypeSet intfs;
build_type_maps(scope, intfs_to_classes, intfs);
TypeMap single_impl;
collect_single_impl(intfs_to_classes, single_impl);
std::unique_ptr<SingleImplAnalysis> single_impls =
SingleImplAnalysis::analyze(
scope, stores, single_impl, intfs, m_pass_config);
auto optimized = optimize(
std::move(single_impls), ch, scope, m_pass_config);
if (optimized == 0 || ++max_steps >= MAX_PASSES) break;
removed_count += optimized;
assert(scope_size > scope.size());
}
TRACE(INTF, 1, "Removed interfaces %ld\n", removed_count);
TRACE(INTF, 1,
"Updated invoke-interface to invoke-virtual %ld\n",
s_invoke_intf_count - previous_invoke_intf_count);
mgr.incr_metric(METRIC_REMOVED_INTERFACES, removed_count);
mgr.incr_metric(METRIC_INVOKE_INT_TO_VIRT,
s_invoke_intf_count - previous_invoke_intf_count);
post_dexen_changes(scope, stores);
}
void AccessMarkingPass::run_pass(DexStoresVector& stores,
ConfigFiles& /* conf */,
PassManager& pm) {
auto scope = build_class_scope(stores);
ClassHierarchy ch = build_type_hierarchy(scope);
SignatureMap sm = build_signature_map(ch);
if (m_finalize_classes) {
auto n_classes_final = mark_classes_final(scope, ch);
pm.incr_metric("finalized_classes", n_classes_final);
TRACE(ACCESS, 1, "Finalized %lu classes\n", n_classes_final);
}
if (m_finalize_methods) {
auto n_methods_final = mark_methods_final(scope, ch);
pm.incr_metric("finalized_methods", n_methods_final);
TRACE(ACCESS, 1, "Finalized %lu methods\n", n_methods_final);
}
if (m_finalize_fields) {
auto n_fields_final = mark_fields_final(scope);
pm.incr_metric("finalized_fields", n_fields_final);
TRACE(ACCESS, 1, "Finalized %lu fields\n", n_fields_final);
}
auto candidates = devirtualize(sm);
auto dmethods = direct_methods(scope);
candidates.insert(candidates.end(), dmethods.begin(), dmethods.end());
if (m_privatize_methods) {
auto privates = find_private_methods(scope, candidates);
fix_call_sites_private(scope, privates);
mark_methods_private(privates);
pm.incr_metric("privatized_methods", privates.size());
TRACE(ACCESS, 1, "Privatized %lu methods\n", privates.size());
}
}
bool should_rename(DexClass *clazz,
std::vector<std::string>& pre_patterns,
std::vector<std::string>& post_patterns,
std::unordered_set<const DexType*>& untouchables,
bool rename_annotations,
PassManager& mgr) {
if (!rename_annotations && is_annotation(clazz)) {
mgr.incr_metric(METRIC_CANT_RENAME_ANNOTATION, 1);
return false;
}
if (untouchables.count(clazz->get_type())) {
mgr.incr_metric(METRIC_CANT_RENAME_UNTOUCHABLE, 1);
return false;
}
auto chstring = clazz->get_type()->get_name()->c_str();
/* We're assuming anonymous classes are safe always safe to rename. */
auto last_cash = strrchr(chstring, '$');
if (last_cash != nullptr) {
auto val = *++last_cash;
if (val >= '0' && val <= '9') {
match_inner++;
return true;
}
}
/* Check for more aggressive, but finer grained filters first */
for (auto p : pre_patterns) {
auto substr = strstr(chstring, p.c_str());
if (substr != nullptr) {
if (p.length() > 1) {
match_long++;
} else {
match_short++;
}
return true;
}
}
if (!can_rename(clazz) && !can_delete(clazz)) {
mgr.incr_metric(METRIC_CANT_RENAME_AND_CANT_DELETE, 1);
return false;
}
/* Check for wider, less precise filters */
for (auto p : post_patterns) {
auto substr = strstr(chstring, p.c_str());
if (substr != nullptr) {
if (p.length() > 1) {
match_long++;
} else {
match_short++;
}
return true;
}
}
mgr.incr_metric(METRIC_NOT_WHITELISTED, 1);
return false;
}
void PassImpl::run_pass(DexStoresVector& stores,
ConfigFiles& config,
PassManager& mgr) {
if (m_config.create_runtime_asserts) {
m_config.runtime_assert =
RuntimeAssertTransform::Config(config.get_proguard_map());
}
auto scope = build_class_scope(stores);
run(scope);
mgr.incr_metric("branches_removed", m_transform_stats.branches_removed);
mgr.incr_metric("materialized_consts", m_transform_stats.materialized_consts);
mgr.incr_metric("constant_fields", m_stats.constant_fields);
mgr.incr_metric("constant_methods", m_stats.constant_methods);
}
void RemoveInterfacePass::run_pass(DexStoresVector& stores,
ConfigFiles&,
PassManager& mgr) {
auto scope = build_class_scope(stores);
TypeSystem type_system(scope);
for (const auto root : m_interface_roots) {
remove_interfaces_for_root(scope, stores, root, type_system);
}
mgr.incr_metric("num_total_interface", m_total_num_interface);
mgr.incr_metric("num_interface_excluded", m_num_interface_excluded);
mgr.incr_metric("num_interface_removed", m_num_interface_removed);
TRACE(RM_INTF, 5, "total number of interfaces %ld\n", m_total_num_interface);
TRACE(RM_INTF, 5, "number of excluded interfaces %ld\n",
m_num_interface_excluded);
TRACE(RM_INTF, 5, "number of removed interfaces %ld\n",
m_num_interface_removed);
for (const auto& stat : m_dispatch_stats) {
std::stringstream metric;
metric << "num_dispatch_" << stat.first;
mgr.incr_metric(metric.str(), stat.second);
}
}
void RemoveUnreachablePass::run_pass(DexStoresVector& stores,
ConfigFiles& conf,
PassManager& pm) {
// Store names of removed classes and methods
ConcurrentSet<std::string> removed_symbols;
if (pm.no_proguard_rules()) {
TRACE(RMU,
1,
"RemoveUnreachablePass not run because no "
"ProGuard configuration was provided.");
return;
}
bool output_unreachable_symbols = pm.get_current_pass_info()->repeat == 0 &&
!m_unreachable_symbols_file_name.empty();
int num_ignore_check_strings = 0;
auto reachables = reachability::compute_reachable_objects(
stores, m_ignore_sets, &num_ignore_check_strings);
reachability::ObjectCounts before = reachability::count_objects(stores);
TRACE(RMU, 1, "before: %lu classes, %lu fields, %lu methods\n",
before.num_classes, before.num_fields, before.num_methods);
reachability::sweep(stores, *reachables,
output_unreachable_symbols ? &removed_symbols : nullptr);
reachability::ObjectCounts after = reachability::count_objects(stores);
TRACE(RMU, 1, "after: %lu classes, %lu fields, %lu methods\n",
after.num_classes, after.num_fields, after.num_methods);
pm.incr_metric("num_ignore_check_strings", num_ignore_check_strings);
pm.incr_metric("classes_removed", before.num_classes - after.num_classes);
pm.incr_metric("fields_removed", before.num_fields - after.num_fields);
pm.incr_metric("methods_removed", before.num_methods - after.num_methods);
if (output_unreachable_symbols) {
std::string filepath = conf.metafile(m_unreachable_symbols_file_name);
write_out_removed_symbols(filepath, removed_symbols);
}
}
void RenameClassesPass::run_pass(DexStoresVector& stores,
ConfigFiles& cfg,
PassManager& mgr) {
const JsonWrapper& json_cfg = cfg.get_json_config();
if (json_cfg.get("emit_name_based_locators", false)) {
// TODO: Purge the old RenameClassesPass entirely everywhere.
fprintf(stderr,
"[RenameClassesPass] error: Configuration option "
"emit_locator_strings is not compatible with RenameClassesPass. "
"Upgrade to RenameClassesPassV2 instead.\n");
exit(EXIT_FAILURE);
}
auto scope = build_class_scope(stores);
ClassHierarchy ch = build_type_hierarchy(scope);
std::unordered_set<const DexType*> untouchables;
for (const auto& base : m_untouchable_hierarchies) {
auto base_type = DexType::get_type(base.c_str());
if (base_type != nullptr) {
untouchables.insert(base_type);
TypeSet children;
get_all_children(ch, base_type, children);
untouchables.insert(children.begin(), children.end());
}
}
mgr.incr_metric(METRIC_CLASSES_IN_SCOPE, scope.size());
rename_classes(
scope, m_pre_filter_whitelist, m_post_filter_whitelist,
untouchables, m_rename_annotations, mgr);
TRACE(RENAME, 1,
"renamed classes: %d anon classes, %d from single char patterns, "
"%d from multi char patterns\n",
match_inner,
match_short,
match_long);
TRACE(RENAME, 1, "String savings, at least %d bytes \n",
base_strings_size - ren_strings_size);
}
void RegAllocPass::run_pass(DexStoresVector& stores,
ConfigFiles&,
PassManager& mgr) {
using Output = graph_coloring::Allocator::Stats;
auto scope = build_class_scope(stores);
auto stats = walk::parallel::reduce_methods<Output>(
scope,
[this](DexMethod* m) { // mapper
graph_coloring::Allocator::Stats stats;
if (m->get_code() == nullptr) {
return stats;
}
auto& code = *m->get_code();
TRACE(REG, 3, "Handling %s:\n", SHOW(m));
TRACE(REG,
5,
"regs:%d code:\n%s\n",
code.get_registers_size(),
SHOW(&code));
try {
live_range::renumber_registers(&code, /* width_aware */ true);
// The transformations below all require a CFG. Build it once
// here instead of requiring each transform to build it.
code.build_cfg(/* editable */ false);
graph_coloring::Allocator allocator(m_allocator_config);
allocator.allocate(m);
stats.accumulate(allocator.get_stats());
TRACE(REG,
5,
"After alloc: regs:%d code:\n%s\n",
code.get_registers_size(),
SHOW(&code));
} catch (std::exception&) {
fprintf(stderr, "Failed to allocate %s\n", SHOW(m));
fprintf(stderr, "%s\n", SHOW(code.cfg()));
throw;
}
return stats;
},
[](Output a, Output b) { // reducer
a.accumulate(b);
return a;
});
TRACE(REG, 1, "Total reiteration count: %lu\n", stats.reiteration_count);
TRACE(REG, 1, "Total Params spilled early: %lu\n", stats.params_spill_early);
TRACE(REG, 1, "Total spill count: %lu\n", stats.moves_inserted());
TRACE(REG, 1, " Total param spills: %lu\n", stats.param_spill_moves);
TRACE(REG, 1, " Total range spills: %lu\n", stats.range_spill_moves);
TRACE(REG, 1, " Total global spills: %lu\n", stats.global_spill_moves);
TRACE(REG, 1, " Total splits: %lu\n", stats.split_moves);
TRACE(REG, 1, "Total coalesce count: %lu\n", stats.moves_coalesced);
TRACE(REG, 1, "Total net moves: %ld\n", stats.net_moves());
mgr.incr_metric("param spilled too early", stats.params_spill_early);
mgr.incr_metric("reiteration_count", stats.reiteration_count);
mgr.incr_metric("spill_count", stats.moves_inserted());
mgr.incr_metric("coalesce_count", stats.moves_coalesced);
mgr.incr_metric("net_moves", stats.net_moves());
mgr.record_running_regalloc();
}
void rename_classes(
Scope& scope,
std::vector<std::string>& pre_whitelist_patterns,
std::vector<std::string>& post_whitelist_patterns,
std::unordered_set<const DexType*>& untouchables,
bool rename_annotations,
PassManager& mgr) {
unpackage_private(scope);
int clazz_ident = 0;
std::map<DexString*, DexString*, dexstrings_comparator> aliases;
for(auto clazz: scope) {
if (!should_rename(
clazz, pre_whitelist_patterns, post_whitelist_patterns,
untouchables, rename_annotations, mgr)) {
continue;
}
char clzname[4];
get_next_ident(clzname, clazz_ident);
auto dtype = clazz->get_type();
auto oldname = dtype->get_name();
// The X helps our hacked Dalvik classloader recognize that a
// class name is the output of the redex renamer and thus will
// never be found in the Android platform.
// The $ indicates that the class was originally an inner class.
// Some code, most notably android instrumentation runner, uses
// this information to decide whether or not to classload the class.
bool inner = strrchr(oldname->c_str(), '$');
char descriptor[10];
sprintf(descriptor, "LX%s%s;", inner ? "$" : "", clzname);
auto dstring = DexString::make_string(descriptor);
aliases[oldname] = dstring;
dtype->set_name(dstring);
std::string old_str(oldname->c_str());
std::string new_str(descriptor);
base_strings_size += strlen(oldname->c_str());
base_strings_size += strlen(dstring->c_str());
TRACE(RENAME, 4, "'%s'->'%s'\n", oldname->c_str(), descriptor);
while (1) {
std::string arrayop("[");
arrayop += oldname->c_str();
oldname = DexString::get_string(arrayop.c_str());
if (oldname == nullptr) {
break;
}
auto arraytype = DexType::get_type(oldname);
if (arraytype == nullptr) {
break;
}
std::string newarraytype("[");
newarraytype += dstring->c_str();
dstring = DexString::make_string(newarraytype.c_str());
aliases[oldname] = dstring;
arraytype->set_name(dstring);
}
}
mgr.incr_metric(METRIC_RENAMED_CLASSES, match_short + match_long + match_inner);
/* Now we need to re-write the Signature annotations. They use
* Strings rather than Type's, so they have to be explicitly
* handled.
*/
/* Generics of the form Type<> turn into the Type string
* sans the ';'. So, we have to alias those if they
* exist. Signature annotations suck.
*/
for (auto apair : aliases) {
char buf[MAX_DESCRIPTOR_LENGTH];
const char *sourcestr = apair.first->c_str();
size_t sourcelen = strlen(sourcestr);
if (sourcestr[sourcelen - 1] != ';') continue;
strcpy(buf, sourcestr);
buf[sourcelen - 1] = '\0';
auto dstring = DexString::get_string(buf);
if (dstring == nullptr) continue;
strcpy(buf, apair.second->c_str());
buf[strlen(apair.second->c_str()) - 1] = '\0';
auto target = DexString::make_string(buf);
aliases[dstring] = target;
}
walk::annotations(scope, [&](DexAnnotation* anno) {
static DexType *dalviksig =
DexType::get_type("Ldalvik/annotation/Signature;");
if (anno->type() != dalviksig) return;
auto elems = anno->anno_elems();
for (auto elem : elems) {
auto ev = elem.encoded_value;
if (ev->evtype() != DEVT_ARRAY) continue;
auto arrayev = static_cast<DexEncodedValueArray*>(ev);
auto const& evs = arrayev->evalues();
for (auto strev : *evs) {
if (strev->evtype() != DEVT_STRING) continue;
auto stringev = static_cast<DexEncodedValueString*>(strev);
if (aliases.count(stringev->string())) {
TRACE(RENAME, 5, "Rewriting Signature from '%s' to '%s'\n",
stringev->string()->c_str(),
aliases[stringev->string()]->c_str());
stringev->string(aliases[stringev->string()]);
}
}
}
});
}
void RemoveBuildersPass::run_pass(DexStoresVector& stores,
ConfigFiles&,
PassManager& mgr) {
if (mgr.no_proguard_rules()) {
TRACE(BUILDERS,
1,
"RemoveBuildersPass did not run because no Proguard configuration "
"was provided.");
return;
}
// Initialize couters.
b_counter = {0, 0, 0, 0};
auto obj_type = get_object_type();
auto scope = build_class_scope(stores);
for (DexClass* cls : scope) {
if (is_annotation(cls) || is_interface(cls) ||
cls->get_super_class() != obj_type) {
continue;
}
if (has_builder_name(cls->get_type())) {
m_builders.emplace(cls->get_type());
}
}
std::unordered_set<DexType*> escaped_builders;
walk::methods(scope, [&](DexMethod* m) {
auto builders = created_builders(m);
for (DexType* builder : builders) {
if (escapes_stack(builder, m)) {
TRACE(BUILDERS,
3,
"%s escapes in %s\n",
SHOW(builder),
m->get_deobfuscated_name().c_str());
escaped_builders.emplace(builder);
}
}
});
std::unordered_set<DexType*> stack_only_builders;
for (DexType* builder : m_builders) {
if (escaped_builders.find(builder) == escaped_builders.end()) {
stack_only_builders.emplace(builder);
}
}
std::unordered_set<DexType*> builders_and_supers;
for (DexType* builder : stack_only_builders) {
DexType* cls = builder;
while (cls != nullptr && cls != obj_type) {
builders_and_supers.emplace(cls);
cls = type_class(cls)->get_super_class();
}
}
std::unordered_set<DexType*> this_escapes;
for (DexType* cls_ty : builders_and_supers) {
DexClass* cls = type_class(cls_ty);
if (cls->is_external() ||
this_arg_escapes(cls, m_enable_buildee_constr_change)) {
this_escapes.emplace(cls_ty);
}
}
// set of builders that neither escape the stack nor pass their 'this' arg
// to another function
std::unordered_set<DexType*> no_escapes;
for (DexType* builder : stack_only_builders) {
DexType* cls = builder;
bool hierarchy_has_escape = false;
while (cls != nullptr) {
if (this_escapes.find(cls) != this_escapes.end()) {
hierarchy_has_escape = true;
break;
}
cls = type_class(cls)->get_super_class();
}
if (!hierarchy_has_escape) {
no_escapes.emplace(builder);
}
}
size_t dmethod_count = 0;
size_t vmethod_count = 0;
size_t build_count = 0;
for (DexType* builder : no_escapes) {
auto cls = type_class(builder);
auto buildee = get_buildee(builder);
dmethod_count += cls->get_dmethods().size();
vmethod_count += cls->get_vmethods().size();
for (DexMethod* m : cls->get_vmethods()) {
if (m->get_proto()->get_rtype() == buildee) {
build_count++;
}
}
}
std::unordered_set<DexClass*> trivial_builders =
get_trivial_builders(m_builders, no_escapes);
std::unordered_set<DexClass*> kept_builders =
get_builders_with_subclasses(scope);
PassConfig pc(mgr.get_config());
BuilderTransform b_transform(pc, scope, stores, false);
// Inline non init methods.
std::unordered_set<DexClass*> removed_builders;
walk::methods(scope, [&](DexMethod* method) {
auto builders = created_builders(method);
for (DexType* builder : builders) {
if (method->get_class() == builder) {
continue;
}
DexClass* builder_cls = type_class(builder);
// Filter out builders that we cannot remove.
if (kept_builders.find(builder_cls) != kept_builders.end()) {
continue;
}
// Check it is a trivial one.
if (trivial_builders.find(builder_cls) != trivial_builders.end()) {
DexMethod* method_copy = DexMethod::make_method_from(
method,
method->get_class(),
DexString::make_string(method->get_name()->str() +
"$redex_builders"));
bool was_not_removed =
!b_transform.inline_methods(
method, builder, &get_non_init_methods) ||
!remove_builder_from(method, builder_cls, b_transform);
if (was_not_removed) {
kept_builders.emplace(builder_cls);
method->set_code(method_copy->release_code());
} else {
b_counter.methods_cleared++;
removed_builders.emplace(builder_cls);
}
DexMethod::erase_method(method_copy);
}
}
});
// No need to remove the builders here, since `RemoveUnreachable` will
// take care of it.
gather_removal_builder_stats(removed_builders, kept_builders);
mgr.set_metric("total_builders", m_builders.size());
mgr.set_metric("stack_only_builders", stack_only_builders.size());
mgr.set_metric("no_escapes", no_escapes.size());
mgr.incr_metric(METRIC_CLASSES_REMOVED, b_counter.classes_removed);
mgr.incr_metric(METRIC_METHODS_REMOVED, b_counter.methods_removed);
mgr.incr_metric(METRIC_FIELDS_REMOVED, b_counter.fields_removed);
mgr.incr_metric(METRIC_METHODS_CLEARED, b_counter.methods_cleared);
TRACE(BUILDERS, 1, "Total builders: %d\n", m_builders.size());
TRACE(BUILDERS, 1, "Stack-only builders: %d\n", stack_only_builders.size());
TRACE(BUILDERS,
1,
"Stack-only builders that don't let `this` escape: %d\n",
no_escapes.size());
TRACE(BUILDERS, 1, "Stats for unescaping builders:\n");
TRACE(BUILDERS, 1, "\tdmethods: %d\n", dmethod_count);
TRACE(BUILDERS, 1, "\tvmethods: %d\n", vmethod_count);
TRACE(BUILDERS, 1, "\tbuild methods: %d\n", build_count);
TRACE(BUILDERS, 1, "Trivial builders: %d\n", trivial_builders.size());
TRACE(BUILDERS, 1, "Classes removed: %d\n", b_counter.classes_removed);
TRACE(BUILDERS, 1, "Methods removed: %d\n", b_counter.methods_removed);
TRACE(BUILDERS, 1, "Fields removed: %d\n", b_counter.fields_removed);
TRACE(BUILDERS, 1, "Methods cleared: %d\n", b_counter.methods_cleared);
}
static void strip_src_strings(
DexStoresVector& stores, const char* map_path, PassManager& mgr) {
size_t shortened = 0;
size_t string_savings = 0;
std::unordered_map<DexString*, std::vector<DexString*>> global_src_strings;
std::unordered_set<DexString*> shortened_used;
for (auto& classes : DexStoreClassesIterator(stores)) {
for (auto const& clazz : classes) {
auto src_string = clazz->get_source_file();
if (src_string) {
// inserting actual source files into this set will cause them to not
// get used --- as the whole point of this analysis is to substitute
// source file strings
shortened_used.insert(src_string);
}
}
}
for (auto& classes : DexStoreClassesIterator(stores)) {
std::unordered_map<DexString*, DexString*> src_to_shortened;
std::vector<DexString*> current_dex_strings;
for (auto const& clazz : classes) {
clazz->gather_strings(current_dex_strings);
}
sort_unique(current_dex_strings, compare_dexstrings);
// reverse current_dex_strings vector, so that we prefer strings that will
// get smaller indices
std::reverse(std::begin(current_dex_strings),
std::end(current_dex_strings));
for (auto const& clazz : classes) {
auto src_string = clazz->get_source_file();
if (!src_string) {
continue;
}
DexString* shortened_src_string = nullptr;
if (src_to_shortened.count(src_string) == 0) {
shortened_src_string =
get_suitable_string(shortened_used, current_dex_strings);
if (!shortened_src_string) {
opt_warn(UNSHORTENED_SRC_STRING, "%s\n", SHOW(src_string));
shortened_src_string = src_string;
} else {
shortened++;
string_savings += strlen(src_string->c_str());
}
src_to_shortened[src_string] = shortened_src_string;
shortened_used.emplace(shortened_src_string);
global_src_strings[src_string].push_back(shortened_src_string);
} else {
shortened_src_string = src_to_shortened[src_string];
}
clazz->set_source_file(shortened_src_string);
}
}
TRACE(SHORTEN, 1, "src strings shortened %ld, %lu bytes saved\n", shortened,
string_savings);
mgr.incr_metric(METRIC_SHORTENED_STRINGS, shortened);
mgr.incr_metric(METRIC_BYTES_SAVED, string_savings);
// generate mapping
FILE* fd = fopen(map_path, "w");
if (fd == nullptr) {
perror("Error writing mapping file");
return;
}
for (auto it : global_src_strings) {
auto desc_vector = it.second;
sort_unique(desc_vector);
fprintf(fd, "%s ->", it.first->c_str());
for (auto str : desc_vector) {
fprintf(fd, " %s,", str->c_str());
}
fprintf(fd, "\n");
}
fclose(fd);
}
