void dataobject::check_idcol(dataobject* data) { if(data->ncol() == 0) {return;} dvec udata; int i; int ndata = data->nrow(); int idcol = data->idcol(); udata.resize(ndata); for(i=0; i < ndata; i++) udata[i] = data->get_value(i,idcol); dvec uthis = this->return_uid(); sort_unique(uthis); sort_unique(udata); dvec inter; std::set_intersection(uthis.begin(), uthis.end(), udata.begin(), udata.end(), std::back_inserter(inter)); if(inter!=uthis) Rcpp::stop("ID found in the data set, but not in idata."); }
void remove_unused_fields(Scope& scope, const std::vector<std::string>& remove_members) { std::vector<DexField*> moveable_fields; std::vector<DexClass*> smallscope; uint32_t aflags = ACC_STATIC | ACC_FINAL; for (auto clazz : scope) { bool found = can_delete(clazz); if (!found) { auto name = clazz->get_name()->c_str(); for (const auto& name_prefix : remove_members) { if (strstr(name, name_prefix.c_str()) != nullptr) { found = true; break; } } if (!found) { TRACE(FINALINLINE, 2, "Cannot delete: %s\n", SHOW(clazz)); continue; } } auto sfields = clazz->get_sfields(); for (auto sfield : sfields) { if ((sfield->get_access() & aflags) != aflags) continue; auto value = sfield->get_static_value(); if (value == nullptr && !is_primitive(sfield->get_type())) continue; if (!found && !can_delete(sfield)) continue; moveable_fields.push_back(sfield); smallscope.push_back(clazz); } } sort_unique(smallscope); std::unordered_set<DexField*> field_target = get_field_target(scope, moveable_fields); std::unordered_set<DexField*> dead_fields; for (auto field : moveable_fields) { if (field_target.count(field) == 0) { dead_fields.insert(field); } } TRACE(FINALINLINE, 1, "Removable fields %lu/%lu\n", dead_fields.size(), moveable_fields.size()); TRACE(FINALINLINE, 1, "Unhandled inline %ld\n", unhandled_inline); for (auto clazz : smallscope) { auto& sfields = clazz->get_sfields(); auto iter = sfields.begin(); while (iter != sfields.end()) { auto todel = iter++; if (dead_fields.count(*todel) > 0) { sfields.erase(todel); } } } }
void remove_unused_fields(Scope& scope, const std::unordered_set<DexType*>& keep_annos, const std::unordered_set<DexField*>& keep_members) { std::vector<DexField*> moveable_fields; std::vector<DexClass*> smallscope; uint32_t aflags = ACC_STATIC | ACC_FINAL; for (auto clazz : scope) { if (!can_delete(clazz)) { continue; } auto sfields = clazz->get_sfields(); for (auto sfield : sfields) { if ((sfield->get_access() & aflags) != aflags) continue; auto value = sfield->get_static_value(); if (value == nullptr && !is_primitive(sfield->get_type())) continue; if (is_kept_by_annotation(sfield, keep_annos)) continue; if (is_kept_member(sfield, keep_members)) continue; moveable_fields.push_back(sfield); smallscope.push_back(clazz); } } sort_unique(smallscope); std::unordered_set<DexField*> field_target = get_field_target(scope, moveable_fields); std::unordered_set<DexField*> dead_fields; for (auto field : moveable_fields) { if (field_target.count(field) == 0) { dead_fields.insert(field); } } TRACE(FINALINLINE, 1, "Removable fields %lu/%lu\n", dead_fields.size(), moveable_fields.size()); TRACE(FINALINLINE, 1, "Unhandled inline %ld\n", unhandled_inline); for (auto clazz : smallscope) { auto& sfields = clazz->get_sfields(); auto iter = sfields.begin(); while (iter != sfields.end()) { auto todel = iter++; if (dead_fields.count(*todel) > 0) { sfields.erase(todel); } } } }
std::unordered_set<DexField*> get_called_field_defs(Scope& scope) { std::vector<DexField*> field_refs; walk_methods(scope, [&](DexMethod* method) { method->gather_fields(field_refs); }); sort_unique(field_refs); /* Okay, now we have a complete list of field refs * for this particular dex. Map to the def actually invoked. */ std::unordered_set<DexField*> field_defs; for (auto field_ref : field_refs) { auto field_def = resolve_field(field_ref); if (field_def == nullptr || !field_def->is_concrete()) continue; field_defs.insert(field_def); } return field_defs; }
void CrossDexRefMinimizer::insert(DexClass* cls) { always_assert(m_class_infos.count(cls) == 0); ++m_stats.classes; CrossDexRefMinimizer::ClassInfo& class_info = m_class_infos .insert({cls, CrossDexRefMinimizer::ClassInfo(m_next_index++)}) .first->second; // Collect all relevant references that contribute to cross-dex metadata // entries. // We don't bother with protos and type_lists, as they are directly related // to method refs (I tried, didn't help). std::vector<DexMethodRef*> method_refs; std::vector<DexFieldRef*> field_refs; std::vector<DexType*> types; std::vector<DexString*> strings; cls->gather_methods(method_refs); sort_unique(method_refs); cls->gather_fields(field_refs); sort_unique(field_refs); cls->gather_types(types); sort_unique(types); cls->gather_strings(strings); sort_unique(strings); auto& refs = class_info.refs; uint64_t refs_weight = 0; refs.reserve(method_refs.size() + field_refs.size() + types.size() + strings.size()); // Record all references with a particular weight. // The weights are somewhat arbitrary, but they were chosen after trying many // different values and observing the effect on APK size. // TODO: Try some other variations. for (auto mref : method_refs) { uint32_t weight = m_config.method_ref_weight; refs.emplace_back(mref, weight); refs_weight += weight; } for (auto type : types) { uint32_t weight = m_config.type_ref_weight; refs.emplace_back(type, weight); refs_weight += weight; } for (auto string : strings) { uint32_t weight = m_config.string_ref_weight; refs.emplace_back(string, weight); refs_weight += weight; } for (auto fref : field_refs) { uint32_t weight = m_config.field_ref_weight; refs.emplace_back(fref, weight); refs_weight += weight; } class_info.refs_weight = refs_weight; std::unordered_map<DexClass*, CrossDexRefMinimizer::ClassInfoDelta> affected_classes; for (const std::pair<void*, uint32_t>& p : refs) { void* ref = p.first; uint32_t weight = p.second; auto& classes = m_ref_classes[ref]; size_t frequency = classes.size(); // We record the need to undo (subtract weight of) a previously claimed // infrequent ref. The actual undoing happens later in // reprioritize. if (frequency > 0 && frequency <= INFREQUENT_REFS_COUNT) { for (DexClass* affected_class : classes) { always_assert(affected_class != cls); affected_classes[affected_class] .infrequent_refs_weight[frequency - 1] -= weight; } } ++frequency; // We are recording a new infrequent unapplied ref, if any. // This happens immediately for the to be inserted class cls, // so that it can be used right away by the upcoming // class_info.get_priority() call, while all other change requests happen // later in reprioritize. if (frequency <= INFREQUENT_REFS_COUNT) { for (DexClass* affected_class : classes) { affected_classes[affected_class] .infrequent_refs_weight[frequency - 1] += weight; } class_info.infrequent_refs_weight[frequency - 1] += weight; } // There's an implicit invariant that class_info and the keys of // affected_classes are disjoint, so we are not going to reprioritize // the class that we are adding here. classes.emplace(cls); } const auto priority = class_info.get_priority(); m_prioritized_classes.insert(cls, priority); TRACE(IDEX, 4, "[dex ordering] Inserting class {%s} with priority %016lx; index %u; " "%s infrequent refs weights, %u total refs\n", SHOW(cls), priority, class_info.index, format_infrequent_refs_array(class_info.infrequent_refs_weight).c_str(), refs.size()); reprioritize(affected_classes); }
bool ternary_table_updates_check_01(TERNARY_TABLE *table, TERNARY_TABLE_UPDATES *updates) { sort_unique(updates->inserts); return table_updates_check_key<cols_01>(updates->inserts, updates->deletes, table->unshifted); }
// [[Rcpp::export]] SEXP write_worksheet_xml_2( std::string prior , std::string post , Reference sheet_data , CharacterVector row_heights , std::string R_fileName){ // open file and write header XML const char * s = R_fileName.c_str(); std::ofstream xmlFile; xmlFile.open (s); xmlFile << "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>"; xmlFile << prior; IntegerVector cell_row = sheet_data.field("rows"); // If no data write childless node and return if(cell_row.size() == 0){ xmlFile << "<sheetData/>"; xmlFile << post; xmlFile.close(); return Rcpp::wrap(0); } // sheet_data will be in order, jsut need to check for row_heights CharacterVector cell_col = int_2_cell_ref(sheet_data.field("cols")); CharacterVector cell_types = map_cell_types_to_char(sheet_data.field("t")); CharacterVector cell_value = sheet_data.field("v"); CharacterVector cell_fn = sheet_data.field("f"); CharacterVector style_id = sheet_data.field("style_id"); CharacterVector unique_rows(sort_unique(cell_row)); CharacterVector row_heights_rows = row_heights.attr("names"); size_t n_row_heights = row_heights.size(); size_t n = cell_row.size(); size_t k = unique_rows.size(); std::string xml; std::string cell_xml; size_t j = 0; size_t h = 0; String current_row = unique_rows[0]; bool row_has_data = true; xmlFile << "<sheetData>"; for(size_t i = 0; i < k; i++){ cell_xml = ""; row_has_data = true; while(current_row == unique_rows[i]){ row_has_data = true; j += 1; if(CharacterVector::is_na(cell_col[j-1])){ //If r IS NA we have no row data we only have a rowHeight row_has_data = false; if(j == n) break; current_row = cell_row[j]; break; } //cell XML strings cell_xml += "<c r=\"" + cell_col[j-1] + itos(cell_row[j-1]); if(!CharacterVector::is_na(style_id[j-1])) cell_xml += "\" s=\"" + style_id[j-1]; //If we have a t value we must have a v value if(!CharacterVector::is_na(cell_types[j-1])){ //If we have a c value we might have an f value if(CharacterVector::is_na(cell_fn[j-1])){ // no function cell_xml += "\" t=\"" + cell_types[j-1] + "\"><v>" + cell_value[j-1] + "</v></c>"; }else{ if(CharacterVector::is_na(cell_value[j-1])){ // If v is NA cell_xml += "\" t=\"" + cell_types[j-1] + "\">" + cell_fn[j-1] + "</c>"; }else{ cell_xml += "\" t=\"" + cell_types[j-1] + "\">" + cell_fn[j-1] + "<v>" + cell_value[j-1] + "</v></c>"; } } }else if(!CharacterVector::is_na(cell_fn[j-1])){ cell_xml += "\">" + cell_fn[j-1] + "</c>"; }else{ cell_xml += "\"/>"; } if(j == n) break; current_row = cell_row[j]; } if(h < n_row_heights){ if((unique_rows[i] == row_heights_rows[h]) & row_has_data){ // this row has a row height and cell_xml data xmlFile << "<row r=\"" + unique_rows[i] + "\" ht=\"" + row_heights[h] + "\" customHeight=\"1\">" + cell_xml + "</row>"; h++; }else if(row_has_data){ xmlFile << "<row r=\"" + unique_rows[i] + "\">" + cell_xml + "</row>"; }else{ xmlFile << "<row r=\"" + unique_rows[i] + "\" ht=\"" + row_heights[h] + "\" customHeight=\"1\"/>"; h++; } }else{ xmlFile << "<row r=\"" + unique_rows[i] + "\">" + cell_xml + "</row>"; } } // write closing tag and XML post data xmlFile << "</sheetData>"; xmlFile << post; //close file xmlFile.close(); return wrap(0); }
void comm_mesh_rebalance( BulkData & M , const CoordinateField & node_coord_field , const WeightField * const elem_weight_field , std::vector<OctTreeKey> & cut_keys ) { const MetaData & mesh_meta_data = M.mesh_meta_data(); Part * const uses_part = & mesh_meta_data.locally_used_part(); Part * const owns_part = & mesh_meta_data.locally_owned_part(); const unsigned p_size = M.parallel_size(); const unsigned p_rank = M.parallel_rank(); //-------------------------------------------------------------------- // The node_coord_field must be up to date on all processors // so that the element oct tree keys are parallel consistent. // It is assumed that the shared node_coord_field values are // already consistent. { const FieldBase * const ptr = & node_coord_field ; std::vector< const FieldBase *> tmp ; tmp.push_back( ptr ); const std::vector<EntityProc> & aura_domain = M.ghost_source(); const std::vector<EntityProc> & aura_range = M.ghost_destination(); communicate_field_data( M , aura_domain , aura_range , tmp , false ); } //-------------------------------------------------------------------- // Generate global oct-tree keys for local element centroids // and cuts for the global element centroids. double bounds[4] ; global_coordinate_bounds( M , node_coord_field , bounds ); cut_keys.assign( p_size , OctTreeKey() ); OctTreeKey * const cut_begin = & cut_keys[0] ; OctTreeKey * const cut_first = cut_begin + 1 ; OctTreeKey * const cut_end = cut_begin + p_size ; global_element_cuts( M , bounds , node_coord_field , elem_weight_field , cut_begin ); //-------------------------------------------------------------------- // Mapping of *all* elements to load balanced processor, // even the aura elements. // This requires that the node coordinates on the aura // elements be up to date. { std::vector< const FieldBase * > tmp ; const FieldBase * const tmp_coord = & node_coord_field ; tmp.push_back( tmp_coord ); const std::vector<EntityProc> & d = M.ghost_source(); const std::vector<EntityProc> & r = M.ghost_destination(); communicate_field_data( M , d , r , tmp , false ); } { const EntitySet & elem_set = M.entities( Element ); const EntitySet::iterator i_end = elem_set.end(); EntitySet::iterator i = elem_set.begin(); while ( i != i_end ) { Entity & elem = *i ; ++i ; const OctTreeKey k = elem_key( bounds , node_coord_field , elem ); const unsigned p = std::upper_bound(cut_first, cut_end, k) - cut_first ; M.change_entity_owner( elem , p ); } } //-------------------------------------------------------------------- // Fill 'rebal' with all uses entities' rebalancing processors std::vector<EntityProc> rebal ; rebal_elem_entities( M , Node , rebal ); rebal_elem_entities( M , Edge , rebal ); rebal_elem_entities( M , Face , rebal ); { const Part & part_uses = * uses_part ; const KernelSet & elem_kernels = M.kernels( Element ); const KernelSet::const_iterator i_end = elem_kernels.end(); KernelSet::const_iterator i = elem_kernels.begin(); while ( i != i_end ) { const Kernel & kernel = *i ; ++i ; if ( kernel.has_superset( part_uses ) ) { const Kernel::iterator j_end = kernel.end(); Kernel::iterator j = kernel.begin(); while ( j != j_end ) { Entity * const entity = *j ; ++j ; const unsigned p = entity->owner_rank(); EntityProc tmp( entity , p ); rebal.push_back( tmp ); } } } } // The 'other' entities rebalance based upon the entities // that they use. This may lead to more sharing entities. // Thus 'rebal' is input and then updated. rebal_other_entities( M , Particle , rebal ); rebal_other_entities( M , Constraint , rebal ); // 'rebal' now contains the rebalancing (entity,processor) pairs // for every non-aura entity. Can now delete the aura entities. remove_aura( M ); // Copy entities to new processors according to 'rebal'. // Only send the owned entities. // Include all processors associated with the entity in 'rebal'. // Unpack all nodes, then all edges, then all faces, then all elements, // from each processor. // The owner of a shared entity is the max-rank processor. // Add received entities to shared if more than one processor. { const RebalanceComm manager ; std::vector<EntityProc> recv_rebal ; communicate_entities( manager , M , M , rebal , recv_rebal , false ); // Destroy not-retained entities, they have been packed. // Remove the corresponding entries in 'rebal' destroy_not_retained( M , rebal ); rebal.insert( rebal.end() , recv_rebal.begin() , recv_rebal.end() ); sort_unique( rebal ); } // The 'rebal' should contain a reference to every non-aura entity // on the local processor. These references include every // processor on which the entity now resides, including the // local processor. { // Set parallel ownership and sharing parts. std::vector<EntityProc>::iterator ish ; for ( ish = rebal.begin() ; ish != rebal.end() ; ) { Entity & e = * ish->first ; for ( ; ish != rebal.end() && ish->first == & e ; ++ish ); const bool is_owned = p_rank == e.owner_rank() ; // Change ownership. std::vector<Part*> add_parts ; std::vector<Part*> remove_parts ; if ( is_owned ) { add_parts.push_back( owns_part ); } else { remove_parts.push_back( owns_part ); } M.change_entity_parts( e , add_parts , remove_parts ); } // Remove references to the local processor, // the remaining entries define the sharing. for ( ish = rebal.end() ; ish != rebal.begin() ; ) { --ish ; if ( p_rank == ish->second ) { ish = rebal.erase( ish ); } } M.set_shares( rebal ); } // Establish new aura comm_mesh_regenerate_aura( M ); }
static void strip_src_strings( DexStoresVector& stores, const char* map_path, PassManager& mgr) { size_t shortened = 0; size_t string_savings = 0; std::unordered_map<DexString*, std::vector<DexString*>> global_src_strings; std::unordered_set<DexString*> shortened_used; for (auto& classes : DexStoreClassesIterator(stores)) { for (auto const& clazz : classes) { auto src_string = clazz->get_source_file(); if (src_string) { // inserting actual source files into this set will cause them to not // get used --- as the whole point of this analysis is to substitute // source file strings shortened_used.insert(src_string); } } } for (auto& classes : DexStoreClassesIterator(stores)) { std::unordered_map<DexString*, DexString*> src_to_shortened; std::vector<DexString*> current_dex_strings; for (auto const& clazz : classes) { clazz->gather_strings(current_dex_strings); } sort_unique(current_dex_strings, compare_dexstrings); // reverse current_dex_strings vector, so that we prefer strings that will // get smaller indices std::reverse(std::begin(current_dex_strings), std::end(current_dex_strings)); for (auto const& clazz : classes) { auto src_string = clazz->get_source_file(); if (!src_string) { continue; } DexString* shortened_src_string = nullptr; if (src_to_shortened.count(src_string) == 0) { shortened_src_string = get_suitable_string(shortened_used, current_dex_strings); if (!shortened_src_string) { opt_warn(UNSHORTENED_SRC_STRING, "%s\n", SHOW(src_string)); shortened_src_string = src_string; } else { shortened++; string_savings += strlen(src_string->c_str()); } src_to_shortened[src_string] = shortened_src_string; shortened_used.emplace(shortened_src_string); global_src_strings[src_string].push_back(shortened_src_string); } else { shortened_src_string = src_to_shortened[src_string]; } clazz->set_source_file(shortened_src_string); } } TRACE(SHORTEN, 1, "src strings shortened %ld, %lu bytes saved\n", shortened, string_savings); mgr.incr_metric(METRIC_SHORTENED_STRINGS, shortened); mgr.incr_metric(METRIC_BYTES_SAVED, string_savings); // generate mapping FILE* fd = fopen(map_path, "w"); if (fd == nullptr) { perror("Error writing mapping file"); return; } for (auto it : global_src_strings) { auto desc_vector = it.second; sort_unique(desc_vector); fprintf(fd, "%s ->", it.first->c_str()); for (auto str : desc_vector) { fprintf(fd, " %s,", str->c_str()); } fprintf(fd, "\n"); } fclose(fd); }