void dataobject::check_idcol(dataobject* data) {
if(data->ncol() == 0) {return;}
dvec udata;
int i;
int ndata = data->nrow();
int idcol = data->idcol();
udata.resize(ndata);
for(i=0; i < ndata; i++) udata[i] = data->get_value(i,idcol);
dvec uthis = this->return_uid();
sort_unique(uthis);
sort_unique(udata);
dvec inter;
std::set_intersection(uthis.begin(), uthis.end(),
udata.begin(), udata.end(),
std::back_inserter(inter));
if(inter!=uthis) Rcpp::stop("ID found in the data set, but not in idata.");
}
void remove_unused_fields(Scope& scope,
const std::vector<std::string>& remove_members) {
std::vector<DexField*> moveable_fields;
std::vector<DexClass*> smallscope;
uint32_t aflags = ACC_STATIC | ACC_FINAL;
for (auto clazz : scope) {
bool found = can_delete(clazz);
if (!found) {
auto name = clazz->get_name()->c_str();
for (const auto& name_prefix : remove_members) {
if (strstr(name, name_prefix.c_str()) != nullptr) {
found = true;
break;
}
}
if (!found) {
TRACE(FINALINLINE, 2, "Cannot delete: %s\n", SHOW(clazz));
continue;
}
}
auto sfields = clazz->get_sfields();
for (auto sfield : sfields) {
if ((sfield->get_access() & aflags) != aflags) continue;
auto value = sfield->get_static_value();
if (value == nullptr && !is_primitive(sfield->get_type())) continue;
if (!found && !can_delete(sfield)) continue;
moveable_fields.push_back(sfield);
smallscope.push_back(clazz);
}
}
sort_unique(smallscope);
std::unordered_set<DexField*> field_target =
get_field_target(scope, moveable_fields);
std::unordered_set<DexField*> dead_fields;
for (auto field : moveable_fields) {
if (field_target.count(field) == 0) {
dead_fields.insert(field);
}
}
TRACE(FINALINLINE, 1,
"Removable fields %lu/%lu\n",
dead_fields.size(),
moveable_fields.size());
TRACE(FINALINLINE, 1, "Unhandled inline %ld\n", unhandled_inline);
for (auto clazz : smallscope) {
auto& sfields = clazz->get_sfields();
auto iter = sfields.begin();
while (iter != sfields.end()) {
auto todel = iter++;
if (dead_fields.count(*todel) > 0) {
sfields.erase(todel);
}
}
}
}
void remove_unused_fields(Scope& scope,
const std::unordered_set<DexType*>& keep_annos,
const std::unordered_set<DexField*>& keep_members) {
std::vector<DexField*> moveable_fields;
std::vector<DexClass*> smallscope;
uint32_t aflags = ACC_STATIC | ACC_FINAL;
for (auto clazz : scope) {
if (!can_delete(clazz)) {
continue;
}
auto sfields = clazz->get_sfields();
for (auto sfield : sfields) {
if ((sfield->get_access() & aflags) != aflags) continue;
auto value = sfield->get_static_value();
if (value == nullptr && !is_primitive(sfield->get_type())) continue;
if (is_kept_by_annotation(sfield, keep_annos)) continue;
if (is_kept_member(sfield, keep_members)) continue;
moveable_fields.push_back(sfield);
smallscope.push_back(clazz);
}
}
sort_unique(smallscope);
std::unordered_set<DexField*> field_target =
get_field_target(scope, moveable_fields);
std::unordered_set<DexField*> dead_fields;
for (auto field : moveable_fields) {
if (field_target.count(field) == 0) {
dead_fields.insert(field);
}
}
TRACE(FINALINLINE, 1,
"Removable fields %lu/%lu\n",
dead_fields.size(),
moveable_fields.size());
TRACE(FINALINLINE, 1, "Unhandled inline %ld\n", unhandled_inline);
for (auto clazz : smallscope) {
auto& sfields = clazz->get_sfields();
auto iter = sfields.begin();
while (iter != sfields.end()) {
auto todel = iter++;
if (dead_fields.count(*todel) > 0) {
sfields.erase(todel);
}
}
}
}
std::unordered_set<DexField*> get_called_field_defs(Scope& scope) {
std::vector<DexField*> field_refs;
walk_methods(scope,
[&](DexMethod* method) { method->gather_fields(field_refs); });
sort_unique(field_refs);
/* Okay, now we have a complete list of field refs
* for this particular dex. Map to the def actually invoked.
*/
std::unordered_set<DexField*> field_defs;
for (auto field_ref : field_refs) {
auto field_def = resolve_field(field_ref);
if (field_def == nullptr || !field_def->is_concrete()) continue;
field_defs.insert(field_def);
}
return field_defs;
}
void CrossDexRefMinimizer::insert(DexClass* cls) {
always_assert(m_class_infos.count(cls) == 0);
++m_stats.classes;
CrossDexRefMinimizer::ClassInfo& class_info =
m_class_infos
.insert({cls, CrossDexRefMinimizer::ClassInfo(m_next_index++)})
.first->second;
// Collect all relevant references that contribute to cross-dex metadata
// entries.
// We don't bother with protos and type_lists, as they are directly related
// to method refs (I tried, didn't help).
std::vector<DexMethodRef*> method_refs;
std::vector<DexFieldRef*> field_refs;
std::vector<DexType*> types;
std::vector<DexString*> strings;
cls->gather_methods(method_refs);
sort_unique(method_refs);
cls->gather_fields(field_refs);
sort_unique(field_refs);
cls->gather_types(types);
sort_unique(types);
cls->gather_strings(strings);
sort_unique(strings);
auto& refs = class_info.refs;
uint64_t refs_weight = 0;
refs.reserve(method_refs.size() + field_refs.size() + types.size() +
strings.size());
// Record all references with a particular weight.
// The weights are somewhat arbitrary, but they were chosen after trying many
// different values and observing the effect on APK size.
// TODO: Try some other variations.
for (auto mref : method_refs) {
uint32_t weight = m_config.method_ref_weight;
refs.emplace_back(mref, weight);
refs_weight += weight;
}
for (auto type : types) {
uint32_t weight = m_config.type_ref_weight;
refs.emplace_back(type, weight);
refs_weight += weight;
}
for (auto string : strings) {
uint32_t weight = m_config.string_ref_weight;
refs.emplace_back(string, weight);
refs_weight += weight;
}
for (auto fref : field_refs) {
uint32_t weight = m_config.field_ref_weight;
refs.emplace_back(fref, weight);
refs_weight += weight;
}
class_info.refs_weight = refs_weight;
std::unordered_map<DexClass*, CrossDexRefMinimizer::ClassInfoDelta>
affected_classes;
for (const std::pair<void*, uint32_t>& p : refs) {
void* ref = p.first;
uint32_t weight = p.second;
auto& classes = m_ref_classes[ref];
size_t frequency = classes.size();
// We record the need to undo (subtract weight of) a previously claimed
// infrequent ref. The actual undoing happens later in
// reprioritize.
if (frequency > 0 && frequency <= INFREQUENT_REFS_COUNT) {
for (DexClass* affected_class : classes) {
always_assert(affected_class != cls);
affected_classes[affected_class]
.infrequent_refs_weight[frequency - 1] -= weight;
}
}
++frequency;
// We are recording a new infrequent unapplied ref, if any.
// This happens immediately for the to be inserted class cls,
// so that it can be used right away by the upcoming
// class_info.get_priority() call, while all other change requests happen
// later in reprioritize.
if (frequency <= INFREQUENT_REFS_COUNT) {
for (DexClass* affected_class : classes) {
affected_classes[affected_class]
.infrequent_refs_weight[frequency - 1] += weight;
}
class_info.infrequent_refs_weight[frequency - 1] += weight;
}
// There's an implicit invariant that class_info and the keys of
// affected_classes are disjoint, so we are not going to reprioritize
// the class that we are adding here.
classes.emplace(cls);
}
const auto priority = class_info.get_priority();
m_prioritized_classes.insert(cls, priority);
TRACE(IDEX, 4,
"[dex ordering] Inserting class {%s} with priority %016lx; index %u; "
"%s infrequent refs weights, %u total refs\n",
SHOW(cls), priority, class_info.index,
format_infrequent_refs_array(class_info.infrequent_refs_weight).c_str(),
refs.size());
reprioritize(affected_classes);
}
bool ternary_table_updates_check_01(TERNARY_TABLE *table, TERNARY_TABLE_UPDATES *updates) {
sort_unique(updates->inserts);
return table_updates_check_key<cols_01>(updates->inserts, updates->deletes, table->unshifted);
}
// [[Rcpp::export]]
SEXP write_worksheet_xml_2( std::string prior
, std::string post
, Reference sheet_data
, CharacterVector row_heights
, std::string R_fileName){
// open file and write header XML
const char * s = R_fileName.c_str();
std::ofstream xmlFile;
xmlFile.open (s);
xmlFile << "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>";
xmlFile << prior;
IntegerVector cell_row = sheet_data.field("rows");
// If no data write childless node and return
if(cell_row.size() == 0){
xmlFile << "<sheetData/>";
xmlFile << post;
xmlFile.close();
return Rcpp::wrap(0);
}
// sheet_data will be in order, jsut need to check for row_heights
CharacterVector cell_col = int_2_cell_ref(sheet_data.field("cols"));
CharacterVector cell_types = map_cell_types_to_char(sheet_data.field("t"));
CharacterVector cell_value = sheet_data.field("v");
CharacterVector cell_fn = sheet_data.field("f");
CharacterVector style_id = sheet_data.field("style_id");
CharacterVector unique_rows(sort_unique(cell_row));
CharacterVector row_heights_rows = row_heights.attr("names");
size_t n_row_heights = row_heights.size();
size_t n = cell_row.size();
size_t k = unique_rows.size();
std::string xml;
std::string cell_xml;
size_t j = 0;
size_t h = 0;
String current_row = unique_rows[0];
bool row_has_data = true;
xmlFile << "<sheetData>";
for(size_t i = 0; i < k; i++){
cell_xml = "";
row_has_data = true;
while(current_row == unique_rows[i]){
row_has_data = true;
j += 1;
if(CharacterVector::is_na(cell_col[j-1])){ //If r IS NA we have no row data we only have a rowHeight
row_has_data = false;
if(j == n)
break;
current_row = cell_row[j];
break;
}
//cell XML strings
cell_xml += "<c r=\"" + cell_col[j-1] + itos(cell_row[j-1]);
if(!CharacterVector::is_na(style_id[j-1]))
cell_xml += "\" s=\"" + style_id[j-1];
//If we have a t value we must have a v value
if(!CharacterVector::is_na(cell_types[j-1])){
//If we have a c value we might have an f value
if(CharacterVector::is_na(cell_fn[j-1])){ // no function
cell_xml += "\" t=\"" + cell_types[j-1] + "\"><v>" + cell_value[j-1] + "</v></c>";
}else{
if(CharacterVector::is_na(cell_value[j-1])){ // If v is NA
cell_xml += "\" t=\"" + cell_types[j-1] + "\">" + cell_fn[j-1] + "</c>";
}else{
cell_xml += "\" t=\"" + cell_types[j-1] + "\">" + cell_fn[j-1] + "<v>" + cell_value[j-1] + "</v></c>";
}
}
}else if(!CharacterVector::is_na(cell_fn[j-1])){
cell_xml += "\">" + cell_fn[j-1] + "</c>";
}else{
cell_xml += "\"/>";
}
if(j == n)
break;
current_row = cell_row[j];
}
if(h < n_row_heights){
if((unique_rows[i] == row_heights_rows[h]) & row_has_data){ // this row has a row height and cell_xml data
xmlFile << "<row r=\"" + unique_rows[i] + "\" ht=\"" + row_heights[h] + "\" customHeight=\"1\">" + cell_xml + "</row>";
h++;
}else if(row_has_data){
xmlFile << "<row r=\"" + unique_rows[i] + "\">" + cell_xml + "</row>";
}else{
xmlFile << "<row r=\"" + unique_rows[i] + "\" ht=\"" + row_heights[h] + "\" customHeight=\"1\"/>";
h++;
}
}else{
xmlFile << "<row r=\"" + unique_rows[i] + "\">" + cell_xml + "</row>";
}
}
// write closing tag and XML post data
xmlFile << "</sheetData>";
xmlFile << post;
//close file
xmlFile.close();
return wrap(0);
}
void comm_mesh_rebalance( BulkData & M ,
const CoordinateField & node_coord_field ,
const WeightField * const elem_weight_field ,
std::vector<OctTreeKey> & cut_keys )
{
const MetaData & mesh_meta_data = M.mesh_meta_data();
Part * const uses_part = & mesh_meta_data.locally_used_part();
Part * const owns_part = & mesh_meta_data.locally_owned_part();
const unsigned p_size = M.parallel_size();
const unsigned p_rank = M.parallel_rank();
//--------------------------------------------------------------------
// The node_coord_field must be up to date on all processors
// so that the element oct tree keys are parallel consistent.
// It is assumed that the shared node_coord_field values are
// already consistent.
{
const FieldBase * const ptr = & node_coord_field ;
std::vector< const FieldBase *> tmp ;
tmp.push_back( ptr );
const std::vector<EntityProc> & aura_domain = M.ghost_source();
const std::vector<EntityProc> & aura_range = M.ghost_destination();
communicate_field_data( M , aura_domain , aura_range , tmp , false );
}
//--------------------------------------------------------------------
// Generate global oct-tree keys for local element centroids
// and cuts for the global element centroids.
double bounds[4] ;
global_coordinate_bounds( M , node_coord_field , bounds );
cut_keys.assign( p_size , OctTreeKey() );
OctTreeKey * const cut_begin = & cut_keys[0] ;
OctTreeKey * const cut_first = cut_begin + 1 ;
OctTreeKey * const cut_end = cut_begin + p_size ;
global_element_cuts( M , bounds , node_coord_field ,
elem_weight_field , cut_begin );
//--------------------------------------------------------------------
// Mapping of *all* elements to load balanced processor,
// even the aura elements.
// This requires that the node coordinates on the aura
// elements be up to date.
{
std::vector< const FieldBase * > tmp ;
const FieldBase * const tmp_coord = & node_coord_field ;
tmp.push_back( tmp_coord );
const std::vector<EntityProc> & d = M.ghost_source();
const std::vector<EntityProc> & r = M.ghost_destination();
communicate_field_data( M , d , r , tmp , false );
}
{
const EntitySet & elem_set = M.entities( Element );
const EntitySet::iterator i_end = elem_set.end();
EntitySet::iterator i = elem_set.begin();
while ( i != i_end ) {
Entity & elem = *i ; ++i ;
const OctTreeKey k = elem_key( bounds , node_coord_field , elem );
const unsigned p = std::upper_bound(cut_first, cut_end, k) - cut_first ;
M.change_entity_owner( elem , p );
}
}
//--------------------------------------------------------------------
// Fill 'rebal' with all uses entities' rebalancing processors
std::vector<EntityProc> rebal ;
rebal_elem_entities( M , Node , rebal );
rebal_elem_entities( M , Edge , rebal );
rebal_elem_entities( M , Face , rebal );
{
const Part & part_uses = * uses_part ;
const KernelSet & elem_kernels = M.kernels( Element );
const KernelSet::const_iterator i_end = elem_kernels.end();
KernelSet::const_iterator i = elem_kernels.begin();
while ( i != i_end ) {
const Kernel & kernel = *i ; ++i ;
if ( kernel.has_superset( part_uses ) ) {
const Kernel::iterator j_end = kernel.end();
Kernel::iterator j = kernel.begin();
while ( j != j_end ) {
Entity * const entity = *j ; ++j ;
const unsigned p = entity->owner_rank();
EntityProc tmp( entity , p );
rebal.push_back( tmp );
}
}
}
}
// The 'other' entities rebalance based upon the entities
// that they use. This may lead to more sharing entities.
// Thus 'rebal' is input and then updated.
rebal_other_entities( M , Particle , rebal );
rebal_other_entities( M , Constraint , rebal );
// 'rebal' now contains the rebalancing (entity,processor) pairs
// for every non-aura entity. Can now delete the aura entities.
remove_aura( M );
// Copy entities to new processors according to 'rebal'.
// Only send the owned entities.
// Include all processors associated with the entity in 'rebal'.
// Unpack all nodes, then all edges, then all faces, then all elements,
// from each processor.
// The owner of a shared entity is the max-rank processor.
// Add received entities to shared if more than one processor.
{
const RebalanceComm manager ;
std::vector<EntityProc> recv_rebal ;
communicate_entities( manager , M , M , rebal , recv_rebal , false );
// Destroy not-retained entities, they have been packed.
// Remove the corresponding entries in 'rebal'
destroy_not_retained( M , rebal );
rebal.insert( rebal.end() , recv_rebal.begin() , recv_rebal.end() );
sort_unique( rebal );
}
// The 'rebal' should contain a reference to every non-aura entity
// on the local processor. These references include every
// processor on which the entity now resides, including the
// local processor.
{ // Set parallel ownership and sharing parts.
std::vector<EntityProc>::iterator ish ;
for ( ish = rebal.begin() ; ish != rebal.end() ; ) {
Entity & e = * ish->first ;
for ( ; ish != rebal.end() && ish->first == & e ; ++ish );
const bool is_owned = p_rank == e.owner_rank() ;
// Change ownership.
std::vector<Part*> add_parts ;
std::vector<Part*> remove_parts ;
if ( is_owned ) { add_parts.push_back( owns_part ); }
else { remove_parts.push_back( owns_part ); }
M.change_entity_parts( e , add_parts , remove_parts );
}
// Remove references to the local processor,
// the remaining entries define the sharing.
for ( ish = rebal.end() ; ish != rebal.begin() ; ) {
--ish ;
if ( p_rank == ish->second ) { ish = rebal.erase( ish ); }
}
M.set_shares( rebal );
}
// Establish new aura
comm_mesh_regenerate_aura( M );
}
static void strip_src_strings(
DexStoresVector& stores, const char* map_path, PassManager& mgr) {
size_t shortened = 0;
size_t string_savings = 0;
std::unordered_map<DexString*, std::vector<DexString*>> global_src_strings;
std::unordered_set<DexString*> shortened_used;
for (auto& classes : DexStoreClassesIterator(stores)) {
for (auto const& clazz : classes) {
auto src_string = clazz->get_source_file();
if (src_string) {
// inserting actual source files into this set will cause them to not
// get used --- as the whole point of this analysis is to substitute
// source file strings
shortened_used.insert(src_string);
}
}
}
for (auto& classes : DexStoreClassesIterator(stores)) {
std::unordered_map<DexString*, DexString*> src_to_shortened;
std::vector<DexString*> current_dex_strings;
for (auto const& clazz : classes) {
clazz->gather_strings(current_dex_strings);
}
sort_unique(current_dex_strings, compare_dexstrings);
// reverse current_dex_strings vector, so that we prefer strings that will
// get smaller indices
std::reverse(std::begin(current_dex_strings),
std::end(current_dex_strings));
for (auto const& clazz : classes) {
auto src_string = clazz->get_source_file();
if (!src_string) {
continue;
}
DexString* shortened_src_string = nullptr;
if (src_to_shortened.count(src_string) == 0) {
shortened_src_string =
get_suitable_string(shortened_used, current_dex_strings);
if (!shortened_src_string) {
opt_warn(UNSHORTENED_SRC_STRING, "%s\n", SHOW(src_string));
shortened_src_string = src_string;
} else {
shortened++;
string_savings += strlen(src_string->c_str());
}
src_to_shortened[src_string] = shortened_src_string;
shortened_used.emplace(shortened_src_string);
global_src_strings[src_string].push_back(shortened_src_string);
} else {
shortened_src_string = src_to_shortened[src_string];
}
clazz->set_source_file(shortened_src_string);
}
}
TRACE(SHORTEN, 1, "src strings shortened %ld, %lu bytes saved\n", shortened,
string_savings);
mgr.incr_metric(METRIC_SHORTENED_STRINGS, shortened);
mgr.incr_metric(METRIC_BYTES_SAVED, string_savings);
// generate mapping
FILE* fd = fopen(map_path, "w");
if (fd == nullptr) {
perror("Error writing mapping file");
return;
}
for (auto it : global_src_strings) {
auto desc_vector = it.second;
sort_unique(desc_vector);
fprintf(fd, "%s ->", it.first->c_str());
for (auto str : desc_vector) {
fprintf(fd, " %s,", str->c_str());
}
fprintf(fd, "\n");
}
fclose(fd);
}
