void
sort_view(FileView *v)
{
int i;
if(v->sort[0] > SK_LAST)
{
/* Completely skip sorting if primary key isn't set. */
return;
}
view = v;
custom_view = flist_custom_active(v);
i = SK_COUNT;
while(--i >= 0)
{
const char sorting_key = view->sort[i];
if(abs(sorting_key) > SK_LAST)
{
continue;
}
sort_by_key(sorting_key);
}
if(!ui_view_sort_list_contains(v->sort, SK_BY_DIR))
{
sort_by_key(SK_BY_DIR);
}
}
/* Sorts specified range of entries according to sorting groups option. */
static void
sort_by_groups(dir_entry_t *entries, size_t nentries)
{
char **groups = NULL;
int ngroups = 0;
const int optimize = (view_sort_groups != view->sort_groups_g);
int i;
char *const copy = strdup(view_sort_groups);
char *group = copy, *state = NULL;
while((group = split_and_get(group, ',', &state)) != NULL)
{
ngroups = add_to_string_array(&groups, ngroups, 1, group);
}
free(copy);
for(i = ngroups - (optimize ? 1 : 0); i >= 1; --i)
{
regex_t regex;
(void)regcomp(®ex, groups[i], REG_EXTENDED | REG_ICASE);
sort_by_key(entries, nentries, SK_BY_GROUPS, ®ex);
regfree(®ex);
}
if(optimize && ngroups != 0)
{
sort_by_key(entries, nentries, SK_BY_GROUPS, &view->primary_group);
}
free_string_array(groups, ngroups);
}
/* Sorts sequence of file entries (plain list, not tree). */
static void
sort_sequence(dir_entry_t *entries, size_t nentries)
{
int i = SK_COUNT;
while(--i >= 0)
{
const char sorting_key = view_sort[i];
if(abs(sorting_key) > SK_LAST)
{
continue;
}
if(sorting_key == SK_BY_GROUPS)
{
sort_by_groups(entries, nentries);
continue;
}
sort_by_key(entries, nentries, sorting_key, NULL);
}
if(!ui_view_sort_list_contains(view_sort, SK_BY_DIR))
{
sort_by_key(entries, nentries, SK_BY_DIR, NULL);
}
}
TEST(GroupByKey, Empty)
{
std::vector<int> A{3,7,8,1,3,8,1,7,5,1};
sort_by_key(A);
for (auto a: A) {
std::cout << a << std::endl;
}
}
int main (void)
{
std::vector<int> A{3,7,8,1,3,8,1,7,5,1};
sort_by_key(A);
for (auto a: A) {
std::cout << a << std::endl;
}
return 0;
}
void depthSortCUDA(float4 *pos, float *depth, int *indices, float4 modelViewZ, int numParticles)
{
for (int i = 0; i < numParticles; i++)
{
float4 p = pos[i];
float z = dot(make_float4(p.x, p.y, p.z, 1.0f), modelViewZ);
depth[i] = z;
indices[i] = i;
}
sort_by_key(depth, indices, numParticles);
}
void
sort_view(FileView *v)
{
int i;
view = v;
i = SORT_OPTION_COUNT;
while(--i >= 0)
{
const char sorting_key = view->sort[i];
if(abs(sorting_key) > LAST_SORT_OPTION)
{
continue;
}
sort_by_key(sorting_key);
}
if(!ui_view_sort_list_contains(v->sort, SORT_BY_TYPE))
{
sort_by_key(SORT_BY_TYPE);
}
}
void score(pcooc_vector_t ** cooc_vec_ptr,/*pcooc_vector_t * res_vec,*/global_score_type * g_scores,unsigned long mem,unsigned lagg,
const char * testset,const char * ngramsfname,pindicator_vector_t * pairs2keep )
{
double phase1span,phase2span;
global_score_type gs;
// cout<<"Scoring started..."<<endl;
Crono timer;
timer.start();
pcooc_vector_t * cooc_vec=*cooc_vec_ptr;
pcooc_vector_t * res_vec=new pcooc_vector_t(cooc_vec->nb_sub_vecs());
cooc_vec->print_sizes();
gs.all_occurs=cooc_vec->size();
cout<<endl;
cout<<"-------------------------\n"
"Sorting (first run)\n"
"-------------------------"<<endl;
sort_by_key (cooc_vec,mem);
timer.stop();
phase1span=timer.span();
cout<<"\nDone sorting partial sets\nPartial sorting took "<<timer.formatted_span()<<endl;
cout<<"\n-------------------------\n"
"Merging partial sorts\n"
"-------------------------"<<endl;
timer.reset();
timer.start();
aggregation_func selector=&max_double;
switch(lagg){
case OCC_AGG: selector=&max_occur;break;
case AVG_AGG: selector=&average;break;
}
Merger<cooc_type> * /*pcooc_vector_t::merge_iterator*/ g_merger=new Merger<cooc_type>(cooc_vec);//->get_merge_iterator();
#pragma omp parallel shared(cooc_vec,res_vec, g_merger,selector)
{
g_merger->init(omp_get_thread_num());
#pragma omp barrier
while(true){
score_one_source(res_vec,g_merger,selector);
if(!(*g_merger))
break;
// cout<<omp_get_thread_num()<<": "<</*g_merger.count()<<*/"\t";
}
}
timer.stop();
cout<<"Done merging in: "<<timer.formatted_span()<<endl;
cout<<//"\n-------------------------------------------------------\n"<<
"\nFirst run of scoring done \nFirst run, including partial sorting, took "<<
timer.formatted_span(phase1span)<<endl;
delete g_merger;
delete *cooc_vec_ptr;
// cooc_vec->clear();
//second sort by target sentences IDs
*cooc_vec_ptr=new pcooc_vector_t(res_vec->nb_sub_vecs());
cooc_vec=*cooc_vec_ptr;
phase1span+=timer.span();
timer.reset();
timer.start();
cout<<"\nNumber of unique pairs: "<<res_vec->size()<<endl;
res_vec->print_sizes();
cout<<endl;
cout<<"---------------------------\nResorting (second run)\n---------------------------"<<endl;
sort_by_key (res_vec,mem);
timer.stop();
phase2span=timer.span();
cout<<"\nDone sorting partial sets\nPartial sorting took "<<timer.formatted_span()<<endl;
/*+++++++++++++++++++++++++++Allow subsampling at this stage++++++++++++++++++++++++++++++++++*/
pngram_vector_t * ngrams=NULL;
if(testset != NULL || ngramsfname!= NULL){
ngrams= new pngram_vector_t(NB_WORKERS);
timer.reset();
timer.start();
if(ngramsfname!= NULL){
cout<<"\n########################\n"
"Loading the n-gram list\n"
"########################\n"<<endl;
pload_ngrams(ngramsfname, ngrams);
cout<<"Done loading n-grams!\n"<<"n-gram list loaded in ";
}else{
cout<<"\n########################\n"
"Creating the n-gram list\n"
"########################\n"<<endl;
pget_ngrams(testset, ngrams);
cout<<"Done creating n-grams!\n"<<"n-gram list created in ";
}
timer.stop();
cout<<timer.formatted_span()<<endl;
timer.reset();
cout<<"\n------------------------\n"
"Sorting the n-gram list\n"
"------------------------\n"<<endl;
timer.start();
sort_unique_ngrams(ngrams,mem);
timer.stop();
cout<<"Sorting n-grams took "<<timer.formatted_span()<<endl;
if(testset){
cout<<"\n------------------------\n"
"Saving the n-gram list\n"
"------------------------\n"<<endl;
timer.reset();
timer.start();
psave_ngrams(rsplit(testset,PATH_SEP).second.append(".ngrams").c_str(),ngrams);
timer.stop();
cout<<"Saving n-grams took "<<timer.formatted_span()<<endl;
}
}
/*++++++++++++++++++++++++++END of ngram creation/loading+++++++++++++++++++++++++++++++++++++*/
if(!ngrams)
cout<<"\n-------------------------\n"
"Merging partial sorts\n"
"-------------------------"<<endl;
else
cout<<"\n---------------------------------------\n"
"Merging and sub-sampling partial sorts\n"
"---------------------------------------"<<endl;
timer.reset();
timer.start();
Merger<ngram_t> * /*pcooc_vector_t::merge_iterator*/ g_ngmerger=NULL;
if(ngrams)
g_ngmerger=new Merger<ngram_t>(ngrams);
g_merger=new Merger<cooc_type>(res_vec);//itr=res_vec->get_merge_iterator(cmp_by_val);
//int more=0;
/*
tmp_vec.clear();
unsigned long tocc,
invn[3];
cooc_type::key_type new_trgt;
cout<<"Merging the sorted parts\n";
*/
unsigned long big_n1=0,
big_n2=0,big_n3=0,big_n4=0,
n_matches=0;
#pragma omp parallel shared(g_merger,g_ngmerger,cooc_vec,res_vec,ngrams,pairs2keep),\
reduction(+:big_n1),reduction(+:big_n2),reduction(+:big_n3),reduction(+:big_n4), reduction(+:n_matches)
{
g_merger->init(omp_get_thread_num());
if(g_ngmerger)
g_ngmerger->init(omp_get_thread_num());
#pragma omp barrier
while(true){
n_matches+=score_one_source(cooc_vec,&big_n1,&big_n2,&big_n3,&big_n4,g_merger,g_ngmerger,pairs2keep);//score_one_source(&lock,&lq,res_vec,g_merger);
if(!(*g_merger))
break;
// cout<<omp_get_thread_num()<<": "<</*g_merger.count()<<*/"\t";
}
}
timer.stop();
cout<<"Done merging in: "<<timer.formatted_span()<<endl;
cout<<"Disposing unneeded resources\n";
timer.start();
delete g_merger;
delete res_vec;
if(ngrams){
cout<<"\nNumber of matching pairs to provided ngrams: "<<n_matches<<" ("<<100*double(n_matches)/cooc_vec->size()<<"%)"<<endl;
delete g_ngmerger;
delete ngrams;
}
gs.big_n1=big_n1;
gs.big_n2=big_n2;
gs.big_n3=big_n3;
gs.big_n4=big_n4;
//cout<<"Phrases more than 4 times: "<<more<<endl;
gs.all=cooc_vec->size();
if(g_scores){
if(gs.big_n1==0){
cerr<<"Warning: N1 was found to be zero, taking N1=1!\n";
gs.big_n1=1;
}
if(gs.big_n2==0){
cerr<<"Warning: N2 was found to be zero, taking N2=1!\n";
gs.big_n2=1;
}
if(gs.big_n3==0){
cerr<<"Warning: N3 was found to be zero, taking N3=1!\n";
gs.big_n3=1;
}
memcpy(g_scores,&gs,sizeof(global_score_type));
}
timer.stop();
cout<<"\nSecond run of scoring done\nSecond run, including partial sorting, took "<<
timer.formatted_span(phase2span)<<endl;
cout<<"\nTwo phase phrase scoring took "<<timer.formatted_span(phase1span+phase2span)<<endl<<endl;
//calc corresponding scores
}
