int main(int argc, char *argv[]) { int n_search; int i_halo; char filename_in[SID_MAX_FILENAME_LENGTH]; char group_text_prefix[4]; int n_files; int k_read; int max_n_groups; int l_read; int n_groups; int * n_particles_i; int * n_particles_j; int j_read; int mode; int n_groups_i; int n_groups_j; int j_halo; int i_read; int i_read_start; int i_read_stop; SID_fp fp_in; SID_Init(&argc, &argv, NULL); // Fetch user inputs char filename_root_in[SID_MAX_FILENAME_LENGTH]; char filename_catalog_root[SID_MAX_FILENAME_LENGTH]; char filename_halo_version[SID_MAX_FILENAME_LENGTH]; strcpy(filename_root_in, argv[1]); strcpy(filename_halo_version, argv[2]); if(!strcmp(argv[3], "groups") || !strcmp(argv[3], "group")) mode = MATCH_GROUPS; else if(!strcmp(argv[3], "subgroups") || !strcmp(argv[3], "subgroup")) mode = MATCH_SUBGROUPS; else { SID_exit_error("Invalid mode selection {%s}. Should be 'group' or 'subgroup'.", SID_ERROR_SYNTAX, argv[3]); } i_read = atoi(argv[4]); j_read = atoi(argv[5]); int flag_SSimPL_base = GBP_TRUE; if(argc == 7) { flag_SSimPL_base = GBP_FALSE; strcpy(filename_catalog_root, argv[6]); sprintf(filename_catalog_root, "%s/halos/%s", argv[6], filename_halo_version); } else sprintf(filename_catalog_root, "%s/halos/%s", filename_root_in, filename_halo_version); SID_log("Searching match information for halo #%d in file #%d of {%s}...", SID_LOG_OPEN | SID_LOG_TIMER, i_halo, i_read, filename_root_in); // Convert filename_root to filename switch(mode) { case MATCH_SUBGROUPS: sprintf(group_text_prefix, "sub"); break; case MATCH_GROUPS: sprintf(group_text_prefix, ""); break; } // Set the standard SSiMPL match file path char filename_root[SID_MAX_FILENAME_LENGTH]; if(flag_SSimPL_base) sprintf(filename_root, "%s/trees/matches/%03d/", filename_root_in, i_read); else sprintf(filename_root, "%s_", filename_root_in); // Read header information int i_read_in; int j_read_in; int n_groups_1; int n_groups_2; float score_rank_index; sprintf(filename_in, "%s%sgroup_matches_%03d_%03d.dat", filename_root, group_text_prefix, i_read, j_read); SID_fopen(filename_in, "r", &fp_in); SID_fread(&i_read_in, sizeof(int), 1, &fp_in); SID_log("i_read =%d", SID_LOG_COMMENT, i_read_in); SID_fread(&j_read_in, sizeof(int), 1, &fp_in); SID_log("j_read =%d", SID_LOG_COMMENT, j_read_in); SID_fread(&n_groups_i, sizeof(int), 1, &fp_in); SID_log("n_groups_i=%d", SID_LOG_COMMENT, n_groups_i); SID_fread(&n_groups_j, sizeof(int), 1, &fp_in); SID_log("n_groups_j=%d", SID_LOG_COMMENT, n_groups_j); SID_fread(&score_rank_index, sizeof(int), 1, &fp_in); SID_log("score_idx =%f", SID_LOG_COMMENT, score_rank_index); // Allocate RAM int * match = (int *)SID_malloc(sizeof(int) * n_groups_i); float *score = (float *)SID_malloc(sizeof(float) * n_groups_i); int * count = (int *)SID_malloc(sizeof(int) * n_groups_i); // Read arrays SID_fread(match, sizeof(int), n_groups_i, &fp_in); SID_fread(score, sizeof(float), n_groups_i, &fp_in); SID_fread(count, sizeof(int), n_groups_i, &fp_in); // Close file SID_fclose(&fp_in); // Read halo sizes from header file SID_log("Reading halo sizes...", SID_LOG_OPEN); int *n_p_i = (int *)SID_malloc(sizeof(int) * n_groups_i); sprintf(filename_in, "%s_%03d.catalog_%sgroups", filename_catalog_root, i_read, group_text_prefix); SID_fopen(filename_in, "r", &fp_in); int n_cat_i; int offset_size_i; SID_fread(&n_cat_i, sizeof(int), 1, &fp_in); SID_fread(&offset_size_i, sizeof(int), 1, &fp_in); if(n_cat_i != n_groups_i) SID_exit_error("Catalog 'i' halo counts don't match (ie %d!=%d)", SID_ERROR_LOGIC, n_cat_i, n_groups_i); SID_fread(n_p_i, sizeof(int), n_cat_i, &fp_in); SID_fclose(&fp_in); int *n_p_j = (int *)SID_malloc(sizeof(int) * n_groups_j); sprintf(filename_in, "%s_%03d.catalog_%sgroups", filename_catalog_root, j_read, group_text_prefix); SID_fopen(filename_in, "r", &fp_in); int n_cat_j; int offset_size_j; SID_fread(&n_cat_j, sizeof(int), 1, &fp_in); SID_fread(&offset_size_j, sizeof(int), 1, &fp_in); if(n_cat_j != n_groups_j) SID_exit_error("Catalog 'i' halo counts don't match (ie %d!=%d)", SID_ERROR_LOGIC, n_cat_j, n_groups_j); SID_fread(n_p_j, sizeof(int), n_cat_j, &fp_in); SID_fclose(&fp_in); SID_log("Done.", SID_LOG_CLOSE); // Print results for(k_read = 0; k_read < n_groups_i; k_read++) { if(match[k_read] >= 0) printf("%7d %7d %7d %7d %7d %le %le %le\n", k_read, match[k_read], n_p_i[k_read], n_p_j[match[k_read]], count[k_read], score[k_read], maximum_match_score(n_p_i[k_read]), match_score_f_goodness(score[k_read], n_p_i[k_read])); } // Clean-up SID_free(SID_FARG match); SID_free(SID_FARG score); SID_free(SID_FARG count); SID_free(SID_FARG n_p_i); SID_free(SID_FARG n_p_j); SID_log("Done.", SID_LOG_CLOSE); SID_Finalize(); }
int main(int argc, char *argv[]) { int n_search; int i_halo; char filename_in[SID_MAX_FILENAME_LENGTH]; char group_text_prefix[4]; int n_files; int k_read; int l_read; int * n_particles_i; int * n_particles_j; int j_read; int mode; int j_halo; int i_read; int i_read_start; int i_read_stop; SID_fp fp_in; SID_Init(&argc, &argv, NULL); // Fetch user inputs char filename_SSimPL_root[SID_MAX_FILENAME_LENGTH]; strcpy(filename_SSimPL_root, argv[1]); SID_log("Checking the integrity of the match files for {%s}...", SID_LOG_OPEN | SID_LOG_TIMER, filename_SSimPL_root); int *n_groups = NULL; int *n_subgroups = NULL; for(int i_type = 0; i_type < 2; i_type++) { // Convert filename_root to filename switch(i_type) { case 0: mode = MATCH_SUBGROUPS; sprintf(group_text_prefix, "sub"); break; case 1: mode = MATCH_GROUPS; sprintf(group_text_prefix, ""); break; } SID_log("Processing %sgroups...", SID_LOG_OPEN | SID_LOG_TIMER, group_text_prefix); // Set the standard SSiMPL match file path char filename_root_in[SID_MAX_FILENAME_LENGTH]; sprintf(filename_root_in, "%s/trees/matches/", filename_SSimPL_root); // Read halo sizes from header file SID_log("Processing header file...", SID_LOG_OPEN | SID_LOG_TIMER); sprintf(filename_in, "%s/%sgroup_matches_header.dat", filename_root_in, group_text_prefix); SID_fopen(filename_in, "r", &fp_in); SID_fread(&i_read_start, sizeof(int), 1, &fp_in); SID_fread(&i_read_stop, sizeof(int), 1, &fp_in); SID_fread(&n_search, sizeof(int), 1, &fp_in); SID_fread(&n_files, sizeof(int), 1, &fp_in); int *n_halos = NULL; switch(mode) { case MATCH_SUBGROUPS: n_subgroups = (int *)SID_malloc(sizeof(int) * n_files); n_halos = n_subgroups; break; case MATCH_GROUPS: n_groups = (int *)SID_malloc(sizeof(int) * n_files); n_halos = n_groups; break; } if(mode == MATCH_GROUPS) SID_log("Halo counts (snap/No. groups/No. subgroups):", SID_LOG_OPEN); for(k_read = 0; k_read < n_files; k_read++) { SID_fread(&l_read, sizeof(int), 1, &fp_in); SID_fread(&(n_halos[k_read]), sizeof(int), 1, &fp_in); SID_fskip(sizeof(int), n_halos[k_read], &fp_in); if(mode == MATCH_GROUPS) { int *n_subgroups_group = (int *)SID_malloc(sizeof(int) * n_halos[k_read]); SID_fread(n_subgroups_group, sizeof(int), n_halos[k_read], &fp_in); int n_subgroups_test = 0; for(int i_test = 0; i_test < n_halos[k_read]; i_test++) n_subgroups_test += n_subgroups_group[i_test]; if(n_subgroups[k_read] != n_subgroups_test) SID_log("Error in %s header: l_read=%3d k_read=%3d n_subgroups: %d!=%d\n", SID_LOG_COMMENT, l_read, k_read, n_subgroups[k_read], n_subgroups_test); SID_free(SID_FARG n_subgroups_group); } if(mode == MATCH_GROUPS) SID_log("%03d %d %d", SID_LOG_COMMENT, k_read, n_groups[k_read], n_subgroups[k_read]); } if(mode == MATCH_GROUPS) SID_log("", SID_LOG_CLOSE | SID_LOG_NOPRINT); SID_fclose(&fp_in); SID_log("Done.", SID_LOG_CLOSE); SID_log("Processing match files...", SID_LOG_OPEN | SID_LOG_TIMER); for(int i_read = i_read_start; i_read < i_read_stop; i_read++) { for(int j_read = GBP_MAX(0, i_read - n_search); j_read < GBP_MIN(i_read_stop, i_read + n_search); j_read++) { if(i_read != j_read) { sprintf(filename_in, "%s/%03d/%sgroup_matches_%03d_%03d.dat", filename_root_in, i_read, group_text_prefix, i_read, j_read); SID_log("Processing {%s}...", SID_LOG_OPEN, filename_in); // Read header information int i_read_in; int j_read_in; int n_groups_i; int n_groups_j; SID_fopen(filename_in, "r", &fp_in); SID_fread(&i_read_in, sizeof(int), 1, &fp_in); SID_fread(&j_read_in, sizeof(int), 1, &fp_in); SID_fread(&n_groups_i, sizeof(int), 1, &fp_in); SID_fread(&n_groups_j, sizeof(int), 1, &fp_in); if(i_read_in != i_read || j_read_in != j_read || n_groups_i != n_halos[n_files - i_read_in - 1] || n_groups_j != n_halos[n_files - j_read_in - 1]) SID_log("Error in matching file: i_read=%3d j_read=%3d n_i_in=%d n_i=%d n_j_in=%d n_j=%d\n", SID_LOG_COMMENT, i_read, j_read, n_groups_i, n_halos[n_files - i_read_in - 1], n_groups_j, n_halos[n_files - j_read_in - 1]); // Read matches int match; for(k_read = 0; k_read < n_groups_i; k_read++) SID_fread(&match, sizeof(int), 1, &fp_in); // Read indices size_t indices; for(k_read = 0; k_read < n_groups_i; k_read++) SID_fread(&indices, sizeof(size_t), 1, &fp_in); // Read scores float score; for(k_read = 0; k_read < n_groups_i; k_read++) SID_fread(&score, sizeof(float), 1, &fp_in); // Close file SID_fclose(&fp_in); SID_log("Done.", SID_LOG_CLOSE); } } } SID_log("Done.", SID_LOG_CLOSE); SID_log("Done.", SID_LOG_CLOSE); } SID_free(SID_FARG n_groups); SID_free(SID_FARG n_subgroups); SID_log("Done.", SID_LOG_CLOSE); SID_Finalize(); }
int main(int argc, char *argv[]){ int n_search; int i_halo; char filename_SSimPL_root[MAX_FILENAME_LENGTH]; char filename_in[MAX_FILENAME_LENGTH]; char group_text_prefix[4]; int n_files; int k_read; int max_n_groups; int l_read; int n_groups; int j_read; int mode; int n_groups_i; int n_groups_j; int j_halo; int match; int i_read; int i_read_start; int i_read_stop; SID_fp fp_in; SID_init(&argc,&argv,NULL,NULL); // Fetch user inputs strcpy(filename_SSimPL_root,argv[1]); if(!strcmp(argv[2],"groups") || !strcmp(argv[2],"group")) mode=MATCH_GROUPS; else if(!strcmp(argv[2],"subgroups") || !strcmp(argv[2],"subgroup")) mode=MATCH_SUBGROUPS; else{ SID_log("Invalid mode selection {%s}. Should be 'group' or 'subgroup'.",SID_LOG_COMMENT,argv[2]); SID_exit(ERROR_SYNTAX); } i_read=atoi(argv[3]); j_read=atoi(argv[4]); SID_log("Searching match information for halo #%d in file #%d of {%s}...",SID_LOG_OPEN|SID_LOG_TIMER,i_halo,i_read,filename_SSimPL_root); // Convert filename_root to filename switch(mode){ case MATCH_SUBGROUPS: sprintf(group_text_prefix,"sub"); break; case MATCH_GROUPS: sprintf(group_text_prefix,""); break; } // Set the standard SSiMPL match file path char filename_root_in[MAX_FILENAME_LENGTH]; sprintf(filename_root_in,"%s/trees/matches/",filename_SSimPL_root); // Set the output file char filename_base[MAX_FILENAME_LENGTH]; char filename_out[MAX_FILENAME_LENGTH]; sprintf(filename_base,filename_SSimPL_root); if(!strcmp(&(filename_base[strlen(filename_base)-1]),"/")) strcpy(&(filename_base[strlen(filename_base)-1]),"\0"); strip_path(filename_base); sprintf(filename_out,"%s_%d_%d_2way_matches.txt",filename_base,i_read,j_read); // Read header information SID_log("Reading header information...",SID_LOG_OPEN); sprintf(filename_in,"%s/%sgroup_matches_header.dat",filename_root_in,group_text_prefix); SID_fopen(filename_in,"r",&fp_in); SID_fread(&i_read_start,sizeof(int),1,&fp_in);SID_log("snap start =%d",SID_LOG_COMMENT,i_read_start); SID_fread(&i_read_stop, sizeof(int),1,&fp_in);SID_log("snap stop =%d",SID_LOG_COMMENT,i_read_stop); SID_fread(&n_search, sizeof(int),1,&fp_in);SID_log("search range=%d",SID_LOG_COMMENT,n_search); SID_fread(&n_files, sizeof(int),1,&fp_in);SID_log("# of files =%d",SID_LOG_COMMENT,n_files); for(k_read=0,max_n_groups=0;k_read<n_files;k_read++){ SID_fread(&l_read, sizeof(int),1,&fp_in); SID_fread(&n_groups,sizeof(int),1,&fp_in); SID_fseek(&fp_in, sizeof(int),n_groups,SID_SEEK_CUR); if(mode==MATCH_GROUPS) SID_fseek(&fp_in, sizeof(int),n_groups,SID_SEEK_CUR); max_n_groups=MAX(max_n_groups,n_groups); } SID_log("Max # groups=%d",SID_LOG_COMMENT,max_n_groups); SID_fclose(&fp_in); SID_log("Done.",SID_LOG_CLOSE); // Initialize some arrays int *n_particles_i =(int *)SID_malloc(sizeof(int) *max_n_groups); int *n_particles_j =(int *)SID_malloc(sizeof(int) *max_n_groups); int *match_forward_ids =(int *)SID_malloc(sizeof(int) *max_n_groups); size_t *match_forward_index =(size_t *)SID_malloc(sizeof(size_t)*max_n_groups); float *match_forward_score =(float *)SID_malloc(sizeof(float) *max_n_groups); char *match_forward_2way =(char *)SID_malloc(sizeof(char) *max_n_groups); int *match_backward_ids =(int *)SID_malloc(sizeof(int) *max_n_groups); size_t *match_backward_index=(size_t *)SID_malloc(sizeof(size_t)*max_n_groups); float *match_backward_score=(float *)SID_malloc(sizeof(float) *max_n_groups); char *match_backward_2way =(char *)SID_malloc(sizeof(char) *max_n_groups); // Loop over all matching combinations SID_log("Reading forward matches...",SID_LOG_OPEN|SID_LOG_TIMER); SID_set_verbosity(SID_SET_VERBOSITY_DEFAULT); read_matches(filename_root_in, i_read, j_read, max_n_groups, mode, &n_groups_i, &n_groups_j, n_particles_i, n_particles_j, NULL, NULL, match_forward_ids, match_forward_score, match_forward_index, match_forward_2way, FALSE); SID_log("Done.",SID_LOG_CLOSE); SID_log("Processing backwards matches...",SID_LOG_OPEN|SID_LOG_TIMER); read_matches(filename_root_in, j_read, i_read, max_n_groups, mode, &n_groups_j, &n_groups_i, n_particles_j, n_particles_i, NULL, NULL, match_backward_ids, match_backward_score, match_backward_index, match_backward_2way, FALSE); SID_log("Done.",SID_LOG_CLOSE); // Open output file FILE *fp_out; fp_out=fopen(filename_out,"w"); int i_column=1; fprintf(fp_out,"# Column (%02d): Halo index for snapshot %d\n", i_column++,i_read); fprintf(fp_out,"# (%02d): Halo index for snapshot %d\n", i_column++,j_read); fprintf(fp_out,"# (%02d): No. particles in snapshot %d\n", i_column++,i_read); fprintf(fp_out,"# (%02d): No. particles in snapshot %d\n", i_column++,j_read); fprintf(fp_out,"# (%02d): Forward match score\n", i_column++); fprintf(fp_out,"# (%02d): Forward match score/min match score\n",i_column++); fprintf(fp_out,"# (%02d): Backward match score\n", i_column++); fprintf(fp_out,"# (%02d): Backward match score/min match score\n",i_column++); for(int i_halo=0;i_halo<n_groups_i;i_halo++){ int j_halo=match_forward_ids[i_halo]; if(match_forward_2way[i_halo]){ if(j_halo<0 || j_halo>n_groups_j) SID_trap_error("There's an invalid match id (ie. %d<0 || %d>%d) attached to a 2-way match!",ERROR_LOGIC,j_halo,j_halo,n_groups_j); fprintf(fp_out,"%7d %7d %6d %6d %10.3le %10.3le %10.3le %10.3le\n", i_halo, j_halo, n_particles_i[i_halo], n_particles_j[j_halo], match_forward_score[i_halo], match_forward_score[i_halo]/minimum_match_score((double)n_particles_i[i_halo]), match_backward_score[j_halo], match_backward_score[j_halo]/minimum_match_score((double)n_particles_j[j_halo])); } } fclose(fp_out); // Clean-up SID_free(SID_FARG n_particles_i); SID_free(SID_FARG n_particles_j); SID_free(SID_FARG match_forward_ids); SID_free(SID_FARG match_forward_index); SID_free(SID_FARG match_forward_score); SID_free(SID_FARG match_forward_2way); SID_free(SID_FARG match_backward_ids); SID_free(SID_FARG match_backward_index); SID_free(SID_FARG match_backward_score); SID_free(SID_FARG match_backward_2way); SID_log("Done.",SID_LOG_CLOSE); SID_exit(ERROR_NONE); }
void read_matches(char *filename_in_dir, int i_read_in, int j_read_in, int n_halos_max, int mode, int *n_groups_i, int *n_groups_j, int *n_particles_i_in, int *n_particles_j_in, int *n_sub_group_i_in, int *n_sub_group_j_in, int *match_ids, float *match_score, size_t *match_index, char *match_flag_two_way, int flag_reject_bad_matches){ char group_text_prefix[5]; char filename_in[MAX_FILENAME_LENGTH]; SID_fp fp_in; int k_read; int l_read; int n_search; int n_matches; size_t offset; int flag_continue; int i_read_file; int j_read_file; int n_groups_file; int n_groups_file_1; int n_groups_file_2; int n_groups; int n_groups_i_file; int n_groups_j_file; int *n_sub_group_i; int *n_sub_group_j; int flag_alloc_n_sub_i=FALSE; int flag_alloc_n_sub_j=FALSE; if(i_read_in==j_read_in) SID_trap_error("i_read=j_read in read_matches",ERROR_LOGIC); switch(mode){ case MATCH_SUBGROUPS: sprintf(group_text_prefix,"sub"); break; case MATCH_GROUPS: sprintf(group_text_prefix,""); // We need n_subgroups arrays for removal of bad groups // if they have not been passed to us. if(n_sub_group_i_in==NULL){ flag_alloc_n_sub_i=TRUE; n_sub_group_i =(int *)SID_malloc(sizeof(int)*n_halos_max); } else n_sub_group_i=n_sub_group_i_in; if(n_sub_group_j_in==NULL){ flag_alloc_n_sub_j=TRUE; n_sub_group_j =(int *)SID_malloc(sizeof(int)*n_halos_max); } else n_sub_group_j=n_sub_group_j_in; break; } // Since we need the particle counts for the goodness of match criterion, // create temporary arrays in case we weren't passed an array for them. int *n_particles_i=n_particles_i_in; int *n_particles_j=n_particles_j_in; // 1) We always need n_particles_i int flag_alloc_n_particles_i=FALSE; int flag_alloc_n_particles_j=FALSE; if(n_particles_i==NULL) flag_alloc_n_particles_i=TRUE; if(n_particles_j==NULL) flag_alloc_n_particles_j=TRUE; // 2) We need n_particles_j if doing 2-way checks if(match_flag_two_way!=NULL && n_particles_j==NULL) flag_alloc_n_particles_j=TRUE; // Read the needed info from the header file int i_read; int i_read_start; int i_read_stop; int n_search_total; int n_files; int n_groups_in; int counter=0; char filename_in_name[256]; strcpy(filename_in_name,filename_in_dir); strip_path(filename_in_name); sprintf(filename_in,"%s/%sgroup_matches_header.dat",filename_in_dir,group_text_prefix); SID_fopen(filename_in,"r",&fp_in); SID_fread(&i_read_start, sizeof(int),1,&fp_in); SID_fread(&i_read_stop, sizeof(int),1,&fp_in); SID_fread(&n_search_total,sizeof(int),1,&fp_in); SID_fread(&n_files, sizeof(int),1,&fp_in); for(i_read=i_read_stop;i_read>=i_read_start && counter<2;i_read--){ SID_fread(&i_read_file, sizeof(int),1,&fp_in); if(i_read_file==i_read_in){ SID_fread(n_groups_i,sizeof(int),1,&fp_in); if((*n_groups_i)>0){ // Create a temporary array for n_particles_i if we were not passed one if(flag_alloc_n_particles_i) n_particles_i=(int *)SID_malloc(sizeof(int)*(*n_groups_i)); if(n_particles_i!=NULL) SID_fread_ordered(n_particles_i,sizeof(int),(size_t)(*n_groups_i),&fp_in); else SID_fskip(sizeof(int),(*n_groups_i),&fp_in); if(mode==MATCH_GROUPS){ if(n_sub_group_i!=NULL) SID_fread_ordered(n_sub_group_i,sizeof(int),(size_t)(*n_groups_i),&fp_in); else SID_fskip(sizeof(int),(*n_groups_i),&fp_in); } } counter++; } else if(i_read_file==j_read_in){ SID_fread(n_groups_j,sizeof(int),1,&fp_in); if((*n_groups_j)>0){ if(flag_alloc_n_particles_j) n_particles_j=(int *)SID_malloc(sizeof(int)*(*n_groups_j)); if(n_particles_j!=NULL) SID_fread_ordered(n_particles_j,sizeof(int),(size_t)(*n_groups_j),&fp_in); else SID_fskip(sizeof(int),(*n_groups_j),&fp_in); if(mode==MATCH_GROUPS){ if(n_sub_group_j!=NULL) SID_fread_ordered(n_sub_group_j,sizeof(int),(size_t)(*n_groups_j),&fp_in); else SID_fskip(sizeof(int),(*n_groups_j),&fp_in); } } counter++; } else{ SID_fread(&n_groups_in,sizeof(int),1,&fp_in); if(n_groups_in>0){ SID_fskip(sizeof(int),n_groups_in,&fp_in); if(mode==MATCH_GROUPS) SID_fskip(sizeof(int),n_groups_in,&fp_in); } } } SID_fclose(&fp_in); // Read the matching file char filename_cat1[256]; char filename_cat2[256]; char filename_in_dir_snap[256]; sprintf(filename_cat1,"%03d",i_read_in); sprintf(filename_cat2,"%03d",j_read_in); sprintf(filename_in_dir_snap,"%s/%s",filename_in_dir,filename_cat1); if(filename_in_dir!=NULL) sprintf(filename_in,"%s/%sgroup_matches_%s_%s.dat",filename_in_dir_snap,group_text_prefix,filename_cat1,filename_cat2); else sprintf(filename_in,"%s_%sgroup_matches_%s_%s.dat",filename_in_name, group_text_prefix,filename_cat1,filename_cat2); SID_fopen(filename_in,"r",&fp_in); SID_fread(&i_read_file,sizeof(int),1,&fp_in); SID_fread(&j_read_file,sizeof(int),1,&fp_in); SID_fread(n_groups_i, sizeof(int),1,&fp_in); SID_fread(n_groups_j, sizeof(int),1,&fp_in); // Read matching data SID_fread(match_ids, sizeof(int), (*n_groups_i),&fp_in); SID_fread(match_index,sizeof(size_t),(*n_groups_i),&fp_in); SID_fread(match_score,sizeof(float), (*n_groups_i),&fp_in); SID_fclose(&fp_in); // If one of the catalogs is empty, set to no-match defaults if((*n_groups_i)<=0 || (*n_groups_j)<=0){ for(int i_halo=0;i_halo<(*n_groups_i);i_halo++){ match_ids[i_halo] =-1; match_score[i_halo]= 0.; } if(match_flag_two_way!=NULL){ for(int i_halo=0;i_halo<(*n_groups_i);i_halo++) match_flag_two_way[i_halo]=FALSE; } } else{ // If we are reading groups, nullify all matches // between halos with no substructures. int i_halo; if(mode==MATCH_GROUPS){ size_t *match_index_temp; for(i_halo=0;i_halo<(*n_groups_i);i_halo++){ if(n_sub_group_i[i_halo]<=0){ match_ids[i_halo] =-1; match_score[i_halo]= 0.; } else if(match_ids[i_halo]>=0 && (*n_groups_j)>0){ if(n_sub_group_j[match_ids[i_halo]]<=0){ match_ids[i_halo] =-1; match_score[i_halo]= 0.; } } } merge_sort(match_ids,(size_t)(*n_groups_i),&match_index_temp,SID_INT,SORT_COMPUTE_INDEX,SORT_COMPUTE_NOT_INPLACE); memcpy(match_index,match_index_temp,(*n_groups_i)*sizeof(size_t)); SID_free(SID_FARG match_index_temp); } // Apply a goodness-of-fit criterion and check that the maximum allowed score has not been exceeded for(i_halo=0;i_halo<(*n_groups_i);i_halo++){ if(match_ids[i_halo]>=0){ if(flag_reject_bad_matches && !check_validity_of_match(n_particles_i[i_halo],match_score[i_halo])) match_ids[i_halo]=-1; } } // Since we may have changed some matches with the goodness // of fit criterion, we need to re-perform the sort size_t *match_index_temp=NULL; merge_sort(match_ids,(size_t)(*n_groups_i),&match_index_temp,SID_INT,SORT_COMPUTE_INDEX,SORT_COMPUTE_NOT_INPLACE); memcpy(match_index,match_index_temp,(*n_groups_i)*sizeof(size_t)); SID_free(SID_FARG match_index_temp); // Determine if the matches are two-way if we have been asked to check this if(match_flag_two_way!=NULL && (*n_groups_i)>0){ // We're going to need the particle counts in the target catalog for checking the goodness of return matches. Make sure we have them. if(n_particles_j==NULL) SID_trap_error("Target catalog halo sizes are not defined in read_matches() but are needed for checking two-way match flags.",ERROR_LOGIC); // Flip the file names for reading the return matches sprintf(filename_in_dir_snap,"%s/%s",filename_in_dir,filename_cat2); if(filename_in_dir!=NULL) sprintf(filename_in,"%s/%sgroup_matches_%s_%s.dat",filename_in_dir_snap,group_text_prefix,filename_cat2,filename_cat1); else sprintf(filename_in,"%s_%sgroup_matches_%s_%s.dat",filename_in_name, group_text_prefix,filename_cat2,filename_cat1); // Open two files, one for reading the IDs and one for matching the scores of the matching file int i_read_file_check; int j_read_file_check; int n_groups_i_check; int n_groups_j_check; SID_fp fp_check_ids; SID_fp fp_check_score; SID_fopen(filename_in,"r",&fp_check_ids); SID_fopen(filename_in,"r",&fp_check_score); SID_fread(&j_read_file_check,sizeof(int),1,&fp_check_ids); SID_fread(&i_read_file_check,sizeof(int),1,&fp_check_ids); SID_fread(&n_groups_j_check, sizeof(int),1,&fp_check_ids); SID_fread(&n_groups_i_check, sizeof(int),1,&fp_check_ids); // Check that we have the right files if(n_groups_i_check!=(*n_groups_i)) SID_trap_error("Source halo counts don't match (ie. %d!=%d) in two-way check in read_matches().",ERROR_LOGIC,n_groups_i_check,(*n_groups_i)); if(n_groups_j_check!=(*n_groups_j)) SID_trap_error("Target halo counts don't match (ie. %d!=%d) in two-way check in read_matches().",ERROR_LOGIC,n_groups_j_check,(*n_groups_j)); if(i_read_file_check!=i_read_file) SID_trap_error("Source file numbers don't match (ie. %d!=%d) in two-way check in read_matches().",ERROR_LOGIC,i_read_file_check,i_read_file); if(j_read_file_check!=j_read_file) SID_trap_error("Target file numbers don't match (ie. %d!=%d) in two-way check in read_matches().",ERROR_LOGIC,j_read_file_check,j_read_file); // Skip to the beginning of the relevant block for the score-reading file pointer SID_fskip(sizeof(int), 4, &fp_check_score); // header SID_fskip(sizeof(int), (*n_groups_j),&fp_check_score); // ids SID_fskip(sizeof(size_t),(*n_groups_j),&fp_check_score); // indices // Set everything to being a one-way match unless subsequently changed for(i_halo=0;i_halo<(*n_groups_i);i_halo++) match_flag_two_way[i_halo]=FALSE; // Read matching data in buffered chunks int n_good =0; int n_2way =0; int n_buffer =1024*1024; int n_chunk =0; int n_remaining =(*n_groups_j); int *buffer_ids =(int *)SID_malloc(sizeof(int) *n_buffer); float *buffer_score=(float *)SID_malloc(sizeof(float)*n_buffer); for(int j_halo=0;n_remaining>0;n_remaining-=n_chunk){ n_chunk=MIN(n_remaining,n_buffer); SID_fread(buffer_ids, sizeof(int), n_chunk,&fp_check_ids); SID_fread(buffer_score,sizeof(float),n_chunk,&fp_check_score); for(int k_halo=0;k_halo<n_chunk;k_halo++,j_halo++){ int id_i=buffer_ids[k_halo]; if(id_i>=0){ if(id_i>=(*n_groups_i)) SID_trap_error("Allowed matching index has been exceeded i(ie %d>=%d) while determining two-way match flags.", ERROR_LOGIC,id_i,(*n_groups_i)); // Do this check first to avoid having to check if both needed n_particles_* references are defined int id_j=match_ids[id_i]; if(id_j==j_halo){ if(!flag_reject_bad_matches || check_validity_of_match(n_particles_j[j_halo],buffer_score[k_halo])){ match_flag_two_way[id_i]=TRUE; n_2way++; } n_good++; } } } } //SID_log("n_good=%d n_2way=%d",SID_LOG_COMMENT,n_good,n_2way); SID_fclose(&fp_check_ids); SID_fclose(&fp_check_score); SID_free(SID_FARG buffer_ids); SID_free(SID_FARG buffer_score); } } // If any of these arrays are temporary, free them. if(flag_alloc_n_sub_i) SID_free(SID_FARG n_sub_group_i); if(flag_alloc_n_sub_j) SID_free(SID_FARG n_sub_group_j); if(flag_alloc_n_particles_i) SID_free(SID_FARG n_particles_i); if(flag_alloc_n_particles_j) SID_free(SID_FARG n_particles_j); }
size_t SID_fread_chunked_ordered(void *buffer, size_t n_x_read_local, SID_fp *fp){ int i_chunk; int i_group; int i_rank; size_t i_x_offset_local; size_t n_x_read_local_bcast; size_t i_x_read_chunk; size_t i_x_offset_global; size_t i_x_chunk; size_t n_skip; size_t n_x_chunk; size_t n_x_chunk_max; size_t header_size; size_t r_val=0; char filename_chunk[256]; // Operates the same way as SID_fread_chunked() except i_x_offset_local is // computed here under the assumption that reads are done in order by rank i_x_offset_local=0; #if USE_MPI for(i_rank=0;i_rank<SID.n_proc;i_rank++){ n_x_read_local_bcast=n_x_read_local; SID_Bcast(&n_x_read_local_bcast,sizeof(size_t),i_rank,SID.COMM_WORLD); if(i_rank<SID.My_rank) i_x_offset_local+=n_x_read_local_bcast; } #endif i_x_offset_global=fp->last_item; for(i_chunk=0,i_x_read_chunk=0,i_x_chunk=i_x_offset_local+i_x_offset_global; i_chunk<fp->chunked_header.n_chunk; i_chunk++){ if(fp->i_x_start_chunk[i_chunk]<=i_x_chunk && fp->i_x_last_chunk[i_chunk]>=i_x_chunk){ n_skip =i_x_chunk-fp->i_x_start_chunk[i_chunk]; n_x_chunk=MIN(n_x_read_local-i_x_read_chunk,fp->i_x_step_chunk[i_chunk]-n_skip); } else{ n_x_chunk=0; n_skip =0; } SID_Allreduce(&n_x_chunk,&n_x_chunk_max,1,SID_SIZE_T,SID_MAX,SID.COMM_WORLD); if(n_x_chunk_max>0){ sprintf(filename_chunk,"%s.%d",fp->filename_root,i_chunk); #if !USE_MPI_IO for(i_group=0;i_group<SID.n_proc;i_group++){ if(SID.My_group==i_group && n_x_chunk>0){ #endif SID_fopen(filename_chunk,"r",fp); if(n_x_chunk>0){ SID_fseek(fp, 1, fp->header_offset[i_chunk]+n_skip*fp->chunked_header.item_size, SID_SEEK_SET); SID_fread((char *)buffer+i_x_read_chunk*fp->chunked_header.item_size, fp->chunked_header.item_size, n_x_chunk, fp); i_x_chunk +=n_x_chunk; i_x_read_chunk+=n_x_chunk; } SID_fclose(fp); #if !USE_MPI_IO } SID_Barrier(SID.COMM_WORLD); } #endif } } SID_Allreduce(&i_x_chunk,&(fp->last_item),1,SID_SIZE_T,SID_MAX,SID.COMM_WORLD); //fp->last_item--; return(r_val); }
void read_trees_horizontal(void **groups_in, int * n_groups_in, void **subgroups_in, int * n_subgroups_in, int * n_subgroups_group, int * n_trees_subgroup_in, int * n_trees_group_in, int i_file, // tree snapshot index int i_read, // actual snapshot index int j_file, int n_wrap, char * filename_output_dir, int mode) { char filename_output_dir_horizontal[SID_MAX_FILENAME_LENGTH]; char filename_output_dir_horizontal_trees[SID_MAX_FILENAME_LENGTH]; char filename_in[SID_MAX_FILENAME_LENGTH]; char filename_output_file_root[SID_MAX_FILENAME_LENGTH]; SID_fp fp_in; SID_log("Reading horizontal trees for snapshot #%03d...", SID_LOG_OPEN, i_read); // Parse the mode and set things up accordingly if(SID_CHECK_BITFIELD_SWITCH(mode, TREE_HORIZONTAL_STORE_EXTENDED) && SID_CHECK_BITFIELD_SWITCH(mode, TREE_HORIZONTAL_STORE_GHOSTS)) SID_exit_error("Too many storage modes chosen in read_trees_horizontal.", SID_ERROR_LOGIC); int flag_read_extended = SID_CHECK_BITFIELD_SWITCH(mode, TREE_HORIZONTAL_READ_EXTENDED); int flag_store_extended = SID_CHECK_BITFIELD_SWITCH(mode, TREE_HORIZONTAL_STORE_EXTENDED); int flag_store_ghosts = SID_CHECK_BITFIELD_SWITCH(mode, TREE_HORIZONTAL_STORE_GHOSTS); if(flag_store_ghosts) SID_exit_error("Ghost processing not supported in read_trees_horizontal().", SID_ERROR_LOGIC); if(!flag_read_extended) SID_exit_error("Reading of non-extended horizontal trees not yet implemented in read_trees_horizontal().", SID_ERROR_LOGIC); // Set filename and open file strcpy(filename_output_file_root, filename_output_dir); strip_path(filename_output_file_root); sprintf(filename_output_dir_horizontal, "%s/horizontal", filename_output_dir); sprintf(filename_output_dir_horizontal_trees, "%s/trees", filename_output_dir_horizontal); if(flag_read_extended) sprintf(filename_in, "%s/horizontal_trees_tmp_%03d.dat", filename_output_dir_horizontal_trees, i_read); else sprintf(filename_in, "%s/horizontal_trees_%03d.dat", filename_output_dir_horizontal_trees, i_read); SID_fopen(filename_in, "r", &fp_in); // Read header int n_step_in; int n_search_in; int n_groups_max_in; int n_subgroups_max_in; SID_fread_all(&n_step_in, sizeof(int), 1, &fp_in); SID_fread_all(&n_search_in, sizeof(int), 1, &fp_in); SID_fread_all(n_groups_in, sizeof(int), 1, &fp_in); SID_fread_all(n_subgroups_in, sizeof(int), 1, &fp_in); SID_fread_all(&n_groups_max_in, sizeof(int), 1, &fp_in); SID_fread_all(&n_subgroups_max_in, sizeof(int), 1, &fp_in); SID_fread_all(n_trees_subgroup_in, sizeof(int), 1, &fp_in); SID_fread_all(n_trees_group_in, sizeof(int), 1, &fp_in); tree_horizontal_extended_info *groups_extended = NULL; tree_horizontal_extended_info *subgroups_extended = NULL; if(flag_store_extended) { groups_extended = (tree_horizontal_extended_info *)groups_in[i_file % n_wrap]; subgroups_extended = (tree_horizontal_extended_info *)subgroups_in[i_file % n_wrap]; } // Perform read int i_group; int i_subgroup; for(i_group = 0, i_subgroup = 0; i_group < (*n_groups_in); i_group++) { // Read groups int group_id; int group_type; int group_descendant_id; int group_tree_id; int group_file_offset; int group_index; int group_n_particles_peak = 0; // default when reading extended format files SID_fread_all(&group_id, sizeof(int), 1, &fp_in); SID_fread_all(&group_type, sizeof(int), 1, &fp_in); SID_fread_all(&group_descendant_id, sizeof(int), 1, &fp_in); SID_fread_all(&group_tree_id, sizeof(int), 1, &fp_in); SID_fread_all(&group_file_offset, sizeof(int), 1, &fp_in); SID_fread_all(&group_index, sizeof(int), 1, &fp_in); if(!flag_read_extended) SID_fread_all(&group_n_particles_peak, sizeof(int), 1, &fp_in); if(!check_validity_of_tree_case_flag(group_type)) SID_exit_error("Invalid match type (%d) for i_group=%d", SID_ERROR_LOGIC, group_type, i_group); if(flag_store_extended) { groups_extended[i_group].id = group_id; groups_extended[i_group].type = group_type; groups_extended[i_group].descendant_id = group_descendant_id; groups_extended[i_group].tree_id = group_tree_id; groups_extended[i_group].descendant_file_offset = group_file_offset; groups_extended[i_group].descendant_index = group_index; } if(flag_read_extended) { int group_file_root; int group_n_particles; int group_n_particles_parent; int group_n_particles_desc; int group_n_particles_proj; float group_score_desc; float group_score_prog; int group_snap_bridge; int group_file_bridge; int group_index_bridge; int group_id_bridge; int group_first_progenitor_file; int group_first_progenitor_index; int group_next_progenitor_file; int group_next_progenitor_index; SID_fread(&group_file_root, sizeof(int), 1, &fp_in); SID_fread(&group_n_particles, sizeof(int), 1, &fp_in); SID_fread(&group_n_particles_parent, sizeof(int), 1, &fp_in); SID_fread(&group_n_particles_desc, sizeof(int), 1, &fp_in); SID_fread(&group_n_particles_proj, sizeof(int), 1, &fp_in); SID_fread(&group_score_desc, sizeof(float), 1, &fp_in); SID_fread(&group_score_prog, sizeof(float), 1, &fp_in); SID_fread(&group_snap_bridge, sizeof(int), 1, &fp_in); SID_fread(&group_file_bridge, sizeof(int), 1, &fp_in); SID_fread(&group_index_bridge, sizeof(int), 1, &fp_in); SID_fread(&group_id_bridge, sizeof(int), 1, &fp_in); SID_fread(&group_first_progenitor_file, sizeof(int), 1, &fp_in); SID_fread(&group_first_progenitor_index, sizeof(int), 1, &fp_in); SID_fread(&group_next_progenitor_file, sizeof(int), 1, &fp_in); SID_fread(&group_next_progenitor_index, sizeof(int), 1, &fp_in); if(flag_store_extended) { groups_extended[i_group].file_root = group_file_root; groups_extended[i_group].n_particles_peak = 0; groups_extended[i_group].parent_id = group_id; groups_extended[i_group].substructure_index = 0; groups_extended[i_group].n_particles = group_n_particles; groups_extended[i_group].n_particles_parent = group_n_particles_parent; groups_extended[i_group].n_particles_desc = group_n_particles_desc; groups_extended[i_group].n_particles_proj = group_n_particles_proj; groups_extended[i_group].score_desc = group_score_desc; groups_extended[i_group].score_prog = group_score_prog; // Only read the bridge information if the halo is marked // TREE_CASE_FRAGMENTED_NEW. This prevents us from overwriting // the bridge info we need to propagate in order to check for // when a fragmented halo returns to its bridge. if(SID_CHECK_BITFIELD_SWITCH(group_type, TREE_CASE_FRAGMENTED_NEW)) { groups_extended[i_group].snap_bridge = group_snap_bridge; groups_extended[i_group].file_bridge = group_file_bridge; groups_extended[i_group].index_bridge = group_index_bridge; groups_extended[i_group].id_bridge = group_id_bridge; } else { groups_extended[i_group].snap_bridge = -1; groups_extended[i_group].file_bridge = -1; groups_extended[i_group].index_bridge = -1; groups_extended[i_group].id_bridge = -1; } groups_extended[i_group].first_progenitor_file = group_first_progenitor_file; groups_extended[i_group].first_progenitor_index = group_first_progenitor_index; groups_extended[i_group].next_progenitor_file = group_next_progenitor_file; groups_extended[i_group].next_progenitor_index = group_next_progenitor_index; } } SID_fread_all(&(n_subgroups_group[i_group]), sizeof(int), 1, &fp_in); int j_subgroup; for(j_subgroup = 0; j_subgroup < n_subgroups_group[i_group]; i_subgroup++, j_subgroup++) { // Read subgroups int subgroup_id; int subgroup_type; int subgroup_descendant_id; int subgroup_tree_id; int subgroup_file_offset; int subgroup_index; int subgroup_n_particles_peak = 0; // default when reading extended format files SID_fread_all(&subgroup_id, sizeof(int), 1, &fp_in); SID_fread_all(&subgroup_type, sizeof(int), 1, &fp_in); SID_fread_all(&subgroup_descendant_id, sizeof(int), 1, &fp_in); SID_fread_all(&subgroup_tree_id, sizeof(int), 1, &fp_in); SID_fread_all(&subgroup_file_offset, sizeof(int), 1, &fp_in); SID_fread_all(&subgroup_index, sizeof(int), 1, &fp_in); if(!flag_read_extended) SID_fread_all(&subgroup_n_particles_peak, sizeof(int), 1, &fp_in); if(!check_validity_of_tree_case_flag(subgroup_type)) SID_exit_error("Invalid match type (%d) for i_group,j_subgroup,i_subgroup=%d,%d,%d", SID_ERROR_LOGIC, subgroup_type, i_group, j_subgroup, i_subgroup); if(flag_store_extended) { subgroups_extended[i_subgroup].id = subgroup_id; subgroups_extended[i_subgroup].type = subgroup_type; subgroups_extended[i_subgroup].descendant_id = subgroup_descendant_id; subgroups_extended[i_subgroup].tree_id = subgroup_tree_id; subgroups_extended[i_subgroup].descendant_file_offset = subgroup_file_offset; subgroups_extended[i_subgroup].descendant_index = subgroup_index; } if(flag_read_extended) { int subgroup_file_root; int subgroup_n_particles; int subgroup_n_particles_parent; int subgroup_n_particles_desc; int subgroup_n_particles_proj; float subgroup_score_desc; float subgroup_score_prog; int subgroup_snap_bridge; int subgroup_file_bridge; int subgroup_index_bridge; int subgroup_id_bridge; int subgroup_first_progenitor_file; int subgroup_first_progenitor_index; int subgroup_next_progenitor_file; int subgroup_next_progenitor_index; SID_fread(&subgroup_file_root, sizeof(int), 1, &fp_in); SID_fread(&subgroup_n_particles, sizeof(int), 1, &fp_in); SID_fread(&subgroup_n_particles_parent, sizeof(int), 1, &fp_in); SID_fread(&subgroup_n_particles_desc, sizeof(int), 1, &fp_in); SID_fread(&subgroup_n_particles_proj, sizeof(int), 1, &fp_in); SID_fread(&subgroup_score_desc, sizeof(float), 1, &fp_in); SID_fread(&subgroup_score_prog, sizeof(float), 1, &fp_in); SID_fread(&subgroup_snap_bridge, sizeof(int), 1, &fp_in); SID_fread(&subgroup_file_bridge, sizeof(int), 1, &fp_in); SID_fread(&subgroup_index_bridge, sizeof(int), 1, &fp_in); SID_fread(&subgroup_id_bridge, sizeof(int), 1, &fp_in); SID_fread(&subgroup_first_progenitor_file, sizeof(int), 1, &fp_in); SID_fread(&subgroup_first_progenitor_index, sizeof(int), 1, &fp_in); SID_fread(&subgroup_next_progenitor_file, sizeof(int), 1, &fp_in); SID_fread(&subgroup_next_progenitor_index, sizeof(int), 1, &fp_in); if(flag_store_extended) { subgroups_extended[i_subgroup].file_root = subgroup_file_root; subgroups_extended[i_subgroup].n_particles_peak = 0; subgroups_extended[i_subgroup].parent_id = group_id; subgroups_extended[i_subgroup].substructure_index = i_subgroup; subgroups_extended[i_subgroup].n_particles = subgroup_n_particles; subgroups_extended[i_subgroup].n_particles_parent = subgroup_n_particles_parent; subgroups_extended[i_subgroup].n_particles_desc = subgroup_n_particles_desc; subgroups_extended[i_subgroup].n_particles_proj = subgroup_n_particles_proj; subgroups_extended[i_subgroup].score_desc = subgroup_score_desc; subgroups_extended[i_subgroup].score_prog = subgroup_score_prog; // Only read the bridge information if the halo is marked // TREE_CASE_FRAGMENTED_NEW. This prevents us from overwriting // the bridge info we need to propagate in order to check for // when a fragmented halo returns to its bridge. if(SID_CHECK_BITFIELD_SWITCH(subgroup_type, TREE_CASE_FRAGMENTED_NEW)) { subgroups_extended[i_subgroup].snap_bridge = subgroup_snap_bridge; subgroups_extended[i_subgroup].file_bridge = subgroup_file_bridge; subgroups_extended[i_subgroup].index_bridge = subgroup_index_bridge; subgroups_extended[i_subgroup].id_bridge = subgroup_id_bridge; } else { subgroups_extended[i_subgroup].snap_bridge = -1; subgroups_extended[i_subgroup].file_bridge = -1; subgroups_extended[i_subgroup].index_bridge = -1; subgroups_extended[i_subgroup].id_bridge = -1; } subgroups_extended[i_subgroup].first_progenitor_file = subgroup_first_progenitor_file; subgroups_extended[i_subgroup].first_progenitor_index = subgroup_first_progenitor_index; subgroups_extended[i_subgroup].next_progenitor_file = subgroup_next_progenitor_file; subgroups_extended[i_subgroup].next_progenitor_index = subgroup_next_progenitor_index; } } } } if(flag_store_extended) { for(; i_group < n_groups_max_in; i_group++) groups_extended[i_group].type = TREE_CASE_INVALID; for(; i_subgroup < n_subgroups_max_in; i_subgroup++) subgroups_extended[i_subgroup].type = TREE_CASE_INVALID; } SID_fclose(&fp_in); SID_log("Done.", SID_LOG_CLOSE); }